debuggers.hg

changeset 21195:b010b792c0f8

xl: Migration support

Implement "xl migrate".

ssh is used as the transport by default, although this can be
overridden by specifying a different sshcommand. This is a very
standard approach nowadays and avoids the need for daemons at the
target host in the default configuration, while providing flexibility
to admins. (In the future it might be nice to support plain
unencrypted migration over TCP, which we do not rule out now, although
it is not currently implemented.)

Properties of the migration protocol:
* The domain on the target machine is named "<domname>--incoming"
while it is being transferred.
* The domain on the source machine is renamed
"<domain>--migratedaway"
before we give the target permission to rename and unpause.
* The locking in libxl_domain_rename ensures that of two
simultaneous migration attempts no more than one will succeed.
* We go to some considerable effort to avoid leaving the domain in
a bad state if something goes wrong with one of the ends or the
network, although there is still (inevitably) a possibility of a
unresolvable state (in case of very badly timed network failure)
which is probably best resolved by destroying the domain at both
ends.

Incidental changes:
create_domain now returns a libxl error code rather than exiting on
error.
New ERROR_BADFAIL error code for reporting unpleasant failures.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Apr 12 17:47:16 2010 +0100 (2010-04-12)
parents f0c305aaa6a2
children 78488a63bbc2
files tools/libxl/libxl.h tools/libxl/xl.c
line diff
     1.1 --- a/tools/libxl/libxl.h	Mon Apr 12 17:46:39 2010 +0100
     1.2 +++ b/tools/libxl/libxl.h	Mon Apr 12 17:47:16 2010 +0100
     1.3 @@ -242,6 +242,7 @@ enum {
     1.4      ERROR_NI = -3,
     1.5      ERROR_NOMEM = -4,
     1.6      ERROR_INVAL = -5,
     1.7 +    ERROR_BADFAIL = -6,
     1.8  };
     1.9  
    1.10  #define LIBXL_VERSION 0
     2.1 --- a/tools/libxl/xl.c	Mon Apr 12 17:46:39 2010 +0100
     2.2 +++ b/tools/libxl/xl.c	Mon Apr 12 17:47:16 2010 +0100
     2.3 @@ -17,6 +17,7 @@
     2.4  #include "libxl_osdeps.h"
     2.5  
     2.6  #include <stdio.h>
     2.7 +#include <assert.h>
     2.8  #include <stdlib.h>
     2.9  #include <string.h>
    2.10  #include <unistd.h>
    2.11 @@ -47,9 +48,26 @@ static struct libxl_ctx ctx;
    2.12  static uint32_t domid;
    2.13  static const char *common_domname;
    2.14  
    2.15 +
    2.16  static const char savefileheader_magic[32]=
    2.17      "Xen saved domain, xl format\n \0 \r";
    2.18  
    2.19 +static const char migrate_receiver_banner[]=
    2.20 +    "xl migration receiver ready, send binary domain data.\n";
    2.21 +static const char migrate_receiver_ready[]=
    2.22 +    "domain received, ready to unpause";
    2.23 +static const char migrate_permission_to_go[]=
    2.24 +    "domain is yours, you are cleared to unpause";
    2.25 +static const char migrate_report[]=
    2.26 +    "my copy unpause results are as follows";
    2.27 +  /* followed by one byte:
    2.28 +   *     0: everything went well, domain is running
    2.29 +   *            next thing is we all exit
    2.30 +   * non-0: things went badly
    2.31 +   *            next thing should be a migrate_permission_to_go
    2.32 +   *            from target to source
    2.33 +   */
    2.34 +
    2.35  struct save_file_header {
    2.36      char magic[32]; /* savefileheader_magic */
    2.37      /* All uint32_ts are in domain's byte order. */
    2.38 @@ -752,7 +770,7 @@ static void *xrealloc(void *ptr, size_t 
    2.39      return r;
    2.40  }
    2.41  
    2.42 -static void create_domain(int debug, int daemonize, const char *config_file, const char *restore_file, int paused)
    2.43 +static int create_domain(int debug, int daemonize, const char *config_file, const char *restore_file, int paused, int migrate_fd /* -1 means none */, char **migration_domname_r)
    2.44  {
    2.45      libxl_domain_create_info info1;
    2.46      libxl_domain_build_info info2;
    2.47 @@ -777,30 +795,25 @@ static void create_domain(int debug, int
    2.48  
    2.49      memset(&dm_info, 0x00, sizeof(dm_info));
    2.50  
    2.51 -    if (libxl_ctx_init(&ctx, LIBXL_VERSION)) {
    2.52 -        fprintf(stderr, "cannot init xl context\n");
    2.53 -        exit(1);
    2.54 -    }
    2.55 -    libxl_ctx_set_log(&ctx, log_callback, NULL);
    2.56 -
    2.57      if (restore_file) {
    2.58          uint8_t *optdata_begin = 0;
    2.59          const uint8_t *optdata_here = 0;
    2.60          union { uint32_t u32; char b[4]; } u32buf;
    2.61          uint32_t badflags;
    2.62  
    2.63 -        restore_fd = open(restore_file, O_RDONLY);
    2.64 +        restore_fd = migrate_fd >= 0 ? migrate_fd :
    2.65 +            open(restore_file, O_RDONLY);
    2.66  
    2.67          CHK_ERRNO( libxl_read_exactly(&ctx, restore_fd, &hdr,
    2.68                     sizeof(hdr), restore_file, "header") );
    2.69          if (memcmp(hdr.magic, savefileheader_magic, sizeof(hdr.magic))) {
    2.70              fprintf(stderr, "File has wrong magic number -"
    2.71                      " corrupt or for a different tool?\n");
    2.72 -            exit(2);
    2.73 +            return ERROR_INVAL;
    2.74          }
    2.75          if (hdr.byteorder != SAVEFILE_BYTEORDER_VALUE) {
    2.76              fprintf(stderr, "File has wrong byte order\n");
    2.77 -            exit(2);
    2.78 +            return ERROR_INVAL;
    2.79          }
    2.80          fprintf(stderr, "Loading new save file %s"
    2.81                  " (new xl fmt info"
    2.82 @@ -813,7 +826,7 @@ static void create_domain(int debug, int
    2.83              fprintf(stderr, "Savefile has mandatory flag(s) 0x%"PRIx32" "
    2.84                      "which are not supported; need newer xl\n",
    2.85                      badflags);
    2.86 -            exit(2);
    2.87 +            return ERROR_INVAL;
    2.88          }
    2.89          if (hdr.optional_data_len) {
    2.90              optdata_begin = xmalloc(hdr.optional_data_len);
    2.91 @@ -822,12 +835,13 @@ static void create_domain(int debug, int
    2.92          }
    2.93  
    2.94  #define OPTDATA_LEFT  (hdr.optional_data_len - (optdata_here - optdata_begin))
    2.95 -#define WITH_OPTDATA(amt, body)                                         \
    2.96 -            if (OPTDATA_LEFT < (amt)) {                                 \
    2.97 -                fprintf(stderr, "Savefile truncated.\n"); exit(2);      \
    2.98 -            } else {                                                    \
    2.99 -                body;                                                   \
   2.100 -                optdata_here += (amt);                                  \
   2.101 +#define WITH_OPTDATA(amt, body)                                 \
   2.102 +            if (OPTDATA_LEFT < (amt)) {                         \
   2.103 +                fprintf(stderr, "Savefile truncated.\n");       \
   2.104 +                return ERROR_INVAL;                             \
   2.105 +            } else {                                            \
   2.106 +                body;                                           \
   2.107 +                optdata_here += (amt);                          \
   2.108              }
   2.109  
   2.110          optdata_here = optdata_begin;
   2.111 @@ -851,12 +865,12 @@ static void create_domain(int debug, int
   2.112          ret = libxl_read_file_contents(&ctx, config_file,
   2.113                                         &config_data, &config_len);
   2.114          if (ret) { fprintf(stderr, "Failed to read config file: %s: %s\n",
   2.115 -                           config_file, strerror(errno)); exit(1); }
   2.116 +                           config_file, strerror(errno)); return ERROR_FAIL; }
   2.117      } else {
   2.118          if (!config_data) {
   2.119              fprintf(stderr, "Config file not specified and"
   2.120                      " none in save file\n");
   2.121 -            exit(1);
   2.122 +            return ERROR_INVAL;
   2.123          }
   2.124          config_file = "<saved>";
   2.125      }
   2.126 @@ -865,6 +879,17 @@ static void create_domain(int debug, int
   2.127  
   2.128      parse_config_data(config_file, config_data, config_len, &info1, &info2, &disks, &num_disks, &vifs, &num_vifs, &pcidevs, &num_pcidevs, &vfbs, &num_vfbs, &vkbs, &num_vkbs, &dm_info);
   2.129  
   2.130 +    if (migrate_fd >= 0) {
   2.131 +        if (info1.name) {
   2.132 +            /* when we receive a domain we get its name from the config
   2.133 +             * file; and we receive it to a temporary name */
   2.134 +            assert(!common_domname);
   2.135 +            common_domname = info1.name;
   2.136 +            asprintf(migration_domname_r, "%s--incoming", info1.name);
   2.137 +            info1.name = *migration_domname_r;
   2.138 +        }
   2.139 +    }
   2.140 +
   2.141      if (debug)
   2.142          printf_info(&info1, &info2, disks, num_disks, vifs, num_vifs, pcidevs, num_pcidevs, vfbs, num_vfbs, vkbs, num_vkbs, &dm_info);
   2.143  
   2.144 @@ -874,14 +899,14 @@ start:
   2.145      ret = libxl_domain_make(&ctx, &info1, &domid);
   2.146      if (ret) {
   2.147          fprintf(stderr, "cannot make domain: %d\n", ret);
   2.148 -        exit(1);
   2.149 +        return ERROR_FAIL;
   2.150      }
   2.151  
   2.152      ret = libxl_userdata_store(&ctx, domid, "xl",
   2.153                                      config_data, config_len);
   2.154      if (ret) {
   2.155          perror("cannot save config file");
   2.156 -        exit(1);
   2.157 +        return ERROR_FAIL;
   2.158      }
   2.159  
   2.160      if (!restore_file || !need_daemon) {
   2.161 @@ -892,12 +917,11 @@ start:
   2.162          ret = libxl_domain_build(&ctx, &info2, domid, &state);
   2.163      } else {
   2.164          ret = libxl_domain_restore(&ctx, &info2, domid, restore_fd, &state, &dm_info);
   2.165 -        close(restore_fd);
   2.166      }
   2.167  
   2.168      if (ret) {
   2.169          fprintf(stderr, "cannot (re-)build domain: %d\n", ret);
   2.170 -        exit(1);
   2.171 +        return ERROR_FAIL;
   2.172      }
   2.173  
   2.174      for (i = 0; i < num_disks; i++) {
   2.175 @@ -905,7 +929,7 @@ start:
   2.176          ret = libxl_device_disk_add(&ctx, domid, &disks[i]);
   2.177          if (ret) {
   2.178              fprintf(stderr, "cannot add disk %d to domain: %d\n", i, ret);
   2.179 -            exit(1);
   2.180 +            return ERROR_FAIL;
   2.181          }
   2.182      }
   2.183      for (i = 0; i < num_vifs; i++) {
   2.184 @@ -913,7 +937,7 @@ start:
   2.185          ret = libxl_device_nic_add(&ctx, domid, &vifs[i]);
   2.186          if (ret) {
   2.187              fprintf(stderr, "cannot add nic %d to domain: %d\n", i, ret);
   2.188 -            exit(1);
   2.189 +            return ERROR_FAIL;
   2.190          }
   2.191      }
   2.192      if (info1.hvm) {
   2.193 @@ -945,7 +969,7 @@ start:
   2.194          libxl_domain_unpause(&ctx, domid);
   2.195  
   2.196      if (!daemonize)
   2.197 -        exit(0);
   2.198 +        return 0; /* caller gets success in parent */
   2.199  
   2.200      if (need_daemon) {
   2.201          char *fullname, *name;
   2.202 @@ -1050,12 +1074,7 @@ start:
   2.203      }
   2.204  
   2.205      close(logfile);
   2.206 -    free(disks);
   2.207 -    free(vifs);
   2.208 -    free(vfbs);
   2.209 -    free(vkbs);
   2.210 -    free(pcidevs);
   2.211 -    free(config_data);
   2.212 +    exit(0);
   2.213  }
   2.214  
   2.215  static void help(char *command)
   2.216 @@ -1119,6 +1138,25 @@ static void help(char *command)
   2.217          printf("-h                     Print this help.\n");
   2.218          printf("-p                     Do not unpause domain after restoring it.\n");
   2.219          printf("-e                     Do not wait in the background for the death of the domain.\n");
   2.220 +        printf("-d                     Enable debug messages.\n");
   2.221 +    } else if(!strcmp(command, "migrate")) {
   2.222 +        printf("Usage: xl migrate [options] <Domain> <host>\n\n");
   2.223 +        printf("Save a domain state to restore later.\n\n");
   2.224 +        printf("Options:\n\n");
   2.225 +        printf("-h                     Print this help.\n");
   2.226 +        printf("-C <config>            Send <config> instead of config file from creation.\n");
   2.227 +        printf("-s <sshcommand>        Use <sshcommand> instead of ssh.  String will be passed to sh.  If empty, run <host> instead of ssh <host> xl migrate-receive [-d -e]\n");
   2.228 +        printf("-e                     Do not wait in the background (on <host>) for the death of the domain.\n");
   2.229 +    } else if(!strcmp(command, "migrate-receive")) {
   2.230 +        printf("Usage: xl migrate-receive  - for internal use only");
   2.231 +    } else if(!strcmp(command, "restore")) {
   2.232 +        printf("Usage: xl restore [options] [<ConfigFile>] <CheckpointFile>\n\n");
   2.233 +        printf("Restore a domain from a saved state.\n\n");
   2.234 +        printf("Options:\n\n");
   2.235 +        printf("-h                     Print this help.\n");
   2.236 +        printf("-O                     Old (configless) xl save format.\n");
   2.237 +        printf("-p                     Do not unpause domain after restoring it.\n");
   2.238 +        printf("-e                     Do not wait in the background for the death of the domain.\n");
   2.239      } else if(!strcmp(command, "destroy")) {
   2.240          printf("Usage: xl destroy <Domain>\n\n");
   2.241          printf("Terminate a domain immediately.\n\n");
   2.242 @@ -1523,23 +1561,39 @@ void list_vm(void)
   2.243      free(info);
   2.244  }
   2.245  
   2.246 -int save_domain(char *p, char *filename, int checkpoint,
   2.247 -                const char *override_config_file)
   2.248 +static void save_domain_core_begin(char *domain_spec,
   2.249 +                                   const char *override_config_file,
   2.250 +                                   uint8_t **config_data_r,
   2.251 +                                   int *config_len_r)
   2.252  {
   2.253 -    int fd, rc;
   2.254 +    int rc;
   2.255 +
   2.256 +    find_domain(domain_spec);
   2.257 +
   2.258 +    /* configuration file in optional data: */
   2.259 +
   2.260 +    if (override_config_file) {
   2.261 +        void *config_v = 0;
   2.262 +        rc = libxl_read_file_contents(&ctx, override_config_file,
   2.263 +                                      &config_v, config_len_r);
   2.264 +        *config_data_r = config_v;
   2.265 +    } else {
   2.266 +        rc = libxl_userdata_retrieve(&ctx, domid, "xl",
   2.267 +                                     config_data_r, config_len_r);
   2.268 +    }
   2.269 +    if (rc) {
   2.270 +        fputs("Unable to get config file\n",stderr);
   2.271 +        exit(2);
   2.272 +    }
   2.273 +}
   2.274 +
   2.275 +void save_domain_core_writeconfig(int fd, const char *filename,
   2.276 +                                  const uint8_t *config_data, int config_len)
   2.277 +{
   2.278      struct save_file_header hdr;
   2.279 -    uint8_t *config_data = 0;
   2.280 -    int config_len;
   2.281      uint8_t *optdata_begin;
   2.282      union { uint32_t u32; char b[4]; } u32buf;
   2.283  
   2.284 -    find_domain(p);
   2.285 -    fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0644);
   2.286 -    if (fd < 0) {
   2.287 -        fprintf(stderr, "Failed to open temp file %s for writing\n", filename);
   2.288 -        exit(2);
   2.289 -    }
   2.290 -
   2.291      memset(&hdr, 0, sizeof(hdr));
   2.292      memcpy(hdr.magic, savefileheader_magic, sizeof(hdr.magic));
   2.293      hdr.byteorder = SAVEFILE_BYTEORDER_VALUE;
   2.294 @@ -1555,25 +1609,6 @@ int save_domain(char *p, char *filename,
   2.295      }                                                                   \
   2.296                            })
   2.297  
   2.298 -    /* configuration file in optional data: */
   2.299 -
   2.300 -    if (override_config_file) {
   2.301 -        void *config_v = 0;
   2.302 -        rc = libxl_read_file_contents(&ctx, override_config_file,
   2.303 -                                      &config_v, &config_len);
   2.304 -        config_data = config_v;
   2.305 -    } else {
   2.306 -        rc = libxl_userdata_retrieve(&ctx, domid, "xl",
   2.307 -                                          &config_data, &config_len);
   2.308 -    }
   2.309 -    if (rc) {
   2.310 -        fputs("Unable to get config file\n",stderr);
   2.311 -        exit(2);
   2.312 -    }
   2.313 -    if (!config_len) {
   2.314 -        fputs(" Savefile will not contain xl domain config\n", stderr);
   2.315 -    }
   2.316 -
   2.317      u32buf.u32 = config_len;
   2.318      ADD_OPTDATA(u32buf.b,    4);
   2.319      ADD_OPTDATA(config_data, config_len);
   2.320 @@ -1589,6 +1624,28 @@ int save_domain(char *p, char *filename,
   2.321              " 0x%"PRIx32"/0x%"PRIx32"/%"PRIu32")\n",
   2.322              filename, hdr.mandatory_flags, hdr.optional_flags,
   2.323              hdr.optional_data_len);
   2.324 +}
   2.325 +
   2.326 +int save_domain(char *p, char *filename, int checkpoint,
   2.327 +                const char *override_config_file)
   2.328 +{
   2.329 +    int fd;
   2.330 +    uint8_t *config_data;
   2.331 +    int config_len;
   2.332 +
   2.333 +    save_domain_core_begin(p, override_config_file, &config_data, &config_len);
   2.334 +
   2.335 +    if (!config_len) {
   2.336 +        fputs(" Savefile will not contain xl domain config\n", stderr);
   2.337 +    }
   2.338 +
   2.339 +    fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 0644);
   2.340 +    if (fd < 0) {
   2.341 +        fprintf(stderr, "Failed to open temp file %s for writing\n", filename);
   2.342 +        exit(2);
   2.343 +    }
   2.344 +
   2.345 +    save_domain_core_writeconfig(fd, filename, config_data, config_len);
   2.346  
   2.347      libxl_domain_suspend(&ctx, NULL, domid, fd);
   2.348      close(fd);
   2.349 @@ -1601,12 +1658,355 @@ int save_domain(char *p, char *filename,
   2.350      exit(0);
   2.351  }
   2.352  
   2.353 +static int migrate_read_fixedmessage(int fd, const void *msg, int msgsz,
   2.354 +                                     const char *what, const char *rune) {
   2.355 +    char buf[msgsz];
   2.356 +    const char *stream;
   2.357 +    int rc;
   2.358 +
   2.359 +    stream = rune ? "migration receiver stream" : "migration stream";
   2.360 +    rc = libxl_read_exactly(&ctx, fd, buf, msgsz, stream, what);
   2.361 +    if (rc) return ERROR_FAIL;
   2.362 +
   2.363 +    if (memcmp(buf, msg, msgsz)) {
   2.364 +        fprintf(stderr, "%s contained unexpected data instead of %s\n",
   2.365 +                stream, what);
   2.366 +        if (rune)
   2.367 +            fprintf(stderr, "(command run was: %s )\n", rune);
   2.368 +        return ERROR_FAIL;
   2.369 +    }
   2.370 +    return 0;
   2.371 +}
   2.372 +
   2.373 +static void migration_child_report(pid_t migration_child, int recv_fd) {
   2.374 +    pid_t child;
   2.375 +    int status, sr;
   2.376 +    struct timeval now, waituntil, timeout;
   2.377 +    static const struct timeval pollinterval = { 0, 1000 }; /* 1ms */
   2.378 +
   2.379 +    if (!migration_child) return;
   2.380 +
   2.381 +    CHK_ERRNO( gettimeofday(&waituntil, 0) );
   2.382 +    waituntil.tv_sec += 2;
   2.383 +
   2.384 +    for (;;) {
   2.385 +        child = waitpid(migration_child, &status, WNOHANG);
   2.386 +
   2.387 +        if (child == migration_child) {
   2.388 +            if (status)
   2.389 +                libxl_report_child_exitstatus(&ctx, XL_LOG_INFO,
   2.390 +                                              "migration target process",
   2.391 +                                              migration_child, status);
   2.392 +            break;
   2.393 +        }
   2.394 +        if (child == -1) {
   2.395 +            if (errno == EINTR) continue;
   2.396 +            fprintf(stderr, "wait for migration child [%ld] failed: %s\n",
   2.397 +                    (long)migration_child, strerror(errno));
   2.398 +            break;
   2.399 +        }
   2.400 +        assert(child == 0);
   2.401 +
   2.402 +        CHK_ERRNO( gettimeofday(&now, 0) );
   2.403 +        if (timercmp(&now, &waituntil, >)) {
   2.404 +            fprintf(stderr, "migration child [%ld] not exiting, no longer"
   2.405 +                    " waiting (exit status will be unreported)\n",
   2.406 +                    (long)migration_child);
   2.407 +            break;
   2.408 +        }
   2.409 +        timersub(&waituntil, &now, &timeout);
   2.410 +
   2.411 +        if (recv_fd >= 0) {
   2.412 +            fd_set readfds, exceptfds;
   2.413 +            FD_ZERO(&readfds);
   2.414 +            FD_ZERO(&exceptfds);
   2.415 +            FD_SET(recv_fd, &readfds);
   2.416 +            FD_SET(recv_fd, &exceptfds);
   2.417 +            sr = select(recv_fd+1, &readfds,0,&exceptfds, &timeout);
   2.418 +        } else {
   2.419 +            if (timercmp(&timeout, &pollinterval, >))
   2.420 +                timeout = pollinterval;
   2.421 +            sr = select(0,0,0,0, &timeout);
   2.422 +        }
   2.423 +        if (sr > 0) {
   2.424 +            recv_fd = -1;
   2.425 +        } else if (sr == 0) {
   2.426 +        } else if (sr == -1) {
   2.427 +            if (errno != EINTR) {
   2.428 +                fprintf(stderr, "migration child [%ld] exit wait select"
   2.429 +                        " failed unexpectedly: %s\n",
   2.430 +                        (long)migration_child, strerror(errno));
   2.431 +                break;
   2.432 +            }
   2.433 +        }
   2.434 +    }
   2.435 +    migration_child = 0;
   2.436 +}
   2.437 +
   2.438 +static void migrate_domain(char *domain_spec, const char *rune,
   2.439 +                           const char *override_config_file)
   2.440 +{
   2.441 +    pid_t child = -1;
   2.442 +    int rc;
   2.443 +    int sendpipe[2], recvpipe[2];
   2.444 +    int send_fd, recv_fd;
   2.445 +    libxl_domain_suspend_info suspinfo;
   2.446 +    char *away_domname;
   2.447 +    char rc_buf;
   2.448 +    uint8_t *config_data;
   2.449 +    int config_len;
   2.450 +
   2.451 +    save_domain_core_begin(domain_spec, override_config_file,
   2.452 +                           &config_data, &config_len);
   2.453 +
   2.454 +    if (!common_domname) {
   2.455 +        common_domname = libxl_domid_to_name(&ctx, domid);
   2.456 +        /* libxl_domid_to_name fails ?  don't bother with names then */
   2.457 +    }
   2.458 +
   2.459 +    if (!config_len) {
   2.460 +        fprintf(stderr, "No config file stored for running domain and "
   2.461 +                "none supplied - cannot migrate.\n");
   2.462 +        exit(1);
   2.463 +    }
   2.464 +
   2.465 +    MUST( libxl_pipe(&ctx, sendpipe) );
   2.466 +    MUST( libxl_pipe(&ctx, recvpipe) );
   2.467 +
   2.468 +    child = libxl_fork(&ctx);
   2.469 +    if (child==-1) exit(1);
   2.470 +
   2.471 +    if (!child) {
   2.472 +        dup2(sendpipe[0], 0);
   2.473 +        dup2(recvpipe[1], 1);
   2.474 +        close(sendpipe[0]); close(sendpipe[1]);
   2.475 +        close(recvpipe[0]); close(recvpipe[1]);
   2.476 +        execlp("sh","sh","-c",rune,(char*)0);
   2.477 +        perror("failed to exec sh");
   2.478 +        exit(-1);
   2.479 +    }
   2.480 +
   2.481 +    close(sendpipe[0]);
   2.482 +    close(recvpipe[1]);
   2.483 +    send_fd = sendpipe[1];
   2.484 +    recv_fd = recvpipe[0];
   2.485 +
   2.486 +    signal(SIGPIPE, SIG_IGN);
   2.487 +    /* if receiver dies, we get an error and can clean up
   2.488 +       rather than just dying */
   2.489 +
   2.490 +    rc = migrate_read_fixedmessage(recv_fd, migrate_receiver_banner,
   2.491 +                                   sizeof(migrate_receiver_banner)-1,
   2.492 +                                   "banner", rune);
   2.493 +    if (rc) {
   2.494 +        close(send_fd);
   2.495 +        migration_child_report(child, recv_fd);
   2.496 +        exit(-rc);
   2.497 +    }
   2.498 +
   2.499 +    save_domain_core_writeconfig(send_fd, "migration stream",
   2.500 +                                 config_data, config_len);
   2.501 +
   2.502 +    memset(&suspinfo, 0, sizeof(suspinfo));
   2.503 +    suspinfo.flags |= XL_SUSPEND_LIVE;
   2.504 +    rc = libxl_domain_suspend(&ctx, &suspinfo, domid, send_fd);
   2.505 +    if (rc) {
   2.506 +        fprintf(stderr, "migration sender: libxl_domain_suspend failed"
   2.507 +                " (rc=%d)\n", rc);
   2.508 +        goto failed_resume;
   2.509 +    }
   2.510 +
   2.511 +    fprintf(stderr, "migration sender: Transfer complete.\n");
   2.512 +
   2.513 +    rc = migrate_read_fixedmessage(recv_fd, migrate_receiver_ready,
   2.514 +                                   sizeof(migrate_receiver_ready),
   2.515 +                                   "ready message", rune);
   2.516 +    if (rc) goto failed_resume;
   2.517 +
   2.518 +    /* right, at this point we are about give the destination
   2.519 +     * permission to rename and resume, so we must first rename the
   2.520 +     * domain away ourselves */
   2.521 +
   2.522 +    fprintf(stderr, "migration sender: Target has acknowledged transfer.\n");
   2.523 +
   2.524 +    if (common_domname) {
   2.525 +        asprintf(&away_domname, "%s--migratedaway", common_domname);
   2.526 +        rc = libxl_domain_rename(&ctx, domid,
   2.527 +                                 common_domname, away_domname, 0);
   2.528 +        if (rc) goto failed_resume;
   2.529 +    }
   2.530 +
   2.531 +    /* point of no return - as soon as we have tried to say
   2.532 +     * "go" to the receiver, it's not safe to carry on.  We leave
   2.533 +     * the domain renamed to %s--migratedaway in case that's helpful.
   2.534 +     */
   2.535 +
   2.536 +    fprintf(stderr, "migration sender: Giving target permission to start.\n");
   2.537 +
   2.538 +    rc = libxl_write_exactly(&ctx, send_fd,
   2.539 +                             migrate_permission_to_go,
   2.540 +                             sizeof(migrate_permission_to_go),
   2.541 +                             "migration stream", "GO message");
   2.542 +    if (rc) goto failed_badly;
   2.543 +
   2.544 +    rc = migrate_read_fixedmessage(recv_fd, migrate_report,
   2.545 +                                   sizeof(migrate_report),
   2.546 +                                   "success/failure report message", rune);
   2.547 +    if (rc) goto failed_badly;
   2.548 +
   2.549 +    rc = libxl_read_exactly(&ctx, recv_fd,
   2.550 +                            &rc_buf, 1,
   2.551 +                            "migration ack stream", "success/failure status");
   2.552 +    if (rc) goto failed_badly;
   2.553 +
   2.554 +    if (rc_buf) {
   2.555 +        fprintf(stderr, "migration sender: Target reports startup failure"
   2.556 +                " (status code %d).\n", rc_buf);
   2.557 +
   2.558 +        rc = migrate_read_fixedmessage(recv_fd, migrate_permission_to_go,
   2.559 +                                       sizeof(migrate_permission_to_go),
   2.560 +                                       "permission for sender to resume",
   2.561 +                                       rune);
   2.562 +        if (rc) goto failed_badly;
   2.563 +
   2.564 +        fprintf(stderr, "migration sender: Trying to resume at our end.\n");
   2.565 +
   2.566 +        if (common_domname) {
   2.567 +            libxl_domain_rename(&ctx, domid,
   2.568 +                                away_domname, common_domname, 0);
   2.569 +        }
   2.570 +        rc = libxl_domain_resume(&ctx, domid);
   2.571 +        if (!rc) fprintf(stderr, "migration sender: Resumed OK.\n");
   2.572 +
   2.573 +        fprintf(stderr, "Migration failed due to problems at target.\n");
   2.574 +        exit(-ERROR_FAIL);
   2.575 +    }
   2.576 +
   2.577 +    fprintf(stderr, "migration sender: Target reports successful startup.\n");
   2.578 +    libxl_domain_destroy(&ctx, domid, 1); /* bang! */
   2.579 +    fprintf(stderr, "Migration successful.\n");
   2.580 +    exit(0);
   2.581 +
   2.582 + failed_resume:
   2.583 +    close(send_fd);
   2.584 +    migration_child_report(child, recv_fd);
   2.585 +    fprintf(stderr, "Migration failed, resuming at sender.\n");
   2.586 +    libxl_domain_resume(&ctx, domid);
   2.587 +    exit(-ERROR_FAIL);
   2.588 +
   2.589 + failed_badly:
   2.590 +    fprintf(stderr,
   2.591 + "** Migration failed during final handshake **\n"
   2.592 + "Domain state is now undefined !\n"
   2.593 + "Please CHECK AT BOTH ENDS for running instances, before renaming and\n"
   2.594 + " resuming at most one instance.  Two simultaneous instances of the domain\n"
   2.595 + " would probably result in SEVERE DATA LOSS and it is now your\n"
   2.596 + " responsibility to avoid that.  Sorry.\n");
   2.597 +
   2.598 +    close(send_fd);
   2.599 +    migration_child_report(child, recv_fd);
   2.600 +    exit(-ERROR_BADFAIL);
   2.601 +}
   2.602 +
   2.603 +static void migrate_receive(int debug, int daemonize)
   2.604 +{
   2.605 +    int rc, rc2;
   2.606 +    char rc_buf;
   2.607 +    char *migration_domname;
   2.608 +
   2.609 +    signal(SIGPIPE, SIG_IGN);
   2.610 +    /* if we get SIGPIPE we'd rather just have it as an error */
   2.611 +
   2.612 +    fprintf(stderr, "migration target: Ready to receive domain.\n");
   2.613 +
   2.614 +    CHK_ERRNO( libxl_write_exactly(&ctx, 1,
   2.615 +                                   migrate_receiver_banner,
   2.616 +                                   sizeof(migrate_receiver_banner)-1,
   2.617 +                                   "migration ack stream",
   2.618 +                                   "banner") );
   2.619 +
   2.620 +    rc = create_domain(debug, daemonize,
   2.621 +                       0 /* no config file, use incoming */,
   2.622 +                       "incoming migration stream", 1,
   2.623 +                       0, &migration_domname);
   2.624 +    if (rc) {
   2.625 +        fprintf(stderr, "migration target: Domain creation failed"
   2.626 +                " (code %d).\n", rc);
   2.627 +        exit(-rc);
   2.628 +    }
   2.629 +
   2.630 +    fprintf(stderr, "migration target: Transfer complete,"
   2.631 +            " requesting permission to start domain.\n");
   2.632 +
   2.633 +    rc = libxl_write_exactly(&ctx, 1,
   2.634 +                             migrate_receiver_ready,
   2.635 +                             sizeof(migrate_receiver_ready),
   2.636 +                             "migration ack stream", "ready message");
   2.637 +    if (rc) exit(-rc);
   2.638 +
   2.639 +    rc = migrate_read_fixedmessage(0, migrate_permission_to_go,
   2.640 +                                   sizeof(migrate_permission_to_go),
   2.641 +                                   "GO message", 0);
   2.642 +    if (rc) goto perhaps_destroy_notify_rc;
   2.643 +
   2.644 +    fprintf(stderr, "migration target: Got permission, starting domain.\n");
   2.645 +
   2.646 +    if (migration_domname) {
   2.647 +        rc = libxl_domain_rename(&ctx, domid,
   2.648 +                                 migration_domname, common_domname, 0);
   2.649 +        if (rc) goto perhaps_destroy_notify_rc;
   2.650 +    }
   2.651 +
   2.652 +    rc = libxl_domain_unpause(&ctx, domid);
   2.653 +    if (rc) goto perhaps_destroy_notify_rc;
   2.654 +
   2.655 +    fprintf(stderr, "migration target: Domain started successsfully.\n");
   2.656 +    rc = 0;
   2.657 +
   2.658 + perhaps_destroy_notify_rc:
   2.659 +    rc2 = libxl_write_exactly(&ctx, 1,
   2.660 +                              migrate_report, sizeof(migrate_report),
   2.661 +                              "migration ack stream",
   2.662 +                              "success/failure report");
   2.663 +    if (rc2) exit(-ERROR_BADFAIL);
   2.664 +
   2.665 +    rc_buf = -rc;
   2.666 +    assert(!!rc_buf == !!rc);
   2.667 +    rc2 = libxl_write_exactly(&ctx, 1, &rc_buf, 1,
   2.668 +                              "migration ack stream",
   2.669 +                              "success/failure code");
   2.670 +    if (rc2) exit(-ERROR_BADFAIL);
   2.671 +
   2.672 +    if (rc) {
   2.673 +        fprintf(stderr, "migration target: Failure, destroying our copy.\n");
   2.674 +
   2.675 +        rc2 = libxl_domain_destroy(&ctx, domid, 1);
   2.676 +        if (rc2) {
   2.677 +            fprintf(stderr, "migration target: Failed to destroy our copy"
   2.678 +                    " (code %d).\n", rc2);
   2.679 +            exit(-ERROR_BADFAIL);
   2.680 +        }
   2.681 +
   2.682 +        fprintf(stderr, "migration target: Cleanup OK, granting sender"
   2.683 +                " permission to resume.\n");
   2.684 +
   2.685 +        rc2 = libxl_write_exactly(&ctx, 1,
   2.686 +                                  migrate_permission_to_go,
   2.687 +                                  sizeof(migrate_permission_to_go),
   2.688 +                                  "migration ack stream",
   2.689 +                                  "permission to sender to have domain back");
   2.690 +        if (rc2) exit(-ERROR_BADFAIL);
   2.691 +    }
   2.692 +
   2.693 +    exit(0);
   2.694 +}
   2.695 +
   2.696  int main_restore(int argc, char **argv)
   2.697  {
   2.698      char *checkpoint_file = NULL;
   2.699      char *config_file = NULL;
   2.700      int paused = 0, debug = 0, daemonize = 1;
   2.701 -    int opt;
   2.702 +    int opt, rc;
   2.703  
   2.704      while ((opt = getopt(argc, argv, "hpde")) != -1) {
   2.705          switch (opt) {
   2.706 @@ -1637,7 +2037,39 @@ int main_restore(int argc, char **argv)
   2.707          help("restore");
   2.708          exit(2);
   2.709      }
   2.710 -    create_domain(debug, daemonize, config_file, checkpoint_file, paused);
   2.711 +    rc = create_domain(debug, daemonize, config_file,
   2.712 +                       checkpoint_file, paused, -1, 0);
   2.713 +    exit(-rc);
   2.714 +}
   2.715 +
   2.716 +int main_migrate_receive(int argc, char **argv)
   2.717 +{
   2.718 +    int debug = 0, daemonize = 1;
   2.719 +    int opt;
   2.720 +
   2.721 +    while ((opt = getopt(argc, argv, "hed")) != -1) {
   2.722 +        switch (opt) {
   2.723 +        case 'h':
   2.724 +            help("restore");
   2.725 +            exit(2);
   2.726 +            break;
   2.727 +        case 'e':
   2.728 +            daemonize = 0;
   2.729 +            break;
   2.730 +        case 'd':
   2.731 +            debug = 1;
   2.732 +            break;
   2.733 +        default:
   2.734 +            fprintf(stderr, "option not supported\n");
   2.735 +            break;
   2.736 +        }
   2.737 +    }
   2.738 +
   2.739 +    if (argc-optind != 0) {
   2.740 +        help("restore");
   2.741 +        exit(2);
   2.742 +    }
   2.743 +    migrate_receive(debug, daemonize);
   2.744      exit(0);
   2.745  }
   2.746  
   2.747 @@ -1674,6 +2106,59 @@ int main_save(int argc, char **argv)
   2.748      exit(0);
   2.749  }
   2.750  
   2.751 +int main_migrate(int argc, char **argv)
   2.752 +{
   2.753 +    char *p = NULL;
   2.754 +    const char *config_filename = NULL;
   2.755 +    const char *ssh_command = "ssh";
   2.756 +    char *rune = NULL;
   2.757 +    char *host;
   2.758 +    int opt, daemonize = 1, debug = 0;
   2.759 +
   2.760 +    while ((opt = getopt(argc, argv, "hC:s:ed")) != -1) {
   2.761 +        switch (opt) {
   2.762 +        case 'h':
   2.763 +            help("migrate");
   2.764 +            exit(0);
   2.765 +        case 'C':
   2.766 +            config_filename = optarg;
   2.767 +            break;
   2.768 +        case 's':
   2.769 +            ssh_command = optarg;
   2.770 +            break;
   2.771 +        case 'e':
   2.772 +            daemonize = 0;
   2.773 +            break;
   2.774 +        case 'd':
   2.775 +            debug = 1;
   2.776 +            break;
   2.777 +        default:
   2.778 +            fprintf(stderr, "option not supported\n");
   2.779 +            break;
   2.780 +        }
   2.781 +    }
   2.782 +
   2.783 +    if (argc-optind < 2 || argc-optind > 2) {
   2.784 +        help("save");
   2.785 +        exit(2);
   2.786 +    }
   2.787 +
   2.788 +    p = argv[optind];
   2.789 +    host = argv[optind + 1];
   2.790 +
   2.791 +    if (!ssh_command[0]) {
   2.792 +        rune= host;
   2.793 +    } else {
   2.794 +        asprintf(&rune, "exec %s %s xl migrate-receive%s%s",
   2.795 +                 ssh_command, host,
   2.796 +                 daemonize ? "" : " -e",
   2.797 +                 debug ? " -d" : "");
   2.798 +    }
   2.799 +
   2.800 +    migrate_domain(p, rune, config_filename);
   2.801 +    exit(0);
   2.802 +}
   2.803 +
   2.804  int main_pause(int argc, char **argv)
   2.805  {
   2.806      int opt;
   2.807 @@ -1799,7 +2284,7 @@ int main_create(int argc, char **argv)
   2.808  {
   2.809      char *filename = NULL;
   2.810      int debug = 0, daemonize = 1;
   2.811 -    int opt;
   2.812 +    int opt, rc;
   2.813  
   2.814      while ((opt = getopt(argc, argv, "hde")) != -1) {
   2.815          switch (opt) {
   2.816 @@ -1824,8 +2309,9 @@ int main_create(int argc, char **argv)
   2.817      }
   2.818  
   2.819      filename = argv[optind];
   2.820 -    create_domain(debug, daemonize, filename, NULL, 0);
   2.821 -    exit(0);
   2.822 +    rc = create_domain(debug, daemonize, filename, NULL, 0,
   2.823 +                       -1, 0);
   2.824 +    exit(-rc);
   2.825  }
   2.826  
   2.827  void button_press(char *p, char *b)
   2.828 @@ -2189,8 +2675,12 @@ int main(int argc, char **argv)
   2.829          main_console(argc - 1, argv + 1);
   2.830      } else if (!strcmp(argv[1], "save")) {
   2.831          main_save(argc - 1, argv + 1);
   2.832 +    } else if (!strcmp(argv[1], "migrate")) {
   2.833 +        main_migrate(argc - 1, argv + 1);
   2.834      } else if (!strcmp(argv[1], "restore")) {
   2.835          main_restore(argc - 1, argv + 1);
   2.836 +    } else if (!strcmp(argv[1], "migrate-receive")) {
   2.837 +        main_migrate_receive(argc - 1, argv + 1);
   2.838      } else if (!strcmp(argv[1], "cd-insert")) {
   2.839          main_cd_insert(argc - 1, argv + 1);
   2.840      } else if (!strcmp(argv[1], "cd-eject")) {