debuggers.hg

view tools/python/xen/xend/XendCheckpoint.py @ 17002:5d84464dc1fc

Fix save/restore failure cleanup

The save failure cleanup introduced in 13543:207523704fb1 is
incorrect: if we didn't get as far as actually suspending the domain,
then the guest domain will not be expecting the devices to be removed
(seen on both Linux and Solaris, which don't expect a 'Closing' state
when they hold the device open). Only re-jig devices if we definitely
shut the domain down.

Signed-off-by: John Levon <john.levon@sun.com>
author Keir Fraser <keir.fraser@citrix.com>
date Sat Feb 02 10:51:54 2008 +0000 (2008-02-02)
parents 4942f9909ec8
children 5f7bfdeb8748
line source
1 # Copyright (C) 2005 Christian Limpach <Christian.Limpach@cl.cam.ac.uk>
2 # Copyright (C) 2005 XenSource Ltd
4 # This file is subject to the terms and conditions of the GNU General
5 # Public License. See the file "COPYING" in the main directory of
6 # this archive for more details.
8 import os
9 import os.path
10 import re
11 import string
12 import threading
13 import fcntl
14 from struct import pack, unpack, calcsize
16 from xen.util.xpopen import xPopen3
17 import xen.util.auxbin
18 import xen.lowlevel.xc
20 from xen.xend import balloon, sxp, image
21 from xen.xend.XendError import XendError, VmError
22 from xen.xend.XendLogging import log
23 from xen.xend.XendConfig import XendConfig
24 from xen.xend.XendConstants import *
26 SIGNATURE = "LinuxGuestRecord"
27 QEMU_SIGNATURE = "QemuDeviceModelRecord"
28 dm_batch = 512
29 XC_SAVE = "xc_save"
30 XC_RESTORE = "xc_restore"
33 sizeof_int = calcsize("i")
34 sizeof_unsigned_int = calcsize("I")
35 sizeof_unsigned_long = calcsize("L")
38 xc = xen.lowlevel.xc.xc()
41 def write_exact(fd, buf, errmsg):
42 if os.write(fd, buf) != len(buf):
43 raise XendError(errmsg)
46 def read_exact(fd, size, errmsg):
47 buf = ''
48 while size != 0:
49 readstr = os.read(fd, size)
50 if not len(readstr):
51 log.error("read_exact: EOF trying to read %d (buf='%s')" % \
52 (size, buf))
53 raise XendError(errmsg)
54 size = size - len(readstr)
55 buf = buf + readstr
56 return buf
59 def save(fd, dominfo, network, live, dst, checkpoint=False):
60 write_exact(fd, SIGNATURE, "could not write guest state file: signature")
62 config = sxp.to_string(dominfo.sxpr())
64 domain_name = dominfo.getName()
65 # Rename the domain temporarily, so that we don't get a name clash if this
66 # domain is migrating (live or non-live) to the local host. Doing such a
67 # thing is useful for debugging.
68 dominfo.setName('migrating-' + domain_name)
70 done_suspend = 0
72 try:
73 dominfo.migrateDevices(network, dst, DEV_MIGRATE_STEP1, domain_name)
75 write_exact(fd, pack("!i", len(config)),
76 "could not write guest state file: config len")
77 write_exact(fd, config, "could not write guest state file: config")
79 image_cfg = dominfo.info.get('image', {})
80 hvm = dominfo.info.is_hvm()
82 # xc_save takes three customization parameters: maxit, max_f, and
83 # flags the last controls whether or not save is 'live', while the
84 # first two further customize behaviour when 'live' save is
85 # enabled. Passing "0" simply uses the defaults compiled into
86 # libxenguest; see the comments and/or code in xc_linux_save() for
87 # more information.
88 cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd),
89 str(dominfo.getDomid()), "0", "0",
90 str(int(live) | (int(hvm) << 2)) ]
91 log.debug("[xc_save]: %s", string.join(cmd))
93 def saveInputHandler(line, tochild):
94 log.debug("In saveInputHandler %s", line)
95 if line == "suspend":
96 log.debug("Suspending %d ...", dominfo.getDomid())
97 dominfo.shutdown('suspend')
98 dominfo.waitForShutdown()
99 done_suspend = 1
100 dominfo.migrateDevices(network, dst, DEV_MIGRATE_STEP2,
101 domain_name)
102 log.info("Domain %d suspended.", dominfo.getDomid())
103 dominfo.migrateDevices(network, dst, DEV_MIGRATE_STEP3,
104 domain_name)
105 if hvm:
106 dominfo.image.saveDeviceModel()
108 tochild.write("done\n")
109 tochild.flush()
110 log.debug('Written done')
112 forkHelper(cmd, fd, saveInputHandler, False)
114 # put qemu device model state
115 if os.path.exists("/var/lib/xen/qemu-save.%d" % dominfo.getDomid()):
116 write_exact(fd, QEMU_SIGNATURE, "could not write qemu signature")
117 qemu_fd = os.open("/var/lib/xen/qemu-save.%d" % dominfo.getDomid(),
118 os.O_RDONLY)
119 while True:
120 buf = os.read(qemu_fd, dm_batch)
121 if len(buf):
122 write_exact(fd, buf, "could not write device model state")
123 else:
124 break
125 os.close(qemu_fd)
126 os.remove("/var/lib/xen/qemu-save.%d" % dominfo.getDomid())
128 if checkpoint:
129 dominfo.resumeDomain()
130 else:
131 dominfo.destroy()
132 dominfo.testDeviceComplete()
133 try:
134 dominfo.setName(domain_name, False)
135 except VmError:
136 # Ignore this. The name conflict (hopefully) arises because we
137 # are doing localhost migration; if we are doing a suspend of a
138 # persistent VM, we need the rename, and don't expect the
139 # conflict. This needs more thought.
140 pass
142 except Exception, exn:
143 log.exception("Save failed on domain %s (%s).", domain_name,
144 dominfo.getDomid())
146 # If we didn't get as far as suspending the domain (for
147 # example, we couldn't balloon enough memory for the new
148 # domain), then we don't want to re-plumb the devices, as the
149 # domU will not be expecting it.
150 if done_suspend:
151 log.debug("XendCheckpoint.save: resumeDomain")
152 dominfo.resumeDomain()
154 try:
155 dominfo.setName(domain_name)
156 except:
157 log.exception("Failed to reset the migrating domain's name")
159 raise exn
162 def restore(xd, fd, dominfo = None, paused = False, relocating = False):
163 signature = read_exact(fd, len(SIGNATURE),
164 "not a valid guest state file: signature read")
165 if signature != SIGNATURE:
166 raise XendError("not a valid guest state file: found '%s'" %
167 signature)
169 l = read_exact(fd, sizeof_int,
170 "not a valid guest state file: config size read")
171 vmconfig_size = unpack("!i", l)[0]
172 vmconfig_buf = read_exact(fd, vmconfig_size,
173 "not a valid guest state file: config read")
175 p = sxp.Parser()
176 p.input(vmconfig_buf)
177 if not p.ready:
178 raise XendError("not a valid guest state file: config parse")
180 vmconfig = p.get_val()
182 if not relocating:
183 domconfig = XendConfig(sxp_obj = vmconfig)
184 othervm = xd.domain_lookup_nr(domconfig["name_label"])
185 if othervm is None or othervm.domid is None:
186 othervm = xd.domain_lookup_nr(domconfig["uuid"])
187 if othervm is not None and othervm.domid is not None:
188 raise VmError("Domain '%s' already exists with ID '%d'" % (domconfig["name_label"], othervm.domid))
190 if dominfo:
191 dominfo.resume()
192 else:
193 dominfo = xd.restore_(vmconfig)
195 store_port = dominfo.getStorePort()
196 console_port = dominfo.getConsolePort()
198 assert store_port
199 assert console_port
201 # if hvm, pass mem size to calculate the store_mfn
202 image_cfg = dominfo.info.get('image', {})
203 is_hvm = dominfo.info.is_hvm()
204 if is_hvm:
205 apic = int(dominfo.info['platform'].get('apic', 0))
206 pae = int(dominfo.info['platform'].get('pae', 0))
207 log.info("restore hvm domain %d, apic=%d, pae=%d",
208 dominfo.domid, apic, pae)
209 else:
210 apic = 0
211 pae = 0
213 try:
214 restore_image = image.create(dominfo, dominfo.info)
215 memory = restore_image.getRequiredAvailableMemory(
216 dominfo.info['memory_dynamic_max'] / 1024)
217 maxmem = restore_image.getRequiredAvailableMemory(
218 dominfo.info['memory_static_max'] / 1024)
219 shadow = restore_image.getRequiredShadowMemory(
220 dominfo.info['shadow_memory'] * 1024,
221 dominfo.info['memory_static_max'] / 1024)
223 log.debug("restore:shadow=0x%x, _static_max=0x%x, _static_min=0x%x, ",
224 dominfo.info['shadow_memory'],
225 dominfo.info['memory_static_max'],
226 dominfo.info['memory_static_min'])
228 # Round shadow up to a multiple of a MiB, as shadow_mem_control
229 # takes MiB and we must not round down and end up under-providing.
230 shadow = ((shadow + 1023) / 1024) * 1024
232 # set memory limit
233 xc.domain_setmaxmem(dominfo.getDomid(), maxmem)
235 balloon.free(memory + shadow)
237 shadow_cur = xc.shadow_mem_control(dominfo.getDomid(), shadow / 1024)
238 dominfo.info['shadow_memory'] = shadow_cur
240 cmd = map(str, [xen.util.auxbin.pathTo(XC_RESTORE),
241 fd, dominfo.getDomid(),
242 store_port, console_port, int(is_hvm), pae, apic])
243 log.debug("[xc_restore]: %s", string.join(cmd))
245 handler = RestoreInputHandler()
247 forkHelper(cmd, fd, handler.handler, True)
249 # We don't want to pass this fd to any other children -- we
250 # might need to recover the disk space that backs it.
251 try:
252 flags = fcntl.fcntl(fd, fcntl.F_GETFD)
253 flags |= fcntl.FD_CLOEXEC
254 fcntl.fcntl(fd, fcntl.F_SETFD, flags)
255 except:
256 pass
258 if handler.store_mfn is None:
259 raise XendError('Could not read store MFN')
261 if not is_hvm and handler.console_mfn is None:
262 raise XendError('Could not read console MFN')
264 # get qemu state and create a tmp file for dm restore
265 # Even PV guests may have QEMU stat, but its not currently
266 # used so only bother with HVM currently.
267 if is_hvm:
268 qemu_signature = read_exact(fd, len(QEMU_SIGNATURE),
269 "invalid device model signature read")
270 if qemu_signature != QEMU_SIGNATURE:
271 raise XendError("not a valid device model state: found '%s'" %
272 qemu_signature)
273 qemu_fd = os.open("/var/lib/xen/qemu-save.%d" % dominfo.getDomid(),
274 os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
275 while True:
276 buf = os.read(fd, dm_batch)
277 if len(buf):
278 write_exact(qemu_fd, buf,
279 "could not write dm state to tmp file")
280 else:
281 break
282 os.close(qemu_fd)
285 os.read(fd, 1) # Wait for source to close connection
287 dominfo.completeRestore(handler.store_mfn, handler.console_mfn)
289 #
290 # We shouldn't hold the domains_lock over a waitForDevices
291 # As this function sometime gets called holding this lock,
292 # we must release it and re-acquire it appropriately
293 #
294 from xen.xend import XendDomain
296 lock = True;
297 try:
298 XendDomain.instance().domains_lock.release()
299 except:
300 lock = False;
302 try:
303 dominfo.waitForDevices() # Wait for backends to set up
304 except Exception, exn:
305 log.exception(exn)
307 if lock:
308 XendDomain.instance().domains_lock.acquire()
310 if not paused:
311 dominfo.unpause()
313 return dominfo
314 except:
315 dominfo.destroy()
316 raise
319 class RestoreInputHandler:
320 def __init__(self):
321 self.store_mfn = None
322 self.console_mfn = None
325 def handler(self, line, _):
326 m = re.match(r"^(store-mfn) (\d+)$", line)
327 if m:
328 self.store_mfn = int(m.group(2))
329 else:
330 m = re.match(r"^(console-mfn) (\d+)$", line)
331 if m:
332 self.console_mfn = int(m.group(2))
335 def forkHelper(cmd, fd, inputHandler, closeToChild):
336 child = xPopen3(cmd, True, -1, [fd, xc.handle()])
338 if closeToChild:
339 child.tochild.close()
341 thread = threading.Thread(target = slurp, args = (child.childerr,))
342 thread.start()
344 try:
345 try:
346 while 1:
347 line = child.fromchild.readline()
348 if line == "":
349 break
350 else:
351 line = line.rstrip()
352 log.debug('%s', line)
353 inputHandler(line, child.tochild)
355 except IOError, exn:
356 raise XendError('Error reading from child process for %s: %s' %
357 (cmd, exn))
358 finally:
359 child.fromchild.close()
360 if not closeToChild:
361 child.tochild.close()
362 thread.join()
363 child.childerr.close()
364 status = child.wait()
366 if status >> 8 == 127:
367 raise XendError("%s failed: popen failed" % string.join(cmd))
368 elif status != 0:
369 raise XendError("%s failed" % string.join(cmd))
372 def slurp(infile):
373 while 1:
374 line = infile.readline()
375 if line == "":
376 break
377 else:
378 line = line.strip()
379 m = re.match(r"^ERROR: (.*)", line)
380 if m is None:
381 log.info('%s', line)
382 else:
383 log.error('%s', m.group(1))