\documentclass[xcolor={dvipsnames}]{beamer}

\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
\usepackage{xcolor,colortbl}
\usepackage{fancyvrb}
\usepackage{minted}
\usepackage{authoraftertitle}
\usepackage[normalem]{ulem}

\usetheme{Boadilla}

\author{Andrew Cooper}
\title{CPUID handling for guests}
\institute{Citrix XenServer}
\date{Friday $26^{th}$ August 2016}

\begin{document}

\frame{\titlepage}

\begin{frame}[fragile]{The CPUID instruction}

  \begin{itemize}
    \item Introduced in 1993
      \begin{itemize}
        \item Takes input parameters in \verb!%eax! and \verb!%ecx!
        \item Returns values in \verb!%eax!, \verb!%ebx!, \verb!%ecx! and \verb!%edx!
      \end{itemize}
    \item<2-> Return information about the processor
      \begin{itemize}
        \item Identifying information (\verb!GenuineIntel!, \verb!AuthenticAMD!, etc)
        \item Feature information (available instructions, MSRs, etc)
        \item Topology information (sockets, cores, threads, caches, TLBs, etc)
      \end{itemize}
    \item<3-> Unpriveleged
      \begin{itemize}
        \item Useable by userspace
        \item Doesn't trap to supervisor mode
      \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}[fragile]{OS expectations}

  \begin{itemize}
    \item CPUID information don't change after boot
    \item<2-> Some kernels binary patch themselves (e.g. Alternatives framework)
      \begin{itemize}
        \item e.g. New features in fastpaths (SMAP)
        \item e.g. Errata workarounds
      \end{itemize}
    \item<3-> Userspace function dispatching at process start
      \begin{itemize}
        \item e.g. Most efficient \verb!memset()!
        \item e.g. Hardware crypto if available
        \item e.g. Audio or video processing
      \end{itemize}
    \item<4-> Migration modelled as suspend/resume
      \begin{itemize}
        \item Not a reboot
      \end{itemize}
    \item<5-> Guest must not observe a loss of dependent features
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{How compatible does hardware need to be?}

  \begin{itemize}
    \item All features previously used need to continue functioning
      \begin{itemize}
        \item All instructions, MSRs, etc
      \end{itemize}
    \item<2-> Available features controlled by:
      \begin{itemize}
        \item CPU family, model and stepping
        \item Firmware version and settings
        \item Microcode version
        \item Xen's hardware support and administrator choices
      \end{itemize}
    \item<3-> Typically, ``identical'' servers are not
      \begin{itemize}
        \item Different CPUs, especially on RMA'd hardware
        \item Configuring firmware is tedious and error prone
      \end{itemize}
    \item<4-> Must lie to guests (for their own good)
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Controlling a guests view of information}

  \begin{itemize}
    \item HVM guests
      \begin{itemize}
        \item \verb!CPUID! instruction traps to hypervisor
        \item Xen can control all information seen
      \end{itemize}
    \item<2-> PV guests
      \begin{itemize}
        \item \verb!CPUID! instruction doesn't trap
        \item Xen has no direct control
      \end{itemize}
    \item<3-> PV Emulated \verb!CPUID!
      \begin{itemize}
        \item Software can opt in to Xen's control
        \item Adhoc use in PV guest kernels, but not by userspace
      \end{itemize}
    \item<4-> \verb!CPUID! Faulting
      \begin{itemize}
        \item Non-architectural, but available in Intel IvyBridge and later
        \item Causes \verb!CPUID! to fault with \verb!#GP(0)!
        \item Xen can control all information seen
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Controlling a guests view of information (2)}

  \begin{itemize}
    \item \verb!CPUID! Masking
      \begin{itemize}
        \item Non-architectural, ``documented'' only in whitepapers
      \end{itemize}
    \item<2-> AMD (All hardware Xen can run on)
      \begin{itemize}
        \item Override MSRs
        \item Must be careful not to advertise features unsupported in silicon
        \item Covers basic and extended feature leaves
      \end{itemize}
    \item<3-> Intel (only some Nehalem/Westmere processors, SandyBridge)
      \begin{itemize}
        \item AND mask against the real hardware values
        \item Different MSRs on different hardware
        \item Covers basic, extended and xsave feature leaves
      \end{itemize}
    \item<4-> Magic \verb!CPUID! bits
      \begin{itemize}
        \item \verb!APIC! and \verb!OSXSAVE! bits fast forwarded from other state
        \item Interaction with masking completely undocumented
        \item Behaviour reverse engineered, hopefully right
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Problematic areas to control}

  \begin{itemize}
    \item Guests can probe for some features
      \begin{itemize}
        \item 1GB Superpage support
        \item AES instructions
      \end{itemize}
    \item<2-> Some features don't have feature bits
      \begin{itemize}
        \item AMD's Long Mode Segment Limit Enable
      \end{itemize}
    \item<3-> FPU pipeline behaviour exposed directly to guests
      \begin{itemize}
        \item \verb!MXCSR_MASK!
        \item Intel's \verb!FPDP! and \verb!FPCSDS!
      \end{itemize}
    \item<4-> Some feature bits affect the interpretation of other leaves
      \begin{itemize}
        \item \verb!CMP_LEGACY!, \verb!HTT! and \verb!X2APIC! affect the
          topology interpretation
        \item Can't control topology information with masking
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{CPUID handling in Xen}

  \begin{itemize}
    \item No real develoment since the uni-vcpu guest days
      \begin{itemize}
        \item No concept of topology information
      \end{itemize}
    \item<2-> No auditing of the toolstack-provided information
      \begin{itemize}
        \item Lots of runtime adjustment to regain plausibility
      \end{itemize}
    \item<3-> Inconsistent whitelist/blacklist approach
      \begin{itemize}
        \item Different approaches between different leaves
        \item Different approaches between different guest types
        \item Some information passed straight through from hardware
      \end{itemize}
    \item<4-> No knowledge of feature dependency
      \begin{itemize}
        \item Buggy assumptions in guests result in crashes
      \end{itemize}
    \item<5-> No way for the toolstack to evaluate the hardware capability
      \begin{itemize}
        \item Feature internals exposed in the ABI, with no API
      \end{itemize}
    \item<6-> PV dependence on leaked \verb!CPUID! information
      \begin{itemize}
        \item Hardware domain for C/P states, MTRRs
        \item Control domain for building guests
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{CPUID-related improvements in Xen 4.7}
  \begin{itemize}
    \item Introduce ``a featureset'' in the API/ABI
      \begin{itemize}
        \item Keys as specified in architecture manuals
        \item Can be treated as opaque by higher level software
      \end{itemize}
    \item<2-> At boot, Xen calculates featuresets for itself and the toolstack:
  \end{itemize}
  \begin{description}
    \item<2->[raw]  Features supported by hardware
    \item<2->[host] Features used by Xen (after errata, command line, etc)
    \item<2->[pv]   Maximum featureset available for PV guests
    \item<2->[hvm]  Maximum featureset available for HVM guests
  \end{description}
  \begin{itemize}
    \item<3-> Shared Xen/libxc algorithm for feature dependencies
      \begin{itemize}
        \item Provides consistent logic between Xen and libxc
        \item Build-time calculations to avoid complicated runtime logic
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[fragile]{Future work}
  \begin{itemize}
    \item Extend the featureset concept to a full \verb!CPUID! policy
      \begin{itemize}
        \item Needs guest topology information
        \item Will allow control of \verb!MAXPHYSADDR!
      \end{itemize}
    \item<2-> Improved hypercalls for policies
      \begin{itemize}
        \item \verb!XEN_DOMCTL_get_cpuid_policy!
        \item Xen to audit policy plausibility at hypercall time, not runtime
      \end{itemize}
    \item<3-> More migration stream work
      \begin{itemize}
        \item \verb!CPUID! policy should be in the migration stream
        \item Avoid mistakes when regenerating
        \item Can finaly audit RAM/CPU state against reality
        \item Allows for a pre-migrate check at the destination
      \end{itemize}
    \item<4-> MSRs
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{\MyTitle}
  \begin{center}
    Any Questions?
  \end{center}
\end{frame}

\end{document}

\iffalse

\begin{frame}
  \frametitle{...}
\end{frame}

  \begin{itemize}
    \item
  \end{itemize}

\fi
