/*

 * This program is free software; you can redistribute it and/or modify it

 * under the terms and conditions of the GNU General Public License,

 * version 2, as published by the Free Software Foundation.

 *

 * This program is distributed in the hope it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for

 * more details.

 *

 * You should have received a copy of the GNU General Public License along with

 * this program; If not, see <http://www.gnu.org/licenses/>.

 */

#include <xen/sched.h>

#include <xen/iommu.h>

#include <xen/paging.h>

#include <xen/guest_access.h>

#include <xen/event.h>

#include <xen/softirq.h>

#include <xen/keyhandler.h>

#include <xsm/xsm.h>

static int parse_iommu_param(const char *s);

static void iommu_dump_p2m_table(unsigned char key);

unsigned int __read_mostly iommu_dev_iotlb_timeout = 1000;

integer_param("iommu_dev_iotlb_timeout", iommu_dev_iotlb_timeout);

/*

 * The 'iommu' parameter enables the IOMMU.  Optional comma separated

 * value may contain:

 *

 *   off|no|false|disable       Disable IOMMU (default)

 *   force|required             Don't boot unless IOMMU is enabled

 *   no-intremap                Disable interrupt remapping

 *   intpost                    Enable VT-d Interrupt posting

 *   verbose                    Be more verbose

 *   debug                      Enable debugging messages and checks

 *   workaround_bios_bug        Workaround some bios issue to still enable

 *                              VT-d, don't guarantee security

 *   dom0-passthrough           No DMA translation at all for Dom0

 *   dom0-strict                No 1:1 memory mapping for Dom0

 *   no-sharept                 Don't share VT-d and EPT page tables

 *   no-snoop                   Disable VT-d Snoop Control

 *   no-qinval                  Disable VT-d Queued Invalidation

 *   no-igfx                    Disable VT-d for IGD devices (insecure)

 *   no-amd-iommu-perdev-intremap Don't use per-device interrupt remapping

 *                              tables (insecure)

 */

custom_param("iommu", parse_iommu_param);

bool_t __initdata iommu_enable = 1;

bool_t __read_mostly iommu_enabled;

bool_t __read_mostly force_iommu;

bool_t __hwdom_initdata iommu_dom0_strict;

bool_t __read_mostly iommu_verbose;

bool_t __read_mostly iommu_workaround_bios_bug;

bool_t __read_mostly iommu_igfx = 1;

bool_t __read_mostly iommu_passthrough;

bool_t __read_mostly iommu_snoop = 1;

bool_t __read_mostly iommu_qinval = 1;

bool_t __read_mostly iommu_intremap = 1;

/*

 * In the current implementation of VT-d posted interrupts, in some extreme

 * cases, the per cpu list which saves the blocked vCPU will be very long,

 * and this will affect the interrupt latency, so let this feature off by

 * default until we find a good solution to resolve it.

 */

bool_t __read_mostly iommu_intpost;

bool_t __read_mostly iommu_hap_pt_share = 1;

bool_t __read_mostly iommu_debug;

bool_t __read_mostly amd_iommu_perdev_intremap = 1;

DEFINE_PER_CPU(bool_t, iommu_dont_flush_iotlb);

DEFINE_SPINLOCK(iommu_pt_cleanup_lock);

PAGE_LIST_HEAD(iommu_pt_cleanup_list);

static struct tasklet iommu_pt_cleanup_tasklet;

static int __init parse_iommu_param(const char *s)

{

    const char *ss;

    int val, b, rc = 0;

    do {

        val = !!strncmp(s, "no-", 3);

        if ( !val )

            s += 3;

        ss = strchr(s, ',');

        if ( !ss )

            ss = strchr(s, '\0');

        b = parse_bool(s, ss);

        if ( b >= 0 )

            iommu_enable = b;

        else if ( !strncmp(s, "force", ss - s) ||

                  !strncmp(s, "required", ss - s) )

            force_iommu = val;

        else if ( !strncmp(s, "workaround_bios_bug", ss - s) )

            iommu_workaround_bios_bug = val;

        else if ( !strncmp(s, "igfx", ss - s) )

            iommu_igfx = val;

        else if ( !strncmp(s, "verbose", ss - s) )

            iommu_verbose = val;

        else if ( !strncmp(s, "snoop", ss - s) )

            iommu_snoop = val;

        else if ( !strncmp(s, "qinval", ss - s) )

            iommu_qinval = val;

        else if ( !strncmp(s, "intremap", ss - s) )

            iommu_intremap = val;

        else if ( !strncmp(s, "intpost", ss - s) )

            iommu_intpost = val;

        else if ( !strncmp(s, "debug", ss - s) )

        {

            iommu_debug = val;

            if ( val )

                iommu_verbose = 1;

        }

        else if ( !strncmp(s, "amd-iommu-perdev-intremap", ss - s) )

            amd_iommu_perdev_intremap = val;

        else if ( !strncmp(s, "dom0-passthrough", ss - s) )

            iommu_passthrough = val;

        else if ( !strncmp(s, "dom0-strict", ss - s) )

            iommu_dom0_strict = val;

        else if ( !strncmp(s, "sharept", ss - s) )

            iommu_hap_pt_share = val;

        else

            rc = -EINVAL;

        s = ss + 1;

    } while ( *ss );

    return rc;

}

int iommu_domain_init(struct domain *d)

{

    struct domain_iommu *hd = dom_iommu(d);

    int ret = 0;

    ret = arch_iommu_domain_init(d);

    if ( ret )

        return ret;

    if ( !iommu_enabled )

        return 0;

    hd->platform_ops = iommu_get_ops();

    return hd->platform_ops->init(d);

}

static void __hwdom_init check_hwdom_reqs(struct domain *d)

{

    if ( !paging_mode_translate(d) )

        return;

    arch_iommu_check_autotranslated_hwdom(d);

    if ( iommu_passthrough )

        panic("Dom0 uses paging translated mode, dom0-passthrough must not be "

              "enabled\n");

    iommu_dom0_strict = 1;

}

void __hwdom_init iommu_hwdom_init(struct domain *d)

{

    const struct domain_iommu *hd = dom_iommu(d);

    check_hwdom_reqs(d);

    if ( !iommu_enabled )

        return;

    register_keyhandler('o', &iommu_dump_p2m_table, "dump iommu p2m table", 0);

    d->need_iommu = !!iommu_dom0_strict;

    if ( need_iommu(d) && !iommu_use_hap_pt(d) )

    {

        struct page_info *page;

        unsigned int i = 0;

        int rc = 0;

        page_list_for_each ( page, &d->page_list )

        {

            unsigned long mfn = page_to_mfn(page);

            unsigned long gfn = mfn_to_gmfn(d, mfn);

            unsigned int mapping = IOMMUF_readable;

            int ret;

            if ( ((page->u.inuse.type_info & PGT_count_mask) == 0) ||

                 ((page->u.inuse.type_info & PGT_type_mask)

                  == PGT_writable_page) )

                mapping |= IOMMUF_writable;

            ret = hd->platform_ops->map_page(d, gfn, mfn, mapping);

            if ( !rc )

                rc = ret;

            if ( !(i++ & 0xfffff) )

                process_pending_softirqs();

        }

        if ( rc )

            printk(XENLOG_WARNING "d%d: IOMMU mapping failed: %d\n",

                   d->domain_id, rc);

    }

    return hd->platform_ops->hwdom_init(d);

}

void iommu_teardown(struct domain *d)

{

    const struct domain_iommu *hd = dom_iommu(d);

    d->need_iommu = 0;

    hd->platform_ops->teardown(d);

    tasklet_schedule(&iommu_pt_cleanup_tasklet);

}

int iommu_construct(struct domain *d)

{

    if ( need_iommu(d) > 0 )

        return 0;

    if ( !iommu_use_hap_pt(d) )

    {

        int rc;

        rc = arch_iommu_populate_page_table(d);

        if ( rc )

            return rc;

    }

    d->need_iommu = 1;

    /*

     * There may be dirty cache lines when a device is assigned

     * and before need_iommu(d) becoming true, this will cause

     * memory_type_changed lose effect if memory type changes.

     * Call memory_type_changed here to amend this.

     */

    memory_type_changed(d);

    return 0;

}

void iommu_domain_destroy(struct domain *d)

{

    if ( !iommu_enabled || !dom_iommu(d)->platform_ops )

        return;

    iommu_teardown(d);

    arch_iommu_domain_destroy(d);

}

int iommu_map_page(struct domain *d, unsigned long gfn, unsigned long mfn,

                   unsigned int flags)

{

    const struct domain_iommu *hd = dom_iommu(d);

    int rc;

    if ( !iommu_enabled || !hd->platform_ops )

        return 0;

    rc = hd->platform_ops->map_page(d, gfn, mfn, flags);

    if ( unlikely(rc) )

    {

        if ( !d->is_shutting_down && printk_ratelimit() )

            printk(XENLOG_ERR

                   "d%d: IOMMU mapping gfn %#lx to mfn %#lx failed: %d\n",

                   d->domain_id, gfn, mfn, rc);

        if ( !is_hardware_domain(d) )

            domain_crash(d);

    }

    return rc;

}

int iommu_unmap_page(struct domain *d, unsigned long gfn)

{

    const struct domain_iommu *hd = dom_iommu(d);

    int rc;

    if ( !iommu_enabled || !hd->platform_ops )

        return 0;

    rc = hd->platform_ops->unmap_page(d, gfn);

    if ( unlikely(rc) )

    {

        if ( !d->is_shutting_down && printk_ratelimit() )

            printk(XENLOG_ERR

                   "d%d: IOMMU unmapping gfn %#lx failed: %d\n",

                   d->domain_id, gfn, rc);

        if ( !is_hardware_domain(d) )

            domain_crash(d);

    }

    return rc;

}

static void iommu_free_pagetables(unsigned long unused)

{

    do {

        struct page_info *pg;

        spin_lock(&iommu_pt_cleanup_lock);

        pg = page_list_remove_head(&iommu_pt_cleanup_list);

        spin_unlock(&iommu_pt_cleanup_lock);

        if ( !pg )

            return;

        iommu_get_ops()->free_page_table(pg);

    } while ( !softirq_pending(smp_processor_id()) );

    tasklet_schedule_on_cpu(&iommu_pt_cleanup_tasklet,

                            cpumask_cycle(smp_processor_id(), &cpu_online_map));

}

int iommu_iotlb_flush(struct domain *d, unsigned long gfn,

                      unsigned int page_count)

{

    const struct domain_iommu *hd = dom_iommu(d);

    int rc;

    if ( !iommu_enabled || !hd->platform_ops || !hd->platform_ops->iotlb_flush )

        return 0;

    rc = hd->platform_ops->iotlb_flush(d, gfn, page_count);

    if ( unlikely(rc) )

    {

        if ( !d->is_shutting_down && printk_ratelimit() )

            printk(XENLOG_ERR

                   "d%d: IOMMU IOTLB flush failed: %d, gfn %#lx, page count %u\n",

                   d->domain_id, rc, gfn, page_count);

        if ( !is_hardware_domain(d) )

            domain_crash(d);

    }

    return rc;

}

int iommu_iotlb_flush_all(struct domain *d)

{

    const struct domain_iommu *hd = dom_iommu(d);

    int rc;

    if ( !iommu_enabled || !hd->platform_ops || !hd->platform_ops->iotlb_flush_all )

        return 0;

    rc = hd->platform_ops->iotlb_flush_all(d);

    if ( unlikely(rc) )

    {

        if ( !d->is_shutting_down && printk_ratelimit() )

            printk(XENLOG_ERR

                   "d%d: IOMMU IOTLB flush all failed: %d\n",

                   d->domain_id, rc);

        if ( !is_hardware_domain(d) )

            domain_crash(d);

    }

    return rc;

}

int __init iommu_setup(void)

{

    int rc = -ENODEV;

    bool_t force_intremap = force_iommu && iommu_intremap;

    if ( iommu_dom0_strict )

        iommu_passthrough = 0;

    if ( iommu_enable )

    {

        rc = iommu_hardware_setup();

        iommu_enabled = (rc == 0);

    }

    if ( !iommu_enabled )

        iommu_intremap = 0;

    if ( (force_iommu && !iommu_enabled) ||

         (force_intremap && !iommu_intremap) )

        panic("Couldn't enable %s and iommu=required/force",

              !iommu_enabled ? "IOMMU" : "Interrupt Remapping");

    if ( !iommu_intremap )

        iommu_intpost = 0;

    if ( !iommu_enabled )

    {

        iommu_snoop = 0;

        iommu_passthrough = 0;

        iommu_dom0_strict = 0;

    }

    printk("I/O virtualisation %sabled\n", iommu_enabled ? "en" : "dis");

    if ( iommu_enabled )

    {

        printk(" - Dom0 mode: %s\n",

               iommu_passthrough ? "Passthrough" :

               iommu_dom0_strict ? "Strict" : "Relaxed");

        printk("Interrupt remapping %sabled\n", iommu_intremap ? "en" : "dis");

        tasklet_init(&iommu_pt_cleanup_tasklet, iommu_free_pagetables, 0);

    }

    return rc;

}

int iommu_suspend()

{

    if ( iommu_enabled )

        return iommu_get_ops()->suspend();

    return 0;

}

void iommu_resume()

{

    if ( iommu_enabled )

        iommu_get_ops()->resume();

}

int iommu_do_domctl(

    struct xen_domctl *domctl, struct domain *d,

    XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)

{

    int ret = -ENODEV;

    if ( !iommu_enabled )

        return -ENOSYS;

#ifdef CONFIG_HAS_PCI

    ret = iommu_do_pci_domctl(domctl, d, u_domctl);

#endif

#ifdef CONFIG_HAS_DEVICE_TREE

    if ( ret == -ENODEV )

        ret = iommu_do_dt_domctl(domctl, d, u_domctl);

#endif

    return ret;

}

void iommu_share_p2m_table(struct domain* d)

{

    if ( iommu_enabled && iommu_use_hap_pt(d) )

        iommu_get_ops()->share_p2m(d);

}

void iommu_crash_shutdown(void)

{

    if ( iommu_enabled )

        iommu_get_ops()->crash_shutdown();

    iommu_enabled = iommu_intremap = iommu_intpost = 0;

}

int iommu_get_reserved_device_memory(iommu_grdm_t *func, void *ctxt)

{

    const struct iommu_ops *ops;

    if ( !iommu_enabled )

        return 0;

    ops = iommu_get_ops();

    if ( !ops->get_reserved_device_memory )

        return 0;

    return ops->get_reserved_device_memory(func, ctxt);

}

bool_t iommu_has_feature(struct domain *d, enum iommu_feature feature)

{

    if ( !iommu_enabled )

        return 0;

    return test_bit(feature, dom_iommu(d)->features);

}

static void iommu_dump_p2m_table(unsigned char key)

{

    struct domain *d;

    const struct iommu_ops *ops;

    if ( !iommu_enabled )

    {

        printk("IOMMU not enabled!\n");

        return;

    }

    ops = iommu_get_ops();

    for_each_domain(d)

    {

        if ( is_hardware_domain(d) || need_iommu(d) <= 0 )

            continue;

        if ( iommu_use_hap_pt(d) )

        {

            printk("\ndomain%d IOMMU p2m table shared with MMU: \n", d->domain_id);

            continue;

        }

        printk("\ndomain%d IOMMU p2m table: \n", d->domain_id);

        ops->dump_p2m_table(d);

    }

}

/*

 * Local variables:

 * mode: C

 * c-file-style: "BSD"

 * c-basic-offset: 4

 * indent-tabs-mode: nil

 * End:

 */