| /* |
| * Copyright (C) 2012 - Virtual Open Systems and Columbia University |
| * Author: Christoffer Dall <c.dall@virtualopensystems.com> |
| * |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License, version 2, as |
| * published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, write to the Free Software |
| * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| */ |
| |
| #include <linux/linkage.h> |
| #include <linux/const.h> |
| #include <asm/unified.h> |
| #include <asm/page.h> |
| #include <asm/ptrace.h> |
| #include <asm/asm-offsets.h> |
| #include <asm/kvm_asm.h> |
| #include <asm/kvm_arm.h> |
| #include <asm/vfpmacros.h> |
| #include "interrupts_head.S" |
| |
| .text |
| |
| __kvm_hyp_code_start: |
| .globl __kvm_hyp_code_start |
| |
| /******************************************************************** |
| * Flush per-VMID TLBs |
| * |
| * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); |
| * |
| * We rely on the hardware to broadcast the TLB invalidation to all CPUs |
| * inside the inner-shareable domain (which is the case for all v7 |
| * implementations). If we come across a non-IS SMP implementation, we'll |
| * have to use an IPI based mechanism. Until then, we stick to the simple |
| * hardware assisted version. |
| * |
| * As v7 does not support flushing per IPA, just nuke the whole TLB |
| * instead, ignoring the ipa value. |
| */ |
| ENTRY(__kvm_tlb_flush_vmid_ipa) |
| push {r2, r3} |
| |
| dsb ishst |
| add r0, r0, #KVM_VTTBR |
| ldrd r2, r3, [r0] |
| mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR |
| isb |
| mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored) |
| dsb ish |
| isb |
| mov r2, #0 |
| mov r3, #0 |
| mcrr p15, 6, r2, r3, c2 @ Back to VMID #0 |
| isb @ Not necessary if followed by eret |
| |
| pop {r2, r3} |
| bx lr |
| ENDPROC(__kvm_tlb_flush_vmid_ipa) |
| |
| /** |
| * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs |
| * |
| * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address |
| * parameter |
| */ |
| |
| ENTRY(__kvm_tlb_flush_vmid) |
| b __kvm_tlb_flush_vmid_ipa |
| ENDPROC(__kvm_tlb_flush_vmid) |
| |
| /******************************************************************** |
| * Flush TLBs and instruction caches of all CPUs inside the inner-shareable |
| * domain, for all VMIDs |
| * |
| * void __kvm_flush_vm_context(void); |
| */ |
| ENTRY(__kvm_flush_vm_context) |
| mov r0, #0 @ rn parameter for c15 flushes is SBZ |
| |
| /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */ |
| mcr p15, 4, r0, c8, c3, 4 |
| /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */ |
| mcr p15, 0, r0, c7, c1, 0 |
| dsb ish |
| isb @ Not necessary if followed by eret |
| |
| bx lr |
| ENDPROC(__kvm_flush_vm_context) |
| |
| |
| /******************************************************************** |
| * Hypervisor world-switch code |
| * |
| * |
| * int __kvm_vcpu_run(struct kvm_vcpu *vcpu) |
| */ |
| ENTRY(__kvm_vcpu_run) |
| @ Save the vcpu pointer |
| mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR |
| |
| save_host_regs |
| |
| restore_vgic_state |
| restore_timer_state |
| |
| @ Store hardware CP15 state and load guest state |
| read_cp15_state store_to_vcpu = 0 |
| write_cp15_state read_from_vcpu = 1 |
| |
| @ If the host kernel has not been configured with VFPv3 support, |
| @ then it is safer if we deny guests from using it as well. |
| #ifdef CONFIG_VFPv3 |
| @ Set FPEXC_EN so the guest doesn't trap floating point instructions |
| VFPFMRX r2, FPEXC @ VMRS |
| push {r2} |
| orr r2, r2, #FPEXC_EN |
| VFPFMXR FPEXC, r2 @ VMSR |
| #endif |
| |
| @ Configure Hyp-role |
| configure_hyp_role vmentry |
| |
| @ Trap coprocessor CRx accesses |
| set_hstr vmentry |
| set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)) |
| set_hdcr vmentry |
| |
| @ Write configured ID register into MIDR alias |
| ldr r1, [vcpu, #VCPU_MIDR] |
| mcr p15, 4, r1, c0, c0, 0 |
| |
| @ Write guest view of MPIDR into VMPIDR |
| ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)] |
| mcr p15, 4, r1, c0, c0, 5 |
| |
| @ Set up guest memory translation |
| ldr r1, [vcpu, #VCPU_KVM] |
| add r1, r1, #KVM_VTTBR |
| ldrd r2, r3, [r1] |
| mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR |
| |
| @ We're all done, just restore the GPRs and go to the guest |
| restore_guest_regs |
| clrex @ Clear exclusive monitor |
| eret |
| |
| __kvm_vcpu_return: |
| /* |
| * return convention: |
| * guest r0, r1, r2 saved on the stack |
| * r0: vcpu pointer |
| * r1: exception code |
| */ |
| save_guest_regs |
| |
| @ Set VMID == 0 |
| mov r2, #0 |
| mov r3, #0 |
| mcrr p15, 6, r2, r3, c2 @ Write VTTBR |
| |
| @ Don't trap coprocessor accesses for host kernel |
| set_hstr vmexit |
| set_hdcr vmexit |
| set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)), after_vfp_restore |
| |
| #ifdef CONFIG_VFPv3 |
| @ Switch VFP/NEON hardware state to the host's |
| add r7, vcpu, #VCPU_VFP_GUEST |
| store_vfp_state r7 |
| add r7, vcpu, #VCPU_VFP_HOST |
| ldr r7, [r7] |
| restore_vfp_state r7 |
| |
| after_vfp_restore: |
| @ Restore FPEXC_EN which we clobbered on entry |
| pop {r2} |
| VFPFMXR FPEXC, r2 |
| #else |
| after_vfp_restore: |
| #endif |
| |
| @ Reset Hyp-role |
| configure_hyp_role vmexit |
| |
| @ Let host read hardware MIDR |
| mrc p15, 0, r2, c0, c0, 0 |
| mcr p15, 4, r2, c0, c0, 0 |
| |
| @ Back to hardware MPIDR |
| mrc p15, 0, r2, c0, c0, 5 |
| mcr p15, 4, r2, c0, c0, 5 |
| |
| @ Store guest CP15 state and restore host state |
| read_cp15_state store_to_vcpu = 1 |
| write_cp15_state read_from_vcpu = 0 |
| |
| save_timer_state |
| save_vgic_state |
| |
| restore_host_regs |
| clrex @ Clear exclusive monitor |
| #ifndef CONFIG_CPU_ENDIAN_BE8 |
| mov r0, r1 @ Return the return code |
| mov r1, #0 @ Clear upper bits in return value |
| #else |
| @ r1 already has return code |
| mov r0, #0 @ Clear upper bits in return value |
| #endif /* CONFIG_CPU_ENDIAN_BE8 */ |
| bx lr @ return to IOCTL |
| |
| /******************************************************************** |
| * Call function in Hyp mode |
| * |
| * |
| * u64 kvm_call_hyp(void *hypfn, ...); |
| * |
| * This is not really a variadic function in the classic C-way and care must |
| * be taken when calling this to ensure parameters are passed in registers |
| * only, since the stack will change between the caller and the callee. |
| * |
| * Call the function with the first argument containing a pointer to the |
| * function you wish to call in Hyp mode, and subsequent arguments will be |
| * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the |
| * function pointer can be passed). The function being called must be mapped |
| * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are |
| * passed in r0 and r1. |
| * |
| * A function pointer with a value of 0xffffffff has a special meaning, |
| * and is used to implement __hyp_get_vectors in the same way as in |
| * arch/arm/kernel/hyp_stub.S. |
| * |
| * The calling convention follows the standard AAPCS: |
| * r0 - r3: caller save |
| * r12: caller save |
| * rest: callee save |
| */ |
| ENTRY(kvm_call_hyp) |
| hvc #0 |
| bx lr |
| |
| /******************************************************************** |
| * Hypervisor exception vector and handlers |
| * |
| * |
| * The KVM/ARM Hypervisor ABI is defined as follows: |
| * |
| * Entry to Hyp mode from the host kernel will happen _only_ when an HVC |
| * instruction is issued since all traps are disabled when running the host |
| * kernel as per the Hyp-mode initialization at boot time. |
| * |
| * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc |
| * below) when the HVC instruction is called from SVC mode (i.e. a guest or the |
| * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC |
| * instructions are called from within Hyp-mode. |
| * |
| * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode): |
| * Switching to Hyp mode is done through a simple HVC #0 instruction. The |
| * exception vector code will check that the HVC comes from VMID==0 and if |
| * so will push the necessary state (SPSR, lr_usr) on the Hyp stack. |
| * - r0 contains a pointer to a HYP function |
| * - r1, r2, and r3 contain arguments to the above function. |
| * - The HYP function will be called with its arguments in r0, r1 and r2. |
| * On HYP function return, we return directly to SVC. |
| * |
| * Note that the above is used to execute code in Hyp-mode from a host-kernel |
| * point of view, and is a different concept from performing a world-switch and |
| * executing guest code SVC mode (with a VMID != 0). |
| */ |
| |
| /* Handle undef, svc, pabt, or dabt by crashing with a user notice */ |
| .macro bad_exception exception_code, panic_str |
| push {r0-r2} |
| mrrc p15, 6, r0, r1, c2 @ Read VTTBR |
| lsr r1, r1, #16 |
| ands r1, r1, #0xff |
| beq 99f |
| |
| load_vcpu @ Load VCPU pointer |
| .if \exception_code == ARM_EXCEPTION_DATA_ABORT |
| mrc p15, 4, r2, c5, c2, 0 @ HSR |
| mrc p15, 4, r1, c6, c0, 0 @ HDFAR |
| str r2, [vcpu, #VCPU_HSR] |
| str r1, [vcpu, #VCPU_HxFAR] |
| .endif |
| .if \exception_code == ARM_EXCEPTION_PREF_ABORT |
| mrc p15, 4, r2, c5, c2, 0 @ HSR |
| mrc p15, 4, r1, c6, c0, 2 @ HIFAR |
| str r2, [vcpu, #VCPU_HSR] |
| str r1, [vcpu, #VCPU_HxFAR] |
| .endif |
| mov r1, #\exception_code |
| b __kvm_vcpu_return |
| |
| @ We were in the host already. Let's craft a panic-ing return to SVC. |
| 99: mrs r2, cpsr |
| bic r2, r2, #MODE_MASK |
| orr r2, r2, #SVC_MODE |
| THUMB( orr r2, r2, #PSR_T_BIT ) |
| msr spsr_cxsf, r2 |
| mrs r1, ELR_hyp |
| ldr r2, =BSYM(panic) |
| msr ELR_hyp, r2 |
| ldr r0, =\panic_str |
| clrex @ Clear exclusive monitor |
| eret |
| .endm |
| |
| .text |
| |
| .align 5 |
| __kvm_hyp_vector: |
| .globl __kvm_hyp_vector |
| |
| @ Hyp-mode exception vector |
| W(b) hyp_reset |
| W(b) hyp_undef |
| W(b) hyp_svc |
| W(b) hyp_pabt |
| W(b) hyp_dabt |
| W(b) hyp_hvc |
| W(b) hyp_irq |
| W(b) hyp_fiq |
| |
| .align |
| hyp_reset: |
| b hyp_reset |
| |
| .align |
| hyp_undef: |
| bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str |
| |
| .align |
| hyp_svc: |
| bad_exception ARM_EXCEPTION_HVC, svc_die_str |
| |
| .align |
| hyp_pabt: |
| bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str |
| |
| .align |
| hyp_dabt: |
| bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str |
| |
| .align |
| hyp_hvc: |
| /* |
| * Getting here is either becuase of a trap from a guest or from calling |
| * HVC from the host kernel, which means "switch to Hyp mode". |
| */ |
| push {r0, r1, r2} |
| |
| @ Check syndrome register |
| mrc p15, 4, r1, c5, c2, 0 @ HSR |
| lsr r0, r1, #HSR_EC_SHIFT |
| #ifdef CONFIG_VFPv3 |
| cmp r0, #HSR_EC_CP_0_13 |
| beq switch_to_guest_vfp |
| #endif |
| cmp r0, #HSR_EC_HVC |
| bne guest_trap @ Not HVC instr. |
| |
| /* |
| * Let's check if the HVC came from VMID 0 and allow simple |
| * switch to Hyp mode |
| */ |
| mrrc p15, 6, r0, r2, c2 |
| lsr r2, r2, #16 |
| and r2, r2, #0xff |
| cmp r2, #0 |
| bne guest_trap @ Guest called HVC |
| |
| host_switch_to_hyp: |
| pop {r0, r1, r2} |
| |
| /* Check for __hyp_get_vectors */ |
| cmp r0, #-1 |
| mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR |
| beq 1f |
| |
| push {lr} |
| mrs lr, SPSR |
| push {lr} |
| |
| mov lr, r0 |
| mov r0, r1 |
| mov r1, r2 |
| mov r2, r3 |
| |
| THUMB( orr lr, #1) |
| blx lr @ Call the HYP function |
| |
| pop {lr} |
| msr SPSR_csxf, lr |
| pop {lr} |
| 1: eret |
| |
| guest_trap: |
| load_vcpu @ Load VCPU pointer to r0 |
| str r1, [vcpu, #VCPU_HSR] |
| |
| @ Check if we need the fault information |
| lsr r1, r1, #HSR_EC_SHIFT |
| cmp r1, #HSR_EC_IABT |
| mrceq p15, 4, r2, c6, c0, 2 @ HIFAR |
| beq 2f |
| cmp r1, #HSR_EC_DABT |
| bne 1f |
| mrc p15, 4, r2, c6, c0, 0 @ HDFAR |
| |
| 2: str r2, [vcpu, #VCPU_HxFAR] |
| |
| /* |
| * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode: |
| * |
| * Abort on the stage 2 translation for a memory access from a |
| * Non-secure PL1 or PL0 mode: |
| * |
| * For any Access flag fault or Translation fault, and also for any |
| * Permission fault on the stage 2 translation of a memory access |
| * made as part of a translation table walk for a stage 1 translation, |
| * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR |
| * is UNKNOWN. |
| */ |
| |
| /* Check for permission fault, and S1PTW */ |
| mrc p15, 4, r1, c5, c2, 0 @ HSR |
| and r0, r1, #HSR_FSC_TYPE |
| cmp r0, #FSC_PERM |
| tsteq r1, #(1 << 7) @ S1PTW |
| mrcne p15, 4, r2, c6, c0, 4 @ HPFAR |
| bne 3f |
| |
| /* Preserve PAR */ |
| mrrc p15, 0, r0, r1, c7 @ PAR |
| push {r0, r1} |
| |
| /* Resolve IPA using the xFAR */ |
| mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR |
| isb |
| mrrc p15, 0, r0, r1, c7 @ PAR |
| tst r0, #1 |
| bne 4f @ Failed translation |
| ubfx r2, r0, #12, #20 |
| lsl r2, r2, #4 |
| orr r2, r2, r1, lsl #24 |
| |
| /* Restore PAR */ |
| pop {r0, r1} |
| mcrr p15, 0, r0, r1, c7 @ PAR |
| |
| 3: load_vcpu @ Load VCPU pointer to r0 |
| str r2, [r0, #VCPU_HPFAR] |
| |
| 1: mov r1, #ARM_EXCEPTION_HVC |
| b __kvm_vcpu_return |
| |
| 4: pop {r0, r1} @ Failed translation, return to guest |
| mcrr p15, 0, r0, r1, c7 @ PAR |
| clrex |
| pop {r0, r1, r2} |
| eret |
| |
| /* |
| * If VFPv3 support is not available, then we will not switch the VFP |
| * registers; however cp10 and cp11 accesses will still trap and fallback |
| * to the regular coprocessor emulation code, which currently will |
| * inject an undefined exception to the guest. |
| */ |
| #ifdef CONFIG_VFPv3 |
| switch_to_guest_vfp: |
| load_vcpu @ Load VCPU pointer to r0 |
| push {r3-r7} |
| |
| @ NEON/VFP used. Turn on VFP access. |
| set_hcptr vmtrap, (HCPTR_TCP(10) | HCPTR_TCP(11)) |
| |
| @ Switch VFP/NEON hardware state to the guest's |
| add r7, r0, #VCPU_VFP_HOST |
| ldr r7, [r7] |
| store_vfp_state r7 |
| add r7, r0, #VCPU_VFP_GUEST |
| restore_vfp_state r7 |
| |
| pop {r3-r7} |
| pop {r0-r2} |
| clrex |
| eret |
| #endif |
| |
| .align |
| hyp_irq: |
| push {r0, r1, r2} |
| mov r1, #ARM_EXCEPTION_IRQ |
| load_vcpu @ Load VCPU pointer to r0 |
| b __kvm_vcpu_return |
| |
| .align |
| hyp_fiq: |
| b hyp_fiq |
| |
| .ltorg |
| |
| __kvm_hyp_code_end: |
| .globl __kvm_hyp_code_end |
| |
| .section ".rodata" |
| |
| und_die_str: |
| .ascii "unexpected undefined exception in Hyp mode at: %#08x\n" |
| pabt_die_str: |
| .ascii "unexpected prefetch abort in Hyp mode at: %#08x\n" |
| dabt_die_str: |
| .ascii "unexpected data abort in Hyp mode at: %#08x\n" |
| svc_die_str: |
| .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n" |