RE: [PATCH v1 23/36] target/riscv: Add hypvervisor trap support

[Jiangyifei]

Hi Alistair, 

When I boot a riscv OS (the host) on the QEMU emulator and then boot a
guest riscv OS in the host. The guest riscv OS fails to boot. 

The riscv OS is a linux kernel in the riscv_kvm_v10 branch, the qemu emulator
is the riscv-hyp-ext-v0.5.next branch.

The QEMU command line:
    ./qemu-system-riscv64 -M virt -m 4096M -cpu rv64,x-h=true -nographic \
        -smp 8 \
        -name guest=riscv-test,debug-threads=on \
        -kernel ./fw_jump.elf \
        -device loader,file=./Image,addr=0x80200000 \
        -drive file=./hyp.img,format=raw,id=hd0 \
        -device virtio-blk-device,drive=hd0 \
        -append "root=/dev/vda rw console=ttyS0 earlycon=sbi"

The kvmtool command line:
    ./lkvm-static run -m 2048 -c2 --console serial -p "console=ttyS0 \
        earlycon=uart8250,mmio,0x3f8" -k ./Image -d guest.img

The error message is:
    [   46.136740] kvm [128]: VCPU exit error -14
    [   46.137967] kvm [128]: SEPC=0x3febbf234e SSTATUS=0x4020 HSTATUS=0x380
    [   46.140075] kvm [128]: SCAUSE=0x8 STVAL=0x0
    KVM_RUN failed: Bad address

According to the error message printed by kvm, the exception of Environment
call from VU-mode should to be delivered to VS-mode, instead of delivered
to kvm in HS-mode.

The problem is that the FORCE_HS_EXCEP flag is set when there is a pending
irq that should trap to V==0, but this bit is not cleared in  
riscv_cpu_do_interrupt
when trapping to M-mode. So clearing FORCE_HS_EXCEP flag when trap to
M-mode. The patch is as follows:

Signed-off-by: Jiang Yifei <address@hidden>
---
target/riscv/cpu_helper.c | 1 +
1 file changed, 1 insertion(+)

diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c
index 7166e61..cb6a1d6 100644
--- a/target/riscv/cpu_helper.c
+++ b/target/riscv/cpu_helper.c
@@ -954,6 +954,7 @@ void riscv_cpu_do_interrupt(CPUState *cs)

            /* Trapping to M mode, virt is disabled */
            riscv_cpu_set_virt_enabled(env, 0);
+            riscv_cpu_set_force_hs_excep(env, 0);
        }

        s = *env->mstatus;
--

Regards,
Yifei

> -----Original Message-----
> From: Qemu-riscv [mailto:qemu-riscv-
> bounces+jiangyifei=address@hidden] On Behalf Of Alistair Francis
> Sent: Tuesday, December 10, 2019 2:12 AM
> To: address@hidden; address@hidden
> Cc: address@hidden; address@hidden; address@hidden
> Subject: [PATCH v1 23/36] target/riscv: Add hypvervisor trap support
> 
> Signed-off-by: Alistair Francis <address@hidden>
> Reviewed-by: Palmer Dabbelt <address@hidden>
> ---
>  target/riscv/cpu_helper.c | 65 +++++++++++++++++++++++++++++++++---
> ---
>  1 file changed, 55 insertions(+), 10 deletions(-)
> 
> diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index
> 1b747abf93..2c6d2bc3a3 100644
> --- a/target/riscv/cpu_helper.c
> +++ b/target/riscv/cpu_helper.c
> @@ -641,6 +641,7 @@ void riscv_cpu_do_interrupt(CPUState *cs)
> 
>      RISCVCPU *cpu = RISCV_CPU(cs);
>      CPURISCVState *env = &cpu->env;
> +    target_ulong s;
> 
>      /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
>       * so we mask off the MSB and separate into trap type and cause.
> @@ -650,13 +651,6 @@ void riscv_cpu_do_interrupt(CPUState *cs)
>      target_ulong deleg = async ? env->mideleg : env->medeleg;
>      target_ulong tval = 0;
> 
> -    static const int ecall_cause_map[] = {
> -        [PRV_U] = RISCV_EXCP_U_ECALL,
> -        [PRV_S] = RISCV_EXCP_S_ECALL,
> -        [PRV_H] = RISCV_EXCP_VS_ECALL,
> -        [PRV_M] = RISCV_EXCP_M_ECALL
> -    };
> -
>      if (!async) {
>          /* set tval to badaddr for traps with address information */
>          switch (cause) {
> @@ -680,7 +674,16 @@ void riscv_cpu_do_interrupt(CPUState *cs)
>          /* ecall is dispatched as one cause so translate based on mode */
>          if (cause == RISCV_EXCP_U_ECALL) {
>              assert(env->priv <= 3);
> -            cause = ecall_cause_map[env->priv];
> +
> +            if (env->priv == PRV_M) {
> +                cause = RISCV_EXCP_M_ECALL;
> +            } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
> +                cause = RISCV_EXCP_VS_ECALL;
> +            } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
> +                cause = RISCV_EXCP_S_ECALL;
> +            } else if (env->priv == PRV_U) {
> +                cause = RISCV_EXCP_U_ECALL;
> +            }
>          }
>      }
> 
> @@ -690,7 +693,36 @@ void riscv_cpu_do_interrupt(CPUState *cs)
>      if (env->priv <= PRV_S &&
>              cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
>          /* handle the trap in S-mode */
> -        target_ulong s = *env->mstatus;
> +        if (riscv_has_ext(env, RVH)) {
> +            target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
> +
> +            if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
> +                !riscv_cpu_force_hs_excep_enabled(env)) {
> +                /* Trap to VS mode */
> +            } else if (riscv_cpu_virt_enabled(env)) {
> +                /* Trap into HS mode, from virt */
> +                riscv_cpu_swap_hypervisor_regs(env);
> +                env->hstatus = set_field(env->hstatus, HSTATUS_SP2V,
> +                                         get_field(env->hstatus, 
> HSTATUS_SPV));
> +                env->hstatus = set_field(env->hstatus, HSTATUS_SP2P,
> +                                         get_field(*env->mstatus, 
> SSTATUS_SPP));
> +                env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
> +                                         riscv_cpu_virt_enabled(env));
> +
> +                riscv_cpu_set_virt_enabled(env, 0);
> +                riscv_cpu_set_force_hs_excep(env, 0);
> +            } else {
> +                /* Trap into HS mode */
> +                env->hstatus = set_field(env->hstatus, HSTATUS_SP2V,
> +                                         get_field(env->hstatus, 
> HSTATUS_SPV));
> +                env->hstatus = set_field(env->hstatus, HSTATUS_SP2P,
> +                                         get_field(*env->mstatus, 
> SSTATUS_SPP));
> +                env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
> +                                         riscv_cpu_virt_enabled(env));
> +            }
> +        }
> +
> +        s = *env->mstatus;
>          s = set_field(s, MSTATUS_SPIE, env->priv_ver >=
> PRIV_VERSION_1_10_0 ?
>              get_field(s, MSTATUS_SIE) : get_field(s, MSTATUS_UIE << 
> env->priv));
>          s = set_field(s, MSTATUS_SPP, env->priv); @@ -704,7 +736,20 @@ void
> riscv_cpu_do_interrupt(CPUState *cs)
>          riscv_cpu_set_mode(env, PRV_S);
>      } else {
>          /* handle the trap in M-mode */
> -        target_ulong s = *env->mstatus;
> +        if (riscv_has_ext(env, RVH)) {
> +            if (riscv_cpu_virt_enabled(env)) {
> +                riscv_cpu_swap_hypervisor_regs(env);
> +            }
> +            *env->mstatus = set_field(*env->mstatus, MSTATUS_MPV,
> +                                      riscv_cpu_virt_enabled(env));
> +            *env->mstatus = set_field(*env->mstatus, MSTATUS_MTL,
> +
> + riscv_cpu_force_hs_excep_enabled(env));
> +
> +            /* Trapping to M mode, virt is disabled */
> +            riscv_cpu_set_virt_enabled(env, 0);
> +        }
> +
> +        s = *env->mstatus;
>          s = set_field(s, MSTATUS_MPIE, env->priv_ver >=
> PRIV_VERSION_1_10_0 ?
>              get_field(s, MSTATUS_MIE) : get_field(s, MSTATUS_UIE << env-
> >priv));
>          s = set_field(s, MSTATUS_MPP, env->priv);
> --
> 2.24.0
>