--- upstream/dynamips-0.2.6-RC5/mips64_jit.c 2007/10/06 16:09:07 6 +++ upstream/dynamips-0.2.7-RC1/mips64_jit.c 2007/10/06 16:23:47 7 @@ -1,5 +1,5 @@ /* - * Cisco 7200 (Predator) simulation platform. + * Cisco router simulation platform. * Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr) * * MIPS64 JIT compiler. @@ -16,24 +16,24 @@ #include #include -#include ARCH_INC_FILE - -#include "rbtree.h" -#include "cp0.h" -#include "memory.h" #include "cpu.h" #include "device.h" #include "mips64.h" +#include "mips64_cp0.h" #include "mips64_exec.h" +#include "mips64_jit.h" #include "insn_lookup.h" +#include "memory.h" #include "ptask.h" +#include MIPS64_ARCH_INC_FILE + #if DEBUG_BLOCK_TIMESTAMP static volatile m_uint64_t jit_jiffies = 0; #endif /* MIPS jump instructions for block scan */ -struct insn_jump mips64_insn_jumps[] = { +struct mips64_insn_jump mips64_insn_jumps[] = { { "b" , 0xffff0000, 0x10000000, 16, 1 }, { "bal" , 0xffff0000, 0x04110000, 16, 1 }, { "beq" , 0xfc000000, 0x10000000, 16, 1 }, @@ -64,12 +64,12 @@ return(&mips64_insn_tags[index]); } -static int mips64_jit_chk_lo(struct insn_tag *tag,int value) +static int mips64_jit_chk_lo(struct mips64_insn_tag *tag,int value) { return((value & tag->mask) == (tag->value & 0xFFFF)); } -static int mips64_jit_chk_hi(struct insn_tag *tag,int value) +static int mips64_jit_chk_hi(struct mips64_insn_tag *tag,int value) { return((value & (tag->mask >> 16)) == (tag->value >> 16)); } @@ -82,7 +82,7 @@ for(i=0,count=0;mips64_insn_tags[i].emit;i++) count++; - ilt = ilt_create(count, + ilt = ilt_create(count+1, (ilt_get_insn_cbk_t)mips64_jit_get_insn, (ilt_check_cbk_t)mips64_jit_chk_lo, (ilt_check_cbk_t)mips64_jit_chk_hi); @@ -141,7 +141,8 @@ } printf("CPU%u: carved JIT exec zone of %lu Mb into %lu pages of %u Kb.\n", - cpu->id,(u_long)(cpu->exec_page_area_size / 1048576), + cpu->gen->id, + (u_long)(cpu->exec_page_area_size / 1048576), (u_long)cpu->exec_page_count,MIPS_JIT_BUFSIZE / 1024); return(0); } @@ -149,18 +150,18 @@ /* Flush the JIT */ u_int mips64_jit_flush(cpu_mips_t *cpu,u_int threshold) { - insn_block_t *p,*next; + mips64_jit_tcb_t *p,*next; u_int count = 0; if (!threshold) threshold = (u_int)(-1); /* UINT_MAX not defined everywhere */ - for(p=cpu->insn_block_list;p;p=next) { + for(p=cpu->tcb_list;p;p=next) { next = p->next; if (p->acc_count <= threshold) { cpu->exec_phys_map[p->phys_page] = NULL; - insn_block_free(cpu,p,TRUE); + mips64_jit_tcb_free(cpu,p,TRUE); count++; } } @@ -172,13 +173,13 @@ /* Shutdown the JIT */ void mips64_jit_shutdown(cpu_mips_t *cpu) { - insn_block_t *p,*next; + mips64_jit_tcb_t *p,*next; /* Flush the JIT */ mips64_jit_flush(cpu,0); /* Free the instruction blocks */ - for(p=cpu->insn_block_free_list;p;p=next) { + for(p=cpu->tcb_free_list;p;p=next) { next = p->next; free(p); } @@ -204,12 +205,13 @@ if (unlikely(!cpu->exec_page_free_list)) { if (cpu->jit_flush_method) { - cpu_log(cpu,"JIT","flushing data structures (compiled pages=%u)\n", + cpu_log(cpu->gen, + "JIT","flushing data structures (compiled pages=%u)\n", cpu->compiled_pages); mips64_jit_flush(cpu,0); } else { count = mips64_jit_flush(cpu,100); - cpu_log(cpu,"JIT","partial JIT flush (count=%u)\n",count); + cpu_log(cpu->gen,"JIT","partial JIT flush (count=%u)\n",count); if (!cpu->exec_page_free_list) mips64_jit_flush(cpu,0); @@ -238,9 +240,9 @@ } /* Find the JIT code emitter for the specified MIPS instruction */ -struct insn_tag *insn_tag_find(mips_insn_t ins) +static struct mips64_insn_tag *insn_tag_find(mips_insn_t ins) { - struct insn_tag *tag = NULL; + struct mips64_insn_tag *tag = NULL; int index; index = ilt_lookup(ilt,ins); @@ -249,9 +251,9 @@ } /* Check if the specified MIPS instruction is a jump */ -struct insn_jump *insn_jump_find(mips_insn_t ins) +static struct mips64_insn_jump *insn_jump_find(mips_insn_t ins) { - struct insn_jump *jump = NULL; + struct mips64_insn_jump *jump = NULL; int i; for(i=0;mips64_insn_jumps[i].name;i++) @@ -264,14 +266,14 @@ } /* Fetch a MIPS instruction */ -static forced_inline mips_insn_t insn_fetch(insn_block_t *b) +static forced_inline mips_insn_t insn_fetch(mips64_jit_tcb_t *b) { return(vmtoh32(b->mips_code[b->mips_trans_pos])); } /* Emit a breakpoint if necessary */ #if BREAKPOINT_ENABLE -static void insn_emit_breakpoint(cpu_mips_t *cpu,insn_block_t *b) +static void insn_emit_breakpoint(cpu_mips_t *cpu,mips64_jit_tcb_t *b) { m_uint64_t pc; int i; @@ -286,25 +288,66 @@ } #endif /* BREAKPOINT_ENABLE */ +/* Get host pointer for the physical address */ +static inline void *physmem_get_hptr(vm_instance_t *vm,m_uint64_t paddr, + u_int op_size,u_int op_type, + m_uint64_t *data) +{ + struct vdevice *dev; + m_uint32_t offset; + void *ptr; + int cow; + + if (!(dev = dev_lookup(vm,paddr,FALSE))) + return NULL; + + if (dev->flags & VDEVICE_FLAG_SPARSE) { + ptr = (void *)dev_sparse_get_host_addr(vm,dev,paddr,op_type,&cow); + if (!ptr) return NULL; + + return(ptr + (paddr & VM_PAGE_IMASK)); + } + + if ((dev->host_addr != 0) && !(dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) + return((void *)dev->host_addr + (paddr - dev->phys_addr)); + + if (op_size == 0) + return NULL; + + offset = paddr - dev->phys_addr; + return(dev->handler(vm->boot_cpu,dev,offset,op_size,op_type,data)); +} + /* Fetch a MIPS instruction and emit corresponding translated code */ -struct insn_tag *insn_fetch_and_emit(cpu_mips_t *cpu,insn_block_t *block, - int delay_slot) +struct mips64_insn_tag *mips64_jit_fetch_and_emit(cpu_mips_t *cpu, + mips64_jit_tcb_t *block, + int delay_slot) { - struct insn_tag *tag; + struct mips64_insn_tag *tag; mips_insn_t code; code = insn_fetch(block); tag = insn_tag_find(code); assert(tag); - + + /* Branch-delay slot is in another page: slow exec */ + if ((block->mips_trans_pos == (MIPS_INSN_PER_PAGE-1)) && !tag->delay_slot) { + block->jit_insn_ptr[block->mips_trans_pos] = block->jit_ptr; + + mips64_set_pc(block,block->start_pc + (block->mips_trans_pos << 2)); + mips64_emit_single_step(block,code); + mips64_jit_tcb_push_epilog(block); + block->mips_trans_pos++; + return tag; + } + if (delay_slot && !tag->delay_slot) { mips64_emit_invalid_delay_slot(block); return NULL; } - if (!delay_slot) { + if (!delay_slot) block->jit_insn_ptr[block->mips_trans_pos] = block->jit_ptr; - } if (delay_slot != 2) block->mips_trans_pos++; @@ -331,18 +374,18 @@ } /* Add end of JIT block */ -void insn_block_add_end(insn_block_t *b) +static void mips64_jit_tcb_add_end(mips64_jit_tcb_t *b) { mips64_set_pc(b,b->start_pc+(b->mips_trans_pos<<2)); - insn_block_push_epilog(b); + mips64_jit_tcb_push_epilog(b); } /* Record a patch to apply in a compiled block */ -int insn_block_record_patch(insn_block_t *block,u_char *jit_ptr, - m_uint64_t vaddr) +int mips64_jit_tcb_record_patch(mips64_jit_tcb_t *block,u_char *jit_ptr, + m_uint64_t vaddr) { - struct insn_patch_table *ipt = block->patch_table; - struct insn_patch *patch; + struct mips64_jit_patch_table *ipt = block->patch_table; + struct mips64_insn_patch *patch; /* pc must be 32-bit aligned */ if (vaddr & 0x03) { @@ -352,12 +395,13 @@ return(-1); } - if (!ipt || (ipt->cur_patch >= INSN_PATCH_TABLE_SIZE)) + if (!ipt || (ipt->cur_patch >= MIPS64_INSN_PATCH_TABLE_SIZE)) { /* full table or no table, create a new one */ ipt = malloc(sizeof(*ipt)); if (!ipt) { - fprintf(stderr,"%% Unable to create patch table.\n"); + fprintf(stderr,"Block 0x%8.8llx: unable to create patch table.\n", + block->start_pc); return(-1); } @@ -379,10 +423,11 @@ } /* Apply all patches */ -int insn_block_apply_patches(cpu_mips_t *cpu,insn_block_t *block) +static int mips64_jit_tcb_apply_patches(cpu_mips_t *cpu, + mips64_jit_tcb_t *block) { - struct insn_patch_table *ipt; - struct insn_patch *patch; + struct mips64_jit_patch_table *ipt; + struct mips64_insn_patch *patch; u_char *jit_dst; int i; @@ -390,7 +435,7 @@ for(i=0;icur_patch;i++) { patch = &ipt->patches[i]; - jit_dst = insn_block_get_jit_ptr(block,patch->mips_pc); + jit_dst = mips64_jit_tcb_get_host_ptr(block,patch->mips_pc); if (jit_dst) { #if DEBUG_BLOCK_PATCH @@ -398,7 +443,7 @@ "[JIT:%p->mips:0x%8.8llx=JIT:%p]\n", block->start_pc,patch->jit_insn,patch->mips_pc,jit_dst); #endif - insn_block_set_patch(patch->jit_insn,jit_dst); + mips64_jit_tcb_set_patch(patch->jit_insn,jit_dst); } } @@ -406,9 +451,9 @@ } /* Free the patch table */ -static void insn_block_free_patches(insn_block_t *block) +static void mips64_jit_tcb_free_patches(mips64_jit_tcb_t *block) { - struct insn_patch_table *p,*next; + struct mips64_jit_patch_table *p,*next; for(p=block->patch_table;p;p=next) { next = p->next; @@ -419,7 +464,8 @@ } /* Adjust the JIT buffer if its size is not sufficient */ -int insn_block_adjust_jit_buffer(cpu_mips_t *cpu,insn_block_t *block) +static int mips64_jit_tcb_adjust_buffer(cpu_mips_t *cpu, + mips64_jit_tcb_t *block) { insn_exec_page_t *new_buffer; @@ -430,7 +476,7 @@ printf("Block 0x%llx: adjusting JIT buffer...\n",block->start_pc); #endif - if (block->jit_chunk_pos >= INSN_MAX_CHUNKS) { + if (block->jit_chunk_pos >= MIPS_JIT_MAX_CHUNKS) { fprintf(stderr,"Block 0x%llx: too many JIT chunks.\n",block->start_pc); return(-1); } @@ -443,19 +489,19 @@ block->jit_buffer = new_buffer; /* jump to the new exec page (link) */ - insn_block_set_jump(block->jit_ptr,new_buffer->ptr); + mips64_jit_tcb_set_jump(block->jit_ptr,new_buffer->ptr); block->jit_ptr = new_buffer->ptr; return(0); } /* Allocate an instruction block */ -static inline insn_block_t *insn_block_alloc(cpu_mips_t *cpu) +static inline mips64_jit_tcb_t *mips64_jit_tcb_alloc(cpu_mips_t *cpu) { - insn_block_t *p; + mips64_jit_tcb_t *p; - if (cpu->insn_block_free_list) { - p = cpu->insn_block_free_list; - cpu->insn_block_free_list = p->next; + if (cpu->tcb_free_list) { + p = cpu->tcb_free_list; + cpu->tcb_free_list = p->next; } else { if (!(p = malloc(sizeof(*p)))) return NULL; @@ -466,7 +512,8 @@ } /* Free an instruction block */ -void insn_block_free(cpu_mips_t *cpu,insn_block_t *block,int list_removal) +void mips64_jit_tcb_free(cpu_mips_t *cpu,mips64_jit_tcb_t *block, + int list_removal) { int i; @@ -476,19 +523,19 @@ if (block->next) block->next->prev = block->prev; else - cpu->insn_block_last = block->prev; + cpu->tcb_last = block->prev; if (block->prev) block->prev->next = block->next; else - cpu->insn_block_list = block->next; + cpu->tcb_list = block->next; } /* Free the patch tables */ - insn_block_free_patches(block); + mips64_jit_tcb_free_patches(block); /* Free code pages */ - for(i=0;ijit_chunks[i]); /* Free the current JIT buffer */ @@ -498,17 +545,18 @@ free(block->jit_insn_ptr); /* Make the block return to the free list */ - block->next = cpu->insn_block_free_list; - cpu->insn_block_free_list = block; + block->next = cpu->tcb_free_list; + cpu->tcb_free_list = block; } } /* Create an instruction block */ -static insn_block_t *insn_block_create(cpu_mips_t *cpu,m_uint64_t vaddr) +static mips64_jit_tcb_t *mips64_jit_tcb_create(cpu_mips_t *cpu, + m_uint64_t vaddr) { - insn_block_t *block = NULL; + mips64_jit_tcb_t *block = NULL; - if (!(block = insn_block_alloc(cpu))) + if (!(block = mips64_jit_tcb_alloc(cpu))) goto err_block_alloc; block->start_pc = vaddr; @@ -533,7 +581,7 @@ err_lookup: err_jit_alloc: - insn_block_free(cpu,block,FALSE); + mips64_jit_tcb_free(cpu,block,FALSE); err_block_alloc: fprintf(stderr,"%% Unable to create instruction block for vaddr=0x%llx\n", vaddr); @@ -541,16 +589,17 @@ } /* Compile a MIPS instruction page */ -static inline insn_block_t *insn_page_compile(cpu_mips_t *cpu,m_uint64_t vaddr) +static inline +mips64_jit_tcb_t *mips64_jit_tcb_compile(cpu_mips_t *cpu,m_uint64_t vaddr) { + mips64_jit_tcb_t *block; + struct mips64_insn_tag *tag; m_uint64_t page_addr; - insn_block_t *block; - struct insn_tag *tag; size_t len; page_addr = vaddr & ~(m_uint64_t)MIPS_MIN_PAGE_IMASK; - if (unlikely(!(block = insn_block_create(cpu,page_addr)))) { + if (unlikely(!(block = mips64_jit_tcb_create(cpu,page_addr)))) { fprintf(stderr,"insn_page_compile: unable to create JIT block.\n"); return NULL; } @@ -566,9 +615,9 @@ /* Emit native code for each instruction */ block->mips_trans_pos = 0; - while(block->mips_trans_pos < (MIPS_MIN_PAGE_SIZE/sizeof(mips_insn_t))) + while(block->mips_trans_pos < MIPS_INSN_PER_PAGE) { - if (unlikely(!(tag = insn_fetch_and_emit(cpu,block,0)))) { + if (unlikely(!(tag = mips64_jit_fetch_and_emit(cpu,block,0)))) { fprintf(stderr,"insn_page_compile: unable to fetch instruction.\n"); goto error; } @@ -578,34 +627,35 @@ block->start_pc,tag->mask,tag->value); #endif - insn_block_adjust_jit_buffer(cpu,block); + mips64_jit_tcb_adjust_buffer(cpu,block); } - insn_block_add_end(block); - insn_block_apply_patches(cpu,block); - insn_block_free_patches(block); + mips64_jit_tcb_add_end(block); + mips64_jit_tcb_apply_patches(cpu,block); + mips64_jit_tcb_free_patches(block); /* Add the block to the linked list */ - block->next = cpu->insn_block_list; + block->next = cpu->tcb_list; block->prev = NULL; - if (cpu->insn_block_list) - cpu->insn_block_list->prev = block; + if (cpu->tcb_list) + cpu->tcb_list->prev = block; else - cpu->insn_block_last = block; + cpu->tcb_last = block; - cpu->insn_block_list = block; + cpu->tcb_list = block; cpu->compiled_pages++; return block; error: - insn_block_free(cpu,block,FALSE); + mips64_jit_tcb_free(cpu,block,FALSE); return NULL; } /* Run a compiled MIPS instruction block */ -static forced_inline void insn_block_run(cpu_mips_t *cpu,insn_block_t *block) +static forced_inline +void mips64_jit_tcb_run(cpu_mips_t *cpu,mips64_jit_tcb_t *block) { #if DEBUG_SYM_TREE struct symbol *sym = NULL; @@ -613,10 +663,10 @@ #endif if (unlikely(cpu->pc & 0x03)) { - fprintf(stderr,"insn_block_run: Invalid PC 0x%llx.\n",cpu->pc); - mips64_dump_regs(cpu); - tlb_dump(cpu); - cpu_stop(cpu); + fprintf(stderr,"mips64_jit_tcb_run: Invalid PC 0x%llx.\n",cpu->pc); + mips64_dump_regs(cpu->gen); + mips64_tlb_dump(cpu->gen); + cpu_stop(cpu->gen); return; } @@ -624,7 +674,7 @@ if (cpu->sym_trace && cpu->sym_tree) { if ((sym = mips64_sym_lookup(cpu,cpu->pc)) != NULL) { - cpu_log(cpu,"insn_block_run(start)", + cpu_log(cpu,"mips64_jit_tcb_run(start)", "%s (PC=0x%llx) RA = 0x%llx\na0=0x%llx, " "a1=0x%llx, a2=0x%llx, a3=0x%llx\n", sym->name, cpu->pc, cpu->gpr[MIPS_GPR_RA], @@ -636,24 +686,24 @@ #endif /* Execute JIT compiled code */ - insn_block_exec_jit_code(cpu,block); + mips64_jit_tcb_exec(cpu,block); #if DEBUG_SYM_TREE if (mark) { - cpu_log(cpu,"insn_block_run(end)","%s, v0 = 0x%llx\n", + cpu_log(cpu,"mips64_jit_tcb_run(end)","%s, v0 = 0x%llx\n", sym->name,cpu->gpr[MIPS_GPR_V0]); } #endif } /* Check if the specified address belongs to the specified block */ -int insn_block_local_addr(insn_block_t *block,m_uint64_t vaddr, - u_char **jit_addr) +int mips64_jit_tcb_local_addr(mips64_jit_tcb_t *block,m_uint64_t vaddr, + u_char **jit_addr) { if ((vaddr >= block->start_pc) && ((vaddr - block->start_pc) < MIPS_MIN_PAGE_SIZE)) { - *jit_addr = insn_block_get_jit_ptr(block,vaddr); + *jit_addr = mips64_jit_tcb_get_host_ptr(block,vaddr); return(1); } @@ -661,7 +711,8 @@ } /* Check if PC register matches the compiled block virtual address */ -static forced_inline int insn_block_match(cpu_mips_t *cpu,insn_block_t *block) +static forced_inline +int mips64_jit_tcb_match(cpu_mips_t *cpu,mips64_jit_tcb_t *block) { m_uint64_t vpage; @@ -669,11 +720,12 @@ return(block->start_pc == vpage); } -/* Execute a compiled MIPS code */ -void *insn_block_execute(cpu_mips_t *cpu) -{ +/* Execute compiled MIPS code */ +void *mips64_jit_run_cpu(cpu_gen_t *gen) +{ + cpu_mips_t *cpu = CPU_MIPS64(gen); pthread_t timer_irq_thread; - insn_block_t *block; + mips64_jit_tcb_t *block; m_uint32_t phys_page; int timer_irq_check = 0; @@ -682,18 +734,18 @@ { fprintf(stderr, "VM '%s': unable to create Timer IRQ thread for CPU%u.\n", - cpu->vm->name,cpu->id); - cpu_stop(cpu); + cpu->vm->name,gen->id); + cpu_stop(cpu->gen); return NULL; } - cpu->cpu_thread_running = TRUE; + gen->cpu_thread_running = TRUE; start_cpu: - cpu->idle_count = 0; + gen->idle_count = 0; for(;;) { - if (unlikely(cpu->state != MIPS_CPU_RUNNING)) + if (unlikely(gen->state != CPU_STATE_RUNNING)) break; #if DEBUG_BLOCK_PERF_CNT @@ -701,9 +753,9 @@ #endif /* Handle virtual idle loop */ if (unlikely(cpu->pc == cpu->idle_pc)) { - if (++cpu->idle_count == cpu->idle_max) { - mips64_idle_loop(cpu); - cpu->idle_count = 0; + if (++gen->idle_count == gen->idle_max) { + cpu_idle_loop(gen); + gen->idle_count = 0; } } @@ -721,27 +773,27 @@ /* Get the physical page address corresponding to PC register */ if (unlikely(cpu->translate(cpu,cpu->pc,&phys_page))) { fprintf(stderr,"VM '%s': no physical page for CPU%u PC=0x%llx\n", - cpu->vm->name,cpu->id,cpu->pc); - cpu_stop(cpu); + cpu->vm->name,gen->id,cpu->pc); + cpu_stop(gen); break; } block = cpu->exec_phys_map[phys_page]; /* No block found, compile the page */ - if (unlikely(!block) || unlikely(!insn_block_match(cpu,block))) + if (unlikely(!block) || unlikely(!mips64_jit_tcb_match(cpu,block))) { if (block != NULL) { - insn_block_free(cpu,block,TRUE); + mips64_jit_tcb_free(cpu,block,TRUE); cpu->exec_phys_map[phys_page] = NULL; } - block = insn_page_compile(cpu,cpu->pc); + block = mips64_jit_tcb_compile(cpu,cpu->pc); if (unlikely(!block)) { fprintf(stderr, "VM '%s': unable to compile block for CPU%u PC=0x%llx\n", - cpu->vm->name,cpu->id,cpu->pc); - cpu_stop(cpu); + cpu->vm->name,gen->id,cpu->pc); + cpu_stop(gen); break; } @@ -753,25 +805,25 @@ block->tm_last_use = jit_jiffies++; #endif block->acc_count++; - insn_block_run(cpu,block); + mips64_jit_tcb_run(cpu,block); } if (!cpu->pc) { - cpu_stop(cpu); - cpu_log(cpu,"JIT","PC=0, halting CPU.\n"); + cpu_stop(gen); + cpu_log(gen,"JIT","PC=0, halting CPU.\n"); } /* Check regularly if the CPU has been restarted */ - while(cpu->cpu_thread_running) { - cpu->seq_state++; + while(gen->cpu_thread_running) { + gen->seq_state++; - switch(cpu->state) { - case MIPS_CPU_RUNNING: - cpu->state = MIPS_CPU_RUNNING; + switch(gen->state) { + case CPU_STATE_RUNNING: + gen->state = CPU_STATE_RUNNING; goto start_cpu; - case MIPS_CPU_HALTED: - cpu->cpu_thread_running = FALSE; + case CPU_STATE_HALTED: + gen->cpu_thread_running = FALSE; pthread_join(timer_irq_thread,NULL); break; }