dynamips/trunk/ppc32_jit.c

/*
 * Cisco router simulation platform.
 * Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
 *
 * PPC32 JIT compiler.
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <signal.h>
#include <fcntl.h>
#include <assert.h>

#include "cpu.h"
#include "device.h"
#include "ppc32.h"
#include "ppc32_exec.h"
#include "ppc32_jit.h"
#include "insn_lookup.h"
#include "memory.h"
#include "ptask.h"

#include PPC32_ARCH_INC_FILE

/* Instruction Lookup Table */
static insn_lookup_t *ilt = NULL;

static void *ppc32_jit_get_insn(int index)
{
   return(&ppc32_insn_tags[index]);
}

static int ppc32_jit_chk_lo(struct ppc32_insn_tag *tag,int value)
{
   return((value & tag->mask) == (tag->value & 0xFFFF));
}

static int ppc32_jit_chk_hi(struct ppc32_insn_tag *tag,int value)
{
   return((value & (tag->mask >> 16)) == (tag->value >> 16));
}

/* Initialize instruction lookup table */
void ppc32_jit_create_ilt(void)
{
   int i,count;

   for(i=0,count=0;ppc32_insn_tags[i].emit;i++)
      count++;

   ilt = ilt_create(count+1,
                    (ilt_get_insn_cbk_t)ppc32_jit_get_insn,
                    (ilt_check_cbk_t)ppc32_jit_chk_lo,
                    (ilt_check_cbk_t)ppc32_jit_chk_hi);
}

/* Initialize the JIT structure */
int ppc32_jit_init(cpu_ppc_t *cpu)
{
   insn_exec_page_t *cp;
   u_char *cp_addr;
   u_int area_size;
   size_t len;
   int i;

   /* Physical mapping for executable pages */
   len = 1048576 * sizeof(void *);
   cpu->exec_phys_map = m_memalign(4096,len);
   memset(cpu->exec_phys_map,0,len);

   /* Get area size */
   if (!(area_size = cpu->vm->exec_area_size))
      area_size = PPC_EXEC_AREA_SIZE;

   /* Create executable page area */
   cpu->exec_page_area_size = area_size * 1048576;
   cpu->exec_page_area = mmap(NULL,cpu->exec_page_area_size,
                              PROT_EXEC|PROT_READ|PROT_WRITE,
                              MAP_SHARED|MAP_ANONYMOUS,-1,(off_t)0);

   if (!cpu->exec_page_area) {
      fprintf(stderr,
              "ppc32_jit_init: unable to create exec area (size %lu)\n",
              (u_long)cpu->exec_page_area_size);
      return(-1);
   }

   /* Carve the executable page area */
   cpu->exec_page_count = cpu->exec_page_area_size / PPC_JIT_BUFSIZE;

   cpu->exec_page_array = calloc(cpu->exec_page_count,
                                 sizeof(insn_exec_page_t));
   
   if (!cpu->exec_page_array) {
      fprintf(stderr,"ppc32_jit_init: unable to create exec page array\n");
      return(-1);
   }

   for(i=0,cp_addr=cpu->exec_page_area;i<cpu->exec_page_count;i++) {
      cp = &cpu->exec_page_array[i];

      cp->ptr = cp_addr;
      cp_addr += PPC_JIT_BUFSIZE;

      cp->next = cpu->exec_page_free_list;
      cpu->exec_page_free_list = cp;
   }

   printf("CPU%u: carved JIT exec zone of %lu Mb into %lu pages of %u Kb.\n",
          cpu->gen->id,
          (u_long)(cpu->exec_page_area_size / 1048576),
          (u_long)cpu->exec_page_count,PPC_JIT_BUFSIZE / 1024);
   return(0);
}

/* Flush the JIT */
u_int ppc32_jit_flush(cpu_ppc_t *cpu,u_int threshold)
{
   ppc32_jit_tcb_t *p,*next;
   u_int count = 0;

   if (!threshold)
      threshold = (u_int)(-1);  /* UINT_MAX not defined everywhere */

   for(p=cpu->tcb_list;p;p=next) {
      next = p->next;

      if (p->acc_count <= threshold) {
         cpu->exec_phys_map[p->phys_page] = NULL;
         ppc32_jit_tcb_free(cpu,p,TRUE);
         count++;
      }
   }

   cpu->compiled_pages -= count;
   return(count);
}

/* Shutdown the JIT */
void ppc32_jit_shutdown(cpu_ppc_t *cpu)
{   
   ppc32_jit_tcb_t *p,*next;

   /* Flush the JIT */
   ppc32_jit_flush(cpu,0);

   /* Free the instruction blocks */
   for(p=cpu->tcb_free_list;p;p=next) {
      next = p->next;
      free(p);
   }

   /* Unmap the executable page area */
   if (cpu->exec_page_area)
      munmap(cpu->exec_page_area,cpu->exec_page_area_size);

   /* Free the exec page array */
   free(cpu->exec_page_array);

   /* Free physical mapping for executable pages */
   free(cpu->exec_phys_map);   
}

/* Allocate an exec page */
static inline insn_exec_page_t *exec_page_alloc(cpu_ppc_t *cpu)
{
   insn_exec_page_t *p;
   u_int count;

   /* If the free list is empty, flush JIT */
   if (unlikely(!cpu->exec_page_free_list)) 
   {
      if (cpu->jit_flush_method) {
         cpu_log(cpu->gen,
                 "JIT","flushing data structures (compiled pages=%u)\n",
                 cpu->compiled_pages);
         ppc32_jit_flush(cpu,0);
      } else {
         count = ppc32_jit_flush(cpu,100);
         cpu_log(cpu->gen,"JIT","partial JIT flush (count=%u)\n",count);

         if (!cpu->exec_page_free_list)
            ppc32_jit_flush(cpu,0);
      }
      
      /* Use both methods alternatively */
      cpu->jit_flush_method = 1 - cpu->jit_flush_method;
   }

   if (unlikely(!(p = cpu->exec_page_free_list)))
      return NULL;
   
   cpu->exec_page_free_list = p->next;
   cpu->exec_page_alloc++;
   return p;
}

/* Free an exec page and returns it to the pool */
static inline void exec_page_free(cpu_ppc_t *cpu,insn_exec_page_t *p)
{
   if (p) {
      p->next = cpu->exec_page_free_list;
      cpu->exec_page_free_list = p;
      cpu->exec_page_alloc--;
   }
}

/* Find the JIT code emitter for the specified PowerPC instruction */
static struct ppc32_insn_tag *insn_tag_find(ppc_insn_t ins)
{
   struct ppc32_insn_tag *tag = NULL;
   int index;

   index = ilt_lookup(ilt,ins);
   tag = ppc32_jit_get_insn(index);
   return tag;
}

/* Fetch a PowerPC instruction */
static forced_inline ppc_insn_t insn_fetch(ppc32_jit_tcb_t *b)
{
   return(vmtoh32(b->ppc_code[b->ppc_trans_pos]));
}

/* Emit a breakpoint if necessary */
#if BREAKPOINT_ENABLE
static void insn_emit_breakpoint(cpu_ppc_t *cpu,ppc32_jit_tcb_t *b)
{
   m_uint32_t ia;
   int i;

   ia = b->start_ia+((b->ppc_trans_pos-1)<<2);

   for(i=0;i<PPC32_MAX_BREAKPOINTS;i++)
      if (ia == cpu->breakpoints[i]) {
         ppc32_emit_breakpoint(b);
         break;
      }
}
#endif /* BREAKPOINT_ENABLE */

/* Fetch a PowerPC instruction and emit corresponding translated code */
struct ppc32_insn_tag *ppc32_jit_fetch_and_emit(cpu_ppc_t *cpu,
                                                ppc32_jit_tcb_t *block)
{
   struct ppc32_insn_tag *tag;
   ppc_insn_t code;

   code = insn_fetch(block);
   tag = insn_tag_find(code);
   assert(tag);
   
   block->jit_insn_ptr[block->ppc_trans_pos] = block->jit_ptr;
   block->ppc_trans_pos++;

#if DEBUG_INSN_PERF_CNT
   ppc32_inc_perf_counter(block);
#endif

#if BREAKPOINT_ENABLE
   if (cpu->breakpoints_enabled)
      insn_emit_breakpoint(cpu,block);
#endif

   tag->emit(cpu,block,code);
   return tag;
}

/* Add end of JIT block */
static void ppc32_jit_tcb_add_end(ppc32_jit_tcb_t *b)
{
   ppc32_set_ia(b,b->start_ia+(b->ppc_trans_pos<<2));
   ppc32_jit_tcb_push_epilog(b);
}

/* Record a patch to apply in a compiled block */
int ppc32_jit_tcb_record_patch(ppc32_jit_tcb_t *block,u_char *jit_ptr,
                               m_uint32_t vaddr)
{
   struct ppc32_jit_patch_table *ipt = block->patch_table;
   struct ppc32_insn_patch *patch;

   /* pc must be 32-bit aligned */
   if (vaddr & 0x03) {
      fprintf(stderr,"Block 0x%8.8x: trying to record an invalid IA "
              "(0x%8.8x) - ppc_trans_pos=%d.\n",
              block->start_ia,vaddr,block->ppc_trans_pos);
      return(-1);
   }

   if (!ipt || (ipt->cur_patch >= PPC32_INSN_PATCH_TABLE_SIZE))
   {
      /* full table or no table, create a new one */
      ipt = malloc(sizeof(*ipt));
      if (!ipt) {
         fprintf(stderr,"Block 0x%8.8x: unable to create patch table.\n",
                 block->start_ia);
         return(-1);
      }

      memset(ipt,0,sizeof(*ipt));
      ipt->next = block->patch_table;
      block->patch_table = ipt;
   }

#if DEBUG_BLOCK_PATCH
   printf("Block 0x%8.8x: recording patch [JIT:%p->ppc:0x%8.8x], "
          "MTP=%d\n",block->start_ia,jit_ptr,vaddr,block->ppc_trans_pos);
#endif

   patch = &ipt->patches[ipt->cur_patch];
   patch->jit_insn = jit_ptr;
   patch->ppc_ia = vaddr;
   ipt->cur_patch++;   
   return(0);
}

/* Apply all patches */
static int ppc32_jit_tcb_apply_patches(cpu_ppc_t *cpu,
                                        ppc32_jit_tcb_t *block)
{
   struct ppc32_jit_patch_table *ipt;
   struct ppc32_insn_patch *patch;
   u_char *jit_dst;
   int i;

   for(ipt=block->patch_table;ipt;ipt=ipt->next)
      for(i=0;i<ipt->cur_patch;i++) 
      {
         patch = &ipt->patches[i];
         jit_dst = ppc32_jit_tcb_get_host_ptr(block,patch->ppc_ia);

         if (jit_dst) {
#if DEBUG_BLOCK_PATCH
            printf("Block 0x%8.8x: applying patch "
                   "[JIT:%p->ppc:0x%8.8x=JIT:%p]\n",
                   block->start_ia,patch->jit_insn,patch->ppc_ia,jit_dst);
#endif
            ppc32_jit_tcb_set_patch(patch->jit_insn,jit_dst);
         }
      }

   return(0);
}

/* Free the patch table */
static void ppc32_jit_tcb_free_patches(ppc32_jit_tcb_t *block)
{
   struct ppc32_jit_patch_table *p,*next;

   for(p=block->patch_table;p;p=next) {
      next = p->next;
      free(p);
   }

   block->patch_table = NULL;
}

/* Adjust the JIT buffer if its size is not sufficient */
static int ppc32_jit_tcb_adjust_buffer(cpu_ppc_t *cpu,
                                       ppc32_jit_tcb_t *block)
{
   insn_exec_page_t *new_buffer;

   if ((block->jit_ptr - block->jit_buffer->ptr) <= (PPC_JIT_BUFSIZE - 512))
      return(0);

#if DEBUG_BLOCK_CHUNK  
   printf("Block 0x%8.8x: adjusting JIT buffer...\n",block->start_ia);
#endif

   if (block->jit_chunk_pos >= PPC_JIT_MAX_CHUNKS) {
      fprintf(stderr,"Block 0x%8.8x: too many JIT chunks.\n",block->start_ia);
      return(-1);
   }

   if (!(new_buffer = exec_page_alloc(cpu)))
      return(-1);

   /* record the new exec page */
   block->jit_chunks[block->jit_chunk_pos++] = block->jit_buffer;
   block->jit_buffer = new_buffer;

   /* jump to the new exec page (link) */
   ppc32_jit_tcb_set_jump(block->jit_ptr,new_buffer->ptr);
   block->jit_ptr = new_buffer->ptr;
   return(0);
}

/* Allocate an instruction block */
static inline ppc32_jit_tcb_t *ppc32_jit_tcb_alloc(cpu_ppc_t *cpu)
{
   ppc32_jit_tcb_t *p;

   if (cpu->tcb_free_list) {
      p = cpu->tcb_free_list;
      cpu->tcb_free_list = p->next;
   } else {
      if (!(p = malloc(sizeof(*p))))
         return NULL;
   }

   memset(p,0,sizeof(*p));
   return p;
}

/* Free an instruction block */
void ppc32_jit_tcb_free(cpu_ppc_t *cpu,ppc32_jit_tcb_t *block,
                         int list_removal)
{
   int i;

   if (block) {
      if (list_removal) {
         /* Remove the block from the linked list */
         if (block->next)
            block->next->prev = block->prev;
         else
            cpu->tcb_last = block->prev;

         if (block->prev)
            block->prev->next = block->next;
         else
            cpu->tcb_list = block->next;
      }

      /* Free the patch tables */
      ppc32_jit_tcb_free_patches(block);

      /* Free code pages */
      for(i=0;i<PPC_JIT_MAX_CHUNKS;i++)
         exec_page_free(cpu,block->jit_chunks[i]);

      /* Free the current JIT buffer */
      exec_page_free(cpu,block->jit_buffer);

      /* Free the PowerPC-to-native code mapping */
      free(block->jit_insn_ptr);

      /* Make the block return to the free list */
      block->next = cpu->tcb_free_list;
      cpu->tcb_free_list = block;
   }
}

/* Create an instruction block */
static ppc32_jit_tcb_t *ppc32_jit_tcb_create(cpu_ppc_t *cpu,m_uint32_t vaddr)
{
   ppc32_jit_tcb_t *block = NULL;

   if (!(block = ppc32_jit_tcb_alloc(cpu)))
      goto err_block_alloc;

   block->start_ia = vaddr;

   /* Allocate the first JIT buffer */
   if (!(block->jit_buffer = exec_page_alloc(cpu)))
      goto err_jit_alloc;

   block->jit_ptr = block->jit_buffer->ptr;
   block->ppc_code = cpu->mem_op_lookup(cpu,block->start_ia,PPC32_MTS_ICACHE);

   if (!block->ppc_code) {
      fprintf(stderr,"%% No memory map for code execution at 0x%8.8x\n",
              block->start_ia);
      goto err_lookup;
   }

#if DEBUG_BLOCK_TIMESTAMP
   block->tm_first_use = block->tm_last_use = jit_jiffies;
#endif
   return block;

 err_lookup:
 err_jit_alloc:
   ppc32_jit_tcb_free(cpu,block,FALSE);
 err_block_alloc:
   fprintf(stderr,"%% Unable to create instruction block for vaddr=0x%8.8x\n", 
           vaddr);
   return NULL;
}

/* Compile a PowerPC instruction page */
static inline 
ppc32_jit_tcb_t *ppc32_jit_tcb_compile(cpu_ppc_t *cpu,m_uint32_t vaddr)
{  
   ppc32_jit_tcb_t *block;
   struct ppc32_insn_tag *tag;
   m_uint32_t page_addr;
   size_t len;

   page_addr = vaddr & ~PPC32_MIN_PAGE_IMASK;

   if (unlikely(!(block = ppc32_jit_tcb_create(cpu,page_addr)))) {
      fprintf(stderr,"insn_page_compile: unable to create JIT block.\n");
      return NULL;
   }

   /* Allocate the array used to convert PPC code ptr to native code ptr */
   len = PPC32_MIN_PAGE_SIZE / sizeof(ppc_insn_t);

   if (!(block->jit_insn_ptr = calloc(len,sizeof(u_char *)))) {
      fprintf(stderr,"insn_page_compile: unable to create JIT mappings.\n");
      goto error;
   }

   /* Emit native code for each instruction */
   block->ppc_trans_pos = 0;

   while(block->ppc_trans_pos < (PPC32_MIN_PAGE_SIZE/sizeof(ppc_insn_t)))
   {
      if (unlikely(!(tag = ppc32_jit_fetch_and_emit(cpu,block)))) {
         fprintf(stderr,"insn_page_compile: unable to fetch instruction.\n");
         goto error;
      }

#if DEBUG_BLOCK_COMPILE
      printf("Page 0x%8.8x: emitted tag 0x%8.8x/0x%8.8x\n",
             block->start_ia,tag->mask,tag->value);
#endif

      ppc32_jit_tcb_adjust_buffer(cpu,block);
   }

   ppc32_jit_tcb_add_end(block);
   ppc32_jit_tcb_apply_patches(cpu,block);
   ppc32_jit_tcb_free_patches(block);

   /* Add the block to the linked list */
   block->next = cpu->tcb_list;
   block->prev = NULL;

   if (cpu->tcb_list)
      cpu->tcb_list->prev = block;
   else
      cpu->tcb_last = block;

   cpu->tcb_list = block;
   
   cpu->compiled_pages++;
   return block;

 error:
   ppc32_jit_tcb_free(cpu,block,FALSE);
   return NULL;
}

/* Run a compiled PowerPC instruction block */
static forced_inline 
void ppc32_jit_tcb_run(cpu_ppc_t *cpu,ppc32_jit_tcb_t *block)
{
   if (unlikely(cpu->ia & 0x03)) {
      fprintf(stderr,"ppc32_jit_tcb_run: Invalid IA 0x%8.8x.\n",cpu->ia);
      ppc32_dump_regs(cpu->gen);
      ppc32_dump_mmu(cpu->gen);
      cpu_stop(cpu->gen);
      return;
   }

   /* Execute JIT compiled code */
   ppc32_jit_tcb_exec(cpu,block);
}

/* Check if the specified address belongs to the specified block */
int ppc32_jit_tcb_local_addr(ppc32_jit_tcb_t *block,m_uint32_t vaddr,
                             u_char **jit_addr)
{
   if ((vaddr >= block->start_ia) && 
       ((vaddr - block->start_ia) < PPC32_MIN_PAGE_SIZE))
   {
      *jit_addr = ppc32_jit_tcb_get_host_ptr(block,vaddr);
      return(1);
   }

   return(0);
}

/* Check if PC register matches the compiled block virtual address */
static forced_inline 
int ppc32_jit_tcb_match(cpu_ppc_t *cpu,ppc32_jit_tcb_t *block)
{
   m_uint32_t vpage;

   vpage = cpu->ia & ~PPC32_MIN_PAGE_IMASK;
   return(block->start_ia == vpage);
}

/* Execute compiled PowerPC code */
void *ppc32_jit_run_cpu(cpu_gen_t *gen)
{    
   cpu_ppc_t *cpu = CPU_PPC32(gen);
   pthread_t timer_irq_thread;
   ppc32_jit_tcb_t *block;
   m_uint32_t phys_page;
   int timer_irq_check = 0;

   if (pthread_create(&timer_irq_thread,NULL,
                      (void *)ppc32_timer_irq_run,cpu)) 
   {
      fprintf(stderr,
              "VM '%s': unable to create Timer IRQ thread for CPU%u.\n",
              cpu->vm->name,gen->id);
      cpu_stop(cpu->gen);
      return NULL;
   }

   gen->cpu_thread_running = TRUE;

 start_cpu:   
   gen->idle_count = 0;

   for(;;) {
      if (unlikely(gen->state != CPU_STATE_RUNNING))
         break;

#if DEBUG_BLOCK_PERF_CNT
      cpu->perf_counter++;
#endif
      /* Handle virtual idle loop */
      if (unlikely(cpu->ia == cpu->idle_pc)) {
         if (++gen->idle_count == gen->idle_max) {
            cpu_idle_loop(gen);
            gen->idle_count = 0;
         }
      }

      /* Check IRQs */
      if (unlikely(cpu->irq_check))
         ppc32_trigger_irq(cpu);

      /* Handle the virtual CPU clock */
      if (++timer_irq_check == cpu->timer_irq_check_itv) {
         timer_irq_check = 0;

         if (cpu->timer_irq_pending && !cpu->irq_disable &&
             (cpu->msr & PPC32_MSR_EE))
         {
            cpu->timer_irq_armed = 0;
            cpu->timer_irq_pending--;

            vm_set_irq(cpu->vm,0);
            //ppc32_trigger_timer_irq(cpu);
         }
      }

      /* Get the physical page address corresponding to PC register */
      if (unlikely(cpu->translate(cpu,cpu->ia,PPC32_MTS_ICACHE,&phys_page))) {
         fprintf(stderr,"VM '%s': no physical page for CPU%u PC=0x%8.8xx\n",
                 cpu->vm->name,gen->id,cpu->ia);
         cpu_stop(gen);
         break;
      }

      block = cpu->exec_phys_map[phys_page];

      /* No block found, compile the page */
      if (unlikely(!block) || unlikely(!ppc32_jit_tcb_match(cpu,block))) 
      {
         if (block != NULL) {
            ppc32_jit_tcb_free(cpu,block,TRUE);
            cpu->exec_phys_map[phys_page] = NULL;
         }

         block = ppc32_jit_tcb_compile(cpu,cpu->ia);
         if (unlikely(!block)) {
            fprintf(stderr,
                    "VM '%s': unable to compile block for CPU%u IA=0x%8.8x\n",
                    cpu->vm->name,gen->id,cpu->ia);
            cpu_stop(gen);
            break;
         }

         block->phys_page = phys_page;
         cpu->exec_phys_map[phys_page] = block;
      }

#if DEBUG_BLOCK_TIMESTAMP
      block->tm_last_use = jit_jiffies++;
#endif
      block->acc_count++;
      ppc32_jit_tcb_run(cpu,block);
   }
      
   if (!cpu->ia) {
      cpu_stop(gen);
      cpu_log(gen,"JIT","IA=0, halting CPU.\n");
   }

   /* Check regularly if the CPU has been restarted */
   while(gen->cpu_thread_running) {
      gen->seq_state++;

      switch(gen->state) {
         case CPU_STATE_RUNNING:
            gen->state = CPU_STATE_RUNNING;
            goto start_cpu;

         case CPU_STATE_HALTED:
            gen->cpu_thread_running = FALSE;
            pthread_join(timer_irq_thread,NULL);
            break;
      }
      
      /* CPU is paused */
      usleep(200000);
   }

   return NULL;
}
1	dpavlin	7	/*
2			* Cisco router simulation platform.
3			* Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
4			*
5			* PPC32 JIT compiler.
6			*/
7
8			#include <stdio.h>
9			#include <stdlib.h>
10			#include <unistd.h>
11			#include <string.h>
12			#include <sys/types.h>
13			#include <sys/stat.h>
14			#include <sys/mman.h>
15			#include <signal.h>
16			#include <fcntl.h>
17			#include <assert.h>
18
19			#include "cpu.h"
20			#include "device.h"
21			#include "ppc32.h"
22			#include "ppc32_exec.h"
23			#include "ppc32_jit.h"
24			#include "insn_lookup.h"
25			#include "memory.h"
26			#include "ptask.h"
27
28			#include PPC32_ARCH_INC_FILE
29
30			/* Instruction Lookup Table */
31			static insn_lookup_t *ilt = NULL;
32
33			static void *ppc32_jit_get_insn(int index)
34			{
35			return(&ppc32_insn_tags[index]);
36			}
37
38			static int ppc32_jit_chk_lo(struct ppc32_insn_tag *tag,int value)
39			{
40			return((value & tag->mask) == (tag->value & 0xFFFF));
41			}
42
43			static int ppc32_jit_chk_hi(struct ppc32_insn_tag *tag,int value)
44			{
45			return((value & (tag->mask >> 16)) == (tag->value >> 16));
46			}
47
48			/* Initialize instruction lookup table */
49			void ppc32_jit_create_ilt(void)
50			{
51			int i,count;
52
53			for(i=0,count=0;ppc32_insn_tags[i].emit;i++)
54			count++;
55
56			ilt = ilt_create(count+1,
57			(ilt_get_insn_cbk_t)ppc32_jit_get_insn,
58			(ilt_check_cbk_t)ppc32_jit_chk_lo,
59			(ilt_check_cbk_t)ppc32_jit_chk_hi);
60			}
61
62			/* Initialize the JIT structure */
63			int ppc32_jit_init(cpu_ppc_t *cpu)
64			{
65			insn_exec_page_t *cp;
66			u_char *cp_addr;
67			u_int area_size;
68			size_t len;
69			int i;
70
71			/* Physical mapping for executable pages */
72			len = 1048576 * sizeof(void *);
73			cpu->exec_phys_map = m_memalign(4096,len);
74			memset(cpu->exec_phys_map,0,len);
75
76			/* Get area size */
77			if (!(area_size = cpu->vm->exec_area_size))
78			area_size = PPC_EXEC_AREA_SIZE;
79
80			/* Create executable page area */
81			cpu->exec_page_area_size = area_size * 1048576;
82			cpu->exec_page_area = mmap(NULL,cpu->exec_page_area_size,
83			PROT_EXEC\|PROT_READ\|PROT_WRITE,
84			MAP_SHARED\|MAP_ANONYMOUS,-1,(off_t)0);
85
86			if (!cpu->exec_page_area) {
87			fprintf(stderr,
88			"ppc32_jit_init: unable to create exec area (size %lu)\n",
89			(u_long)cpu->exec_page_area_size);
90			return(-1);
91			}
92
93			/* Carve the executable page area */
94			cpu->exec_page_count = cpu->exec_page_area_size / PPC_JIT_BUFSIZE;
95
96			cpu->exec_page_array = calloc(cpu->exec_page_count,
97			sizeof(insn_exec_page_t));
98
99			if (!cpu->exec_page_array) {
100			fprintf(stderr,"ppc32_jit_init: unable to create exec page array\n");
101			return(-1);
102			}
103
104			for(i=0,cp_addr=cpu->exec_page_area;i<cpu->exec_page_count;i++) {
105			cp = &cpu->exec_page_array[i];
106
107			cp->ptr = cp_addr;
108			cp_addr += PPC_JIT_BUFSIZE;
109
110			cp->next = cpu->exec_page_free_list;
111			cpu->exec_page_free_list = cp;
112			}
113
114			printf("CPU%u: carved JIT exec zone of %lu Mb into %lu pages of %u Kb.\n",
115			cpu->gen->id,
116			(u_long)(cpu->exec_page_area_size / 1048576),
117			(u_long)cpu->exec_page_count,PPC_JIT_BUFSIZE / 1024);
118			return(0);
119			}
120
121			/* Flush the JIT */
122			u_int ppc32_jit_flush(cpu_ppc_t *cpu,u_int threshold)
123			{
124			ppc32_jit_tcb_t p,next;
125			u_int count = 0;
126
127			if (!threshold)
128			threshold = (u_int)(-1); /* UINT_MAX not defined everywhere */
129
130			for(p=cpu->tcb_list;p;p=next) {
131			next = p->next;
132
133			if (p->acc_count <= threshold) {
134			cpu->exec_phys_map[p->phys_page] = NULL;
135			ppc32_jit_tcb_free(cpu,p,TRUE);
136			count++;
137			}
138			}
139
140			cpu->compiled_pages -= count;
141			return(count);
142			}
143
144			/* Shutdown the JIT */
145			void ppc32_jit_shutdown(cpu_ppc_t *cpu)
146			{
147			ppc32_jit_tcb_t p,next;
148
149			/* Flush the JIT */
150			ppc32_jit_flush(cpu,0);
151
152			/* Free the instruction blocks */
153			for(p=cpu->tcb_free_list;p;p=next) {
154			next = p->next;
155			free(p);
156			}
157
158			/* Unmap the executable page area */
159			if (cpu->exec_page_area)
160			munmap(cpu->exec_page_area,cpu->exec_page_area_size);
161
162			/* Free the exec page array */
163			free(cpu->exec_page_array);
164
165			/* Free physical mapping for executable pages */
166			free(cpu->exec_phys_map);
167			}
168
169			/* Allocate an exec page */
170			static inline insn_exec_page_t exec_page_alloc(cpu_ppc_t cpu)
171			{
172			insn_exec_page_t *p;
173			u_int count;
174
175			/* If the free list is empty, flush JIT */
176			if (unlikely(!cpu->exec_page_free_list))
177			{
178			if (cpu->jit_flush_method) {
179			cpu_log(cpu->gen,
180			"JIT","flushing data structures (compiled pages=%u)\n",
181			cpu->compiled_pages);
182			ppc32_jit_flush(cpu,0);
183			} else {
184			count = ppc32_jit_flush(cpu,100);
185			cpu_log(cpu->gen,"JIT","partial JIT flush (count=%u)\n",count);
186
187			if (!cpu->exec_page_free_list)
188			ppc32_jit_flush(cpu,0);
189			}
190
191			/* Use both methods alternatively */
192			cpu->jit_flush_method = 1 - cpu->jit_flush_method;
193			}
194
195			if (unlikely(!(p = cpu->exec_page_free_list)))
196			return NULL;
197
198			cpu->exec_page_free_list = p->next;
199			cpu->exec_page_alloc++;
200			return p;
201			}
202
203			/* Free an exec page and returns it to the pool */
204			static inline void exec_page_free(cpu_ppc_t cpu,insn_exec_page_t p)
205			{
206			if (p) {
207			p->next = cpu->exec_page_free_list;
208			cpu->exec_page_free_list = p;
209			cpu->exec_page_alloc--;
210			}
211			}
212
213			/* Find the JIT code emitter for the specified PowerPC instruction */
214			static struct ppc32_insn_tag *insn_tag_find(ppc_insn_t ins)
215			{
216			struct ppc32_insn_tag *tag = NULL;
217			int index;
218
219			index = ilt_lookup(ilt,ins);
220			tag = ppc32_jit_get_insn(index);
221			return tag;
222			}
223
224			/* Fetch a PowerPC instruction */
225			static forced_inline ppc_insn_t insn_fetch(ppc32_jit_tcb_t *b)
226			{
227			return(vmtoh32(b->ppc_code[b->ppc_trans_pos]));
228			}
229
230			/* Emit a breakpoint if necessary */
231			#if BREAKPOINT_ENABLE
232			static void insn_emit_breakpoint(cpu_ppc_t cpu,ppc32_jit_tcb_t b)
233			{
234			m_uint32_t ia;
235			int i;
236
237			ia = b->start_ia+((b->ppc_trans_pos-1)<<2);
238
239			for(i=0;i<PPC32_MAX_BREAKPOINTS;i++)
240			if (ia == cpu->breakpoints[i]) {
241			ppc32_emit_breakpoint(b);
242			break;
243			}
244			}
245			#endif /* BREAKPOINT_ENABLE */
246
247			/* Fetch a PowerPC instruction and emit corresponding translated code */
248			struct ppc32_insn_tag ppc32_jit_fetch_and_emit(cpu_ppc_t cpu,
249			ppc32_jit_tcb_t *block)
250			{
251			struct ppc32_insn_tag *tag;
252			ppc_insn_t code;
253
254			code = insn_fetch(block);
255			tag = insn_tag_find(code);
256			assert(tag);
257
258			block->jit_insn_ptr[block->ppc_trans_pos] = block->jit_ptr;
259			block->ppc_trans_pos++;
260
261			#if DEBUG_INSN_PERF_CNT
262			ppc32_inc_perf_counter(block);
263			#endif
264
265			#if BREAKPOINT_ENABLE
266			if (cpu->breakpoints_enabled)
267			insn_emit_breakpoint(cpu,block);
268			#endif
269
270			tag->emit(cpu,block,code);
271			return tag;
272			}
273
274			/* Add end of JIT block */
275			static void ppc32_jit_tcb_add_end(ppc32_jit_tcb_t *b)
276			{
277			ppc32_set_ia(b,b->start_ia+(b->ppc_trans_pos<<2));
278			ppc32_jit_tcb_push_epilog(b);
279			}
280
281			/* Record a patch to apply in a compiled block */
282			int ppc32_jit_tcb_record_patch(ppc32_jit_tcb_t block,u_char jit_ptr,
283			m_uint32_t vaddr)
284			{
285			struct ppc32_jit_patch_table *ipt = block->patch_table;
286			struct ppc32_insn_patch *patch;
287
288			/* pc must be 32-bit aligned */
289			if (vaddr & 0x03) {
290			fprintf(stderr,"Block 0x%8.8x: trying to record an invalid IA "
291			"(0x%8.8x) - ppc_trans_pos=%d.\n",
292			block->start_ia,vaddr,block->ppc_trans_pos);
293			return(-1);
294			}
295
296			if (!ipt \|\| (ipt->cur_patch >= PPC32_INSN_PATCH_TABLE_SIZE))
297			{
298			/* full table or no table, create a new one */
299			ipt = malloc(sizeof(*ipt));
300			if (!ipt) {
301			fprintf(stderr,"Block 0x%8.8x: unable to create patch table.\n",
302			block->start_ia);
303			return(-1);
304			}
305
306			memset(ipt,0,sizeof(*ipt));
307			ipt->next = block->patch_table;
308			block->patch_table = ipt;
309			}
310
311			#if DEBUG_BLOCK_PATCH
312			printf("Block 0x%8.8x: recording patch [JIT:%p->ppc:0x%8.8x], "
313			"MTP=%d\n",block->start_ia,jit_ptr,vaddr,block->ppc_trans_pos);
314			#endif
315
316			patch = &ipt->patches[ipt->cur_patch];
317			patch->jit_insn = jit_ptr;
318			patch->ppc_ia = vaddr;
319			ipt->cur_patch++;
320			return(0);
321			}
322
323			/* Apply all patches */
324			static int ppc32_jit_tcb_apply_patches(cpu_ppc_t *cpu,
325			ppc32_jit_tcb_t *block)
326			{
327			struct ppc32_jit_patch_table *ipt;
328			struct ppc32_insn_patch *patch;
329			u_char *jit_dst;
330			int i;
331
332			for(ipt=block->patch_table;ipt;ipt=ipt->next)
333			for(i=0;i<ipt->cur_patch;i++)
334			{
335			patch = &ipt->patches[i];
336			jit_dst = ppc32_jit_tcb_get_host_ptr(block,patch->ppc_ia);
337
338			if (jit_dst) {
339			#if DEBUG_BLOCK_PATCH
340			printf("Block 0x%8.8x: applying patch "
341			"[JIT:%p->ppc:0x%8.8x=JIT:%p]\n",
342			block->start_ia,patch->jit_insn,patch->ppc_ia,jit_dst);
343			#endif
344			ppc32_jit_tcb_set_patch(patch->jit_insn,jit_dst);
345			}
346			}
347
348			return(0);
349			}
350
351			/* Free the patch table */
352			static void ppc32_jit_tcb_free_patches(ppc32_jit_tcb_t *block)
353			{
354			struct ppc32_jit_patch_table p,next;
355
356			for(p=block->patch_table;p;p=next) {
357			next = p->next;
358			free(p);
359			}
360
361			block->patch_table = NULL;
362			}
363
364			/* Adjust the JIT buffer if its size is not sufficient */
365			static int ppc32_jit_tcb_adjust_buffer(cpu_ppc_t *cpu,
366			ppc32_jit_tcb_t *block)
367			{
368			insn_exec_page_t *new_buffer;
369
370			if ((block->jit_ptr - block->jit_buffer->ptr) <= (PPC_JIT_BUFSIZE - 512))
371			return(0);
372
373			#if DEBUG_BLOCK_CHUNK
374			printf("Block 0x%8.8x: adjusting JIT buffer...\n",block->start_ia);
375			#endif
376
377			if (block->jit_chunk_pos >= PPC_JIT_MAX_CHUNKS) {
378			fprintf(stderr,"Block 0x%8.8x: too many JIT chunks.\n",block->start_ia);
379			return(-1);
380			}
381
382			if (!(new_buffer = exec_page_alloc(cpu)))
383			return(-1);
384
385			/* record the new exec page */
386			block->jit_chunks[block->jit_chunk_pos++] = block->jit_buffer;
387			block->jit_buffer = new_buffer;
388
389			/* jump to the new exec page (link) */
390			ppc32_jit_tcb_set_jump(block->jit_ptr,new_buffer->ptr);
391			block->jit_ptr = new_buffer->ptr;
392			return(0);
393			}
394
395			/* Allocate an instruction block */
396			static inline ppc32_jit_tcb_t ppc32_jit_tcb_alloc(cpu_ppc_t cpu)
397			{
398			ppc32_jit_tcb_t *p;
399
400			if (cpu->tcb_free_list) {
401			p = cpu->tcb_free_list;
402			cpu->tcb_free_list = p->next;
403			} else {
404			if (!(p = malloc(sizeof(*p))))
405			return NULL;
406			}
407
408			memset(p,0,sizeof(*p));
409			return p;
410			}
411
412			/* Free an instruction block */
413			void ppc32_jit_tcb_free(cpu_ppc_t cpu,ppc32_jit_tcb_t block,
414			int list_removal)
415			{
416			int i;
417
418			if (block) {
419			if (list_removal) {
420			/* Remove the block from the linked list */
421			if (block->next)
422			block->next->prev = block->prev;
423			else
424			cpu->tcb_last = block->prev;
425
426			if (block->prev)
427			block->prev->next = block->next;
428			else
429			cpu->tcb_list = block->next;
430			}
431
432			/* Free the patch tables */
433			ppc32_jit_tcb_free_patches(block);
434
435			/* Free code pages */
436			for(i=0;i<PPC_JIT_MAX_CHUNKS;i++)
437			exec_page_free(cpu,block->jit_chunks[i]);
438
439			/* Free the current JIT buffer */
440			exec_page_free(cpu,block->jit_buffer);
441
442			/* Free the PowerPC-to-native code mapping */
443			free(block->jit_insn_ptr);
444
445			/* Make the block return to the free list */
446			block->next = cpu->tcb_free_list;
447			cpu->tcb_free_list = block;
448			}
449			}
450
451			/* Create an instruction block */
452			static ppc32_jit_tcb_t ppc32_jit_tcb_create(cpu_ppc_t cpu,m_uint32_t vaddr)
453			{
454			ppc32_jit_tcb_t *block = NULL;
455
456			if (!(block = ppc32_jit_tcb_alloc(cpu)))
457			goto err_block_alloc;
458
459			block->start_ia = vaddr;
460
461			/* Allocate the first JIT buffer */
462			if (!(block->jit_buffer = exec_page_alloc(cpu)))
463			goto err_jit_alloc;
464
465			block->jit_ptr = block->jit_buffer->ptr;
466			block->ppc_code = cpu->mem_op_lookup(cpu,block->start_ia,PPC32_MTS_ICACHE);
467
468			if (!block->ppc_code) {
469			fprintf(stderr,"%% No memory map for code execution at 0x%8.8x\n",
470			block->start_ia);
471			goto err_lookup;
472			}
473
474			#if DEBUG_BLOCK_TIMESTAMP
475			block->tm_first_use = block->tm_last_use = jit_jiffies;
476			#endif
477			return block;
478
479			err_lookup:
480			err_jit_alloc:
481			ppc32_jit_tcb_free(cpu,block,FALSE);
482			err_block_alloc:
483			fprintf(stderr,"%% Unable to create instruction block for vaddr=0x%8.8x\n",
484			vaddr);
485			return NULL;
486			}
487
488			/* Compile a PowerPC instruction page */
489			static inline
490			ppc32_jit_tcb_t ppc32_jit_tcb_compile(cpu_ppc_t cpu,m_uint32_t vaddr)
491			{
492			ppc32_jit_tcb_t *block;
493			struct ppc32_insn_tag *tag;
494			m_uint32_t page_addr;
495			size_t len;
496
497			page_addr = vaddr & ~PPC32_MIN_PAGE_IMASK;
498
499			if (unlikely(!(block = ppc32_jit_tcb_create(cpu,page_addr)))) {
500			fprintf(stderr,"insn_page_compile: unable to create JIT block.\n");
501			return NULL;
502			}
503
504			/* Allocate the array used to convert PPC code ptr to native code ptr */
505			len = PPC32_MIN_PAGE_SIZE / sizeof(ppc_insn_t);
506
507			if (!(block->jit_insn_ptr = calloc(len,sizeof(u_char *)))) {
508			fprintf(stderr,"insn_page_compile: unable to create JIT mappings.\n");
509			goto error;
510			}
511
512			/* Emit native code for each instruction */
513			block->ppc_trans_pos = 0;
514
515			while(block->ppc_trans_pos < (PPC32_MIN_PAGE_SIZE/sizeof(ppc_insn_t)))
516			{
517			if (unlikely(!(tag = ppc32_jit_fetch_and_emit(cpu,block)))) {
518			fprintf(stderr,"insn_page_compile: unable to fetch instruction.\n");
519			goto error;
520			}
521
522			#if DEBUG_BLOCK_COMPILE
523			printf("Page 0x%8.8x: emitted tag 0x%8.8x/0x%8.8x\n",
524			block->start_ia,tag->mask,tag->value);
525			#endif
526
527			ppc32_jit_tcb_adjust_buffer(cpu,block);
528			}
529
530			ppc32_jit_tcb_add_end(block);
531			ppc32_jit_tcb_apply_patches(cpu,block);
532			ppc32_jit_tcb_free_patches(block);
533
534			/* Add the block to the linked list */
535			block->next = cpu->tcb_list;
536			block->prev = NULL;
537
538			if (cpu->tcb_list)
539			cpu->tcb_list->prev = block;
540			else
541			cpu->tcb_last = block;
542
543			cpu->tcb_list = block;
544
545			cpu->compiled_pages++;
546			return block;
547
548			error:
549			ppc32_jit_tcb_free(cpu,block,FALSE);
550			return NULL;
551			}
552
553			/* Run a compiled PowerPC instruction block */
554			static forced_inline
555			void ppc32_jit_tcb_run(cpu_ppc_t cpu,ppc32_jit_tcb_t block)
556			{
557			if (unlikely(cpu->ia & 0x03)) {
558			fprintf(stderr,"ppc32_jit_tcb_run: Invalid IA 0x%8.8x.\n",cpu->ia);
559			ppc32_dump_regs(cpu->gen);
560			ppc32_dump_mmu(cpu->gen);
561			cpu_stop(cpu->gen);
562			return;
563			}
564
565			/* Execute JIT compiled code */
566			ppc32_jit_tcb_exec(cpu,block);
567			}
568
569			/* Check if the specified address belongs to the specified block */
570			int ppc32_jit_tcb_local_addr(ppc32_jit_tcb_t *block,m_uint32_t vaddr,
571			u_char **jit_addr)
572			{
573			if ((vaddr >= block->start_ia) &&
574			((vaddr - block->start_ia) < PPC32_MIN_PAGE_SIZE))
575			{
576			*jit_addr = ppc32_jit_tcb_get_host_ptr(block,vaddr);
577			return(1);
578			}
579
580			return(0);
581			}
582
583			/* Check if PC register matches the compiled block virtual address */
584			static forced_inline
585			int ppc32_jit_tcb_match(cpu_ppc_t cpu,ppc32_jit_tcb_t block)
586			{
587			m_uint32_t vpage;
588
589			vpage = cpu->ia & ~PPC32_MIN_PAGE_IMASK;
590			return(block->start_ia == vpage);
591			}
592
593			/* Execute compiled PowerPC code */
594			void ppc32_jit_run_cpu(cpu_gen_t gen)
595			{
596			cpu_ppc_t *cpu = CPU_PPC32(gen);
597			pthread_t timer_irq_thread;
598			ppc32_jit_tcb_t *block;
599			m_uint32_t phys_page;
600			int timer_irq_check = 0;
601
602			if (pthread_create(&timer_irq_thread,NULL,
603			(void *)ppc32_timer_irq_run,cpu))
604			{
605			fprintf(stderr,
606			"VM '%s': unable to create Timer IRQ thread for CPU%u.\n",
607			cpu->vm->name,gen->id);
608			cpu_stop(cpu->gen);
609			return NULL;
610			}
611
612			gen->cpu_thread_running = TRUE;
613
614			start_cpu:
615			gen->idle_count = 0;
616
617			for(;;) {
618			if (unlikely(gen->state != CPU_STATE_RUNNING))
619			break;
620
621			#if DEBUG_BLOCK_PERF_CNT
622			cpu->perf_counter++;
623			#endif
624			/* Handle virtual idle loop */
625			if (unlikely(cpu->ia == cpu->idle_pc)) {
626			if (++gen->idle_count == gen->idle_max) {
627			cpu_idle_loop(gen);
628			gen->idle_count = 0;
629			}
630			}
631
632			/* Check IRQs */
633			if (unlikely(cpu->irq_check))
634			ppc32_trigger_irq(cpu);
635
636			/* Handle the virtual CPU clock */
637			if (++timer_irq_check == cpu->timer_irq_check_itv) {
638			timer_irq_check = 0;
639
640			if (cpu->timer_irq_pending && !cpu->irq_disable &&
641			(cpu->msr & PPC32_MSR_EE))
642			{
643			cpu->timer_irq_armed = 0;
644			cpu->timer_irq_pending--;
645
646			vm_set_irq(cpu->vm,0);
647			//ppc32_trigger_timer_irq(cpu);
648			}
649			}
650
651			/* Get the physical page address corresponding to PC register */
652			if (unlikely(cpu->translate(cpu,cpu->ia,PPC32_MTS_ICACHE,&phys_page))) {
653			fprintf(stderr,"VM '%s': no physical page for CPU%u PC=0x%8.8xx\n",
654			cpu->vm->name,gen->id,cpu->ia);
655			cpu_stop(gen);
656			break;
657			}
658
659			block = cpu->exec_phys_map[phys_page];
660
661			/* No block found, compile the page */
662			if (unlikely(!block) \|\| unlikely(!ppc32_jit_tcb_match(cpu,block)))
663			{
664			if (block != NULL) {
665			ppc32_jit_tcb_free(cpu,block,TRUE);
666			cpu->exec_phys_map[phys_page] = NULL;
667			}
668
669			block = ppc32_jit_tcb_compile(cpu,cpu->ia);
670			if (unlikely(!block)) {
671			fprintf(stderr,
672			"VM '%s': unable to compile block for CPU%u IA=0x%8.8x\n",
673			cpu->vm->name,gen->id,cpu->ia);
674			cpu_stop(gen);
675			break;
676			}
677
678			block->phys_page = phys_page;
679			cpu->exec_phys_map[phys_page] = block;
680			}
681
682			#if DEBUG_BLOCK_TIMESTAMP
683			block->tm_last_use = jit_jiffies++;
684			#endif
685			block->acc_count++;
686			ppc32_jit_tcb_run(cpu,block);
687			}
688
689			if (!cpu->ia) {
690			cpu_stop(gen);
691			cpu_log(gen,"JIT","IA=0, halting CPU.\n");
692			}
693
694			/* Check regularly if the CPU has been restarted */
695			while(gen->cpu_thread_running) {
696			gen->seq_state++;
697
698			switch(gen->state) {
699			case CPU_STATE_RUNNING:
700			gen->state = CPU_STATE_RUNNING;
701			goto start_cpu;
702
703			case CPU_STATE_HALTED:
704			gen->cpu_thread_running = FALSE;
705			pthread_join(timer_irq_thread,NULL);
706			break;
707			}
708
709			/* CPU is paused */
710			usleep(200000);
711			}
712
713			return NULL;
714			}