dynamips/trunk/mips_mts.c

/*
 * Cisco router simulation platform.
 * Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
 *
 * Template code for MTS.
 */

#define MTS_ENTRY  MTS_NAME(entry_t)
#define MTS_CACHE(cpu)  ( cpu->mts_u. MTS_NAME(cache) )

/* Forward declarations */
static forced_inline void *MTS_PROTO(access)(cpu_mips_t *cpu,m_uint64_t vaddr,
                                             u_int op_code,u_int op_size,
                                             u_int op_type,m_uint64_t *data);

static fastcall int MTS_PROTO(translate)(cpu_mips_t *cpu,m_uint64_t vaddr,
                                         m_uint32_t *phys_page);

/* Initialize the MTS subsystem for the specified CPU */
int MTS_PROTO(init)(cpu_mips_t *cpu)
{
   size_t len;

   /* Initialize the cache entries to 0 (empty) */
   len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
   if (!(MTS_CACHE(cpu) = malloc(len)))
      return(-1);

   memset(MTS_CACHE(cpu),0xFF,len);
   cpu->mts_lookups = 0;
   cpu->mts_misses  = 0;
   return(0);
}

/* Free memory used by MTS */
void MTS_PROTO(shutdown)(cpu_mips_t *cpu)
{
   /* Free the cache itself */
   free(MTS_CACHE(cpu));
   MTS_CACHE(cpu) = NULL;
}

/* Show MTS detailed information (debugging only!) */
void MTS_PROTO(show_stats)(cpu_gen_t *gen_cpu)
{
   cpu_mips_t *cpu = CPU_MIPS64(gen_cpu);
#if DEBUG_MTS_MAP_VIRT
   MTS_ENTRY *entry;
   u_int i,count;
#endif

   printf("\nCPU%u: MTS%d statistics:\n",cpu->gen->id,MTS_ADDR_SIZE);

#if DEBUG_MTS_MAP_VIRT
   /* Valid hash entries */
   for(count=0,i=0;i<MTS_NAME_UP(HASH_SIZE);i++) {
      entry = &(MTS_CACHE(cpu)[i]);

      if (!(entry->gvpa & MTS_INV_ENTRY_MASK)) {
         printf("    %4u: vaddr=0x%8.8llx, paddr=0x%8.8llx, hpa=%p\n",
                i,(m_uint64_t)entry->gvpa,(m_uint64_t)entry->gppa,
                (void *)entry->hpa);
         count++;
      }
   }

   printf("   %u/%u valid hash entries.\n",count,MTS_NAME_UP(HASH_SIZE));
#endif

   printf("   Total lookups: %llu, misses: %llu, efficiency: %g%%\n",
          cpu->mts_lookups, cpu->mts_misses,
          100 - ((double)(cpu->mts_misses*100)/
                 (double)cpu->mts_lookups));
}

/* Invalidate the complete MTS cache */
void MTS_PROTO(invalidate_cache)(cpu_mips_t *cpu)
{
   size_t len;

   len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
   memset(MTS_CACHE(cpu),0xFF,len);
}

/* Invalidate partially the MTS cache, given a TLB entry index */
void MTS_PROTO(invalidate_tlb_entry)(cpu_mips_t *cpu,u_int tlb_index)
{
   MTS_PROTO(invalidate_cache)(cpu);
} 

/* 
 * MTS mapping.
 *
 * It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5)
 */
static no_inline MTS_ENTRY *
MTS_PROTO(map)(cpu_mips_t *cpu,u_int op_type,mts_map_t *map,
               MTS_ENTRY *entry,MTS_ENTRY *alt_entry)
{
   struct vdevice *dev;
   m_uint32_t offset;
   m_iptr_t host_ptr;
   int cow;

   if (!(dev = dev_lookup(cpu->vm,map->paddr,map->cached)))
      return NULL;

   if (dev->flags & VDEVICE_FLAG_SPARSE) {
      host_ptr = dev_sparse_get_host_addr(cpu->vm,dev,map->paddr,op_type,&cow);

      entry->gvpa  = map->vaddr;
      entry->gppa  = map->paddr;
      entry->hpa   = host_ptr;
      entry->flags = (cow) ? MTS_FLAG_COW : 0;
      return entry;
   }

   if (!dev->host_addr || (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) {
      offset = (map->paddr + map->offset) - dev->phys_addr;

      /* device entries are never stored in virtual TLB */
      alt_entry->hpa   = (dev->id << MTS_DEVID_SHIFT) + offset;
      alt_entry->flags = MTS_FLAG_DEV;
      return alt_entry;
   }

   entry->gvpa  = map->vaddr;
   entry->gppa  = map->paddr;
   entry->hpa   = dev->host_addr + (map->paddr - dev->phys_addr);
   entry->flags = 0;

   return entry;
}

/* MTS lookup */
static void *MTS_PROTO(lookup)(cpu_mips_t *cpu,m_uint64_t vaddr)
{
   m_uint64_t data;
   return(MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LOOKUP,4,MTS_READ,&data));
}

/* === MIPS Memory Operations ============================================= */

/* LB: Load Byte */
fastcall void MTS_PROTO(lb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LB,1,MTS_READ,&data);
   if (likely(haddr != NULL)) data = *(m_uint8_t *)haddr;
   cpu->gpr[reg] = sign_extend(data,8);
}

/* LBU: Load Byte Unsigned */
fastcall void MTS_PROTO(lbu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LBU,1,MTS_READ,&data);
   if (likely(haddr != NULL)) data = *(m_uint8_t *)haddr;
   cpu->gpr[reg] = data & 0xff;
}

/* LH: Load Half-Word */
fastcall void MTS_PROTO(lh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LH,2,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh16(*(m_uint16_t *)haddr);
   cpu->gpr[reg] = sign_extend(data,16);
}

/* LHU: Load Half-Word Unsigned */
fastcall void MTS_PROTO(lhu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LHU,2,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh16(*(m_uint16_t *)haddr);
   cpu->gpr[reg] = data & 0xffff;
}

/* LW: Load Word */
fastcall void MTS_PROTO(lw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LW,4,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr);
   cpu->gpr[reg] = sign_extend(data,32);
}

/* LWU: Load Word Unsigned */
fastcall void MTS_PROTO(lwu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LWU,4,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr);
   cpu->gpr[reg] = data & 0xffffffff;
}

/* LD: Load Double-Word */
fastcall void MTS_PROTO(ld)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LD,8,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh64(*(m_uint64_t *)haddr);
   cpu->gpr[reg] = data;
}

/* SB: Store Byte */
fastcall void MTS_PROTO(sb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   data = cpu->gpr[reg] & 0xff;
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SB,1,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint8_t *)haddr = data;
}

/* SH: Store Half-Word */
fastcall void MTS_PROTO(sh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   data = cpu->gpr[reg] & 0xffff;
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SH,2,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint16_t *)haddr = htovm16(data);
}

/* SW: Store Word */
fastcall void MTS_PROTO(sw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   data = cpu->gpr[reg] & 0xffffffff;
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SW,4,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
}

/* SD: Store Double-Word */
fastcall void MTS_PROTO(sd)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   data = cpu->gpr[reg];
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SD,8,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
}

/* LDC1: Load Double-Word To Coprocessor 1 */
fastcall void MTS_PROTO(ldc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LDC1,8,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh64(*(m_uint64_t *)haddr);
   cpu->fpu.reg[reg] = data;
}

/* LWL: Load Word Left */
fastcall void MTS_PROTO(lwl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;

   naddr = vaddr & ~(0x03);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWL,4,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   m_shift = (vaddr & 0x03) << 3;
   r_mask = (1ULL << m_shift) - 1;
   data <<= m_shift;

   cpu->gpr[reg] &= r_mask;
   cpu->gpr[reg] |= data;
   cpu->gpr[reg] = sign_extend(cpu->gpr[reg],32);
}

/* LWR: Load Word Right */
fastcall void MTS_PROTO(lwr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;

   naddr = vaddr & ~(0x03);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWR,4,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   m_shift = ((vaddr & 0x03) + 1) << 3;
   r_mask = (1ULL << m_shift) - 1;

   data = sign_extend(data >> (32 - m_shift),32);
   r_mask = sign_extend(r_mask,32);

   cpu->gpr[reg] &= ~r_mask;
   cpu->gpr[reg] |= data;
}

/* LDL: Load Double-Word Left */
fastcall void MTS_PROTO(ldl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDL,8,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   m_shift = (vaddr & 0x07) << 3;
   r_mask = (1ULL << m_shift) - 1;
   data <<= m_shift;

   cpu->gpr[reg] &= r_mask;
   cpu->gpr[reg] |= data;
}

/* LDR: Load Double-Word Right */
fastcall void MTS_PROTO(ldr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t r_mask,naddr;
   m_uint64_t data;
   u_int m_shift;
   void *haddr;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDR,8,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   m_shift = ((vaddr & 0x07) + 1) << 3;
   r_mask = (1ULL << m_shift) - 1;
   data >>= (64 - m_shift);
   
   cpu->gpr[reg] &= ~r_mask;
   cpu->gpr[reg] |= data;
}

/* SWL: Store Word Left */
fastcall void MTS_PROTO(swl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;

   naddr = vaddr & ~(0x03ULL);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   r_shift = (vaddr & 0x03) << 3;
   d_mask = 0xffffffff >> r_shift;

   data &= ~d_mask;
   data |= (cpu->gpr[reg] & 0xffffffff) >> r_shift;

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
}

/* SWR: Store Word Right */
fastcall void MTS_PROTO(swr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;

   naddr = vaddr & ~(0x03);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh32(*(m_uint32_t *)haddr);

   r_shift = ((vaddr & 0x03) + 1) << 3;
   d_mask = 0xffffffff >> r_shift;

   data &= d_mask;
   data |= (cpu->gpr[reg] << (32 - r_shift)) & 0xffffffff;

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
}

/* SDL: Store Double-Word Left */
fastcall void MTS_PROTO(sdl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   r_shift = (vaddr & 0x07) << 3;
   d_mask = 0xffffffffffffffffULL >> r_shift;

   data &= ~d_mask;
   data |= cpu->gpr[reg] >> r_shift;

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
}

/* SDR: Store Double-Word Right */
fastcall void MTS_PROTO(sdr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t d_mask,naddr;
   m_uint64_t data;
   u_int r_shift;
   void *haddr;

   naddr = vaddr & ~(0x07);
   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_READ,&data);

   if (likely(haddr != NULL))
      data = vmtoh64(*(m_uint64_t *)haddr);

   r_shift = ((vaddr & 0x07) + 1) << 3;
   d_mask = 0xffffffffffffffffULL >> r_shift;

   data &= d_mask;
   data |= cpu->gpr[reg] << (64 - r_shift);

   haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
}

/* LL: Load Linked */
fastcall void MTS_PROTO(ll)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LL,4,MTS_READ,&data);
   if (likely(haddr != NULL)) data = vmtoh32(*(m_uint32_t *)haddr);

   cpu->gpr[reg] = sign_extend(data,32);
   cpu->ll_bit = 1;
}

/* SC: Store Conditional */
fastcall void MTS_PROTO(sc)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   if (cpu->ll_bit) {
      data = cpu->gpr[reg] & 0xffffffff;
      haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SC,4,MTS_WRITE,&data);
      if (likely(haddr != NULL)) *(m_uint32_t *)haddr = htovm32(data);
   }

   cpu->gpr[reg] = cpu->ll_bit;
}

/* SDC1: Store Double-Word from Coprocessor 1 */
fastcall void MTS_PROTO(sdc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
{
   m_uint64_t data;
   void *haddr;

   data = cpu->fpu.reg[reg];
   haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SDC1,8,MTS_WRITE,&data);
   if (likely(haddr != NULL)) *(m_uint64_t *)haddr = htovm64(data);
}

/* CACHE: Cache operation */
fastcall void MTS_PROTO(cache)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op)
{
   mips64_jit_tcb_t *block;
   m_uint32_t pc_hash;

#if DEBUG_CACHE
   cpu_log(cpu->gen,
           "MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n",
           cpu->pc, vaddr, op & 0x3, op >> 2);
#endif

   if (cpu->exec_blk_map) {
      pc_hash = mips64_jit_get_pc_hash(vaddr);
      block = cpu->exec_blk_map[pc_hash];

      if (block && (block->start_pc == (vaddr & MIPS_MIN_PAGE_MASK))) {
#if DEBUG_CACHE
         cpu_log(cpu->gen,"MTS",
                 "CACHE: removing compiled page at 0x%llx, pc=0x%llx\n",
                 block->start_pc,cpu->pc);
#endif
         cpu->exec_blk_map[pc_hash] = NULL;
         mips64_jit_tcb_free(cpu,block,TRUE);
      }
      else
      {
#if DEBUG_CACHE
         cpu_log(cpu->gen,"MTS",
                 "CACHE: trying to remove page 0x%llx with pc=0x%llx\n",
                 vaddr, cpu->pc);
#endif         
      }
   }
}

/* === MTS Cache Management ============================================= */

/* MTS map/unmap/rebuild "API" functions */
void MTS_PROTO(api_map)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t paddr,
                        m_uint32_t len,int cache_access,int tlb_index)
{
   /* nothing to do, the cache will be filled on-the-fly */
}

void MTS_PROTO(api_unmap)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
                          m_uint32_t val,int tlb_index)
{
   /* Invalidate the TLB entry or the full cache if no index is specified */
   if (tlb_index != -1)
      MTS_PROTO(invalidate_tlb_entry)(cpu,tlb_index);
   else
      MTS_PROTO(invalidate_cache)(cpu);
}

void MTS_PROTO(api_rebuild)(cpu_gen_t *cpu)
{
   MTS_PROTO(invalidate_cache)(CPU_MIPS64(cpu));
}

/* ======================================================================== */

/* Initialize memory access vectors */
void MTS_PROTO(init_memop_vectors)(cpu_mips_t *cpu)
{
   /* XXX TODO:
    *  - LD/SD forbidden in Supervisor/User modes with 32-bit addresses.
    */

   cpu->addr_mode = MTS_ADDR_SIZE;

   /* API vectors */
   cpu->mts_map     = MTS_PROTO(api_map);
   cpu->mts_unmap   = MTS_PROTO(api_unmap);

   /* Memory lookup operation */
   cpu->mem_op_lookup = MTS_PROTO(lookup);

   /* Translation operation */
   cpu->translate = MTS_PROTO(translate);

   /* Shutdown operation */
   cpu->mts_shutdown = MTS_PROTO(shutdown);

   /* Rebuild MTS data structures */
   cpu->gen->mts_rebuild = MTS_PROTO(api_rebuild);

   /* Show statistics */
   cpu->gen->mts_show_stats = MTS_PROTO(show_stats);

   /* Load Operations */
   cpu->mem_op_fn[MIPS_MEMOP_LB] = MTS_PROTO(lb);
   cpu->mem_op_fn[MIPS_MEMOP_LBU] = MTS_PROTO(lbu);
   cpu->mem_op_fn[MIPS_MEMOP_LH] = MTS_PROTO(lh);
   cpu->mem_op_fn[MIPS_MEMOP_LHU] = MTS_PROTO(lhu);
   cpu->mem_op_fn[MIPS_MEMOP_LW] = MTS_PROTO(lw);
   cpu->mem_op_fn[MIPS_MEMOP_LWU] = MTS_PROTO(lwu);
   cpu->mem_op_fn[MIPS_MEMOP_LD] = MTS_PROTO(ld);
   cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
   cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);

   /* Store Operations */
   cpu->mem_op_fn[MIPS_MEMOP_SB] = MTS_PROTO(sb);
   cpu->mem_op_fn[MIPS_MEMOP_SH] = MTS_PROTO(sh);
   cpu->mem_op_fn[MIPS_MEMOP_SW] = MTS_PROTO(sw);
   cpu->mem_op_fn[MIPS_MEMOP_SD] = MTS_PROTO(sd);

   /* Load Left/Right operations */
   cpu->mem_op_fn[MIPS_MEMOP_LWL] = MTS_PROTO(lwl);
   cpu->mem_op_fn[MIPS_MEMOP_LWR] = MTS_PROTO(lwr);
   cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
   cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);

   /* Store Left/Right operations */
   cpu->mem_op_fn[MIPS_MEMOP_SWL] = MTS_PROTO(swl);
   cpu->mem_op_fn[MIPS_MEMOP_SWR] = MTS_PROTO(swr);
   cpu->mem_op_fn[MIPS_MEMOP_SDL] = MTS_PROTO(sdl);
   cpu->mem_op_fn[MIPS_MEMOP_SDR] = MTS_PROTO(sdr);

   /* LL/SC - Load Linked / Store Conditional */
   cpu->mem_op_fn[MIPS_MEMOP_LL] = MTS_PROTO(ll);
   cpu->mem_op_fn[MIPS_MEMOP_SC] = MTS_PROTO(sc);

   /* Coprocessor 1 memory access functions */
   cpu->mem_op_fn[MIPS_MEMOP_LDC1] = MTS_PROTO(ldc1);
   cpu->mem_op_fn[MIPS_MEMOP_SDC1] = MTS_PROTO(sdc1);

   /* Cache Operation */
   cpu->mem_op_fn[MIPS_MEMOP_CACHE] = MTS_PROTO(cache);
}

#undef MTS_ADDR_SIZE
#undef MTS_NAME
#undef MTS_NAME_UP
#undef MTS_PROTO
#undef MTS_PROTO_UP
#undef MTS_ENTRY
#undef MTS_CHUNK
1	dpavlin	4	/*
2			* Cisco router simulation platform.
3			* Copyright (c) 2005,2006 Christophe Fillot (cf@utc.fr)
4			*
5			* Template code for MTS.
6			*/
7
8	dpavlin	7	#define MTS_ENTRY MTS_NAME(entry_t)
9			#define MTS_CACHE(cpu) ( cpu->mts_u. MTS_NAME(cache) )
10	dpavlin	4
11			/* Forward declarations */
12			static forced_inline void MTS_PROTO(access)(cpu_mips_t cpu,m_uint64_t vaddr,
13			u_int op_code,u_int op_size,
14	dpavlin	11	u_int op_type,m_uint64_t *data);
15	dpavlin	4
16			static fastcall int MTS_PROTO(translate)(cpu_mips_t *cpu,m_uint64_t vaddr,
17			m_uint32_t *phys_page);
18
19			/* Initialize the MTS subsystem for the specified CPU */
20			int MTS_PROTO(init)(cpu_mips_t *cpu)
21			{
22			size_t len;
23
24			/* Initialize the cache entries to 0 (empty) */
25	dpavlin	7	len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
26			if (!(MTS_CACHE(cpu) = malloc(len)))
27	dpavlin	4	return(-1);
28
29	dpavlin	7	memset(MTS_CACHE(cpu),0xFF,len);
30	dpavlin	4	cpu->mts_lookups = 0;
31			cpu->mts_misses = 0;
32			return(0);
33			}
34
35			/* Free memory used by MTS */
36			void MTS_PROTO(shutdown)(cpu_mips_t *cpu)
37			{
38			/* Free the cache itself */
39	dpavlin	7	free(MTS_CACHE(cpu));
40			MTS_CACHE(cpu) = NULL;
41	dpavlin	4	}
42
43			/* Show MTS detailed information (debugging only!) */
44	dpavlin	7	void MTS_PROTO(show_stats)(cpu_gen_t *gen_cpu)
45	dpavlin	4	{
46	dpavlin	7	cpu_mips_t *cpu = CPU_MIPS64(gen_cpu);
47	dpavlin	4	#if DEBUG_MTS_MAP_VIRT
48			MTS_ENTRY *entry;
49	dpavlin	7	u_int i,count;
50	dpavlin	4	#endif
51
52	dpavlin	7	printf("\nCPU%u: MTS%d statistics:\n",cpu->gen->id,MTS_ADDR_SIZE);
53	dpavlin	4
54			#if DEBUG_MTS_MAP_VIRT
55			/* Valid hash entries */
56	dpavlin	7	for(count=0,i=0;i<MTS_NAME_UP(HASH_SIZE);i++) {
57			entry = &(MTS_CACHE(cpu)[i]);
58
59			if (!(entry->gvpa & MTS_INV_ENTRY_MASK)) {
60			printf(" %4u: vaddr=0x%8.8llx, paddr=0x%8.8llx, hpa=%p\n",
61			i,(m_uint64_t)entry->gvpa,(m_uint64_t)entry->gppa,
62			(void *)entry->hpa);
63	dpavlin	4	count++;
64			}
65			}
66
67	dpavlin	7	printf(" %u/%u valid hash entries.\n",count,MTS_NAME_UP(HASH_SIZE));
68	dpavlin	4	#endif
69
70	dpavlin	7	printf(" Total lookups: %llu, misses: %llu, efficiency: %g%%\n",
71			cpu->mts_lookups, cpu->mts_misses,
72			100 - ((double)(cpu->mts_misses*100)/
73			(double)cpu->mts_lookups));
74	dpavlin	4	}
75
76			/* Invalidate the complete MTS cache */
77			void MTS_PROTO(invalidate_cache)(cpu_mips_t *cpu)
78			{
79			size_t len;
80
81	dpavlin	7	len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY);
82			memset(MTS_CACHE(cpu),0xFF,len);
83	dpavlin	4	}
84
85			/* Invalidate partially the MTS cache, given a TLB entry index */
86			void MTS_PROTO(invalidate_tlb_entry)(cpu_mips_t *cpu,u_int tlb_index)
87			{
88	dpavlin	7	MTS_PROTO(invalidate_cache)(cpu);
89	dpavlin	4	}
90
91			/*
92			* MTS mapping.
93			*
94			* It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5)
95			*/
96	dpavlin	7	static no_inline MTS_ENTRY *
97			MTS_PROTO(map)(cpu_mips_t cpu,u_int op_type,mts_map_t map,
98			MTS_ENTRY entry,MTS_ENTRY alt_entry)
99	dpavlin	4	{
100			struct vdevice *dev;
101	dpavlin	7	m_uint32_t offset;
102			m_iptr_t host_ptr;
103			int cow;
104	dpavlin	4
105	dpavlin	7	if (!(dev = dev_lookup(cpu->vm,map->paddr,map->cached)))
106			return NULL;
107	dpavlin	4
108	dpavlin	7	if (dev->flags & VDEVICE_FLAG_SPARSE) {
109			host_ptr = dev_sparse_get_host_addr(cpu->vm,dev,map->paddr,op_type,&cow);
110	dpavlin	4
111	dpavlin	7	entry->gvpa = map->vaddr;
112			entry->gppa = map->paddr;
113			entry->hpa = host_ptr;
114			entry->flags = (cow) ? MTS_FLAG_COW : 0;
115			return entry;
116	dpavlin	4	}
117
118	dpavlin	7	if (!dev->host_addr \|\| (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) {
119	dpavlin	11	offset = (map->paddr + map->offset) - dev->phys_addr;
120	dpavlin	4
121	dpavlin	11	/* device entries are never stored in virtual TLB */
122	dpavlin	7	alt_entry->hpa = (dev->id << MTS_DEVID_SHIFT) + offset;
123			alt_entry->flags = MTS_FLAG_DEV;
124			return alt_entry;
125			}
126
127			entry->gvpa = map->vaddr;
128			entry->gppa = map->paddr;
129			entry->hpa = dev->host_addr + (map->paddr - dev->phys_addr);
130			entry->flags = 0;
131
132			return entry;
133	dpavlin	4	}
134
135			/* MTS lookup */
136			static void MTS_PROTO(lookup)(cpu_mips_t cpu,m_uint64_t vaddr)
137			{
138			m_uint64_t data;
139	dpavlin	11	return(MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LOOKUP,4,MTS_READ,&data));
140	dpavlin	4	}
141
142			/* === MIPS Memory Operations ============================================= */
143
144			/* LB: Load Byte */
145	dpavlin	11	fastcall void MTS_PROTO(lb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
146	dpavlin	4	{
147			m_uint64_t data;
148			void *haddr;
149
150	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LB,1,MTS_READ,&data);
151	dpavlin	4	if (likely(haddr != NULL)) data = (m_uint8_t )haddr;
152	dpavlin	11	cpu->gpr[reg] = sign_extend(data,8);
153	dpavlin	4	}
154
155			/* LBU: Load Byte Unsigned */
156	dpavlin	11	fastcall void MTS_PROTO(lbu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
157	dpavlin	4	{
158			m_uint64_t data;
159			void *haddr;
160
161	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LBU,1,MTS_READ,&data);
162	dpavlin	4	if (likely(haddr != NULL)) data = (m_uint8_t )haddr;
163	dpavlin	11	cpu->gpr[reg] = data & 0xff;
164	dpavlin	4	}
165
166			/* LH: Load Half-Word */
167	dpavlin	11	fastcall void MTS_PROTO(lh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
168	dpavlin	4	{
169			m_uint64_t data;
170			void *haddr;
171
172	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LH,2,MTS_READ,&data);
173	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh16((m_uint16_t )haddr);
174	dpavlin	11	cpu->gpr[reg] = sign_extend(data,16);
175	dpavlin	4	}
176
177			/* LHU: Load Half-Word Unsigned */
178	dpavlin	11	fastcall void MTS_PROTO(lhu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
179	dpavlin	4	{
180			m_uint64_t data;
181			void *haddr;
182
183	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LHU,2,MTS_READ,&data);
184	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh16((m_uint16_t )haddr);
185	dpavlin	11	cpu->gpr[reg] = data & 0xffff;
186	dpavlin	4	}
187
188			/* LW: Load Word */
189	dpavlin	11	fastcall void MTS_PROTO(lw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
190	dpavlin	4	{
191			m_uint64_t data;
192			void *haddr;
193
194	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LW,4,MTS_READ,&data);
195	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh32((m_uint32_t )haddr);
196	dpavlin	11	cpu->gpr[reg] = sign_extend(data,32);
197	dpavlin	4	}
198
199			/* LWU: Load Word Unsigned */
200	dpavlin	11	fastcall void MTS_PROTO(lwu)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
201	dpavlin	4	{
202			m_uint64_t data;
203			void *haddr;
204
205	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LWU,4,MTS_READ,&data);
206	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh32((m_uint32_t )haddr);
207	dpavlin	11	cpu->gpr[reg] = data & 0xffffffff;
208	dpavlin	4	}
209
210			/* LD: Load Double-Word */
211	dpavlin	11	fastcall void MTS_PROTO(ld)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
212	dpavlin	4	{
213			m_uint64_t data;
214			void *haddr;
215
216	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LD,8,MTS_READ,&data);
217	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh64((m_uint64_t )haddr);
218	dpavlin	11	cpu->gpr[reg] = data;
219	dpavlin	4	}
220
221			/* SB: Store Byte */
222	dpavlin	11	fastcall void MTS_PROTO(sb)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
223	dpavlin	4	{
224			m_uint64_t data;
225			void *haddr;
226
227			data = cpu->gpr[reg] & 0xff;
228	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SB,1,MTS_WRITE,&data);
229	dpavlin	4	if (likely(haddr != NULL)) (m_uint8_t )haddr = data;
230			}
231
232			/* SH: Store Half-Word */
233	dpavlin	11	fastcall void MTS_PROTO(sh)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
234	dpavlin	4	{
235			m_uint64_t data;
236			void *haddr;
237
238			data = cpu->gpr[reg] & 0xffff;
239	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SH,2,MTS_WRITE,&data);
240	dpavlin	4	if (likely(haddr != NULL)) (m_uint16_t )haddr = htovm16(data);
241			}
242
243			/* SW: Store Word */
244	dpavlin	11	fastcall void MTS_PROTO(sw)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
245	dpavlin	4	{
246			m_uint64_t data;
247			void *haddr;
248
249			data = cpu->gpr[reg] & 0xffffffff;
250	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SW,4,MTS_WRITE,&data);
251	dpavlin	4	if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
252			}
253
254			/* SD: Store Double-Word */
255	dpavlin	11	fastcall void MTS_PROTO(sd)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
256	dpavlin	4	{
257			m_uint64_t data;
258			void *haddr;
259
260			data = cpu->gpr[reg];
261	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SD,8,MTS_WRITE,&data);
262	dpavlin	4	if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
263			}
264
265			/* LDC1: Load Double-Word To Coprocessor 1 */
266	dpavlin	11	fastcall void MTS_PROTO(ldc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
267	dpavlin	4	{
268			m_uint64_t data;
269			void *haddr;
270
271	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LDC1,8,MTS_READ,&data);
272	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh64((m_uint64_t )haddr);
273	dpavlin	11	cpu->fpu.reg[reg] = data;
274	dpavlin	4	}
275
276			/* LWL: Load Word Left */
277	dpavlin	11	fastcall void MTS_PROTO(lwl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
278	dpavlin	4	{
279			m_uint64_t r_mask,naddr;
280			m_uint64_t data;
281			u_int m_shift;
282			void *haddr;
283
284			naddr = vaddr & ~(0x03);
285	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWL,4,MTS_READ,&data);
286	dpavlin	4
287			if (likely(haddr != NULL))
288			data = vmtoh32((m_uint32_t )haddr);
289
290	dpavlin	11	m_shift = (vaddr & 0x03) << 3;
291			r_mask = (1ULL << m_shift) - 1;
292			data <<= m_shift;
293	dpavlin	4
294	dpavlin	11	cpu->gpr[reg] &= r_mask;
295			cpu->gpr[reg] \|= data;
296			cpu->gpr[reg] = sign_extend(cpu->gpr[reg],32);
297	dpavlin	4	}
298
299			/* LWR: Load Word Right */
300	dpavlin	11	fastcall void MTS_PROTO(lwr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
301	dpavlin	4	{
302			m_uint64_t r_mask,naddr;
303			m_uint64_t data;
304			u_int m_shift;
305			void *haddr;
306
307			naddr = vaddr & ~(0x03);
308	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LWR,4,MTS_READ,&data);
309	dpavlin	4
310			if (likely(haddr != NULL))
311			data = vmtoh32((m_uint32_t )haddr);
312
313	dpavlin	11	m_shift = ((vaddr & 0x03) + 1) << 3;
314			r_mask = (1ULL << m_shift) - 1;
315	dpavlin	4
316	dpavlin	11	data = sign_extend(data >> (32 - m_shift),32);
317			r_mask = sign_extend(r_mask,32);
318	dpavlin	4
319	dpavlin	11	cpu->gpr[reg] &= ~r_mask;
320			cpu->gpr[reg] \|= data;
321	dpavlin	4	}
322
323			/* LDL: Load Double-Word Left */
324	dpavlin	11	fastcall void MTS_PROTO(ldl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
325	dpavlin	4	{
326			m_uint64_t r_mask,naddr;
327			m_uint64_t data;
328			u_int m_shift;
329			void *haddr;
330
331			naddr = vaddr & ~(0x07);
332	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDL,8,MTS_READ,&data);
333	dpavlin	4
334			if (likely(haddr != NULL))
335			data = vmtoh64((m_uint64_t )haddr);
336
337	dpavlin	11	m_shift = (vaddr & 0x07) << 3;
338			r_mask = (1ULL << m_shift) - 1;
339			data <<= m_shift;
340	dpavlin	4
341	dpavlin	11	cpu->gpr[reg] &= r_mask;
342			cpu->gpr[reg] \|= data;
343	dpavlin	4	}
344
345			/* LDR: Load Double-Word Right */
346	dpavlin	11	fastcall void MTS_PROTO(ldr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
347	dpavlin	4	{
348			m_uint64_t r_mask,naddr;
349			m_uint64_t data;
350			u_int m_shift;
351			void *haddr;
352
353			naddr = vaddr & ~(0x07);
354	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_LDR,8,MTS_READ,&data);
355	dpavlin	4
356			if (likely(haddr != NULL))
357			data = vmtoh64((m_uint64_t )haddr);
358
359	dpavlin	11	m_shift = ((vaddr & 0x07) + 1) << 3;
360			r_mask = (1ULL << m_shift) - 1;
361			data >>= (64 - m_shift);
362
363			cpu->gpr[reg] &= ~r_mask;
364			cpu->gpr[reg] \|= data;
365	dpavlin	4	}
366
367			/* SWL: Store Word Left */
368	dpavlin	11	fastcall void MTS_PROTO(swl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
369	dpavlin	4	{
370			m_uint64_t d_mask,naddr;
371			m_uint64_t data;
372			u_int r_shift;
373			void *haddr;
374
375			naddr = vaddr & ~(0x03ULL);
376	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_READ,&data);
377	dpavlin	4
378			if (likely(haddr != NULL))
379			data = vmtoh32((m_uint32_t )haddr);
380
381			r_shift = (vaddr & 0x03) << 3;
382			d_mask = 0xffffffff >> r_shift;
383
384			data &= ~d_mask;
385			data \|= (cpu->gpr[reg] & 0xffffffff) >> r_shift;
386
387	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWL,4,MTS_WRITE,&data);
388	dpavlin	4	if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
389			}
390
391			/* SWR: Store Word Right */
392	dpavlin	11	fastcall void MTS_PROTO(swr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
393	dpavlin	4	{
394			m_uint64_t d_mask,naddr;
395			m_uint64_t data;
396			u_int r_shift;
397			void *haddr;
398
399			naddr = vaddr & ~(0x03);
400	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_READ,&data);
401	dpavlin	4
402			if (likely(haddr != NULL))
403			data = vmtoh32((m_uint32_t )haddr);
404
405			r_shift = ((vaddr & 0x03) + 1) << 3;
406			d_mask = 0xffffffff >> r_shift;
407
408			data &= d_mask;
409			data \|= (cpu->gpr[reg] << (32 - r_shift)) & 0xffffffff;
410
411	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SWR,4,MTS_WRITE,&data);
412	dpavlin	4	if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
413			}
414
415			/* SDL: Store Double-Word Left */
416	dpavlin	11	fastcall void MTS_PROTO(sdl)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
417	dpavlin	4	{
418			m_uint64_t d_mask,naddr;
419			m_uint64_t data;
420			u_int r_shift;
421			void *haddr;
422
423			naddr = vaddr & ~(0x07);
424	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_READ,&data);
425	dpavlin	4
426			if (likely(haddr != NULL))
427			data = vmtoh64((m_uint64_t )haddr);
428
429			r_shift = (vaddr & 0x07) << 3;
430			d_mask = 0xffffffffffffffffULL >> r_shift;
431
432			data &= ~d_mask;
433			data \|= cpu->gpr[reg] >> r_shift;
434
435	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDL,8,MTS_WRITE,&data);
436	dpavlin	4	if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
437			}
438
439			/* SDR: Store Double-Word Right */
440	dpavlin	11	fastcall void MTS_PROTO(sdr)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
441	dpavlin	4	{
442			m_uint64_t d_mask,naddr;
443			m_uint64_t data;
444			u_int r_shift;
445			void *haddr;
446
447			naddr = vaddr & ~(0x07);
448	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_READ,&data);
449	dpavlin	4
450			if (likely(haddr != NULL))
451			data = vmtoh64((m_uint64_t )haddr);
452
453			r_shift = ((vaddr & 0x07) + 1) << 3;
454			d_mask = 0xffffffffffffffffULL >> r_shift;
455
456			data &= d_mask;
457			data \|= cpu->gpr[reg] << (64 - r_shift);
458
459	dpavlin	11	haddr = MTS_PROTO(access)(cpu,naddr,MIPS_MEMOP_SDR,8,MTS_WRITE,&data);
460	dpavlin	4	if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
461			}
462
463			/* LL: Load Linked */
464	dpavlin	11	fastcall void MTS_PROTO(ll)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
465	dpavlin	4	{
466			m_uint64_t data;
467			void *haddr;
468
469	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_LL,4,MTS_READ,&data);
470	dpavlin	4	if (likely(haddr != NULL)) data = vmtoh32((m_uint32_t )haddr);
471
472	dpavlin	11	cpu->gpr[reg] = sign_extend(data,32);
473			cpu->ll_bit = 1;
474	dpavlin	4	}
475
476			/* SC: Store Conditional */
477	dpavlin	11	fastcall void MTS_PROTO(sc)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
478	dpavlin	4	{
479			m_uint64_t data;
480			void *haddr;
481
482			if (cpu->ll_bit) {
483			data = cpu->gpr[reg] & 0xffffffff;
484	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SC,4,MTS_WRITE,&data);
485	dpavlin	4	if (likely(haddr != NULL)) (m_uint32_t )haddr = htovm32(data);
486			}
487
488	dpavlin	11	cpu->gpr[reg] = cpu->ll_bit;
489	dpavlin	4	}
490
491			/* SDC1: Store Double-Word from Coprocessor 1 */
492	dpavlin	11	fastcall void MTS_PROTO(sdc1)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int reg)
493	dpavlin	4	{
494			m_uint64_t data;
495			void *haddr;
496
497			data = cpu->fpu.reg[reg];
498	dpavlin	11	haddr = MTS_PROTO(access)(cpu,vaddr,MIPS_MEMOP_SDC1,8,MTS_WRITE,&data);
499	dpavlin	4	if (likely(haddr != NULL)) (m_uint64_t )haddr = htovm64(data);
500			}
501
502			/* CACHE: Cache operation */
503	dpavlin	11	fastcall void MTS_PROTO(cache)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op)
504	dpavlin	4	{
505	dpavlin	7	mips64_jit_tcb_t *block;
506	dpavlin	8	m_uint32_t pc_hash;
507	dpavlin	4
508			#if DEBUG_CACHE
509	dpavlin	7	cpu_log(cpu->gen,
510			"MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n",
511	dpavlin	4	cpu->pc, vaddr, op & 0x3, op >> 2);
512			#endif
513
514	dpavlin	8	if (cpu->exec_blk_map) {
515	dpavlin	9	pc_hash = mips64_jit_get_pc_hash(vaddr);
516	dpavlin	8	block = cpu->exec_blk_map[pc_hash];
517	dpavlin	4
518	dpavlin	9	if (block && (block->start_pc == (vaddr & MIPS_MIN_PAGE_MASK))) {
519	dpavlin	4	#if DEBUG_CACHE
520	dpavlin	8	cpu_log(cpu->gen,"MTS",
521			"CACHE: removing compiled page at 0x%llx, pc=0x%llx\n",
522			block->start_pc,cpu->pc);
523	dpavlin	4	#endif
524	dpavlin	8	cpu->exec_blk_map[pc_hash] = NULL;
525			mips64_jit_tcb_free(cpu,block,TRUE);
526			}
527			else
528			{
529	dpavlin	4	#if DEBUG_CACHE
530	dpavlin	8	cpu_log(cpu->gen,"MTS",
531			"CACHE: trying to remove page 0x%llx with pc=0x%llx\n",
532	dpavlin	9	vaddr, cpu->pc);
533	dpavlin	11	#endif
534	dpavlin	4	}
535			}
536			}
537
538			/* === MTS Cache Management ============================================= */
539
540			/* MTS map/unmap/rebuild "API" functions */
541			void MTS_PROTO(api_map)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint64_t paddr,
542			m_uint32_t len,int cache_access,int tlb_index)
543			{
544			/* nothing to do, the cache will be filled on-the-fly */
545			}
546
547			void MTS_PROTO(api_unmap)(cpu_mips_t *cpu,m_uint64_t vaddr,m_uint32_t len,
548			m_uint32_t val,int tlb_index)
549			{
550			/* Invalidate the TLB entry or the full cache if no index is specified */
551			if (tlb_index != -1)
552			MTS_PROTO(invalidate_tlb_entry)(cpu,tlb_index);
553			else
554			MTS_PROTO(invalidate_cache)(cpu);
555			}
556
557	dpavlin	7	void MTS_PROTO(api_rebuild)(cpu_gen_t *cpu)
558	dpavlin	4	{
559	dpavlin	7	MTS_PROTO(invalidate_cache)(CPU_MIPS64(cpu));
560	dpavlin	4	}
561
562			/* ======================================================================== */
563
564			/* Initialize memory access vectors */
565			void MTS_PROTO(init_memop_vectors)(cpu_mips_t *cpu)
566			{
567			/* XXX TODO:
568			* - LD/SD forbidden in Supervisor/User modes with 32-bit addresses.
569			*/
570
571			cpu->addr_mode = MTS_ADDR_SIZE;
572
573			/* API vectors */
574			cpu->mts_map = MTS_PROTO(api_map);
575			cpu->mts_unmap = MTS_PROTO(api_unmap);
576
577	dpavlin	7	/* Memory lookup operation */
578	dpavlin	4	cpu->mem_op_lookup = MTS_PROTO(lookup);
579
580			/* Translation operation */
581			cpu->translate = MTS_PROTO(translate);
582
583			/* Shutdown operation */
584			cpu->mts_shutdown = MTS_PROTO(shutdown);
585
586	dpavlin	7	/* Rebuild MTS data structures */
587			cpu->gen->mts_rebuild = MTS_PROTO(api_rebuild);
588
589	dpavlin	4	/* Show statistics */
590	dpavlin	7	cpu->gen->mts_show_stats = MTS_PROTO(show_stats);
591	dpavlin	4
592			/* Load Operations */
593			cpu->mem_op_fn[MIPS_MEMOP_LB] = MTS_PROTO(lb);
594			cpu->mem_op_fn[MIPS_MEMOP_LBU] = MTS_PROTO(lbu);
595			cpu->mem_op_fn[MIPS_MEMOP_LH] = MTS_PROTO(lh);
596			cpu->mem_op_fn[MIPS_MEMOP_LHU] = MTS_PROTO(lhu);
597			cpu->mem_op_fn[MIPS_MEMOP_LW] = MTS_PROTO(lw);
598			cpu->mem_op_fn[MIPS_MEMOP_LWU] = MTS_PROTO(lwu);
599			cpu->mem_op_fn[MIPS_MEMOP_LD] = MTS_PROTO(ld);
600			cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
601			cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);
602
603			/* Store Operations */
604			cpu->mem_op_fn[MIPS_MEMOP_SB] = MTS_PROTO(sb);
605			cpu->mem_op_fn[MIPS_MEMOP_SH] = MTS_PROTO(sh);
606			cpu->mem_op_fn[MIPS_MEMOP_SW] = MTS_PROTO(sw);
607			cpu->mem_op_fn[MIPS_MEMOP_SD] = MTS_PROTO(sd);
608
609			/* Load Left/Right operations */
610			cpu->mem_op_fn[MIPS_MEMOP_LWL] = MTS_PROTO(lwl);
611			cpu->mem_op_fn[MIPS_MEMOP_LWR] = MTS_PROTO(lwr);
612			cpu->mem_op_fn[MIPS_MEMOP_LDL] = MTS_PROTO(ldl);
613			cpu->mem_op_fn[MIPS_MEMOP_LDR] = MTS_PROTO(ldr);
614
615			/* Store Left/Right operations */
616			cpu->mem_op_fn[MIPS_MEMOP_SWL] = MTS_PROTO(swl);
617			cpu->mem_op_fn[MIPS_MEMOP_SWR] = MTS_PROTO(swr);
618			cpu->mem_op_fn[MIPS_MEMOP_SDL] = MTS_PROTO(sdl);
619			cpu->mem_op_fn[MIPS_MEMOP_SDR] = MTS_PROTO(sdr);
620
621			/* LL/SC - Load Linked / Store Conditional */
622			cpu->mem_op_fn[MIPS_MEMOP_LL] = MTS_PROTO(ll);
623			cpu->mem_op_fn[MIPS_MEMOP_SC] = MTS_PROTO(sc);
624
625			/* Coprocessor 1 memory access functions */
626			cpu->mem_op_fn[MIPS_MEMOP_LDC1] = MTS_PROTO(ldc1);
627			cpu->mem_op_fn[MIPS_MEMOP_SDC1] = MTS_PROTO(sdc1);
628
629			/* Cache Operation */
630			cpu->mem_op_fn[MIPS_MEMOP_CACHE] = MTS_PROTO(cache);
631			}
632
633			#undef MTS_ADDR_SIZE
634	dpavlin	7	#undef MTS_NAME
635			#undef MTS_NAME_UP
636	dpavlin	4	#undef MTS_PROTO
637			#undef MTS_PROTO_UP
638			#undef MTS_ENTRY
639			#undef MTS_CHUNK