5 |
* Template code for MTS. |
* Template code for MTS. |
6 |
*/ |
*/ |
7 |
|
|
8 |
#define MTS_ENTRY MTS_PROTO(entry_t) |
#define MTS_ENTRY MTS_NAME(entry_t) |
9 |
#define MTS_CHUNK MTS_PROTO(chunk_t) |
#define MTS_CACHE(cpu) ( cpu->mts_u. MTS_NAME(cache) ) |
10 |
|
|
11 |
/* Forward declarations */ |
/* Forward declarations */ |
12 |
static forced_inline void *MTS_PROTO(access)(cpu_mips_t *cpu,m_uint64_t vaddr, |
static forced_inline void *MTS_PROTO(access)(cpu_mips_t *cpu,m_uint64_t vaddr, |
23 |
size_t len; |
size_t len; |
24 |
|
|
25 |
/* Initialize the cache entries to 0 (empty) */ |
/* Initialize the cache entries to 0 (empty) */ |
26 |
len = MTS_PROTO_UP(HASH_SIZE) * sizeof(MTS_ENTRY *); |
len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY); |
27 |
if (!(cpu->mts_cache = malloc(len))) |
if (!(MTS_CACHE(cpu) = malloc(len))) |
28 |
return(-1); |
return(-1); |
29 |
|
|
30 |
memset(cpu->mts_cache,0,len); |
memset(MTS_CACHE(cpu),0xFF,len); |
31 |
cpu->mts_lookups = 0; |
cpu->mts_lookups = 0; |
32 |
cpu->mts_misses = 0; |
cpu->mts_misses = 0; |
|
|
|
|
/* Reset the TLB reverse map (used for selective invalidations) */ |
|
|
memset(cpu->mts_rmap,0,(cpu->cp0.tlb_entries * sizeof(MTS_ENTRY *))); |
|
33 |
return(0); |
return(0); |
34 |
} |
} |
35 |
|
|
36 |
/* Free memory used by MTS */ |
/* Free memory used by MTS */ |
37 |
void MTS_PROTO(shutdown)(cpu_mips_t *cpu) |
void MTS_PROTO(shutdown)(cpu_mips_t *cpu) |
38 |
{ |
{ |
|
MTS_CHUNK *chunk,*next; |
|
|
int i; |
|
|
|
|
|
/* Reset the reverse map */ |
|
|
for(i=0;i<cpu->cp0.tlb_entries;i++) |
|
|
cpu->mts_rmap[i] = NULL; |
|
|
|
|
39 |
/* Free the cache itself */ |
/* Free the cache itself */ |
40 |
free(cpu->mts_cache); |
free(MTS_CACHE(cpu)); |
41 |
cpu->mts_cache = NULL; |
MTS_CACHE(cpu) = NULL; |
|
|
|
|
/* Free the chunks */ |
|
|
for(chunk=cpu->mts_chunk_list;chunk;chunk=next) { |
|
|
next = chunk->next; |
|
|
free(chunk); |
|
|
} |
|
|
|
|
|
for(chunk=cpu->mts_chunk_free_list;chunk;chunk=next) { |
|
|
next = chunk->next; |
|
|
free(chunk); |
|
|
} |
|
|
|
|
|
cpu->mts_chunk_list = cpu->mts_chunk_free_list = NULL; |
|
|
cpu->mts_entry_free_list = NULL; |
|
42 |
} |
} |
43 |
|
|
44 |
/* Show MTS detailed information (debugging only!) */ |
/* Show MTS detailed information (debugging only!) */ |
45 |
void MTS_PROTO(show_stats)(cpu_mips_t *cpu) |
void MTS_PROTO(show_stats)(cpu_gen_t *gen_cpu) |
46 |
{ |
{ |
47 |
MTS_CHUNK *chunk; |
cpu_mips_t *cpu = CPU_MIPS64(gen_cpu); |
48 |
#if DEBUG_MTS_MAP_VIRT |
#if DEBUG_MTS_MAP_VIRT |
49 |
MTS_ENTRY *entry; |
MTS_ENTRY *entry; |
50 |
u_int i; |
u_int i,count; |
51 |
#endif |
#endif |
|
u_int count; |
|
52 |
|
|
53 |
printf("\nCPU%u: MTS%d statistics:\n",cpu->id,MTS_ADDR_SIZE); |
printf("\nCPU%u: MTS%d statistics:\n",cpu->gen->id,MTS_ADDR_SIZE); |
|
|
|
|
printf(" Total lookups: %llu, misses: %llu, efficiency: %g%%\n", |
|
|
cpu->mts_lookups, cpu->mts_misses, |
|
|
100 - ((double)(cpu->mts_misses*100)/ |
|
|
(double)cpu->mts_lookups)); |
|
54 |
|
|
55 |
#if DEBUG_MTS_MAP_VIRT |
#if DEBUG_MTS_MAP_VIRT |
56 |
/* Valid hash entries */ |
/* Valid hash entries */ |
57 |
for(count=0,i=0;i<MTS_PROTO_UP(HASH_SIZE);i++) { |
for(count=0,i=0;i<MTS_NAME_UP(HASH_SIZE);i++) { |
58 |
if ((entry = cpu->mts_cache[i]) != NULL) { |
entry = &(MTS_CACHE(cpu)[i]); |
59 |
printf(" %4u: entry=%p, start=0x%16.16llx, " |
|
60 |
"len=0x%8.8x, action=0x%8.8llx\n", |
if (!(entry->gvpa & MTS_INV_ENTRY_MASK)) { |
61 |
i,entry,(m_uint64_t)entry->start,entry->mask, |
printf(" %4u: vaddr=0x%8.8llx, paddr=0x%8.8llx, hpa=%p\n", |
62 |
(m_uint64_t)entry->action); |
i,(m_uint64_t)entry->gvpa,(m_uint64_t)entry->gppa, |
63 |
|
(void *)entry->hpa); |
64 |
count++; |
count++; |
65 |
} |
} |
66 |
} |
} |
67 |
|
|
68 |
printf(" %u/%u valid hash entries.\n",count,MTS_PROTO_UP(HASH_SIZE)); |
printf(" %u/%u valid hash entries.\n",count,MTS_NAME_UP(HASH_SIZE)); |
|
#endif |
|
|
|
|
|
/* Number of chunks */ |
|
|
for(count=0,chunk=cpu->mts_chunk_list;chunk;chunk=chunk->next) |
|
|
count++; |
|
|
|
|
|
printf(" Number of chunks: %u\n",count); |
|
|
|
|
|
#if DEBUG_MTS_MAP_VIRT |
|
|
/* Reverse map */ |
|
|
for(i=0;i<MIPS64_TLB_MAX_ENTRIES;i++) { |
|
|
for(count=0,entry=cpu->mts_rmap[i];entry;entry=entry->next) |
|
|
count++; |
|
|
|
|
|
if (count > 0) |
|
|
printf(" tlb_rmap[%u]: %u entries\n",i,count); |
|
|
} |
|
69 |
#endif |
#endif |
|
} |
|
70 |
|
|
71 |
/* Allocate a new chunk */ |
printf(" Total lookups: %llu, misses: %llu, efficiency: %g%%\n", |
72 |
static int MTS_PROTO(alloc_chunk)(cpu_mips_t *cpu) |
cpu->mts_lookups, cpu->mts_misses, |
73 |
{ |
100 - ((double)(cpu->mts_misses*100)/ |
74 |
MTS_CHUNK *chunk; |
(double)cpu->mts_lookups)); |
|
|
|
|
/* Try the free list first, then use standard allocation procedure */ |
|
|
if ((chunk = cpu->mts_chunk_free_list) != NULL) { |
|
|
cpu->mts_chunk_free_list = chunk->next; |
|
|
} else { |
|
|
if (!(chunk = malloc(sizeof(*chunk)))) |
|
|
return(-1); |
|
|
} |
|
|
|
|
|
chunk->count = 0; |
|
|
chunk->next = cpu->mts_chunk_list; |
|
|
cpu->mts_chunk_list = chunk; |
|
|
return(0); |
|
|
} |
|
|
|
|
|
/* Allocate a new entry */ |
|
|
static MTS_ENTRY *MTS_PROTO(alloc_entry)(cpu_mips_t *cpu) |
|
|
{ |
|
|
MTS_CHUNK *chunk = cpu->mts_chunk_list; |
|
|
MTS_ENTRY *entry; |
|
|
|
|
|
/* First, try to allocate the entry from the free list */ |
|
|
if ((entry = cpu->mts_entry_free_list) != NULL) { |
|
|
cpu->mts_entry_free_list = ((MTS_ENTRY *)cpu->mts_entry_free_list)->next; |
|
|
return entry; |
|
|
} |
|
|
|
|
|
/* A new chunk is required */ |
|
|
if (!chunk || (chunk->count == MTS_PROTO_UP(CHUNK_SIZE))) { |
|
|
if (MTS_PROTO(alloc_chunk)(cpu) == -1) |
|
|
return NULL; |
|
|
|
|
|
chunk = cpu->mts_chunk_list; |
|
|
} |
|
|
|
|
|
entry = &chunk->entry[chunk->count]; |
|
|
chunk->count++; |
|
|
return entry; |
|
75 |
} |
} |
76 |
|
|
77 |
/* Invalidate the complete MTS cache */ |
/* Invalidate the complete MTS cache */ |
78 |
void MTS_PROTO(invalidate_cache)(cpu_mips_t *cpu) |
void MTS_PROTO(invalidate_cache)(cpu_mips_t *cpu) |
79 |
{ |
{ |
|
MTS_CHUNK *chunk; |
|
80 |
size_t len; |
size_t len; |
|
u_int i; |
|
|
|
|
|
len = MTS_PROTO_UP(HASH_SIZE) * sizeof(MTS_ENTRY *); |
|
|
memset(cpu->mts_cache,0,len); |
|
|
|
|
|
/* Move all chunks to the free list */ |
|
|
while((chunk = cpu->mts_chunk_list) != NULL) { |
|
|
cpu->mts_chunk_list = chunk->next; |
|
|
chunk->next = cpu->mts_chunk_free_list; |
|
|
cpu->mts_chunk_free_list = chunk; |
|
|
} |
|
81 |
|
|
82 |
/* Reset the free list of entries (since they are located in chunks) */ |
len = MTS_NAME_UP(HASH_SIZE) * sizeof(MTS_ENTRY); |
83 |
cpu->mts_entry_free_list = NULL; |
memset(MTS_CACHE(cpu),0xFF,len); |
|
|
|
|
/* Reset the reverse map */ |
|
|
for(i=0;i<cpu->cp0.tlb_entries;i++) |
|
|
cpu->mts_rmap[i] = NULL; |
|
84 |
} |
} |
85 |
|
|
86 |
/* Invalidate partially the MTS cache, given a TLB entry index */ |
/* Invalidate partially the MTS cache, given a TLB entry index */ |
87 |
void MTS_PROTO(invalidate_tlb_entry)(cpu_mips_t *cpu,u_int tlb_index) |
void MTS_PROTO(invalidate_tlb_entry)(cpu_mips_t *cpu,u_int tlb_index) |
88 |
{ |
{ |
89 |
MTS_ENTRY *entry; |
MTS_PROTO(invalidate_cache)(cpu); |
|
|
|
|
for(entry=cpu->mts_rmap[tlb_index];entry;entry=entry->next) { |
|
|
*(entry->pself) = NULL; |
|
|
if (!entry->next) { |
|
|
entry->next = cpu->mts_entry_free_list; |
|
|
break; |
|
|
} |
|
|
} |
|
|
|
|
|
cpu->mts_entry_free_list = cpu->mts_rmap[tlb_index]; |
|
|
cpu->mts_rmap[tlb_index] = NULL; |
|
90 |
} |
} |
91 |
|
|
92 |
/* |
/* |
94 |
* |
* |
95 |
* It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5) |
* It is NOT inlined since it triggers a GCC bug on my config (x86, GCC 3.3.5) |
96 |
*/ |
*/ |
97 |
static no_inline int MTS_PROTO(map)(cpu_mips_t *cpu,m_uint64_t vaddr, |
static no_inline MTS_ENTRY * |
98 |
mts_map_t *map,MTS_ENTRY *entry) |
MTS_PROTO(map)(cpu_mips_t *cpu,u_int op_type,mts_map_t *map, |
99 |
|
MTS_ENTRY *entry,MTS_ENTRY *alt_entry) |
100 |
{ |
{ |
101 |
struct vdevice *dev; |
struct vdevice *dev; |
102 |
m_uint64_t lk_addr; |
m_uint32_t offset; |
103 |
m_uint32_t poffset; |
m_iptr_t host_ptr; |
104 |
|
int cow; |
105 |
lk_addr = map->paddr + (vaddr - map->vaddr); |
|
106 |
|
if (!(dev = dev_lookup(cpu->vm,map->paddr,map->cached))) |
107 |
|
return NULL; |
108 |
|
|
109 |
|
if (dev->flags & VDEVICE_FLAG_SPARSE) { |
110 |
|
host_ptr = dev_sparse_get_host_addr(cpu->vm,dev,map->paddr,op_type,&cow); |
111 |
|
|
112 |
|
entry->gvpa = map->vaddr; |
113 |
|
entry->gppa = map->paddr; |
114 |
|
entry->hpa = host_ptr; |
115 |
|
entry->flags = (cow) ? MTS_FLAG_COW : 0; |
116 |
|
return entry; |
117 |
|
} |
118 |
|
|
119 |
if (!(dev = dev_lookup(cpu->vm,lk_addr,map->cached))) |
if (!dev->host_addr || (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) { |
120 |
return(FALSE); |
offset = map->paddr - dev->phys_addr; |
121 |
|
|
122 |
if (map->paddr > dev->phys_addr) { |
alt_entry->gvpa = map->vaddr; |
123 |
poffset = map->paddr - dev->phys_addr; |
alt_entry->gppa = map->paddr; |
124 |
entry->start = map->vaddr; |
alt_entry->hpa = (dev->id << MTS_DEVID_SHIFT) + offset; |
125 |
entry->phys_page = map->paddr >> MIPS_MIN_PAGE_SHIFT; |
alt_entry->flags = MTS_FLAG_DEV; |
126 |
entry->mask = ~((m_min(map->len,dev->phys_len - poffset)) - 1); |
return alt_entry; |
|
entry->action = poffset; |
|
|
} else { |
|
|
poffset = dev->phys_addr - map->paddr; |
|
|
entry->start = map->vaddr + poffset; |
|
|
entry->phys_page = (map->paddr + poffset) >> MIPS_MIN_PAGE_SHIFT; |
|
|
entry->mask = ~((m_min(map->len - poffset,dev->phys_len)) - 1); |
|
|
entry->action = 0; |
|
127 |
} |
} |
128 |
|
|
129 |
if (!dev->host_addr || (dev->flags & VDEVICE_FLAG_NO_MTS_MMAP)) |
entry->gvpa = map->vaddr; |
130 |
entry->action += (dev->id << MTS_DEVID_SHIFT) | MTS_DEV_MASK; |
entry->gppa = map->paddr; |
131 |
else |
entry->hpa = dev->host_addr + (map->paddr - dev->phys_addr); |
132 |
entry->action += dev->host_addr; |
entry->flags = 0; |
133 |
|
|
134 |
return(TRUE); |
return entry; |
135 |
} |
} |
136 |
|
|
137 |
/* MTS lookup */ |
/* MTS lookup */ |
570 |
/* CACHE: Cache operation */ |
/* CACHE: Cache operation */ |
571 |
fastcall u_int MTS_PROTO(cache)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op) |
fastcall u_int MTS_PROTO(cache)(cpu_mips_t *cpu,m_uint64_t vaddr,u_int op) |
572 |
{ |
{ |
573 |
struct insn_block *block; |
mips64_jit_tcb_t *block; |
574 |
m_uint32_t phys_page; |
m_uint32_t phys_page; |
575 |
|
|
576 |
#if DEBUG_CACHE |
#if DEBUG_CACHE |
577 |
cpu_log(cpu,"MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n", |
cpu_log(cpu->gen, |
578 |
|
"MTS","CACHE: PC=0x%llx, vaddr=0x%llx, cache=%u, code=%u\n", |
579 |
cpu->pc, vaddr, op & 0x3, op >> 2); |
cpu->pc, vaddr, op & 0x3, op >> 2); |
580 |
#endif |
#endif |
581 |
|
|
588 |
((cpu->pc - block->start_pc) >= MIPS_MIN_PAGE_SIZE)) |
((cpu->pc - block->start_pc) >= MIPS_MIN_PAGE_SIZE)) |
589 |
{ |
{ |
590 |
#if DEBUG_CACHE |
#if DEBUG_CACHE |
591 |
cpu_log(cpu,"MTS", |
cpu_log(cpu->gen,"MTS", |
592 |
"CACHE: removing compiled page at 0x%llx, pc=0x%llx\n", |
"CACHE: removing compiled page at 0x%llx, pc=0x%llx\n", |
593 |
block->start_pc,cpu->pc); |
block->start_pc,cpu->pc); |
594 |
#endif |
#endif |
595 |
cpu->exec_phys_map[phys_page] = NULL; |
cpu->exec_phys_map[phys_page] = NULL; |
596 |
insn_block_free(cpu,block,TRUE); |
mips64_jit_tcb_free(cpu,block,TRUE); |
597 |
} |
} |
598 |
else |
else |
599 |
{ |
{ |
600 |
#if DEBUG_CACHE |
#if DEBUG_CACHE |
601 |
cpu_log(cpu,"MTS", |
cpu_log(cpu->gen,"MTS", |
602 |
"CACHE: trying to remove page 0x%llx with pc=0x%llx\n", |
"CACHE: trying to remove page 0x%llx with pc=0x%llx\n", |
603 |
block->start_pc,cpu->pc); |
block->start_pc,cpu->pc); |
604 |
#endif |
#endif |
629 |
MTS_PROTO(invalidate_cache)(cpu); |
MTS_PROTO(invalidate_cache)(cpu); |
630 |
} |
} |
631 |
|
|
632 |
void MTS_PROTO(api_rebuild)(cpu_mips_t *cpu) |
void MTS_PROTO(api_rebuild)(cpu_gen_t *cpu) |
633 |
{ |
{ |
634 |
MTS_PROTO(invalidate_cache)(cpu); |
MTS_PROTO(invalidate_cache)(CPU_MIPS64(cpu)); |
635 |
} |
} |
636 |
|
|
637 |
/* ======================================================================== */ |
/* ======================================================================== */ |
648 |
/* API vectors */ |
/* API vectors */ |
649 |
cpu->mts_map = MTS_PROTO(api_map); |
cpu->mts_map = MTS_PROTO(api_map); |
650 |
cpu->mts_unmap = MTS_PROTO(api_unmap); |
cpu->mts_unmap = MTS_PROTO(api_unmap); |
|
cpu->mts_rebuild = MTS_PROTO(api_rebuild); |
|
651 |
|
|
652 |
/* memory lookup operation */ |
/* Memory lookup operation */ |
653 |
cpu->mem_op_lookup = MTS_PROTO(lookup); |
cpu->mem_op_lookup = MTS_PROTO(lookup); |
654 |
|
|
655 |
/* Translation operation */ |
/* Translation operation */ |
658 |
/* Shutdown operation */ |
/* Shutdown operation */ |
659 |
cpu->mts_shutdown = MTS_PROTO(shutdown); |
cpu->mts_shutdown = MTS_PROTO(shutdown); |
660 |
|
|
661 |
|
/* Rebuild MTS data structures */ |
662 |
|
cpu->gen->mts_rebuild = MTS_PROTO(api_rebuild); |
663 |
|
|
664 |
/* Show statistics */ |
/* Show statistics */ |
665 |
cpu->mts_show_stats = MTS_PROTO(show_stats); |
cpu->gen->mts_show_stats = MTS_PROTO(show_stats); |
666 |
|
|
667 |
/* Load Operations */ |
/* Load Operations */ |
668 |
cpu->mem_op_fn[MIPS_MEMOP_LB] = MTS_PROTO(lb); |
cpu->mem_op_fn[MIPS_MEMOP_LB] = MTS_PROTO(lb); |
706 |
} |
} |
707 |
|
|
708 |
#undef MTS_ADDR_SIZE |
#undef MTS_ADDR_SIZE |
709 |
|
#undef MTS_NAME |
710 |
|
#undef MTS_NAME_UP |
711 |
#undef MTS_PROTO |
#undef MTS_PROTO |
712 |
#undef MTS_PROTO_UP |
#undef MTS_PROTO_UP |
713 |
#undef MTS_ENTRY |
#undef MTS_ENTRY |