1
linux/arch/powerpc/mm/cacheflush.c

222 lines
5.7 KiB
C
Raw Permalink Normal View History

// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/highmem.h>
#include <linux/kprobes.h>
/**
* flush_coherent_icache() - if a CPU has a coherent icache, flush it
* Return true if the cache was flushed, false otherwise
*/
static inline bool flush_coherent_icache(void)
{
/*
* For a snooping icache, we still need a dummy icbi to purge all the
* prefetched instructions from the ifetch buffers. We also need a sync
* before the icbi to order the actual stores to memory that might
* have modified instructions with the icbi.
*/
if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
mb(); /* sync */
icbi((void *)PAGE_OFFSET);
mb(); /* sync */
isync();
return true;
}
return false;
}
/**
* invalidate_icache_range() - Flush the icache by issuing icbi across an address range
* @start: the start address
* @stop: the stop address (exclusive)
*/
static void invalidate_icache_range(unsigned long start, unsigned long stop)
{
unsigned long shift = l1_icache_shift();
unsigned long bytes = l1_icache_bytes();
char *addr = (char *)(start & ~(bytes - 1));
unsigned long size = stop - (unsigned long)addr + (bytes - 1);
unsigned long i;
for (i = 0; i < size >> shift; i++, addr += bytes)
icbi(addr);
mb(); /* sync */
isync();
}
/**
* flush_icache_range: Write any modified data cache blocks out to memory
* and invalidate the corresponding blocks in the instruction cache
*
* Generic code will call this after writing memory, before executing from it.
*
* @start: the start address
* @stop: the stop address (exclusive)
*/
void flush_icache_range(unsigned long start, unsigned long stop)
{
if (flush_coherent_icache())
return;
clean_dcache_range(start, stop);
if (IS_ENABLED(CONFIG_44x)) {
/*
* Flash invalidate on 44x because we are passed kmapped
* addresses and this doesn't work for userspace pages due to
* the virtually tagged icache.
*/
iccci((void *)start);
mb(); /* sync */
isync();
} else
invalidate_icache_range(start, stop);
}
EXPORT_SYMBOL(flush_icache_range);
#ifdef CONFIG_HIGHMEM
/**
* flush_dcache_icache_phys() - Flush a page by its physical address
* @physaddr: the physical address of the page
*/
static void flush_dcache_icache_phys(unsigned long physaddr)
{
unsigned long bytes = l1_dcache_bytes();
unsigned long nb = PAGE_SIZE / bytes;
unsigned long addr = physaddr & PAGE_MASK;
unsigned long msr, msr0;
unsigned long loop1 = addr, loop2 = addr;
msr0 = mfmsr();
msr = msr0 & ~MSR_DR;
/*
* This must remain as ASM to prevent potential memory accesses
* while the data MMU is disabled
*/
asm volatile(
" mtctr %2;\n"
" mtmsr %3;\n"
" isync;\n"
"0: dcbst 0, %0;\n"
" addi %0, %0, %4;\n"
" bdnz 0b;\n"
" sync;\n"
" mtctr %2;\n"
"1: icbi 0, %1;\n"
" addi %1, %1, %4;\n"
" bdnz 1b;\n"
" sync;\n"
" mtmsr %5;\n"
" isync;\n"
: "+&r" (loop1), "+&r" (loop2)
: "r" (nb), "r" (msr), "i" (bytes), "r" (msr0)
: "ctr", "memory");
}
NOKPROBE_SYMBOL(flush_dcache_icache_phys)
#else
static void flush_dcache_icache_phys(unsigned long physaddr)
{
}
#endif
/**
* __flush_dcache_icache(): Flush a particular page from the data cache to RAM.
* Note: this is necessary because the instruction cache does *not*
* snoop from the data cache.
*
* @p: the address of the page to flush
*/
static void __flush_dcache_icache(void *p)
{
powerpc/mem: Help GCC realise __flush_dcache_icache() flushes single pages 'And' the given page address with PAGE_MASK to help GCC. With the patch: 00000024 <__flush_dcache_icache>: 24: 54 63 00 26 rlwinm r3,r3,0,0,19 28: 39 40 00 40 li r10,64 2c: 7c 69 1b 78 mr r9,r3 30: 7d 49 03 a6 mtctr r10 34: 7c 00 48 6c dcbst 0,r9 38: 39 29 00 20 addi r9,r9,32 3c: 7c 00 48 6c dcbst 0,r9 40: 39 29 00 20 addi r9,r9,32 44: 42 00 ff f0 bdnz 34 <__flush_dcache_icache+0x10> 48: 7c 00 04 ac hwsync 4c: 39 20 00 40 li r9,64 50: 7d 29 03 a6 mtctr r9 54: 7c 00 1f ac icbi 0,r3 58: 38 63 00 20 addi r3,r3,32 5c: 7c 00 1f ac icbi 0,r3 60: 38 63 00 20 addi r3,r3,32 64: 42 00 ff f0 bdnz 54 <__flush_dcache_icache+0x30> 68: 7c 00 04 ac hwsync 6c: 4c 00 01 2c isync 70: 4e 80 00 20 blr Without the patch: 00000024 <__flush_dcache_icache>: 24: 54 6a 00 34 rlwinm r10,r3,0,0,26 28: 39 23 10 1f addi r9,r3,4127 2c: 7d 2a 48 50 subf r9,r10,r9 30: 55 29 d9 7f rlwinm. r9,r9,27,5,31 34: 41 82 00 94 beq c8 <__flush_dcache_icache+0xa4> 38: 71 28 00 01 andi. r8,r9,1 3c: 38 c9 ff ff addi r6,r9,-1 40: 7d 48 53 78 mr r8,r10 44: 7d 27 4b 78 mr r7,r9 48: 40 82 00 6c bne b4 <__flush_dcache_icache+0x90> 4c: 54 e7 f8 7e rlwinm r7,r7,31,1,31 50: 7c e9 03 a6 mtctr r7 54: 7c 00 40 6c dcbst 0,r8 58: 39 08 00 20 addi r8,r8,32 5c: 7c 00 40 6c dcbst 0,r8 60: 39 08 00 20 addi r8,r8,32 64: 42 00 ff f0 bdnz 54 <__flush_dcache_icache+0x30> 68: 7c 00 04 ac hwsync 6c: 71 28 00 01 andi. r8,r9,1 70: 39 09 ff ff addi r8,r9,-1 74: 40 82 00 2c bne a0 <__flush_dcache_icache+0x7c> 78: 55 29 f8 7e rlwinm r9,r9,31,1,31 7c: 7d 29 03 a6 mtctr r9 80: 7c 00 57 ac icbi 0,r10 84: 39 4a 00 20 addi r10,r10,32 88: 7c 00 57 ac icbi 0,r10 8c: 39 4a 00 20 addi r10,r10,32 90: 42 00 ff f0 bdnz 80 <__flush_dcache_icache+0x5c> 94: 7c 00 04 ac hwsync 98: 4c 00 01 2c isync 9c: 4e 80 00 20 blr a0: 7c 00 57 ac icbi 0,r10 a4: 2c 08 00 00 cmpwi r8,0 a8: 39 4a 00 20 addi r10,r10,32 ac: 40 82 ff cc bne 78 <__flush_dcache_icache+0x54> b0: 4b ff ff e4 b 94 <__flush_dcache_icache+0x70> b4: 7c 00 50 6c dcbst 0,r10 b8: 2c 06 00 00 cmpwi r6,0 bc: 39 0a 00 20 addi r8,r10,32 c0: 40 82 ff 8c bne 4c <__flush_dcache_icache+0x28> c4: 4b ff ff a4 b 68 <__flush_dcache_icache+0x44> c8: 7c 00 04 ac hwsync cc: 7c 00 04 ac hwsync d0: 4c 00 01 2c isync d4: 4e 80 00 20 blr Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/23030822ea5cd0a122948b10226abe56602dc027.1617895813.git.christophe.leroy@csgroup.eu
2021-04-08 08:30:31 -07:00
unsigned long addr = (unsigned long)p & PAGE_MASK;
clean_dcache_range(addr, addr + PAGE_SIZE);
/*
* We don't flush the icache on 44x. Those have a virtual icache and we
* don't have access to the virtual address here (it's not the page
* vaddr but where it's mapped in user space). The flushing of the
* icache on these is handled elsewhere, when a change in the address
* space occurs, before returning to user space.
*/
if (mmu_has_feature(MMU_FTR_TYPE_44x))
return;
invalidate_icache_range(addr, addr + PAGE_SIZE);
}
void flush_dcache_icache_folio(struct folio *folio)
{
unsigned int i, nr = folio_nr_pages(folio);
if (flush_coherent_icache())
return;
if (!folio_test_highmem(folio)) {
void *addr = folio_address(folio);
for (i = 0; i < nr; i++)
__flush_dcache_icache(addr + i * PAGE_SIZE);
} else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) {
for (i = 0; i < nr; i++) {
void *start = kmap_local_folio(folio, i * PAGE_SIZE);
__flush_dcache_icache(start);
kunmap_local(start);
}
} else {
unsigned long pfn = folio_pfn(folio);
for (i = 0; i < nr; i++)
flush_dcache_icache_phys((pfn + i) * PAGE_SIZE);
}
}
EXPORT_SYMBOL(flush_dcache_icache_folio);
void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
{
clear_page(page);
/*
* We shouldn't have to do this, but some versions of glibc
* require it (ld.so assumes zero filled pages are icache clean)
* - Anton
*/
flush_dcache_page(pg);
}
EXPORT_SYMBOL(clear_user_page);
void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
struct page *pg)
{
copy_page(vto, vfrom);
/*
* We should be able to use the following optimisation, however
* there are two problems.
* Firstly a bug in some versions of binutils meant PLT sections
* were not marked executable.
* Secondly the first word in the GOT section is blrl, used
* to establish the GOT address. Until recently the GOT was
* not marked executable.
* - Anton
*/
#if 0
if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
return;
#endif
flush_dcache_page(pg);
}
void flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
unsigned long addr, int len)
{
void *maddr;
maddr = kmap_local_page(page) + (addr & ~PAGE_MASK);
flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len);
kunmap_local(maddr);
}