Commit 3cf731d6 authored by Lorenzo "Palinuro" Faletra's avatar Lorenzo "Palinuro" Faletra
Browse files

Import Upstream version 4.16.16

parent f34edc87
......@@ -919,6 +919,8 @@ void __init setup_arch(char **cmdline_p)
#ifdef CONFIG_PPC64
if (!radix_enabled())
init_mm.context.slb_addr_limit = DEFAULT_MAP_WINDOW_USER64;
#elif defined(CONFIG_PPC_8xx)
init_mm.context.slb_addr_limit = DEFAULT_MAP_WINDOW;
#else
#error "context.addr_limit not initialized."
#endif
......
......@@ -208,6 +208,12 @@ static void oops_end(unsigned long flags, struct pt_regs *regs,
}
raw_local_irq_restore(flags);
/*
* system_reset_excption handles debugger, crash dump, panic, for 0x100
*/
if (TRAP(regs) == 0x100)
return;
crash_fadump(regs, "die oops");
if (kexec_should_crash(current))
......@@ -272,8 +278,13 @@ void die(const char *str, struct pt_regs *regs, long err)
{
unsigned long flags;
if (debugger(regs))
return;
/*
* system_reset_excption handles debugger, crash dump, panic, for 0x100
*/
if (TRAP(regs) != 0x100) {
if (debugger(regs))
return;
}
flags = oops_begin(regs);
if (__die(str, regs, err))
......@@ -1612,6 +1623,22 @@ void facility_unavailable_exception(struct pt_regs *regs)
value = mfspr(SPRN_FSCR);
status = value >> 56;
if ((hv || status >= 2) &&
(status < ARRAY_SIZE(facility_strings)) &&
facility_strings[status])
facility = facility_strings[status];
/* We should not have taken this interrupt in kernel */
if (!user_mode(regs)) {
pr_emerg("Facility '%s' unavailable (%d) exception in kernel mode at %lx\n",
facility, status, regs->nip);
die("Unexpected facility unavailable exception", regs, SIGABRT);
}
/* We restore the interrupt state now */
if (!arch_irq_disabled_regs(regs))
local_irq_enable();
if (status == FSCR_DSCR_LG) {
/*
* User is accessing the DSCR register using the problem
......@@ -1678,25 +1705,11 @@ void facility_unavailable_exception(struct pt_regs *regs)
return;
}
if ((hv || status >= 2) &&
(status < ARRAY_SIZE(facility_strings)) &&
facility_strings[status])
facility = facility_strings[status];
/* We restore the interrupt state now */
if (!arch_irq_disabled_regs(regs))
local_irq_enable();
pr_err_ratelimited("%sFacility '%s' unavailable (%d), exception at 0x%lx, MSR=%lx\n",
hv ? "Hypervisor " : "", facility, status, regs->nip, regs->msr);
out:
if (user_mode(regs)) {
_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
return;
}
die("Unexpected facility unavailable exception", regs, SIGABRT);
_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
}
#endif
......
......@@ -192,7 +192,7 @@ void set_context(unsigned long id, pgd_t *pgd)
mtspr(SPRN_M_TW, __pa(pgd) - offset);
/* Update context */
mtspr(SPRN_M_CASID, id);
mtspr(SPRN_M_CASID, id - 1);
/* sync */
mb();
}
......
......@@ -553,9 +553,11 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
struct hstate *hstate = hstate_file(file);
int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
#ifdef CONFIG_PPC_RADIX_MMU
if (radix_enabled())
return radix__hugetlb_get_unmapped_area(file, addr, len,
pgoff, flags);
#endif
return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
}
#endif
......
......@@ -331,6 +331,20 @@ int init_new_context(struct task_struct *t, struct mm_struct *mm)
{
pr_hard("initing context for mm @%p\n", mm);
#ifdef CONFIG_PPC_MM_SLICES
if (!mm->context.slb_addr_limit)
mm->context.slb_addr_limit = DEFAULT_MAP_WINDOW;
/*
* We have MMU_NO_CONTEXT set to be ~0. Hence check
* explicitly against context.id == 0. This ensures that we properly
* initialize context slice details for newly allocated mm's (which will
* have id == 0) and don't alter context slice inherited via fork (which
* will have id != 0).
*/
if (mm->context.id == 0)
slice_set_user_psize(mm, mmu_virtual_psize);
#endif
mm->context.id = MMU_NO_CONTEXT;
mm->context.active = 0;
return 0;
......@@ -428,8 +442,8 @@ void __init mmu_context_init(void)
* -- BenH
*/
if (mmu_has_feature(MMU_FTR_TYPE_8xx)) {
first_context = 0;
last_context = 15;
first_context = 1;
last_context = 16;
no_selective_tlbil = true;
} else if (mmu_has_feature(MMU_FTR_TYPE_47x)) {
first_context = 1;
......
......@@ -73,10 +73,12 @@ static void slice_range_to_mask(unsigned long start, unsigned long len,
unsigned long end = start + len - 1;
ret->low_slices = 0;
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
if (SLICE_NUM_HIGH)
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
if (start < SLICE_LOW_TOP) {
unsigned long mend = min(end, (SLICE_LOW_TOP - 1));
unsigned long mend = min(end,
(unsigned long)(SLICE_LOW_TOP - 1));
ret->low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1))
- (1u << GET_LOW_SLICE_INDEX(start));
......@@ -113,11 +115,13 @@ static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)
unsigned long start = slice << SLICE_HIGH_SHIFT;
unsigned long end = start + (1ul << SLICE_HIGH_SHIFT);
#ifdef CONFIG_PPC64
/* Hack, so that each addresses is controlled by exactly one
* of the high or low area bitmaps, the first high area starts
* at 4GB, not 0 */
if (start == 0)
start = SLICE_LOW_TOP;
#endif
return !slice_area_is_free(mm, start, end - start);
}
......@@ -128,7 +132,8 @@ static void slice_mask_for_free(struct mm_struct *mm, struct slice_mask *ret,
unsigned long i;
ret->low_slices = 0;
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
if (SLICE_NUM_HIGH)
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
for (i = 0; i < SLICE_NUM_LOW; i++)
if (!slice_low_has_vma(mm, i))
......@@ -151,7 +156,8 @@ static void slice_mask_for_size(struct mm_struct *mm, int psize, struct slice_ma
u64 lpsizes;
ret->low_slices = 0;
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
if (SLICE_NUM_HIGH)
bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
lpsizes = mm->context.low_slices_psize;
for (i = 0; i < SLICE_NUM_LOW; i++)
......@@ -180,6 +186,10 @@ static int slice_check_fit(struct mm_struct *mm,
*/
unsigned long slice_count = GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit);
if (!SLICE_NUM_HIGH)
return (mask.low_slices & available.low_slices) ==
mask.low_slices;
bitmap_and(result, mask.high_slices,
available.high_slices, slice_count);
......@@ -189,6 +199,7 @@ static int slice_check_fit(struct mm_struct *mm,
static void slice_flush_segments(void *parm)
{
#ifdef CONFIG_PPC64
struct mm_struct *mm = parm;
unsigned long flags;
......@@ -200,6 +211,7 @@ static void slice_flush_segments(void *parm)
local_irq_save(flags);
slb_flush_and_rebolt();
local_irq_restore(flags);
#endif
}
static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psize)
......@@ -388,21 +400,21 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
static inline void slice_or_mask(struct slice_mask *dst, struct slice_mask *src)
{
DECLARE_BITMAP(result, SLICE_NUM_HIGH);
dst->low_slices |= src->low_slices;
bitmap_or(result, dst->high_slices, src->high_slices, SLICE_NUM_HIGH);
bitmap_copy(dst->high_slices, result, SLICE_NUM_HIGH);
if (!SLICE_NUM_HIGH)
return;
bitmap_or(dst->high_slices, dst->high_slices, src->high_slices,
SLICE_NUM_HIGH);
}
static inline void slice_andnot_mask(struct slice_mask *dst, struct slice_mask *src)
{
DECLARE_BITMAP(result, SLICE_NUM_HIGH);
dst->low_slices &= ~src->low_slices;
bitmap_andnot(result, dst->high_slices, src->high_slices, SLICE_NUM_HIGH);
bitmap_copy(dst->high_slices, result, SLICE_NUM_HIGH);
if (!SLICE_NUM_HIGH)
return;
bitmap_andnot(dst->high_slices, dst->high_slices, src->high_slices,
SLICE_NUM_HIGH);
}
#ifdef CONFIG_PPC_64K_PAGES
......@@ -450,14 +462,17 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
* init different masks
*/
mask.low_slices = 0;
bitmap_zero(mask.high_slices, SLICE_NUM_HIGH);
/* silence stupid warning */;
potential_mask.low_slices = 0;
bitmap_zero(potential_mask.high_slices, SLICE_NUM_HIGH);
compat_mask.low_slices = 0;
bitmap_zero(compat_mask.high_slices, SLICE_NUM_HIGH);
if (SLICE_NUM_HIGH) {
bitmap_zero(mask.high_slices, SLICE_NUM_HIGH);
bitmap_zero(potential_mask.high_slices, SLICE_NUM_HIGH);
bitmap_zero(compat_mask.high_slices, SLICE_NUM_HIGH);
}
/* Sanity checks */
BUG_ON(mm->task_size == 0);
......@@ -595,7 +610,9 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
convert:
slice_andnot_mask(&mask, &good_mask);
slice_andnot_mask(&mask, &compat_mask);
if (mask.low_slices || !bitmap_empty(mask.high_slices, SLICE_NUM_HIGH)) {
if (mask.low_slices ||
(SLICE_NUM_HIGH &&
!bitmap_empty(mask.high_slices, SLICE_NUM_HIGH))) {
slice_convert(mm, mask, psize);
if (psize > MMU_PAGE_BASE)
on_each_cpu(slice_flush_segments, mm, 1);
......
......@@ -457,6 +457,16 @@ static void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw)
/* invalid entry */
continue;
/*
* BHRB rolling buffer could very much contain the kernel
* addresses at this point. Check the privileges before
* exporting it to userspace (avoid exposure of regions
* where we could have speculative execution)
*/
if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN) &&
is_kernel_addr(addr))
continue;
/* Branches are read most recent first (ie. mfbhrb 0 is
* the most recent branch).
* There are two types of valid entries:
......@@ -1226,6 +1236,7 @@ static void power_pmu_disable(struct pmu *pmu)
*/
write_mmcr0(cpuhw, val);
mb();
isync();
/*
* Disable instruction sampling if it was enabled
......@@ -1234,12 +1245,26 @@ static void power_pmu_disable(struct pmu *pmu)
mtspr(SPRN_MMCRA,
cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
mb();
isync();
}
cpuhw->disabled = 1;
cpuhw->n_added = 0;
ebb_switch_out(mmcr0);
#ifdef CONFIG_PPC64
/*
* These are readable by userspace, may contain kernel
* addresses and are not switched by context switch, so clear
* them now to avoid leaking anything to userspace in general
* including to another process.
*/
if (ppmu->flags & PPMU_ARCH_207S) {
mtspr(SPRN_SDAR, 0);
mtspr(SPRN_SIAR, 0);
}
#endif
}
local_irq_restore(flags);
......
......@@ -326,6 +326,7 @@ config PPC_BOOK3E_MMU
config PPC_MM_SLICES
bool
default y if PPC_BOOK3S_64
default y if PPC_8xx && HUGETLB_PAGE
default n
config PPC_HAVE_PMU_SUPPORT
......
......@@ -417,6 +417,11 @@ struct npu_context {
void *priv;
};
struct mmio_atsd_reg {
struct npu *npu;
int reg;
};
/*
* Find a free MMIO ATSD register and mark it in use. Return -ENOSPC
* if none are available.
......@@ -426,7 +431,7 @@ static int get_mmio_atsd_reg(struct npu *npu)
int i;
for (i = 0; i < npu->mmio_atsd_count; i++) {
if (!test_and_set_bit(i, &npu->mmio_atsd_usage))
if (!test_and_set_bit_lock(i, &npu->mmio_atsd_usage))
return i;
}
......@@ -435,86 +440,90 @@ static int get_mmio_atsd_reg(struct npu *npu)
static void put_mmio_atsd_reg(struct npu *npu, int reg)
{
clear_bit(reg, &npu->mmio_atsd_usage);
clear_bit_unlock(reg, &npu->mmio_atsd_usage);
}
/* MMIO ATSD register offsets */
#define XTS_ATSD_AVA 1
#define XTS_ATSD_STAT 2
static int mmio_launch_invalidate(struct npu *npu, unsigned long launch,
unsigned long va)
static void mmio_launch_invalidate(struct mmio_atsd_reg *mmio_atsd_reg,
unsigned long launch, unsigned long va)
{
int mmio_atsd_reg;
do {
mmio_atsd_reg = get_mmio_atsd_reg(npu);
cpu_relax();
} while (mmio_atsd_reg < 0);
struct npu *npu = mmio_atsd_reg->npu;
int reg = mmio_atsd_reg->reg;
__raw_writeq(cpu_to_be64(va),
npu->mmio_atsd_regs[mmio_atsd_reg] + XTS_ATSD_AVA);
npu->mmio_atsd_regs[reg] + XTS_ATSD_AVA);
eieio();
__raw_writeq(cpu_to_be64(launch), npu->mmio_atsd_regs[mmio_atsd_reg]);
return mmio_atsd_reg;
__raw_writeq(cpu_to_be64(launch), npu->mmio_atsd_regs[reg]);
}
static int mmio_invalidate_pid(struct npu *npu, unsigned long pid, bool flush)
static void mmio_invalidate_pid(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS],
unsigned long pid, bool flush)
{
int i;
unsigned long launch;
/* IS set to invalidate matching PID */
launch = PPC_BIT(12);
for (i = 0; i <= max_npu2_index; i++) {
if (mmio_atsd_reg[i].reg < 0)
continue;
/* IS set to invalidate matching PID */
launch = PPC_BIT(12);
/* PRS set to process-scoped */
launch |= PPC_BIT(13);
/* PRS set to process-scoped */
launch |= PPC_BIT(13);
/* AP */
launch |= (u64) mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17);
/* AP */
launch |= (u64)
mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17);
/* PID */
launch |= pid << PPC_BITLSHIFT(38);
/* PID */
launch |= pid << PPC_BITLSHIFT(38);
/* No flush */
launch |= !flush << PPC_BITLSHIFT(39);
/* No flush */
launch |= !flush << PPC_BITLSHIFT(39);
/* Invalidating the entire process doesn't use a va */
return mmio_launch_invalidate(npu, launch, 0);
/* Invalidating the entire process doesn't use a va */
mmio_launch_invalidate(&mmio_atsd_reg[i], launch, 0);
}
}
static int mmio_invalidate_va(struct npu *npu, unsigned long va,
unsigned long pid, bool flush)
static void mmio_invalidate_va(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS],
unsigned long va, unsigned long pid, bool flush)
{
int i;
unsigned long launch;
/* IS set to invalidate target VA */
launch = 0;
for (i = 0; i <= max_npu2_index; i++) {
if (mmio_atsd_reg[i].reg < 0)
continue;
/* IS set to invalidate target VA */
launch = 0;
/* PRS set to process scoped */
launch |= PPC_BIT(13);
/* PRS set to process scoped */
launch |= PPC_BIT(13);
/* AP */
launch |= (u64) mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17);
/* AP */
launch |= (u64)
mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17);
/* PID */
launch |= pid << PPC_BITLSHIFT(38);
/* PID */
launch |= pid << PPC_BITLSHIFT(38);
/* No flush */
launch |= !flush << PPC_BITLSHIFT(39);
/* No flush */
launch |= !flush << PPC_BITLSHIFT(39);
return mmio_launch_invalidate(npu, launch, va);
mmio_launch_invalidate(&mmio_atsd_reg[i], launch, va);
}
}
#define mn_to_npu_context(x) container_of(x, struct npu_context, mn)
struct mmio_atsd_reg {
struct npu *npu;
int reg;
};
static void mmio_invalidate_wait(
struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS], bool flush)
struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS])
{
struct npu *npu;
int i, reg;
......@@ -529,16 +538,67 @@ static void mmio_invalidate_wait(
reg = mmio_atsd_reg[i].reg;
while (__raw_readq(npu->mmio_atsd_regs[reg] + XTS_ATSD_STAT))
cpu_relax();
}
}
/*
* Acquires all the address translation shootdown (ATSD) registers required to
* launch an ATSD on all links this npu_context is active on.
*/
static void acquire_atsd_reg(struct npu_context *npu_context,
struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS])
{
int i, j;
struct npu *npu;
struct pci_dev *npdev;
struct pnv_phb *nphb;
put_mmio_atsd_reg(npu, reg);
for (i = 0; i <= max_npu2_index; i++) {
mmio_atsd_reg[i].reg = -1;
for (j = 0; j < NV_MAX_LINKS; j++) {
/*
* There are no ordering requirements with respect to
* the setup of struct npu_context, but to ensure
* consistent behaviour we need to ensure npdev[][] is
* only read once.
*/
npdev = READ_ONCE(npu_context->npdev[i][j]);
if (!npdev)
continue;
nphb = pci_bus_to_host(npdev->bus)->private_data;
npu = &nphb->npu;
mmio_atsd_reg[i].npu = npu;
mmio_atsd_reg[i].reg = get_mmio_atsd_reg(npu);
while (mmio_atsd_reg[i].reg < 0) {
mmio_atsd_reg[i].reg = get_mmio_atsd_reg(npu);
cpu_relax();
}
break;
}
}
}
/*
* Release previously acquired ATSD registers. To avoid deadlocks the registers
* must be released in the same order they were acquired above in
* acquire_atsd_reg.
*/
static void release_atsd_reg(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS])
{
int i;
for (i = 0; i <= max_npu2_index; i++) {
/*
* The GPU requires two flush ATSDs to ensure all entries have
* been flushed. We use PID 0 as it will never be used for a
* process on the GPU.
* We can't rely on npu_context->npdev[][] being the same here
* as when acquire_atsd_reg() was called, hence we use the
* values stored in mmio_atsd_reg during the acquire phase
* rather than re-reading npdev[][].
*/
if (flush)
mmio_invalidate_pid(npu, 0, true);
if (mmio_atsd_reg[i].reg < 0)
continue;
put_mmio_atsd_reg(mmio_atsd_reg[i].npu, mmio_atsd_reg[i].reg);
}
}
......@@ -549,10 +609,6 @@ static void mmio_invalidate_wait(
static void mmio_invalidate(struct npu_context *npu_context, int va,
unsigned long address, bool flush)
{
int i, j;
struct npu *npu;
struct pnv_phb *nphb;
struct pci_dev *npdev;
struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS];
unsigned long pid = npu_context->mm->context.id;
......@@ -568,37 +624,25 @@ static void mmio_invalidate(struct npu_context *npu_context, int va,
* Loop over all the NPUs this process is active on and launch
* an invalidate.
*/
for (i = 0; i <= max_npu2_index; i++) {
mmio_atsd_reg[i].reg = -1;
for (j = 0; j < NV_MAX_LINKS; j++) {
npdev = npu_context->npdev[i][j];
if (!npdev)
continue;
nphb = pci_bus_to_host(npdev->bus)->private_data;
npu = &nphb->npu;
mmio_atsd_reg[i].npu = npu;
if (va)
mmio_atsd_reg[i].reg =
mmio_invalidate_va(npu, address, pid,
flush);
else
mmio_atsd_reg[i].reg =
mmio_invalidate_pid(npu, pid, flush);
/*
* The NPU hardware forwards the shootdown to all GPUs
* so we only have to launch one shootdown per NPU.
*/
break;
}
acquire_atsd_reg(npu_context, mmio_atsd_reg);
if (va)
mmio_invalidate_va(mmio_atsd_reg, address, pid, flush);
else
mmio_invalidate_pid(mmio_atsd_reg, pid, flush);