From 70f8a3ca68d3e1f3344d959981ca55d5f6ec77f7 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Wed, 6 Feb 2019 09:59:15 -0800 Subject: mm: make mm->pinned_vm an atomic64 counter Taking a sleeping lock to _only_ increment a variable is quite the overkill, and pretty much all users do this. Furthermore, some drivers (ie: infiniband and scif) that need pinned semantics can go to quite some trouble to actually delay via workqueue (un)accounting for pinned pages when not possible to acquire it. By making the counter atomic we no longer need to hold the mmap_sem and can simply some code around it for pinned_vm users. The counter is 64-bit such that we need not worry about overflows such as rdma user input controlled from userspace. Reviewed-by: Ira Weiny Reviewed-by: Christoph Lameter Reviewed-by: Daniel Jordan Reviewed-by: Jan Kara Signed-off-by: Davidlohr Bueso Signed-off-by: Jason Gunthorpe --- drivers/misc/mic/scif/scif_rma.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/misc/mic') diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c index 749321eb91ae..2448368f181e 100644 --- a/drivers/misc/mic/scif/scif_rma.c +++ b/drivers/misc/mic/scif/scif_rma.c @@ -285,7 +285,7 @@ __scif_dec_pinned_vm_lock(struct mm_struct *mm, } else { down_write(&mm->mmap_sem); } - mm->pinned_vm -= nr_pages; + atomic64_sub(nr_pages, &mm->pinned_vm); up_write(&mm->mmap_sem); return 0; } @@ -299,7 +299,7 @@ static inline int __scif_check_inc_pinned_vm(struct mm_struct *mm, return 0; locked = nr_pages; - locked += mm->pinned_vm; + locked += atomic64_read(&mm->pinned_vm); lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) { dev_err(scif_info.mdev.this_device, @@ -307,7 +307,7 @@ static inline int __scif_check_inc_pinned_vm(struct mm_struct *mm, locked, lock_limit); return -ENOMEM; } - mm->pinned_vm = locked; + atomic64_set(&mm->pinned_vm, locked); return 0; } -- cgit v1.2.3 From 4f564ff3d4dd887955b25b83d06989610769bf32 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Wed, 6 Feb 2019 09:59:16 -0800 Subject: drivers/mic/scif: do not use mmap_sem The driver uses mmap_sem for both pinned_vm accounting and get_user_pages(). By using gup_fast() and letting the mm handle the lock if needed, we can no longer rely on the semaphore and simplify the whole thing. Reviewed-by: Ira Weiny Signed-off-by: Davidlohr Bueso Signed-off-by: Jason Gunthorpe --- drivers/misc/mic/scif/scif_rma.c | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) (limited to 'drivers/misc/mic') diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c index 2448368f181e..263b8ad507ea 100644 --- a/drivers/misc/mic/scif/scif_rma.c +++ b/drivers/misc/mic/scif/scif_rma.c @@ -272,21 +272,12 @@ static inline void __scif_release_mm(struct mm_struct *mm) static inline int __scif_dec_pinned_vm_lock(struct mm_struct *mm, - int nr_pages, bool try_lock) + int nr_pages) { if (!mm || !nr_pages || !scif_ulimit_check) return 0; - if (try_lock) { - if (!down_write_trylock(&mm->mmap_sem)) { - dev_err(scif_info.mdev.this_device, - "%s %d err\n", __func__, __LINE__); - return -1; - } - } else { - down_write(&mm->mmap_sem); - } + atomic64_sub(nr_pages, &mm->pinned_vm); - up_write(&mm->mmap_sem); return 0; } @@ -298,16 +289,16 @@ static inline int __scif_check_inc_pinned_vm(struct mm_struct *mm, if (!mm || !nr_pages || !scif_ulimit_check) return 0; - locked = nr_pages; - locked += atomic64_read(&mm->pinned_vm); lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + locked = atomic64_add_return(nr_pages, &mm->pinned_vm); + if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) { + atomic64_sub(nr_pages, &mm->pinned_vm); dev_err(scif_info.mdev.this_device, "locked(%lu) > lock_limit(%lu)\n", locked, lock_limit); return -ENOMEM; } - atomic64_set(&mm->pinned_vm, locked); return 0; } @@ -326,7 +317,7 @@ int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window) might_sleep(); if (!window->temp && window->mm) { - __scif_dec_pinned_vm_lock(window->mm, window->nr_pages, 0); + __scif_dec_pinned_vm_lock(window->mm, window->nr_pages); __scif_release_mm(window->mm); window->mm = NULL; } @@ -737,7 +728,7 @@ done: ep->rma_info.dma_chan); } else { if (!__scif_dec_pinned_vm_lock(window->mm, - window->nr_pages, 1)) { + window->nr_pages)) { __scif_release_mm(window->mm); window->mm = NULL; } @@ -1385,28 +1376,23 @@ int __scif_pin_pages(void *addr, size_t len, int *out_prot, prot |= SCIF_PROT_WRITE; retry: mm = current->mm; - down_write(&mm->mmap_sem); if (ulimit) { err = __scif_check_inc_pinned_vm(mm, nr_pages); if (err) { - up_write(&mm->mmap_sem); pinned_pages->nr_pages = 0; goto error_unmap; } } - pinned_pages->nr_pages = get_user_pages( + pinned_pages->nr_pages = get_user_pages_fast( (u64)addr, nr_pages, (prot & SCIF_PROT_WRITE) ? FOLL_WRITE : 0, - pinned_pages->pages, - NULL); - up_write(&mm->mmap_sem); + pinned_pages->pages); if (nr_pages != pinned_pages->nr_pages) { if (try_upgrade) { if (ulimit) - __scif_dec_pinned_vm_lock(mm, - nr_pages, 0); + __scif_dec_pinned_vm_lock(mm, nr_pages); /* Roll back any pinned pages */ for (i = 0; i < pinned_pages->nr_pages; i++) { if (pinned_pages->pages[i]) @@ -1433,7 +1419,7 @@ retry: return err; dec_pinned: if (ulimit) - __scif_dec_pinned_vm_lock(mm, nr_pages, 0); + __scif_dec_pinned_vm_lock(mm, nr_pages); /* Something went wrong! Rollback */ error_unmap: pinned_pages->nr_pages = nr_pages; -- cgit v1.2.3