[PATCHv2 1/2] Handle notification on page eviction

[Richard Braun]

If requested by the user, make libpager call pager_notify_evict when a page
is flushed out by the kernel. Based on work by Ognyan Kulev.

* console/pager.c (pager_notify_evict): New function.
(user_pager_create): Update call to pager_create.
* ext2fs/pager.c: (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* fatfs/pager.c: (pager_notify_evict): New function.
(create_fat_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* isofs/pager.c: (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* libdiskfs/disk-pager.c (diskfs_start_disk_pager): Update definition and call
to pager_create.
* libdiskfs/diskfs-pager.h (diskfs_start_disk_pager): Update declaration.
* libpager/data-request.c (_pager_seqnos_memory_object_data_request): Take
pager's `notify_on_evict' member into account when calling
memory_object_data_supply.
* libpager/data-return.c (_pager_do_write_request): Handle user notification
on page flush.
* libpager/pager-create.c (pager_create): Update definition and set pager's
`notify_on_evict' member.
* libpager/pager.h (pager_create): Update declaration.
(pager_notify_evict): New declaration.
* libpager/priv.h (struct pager): New `notify_on_evict' member.
* storeio/pager.c: (pager_notify_evict): New function.
(dev_get_memory_object): Update call to pager_create.
* tmpfs/pager-stubs.c (pager_notify_evict): New function.
* ufs/pager.c (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
---
 console/pager.c          |   10 +++++-
 ext2fs/pager.c           |   14 ++++++--
 fatfs/pager.c            |   11 +++++--
 isofs/pager.c            |   12 +++++--
 libdiskfs/disk-pager.c   |    6 ++--
 libdiskfs/diskfs-pager.h |    3 +-
 libpager/data-request.c  |    3 +-
 libpager/data-return.c   |   79 ++++++++++++++++++++++++++++++++++++----------
 libpager/pager-create.c  |    4 ++-
 libpager/pager.h         |   23 +++++++++++---
 libpager/priv.h          |    1 +
 storeio/pager.c          |    9 +++++-
 tmpfs/pager-stubs.c      |    8 +++++
 ufs/pager.c              |   11 +++++--
 14 files changed, 158 insertions(+), 36 deletions(-)

diff --git a/console/pager.c b/console/pager.c
index 781ba35..4d0c5cd 100644
--- a/console/pager.c
+++ b/console/pager.c
@@ -95,6 +95,14 @@ pager_unlock_page (struct user_pager_info *pager,
 }
 
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
+
 /* Tell how big the file is. */
 error_t
 pager_report_extent (struct user_pager_info *upi,
@@ -170,7 +178,7 @@ user_pager_create (struct user_pager *user_pager, unsigned 
int npages,
 
   /* XXX Are the values 1 and MEMORY_OBJECT_COPY_DELAY correct? */
   user_pager->pager = pager_create (upi, pager_bucket,
-                                   1, MEMORY_OBJECT_COPY_DELAY);
+                                   1, MEMORY_OBJECT_COPY_DELAY, 0);
   if (!user_pager->pager)
     {
       free (upi);
diff --git a/ext2fs/pager.c b/ext2fs/pager.c
index f740434..9213711 100644
--- a/ext2fs/pager.c
+++ b/ext2fs/pager.c
@@ -511,6 +511,13 @@ pager_write_page (struct user_pager_info *pager, 
vm_offset_t page,
   else
     return file_pager_write_page (pager->node, page, (void *)buf);
 }
+
+void
+pager_notify_evict (struct user_pager_info *pager, vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 
 /* Make page PAGE writable, at least up to ALLOCSIZE.  This function and
    diskfs_grow are the only places that blocks are actually added to the
@@ -776,8 +783,9 @@ create_disk_pager (void)
     ext2_panic ("can't create disk pager: %s", strerror (errno));
   upi->type = DISK;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, store->size,
-                          &disk_image);
+  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, 0,
+                          store->size, &disk_image);
+
 }
 
 /* Call this to create a FILE_DATA pager and return a send right.
@@ -817,7 +825,7 @@ diskfs_get_filemap (struct node *node, vm_prot_t prot)
          diskfs_nref_light (node);
          node->dn->pager =
            pager_create (upi, pager_bucket, MAY_CACHE,
-                         MEMORY_OBJECT_COPY_DELAY);
+                         MEMORY_OBJECT_COPY_DELAY, 0);
          if (node->dn->pager == 0)
            {
              diskfs_nrele_light (node);
diff --git a/fatfs/pager.c b/fatfs/pager.c
index f892c88..8146e64 100644
--- a/fatfs/pager.c
+++ b/fatfs/pager.c
@@ -596,6 +596,13 @@ pager_unlock_page (struct user_pager_info *pager,
   return 0;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* Grow the disk allocated to locked node NODE to be at least SIZE
    bytes, and set NODE->allocsize to the actual allocated size.  (If
    the allocated size is already SIZE bytes, do nothing.)  CRED
@@ -752,7 +759,7 @@ create_fat_pager (void)
   struct user_pager_info *upi = malloc (sizeof (struct user_pager_info));
   upi->type = FAT;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE,
+  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, 0,
                           bytes_per_sector * sectors_per_fat,
                           &fat_image);
 }
@@ -794,7 +801,7 @@ diskfs_get_filemap (struct node *node, vm_prot_t prot)
           diskfs_nref_light (node);
           node->dn->pager =
             pager_create (upi, pager_bucket, MAY_CACHE,
-                          MEMORY_OBJECT_COPY_DELAY);
+                          MEMORY_OBJECT_COPY_DELAY, 0);
           if (node->dn->pager == 0)
             {
               diskfs_nrele_light (node);
diff --git a/isofs/pager.c b/isofs/pager.c
index f93e0c8..7a38ba3 100644
--- a/isofs/pager.c
+++ b/isofs/pager.c
@@ -94,6 +94,13 @@ pager_unlock_page (struct user_pager_info *pager,
   return EROFS;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* Tell how big the file is. */
 error_t
 pager_report_extent (struct user_pager_info *pager,
@@ -137,7 +144,7 @@ create_disk_pager (void)
   upi->type = DISK;
   upi->np = 0;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, 1, store->size, &disk_image);
+  diskfs_start_disk_pager (upi, pager_bucket, 1, 0, store->size, &disk_image);
   upi->p = diskfs_disk_pager;
 }
 
@@ -168,7 +175,8 @@ diskfs_get_filemap (struct node *np, vm_prot_t prot)
        upi->type = FILE_DATA;
        upi->np = np;
        diskfs_nref_light (np);
-       upi->p = pager_create (upi, pager_bucket, 1, MEMORY_OBJECT_COPY_DELAY);
+       upi->p = pager_create (upi, pager_bucket, 1,
+                              MEMORY_OBJECT_COPY_DELAY, 0);
        if (upi->p == 0)
          {
            diskfs_nrele_light (np);
diff --git a/libdiskfs/disk-pager.c b/libdiskfs/disk-pager.c
index 5795a28..8fe8f80 100644
--- a/libdiskfs/disk-pager.c
+++ b/libdiskfs/disk-pager.c
@@ -49,7 +49,8 @@ service_paging_requests (void *arg)
 
 void
 diskfs_start_disk_pager (struct user_pager_info *upi,
-                        struct port_bucket *pager_bucket, int may_cache,
+                        struct port_bucket *pager_bucket,
+                        int may_cache, int notify_on_evict,
                         size_t size, void **image)
 {
   pthread_t thread;
@@ -68,7 +69,8 @@ diskfs_start_disk_pager (struct user_pager_info *upi,
 
   /* Create the pager.  */
   diskfs_disk_pager = pager_create (upi, pager_bucket,
-                                   may_cache, MEMORY_OBJECT_COPY_NONE);
+                                   may_cache, MEMORY_OBJECT_COPY_NONE,
+                                   notify_on_evict);
   assert (diskfs_disk_pager);
 
   /* Get a port to the disk pager.  */
diff --git a/libdiskfs/diskfs-pager.h b/libdiskfs/diskfs-pager.h
index bd0a050..a253069 100644
--- a/libdiskfs/diskfs-pager.h
+++ b/libdiskfs/diskfs-pager.h
@@ -35,7 +35,8 @@ extern __thread struct disk_image_user *diskfs_exception_diu;
    mapped is returned in IMAGE.  INFO, PAGER_BUCKET, & MAY_CACHE are passed
    to `pager_create'.  */
 extern void diskfs_start_disk_pager (struct user_pager_info *info,
-                                    struct port_bucket *pager_bucket, int 
may_cache,
+                                    struct port_bucket *pager_bucket,
+                                    int may_cache, int notify_on_evict,
                                     size_t size, void **image);
 
 extern struct pager *diskfs_disk_pager;
diff --git a/libpager/data-request.c b/libpager/data-request.c
index 4454fac..34b8b43 100644
--- a/libpager/data-request.c
+++ b/libpager/data-request.c
@@ -119,7 +119,8 @@ _pager_seqnos_memory_object_data_request (mach_port_t 
object,
     goto error_read;
 
   memory_object_data_supply (p->memobjcntl, offset, page, length, 1,
-                            write_lock ? VM_PROT_WRITE : VM_PROT_NONE, 0,
+                            write_lock ? VM_PROT_WRITE : VM_PROT_NONE,
+                            p->notify_on_evict ? 1 : 0,
                             MACH_PORT_NULL);
   pthread_mutex_lock (&p->interlock);
   _pager_mark_object_error (p, offset, length, 0);
diff --git a/libpager/data-return.c b/libpager/data-return.c
index c70f0e8..6a3b903 100644
--- a/libpager/data-return.c
+++ b/libpager/data-return.c
@@ -39,6 +39,7 @@ _pager_do_write_request (mach_port_t object,
   struct pager *p;
   short *pm_entries;
   int npages, i;
+  char *notified;
   error_t *pagerrs;
   struct lock_request *lr;
   struct lock_list {struct lock_request *lr;
@@ -71,9 +72,6 @@ _pager_do_write_request (mach_port_t object,
       goto release_out;
     }
 
-  if (! dirty)
-    goto release_out;
-
   if (p->pager_state != NORMAL)
     {
       printf ("pager in wrong state for write\n");
@@ -83,6 +81,11 @@ _pager_do_write_request (mach_port_t object,
   npages = length / __vm_page_size;
   pagerrs = alloca (npages * sizeof (error_t));
 
+  notified = alloca (npages * (sizeof *notified));
+#ifndef NDEBUG
+  memset (notified, -1, npages * (sizeof *notified));
+#endif
+
   _pager_block_termination (p);        /* until we are done with the pagemap
                                   when the write completes. */
 
@@ -90,6 +93,24 @@ _pager_do_write_request (mach_port_t object,
 
   pm_entries = &p->pagemap[offset / __vm_page_size];
 
+  if (! dirty)
+    {
+      munmap ((void *) data, length);
+      if (!kcopy) {
+        /* Prepare notified array.  */
+        for (i = 0; i < npages; i++)
+          notified[i] = (p->notify_on_evict
+                         && ! (pm_entries[i] & PM_PAGEINWAIT));
+
+        _pager_release_seqno (p, seqno);
+        goto notify;
+      }
+      else {
+        _pager_allow_termination (p);
+        goto release_out;
+      }
+    }
+
   /* Make sure there are no other in-progress writes for any of these
      pages before we begin.  This imposes a little more serialization
      than we really have to require (because *all* future writes on
@@ -120,10 +141,6 @@ _pager_do_write_request (mach_port_t object,
     for (i = 0; i < npages; i++)
       pm_entries[i] |= PM_PAGINGOUT | PM_INIT;
 
-  if (!kcopy)
-    for (i = 0; i < npages; i++)
-      pm_entries[i] &= ~PM_INCORE;
-
   /* If this write occurs while a lock is pending, record
      it.  We have to keep this list because a lock request
      might come in while we do the I/O; in that case there
@@ -163,7 +180,10 @@ _pager_do_write_request (mach_port_t object,
   for (i = 0; i < npages; i++)
     {
       if (omitdata & (1 << i))
-       continue;
+       {
+         notified[i] = 0;
+         continue;
+       }
 
       if (pm_entries[i] & PM_WRITEWAIT)
        wakeup = 1;
@@ -179,14 +199,22 @@ _pager_do_write_request (mach_port_t object,
        pm_entries[i] |= PM_INVALID;
 
       if (pm_entries[i] & PM_PAGEINWAIT)
-       memory_object_data_supply (p->memobjcntl,
-                                  offset + (vm_page_size * i),
-                                  data + (vm_page_size * i),
-                                  vm_page_size, 1,
-                                  VM_PROT_NONE, 0, MACH_PORT_NULL);
+       {
+         memory_object_data_supply (p->memobjcntl,
+                                    offset + (vm_page_size * i),
+                                    data + (vm_page_size * i),
+                                    vm_page_size, 1,
+                                    VM_PROT_NONE, 0, MACH_PORT_NULL);
+         notified[i] = 0;
+       }
       else
-       munmap ((caddr_t) (data + (vm_page_size * i)),
-               vm_page_size);
+       {
+         munmap ((void *) (data + (vm_page_size * i)),
+                 vm_page_size);
+         notified[i] = (! kcopy && p->notify_on_evict);
+         if (! kcopy)
+           pm_entries[i] &= ~PM_INCORE;
+       }
 
       pm_entries[i] &= ~(PM_PAGINGOUT | PM_PAGEINWAIT | PM_WRITEWAIT);
     }
@@ -198,10 +226,29 @@ _pager_do_write_request (mach_port_t object,
   if (wakeup)
     pthread_cond_broadcast (&p->wakeup);
 
+ notify:
   _pager_allow_termination (p);
-
   pthread_mutex_unlock (&p->interlock);
 
+  for (i = 0; i < npages; i++)
+    {
+      assert (notified[i] == 0 || notified[i] == 1);
+      if (notified[i])
+       {
+         short *pm_entry = &pm_entries[i];
+
+         /* Do notify user.  */
+         pager_notify_evict (p->upi, offset + (i * vm_page_size));
+
+         /* Clear any error that is left.  Notification on eviction
+            is used only to change association of page, so any
+            error may no longer be valid.  */
+         pthread_mutex_lock (&p->interlock);
+         *pm_entry = SET_PM_ERROR (SET_PM_NEXTERROR (*pm_entry, 0), 0);
+         pthread_mutex_unlock (&p->interlock);
+       }
+    }
+
   ports_port_deref (p);
   return 0;
 
diff --git a/libpager/pager-create.c b/libpager/pager-create.c
index 318c9f1..1aea6e9 100644
--- a/libpager/pager-create.c
+++ b/libpager/pager-create.c
@@ -22,7 +22,8 @@ struct pager *
 pager_create (struct user_pager_info *upi,
              struct port_bucket *bucket,
              boolean_t may_cache,
-             memory_object_copy_strategy_t copy_strategy)
+             memory_object_copy_strategy_t copy_strategy,
+             boolean_t notify_on_evict)
 {
   struct pager *p;
 
@@ -38,6 +39,7 @@ pager_create (struct user_pager_info *upi,
   p->attribute_requests = 0;
   p->may_cache = may_cache;
   p->copy_strategy = copy_strategy;
+  p->notify_on_evict = notify_on_evict;
   p->memobjcntl = MACH_PORT_NULL;
   p->memobjname = MACH_PORT_NULL;
   p->seqno = -1;
diff --git a/libpager/pager.h b/libpager/pager.h
index 99fb384..75ff108 100644
--- a/libpager/pager.h
+++ b/libpager/pager.h
@@ -36,14 +36,17 @@ int pager_demuxer (mach_msg_header_t *inp,
    to receive requests.  U_PAGER will be provided to later calls to
    pager_find_address.  The pager will have one user reference
    created.  MAY_CACHE and COPY_STRATEGY are the original values of
-   those attributes as for memory_object_ready.  Users may create
-   references to pagers by use of the relevant ports library
-   functions.  On errors, return null and set errno.  */
+   those attributes as for memory_object_ready.  If NOTIFY_ON_EVICT is
+   non-zero, pager_notify_evict user callback will be called when page
+   is evicted.  Users may create references to pagers by use of the
+   relevant ports library functions.  On errors, return null and set
+   errno.  */
 struct pager *
 pager_create (struct user_pager_info *u_pager,
              struct port_bucket *bucket,
              boolean_t may_cache,
-             memory_object_copy_strategy_t copy_strategy);
+             memory_object_copy_strategy_t copy_strategy,
+             boolean_t notify_on_evict);
 
 /* Return the user_pager_info struct associated with a pager. */
 struct user_pager_info *
@@ -172,6 +175,18 @@ error_t
 pager_unlock_page (struct user_pager_info *pager,
                   vm_offset_t address);
 
+/* The user must define this function.  It is used when you want be
+   able to change association of pages to backing store.  To use it,
+   pass non-zero value in NOTIFY_ON_EVICT when pager is created with
+   pager_create.  You can change association of page only when
+   pager_notify_evict has been called and you haven't touched page
+   content after that.  Note there is a possibility that a page is
+   evicted, but user is not notified about that.  The user should be
+   able to handle this case.  */
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page);
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
diff --git a/libpager/priv.h b/libpager/priv.h
index e6b2546..7aa0fb4 100644
--- a/libpager/priv.h
+++ b/libpager/priv.h
@@ -49,6 +49,7 @@ struct pager
 
   boolean_t may_cache;
   memory_object_copy_strategy_t copy_strategy;
+  boolean_t notify_on_evict;
 
   /* Interface ports */
   memory_object_control_t memobjcntl;
diff --git a/storeio/pager.c b/storeio/pager.c
index cbae7eb..7d78711 100644
--- a/storeio/pager.c
+++ b/storeio/pager.c
@@ -110,6 +110,13 @@ pager_unlock_page (struct user_pager_info *upi, 
vm_offset_t address)
     return 0;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
@@ -246,7 +253,7 @@ dev_get_memory_object (struct dev *dev, vm_prot_t prot, 
memory_object_t *memobj)
        {
          dev->pager =
            pager_create ((struct user_pager_info *)dev, pager_port_bucket,
-                         1, MEMORY_OBJECT_COPY_DELAY);
+                         1, MEMORY_OBJECT_COPY_DELAY, 0);
          if (dev->pager == NULL)
            {
              pthread_mutex_unlock (&dev->pager_lock);
diff --git a/tmpfs/pager-stubs.c b/tmpfs/pager-stubs.c
index 25d70fe..3cb264b 100644
--- a/tmpfs/pager-stubs.c
+++ b/tmpfs/pager-stubs.c
@@ -57,6 +57,14 @@ pager_unlock_page (struct user_pager_info *pager,
   return EIEIO;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page)
+{
+  abort();
+}
+
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
diff --git a/ufs/pager.c b/ufs/pager.c
index 1e3d140..5d3e44a 100644
--- a/ufs/pager.c
+++ b/ufs/pager.c
@@ -450,6 +450,13 @@ pager_unlock_page (struct user_pager_info *pager,
   return err;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+                   vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* Implement the pager_report_extent callback from the pager library.  See
    <hurd/pager.h> for the interface description. */
 inline error_t
@@ -502,7 +509,7 @@ create_disk_pager (void)
   upi->type = DISK;
   upi->np = 0;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, store->size,
+  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, 0, store->size,
                           &disk_image);
   upi->p = diskfs_disk_pager;
 }
@@ -595,7 +602,7 @@ diskfs_get_filemap (struct node *np, vm_prot_t prot)
        upi->unlocked_pagein_length = 0;
        diskfs_nref_light (np);
        upi->p = pager_create (upi, pager_bucket,
-                              MAY_CACHE, MEMORY_OBJECT_COPY_DELAY);
+                              MAY_CACHE, MEMORY_OBJECT_COPY_DELAY, 0);
        if (upi->p == 0)
          {
            diskfs_nrele_light (np);
-- 
1.7.10.4