bug-hurd
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[PATCH gnumach 1/2] ipc: rework the kernel message buffer cache


From: Justus Winter
Subject: [PATCH gnumach 1/2] ipc: rework the kernel message buffer cache
Date: Thu, 18 Dec 2014 11:30:24 +0100

Keep a per-processor stack of free kernel messages buffers.

* ipc/ipc_kmsg.h (IKM_CACHE_SIZE): New macro.
(struct ipc_mksg_cpu_cache): New type.
(ipc_kmsg_cache): Use the new type for the cache.
(ikm_cache): Drop macro.
(ikm_cache_get, ikm_cache_put): New functions.
(ikm_free): Return buffers instead of freeing them.
(_ikm_free): New version of `ikm_free' that only frees buffers.
* ipc/ipc_kmsg.c (ipc_kmsg_cache): Use new type.
(ipc_kmsg_get, ipc_kmsg_put): Use new functions.
* ipc/mach_msg.c (mach_msg_trap): Likewise.
* kern/exception.c (exception_raise): Likewise.
(exception_parse_reply): Likewise.
* kern/ipc_kobject.c (ipc_kobject_server): Likewise.
---
 ipc/ipc_kmsg.c     | 31 +++++---------------
 ipc/ipc_kmsg.h     | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 ipc/mach_msg.c     | 14 ++++-----
 kern/exception.c   | 28 ++++--------------
 kern/ipc_kobject.c |  6 +---
 5 files changed, 99 insertions(+), 63 deletions(-)

diff --git a/ipc/ipc_kmsg.c b/ipc/ipc_kmsg.c
index 66643fd..e16709c 100644
--- a/ipc/ipc_kmsg.c
+++ b/ipc/ipc_kmsg.c
@@ -73,7 +73,9 @@
 #define ptr_align(x)   \
        ( ( ((vm_offset_t)(x)) + (sizeof(vm_offset_t)-1) ) & 
~(sizeof(vm_offset_t)-1) )
 
-ipc_kmsg_t ipc_kmsg_cache[NCPUS];
+#include <cache.h>
+
+struct ipc_kmsg_cpu_cache ipc_kmsg_cache[NCPUS] __cacheline_aligned;
 
 /*
  *     Routine:        ipc_kmsg_enqueue
@@ -506,23 +508,9 @@ ipc_kmsg_get(
        if ((size < sizeof(mach_msg_header_t)) || (size & 3))
                return MACH_SEND_MSG_TOO_SMALL;
 
-       if (size <= IKM_SAVED_MSG_SIZE) {
-               kmsg = ikm_cache();
-               if (kmsg != IKM_NULL) {
-                       ikm_cache() = IKM_NULL;
-                       ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
-               } else {
-                       kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
-                       if (kmsg == IKM_NULL)
-                               return MACH_SEND_NO_BUFFER;
-                       ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
-               }
-       } else {
-               kmsg = ikm_alloc(size);
-               if (kmsg == IKM_NULL)
-                       return MACH_SEND_NO_BUFFER;
-               ikm_init(kmsg, size);
-       }
+       kmsg = ikm_cache_get (size);
+       if (kmsg == IKM_NULL)
+               return MACH_SEND_NO_BUFFER;
 
        if (copyinmsg(msg, &kmsg->ikm_header, size)) {
                ikm_free(kmsg);
@@ -599,12 +587,7 @@ ipc_kmsg_put(
        else
                mr = MACH_MSG_SUCCESS;
 
-       if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
-           (ikm_cache() == IKM_NULL))
-               ikm_cache() = kmsg;
-       else
-               ikm_free(kmsg);
-
+       ikm_cache_put (kmsg);
        return mr;
 }
 
diff --git a/ipc/ipc_kmsg.h b/ipc/ipc_kmsg.h
index 620785b..c676996 100644
--- a/ipc/ipc_kmsg.h
+++ b/ipc/ipc_kmsg.h
@@ -96,11 +96,36 @@ MACRO_END
  *     The per-processor cache seems to miss less than a per-thread cache,
  *     and it also uses less memory.  Access to the cache doesn't
  *     require locking.
+ *
+ *     The per-processor cache is a stack containing unused kernel
+ *     message buffers.  We choose IKM_CACHE_SIZE so that the size of
+ *     the struct ipc_kmsg_cpu_cache is a multiple of a cache line,
+ *     to prevent it bouncing between per-cpu caches.
+ *
+ *     A kernel message buffer can be allocated using
+ *     `ikm_cache_get', and returned using `ikm_cache_put'.
  */
 
-extern ipc_kmsg_t      ipc_kmsg_cache[NCPUS];
+#define IKM_CACHE_SIZE (((1 << CPU_L1_SHIFT) - sizeof (size_t))        \
+                        / sizeof (ipc_kmsg_t))
+
+struct ipc_kmsg_cpu_cache
+{
+       size_t          count;
+       ipc_kmsg_t      buffers[IKM_CACHE_SIZE];
+};
+
+extern struct ipc_kmsg_cpu_cache ipc_kmsg_cache[NCPUS];
+
+/* Return a kernel message buffer of at least SIZE size, preferably
+   from the cache.  This functions is defined below.  */
+static inline ipc_kmsg_t ikm_cache_get (mach_msg_size_t size)
+       __attribute__ ((always_inline));
 
-#define ikm_cache()     ipc_kmsg_cache[cpu_number()]
+/* Return a kernel message buffer to the cache, or free it.  This
+   functions is defined below.  */
+static inline void ikm_cache_put (ipc_kmsg_t kmsg)
+       __attribute__ ((always_inline));
 
 /*
  *     The size of the kernel message buffers that will be cached.
@@ -150,11 +175,18 @@ MACRO_BEGIN                                               
                \
        register vm_size_t _size = (kmsg)->ikm_size;                    \
                                                                        \
        if ((integer_t)_size > 0)                                       \
-               kfree((vm_offset_t) (kmsg), _size);                     \
+               ikm_cache_put (kmsg);                                   \
        else                                                            \
                ipc_kmsg_free(kmsg);                                    \
 MACRO_END
 
+#define        _ikm_free(kmsg)                                                 
\
+MACRO_BEGIN                                                            \
+       vm_size_t _size = (kmsg)->ikm_size;                             \
+       assert (_size != IKM_SIZE_NETWORK);                             \
+       kfree((vm_offset_t) (kmsg), _size);                             \
+MACRO_END
+
 /*
  *     struct ipc_kmsg_queue is defined in ipc/ipc_kmsg_queue.h
  */
@@ -280,5 +312,50 @@ ipc_kmsg_copyout_pseudo(ipc_kmsg_t, ipc_space_t, vm_map_t);
 
 extern void
 ipc_kmsg_copyout_dest(ipc_kmsg_t, ipc_space_t);
+
+static inline ipc_kmsg_t
+ikm_cache_get (mach_msg_size_t size)
+{
+       ipc_kmsg_t kmsg;
+       struct ipc_kmsg_cpu_cache *c = &ipc_kmsg_cache[cpu_number ()];
+
+       if (size < IKM_SAVED_MSG_SIZE)
+               size = IKM_SAVED_MSG_SIZE;
+
+       if (size > IKM_SAVED_MSG_SIZE
+           || c->count == 0) {
+               kmsg = ikm_alloc (size);
+               if (kmsg)
+                       ikm_init (kmsg, size);
+               return kmsg;
+       }
+
+       c->count -= 1;
+       kmsg = c->buffers[c->count];
+
+#if    MACH_IPC_TEST
+       ikm_check_initialized (kmsg, IKM_SAVED_KMSG_SIZE);
+#endif /* MACH_IPC_TEST */
+       return kmsg;
+}
+
+static inline void
+ikm_cache_put (ipc_kmsg_t kmsg)
+{
+       struct ipc_kmsg_cpu_cache *c = &ipc_kmsg_cache[cpu_number ()];
+
+       if (kmsg->ikm_size != IKM_SAVED_KMSG_SIZE
+           || c->count == IKM_CACHE_SIZE) {
+               _ikm_free (kmsg);
+               return;
+       }
+
+       c->buffers[c->count] = kmsg;
+       c->count += 1;
+
+#if    MACH_IPC_TEST
+       ikm_check_initialized (kmsg, IKM_SAVED_KMSG_SIZE);
+#endif /* MACH_IPC_TEST */
+}
 
 #endif /* _IPC_IPC_KMSG_H_ */
diff --git a/ipc/mach_msg.c b/ipc/mach_msg.c
index 1e122c7..cdf16ea 100644
--- a/ipc/mach_msg.c
+++ b/ipc/mach_msg.c
@@ -451,16 +451,14 @@ mach_msg_trap(
 
                if ((send_size > IKM_SAVED_MSG_SIZE) ||
                    (send_size < sizeof(mach_msg_header_t)) ||
-                   (send_size & 3) ||
-                   ((kmsg = ikm_cache()) == IKM_NULL))
+                   (send_size & 3))
                        goto slow_get;
 
-               ikm_cache() = IKM_NULL;
-               ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
+               kmsg = ikm_cache_get (IKM_SAVED_MSG_SIZE);
 
                if (copyinmsg(msg, &kmsg->ikm_header,
                              send_size)) {
-                       ikm_free(kmsg);
+                       ikm_cache_put (kmsg);
                        goto slow_get;
                }
 
@@ -1263,18 +1261,16 @@ mach_msg_trap(
                 *      We have the reply message data in kmsg,
                 *      and the reply message size in reply_size.
                 *      Just need to copy it out to the user and free kmsg.
-                *      We must check ikm_cache after copyoutmsg.
                 */
 
                ikm_check_initialized(kmsg, kmsg->ikm_size);
 
                if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
                    copyoutmsg(&kmsg->ikm_header, msg,
-                              reply_size) ||
-                   (ikm_cache() != IKM_NULL))
+                              reply_size))
                        goto slow_put;
 
-               ikm_cache() = kmsg;
+               ikm_cache_put (kmsg);
                thread_syscall_return(MACH_MSG_SUCCESS);
                /*NOTREACHED*/
                return MACH_MSG_SUCCESS; /* help for the compiler */
diff --git a/kern/exception.c b/kern/exception.c
index 7954fba..72a4f29 100644
--- a/kern/exception.c
+++ b/kern/exception.c
@@ -348,16 +348,9 @@ exception_raise(
         *      and it will give the buffer back with its reply.
         */
 
-       kmsg = ikm_cache();
-       if (kmsg != IKM_NULL) {
-               ikm_cache() = IKM_NULL;
-               ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
-       } else {
-               kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
-               if (kmsg == IKM_NULL)
-                       panic("exception_raise");
-               ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
-       }
+       kmsg = ikm_cache_get (IKM_SAVED_MSG_SIZE);
+       if (kmsg == IKM_NULL)
+               panic("exception_raise");
 
        /*
         *      We need a reply port for the RPC.
@@ -681,22 +674,19 @@ exception_raise(
 
        /*
         *      Optimized version of ipc_kmsg_put.
-        *      We must check ikm_cache after copyoutmsg.
         */
-
        ikm_check_initialized(kmsg, kmsg->ikm_size);
        assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
 
        if (copyoutmsg(&kmsg->ikm_header, receiver->ith_msg,
-                      sizeof(struct mach_exception)) ||
-           (ikm_cache() != IKM_NULL)) {
+                      sizeof(struct mach_exception))) {
                mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
                                  kmsg->ikm_header.msgh_size);
                thread_syscall_return(mr);
                /*NOTREACHED*/
        }
 
-       ikm_cache() = kmsg;
+       ikm_cache_put (kmsg);
        thread_syscall_return(MACH_MSG_SUCCESS);
        /*NOTREACHED*/
 #ifndef        __GNUC__
@@ -809,13 +799,7 @@ exception_parse_reply(ipc_kmsg_t kmsg)
        }
 
        kr = msg->RetCode;
-
-       if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
-           (ikm_cache() == IKM_NULL))
-               ikm_cache() = kmsg;
-       else
-               ikm_free(kmsg);
-
+       ikm_cache_put (kmsg);
        return kr;
 }
 
diff --git a/kern/ipc_kobject.c b/kern/ipc_kobject.c
index bf22028..27535b0 100644
--- a/kern/ipc_kobject.c
+++ b/kern/ipc_kobject.c
@@ -236,11 +236,7 @@ ipc_kobject_server(request)
                /* like ipc_kmsg_put, but without the copyout */
 
                ikm_check_initialized(request, request->ikm_size);
-               if ((request->ikm_size == IKM_SAVED_KMSG_SIZE) &&
-                   (ikm_cache() == IKM_NULL))
-                       ikm_cache() = request;
-               else
-                       ikm_free(request);
+               ikm_cache_put (request);
        } else {
                /*
                 *      The message contents of the request are intact.
-- 
2.1.3




reply via email to

[Prev in Thread] Current Thread [Next in Thread]