[PATCH] Add MPI variants for RemoteProxy, CollectFromContexts and ReduceOverContexts

[Richard Guenther]

Hi!

This patch adds native MPI variants of the above messaging abstractions.
These patches were tested together with the remaining changes with serial,
Cheetah and MPI.  As POOMA_MPI is never defined (for now), this shouldn't
introduce regressions there, too.  But of course for it alone, this patch
is useless.  More to follow.

Ok?

Richard.


2003Dec30  Richard Guenther <address@hidden>

        * src/Tulip/Messaging.cmpl.cpp: initialize static members for
        POOMA_CHEETAH only.
        src/Tulip/CollectFromContexts.h: add MPI variant.
        src/Tulip/ReduceOverContexts.h: likewise.
        src/Tulip/RemoteProxy.h: likewise.

--- Messaging.cmpl.cpp  2003-12-09 20:30:07.000000000 +0100
+++ /tmp/Messaging.cmpl.cpp     2003-12-30 21:11:27.000000000 +0100
@@ -38,12 +38,15 @@
 #include "Tulip/ReduceOverContexts.h"
 #include "Tulip/RemoteProxy.h"
 #include "Tulip/PatchSizeSyncer.h"
+#include "Tulip/SendReceive.h"

+#if POOMA_CHEETAH
 int  ReduceOverContextsBase::tagBase_m = 0;
 int  CollectFromContextsBase::tagBase_m = 0;
-
 bool RemoteProxyBase::ready_m;
 int  RemoteProxyBase::tag_m = 0;
+#endif
+

 //-----------------------------------------------------------------------------
 // Tag generator creates a set of tags for global use in r2.  There is a
--- CollectFromContexts.h       2003-12-09 20:27:38.000000000 +0100
+++ /tmp/CollectFromContexts.h  2003-12-30 21:11:27.000000000 +0100
@@ -44,12 +44,46 @@
 // Includes:
 //-----------------------------------------------------------------------------

-#include "Pooma/Pooma.h"
 #include "Tulip/Messaging.h"
+#include "Utilities/PAssert.h"

 #include <vector>


+#if !POOMA_MESSAGING
+
+template<class T>
+class CollectFromContexts
+{
+public:
+
+  CollectFromContexts(const T &val, int context = 0, bool valid = true)
+    {
+      PAssert(valid);
+      PAssert(context == 0);
+      value_m = val;
+    }
+
+  T &operator[](int i)
+    {
+      PAssert(i == 0);
+      return value_m;
+    }
+
+  T operator[](int i) const
+    {
+      PAssert(i == 0);
+      return value_m;
+    }
+
+private:
+  T value_m;
+};
+
+
+#else // POOMA_MESSAGING
+
+
 /**
  * This class associates a value with a flag that indicates whether or not
  * it is valid. It takes special care to not read the value if it is invalid.
@@ -108,11 +142,8 @@
 };


-#if POOMA_CHEETAH
-
 namespace Cheetah {

-
 /**
  * This class is used to serialize CollectionValue<T> objects, taking care
  * not to send invalid values.
@@ -177,9 +208,8 @@

 } // namespace Cheetah

-#endif
-

+#if POOMA_CHEETAH
 /**
  * This struct holds a few static quantities that are shared by all
  * instantiations of CollectFromContexts<T>. In particular, we want to
@@ -192,14 +222,14 @@

   static int tagBase_m;
 };
-
+#endif

 /**
  * This class is used to collect all valid values from all contexts.
  */

 template<class T>
-class CollectFromContexts : public CollectFromContextsBase
+class CollectFromContexts
 {
   typedef CollectFromContexts<T> This_t;

@@ -215,13 +245,50 @@
   // to read 'val' unless it is a valid value. Values don't have to
   // valid because not all contexts necessarily contribute to the collection.

-#if POOMA_CHEETAH
+#if POOMA_MPI
+
+  CollectFromContexts(const T &val, int toContext = 0, bool valid = true)
+    : toContext_m(toContext), data_m(Pooma::contexts())
+    {
+      typedef Cheetah::Serialize<Cheetah::CHEETAH, CollectionValue<T> > 
Serialize_t;
+      CollectionValue<T> v(valid, val);
+      // We need to get at the maximum size we need to transfer per context.
+      // With the valid/invalid mechanism we can't use size(v) for this, and
+      // for dynamic types like Grid<> we can't use CV<T>(true, T()) either.
+      // So for these cases we need to communicate the maximum size needed
+      // (but we might be able to optimize this with appropriate type tags).
+      int thislength = Serialize_t::size(v);
+      thislength = (thislength+7)&~7; // round to qword size
+      int length;
+      MPI_Allreduce(&thislength, &length, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+      char *buffer = new char[length];
+      char *recvbuffer = NULL;
+      if (Pooma::context() == toContext)
+        recvbuffer = new char[length*Pooma::contexts()];
+      Serialize_t::pack(v, buffer);
+      MPI_Gather(buffer, length, MPI_CHAR,
+                 recvbuffer, length, MPI_CHAR,
+                 toContext, MPI_COMM_WORLD);
+      delete[] buffer;
+      if (Pooma::context() == toContext) {
+        for (int i=0; i<Pooma::contexts(); ++i) {
+          CollectionValue<T> *v2;
+          Serialize_t::unpack(v2, recvbuffer+i*length);
+          if (v2->valid())
+           data_m[i] = v2->value();
+          Serialize_t::cleanup(v2);
+        }
+        delete[] recvbuffer;
+      }
+    }
+
+#elif POOMA_CHEETAH

   CollectFromContexts(const T &val, int toContext = 0, bool valid = true)
     : toContext_m(toContext), data_m(Pooma::contexts())
     {
-      int tagBase = tagBase_m;
-      tagBase_m += Pooma::contexts();
+      int tagBase = CollectFromContextsBase::tagBase_m;
+      CollectFromContextsBase::tagBase_m += Pooma::contexts();

       if (Pooma::context() == toContext)
        {
@@ -255,20 +322,6 @@
            send(toContext, tagBase + Pooma::context(), v);
        }
     }
-
-  T &operator[](int i)
-    {
-      PAssert(Pooma::context() == toContext_m);
-      PAssert(i >= 0 and i < Pooma::contexts());
-      return data_m[i];
-    }
-
-  T operator[](int i) const
-    {
-      PAssert(Pooma::context() == toContext_m);
-      PAssert(i >= 0 and i < Pooma::contexts());
-      return data_m[i];
-    }

 private:

@@ -283,47 +336,43 @@

     me->toReceive_m--;
   }
-
- // The actual value we're reducing.
-
-  std::vector<T> data_m;

   // The number of messages we're receiving.

   int toReceive_m;

-  // The context we're reducing on.
-
-  int toContext_m;
-
-#else
+#endif
+
+public:

-  CollectFromContexts(const T &val, int = 0, bool valid = true)
-    {
-      PAssert(valid);
-      value_m = val;
-    }
-
   T &operator[](int i)
     {
-      PAssert(i == 0);
-      return value_m;
+      PAssert(Pooma::context() == toContext_m);
+      PAssert(i >= 0 and i < Pooma::contexts());
+      return data_m[i];
     }

   T operator[](int i) const
     {
-      PAssert(i == 0);
-      return value_m;
+      PAssert(Pooma::context() == toContext_m);
+      PAssert(i >= 0 and i < Pooma::contexts());
+      return data_m[i];
     }

 private:

-  T value_m;
+  // The actual value we're reducing.
+
+  std::vector<T> data_m;
+
+  // The context we're reducing on.

-#endif // POOMA_CHEETAH
+  int toContext_m;

 };

+#endif // POOMA_MESSAGING
+
 #endif     // POOMA_MESSAGING_COLLECTFROMCONTEXTS_H

 // ACL:rcsinfo
--- ReduceOverContexts.h        2003-12-09 22:49:22.000000000 +0100
+++ /tmp/ReduceOverContexts.h   2003-12-30 21:11:27.000000000 +0100
@@ -45,8 +45,8 @@
 // Includes:
 //-----------------------------------------------------------------------------

-#include "Pooma/Pooma.h"
 #include "Tulip/Messaging.h"
+#include "Evaluator/OpMask.h"
 #include "Tulip/RemoteProxy.h"
 #include "Evaluator/OpMask.h"

@@ -89,6 +89,7 @@

   bool valid() const { return valid_m; }
   const T &value() const { PAssert(valid()); return val_m; }
+  T &value() { PAssert(valid()); return val_m; }

 private:

@@ -97,7 +98,7 @@
 };


-#if POOMA_CHEETAH
+#if POOMA_MESSAGING

 namespace Cheetah {

@@ -146,6 +147,9 @@

     vp = new ReductionValue<T>(*pvalid, *pval);

+    if (*pvalid)
+      Serialize<CHEETAH, T>::cleanup(pval);
+
     return nBytes;
   }

@@ -159,6 +163,8 @@

 #endif

+
+#if POOMA_CHEETAH
 /**
  * This struct holds a few static quantities that are shared by all
  * instantiations of ReduceOverContexts<T>. In particular, we want to
@@ -171,7 +177,7 @@

   static int tagBase_m;
 };
-
+#endif

 /**
  * This class is used to implement the final reduction over contexts used
@@ -179,7 +185,7 @@
  */

 template<class T, class ReductionOp>
-class ReduceOverContexts : public ReduceOverContextsBase
+class ReduceOverContexts
 {
   typedef ReduceOverContexts<T, ReductionOp> This_t;

@@ -195,13 +201,15 @@
   // to read 'val' unless it is a valid value. Values don't have to
   // valid because not all contexts necessarily contribute to the reduction.

+#if POOMA_MESSAGING
+
 #if POOMA_CHEETAH

   ReduceOverContexts(const T &val, int toContext = 0, bool valid = true)
     : valid_m(false), toContext_m(toContext)
     {
-      int tagBase = tagBase_m;
-      tagBase_m += Pooma::contexts();
+      int tagBase = ReduceOverContextsBase::tagBase_m;
+      ReduceOverContextsBase::tagBase_m += Pooma::contexts();

       if (Pooma::context() == toContext)
        {
@@ -235,6 +243,50 @@
        }
     }

+#elif POOMA_MPI
+
+  ReduceOverContexts(const T &val, int toContext = 0, bool valid = true)
+    : toContext_m(toContext)
+    {
+      typedef Cheetah::Serialize<Cheetah::CHEETAH, ReductionValue<T> > 
Serialize_t;
+      ReductionValue<T> v(valid, val);
+      // invalid size is different (doh!), so use some default for size
+      // strictly speaking this is incorrect, too (see CollectOverContexts),
+      // but we might not have reduction ops over dynamic sized objects...
+      int length = Serialize_t::size(ReductionValue<T>(true, T()));
+      length = (length+7)&~7; // round to qword size
+      char *buffer = new char[length];
+      char *recvbuffer = NULL;
+      if (Pooma::context() == toContext)
+       recvbuffer = new char[length*Pooma::contexts()];
+      Serialize_t::pack(v, buffer);
+      MPI_Gather(buffer, length, MPI_CHAR,
+                 recvbuffer, length, MPI_CHAR,
+                toContext, MPI_COMM_WORLD);
+      delete[] buffer;
+      if (Pooma::context() == toContext) {
+        for (int i=0; i<Pooma::contexts(); ++i) {
+          if (i == toContext) // this we already have in v
+            continue;
+          ReductionValue<T> *v2;
+          Serialize_t::unpack(v2, recvbuffer+i*length);
+          if (v2->valid()) {
+           if (!v.valid())
+             v = *v2;
+           else
+             Unwrap<ReductionOp>::Op_t()(v.value(), v2->value());
+          }
+          Serialize_t::cleanup(v2);
+        }
+        delete[] recvbuffer;
+        if (v.valid())
+          value_m = v.value();
+      }
+   }
+
+#endif
+
+  // FIXME: with a different API we could use MPI_AllGather here...
   void broadcast(T &val)
     {
       RemoteProxy<T> broadcast(value_m, toContext_m);
@@ -254,7 +306,7 @@
       val = value_m;
     }

-#endif // POOMA_CHEETAH
+#endif // POOMA_MESSAGING

   inline operator T() const { return value_m; }

--- RemoteProxy.h       2003-12-30 20:45:38.000000000 +0100
+++ /tmp/RemoteProxy.h  2003-12-30 21:11:27.000000000 +0100
@@ -34,7 +34,13 @@
 /** @file
  * @ingroup Tulip
  * @brief
- * Undocumented.
+ * This is like MPI_Bcast.
+ *
+ * It moves a value from one context to all others.
+ * Special about this is that assigns to a RemoteProxy object
+ * on the owning context is performed to the underlying data,
+ * while on the remote contexts it is just done to the proxy
+ * object.
  */

 #ifndef POOMA_CHEETAH_REMOTE_PROXY_H
@@ -54,20 +60,15 @@
 // Includes:
 //-----------------------------------------------------------------------------

-#include "Pooma/Pooma.h"
+#include "Tulip/Messaging.h"
 #include "Domain/Loc.h"
 #include "Tiny/Vector.h"
-#include "Tulip/Messaging.h"
 #include "Functions/ComponentAccess.h"

-#if POOMA_CHEETAH
-# include "Cheetah/Cheetah.h"
-#endif
-

 // For Cheetah support we need to mark more types not delegate.

-#if POOMA_CHEETAH
+#if POOMA_MESSAGING
 namespace Cheetah {

   /**
@@ -105,6 +106,22 @@
 #endif


+#if POOMA_CHEETAH
+struct RemoteProxyBase
+{
+  /// If we need a remote value, then this flag lets us know when it's
+  /// ready.  This value is static because it is used to block the parse
+  /// thread until the data is received.
+
+  static bool ready_m;
+
+  /// We only need one tag for all the remote proxies.  Perhaps this could
+  /// be packaged with the handler for remote proxies.
+
+  static int tag_m;
+};
+#endif
+
 /**
  * This class is the return type of the remote brick engine operator().
  * We need an object that lets us assign to data on this context, but that
@@ -122,20 +139,6 @@
  * value belongs to.
  */

-struct RemoteProxyBase
-{
-  /// If we need a remote value, then this flag lets us know when it's
-  /// ready.  This value is static because it is used to block the parse
-  /// thread until the data is received.
-
-  static bool ready_m;
-
-  /// We only need one tag for all the remote proxies.  Perhaps this could
-  /// be packaged with the handler for remote proxies.
-
-  static int tag_m;
-};
-
 template<class T>
 class RemoteProxy
 {
@@ -147,16 +150,15 @@
   /// value and broadcast the value to the other contexts.
   /// Otherwise we receive the value from the owning context.

+#if POOMA_CHEETAH
+
   RemoteProxy(T &val, int owningContext = 0)
   {
-#if POOMA_CHEETAH
     int tag = RemoteProxyBase::tag_m++;
-#endif
     if (Pooma::context() == owningContext)
     {
       value_m = &val;

-#if POOMA_CHEETAH
       int toContext;
       for (toContext = 0; toContext < Pooma::contexts(); ++toContext)
       {
@@ -165,11 +167,9 @@
          Pooma::indexHandler()->sendWith(Cheetah::CHEETAH(), toContext, tag, 
val);
        }
       }
-#endif
     }
     else
     {
-#if POOMA_CHEETAH
       storedValue_m = val;
       value_m = &storedValue_m;

@@ -182,10 +182,57 @@
       {
        Pooma::poll();
       }
-#endif
     }
   }

+private:
+  // Handler function for Cheetah.
+
+  static void receive(This_t *me, T &value)
+  {
+    me->storedValue_m = value;
+    RemoteProxyBase::ready_m = true;
+  }
+
+public:
+#elif POOMA_MPI
+
+  RemoteProxy(T &val, int owningContext = 0)
+  {
+    typedef Cheetah::Serialize<Cheetah::CHEETAH, T> Serialize_t;
+    int length = Serialize_t::size(val);
+    // Only the owningContext can possibly know the actual length for
+    // types like std::vector<>. Maybe we can conditionalize this extra
+    // communication on a tag field in the Cheetah::Serialize type.
+    MPI_Bcast(&length, 1, MPI_INT, owningContext, MPI_COMM_WORLD);
+    char *buffer = new char[length];
+    if (Pooma::context() == owningContext)
+      Serialize_t::pack(val, buffer);
+    MPI_Bcast(buffer, length, MPI_CHAR, owningContext, MPI_COMM_WORLD);
+    if (Pooma::context() == owningContext) {
+      value_m = &val;
+    } else {
+      T *nval;
+      Serialize_t::unpack(nval, buffer);
+      storedValue_m = *nval;
+      value_m = &storedValue_m;
+      Serialize_t::cleanup(nval);
+    }
+    delete[] buffer;
+  }
+
+#else
+
+  RemoteProxy(T &val, int owningContext = 0)
+  {
+    if (Pooma::context() == owningContext)
+    {
+      value_m = &val;
+    }
+  }
+
+#endif
+
   RemoteProxy(const RemoteProxy<T> &s)
   {
     if (s.value_m != &s.storedValue_m)
@@ -246,14 +293,6 @@
   }

 private:
-
-  // Handler function for Cheetah.
-
-  static void receive(This_t *me, T &value)
-  {
-    me->storedValue_m = value;
-    RemoteProxyBase::ready_m = true;
-  }

  // Pointer to the actual value represented by this proxy.