Import Boost.MPI with the beginnings of a BBv2-based build system

[SVN r36337]

.gitattributes

@@ -0,0 +1,96 @@
+* text=auto !eol svneol=native#text/plain
+*.gitattributes text svneol=native#text/plain
+
+# Scriptish formats
+*.bat        text svneol=native#text/plain
+*.bsh        text svneol=native#text/x-beanshell
+*.cgi        text svneol=native#text/plain
+*.cmd        text svneol=native#text/plain
+*.js         text svneol=native#text/javascript
+*.php        text svneol=native#text/x-php
+*.pl         text svneol=native#text/x-perl
+*.pm         text svneol=native#text/x-perl
+*.py         text svneol=native#text/x-python
+*.sh         eol=lf svneol=LF#text/x-sh
+configure    eol=lf svneol=LF#text/x-sh
+
+# Image formats
+*.bmp        binary svneol=unset#image/bmp
+*.gif        binary svneol=unset#image/gif
+*.ico        binary svneol=unset#image/ico
+*.jpeg       binary svneol=unset#image/jpeg
+*.jpg        binary svneol=unset#image/jpeg
+*.png        binary svneol=unset#image/png
+*.tif        binary svneol=unset#image/tiff
+*.tiff       binary svneol=unset#image/tiff
+*.svg        text svneol=native#image/svg%2Bxml
+
+# Data formats
+*.pdf        binary svneol=unset#application/pdf
+*.avi        binary svneol=unset#video/avi
+*.doc        binary svneol=unset#application/msword
+*.dsp        text svneol=crlf#text/plain
+*.dsw        text svneol=crlf#text/plain
+*.eps        binary svneol=unset#application/postscript
+*.gz         binary svneol=unset#application/gzip
+*.mov        binary svneol=unset#video/quicktime
+*.mp3        binary svneol=unset#audio/mpeg
+*.ppt        binary svneol=unset#application/vnd.ms-powerpoint
+*.ps         binary svneol=unset#application/postscript
+*.psd        binary svneol=unset#application/photoshop
+*.rdf        binary svneol=unset#text/rdf
+*.rss        text svneol=unset#text/xml
+*.rtf        binary svneol=unset#text/rtf
+*.sln        text svneol=native#text/plain
+*.swf        binary svneol=unset#application/x-shockwave-flash
+*.tgz        binary svneol=unset#application/gzip
+*.vcproj     text svneol=native#text/xml
+*.vcxproj    text svneol=native#text/xml
+*.vsprops    text svneol=native#text/xml
+*.wav        binary svneol=unset#audio/wav
+*.xls        binary svneol=unset#application/vnd.ms-excel
+*.zip        binary svneol=unset#application/zip
+
+# Text formats
+.htaccess    text svneol=native#text/plain
+*.bbk        text svneol=native#text/xml
+*.cmake      text svneol=native#text/plain
+*.css        text svneol=native#text/css
+*.dtd        text svneol=native#text/xml
+*.htm        text svneol=native#text/html
+*.html       text svneol=native#text/html
+*.ini        text svneol=native#text/plain
+*.log        text svneol=native#text/plain
+*.mak        text svneol=native#text/plain
+*.qbk        text svneol=native#text/plain
+*.rst        text svneol=native#text/plain
+*.sql        text svneol=native#text/x-sql
+*.txt        text svneol=native#text/plain
+*.xhtml      text svneol=native#text/xhtml%2Bxml
+*.xml        text svneol=native#text/xml
+*.xsd        text svneol=native#text/xml
+*.xsl        text svneol=native#text/xml
+*.xslt       text svneol=native#text/xml
+*.xul        text svneol=native#text/xul
+*.yml        text svneol=native#text/plain
+boost-no-inspect text svneol=native#text/plain
+CHANGES      text svneol=native#text/plain
+COPYING      text svneol=native#text/plain
+INSTALL      text svneol=native#text/plain
+Jamfile      text svneol=native#text/plain
+Jamroot      text svneol=native#text/plain
+Jamfile.v2   text svneol=native#text/plain
+Jamrules     text svneol=native#text/plain
+Makefile*    text svneol=native#text/plain
+README       text svneol=native#text/plain
+TODO         text svneol=native#text/plain
+
+# Code formats
+*.c          text svneol=native#text/plain
+*.cpp        text svneol=native#text/plain
+*.h          text svneol=native#text/plain
+*.hpp        text svneol=native#text/plain
+*.ipp        text svneol=native#text/plain
+*.tpp        text svneol=native#text/plain
+*.jam        text svneol=native#text/plain
+*.java       text svneol=native#text/plain

build/Jamfile.v2

@@ -0,0 +1,49 @@
+# Copyright (C) 2005, 2006 The Trustees of Indiana University.
+# Copyright (C) 2005 Douglas Gregor <doug.gregor -at- gmail.com>
+
+# Use, modification and distribution is subject to the Boost Software
+# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+#  Authors: Douglas Gregor
+#           Andrew Lumsdaine
+
+import mpi ;
+
+project boost/python 
+  : source-location ../src/mpi
+  ;
+
+if [ mpi.configured ]
+{
+
+lib boost_mpi 
+  : 
+    broadcast.cpp
+    communicator.cpp
+    computation_tree.cpp
+    content_oarchive.cpp
+    environment.cpp
+    mpi_datatype_cache.cpp
+    mpi_datatype_oarchive.cpp
+    packed_iarchive.cpp
+    packed_oarchive.cpp
+    packed_skeleton_iarchive.cpp
+    packed_skeleton_oarchive.cpp
+    point_to_point.cpp
+    request.cpp
+    text_skeleton_oarchive.cpp
+    timer.cpp
+  : # Requirements
+    <library>../../serialization/build//boost_serialization
+    <library>/mpi//mpi
+  : # Default build
+    <link>shared
+  : # Usage requirements
+  ;
+}
+else
+{
+  ECHO "The mpi module has not been initialized; MPI libraries will not be built." ;
+  ECHO "hint: add \"using mpi ;\" to user-config.jam to enable MPI support." ;
+}

build/__init__.py

@@ -0,0 +1 @@
+

doc/Jamfile.v2

@@ -0,0 +1,41 @@
+# Copyright (C) 2005-2006 Douglas Gregor <doug.gregor@gmail.com>
+#
+# Distributed under the Boost Software License, Version 1.0. (See accompanying 
+# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt.)
+project boost/parallel/mpi ;
+
+using quickbook ;
+using doxygen ;
+
+doxygen mpi_autodoc 
+  : ../../../boost/parallel/mpi.hpp
+    ../../../boost/parallel/mpi/allocator.hpp
+    ../../../boost/parallel/mpi/collectives.hpp
+    ../../../boost/parallel/mpi/collectives_fwd.hpp
+    ../../../boost/parallel/mpi/communicator.hpp
+    ../../../boost/parallel/mpi/config.hpp
+    ../../../boost/parallel/mpi/datatype.hpp
+    ../../../boost/parallel/mpi/datatype_fwd.hpp
+    ../../../boost/parallel/mpi/environment.hpp
+    ../../../boost/parallel/mpi/exception.hpp
+    ../../../boost/parallel/mpi/nonblocking.hpp
+    ../../../boost/parallel/mpi/operations.hpp
+    ../../../boost/parallel/mpi/packed_iarchive.hpp
+    ../../../boost/parallel/mpi/packed_oarchive.hpp
+    ../../../boost/parallel/mpi/skeleton_and_content.hpp
+    ../../../boost/parallel/mpi/skeleton_and_content_fwd.hpp
+    ../../../boost/parallel/mpi/status.hpp
+    ../../../boost/parallel/mpi/request.hpp
+    ../../../boost/parallel/mpi/timer.hpp
+    ../../../boost/parallel/mpi/python.hpp
+  : <doxygen:param>MACRO_EXPANSION=YES
+    <doxygen:param>MACRO_ONLY_PREDEF=YES
+    <doxygen:param>"PREDEFINED=BOOST_MPI_HAS_MEMORY_ALLOCATION= MPI_VERSION=2 BOOST_MPI_DOXYGEN="
+  ;
+
+boostbook mpi : mpi.qbk mpi_autodoc 
+              : <xsl:param>boost.root=http://www.boost.org
+              # Uncomment this line when generating PDF output
+              # <xsl:param>max-columns=66
+              ;
+

example/generate_collect.cpp

@@ -0,0 +1,129 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example using Boost.MPI's split() operation on communicators to
+// create separate data-generating processes and data-collecting
+// processes.
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <cstdlib>
+#include <boost/serialization/vector.hpp>
+namespace mpi = boost::parallel::mpi;
+
+enum message_tags { msg_data_packet, msg_broadcast_data, msg_finished };
+
+void generate_data(mpi::communicator local, mpi::communicator world)
+{
+  using std::srand;
+  using std::rand;
+
+  // The rank of the collector within the world communicator
+  int master_collector = local.size();
+
+  srand(time(0) + world.rank());
+
+  // Send out several blocks of random data to the collectors.
+  int num_data_blocks = rand() % 3 + 1;
+  for (int block = 0; block < num_data_blocks; ++block) {
+    // Generate some random data
+    int num_samples = rand() % 1000;
+    std::vector<int> data;
+    for (int i = 0; i < num_samples; ++i) {
+      data.push_back(rand());
+    }
+
+    // Send our data to the master collector process.
+    std::cout << "Generator #" << local.rank() << " sends some data..."
+              << std::endl;
+    world.send(master_collector, msg_data_packet, data);
+  }
+
+  // Wait for all of the generators to complete
+  (local.barrier)();
+
+  // The first generator will send the message to the master collector
+  // indicating that we're done.
+  if (local.rank() == 0)
+    world.send(master_collector, msg_finished);
+}
+
+void collect_data(mpi::communicator local, mpi::communicator world)
+{
+  // The rank of the collector within the world communicator
+  int master_collector = world.size() - local.size();
+
+  if (world.rank() == master_collector) {
+    while (true) {
+      // Wait for a message
+      mpi::status msg = world.probe();
+      if (msg.tag() == msg_data_packet) {
+        // Receive the packet of data
+        std::vector<int> data;
+        world.recv(msg.source(), msg.tag(), data);
+
+        // Tell each of the collectors that we'll be broadcasting some data
+        for (int dest = 1; dest < local.size(); ++dest)
+          local.send(dest, msg_broadcast_data, msg.source());
+
+        // Broadcast the actual data.
+        broadcast(local, data, 0);
+      } else if (msg.tag() == msg_finished) {
+        // Receive the message
+        world.recv(msg.source(), msg.tag());
+
+        // Tell each of the collectors that we're finished
+        for (int dest = 1; dest < local.size(); ++dest)
+          local.send(dest, msg_finished);
+
+        break;
+      }
+    }
+  } else {
+    while (true) {
+      // Wait for a message from the master collector
+      mpi::status msg = local.probe();
+      if (msg.tag() == msg_broadcast_data) {
+        // Receive the broadcast message
+        int originator;
+        local.recv(msg.source(), msg.tag(), originator);
+
+        // Receive the data broadcasted from the master collector
+        std::vector<int> data;
+        broadcast(local, data, 0);
+
+        std::cout << "Collector #" << local.rank()
+                  << " is processing data from generator #" << originator
+                  << "." << std::endl;
+      } else if (msg.tag() == msg_finished) {
+        // Receive the message
+        local.recv(msg.source(), msg.tag());
+
+        break;
+      }
+    }
+  }
+}
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  if (world.size() < 3) {
+    if (world.rank() == 0) {
+      std::cerr << "Error: this example requires at least 3 processes."
+                << std::endl;
+    }
+    env.abort(-1);
+  }
+
+  bool is_generator = world.rank() < 2 * world.size() / 3;
+  mpi::communicator local = world.split(is_generator? 0 : 1);
+  if (is_generator) generate_data(local, world);
+  else collect_data(local, world);
+
+  return 0;
+}

example/generate_collect_optional.cpp

@@ -0,0 +1,113 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example using Boost.MPI's split() operation on communicators to
+// create separate data-generating processes and data-collecting
+// processes using boost::optional for broadcasting.
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <cstdlib>
+#include <boost/serialization/vector.hpp>
+#include <boost/serialization/optional.hpp>
+namespace mpi = boost::parallel::mpi;
+
+enum message_tags { msg_data_packet, msg_finished };
+
+void generate_data(mpi::communicator local, mpi::communicator world)
+{
+  using std::srand;
+  using std::rand;
+
+  // The rank of the collector within the world communicator
+  int master_collector = local.size();
+
+  srand(time(0) + world.rank());
+
+  // Send out several blocks of random data to the collectors.
+  int num_data_blocks = rand() % 3 + 1;
+  for (int block = 0; block < num_data_blocks; ++block) {
+    // Generate some random dataa
+    int num_samples = rand() % 1000;
+    std::vector<int> data;
+    for (int i = 0; i < num_samples; ++i) {
+      data.push_back(rand());
+    }
+
+    // Send our data to the master collector process.
+    std::cout << "Generator #" << local.rank() << " sends some data..."
+              << std::endl;
+    world.send(master_collector, msg_data_packet, data);
+  }
+
+  // Wait for all of the generators to complete
+  (local.barrier)();
+
+  // The first generator will send the message to the master collector
+  // indicating that we're done.
+  if (local.rank() == 0)
+    world.send(master_collector, msg_finished);
+}
+
+void collect_data(mpi::communicator local, mpi::communicator world)
+{
+  // The rank of the collector within the world communicator
+  int master_collector = world.size() - local.size();
+
+  if (world.rank() == master_collector) {
+    while (true) {
+      // Wait for a message
+      mpi::status msg = world.probe();
+      if (msg.tag() == msg_data_packet) {
+        // Receive the packet of data into a boost::optional
+        boost::optional<std::vector<int> > data;
+        data = std::vector<int>();
+        world.recv(msg.source(), msg.source(), *data);
+
+        // Broadcast the actual data.
+        broadcast(local, data, 0);
+      } else if (msg.tag() == msg_finished) {
+        // Receive the message
+        world.recv(msg.source(), msg.tag());
+
+        // Broadcast to each collector to tell them we've finished.
+        boost::optional<std::vector<int> > data;
+        broadcast(local, data, 0);
+        break;
+      }
+    }
+  } else {
+    boost::optional<std::vector<int> > data;
+    do {
+      // Wait for a broadcast from the master collector
+      broadcast(local, data, 0);
+      if (data) {
+        std::cout << "Collector #" << local.rank()
+                  << " is processing data." << std::endl;
+      }
+    } while (data);
+  }
+}
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  if (world.size() < 4) {
+    if (world.rank() == 0) {
+      std::cerr << "Error: this example requires at least 4 processes."
+                << std::endl;
+    }
+    env.abort(-1);
+  }
+
+  bool is_generator = world.rank() < 2 * world.size() / 3;
+  mpi::communicator local = world.split(is_generator? 0 : 1);
+  if (is_generator) generate_data(local, world);
+  else collect_data(local, world);
+
+  return 0;
+}

example/hello_world.cpp

@@ -0,0 +1,33 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A simple Hello, world! example using Boost.MPI message passing.
+
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <boost/serialization/string.hpp> // Needed to send/receive strings!
+namespace mpi = boost::parallel::mpi;
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  if (world.rank() == 0) {
+    world.send(1, 0, std::string("Hello"));
+    std::string msg;
+    world.recv(1, 1, msg);
+    std::cout << msg << "!" << std::endl;
+  } else {
+    std::string msg;
+    world.recv(0, 0, msg);
+    std::cout << msg << ", ";
+    std::cout.flush();
+    world.send(0, 1, std::string("world"));
+  }
+
+  return 0;
+}

example/hello_world_broadcast.cpp

@@ -0,0 +1,28 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A simple Hello, world! example using Boost.MPI broadcast()
+
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <boost/serialization/string.hpp> // Needed to send/receive strings!
+namespace mpi = boost::parallel::mpi;
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  std::string value;
+  if (world.rank() == 0) {
+    value = "Hello, World!";
+  }
+
+  broadcast(world, value, 0);
+
+  std::cout << "Process #" << world.rank() << " says " << value << std::endl;
+  return 0;
+}

example/hello_world_nonblocking.cpp

@@ -0,0 +1,36 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A simple Hello, world! example using Boost.MPI message passing.
+
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <boost/serialization/string.hpp> // Needed to send/receive strings!
+namespace mpi = boost::parallel::mpi;
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  if (world.rank() == 0) {
+    mpi::request reqs[2];
+    std::string msg, out_msg = "Hello";
+    reqs[0] = world.isend(1, 0, out_msg);
+    reqs[1] = world.irecv(1, 1, msg);
+    mpi::wait_all(reqs, reqs + 2);
+    std::cout << msg << "!" << std::endl;
+  } else {
+    mpi::request reqs[2];
+    std::string msg, out_msg = "world";
+    reqs[0] = world.isend(0, 1, out_msg);
+    reqs[1] = world.irecv(0, 0, msg);
+    mpi::wait_all(reqs, reqs + 2);
+    std::cout << msg << ", ";
+  }
+
+  return 0;
+}

example/parallel_example.cpp

@@ -0,0 +1,196 @@
+// Copyright (C) 2005-2006 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example of a parallel Monte Carlo simulation using some nodes to produce
+// data and others to aggregate the data
+#include <iostream>
+
+#include <boost/parallel/mpi.hpp>
+#include <boost/random/parallel.hpp>
+#include <boost/random.hpp>
+#include <boost/foreach.hpp>
+#include <iostream>
+#include <cstdlib>
+
+namespace mpi = boost::parallel::mpi;
+
+enum {sample_tag, sample_skeleton_tag, sample_broadcast_tag, quit_tag};
+
+
+void calculate_samples(int sample_length)
+{
+  int num_samples = 100;
+  std::vector<double> sample(sample_length);
+
+  // setup communicator by splitting
+
+  mpi::communicator world;
+  mpi::communicator calculate_communicator = world.split(0);
+
+  unsigned int num_calculate_ranks = calculate_communicator.size();
+  
+  // the master of the accumulaion ranks is the first of them, hence
+  // with a rank just one after the last calculation rank
+  int master_accumulate_rank = num_calculate_ranks;
+  
+  // the master of the calculation ranks sends the skeleton of the sample 
+  // to the master of the accumulation ranks
+
+  if (world.rank()==0)
+    world.send(master_accumulate_rank,sample_skeleton_tag,mpi::skeleton(sample));
+  
+  // next we extract the content of the sample vector, to be used in sending
+  // the content later on
+  
+  mpi::content sample_content = mpi::get_content(sample);
+  
+  // now intialize the parallel random number generator
+  
+  boost::lcg64 engine(
+        boost::random::stream_number = calculate_communicator.rank(),
+        boost::random::total_streams = calculate_communicator.size()
+      );
+      
+  boost::variate_generator<boost::lcg64&,boost::uniform_real<> > 
+    rng(engine,boost::uniform_real<>());
+    
+  for (unsigned int i=0; i<num_samples/num_calculate_ranks+1;++i) {
+    
+    // calculate sample by filling the vector with random numbers
+    // note that std::generate will not work since it takes the generator
+    // by value, and boost::ref cannot be used as a generator.
+    // boost::ref should be fixed so that it can be used as generator
+    
+    BOOST_FOREACH(double& x, sample)
+      x = rng();
+    
+    // send sample to accumulation ranks
+    // Ideally we want to do this as a broadcast with an inter-communicator 
+    // between the calculation and accumulation ranks. MPI2 should support 
+    // this, but here we present an MPI1 compatible solution.
+    
+    // send content of sample to first (master) accumulation process
+    
+    world.send(master_accumulate_rank,sample_tag,sample_content);
+    
+    // gather some results from all calculation ranks
+    
+    double local_result = sample[0];
+    std::vector<double> gathered_results(calculate_communicator.size());
+    mpi::all_gather(calculate_communicator,local_result,gathered_results);
+  }
+  
+  // we are done: the master tells the accumulation ranks to quit
+  if (world.rank()==0)
+    world.send(master_accumulate_rank,quit_tag);
+}
+
+
+
+void accumulate_samples()
+{
+  std::vector<double> sample;
+
+  // setup the communicator for all accumulation ranks by splitting
+
+  mpi::communicator world;
+  mpi::communicator accumulate_communicator = world.split(1);
+
+  bool is_master_accumulate_rank = accumulate_communicator.rank()==0;
+
+  // the master receives the sample skeleton
+  
+  if (is_master_accumulate_rank) 
+    world.recv(0,sample_skeleton_tag,mpi::skeleton(sample));
+    
+  // and broadcasts it to all accumulation ranks
+  mpi::broadcast(accumulate_communicator,mpi::skeleton(sample),0);
+  
+  // next we extract the content of the sample vector, to be used in receiving
+  // the content later on
+  
+  mpi::content sample_content = mpi::get_content(sample);
+  
+  // accumulate until quit is called
+  double sum=0.;
+  while (true) {
+  
+    
+    // the accumulation master checks whether we should quit
+    if (world.iprobe(0,quit_tag)) {
+      world.recv(0,quit_tag);
+      for (int i=1; i<accumulate_communicator.size();++i)
+        accumulate_communicator.send(i,quit_tag);
+      std::cout << sum << "\n";
+      break; // We're done
+    }
+
+    // the otehr accumulation ranks check whether we should quit
+    if (accumulate_communicator.iprobe(0,quit_tag)) {
+      accumulate_communicator.recv(0,quit_tag);
+      std::cout << sum << "\n";
+      break; // We're done
+    }
+    
+    // check whether the master accumulation rank has received a sample
+    if (world.iprobe(mpi::any_source,sample_tag)) {
+      BOOST_ASSERT(is_master_accumulate_rank);
+      
+      // receive the content
+      world.recv(mpi::any_source,sample_tag,sample_content);
+      
+      // now we need to braodcast
+      // the problam is we do not have a non-blocking broadcast that we could 
+      // abort if we receive a quit message from the master. We thus need to
+      // first tell all accumulation ranks to start a broadcast. If the sample
+      // is small, we could just send the sample in this message, but here we
+      // optimize the code for large samples, so that the overhead of these
+      // sends can be ignored, and we count on an optimized broadcast
+      // implementation with O(log N) complexity
+
+      for (int i=1; i<accumulate_communicator.size();++i)
+        accumulate_communicator.send(i,sample_broadcast_tag);
+      
+      // now broadcast the contents of the sample to all accumulate ranks
+      mpi::broadcast(accumulate_communicator,sample_content,0);
+      
+      // and handle the sample by summing the appropriate value
+      sum += sample[0];
+    }
+  
+    // the other accumulation ranks wait for a mesage to start the broadcast
+    if (accumulate_communicator.iprobe(0,sample_broadcast_tag)) {
+      BOOST_ASSERT(!is_master_accumulate_rank);
+      
+      accumulate_communicator.recv(0,sample_broadcast_tag);
+      
+      // receive broadcast of the sample contents
+      mpi::broadcast(accumulate_communicator,sample_content,0);
+      
+      // and handle the sample
+      
+      // and handle the sample by summing the appropriate value
+      sum += sample[accumulate_communicator.rank()];
+    }
+  }
+}
+
+int main(int argc, char** argv)
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  // half of the processes generate, the others accumulate
+  // the sample size is just the number of accumulation ranks
+  if (world.rank() < world.size()/2)
+    calculate_samples(world.size()-world.size()/2);
+  else
+    accumulate_samples();
+
+  return 0;
+}
+
+

example/python/hello_world.py

@@ -0,0 +1,16 @@
+# Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+# Use, modification and distribution is subject to the Boost Software
+# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+# http:#www.boost.org/LICENSE_1_0.txt)
+
+import boost.parallel.mpi as mpi
+
+if mpi.world.rank == 0:
+  mpi.world.send(1, 0, 'Hello')
+  msg = mpi.world.recv(1, 1)
+  print msg,'!'
+else:
+  msg = mpi.world.recv(0, 0)
+  print msg,', ',
+  mpi.world.send(0, 1, 'world')

example/random_content.cpp

@@ -0,0 +1,98 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example using Boost.MPI's skeletons and content to optimize
+// communication.
+#include <boost/parallel/mpi.hpp>
+#include <boost/serialization/list.hpp>
+#include <algorithm>
+#include <functional>
+#include <numeric>
+#include <iostream>
+#include <stdlib.h>
+namespace mpi = boost::parallel::mpi;
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  if (world.size() < 2 || world.size() > 4) {
+    if (world.rank() == 0)
+      std::cerr << "error: please execute this program with 2-4 processes.\n";
+    world.abort(-1);
+  }
+
+  if (world.rank() == 0) {
+    int list_len = 50;
+    int iterations = 10;
+
+    if (argc > 1) list_len = atoi(argv[1]);
+    if (argc > 2) iterations = atoi(argv[2]);
+
+    if (list_len <= 0) {
+      std::cerr << "error: please specific a list length greater than zero.\n";
+      world.abort(-1);
+    }
+
+    // Generate the list and broadcast its structure
+    std::list<int> l(list_len);
+    broadcast(world, mpi::skeleton(l), 0);
+
+    // Generate content several times and broadcast out that content
+    mpi::content c = mpi::get_content(l);
+    for (int i = 0; i < iterations; ++i) {
+      do {
+        std::generate(l.begin(), l.end(), &random);
+      } while (std::find_if(l.begin(), l.end(),
+                            std::bind1st(std::not_equal_to<int>(), 0))
+               == l.end());
+
+
+      std::cout << "Iteration #" << i << ": sending content"
+                << " (min = " << *std::min_element(l.begin(), l.end())
+                << ", max = " << *std::max_element(l.begin(), l.end())
+                << ", avg = "
+                << std::accumulate(l.begin(), l.end(), 0)/l.size()
+                << ").\n";
+
+      broadcast(world, c, 0);
+    }
+
+    // Notify the slaves that we're done by sending all zeroes
+    std::fill(l.begin(), l.end(), 0);
+    broadcast(world, c, 0);
+
+  } else {
+    // Receive the content and build up our own list
+    std::list<int> l;
+    broadcast(world, mpi::skeleton(l), 0);
+
+    mpi::content c = mpi::get_content(l);
+    int i = 0;
+    do {
+      broadcast(world, c, 0);
+
+      if (std::find_if(l.begin(), l.end(),
+                       std::bind1st(std::not_equal_to<int>(), 0)) == l.end())
+        break;
+
+      if (world.rank() == 1)
+        std::cout << "Iteration #" << i << ": max value = "
+                  << *std::max_element(l.begin(), l.end()) << ".\n";
+      else if (world.rank() == 2)
+        std::cout << "Iteration #" << i << ": min value = "
+                  << *std::min_element(l.begin(), l.end()) << ".\n";
+      else if (world.rank() == 3)
+        std::cout << "Iteration #" << i << ": avg value = "
+                  << std::accumulate(l.begin(), l.end(), 0)/l.size()
+                  << ".\n";
+      ++i;
+    } while (true);
+  }
+
+  return 0;
+}

example/random_gather.cpp

@@ -0,0 +1,32 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example using Boost.MPI's gather()
+
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <cstdlib>
+namespace mpi = boost::parallel::mpi;
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  std::srand(time(0) + world.rank());
+  int my_number = std::rand();
+  if (world.rank() == 0) {
+    std::vector<int> all_numbers;
+    gather(world, my_number, all_numbers, 0);
+    for (int proc = 0; proc < world.size(); ++proc)
+      std::cout << "Process #" << proc << " thought of " << all_numbers[proc]
+                << std::endl;
+  } else {
+    gather(world, my_number, 0);
+  }
+
+  return 0;
+}

example/random_min.cpp

@@ -0,0 +1,30 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example using Boost.MPI's reduce() to compute a minimum value.
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <cstdlib>
+namespace mpi = boost::parallel::mpi;
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  std::srand(time(0) + world.rank());
+  int my_number = std::rand();
+
+  if (world.rank() == 0) {
+    int minimum;
+    reduce(world, my_number, minimum, mpi::minimum<int>(), 0);
+    std::cout << "The minimum value is " << minimum << std::endl;
+  } else {
+    reduce(world, my_number, mpi::minimum<int>(), 0);
+  }
+
+  return 0;
+}

example/reduce_performance_test.cpp

@@ -0,0 +1,138 @@
+// Copyright (C) 2006 Trustees of Indiana University
+//
+// Authors: Douglas Gregor
+//          Andrew Lumsdaine
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Performance test of the reduce() collective
+#include <boost/parallel/mpi.hpp>
+#include <boost/lexical_cast.hpp>
+
+namespace mpi = boost::parallel::mpi;
+
+struct add_int {
+  int operator()(int x, int y) const { return x + y; }
+};
+
+struct wrapped_int
+{
+  wrapped_int() : value(0) { }
+  wrapped_int(int value) : value(value) { }
+
+  template<typename Archiver>
+  void serialize(Archiver& ar, const unsigned int /*version*/) {
+    ar & value;
+  }
+
+  int value;
+};
+
+inline wrapped_int operator+(wrapped_int x, wrapped_int y)
+{
+  return wrapped_int(x.value + y.value);
+}
+
+namespace boost { namespace parallel { namespace mpi {
+  template<> struct is_mpi_datatype<wrapped_int> : mpl::true_ { };
+} } }
+
+struct serialized_int
+{
+  serialized_int() : value(0) { }
+  serialized_int(int value) : value(value) { }
+
+  template<typename Archiver>
+  void serialize(Archiver& ar, const unsigned int /*version*/) {
+    ar & value;
+  }
+
+  int value;
+};
+
+inline serialized_int operator+(serialized_int x, serialized_int y)
+{
+  return serialized_int(x.value + y.value);
+}
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  int repeat_count = 100;
+  int outer_repeat_count = 2;
+
+  if (argc > 1) repeat_count = boost::lexical_cast<int>(argv[1]);
+  if (argc > 2) outer_repeat_count = boost::lexical_cast<int>(argv[2]);
+
+  if (world.rank() == 0)
+    std::cout << "# of processors: " << world.size() << std::endl
+              << "# of iterations: " << repeat_count << std::endl;
+
+  int value = world.rank();
+  int result;
+  wrapped_int wi_value = world.rank();
+  wrapped_int wi_result;
+  serialized_int si_value = world.rank();
+  serialized_int si_result;
+
+  // Spin for a while...
+  for (int i = 0; i < repeat_count/10; ++i) {
+    reduce(world, value, result, std::plus<int>(), 0);
+    reduce(world, value, result, add_int(), 0);
+    reduce(world, wi_value, wi_result, std::plus<wrapped_int>(), 0);
+    reduce(world, si_value, si_result, std::plus<serialized_int>(), 0);
+  }
+
+  for (int outer = 0; outer < outer_repeat_count; ++outer) {
+    // Raw MPI
+    mpi::timer time;
+    for (int i = 0; i < repeat_count; ++i) {
+      MPI_Reduce(&value, &result, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+    }
+    double reduce_raw_mpi_total_time = time.elapsed();
+
+    // MPI_INT/MPI_SUM case
+    time.restart();
+    for (int i = 0; i < repeat_count; ++i) {
+      reduce(world, value, result, std::plus<int>(), 0);
+    }
+    double reduce_int_sum_total_time = time.elapsed();
+
+    // MPI_INT/MPI_Op case
+    time.restart();
+    for (int i = 0; i < repeat_count; ++i) {
+      reduce(world, value, result, add_int(), 0);
+    }
+    double reduce_int_op_total_time = time.elapsed();
+
+    // MPI_Datatype/MPI_Op case
+    time.restart();
+    for (int i = 0; i < repeat_count; ++i) {
+      reduce(world, wi_value, wi_result, std::plus<wrapped_int>(), 0);
+    }
+    double reduce_type_op_total_time = time.elapsed();
+  
+    // Serialized/MPI_Op case
+    time.restart();
+    for (int i = 0; i < repeat_count; ++i) {
+      reduce(world, si_value, si_result, std::plus<serialized_int>(), 0);
+    }
+    double reduce_ser_op_total_time = time.elapsed();
+
+
+    if (world.rank() == 0)
+      std::cout << "\nInvocation\tElapsed Time (seconds)"
+                << "\nRaw MPI\t\t\t" << reduce_raw_mpi_total_time
+                << "\nMPI_INT/MPI_SUM\t\t" << reduce_int_sum_total_time
+                << "\nMPI_INT/MPI_Op\t\t" << reduce_int_op_total_time
+                << "\nMPI_Datatype/MPI_Op\t" << reduce_type_op_total_time
+                << "\nSerialized/MPI_Op\t" << reduce_ser_op_total_time 
+                << std::endl;
+  }
+
+  return 0;
+}

example/string_cat.cpp

@@ -0,0 +1,46 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// An example using Boost.MPI's reduce() to concatenate strings.
+#include <boost/parallel/mpi.hpp>
+#include <iostream>
+#include <string>
+#include <boost/serialization/string.hpp> // Important for sending strings!
+namespace mpi = boost::parallel::mpi;
+
+/* Defining STRING_CONCAT_COMMUTATIVE lies to Boost.MPI by forcing it
+ * to assume that string concatenation is commutative, which it is
+ * not. However, doing so illustrates how the results of a reduction
+ * can change when a non-commutative operator is assumed to be
+ * commutative.
+ */
+#ifdef STRING_CONCAT_COMMUTATIVE
+namespace boost { namespace parallel { namespace mpi {
+
+template<>
+struct is_commutative<std::plus<std::string>, std::string> : mpl::true_ { };
+
+} } } // end namespace boost::parallel::mpi
+#endif
+
+int main(int argc, char* argv[])
+{
+  mpi::environment env(argc, argv);
+  mpi::communicator world;
+
+  std::string names[10] = { "zero ", "one ", "two ", "three ", "four ",
+                            "five ", "six ", "seven ", "eight ", "nine " };
+
+  std::string result;
+  reduce(world,
+         world.rank() < 10? names[world.rank()] : std::string("many "),
+         result, std::plus<std::string>(), 0);
+
+  if (world.rank() == 0)
+    std::cout << "The result is " << result << std::endl;
+
+  return 0;
+}

include/boost/mpi.hpp

@@ -0,0 +1,30 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface
+
+/** @file mpi.hpp
+ *
+ *  This file is a top-level convenience header that includes all of
+ *  the Boost.MPI library headers. Users concerned about compile time
+ *  may wish to include only specific headers from the Boost.MPI
+ *  library.
+ *
+ */
+#ifndef BOOST_MPI_HPP
+#define BOOST_MPI_HPP
+
+#include <boost/parallel/mpi/allocator.hpp>
+#include <boost/parallel/mpi/collectives.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/parallel/mpi/nonblocking.hpp>
+#include <boost/parallel/mpi/operations.hpp>
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+#include <boost/parallel/mpi/timer.hpp>
+
+#endif // BOOST_MPI_HPP

include/boost/mpi/allocator.hpp

@@ -0,0 +1,210 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file allocator.hpp
+ *
+ *  This header provides an STL-compliant allocator that uses the
+ *  MPI-2 memory allocation facilities.
+ */
+#ifndef BOOST_PARALLEL_MPI_ALLOCATOR_HPP
+#define BOOST_PARALLEL_MPI_ALLOCATOR_HPP
+
+#include <boost/parallel/mpi/config.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <cstddef>
+#include <memory>
+#include <boost/limits.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+#if defined(BOOST_MPI_HAS_MEMORY_ALLOCATION)
+template<typename T> class allocator;
+
+/** @brief Allocator specialization for @c void value types.
+ *
+ *  The @c void specialization of @c allocator is useful only for
+ *  rebinding to another, different value type.
+ */
+template<> 
+class allocator<void> 
+{ 
+public: 
+  typedef void* pointer; 
+  typedef const void* const_pointer; 
+  typedef void value_type; 
+
+  template <class U> 
+  struct rebind 
+  { 
+    typedef allocator<U> other; 
+  }; 
+};
+
+/** @brief Standard Library-compliant allocator for the MPI-2 memory
+ *  allocation routines.
+ *
+ *  This allocator provides a standard C++ interface to the @c
+ *  MPI_Alloc_mem and @c MPI_Free_mem routines of MPI-2. It is
+ *  intended to be used with the containers in the Standard Library
+ *  (@c vector, in particular) in cases where the contents of the
+ *  container will be directly transmitted via MPI. This allocator is
+ *  also used internally by the library for character buffers that
+ *  will be used in the transmission of data.
+ *
+ *  The @c allocator class template only provides MPI memory
+ *  allocation when the underlying MPI implementation is either MPI-2
+ *  compliant or is known to provide @c MPI_Alloc_mem and @c
+ *  MPI_Free_mem as extensions. When the MPI memory allocation
+ *  routines are not available, @c allocator is brought in directly
+ *  from namespace @c std, so that standard allocators are used
+ *  throughout. The macro @c BOOST_MPI_HAS_MEMORY_ALLOCATION will be
+ *  defined when the MPI-2 memory allocation facilities are available.
+ */
+template<typename T> 
+class allocator 
+{
+public:
+  /// Holds the size of objects
+  typedef std::size_t     size_type;
+
+  /// Holds the number of elements between two pointers
+  typedef std::ptrdiff_t  difference_type;
+
+  /// A pointer to an object of type @c T
+  typedef T*              pointer;
+
+  /// A pointer to a constant object of type @c T
+  typedef const T*        const_pointer;
+
+  /// A reference to an object of type @c T
+  typedef T&              reference;
+
+  /// A reference to a constant object of type @c T
+  typedef const T&        const_reference;
+
+  /// The type of memory allocated by this allocator
+  typedef T               value_type;
+
+  /** @brief Retrieve the type of an allocator similar to this
+   * allocator but for a different value type.
+   */
+  template <typename U> 
+  struct rebind 
+  { 
+    typedef allocator<U> other; 
+  };
+
+  /** Default-construct an allocator. */
+  allocator() throw() { }
+
+  /** Copy-construct an allocator. */
+  allocator(const allocator&) throw() { }
+
+  /** 
+   * Copy-construct an allocator from another allocator for a
+   * different value type.
+   */
+  template <typename U> 
+  allocator(const allocator<U>&) throw() { }
+
+  /** Destroy an allocator. */
+  ~allocator() throw() { }
+
+  /** Returns the address of object @p x. */
+  pointer address(reference x) const
+  {
+    return &x;
+  }
+
+  /** Returns the address of object @p x. */
+  const_pointer address(const_reference x) const
+  {
+    return &x;
+  }
+
+  /** 
+   *  Allocate enough memory for @p n elements of type @c T.
+   *
+   *  @param n The number of elements for which memory should be
+   *  allocated.
+   *
+   *  @return a pointer to the newly-allocated memory
+   */
+  pointer allocate(size_type n, allocator<void>::const_pointer /*hint*/ = 0)
+  {
+    pointer result;
+    BOOST_MPI_CHECK_RESULT(MPI_Alloc_mem,
+                           (static_cast<MPI_Aint>(n * sizeof(T)), 
+                            MPI_INFO_NULL, 
+                            &result));
+    return result;
+  }
+
+  /**
+   *  Deallocate memory referred to by the pointer @c p.
+   *
+   *  @param p The pointer whose memory should be deallocated. This
+   *  pointer shall have been returned from the @c allocate() function
+   *  and not have already been freed.
+   */
+  void deallocate(pointer p, size_type /*n*/)
+  {
+    BOOST_MPI_CHECK_RESULT(MPI_Free_mem, (p));
+  }
+
+  /** 
+   * Returns the maximum number of elements that can be allocated
+   * with @c allocate().
+   */
+  size_type max_size() const throw()
+  {
+    return (std::numeric_limits<std::size_t>::max)() / sizeof(T);
+  }
+
+  /** Construct a copy of @p val at the location referenced by @c p. */
+  void construct(pointer p, const T& val)
+  {
+    new ((void *)p) T(val);
+  }
+
+  /** Destroy the object referenced by @c p. */
+  void destroy(pointer p)
+  {
+    ((T*)p)->~T();
+  }
+};
+
+/** @brief Compare two allocators for equality.
+ *
+ *  Since MPI allocators have no state, all MPI allocators are equal.
+ *
+ *  @returns @c true
+ */
+template<typename T1, typename T2>
+inline bool operator==(const allocator<T1>&, const allocator<T2>&) throw()
+{
+  return true;
+}
+
+/** @brief Compare two allocators for inequality.
+ *
+ *  Since MPI allocators have no state, all MPI allocators are equal.
+ *
+ *  @returns @c false
+ */
+template<typename T1, typename T2>
+inline bool operator!=(const allocator<T1>&, const allocator<T2>&) throw()
+{
+  return false;
+}
+#else
+// Bring in the default allocator from namespace std.
+using std::allocator;
+#endif
+
+} } } /// end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_ALLOCATOR_HPP

include/boost/mpi/collectives.hpp

@@ -0,0 +1,377 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4. MPI Collectives
+
+/** @file collectives.hpp
+ *
+ *  This header contains MPI collective operations, which implement
+ *  various parallel algorithms that require the coordination of all
+ *  processes within a communicator. The header @c collectives_fwd.hpp
+ *  provides forward declarations for each of these operations. To
+ *  include only specific collective algorithms, use the headers @c
+ *  boost/parallel/mpi/collectives/algorithm_name.hpp.
+ */
+#ifndef BOOST_PARALLEL_MPI_COLLECTIVES_HPP
+#define BOOST_PARALLEL_MPI_COLLECTIVES_HPP
+
+#include <boost/parallel/mpi/communicator.hpp>
+#include <vector>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/**
+ *  @brief Gather the values stored at every process into vectors of
+ *  values from each process.
+ *
+ *  @c all_gather is a collective algorithm that collects the values
+ *  stored at each process into a vector of values indexed by the
+ *  process number they came from. The type @c T of the values may be
+ *  any type that is serializable or has an associated MPI data type.
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Allgather to gather the values.
+ *
+ *    @param comm The communicator over which the all-gather will
+ *    occur.
+ *
+ *    @param value The value to be transmitted by each process.
+ *
+ *    @param values A vector that will be populated with the values
+ *    from each process, indexed by the process ID number. This vector
+ *    will be resized accordingly.
+ */
+template<typename T>
+void
+all_gather(const communicator& comm, const T& value, std::vector<T>& values);
+
+/**
+ *  @brief Combine the values stored by each process into a single
+ *  value available to all processes.
+ *
+ *  @c all_reduce is a collective algorithm that combines the values
+ *  stored by each process into a single value available to all
+ *  processes. The values are combined in a user-defined way,
+ *  specified via a function object. The type @c T of the values may
+ *  be any type that is serializable or has an associated MPI data
+ *  type. One can think of this operation as a @c all_gather, followed
+ *  by an @c std::accumulate() over the gather values and using the
+ *  operation @c op.
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Allreduce to perform the reduction. If possible,
+ *  built-in MPI operations will be used; otherwise, @c all_reduce()
+ *  will create a custom MPI_Op for the call to MPI_Allreduce.
+ *
+ *    @param comm The communicator over which the reduction will
+ *    occur.
+ *
+ *    @param in_value The local value to be combined with the local
+ *    values of every other process.
+ *
+ *    @param out_value Will receive the result of the reduction
+ *    operation. If this parameter is omitted, the outgoing value will
+ *    instead be returned.
+ *
+ *    @param op The binary operation that combines two values of type
+ *    @c T and returns a third value of type @c T. For types @c T that has
+ *    ssociated MPI data types, @c op will either be translated into
+ *    an @c MPI_Op (via @c MPI_Op_create) or, if possible, mapped
+ *    directly to a built-in MPI operation. See @c is_mpi_op in the @c
+ *    operations.hpp header for more details on this mapping. For any
+ *    non-built-in operation, commutativity will be determined by the
+ *    @c is_commmutative trait (also in @c operations.hpp): users are
+ *    encouraged to mark commutative operations as such, because it
+ *    gives the implementation additional lattitude to optimize the
+ *    reduction operation.
+ *
+ *    @returns If no @p out_value parameter is supplied, returns the
+ *    result of the reduction operation.
+ */
+template<typename T, typename Op>
+void
+all_reduce(const communicator& comm, const T& in_value, T& out_value, Op op);
+
+/**
+ * \overload
+ */
+template<typename T, typename Op>
+T all_reduce(const communicator& comm, const T& in_value, Op op);
+
+/**
+ * @brief Broadcast a value from a root process to all other
+ * processes.
+ *
+ * @c broadcast is a collective algorithm that transfers a value from
+ * an arbitrary @p root process to every other process that is part of
+ * the given communicator. The @c broadcast algorithm can transmit any
+ * Serializable value, values that have associated MPI data types,
+ * packed archives, skeletons, and the content of skeletons; see the
+ * @c send primitive for communicators for a complete list. The type
+ * @c T shall be the same for all processes that are a part of the
+ * communicator @p comm, unless packed archives are being transferred:
+ * with packed archives, the root sends a @c packed_oarchive or @c
+ * packed_skeleton_oarchive whereas the other processes receive a
+ * @c packed_iarchive or @c packed_skeleton_iarchve, respectively.
+ *
+ * When the type @c T has an associated MPI data type, this routine
+ * invokes @c MPI_Bcast to perform the broadcast.
+ *
+ *   @param comm The communicator over which the broadcast will
+ *   occur.
+ *
+ *   @param value The value to be transmitted (if the rank of @p comm
+ *   is equal to @p root) or received (if the rank of @p comm is not
+ *   equal to @p root). When the @p value is a @c skeleton_proxy, only
+ *   the skeleton of the object will be broadcast. In this case, the
+ *   @p root will build a skeleton from the object help in the proxy
+ *   and all of the non-roots will reshape the objects held in their
+ *   proxies based on the skeleton sent from the root.
+ *
+ *   @param root The rank/process ID of the process that will be
+ *   transmitting the value.
+ */
+template<typename T>
+void broadcast(const communicator& comm, T& value, int root);
+
+/**
+ * \overload
+ */
+template<typename T>
+void broadcast(const communicator& comm, skeleton_proxy<T>& value, int root);
+
+/**
+ * \overload
+ */
+template<typename T>
+void
+broadcast(const communicator& comm, const skeleton_proxy<T>& value, int root);
+
+/**
+ *  @brief Gather the values stored at every process into a vector at
+ *  the root process.
+ *
+ *  @c gather is a collective algorithm that collects the values
+ *  stored at each process into a vector of values at the @p root
+ *  process. This vector is indexed by the process number that the
+ *  value came from. The type @c T of the values may be any type that
+ *  is serializable or has an associated MPI data type.
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Gather to gather the values.
+ *
+ *    @param comm The communicator over which the gather will occur.
+ *
+ *    @param value The value to be transmitted by each process.
+ *
+ *    @param values A vector that will be populated with the values
+ *    from each process, indexed by the process ID number. This vector
+ *    will be resized accordingly. For non-root processes, this
+ *    parameter may be omitted. If it is still provided, however, it
+ *    will be unchanged.
+ *
+ *    @param root The process ID number that will collect the
+ *    values. This value must be the same on all processes.
+ */
+template<typename T>
+void
+gather(const communicator& comm, const T& value, std::vector<T>& values,
+       int root);
+
+/**
+ * \overload
+ */
+template<typename T>
+void gather(const communicator& comm, const T& value, int root);
+
+/**
+ *  @brief Scatter the values stored at the root to all processes
+ *  within the communicator.
+ *
+ *  @c scatter is a collective algorithm that scatters the values
+ *  stored in the @p root process (inside a vector) to all of the
+ *  processes in the communicator. The vector @p values (only
+ *  significant at the @p root) is indexed by the process number to
+ *  which the corresponding value will be sent. The type @c T of the
+ *  values may be any type that is serializable or has an associated
+ *  MPI data type.
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Scatter to scatter the values.
+ *
+ *    @param comm The communicator over which the gather will occur.
+ *
+ *    @param values A vector that will contain the values to send to
+ *    each process, indexed by the process ID number. For non-root
+ *    processes, this parameter may be omitted. If it is still
+ *    provided, however, it will be unchanged.
+ *
+ *    @param value The value received by each process.
+ *
+ *    @param root The process ID number that will scatter the
+ *    values. This value must be the same on all processes.
+ */
+template<typename T>
+void
+scatter(const communicator& comm, const std::vector<T>& values, T& value,
+        int root);
+
+/**
+ * \overload
+ */
+template<typename T>
+void scatter(const communicator& comm, T& value, int root);
+
+/**
+ *  @brief Send data from every process to every other process.
+ *
+ *  @c all_to_all is a collective algorithm that transmits @c p values
+ *  from every process to every other process. On process i, jth value
+ *  of the @p in_values vector is sent to process j and placed in the
+ *  ith position of the @p out_values vector in process @p j. The type
+ *  @c T of the values may be any type that is serializable or has an
+ *  associated MPI data type.
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Alltoall to scatter the values.
+ *
+ *    @param comm The communicator over which the all-to-all
+ *    communication will occur.
+ *
+ *    @param in_values A vector that will containing the values to
+ *    send to each process, indexed by the process ID number.
+ *
+ *    @param out_values A vector that will be updated to contain the
+ *    values received from other processes. The jth value in the
+ *    vector will come from the procss with rank j.
+ */
+template<typename T>
+void
+all_to_all(const communicator& comm, const std::vector<T>& in_values,
+           std::vector<T>& out_values);
+
+/**
+ *  @brief Combine the values stored by each process into a single
+ *  value at the root.
+ *
+ *  @c reduce is a collective algorithm that combines the values
+ *  stored by each process into a single value at the @c root. The
+ *  values can be combined arbitrarily, specified via a function
+ *  object. The type @c T of the values may be any type that is
+ *  serializable or has an associated MPI data type. One can think of
+ *  this operation as a @c gather to the @p root, followed by an @c
+ *  std::accumulate() over the gathered values and using the operation
+ *  @c op.
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Reduce to perform the reduction. If possible,
+ *  built-in MPI operations will be used; otherwise, @c reduce() will
+ *  create a custom MPI_Op for the call to MPI_Reduce.
+ *
+ *    @param comm The communicator over which the reduction will
+ *    occur.
+ *
+ *    @param in_value The local value to be combined with the local
+ *    values of every other process.
+ *
+ *    @param out_value Will receive the result of the reduction
+ *    operation, but only for the @p root process. Non-root processes
+ *    may omit if parameter; if they choose to supply the parameter,
+ *    it will be unchanged.
+ *
+ *    @param op The binary operation that combines two values of type
+ *    @c T into a third value of type @c T. For types @c T that has
+ *    ssociated MPI data types, @c op will either be translated into
+ *    an @c MPI_Op (via @c MPI_Op_create) or, if possible, mapped
+ *    directly to a built-in MPI operation. See @c is_mpi_op in the @c
+ *    operations.hpp header for more details on this mapping. For any
+ *    non-built-in operation, commutativity will be determined by the
+ *    @c is_commmutative trait (also in @c operations.hpp): users are
+ *    encouraged to mark commutative operations as such, because it
+ *    gives the implementation additional lattitude to optimize the
+ *    reduction operation.
+ *
+ *    @param root The process ID number that will receive the final,
+ *    combined value. This value must be the same on all processes.
+ */
+template<typename T, typename Op>
+void
+reduce(const communicator& comm, const T& in_value, T& out_value, Op op,
+       int root);
+
+/**
+ * \overload
+ */
+template<typename T, typename Op>
+void reduce(const communicator& comm, const T& in_value, Op op, int root);
+
+/**
+ *  @brief Compute a prefix reduction of values from all processes in
+ *  the communicator.
+ *
+ *  @c scan is a collective algorithm that combines the values stored
+ *  by each process with the values of all processes with a smaller
+ *  rank. The values can be arbitrarily combined, specified via a
+ *  function object @p op. The type @c T of the values may be any type
+ *  that is serializable or has an associated MPI data type. One can
+ *  think of this operation as a @c gather to some process, followed
+ *  by an @c std::prefix_sum() over the gathered values using the
+ *  operation @c op. The ith process returns the ith value emitted by
+ *  @c std::prefix_sum().
+ *
+ *  When the type @c T has an associated MPI data type, this routine
+ *  invokes @c MPI_Scan to perform the reduction. If possible,
+ *  built-in MPI operations will be used; otherwise, @c scan() will
+ *  create a custom @c MPI_Op for the call to MPI_Scan.
+ *
+ *    @param comm The communicator over which the prefix reduction
+ *    will occur.
+ *
+ *    @param in_value The local value to be combined with the local
+ *    values of other processes.
+ *
+ *    @param out_value If provided, the ith process will receive the
+ *    value @c op(in_value[0], op(in_value[1], op(..., in_value[i])
+ *    ... )).
+ *
+ *    @param op The binary operation that combines two values of type
+ *    @c T into a third value of type @c T. For types @c T that has
+ *    ssociated MPI data types, @c op will either be translated into
+ *    an @c MPI_Op (via @c MPI_Op_create) or, if possible, mapped
+ *    directly to a built-in MPI operation. See @c is_mpi_op in the @c
+ *    operations.hpp header for more details on this mapping. For any
+ *    non-built-in operation, commutativity will be determined by the
+ *    @c is_commmutative trait (also in @c operations.hpp).
+ *
+ *    @returns If no @p out_value parameter is provided, returns the
+ *    result of prefix reduction.
+ */
+template<typename T, typename Op>
+void
+scan(const communicator& comm, const T& in_value, T& out_value, Op op);
+
+/**
+ * \overload
+ */
+template<typename T, typename Op>
+T
+scan(const communicator& comm, const T& in_value, Op op);
+
+} } } // end namespace boost::parallel::mpi
+#endif // BOOST_PARALLEL_MPI_COLLECTIVES_HPP
+
+#ifndef BOOST_PARALLEL_MPI_COLLECTIVES_FORWARD_ONLY
+// Include implementations of each of the collectives
+#  include <boost/parallel/mpi/collectives/all_gather.hpp>
+#  include <boost/parallel/mpi/collectives/all_reduce.hpp>
+#  include <boost/parallel/mpi/collectives/all_to_all.hpp>
+#  include <boost/parallel/mpi/collectives/broadcast.hpp>
+#  include <boost/parallel/mpi/collectives/gather.hpp>
+#  include <boost/parallel/mpi/collectives/scatter.hpp>
+#  include <boost/parallel/mpi/collectives/reduce.hpp>
+#  include <boost/parallel/mpi/collectives/scan.hpp>
+#endif
+

include/boost/mpi/collectives/all_gather.hpp

@@ -0,0 +1,59 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4.7. Gather-to-all
+#ifndef BOOST_PARALLEL_MPI_ALL_GATHER_HPP
+#define BOOST_PARALLEL_MPI_ALL_GATHER_HPP
+
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <vector>
+#include <boost/serialization/vector.hpp>
+
+// all_gather falls back to gather+broadcast in some cases
+#include <boost/parallel/mpi/collectives/broadcast.hpp>
+#include <boost/parallel/mpi/collectives/gather.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+namespace detail {
+  // We're all-gathering for a type that has an associated MPI
+  // datatype, so we'll use MPI_Gather to do all of the work.
+  template<typename T>
+  void
+  all_gather_impl(const communicator& comm, const T& value,
+                  std::vector<T>& values, mpl::true_)
+  {
+    values.resize(comm.size());
+
+    MPI_Datatype type = boost::parallel::mpi::get_mpi_datatype<T>();
+    BOOST_MPI_CHECK_RESULT(MPI_Allgather,
+                           (const_cast<T*>(&value), 1, type,
+                            &values[0], 1, type, comm));
+  }
+
+  // We're all-gathering for a type that has no associated MPI. So,
+  // we'll do a manual gather followed by a broadcast.
+  template<typename T>
+  void
+  all_gather_impl(const communicator& comm, const T& value,
+                  std::vector<T>& values, mpl::false_)
+  {
+    gather(comm, value, values, 0);
+    broadcast(comm, values, 0);
+  }
+} // end namespace detail
+
+template<typename T>
+void
+all_gather(const communicator& comm, const T& value, std::vector<T>& values)
+{
+  detail::all_gather_impl(comm, value, values, is_mpi_datatype<T>());
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_ALL_GATHER_HPP

include/boost/mpi/collectives/all_reduce.hpp

@@ -0,0 +1,94 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>
+// Copyright (C) 2004 The Trustees of Indiana University
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//   Authors: Douglas Gregor
+//            Andrew Lumsdaine
+
+// Message Passing Interface 1.1 -- Section 4.9.1. Reduce
+#ifndef BOOST_PARALLEL_MPI_ALL_REDUCE_HPP
+#define BOOST_PARALLEL_MPI_ALL_REDUCE_HPP
+
+// All-reduce falls back to reduce() + broadcast() in some cases.
+#include <boost/parallel/mpi/collectives/broadcast.hpp>
+#include <boost/parallel/mpi/collectives/reduce.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+namespace detail {
+  /**********************************************************************
+   * Simple reduction with MPI_Allreduce                                *
+   **********************************************************************/
+  // We are reducing for a type that has an associated MPI
+  // datatype and operation, so we'll use MPI_Allreduce directly.
+  template<typename T, typename Op>
+  void
+  all_reduce_impl(const communicator& comm, const T& in_value, T& out_value,
+                  Op op, mpl::true_ /*is_mpi_op*/,
+                  mpl::true_ /*is_mpi_datatype*/)
+  {
+    BOOST_MPI_CHECK_RESULT(MPI_Allreduce,
+                           (const_cast<T*>(&in_value), &out_value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            is_mpi_op<Op, T>::op(), comm));
+  }
+
+  /**********************************************************************
+   * User-defined reduction with MPI_Allreduce                          *
+   **********************************************************************/
+  // We are reducing at the root for a type that has an associated MPI
+  // datatype but with a custom operation. We'll use MPI_Reduce
+  // directly, but we'll need to create an MPI_Op manually.
+  template<typename T, typename Op>
+  void
+  all_reduce_impl(const communicator& comm, const T& in_value, T& out_value,
+                  Op op, mpl::false_ /*is_mpi_op*/,
+                  mpl::true_ /*is_mpi_datatype*/)
+  {
+    user_op<Op, T> mpi_op(op);
+    BOOST_MPI_CHECK_RESULT(MPI_Allreduce,
+                           (const_cast<T*>(&in_value), &out_value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            mpi_op.get_mpi_op(), comm));
+  }
+
+  /**********************************************************************
+   * User-defined, tree-based reduction for non-MPI data types          *
+   **********************************************************************/
+  // We are reducing at the root for a type that has no associated MPI
+  // datatype and operation, so we'll use a simple tree-based
+  // algorithm.
+  template<typename T, typename Op>
+  void
+  all_reduce_impl(const communicator& comm, const T& in_value, T& out_value,
+                  Op op, mpl::false_ /*is_mpi_op*/,
+                  mpl::false_ /*is_mpi_datatype*/)
+  {
+    reduce(comm, in_value, out_value, op, 0);
+    broadcast(comm, out_value, 0);
+  }
+} // end namespace detail
+
+
+template<typename T, typename Op>
+void
+all_reduce(const communicator& comm, const T& in_value, T& out_value, Op op)
+{
+  detail::all_reduce_impl(comm, in_value, out_value, op,
+                          is_mpi_op<Op, T>(), is_mpi_datatype<T>());
+}
+
+template<typename T, typename Op>
+T all_reduce(const communicator& comm, const T& in_value, Op op)
+{
+  T result;
+  all_reduce(comm, in_value, result, op);
+  return result;
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_ALL_REDUCE_HPP

include/boost/mpi/collectives/all_to_all.hpp

@@ -0,0 +1,129 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4.8. All-to-all
+#ifndef BOOST_PARALLEL_MPI_ALL_TO_ALL_HPP
+#define BOOST_PARALLEL_MPI_ALL_TO_ALL_HPP
+
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <vector>
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/assert.hpp>
+#include <boost/parallel/mpi/collectives_fwd.hpp>
+#include <boost/parallel/mpi/allocator.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+
+template<typename T>
+void
+all_to_all(const communicator& comm, const std::vector<T>& in_values,
+           std::vector<T>& out_values);
+           
+namespace detail {
+  // We're performaing an all-to-all with a type that has an
+  // associated MPI datatype, so we'll use MPI_Alltoall to do all of
+  // the work.
+  template<typename T>
+  void
+  all_to_all_impl(const communicator& comm, const std::vector<T>& in_values,
+                  std::vector<T>& out_values, mpl::true_)
+  {
+    MPI_Datatype type = get_mpi_datatype<T>();
+    out_values.resize(comm.size());
+    BOOST_MPI_CHECK_RESULT(MPI_Alltoall,
+                           (const_cast<T*>(&in_values.front()), 1, type,
+                            &out_values.front(), 1, type, comm));
+  }
+
+  // We're performing an all-to-all with a type that does not have an
+  // associated MPI datatype, so we'll need to serialize
+  // it. Unfortunately, this means that we cannot use MPI_Alltoall, so
+  // we'll just have to send individual messages to the other
+  // processes.
+  template<typename T>
+  void
+  all_to_all_impl(const communicator& comm, const std::vector<T>& in_values,
+                  std::vector<T>& out_values, mpl::false_)
+  {
+    int size = comm.size();
+    int rank = comm.rank();
+
+    out_values.resize(comm.size());
+
+    // The amount of data to be sent to each process
+    std::vector<int> send_sizes(size);
+
+    // The displacements for each outgoing value.
+    std::vector<int> send_disps(size);
+
+    // The buffer that will store all of the outgoing values
+    std::vector<char, allocator<char> > outgoing;
+
+    // Pack the buffer with all of the outgoing values.
+    for (int dest = 0; dest < size; ++dest) {
+      // Keep track of the displacements
+      send_disps[dest] = outgoing.size();
+
+      // Our own value will never be transmitted, so don't pack it.
+      if (dest != rank) {
+        packed_oarchive oa(comm, outgoing);
+        oa << in_values[dest];
+      }
+
+      // Keep track of the sizes
+      send_sizes[dest] = outgoing.size() - send_disps[dest];
+    }
+
+    // Determine how much data each process will receive.
+    std::vector<int> recv_sizes(size);
+    all_to_all(comm, send_sizes, recv_sizes);
+
+    // Prepare a buffer to receive the incoming data.
+    std::vector<int> recv_disps(size);
+    int sum = 0;
+    for (int src = 0; src < size; ++src) {
+      recv_disps[src] = sum;
+      sum += recv_sizes[src];
+    }
+    std::vector<char, allocator<char> > incoming(sum > 0? sum : 1);
+
+    // Transmit the actual data
+    BOOST_MPI_CHECK_RESULT(MPI_Alltoallv,
+                           (&outgoing[0], &send_sizes[0],
+                            &send_disps[0], MPI_PACKED,
+                            &incoming[0], &recv_sizes[0],
+                            &recv_disps[0], MPI_PACKED,
+                            comm));
+
+    // Deserialize data from the iarchive
+    for (int src = 0; src < size; ++src) {
+      if (src == rank) out_values[src] = in_values[src];
+      else {
+        packed_iarchive ia(comm, incoming, boost::archive::no_header,
+                           recv_disps[src]);
+        ia >> out_values[src];
+      }
+    }
+  }
+} // end namespace detail
+
+template<typename T>
+void
+all_to_all(const communicator& comm, const std::vector<T>& in_values,
+           std::vector<T>& out_values)
+{
+  BOOST_ASSERT((int)in_values.size() == comm.size());
+  detail::all_to_all_impl(comm, in_values, out_values, is_mpi_datatype<T>());
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_ALL_TO_ALL_HPP

include/boost/mpi/collectives/broadcast.hpp

@@ -0,0 +1,133 @@
+// Copyright (C) 2005, 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4.4. Broadcast
+#ifndef BOOST_PARALLEL_MPI_BROADCAST_HPP
+#define BOOST_PARALLEL_MPI_BROADCAST_HPP
+
+#include <boost/parallel/mpi/collectives_fwd.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+/************************************************************************
+ * Specializations                                                      *
+ ************************************************************************/
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void
+broadcast<const packed_oarchive>(const communicator& comm,
+                                 const packed_oarchive& oa,
+                                 int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void
+broadcast<packed_oarchive>(const communicator& comm, packed_oarchive& oa,
+                           int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void
+broadcast<packed_iarchive>(const communicator& comm, packed_iarchive& ia,
+                           int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void
+broadcast<const packed_skeleton_oarchive>(const communicator& comm,
+                                          const packed_skeleton_oarchive& oa,
+                                          int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void
+broadcast<packed_skeleton_oarchive>(const communicator& comm,
+                                    packed_skeleton_oarchive& oa, int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void
+broadcast<packed_skeleton_iarchive>(const communicator& comm,
+                                    packed_skeleton_iarchive& ia, int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void broadcast<content>(const communicator& comm, content& c, int root);
+
+/**
+ * INTERNAL ONLY
+ */
+template<>
+void broadcast<const content>(const communicator& comm, const content& c,
+                              int root);
+
+/************************************************************************
+ * broadcast() implementation                                           *
+ ************************************************************************/
+namespace detail {
+  // We're sending a type that has an associated MPI datatype, so
+  // we'll use MPI_Bcast to do all of the work.
+  template<typename T>
+  void broadcast_impl(const communicator& comm, T& value, int root, mpl::true_)
+  {
+    BOOST_MPI_CHECK_RESULT(MPI_Bcast,
+                           (&value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            root, MPI_Comm(comm)));
+  }
+
+  // We're sending a type that does not have an associated MPI
+  // datatype, so we'll need to serialize it. Unfortunately, this
+  // means that we cannot use MPI_Bcast, so we'll just send from the
+  // root to everyone else.
+  template<typename T>
+  void
+  broadcast_impl(const communicator& comm, T& value, int root, mpl::false_)
+  {
+    if (comm.rank() == root) {
+      packed_oarchive oa(comm);
+      oa << value;
+      broadcast(comm, oa, root);
+    } else {
+      packed_iarchive ia(comm);
+      broadcast(comm, ia, root);
+      ia >> value;
+    }
+  }
+} // end namespace detail
+
+template<typename T>
+void broadcast(const communicator& comm, T& value, int root = 0)
+{
+  detail::broadcast_impl(comm, value, root, is_mpi_datatype<T>());
+}
+
+} } } // end namespace boost::parallel::mpi
+
+// If the user has already included skeleton_and_content.hpp, include
+// the code to broadcast skeletons and content.
+#ifdef BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_HPP
+#  include <boost/parallel/mpi/detail/broadcast_sc.hpp>
+#endif
+
+#endif // BOOST_PARALLEL_MPI_BROADCAST_HPP

include/boost/mpi/collectives/gather.hpp

@@ -0,0 +1,125 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4.5. Gather
+#ifndef BOOST_PARALLEL_MPI_GATHER_HPP
+#define BOOST_PARALLEL_MPI_GATHER_HPP
+
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <vector>
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+namespace detail {
+  // We're gathering at the root for a type that has an associated MPI
+  // datatype, so we'll use MPI_Gather to do all of the work.
+  template<typename T>
+  void
+  gather_impl(const communicator& comm, const T& value, std::vector<T>& values,
+              int root, mpl::true_)
+  {
+    values.resize(comm.size());
+
+    MPI_Datatype type = get_mpi_datatype<T>();
+    BOOST_MPI_CHECK_RESULT(MPI_Gather,
+                           (const_cast<T*>(&value), 1, type,
+                            &values[0], 1, type, root, comm));
+  }
+
+  // We're gathering from a non-root for a type that has an associated MPI
+  // datatype, so we'll use MPI_Gather to do all of the work.
+  template<typename T>
+  void
+  gather_impl(const communicator& comm, const T& value, int root, mpl::true_)
+  {
+    MPI_Datatype type = get_mpi_datatype<T>();
+    BOOST_MPI_CHECK_RESULT(MPI_Gather,
+                           (const_cast<T*>(&value), 1, type,
+                            0, 1, type, root, comm));
+  }
+
+  // We're gathering at the root for a type that does not have an
+  // associated MPI datatype, so we'll need to serialize
+  // it. Unfortunately, this means that we cannot use MPI_Gather, so
+  // we'll just have all of the non-root nodes send individual
+  // messages to the root.
+  template<typename T>
+  void
+  gather_impl(const communicator& comm, const T& value, std::vector<T>& values,
+              int root, mpl::false_)
+  {
+    int tag = environment::collectives_tag();
+    int size = comm.size();
+
+    // We know we'll get "size" values
+    values.clear();
+    values.reserve(size);
+
+    MPI_Status status;
+    for (int src = 0; src < size; ++src) {
+      if (src == root) {
+        // Our own value will never be transmitted: just copy it.
+        values.push_back(value);
+      } else {
+        // Receive archive
+        packed_iarchive ia(comm);
+        detail::packed_archive_recv(comm, src, tag, ia, status);
+
+        // Unpack the data
+        T value_from_src;
+        ia >> value_from_src;
+
+        // Save the result
+        values.push_back(value_from_src);
+      }
+    }
+  }
+
+  // We're gathering at a non-root for a type that does not have an
+  // associated MPI datatype, so we'll need to serialize
+  // it. Unfortunately, this means that we cannot use MPI_Gather, so
+  // we'll just have all of the non-root nodes send individual
+  // messages to the root.
+  template<typename T>
+  void
+  gather_impl(const communicator& comm, const T& value, int root, mpl::false_)
+  {
+    int tag = environment::collectives_tag();
+
+    packed_oarchive oa(comm);
+    oa << value;
+    detail::packed_archive_send(comm, root, tag, oa);
+  }
+} // end namespace detail
+
+template<typename T>
+void
+gather(const communicator& comm, const T& value, std::vector<T>& values,
+       int root)
+{
+  if (comm.rank() == root)
+    detail::gather_impl(comm, value, values, root, is_mpi_datatype<T>());
+  else
+    detail::gather_impl(comm, value, root, is_mpi_datatype<T>());
+}
+
+template<typename T>
+void gather(const communicator& comm, const T& value, int root)
+{
+  BOOST_ASSERT(comm.rank() != root);
+  detail::gather_impl(comm, value, root, is_mpi_datatype<T>());
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_GATHER_HPP

include/boost/mpi/collectives/reduce.hpp

@@ -0,0 +1,319 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor@gmail.com>.
+// Copyright (C) 2004 The Trustees of Indiana University
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//   Authors: Douglas Gregor
+//            Andrew Lumsdaine
+
+// Message Passing Interface 1.1 -- Section 4.9.1. Reduce
+#ifndef BOOST_PARALLEL_MPI_REDUCE_HPP
+#define BOOST_PARALLEL_MPI_REDUCE_HPP
+
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+
+// For (de-)serializing sends and receives
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+
+// For packed_[io]archive sends and receives
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/parallel/mpi/detail/computation_tree.hpp>
+#include <boost/parallel/mpi/operations.hpp>
+#include <algorithm>
+#include <exception>
+#include <boost/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+
+/************************************************************************
+ * Implementation details                                               *
+ ************************************************************************/
+namespace detail {
+  /**********************************************************************
+   * Simple reduction with MPI_Reduce                                   *
+   **********************************************************************/
+  // We are reducing at the root for a type that has an associated MPI
+  // datatype and operation, so we'll use MPI_Reduce directly.
+  template<typename T, typename Op>
+  void
+  reduce_impl(const communicator& comm, const T& in_value, T& out_value, Op op,
+              int root, mpl::true_ /*is_mpi_op*/,
+              mpl::true_/*is_mpi_datatype*/)
+  {
+    BOOST_MPI_CHECK_RESULT(MPI_Reduce,
+                           (const_cast<T*>(&in_value), &out_value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            is_mpi_op<Op, T>::op(), root, comm));
+  }
+
+  // We are reducing to the root for a type that has an associated MPI
+  // datatype and operation, so we'll use MPI_Reduce directly.
+  template<typename T, typename Op>
+  void
+  reduce_impl(const communicator& comm, const T& in_value, Op op, int root,
+              mpl::true_ /*is_mpi_op*/, mpl::true_/*is_mpi_datatype*/)
+  {
+    BOOST_MPI_CHECK_RESULT(MPI_Reduce,
+                           (const_cast<T*>(&in_value), 0, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            is_mpi_op<Op, T>::op(), root, comm));
+  }
+
+  // We are reducing at the root for a type that has an associated MPI
+  // datatype but with a custom operation. We'll use MPI_Reduce
+  // directly, but we'll need to create an MPI_Op manually.
+  template<typename T, typename Op>
+  void
+  reduce_impl(const communicator& comm, const T& in_value, T& out_value,
+              Op op, int root, mpl::false_ /*is_mpi_op*/,
+              mpl::true_/*is_mpi_datatype*/)
+  {
+    user_op<Op, T> mpi_op(op);
+    BOOST_MPI_CHECK_RESULT(MPI_Reduce,
+                           (const_cast<T*>(&in_value), &out_value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            mpi_op.get_mpi_op(), root, comm));
+  }
+
+  /**********************************************************************
+   * User-defined reduction with MPI_Reduce                             *
+   **********************************************************************/
+
+  // We are reducing to the root for a type that has an associated MPI
+  // datatype but with a custom operation. We'll use MPI_Reduce
+  // directly, but we'll need to create an MPI_Op manually.
+  template<typename T, typename Op>
+  void
+  reduce_impl(const communicator& comm, const T& in_value, Op op, int root,
+              mpl::false_ /*is_mpi_op*/, mpl::true_/*is_mpi_datatype*/)
+  {
+    user_op<Op, T> mpi_op(op);
+    BOOST_MPI_CHECK_RESULT(MPI_Reduce,
+                           (const_cast<T*>(&in_value), 0, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            mpi_op.get_mpi_op(), root, comm));
+  }
+
+  /**********************************************************************
+   * User-defined, tree-based reduction for non-MPI data types          *
+   **********************************************************************/
+
+  // Commutative reduction
+  template<typename T, typename Op>
+  void
+  tree_reduce_impl(const communicator& comm, const T& in_value, T& out_value,
+                   Op op, int root, mpl::true_ /*is_commutative*/)
+  {
+    out_value = in_value;
+
+    int size = comm.size();
+    int rank = comm.rank();
+
+    // The computation tree we will use.
+    detail::computation_tree tree(rank, size, root);
+
+    int tag = environment::collectives_tag();
+
+    MPI_Status status;
+    int children = 0;
+    for (int child = tree.child_begin();
+         children < tree.branching_factor() && child != root;
+         ++children, child = (child + 1) % size) {
+      // Receive archive
+      packed_iarchive ia(comm);
+      detail::packed_archive_recv(comm, child, tag, ia, status);
+
+      T incoming;
+      ia >> incoming;
+
+      out_value = op(out_value, incoming);
+    }
+
+    // For non-roots, send the result to the parent.
+    if (tree.parent() != rank) {
+      packed_oarchive oa(comm);
+      oa << out_value;
+      detail::packed_archive_send(comm, tree.parent(), tag, oa);
+    }
+  }
+
+  // Commutative reduction from a non-root.
+  template<typename T, typename Op>
+  void
+  tree_reduce_impl(const communicator& comm, const T& in_value, Op op,
+                   int root, mpl::true_ /*is_commutative*/)
+  {
+    T result;
+    detail::tree_reduce_impl(comm, in_value, result, op, root, mpl::true_());
+  }
+
+  // Non-commutative reduction
+  template<typename T, typename Op>
+  void
+  tree_reduce_impl(const communicator& comm, const T& in_value, T& out_value,
+                   Op op, int root, mpl::false_ /*is_commutative*/)
+  {
+    int tag = environment::collectives_tag();
+
+    int left_child = root / 2;
+    int right_child = (root + comm.size()) / 2;
+
+    MPI_Status status;
+    if (left_child != root) {
+      // Receive value from the left child and merge it with the value
+      // we had incoming.
+      packed_iarchive ia(comm);
+      detail::packed_archive_recv(comm, left_child, tag, ia, status);
+      T incoming;
+      ia >> incoming;
+      out_value = op(incoming, in_value);
+    } else {
+      // There was no left value, so copy our incoming value.
+      out_value = in_value;
+    }
+
+    if (right_child != root) {
+      // Receive value from the right child and merge it with the
+      // value we had incoming.
+      packed_iarchive ia(comm);
+      detail::packed_archive_recv(comm, right_child, tag, ia, status);
+      T incoming;
+      ia >> incoming;
+      out_value = op(out_value, incoming);
+    }
+  }
+
+  // Non-commutative reduction from a non-root.
+  template<typename T, typename Op>
+  void
+  tree_reduce_impl(const communicator& comm, const T& in_value, Op op,
+                   int root, mpl::false_ /*is_commutative*/)
+  {
+    int size = comm.size();
+    int rank = comm.rank();
+
+    int tag = environment::collectives_tag();
+
+    // Determine our parents and children in the commutative binary
+    // computation tree.
+    int grandparent = root;
+    int parent = root;
+    int left_bound = 0;
+    int right_bound = size;
+    int left_child, right_child;
+    do {
+      left_child = (left_bound + parent) / 2;
+      right_child = (parent + right_bound) / 2;
+
+      if (rank < parent) {
+        // Go left.
+        grandparent = parent;
+        right_bound = parent;
+        parent = left_child;
+      } else if (rank > parent) {
+        // Go right.
+        grandparent = parent;
+        left_bound = parent + 1;
+        parent = right_child;
+      } else {
+        // We've found the parent/
+        break;
+      }
+    } while (true);
+
+    // Our parent is the grandparent of our children. This is a slight
+    // abuse of notation, but it makes the send-to-parent below make
+    // more sense.
+    parent = grandparent;
+
+    MPI_Status status;
+    T out_value;
+    if (left_child != rank) {
+      // Receive value from the left child and merge it with the value
+      // we had incoming.
+      packed_iarchive ia(comm);
+      detail::packed_archive_recv(comm, left_child, tag, ia, status);
+      T incoming;
+      ia >> incoming;
+      out_value = op(incoming, in_value);
+    } else {
+      // There was no left value, so copy our incoming value.
+      out_value = in_value;
+    }
+
+    if (right_child != rank) {
+      // Receive value from the right child and merge it with the
+      // value we had incoming.
+      packed_iarchive ia(comm);
+      detail::packed_archive_recv(comm, right_child, tag, ia, status);
+      T incoming;
+      ia >> incoming;
+      out_value = op(out_value, incoming);
+    }
+
+    // Send the combined value to our parent.
+    packed_oarchive oa(comm);
+    oa << out_value;
+    detail::packed_archive_send(comm, parent, tag, oa);
+  }
+
+  // We are reducing at the root for a type that has no associated MPI
+  // datatype and operation, so we'll use a simple tree-based
+  // algorithm.
+  template<typename T, typename Op>
+  void
+  reduce_impl(const communicator& comm, const T& in_value, T& out_value, Op op,
+              int root, mpl::false_ /*is_mpi_op*/,
+              mpl::false_/*is_mpi_datatype*/)
+  {
+    detail::tree_reduce_impl(comm, in_value, out_value, op, root,
+                             is_commutative<Op, T>());
+  }
+
+  // We are reducing to the root for a type that has no associated MPI
+  // datatype and operation, so we'll use a simple tree-based
+  // algorithm.
+  template<typename T, typename Op>
+  void
+  reduce_impl(const communicator& comm, const T& in_value, Op op, int root,
+              mpl::false_ /*is_mpi_op*/, mpl::false_/*is_mpi_datatype*/)
+  {
+    detail::tree_reduce_impl(comm, in_value, op, root,
+                             is_commutative<Op, T>());
+  }
+} // end namespace detail
+
+
+template<typename T, typename Op>
+void
+reduce(const communicator& comm, const T& in_value, T& out_value, Op op,
+       int root)
+{
+  if (comm.rank() == root)
+    detail::reduce_impl(comm, in_value, out_value, op, root,
+                        is_mpi_op<Op, T>(), is_mpi_datatype<T>());
+  else
+    detail::reduce_impl(comm, in_value, op, root,
+                        is_mpi_op<Op, T>(), is_mpi_datatype<T>());
+}
+
+template<typename T, typename Op>
+void reduce(const communicator& comm, const T& in_value, Op op, int root)
+{
+  BOOST_ASSERT(comm.rank() != root);
+
+  detail::reduce_impl(comm, in_value, op, root,
+                      is_mpi_op<Op, T>(), is_mpi_datatype<T>());
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_REDUCE_HPP

include/boost/mpi/collectives/scan.hpp

@@ -0,0 +1,151 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor@gmail.com>.
+// Copyright (C) 2004 The Trustees of Indiana University
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//   Authors: Douglas Gregor
+//            Andrew Lumsdaine
+
+// Message Passing Interface 1.1 -- Section 4.9.1. Scan
+#ifndef BOOST_PARALLEL_MPI_SCAN_HPP
+#define BOOST_PARALLEL_MPI_SCAN_HPP
+
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+
+// For (de-)serializing sends and receives
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+
+// For packed_[io]archive sends and receives
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/parallel/mpi/detail/computation_tree.hpp>
+#include <boost/parallel/mpi/operations.hpp>
+#include <algorithm>
+#include <exception>
+#include <boost/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+
+/************************************************************************
+ * Implementation details                                               *
+ ************************************************************************/
+namespace detail {
+  /**********************************************************************
+   * Simple prefix reduction with MPI_Scan                              *
+   **********************************************************************/
+
+  // We are performing prefix reduction for a type that has an
+  // associated MPI datatype and operation, so we'll use MPI_Scan
+  // directly.
+  template<typename T, typename Op>
+  void
+  scan_impl(const communicator& comm, const T& in_value, T& out_value, Op op,
+            mpl::true_ /*is_mpi_op*/, mpl::true_ /*is_mpi_datatype*/)
+  {
+    BOOST_MPI_CHECK_RESULT(MPI_Scan,
+                           (const_cast<T*>(&in_value), &out_value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            is_mpi_op<Op, T>::op(), comm));
+  }
+
+  /**********************************************************************
+   * User-defined prefix reduction with MPI_Scan                        *
+   **********************************************************************/
+
+  // We are performing prefix reduction for a type that has an
+  // associated MPI datatype but with a custom operation. We'll use
+  // MPI_Scan directly, but we'll need to create an MPI_Op manually.
+  template<typename T, typename Op>
+  void
+  scan_impl(const communicator& comm, const T& in_value, T& out_value,
+            Op op, mpl::false_ /*is_mpi_op*/, mpl::true_ /*is_mpi_datatype*/)
+  {
+    user_op<Op, T> mpi_op(op);
+    BOOST_MPI_CHECK_RESULT(MPI_Scan,
+                           (const_cast<T*>(&in_value), &out_value, 1,
+                            boost::parallel::mpi::get_mpi_datatype<T>(),
+                            mpi_op.get_mpi_op(), comm));
+  }
+
+  /**********************************************************************
+   * User-defined, tree-based reduction for non-MPI data types          *
+   **********************************************************************/
+
+  template<typename T, typename Op>
+  void
+  upper_lower_scan(const communicator& comm, const T& in_value, 
+                   T& out_value, Op& op, int lower, int upper)
+  {
+    int tag = environment::collectives_tag();
+    int rank = comm.rank();
+
+    if (lower + 1 == upper) {
+      out_value = in_value;
+    } else {
+      int middle = (lower + upper) / 2;
+      
+      if (rank < middle) {
+        // Lower half
+        upper_lower_scan(comm, in_value, out_value, op, lower, middle);
+
+        // If we're the last process in the lower half, send our value
+        // to everyone in the upper half.
+        if (rank == middle - 1) {
+          for (int p = middle; p < upper; ++p)
+            comm.send(p, tag, out_value);
+        }
+      } else {
+        // Upper half
+        upper_lower_scan(comm, in_value, out_value, op, middle, upper);
+
+        // Receive value from the last process in the lower half.
+        T left_value;
+        comm.recv(middle - 1, tag, left_value);
+
+        // Combine value that came from the left with our value
+        out_value = op(left_value, out_value);
+      }
+    }
+  }
+
+  // We are performing prefix reduction for a type that has no
+  // associated MPI datatype and operation, so we'll use a simple
+  // upper/lower algorithm.
+  template<typename T, typename Op>
+  inline void
+  scan_impl(const communicator& comm, const T& in_value, T& out_value, Op op,
+            mpl::false_ /*is_mpi_op*/, mpl::false_/*is_mpi_datatype*/)
+  {
+    upper_lower_scan(comm, in_value, out_value, op, 0, comm.size());
+  }
+} // end namespace detail
+
+
+template<typename T, typename Op>
+inline void
+scan(const communicator& comm, const T& in_value, T& out_value, Op op)
+{
+  detail::scan_impl(comm, in_value, out_value, op, 
+                    is_mpi_op<Op, T>(), is_mpi_datatype<T>());
+}
+
+template<typename T, typename Op>
+inline T
+scan(const communicator& comm, const T& in_value, Op op)
+{
+  T out_value;
+  detail::scan_impl(comm, in_value, out_value, op, 
+                    is_mpi_op<Op, T>(), is_mpi_datatype<T>());
+  return out_value;
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_SCAN_HPP

include/boost/mpi/collectives/scatter.hpp

@@ -0,0 +1,114 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4.6. Scatter
+#ifndef BOOST_PARALLEL_MPI_SCATTER_HPP
+#define BOOST_PARALLEL_MPI_SCATTER_HPP
+
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <vector>
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+namespace detail {
+  // We're scattering from the root for a type that has an associated MPI
+  // datatype, so we'll use MPI_Scatter to do all of the work.
+  template<typename T>
+  void
+  scatter_impl(const communicator& comm, const std::vector<T>& values,
+               T& value, int root, mpl::true_)
+  {
+    MPI_Datatype type = get_mpi_datatype<T>();
+    BOOST_MPI_CHECK_RESULT(MPI_Scatter,
+                           (const_cast<T*>(&values.front()), 1, type,
+                            &value, 1, type, root, comm));
+  }
+
+  // We're scattering from a non-root for a type that has an associated MPI
+  // datatype, so we'll use MPI_Scatter to do all of the work.
+  template<typename T>
+  void
+  scatter_impl(const communicator& comm, T& value, int root, mpl::true_)
+  {
+    MPI_Datatype type = get_mpi_datatype<T>();
+    BOOST_MPI_CHECK_RESULT(MPI_Scatter,
+                           (0, 1, type,
+                            &value, 1, type,
+                            root, comm));
+  }
+
+  // We're scattering from the root for a type that does not have an
+  // associated MPI datatype, so we'll need to serialize
+  // it. Unfortunately, this means that we cannot use MPI_Scatter, so
+  // we'll just have the root send individual messages to the other
+  // processes.
+  template<typename T>
+  void
+  scatter_impl(const communicator& comm, const std::vector<T>& values,
+               T& value, int root, mpl::false_)
+  {
+    int tag = environment::collectives_tag();
+    int size = comm.size();
+
+    for (int dest = 0; dest < size; ++dest) {
+      if (dest == root) {
+        // Our own value will never be transmitted: just copy it.
+        value = values[dest];
+      } else {
+        // Send archive
+        packed_oarchive oa(comm);
+        oa << values[dest];
+        detail::packed_archive_send(comm, dest, tag, oa);
+      }
+    }
+  }
+
+  // We're scattering to a non-root for a type that does not have an
+  // associated MPI datatype, so we'll need to de-serialize
+  // it. Unfortunately, this means that we cannot use MPI_Scatter, so
+  // we'll just have all of the non-root nodes send individual
+  // messages to the root.
+  template<typename T>
+  void
+  scatter_impl(const communicator& comm, T& value, int root, mpl::false_)
+  {
+    int tag = environment::collectives_tag();
+
+    packed_iarchive ia(comm);
+    MPI_Status status;
+    detail::packed_archive_recv(comm, root, tag, ia, status);
+    ia >> value;
+  }
+} // end namespace detail
+
+template<typename T>
+void
+scatter(const communicator& comm, const std::vector<T>& values, T& value,
+        int root)
+{
+  if (comm.rank() == root)
+    detail::scatter_impl(comm, values, value, root, is_mpi_datatype<T>());
+  else
+    detail::scatter_impl(comm, value, root, is_mpi_datatype<T>());
+}
+
+template<typename T>
+void scatter(const communicator& comm, T& value, int root)
+{
+  BOOST_ASSERT(comm.rank() != root);
+  detail::scatter_impl(comm, value, root, is_mpi_datatype<T>());
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_SCATTER_HPP

include/boost/mpi/collectives_fwd.hpp

@@ -0,0 +1,23 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4. MPI Collectives
+
+/** @file collectives_fwd.hpp
+ *
+ *  This header provides forward declarations for all of the
+ *  collective operations contained in the header @c collectives.hpp.
+ */
+#ifndef BOOST_PARALLEL_MPI_COLLECTIVES_FWD_HPP
+#define BOOST_PARALLEL_MPI_COLLECTIVES_FWD_HPP
+
+/// INTERNAL ONLY
+#define BOOST_PARALLEL_MPI_COLLECTIVES_FORWARD_ONLY
+#include <boost/parallel/mpi/collectives.hpp>
+#undef BOOST_PARALLEL_MPI_COLLECTIVES_FORWARD_ONLY
+
+#endif // BOOST_PARALLEL_MPI_COLLECTIVES_FWD_HPP
+

include/boost/mpi/config.hpp

@@ -0,0 +1,39 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file config.hpp
+ *
+ *  This header provides MPI configuration details that expose the
+ *  capabilities of the underlying MPI implementation.
+ */
+#ifndef BOOST_PARALLEL_MPI_CONFIG_HPP
+#define BOOST_PARALLEL_MPI_CONFIG_HPP
+
+#include <mpi.h>
+
+// If this is an MPI-2 implementation, define configuration macros for
+// the features we are interested in.
+#if defined(MPI_VERSION) && MPI_VERSION == 2
+/** @brief Determine if the MPI implementation has support for memory
+ *  allocation.
+ *
+ *  This macro will be defined when the underlying MPI implementation
+ *  has support for the MPI-2 memory allocation routines @c
+ *  MPI_Alloc_mem and @c MPI_Free_mem. When defined, the @c allocator
+ *  class template will provide Standard Library-compliant access to
+ *  these memory-allocation routines.
+ */
+#  define BOOST_MPI_HAS_MEMORY_ALLOCATION
+#endif
+
+#if defined(LAM_MPI)
+// Configuration for LAM/MPI
+#  define BOOST_MPI_HAS_MEMORY_ALLOCATION
+#elif defined(MPICH_NAME)
+// Configuration for MPICH
+#endif
+
+#endif // BOOST_PARALLEL_MPI_CONFIG_HPP

include/boost/mpi/datatype.hpp

@@ -0,0 +1,293 @@
+// Copyright 2004 The Trustees of Indiana University.
+// Copyright 2005 Matthias Troyer.
+// Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+//           Andrew Lumsdaine
+//           Matthias Troyer
+
+/** @file datatype.hpp
+ *
+ *  This header provides the mapping from C++ types to MPI data types.
+ */
+#ifndef BOOST_PARALLEL_MPI_DATATYPE_HPP
+#define BOOST_PARALLEL_MPI_DATATYPE_HPP
+
+#include <boost/parallel/mpi/datatype_fwd.hpp>
+#include <mpi.h>
+#include <boost/config.hpp>
+#include <boost/mpl/bool.hpp>
+#include <boost/mpl/or.hpp>
+#include <boost/parallel/mpi/detail/mpi_datatype_cache.hpp>
+#include <boost/mpl/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/**
+ *  @brief Type trait that determines if there exists a built-in
+ *  integer MPI data type for a given C++ type.
+ *
+ *  This ytpe trait determines when there is a direct mapping from a
+ *  C++ type to an MPI data type that is classified as an integer data
+ *  type. See @c is_mpi_builtin_datatype for general information about
+ *  built-in MPI data types.
+ */
+template<typename T>
+struct is_mpi_integer_datatype
+  : public boost::mpl::false_ { };
+
+/**
+ *  @brief Type trait that determines if there exists a built-in
+ *  floating point MPI data type for a given C++ type.
+ *
+ *  This ytpe trait determines when there is a direct mapping from a
+ *  C++ type to an MPI data type that is classified as a floating
+ *  point data type. See @c is_mpi_builtin_datatype for general
+ *  information about built-in MPI data types.
+ */
+template<typename T>
+struct is_mpi_floating_point_datatype
+  : public boost::mpl::false_ { };
+
+/**
+ *  @brief Type trait that determines if there exists a built-in
+ *  logical MPI data type for a given C++ type.
+ *
+ *  This ytpe trait determines when there is a direct mapping from a
+ *  C++ type to an MPI data type that is classified as an logical data
+ *  type. See @c is_mpi_builtin_datatype for general information about
+ *  built-in MPI data types.
+ */
+template<typename T>
+struct is_mpi_logical_datatype
+  : public boost::mpl::false_ { };
+
+/**
+ *  @brief Type trait that determines if there exists a built-in
+ *  complex MPI data type for a given C++ type.
+ *
+ *  This ytpe trait determines when there is a direct mapping from a
+ *  C++ type to an MPI data type that is classified as an complex data
+ *  type. See @c is_mpi_builtin_datatype for general information about
+ *  built-in MPI data types.
+ */
+template<typename T>
+struct is_mpi_complex_datatype
+  : public boost::mpl::false_ { };
+
+/**
+ *  @brief Type trait that determines if there exists a built-in
+ *  byte MPI data type for a given C++ type.
+ *
+ *  This ytpe trait determines when there is a direct mapping from a
+ *  C++ type to an MPI data type that is classified as an byte data
+ *  type. See @c is_mpi_builtin_datatype for general information about
+ *  built-in MPI data types.
+ */
+template<typename T>
+struct is_mpi_byte_datatype
+  : public boost::mpl::false_ { };
+
+/** @brief Type trait that determines if there exists a built-in MPI
+ *  data type for a given C++ type.
+ *
+ *  This type trait determines when there is a direct mapping from a
+ *  C++ type to an MPI type. For instance, the C++ @c int type maps
+ *  directly to the MPI type @c MPI_INT. When there is a direct
+ *  mapping from the type @c T to an MPI type, @c
+ *  is_mpi_builtin_datatype will derive from @c mpl::true_ and the MPI
+ *  data type will be accessible via @c get_mpi_datatype. 
+ *
+ *  In general, users should not need to specialize this
+ *  trait. However, if you have an additional C++ type that can map
+ *  directly to only of MPI's built-in types, specialize either this
+ *  trait or one of the traits corresponding to categories of MPI data
+ *  types (@c is_mpi_integer_datatype, @c
+ *  is_mpi_floating_point_datatype, @c is_mpi_logical_datatype, @c
+ *  is_mpi_complex_datatype, or @c is_mpi_builtin_datatype). @c
+ *  is_mpi_builtin_datatype derives @c mpl::true_ if any of the traits
+ *  corresponding to MPI data type categories derived @c mpl::true_.
+ */
+template<typename T>
+struct is_mpi_builtin_datatype
+  : boost::mpl::or_<is_mpi_integer_datatype<T>,
+                    is_mpi_floating_point_datatype<T>,
+                    is_mpi_logical_datatype<T>,
+                    is_mpi_complex_datatype<T>,
+                    is_mpi_byte_datatype<T> >
+{
+};
+
+/** @brief Type trait that determines if a C++ type can be mapped to
+ *  an MPI data type.
+ *
+ *  This type trait determines if it is possible to build an MPI data
+ *  type that represents a C++ data type. When this is the case, @c
+ *  is_mpi_datatype derives @c mpl::true_ and the MPI data type will
+ *  be accessible via @c get_mpi_datatype.
+
+ *  For any C++ type that maps to a built-in MPI data type (see @c
+ *  is_mpi_builtin_datatype), @c is_mpi_data_type is trivially
+ *  true. However, any POD ("Plain Old Data") type containing types
+ *  that themselves can be represented by MPI data types can itself be
+ *  represented as an MPI data type. For instance, a @c point3d class
+ *  containing three @c double values can be represented as an MPI
+ *  data type. To do so, first make the data type Serializable (using
+ *  the Boost.Serialization library); then, specialize the @c
+ *  is_mpi_datatype trait for the point type so that it will derive @c
+ *  mpl::true_:
+ *
+ *    @code
+ *    namespace boost { namespace parallel { namespace mpi {
+ *      template<> struct is_mpi_datatype<point>
+ *        : public mpl::true_ { };
+ *    } } }
+ *    @endcode
+ */
+template<typename T>
+struct is_mpi_datatype
+ : public is_mpi_builtin_datatype<T>
+{
+};
+
+/** @brief Returns an MPI data type for a C++ type.
+ *
+ *  The function creates an MPI data type for the given object @c
+ *  x. The first time it is called for a class @c T, the MPI data type
+ *  is created and cached. Subsequent calls for objects of the same
+ *  type @c T return the cached MPI data type.  The type @c T must
+ *  allow creation of an MPI data type. That is, it must be
+ *  Serializable and @c is_mpi_datatype<T> must derive @c mpl::true_.
+ *
+ *  For fundamental MPI types, a copy of the MPI data type of the MPI
+ *  library is returned.
+ *
+ *  Note that since the data types are cached, the caller should never
+ *  call @c MPI_Type_free() for the MPI data type returned by this
+ *  call.
+ *
+ *  @param x for an optimized call, a constructed object of the type
+ *  should be passed; otherwise, an object will be
+ *  default-constructed.
+ *
+ *  @returns The MPI data type corresponding to type @c T.
+ */
+template<typename T> MPI_Datatype get_mpi_datatype(const T& x)
+{
+  BOOST_MPL_ASSERT((is_mpi_datatype<T>));
+  return detail::mpi_datatype_cache.datatype(x);
+}
+
+// Don't parse this part when we're generating Doxygen documentation.
+#ifndef BOOST_MPI_DOXYGEN
+
+/// INTERNAL ONLY
+#define BOOST_MPI_DATATYPE(CppType, MPIType, Kind)                      \
+template<>                                                              \
+inline MPI_Datatype                                                     \
+get_mpi_datatype< CppType >(const CppType&) { return MPIType; }         \
+                                                                        \
+template<>                                                              \
+ struct BOOST_JOIN(is_mpi_,BOOST_JOIN(Kind,_datatype))< CppType >       \
+: boost::mpl::bool_<true>                                               \
+{}
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(char, MPI_CHAR, builtin);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(short, MPI_SHORT, integer);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(int, MPI_INT, integer);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(long, MPI_LONG, integer);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(float, MPI_FLOAT, floating_point);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(double, MPI_DOUBLE, floating_point);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(long double, MPI_LONG_DOUBLE, floating_point);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(unsigned char, MPI_UNSIGNED_CHAR, builtin);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(unsigned short, MPI_UNSIGNED_SHORT, integer);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(unsigned, MPI_UNSIGNED, integer);
+
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(unsigned long, MPI_UNSIGNED_LONG, integer);
+
+/// INTERNAL ONLY
+#define BOOST_MPI_LIST2(A, B) A, B
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(std::pair<BOOST_MPI_LIST2(float, int)>, MPI_FLOAT_INT, 
+                   builtin);
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(std::pair<BOOST_MPI_LIST2(double, int)>, MPI_DOUBLE_INT, 
+                   builtin);
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(std::pair<BOOST_MPI_LIST2(long double, int)>,
+                   MPI_LONG_DOUBLE_INT, builtin);
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(std::pair<BOOST_MPI_LIST2(long, int>), MPI_LONG_INT, 
+                   builtin);
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(std::pair<BOOST_MPI_LIST2(short, int>), MPI_SHORT_INT, 
+                   builtin);
+/// INTERNAL ONLY
+BOOST_MPI_DATATYPE(std::pair<BOOST_MPI_LIST2(int, int>), MPI_2INT, builtin);
+#undef BOOST_MPI_LIST2
+
+#if 0
+#ifndef BOOST_NO_INTRINSIC_WCHAR_T
+BOOST_MPI_DATATYPE(wchar_t, MPI_WCHAR, builtin);
+#endif
+
+#ifdef BOOST_HAS_LONG_LONG
+BOOST_MPI_DATATYPE(long long, MPI_LONG_LONG_INT, builtin);
+BOOST_MPI_DATATYPE(unsigned long long, MPI_UNSIGNED_LONG_LONG, builtin);
+#endif
+#endif
+
+#endif // Doxygen
+
+namespace detail {
+  inline MPI_Datatype build_mpi_datatype_for_bool()
+  {
+    MPI_Datatype type;
+    MPI_Type_contiguous(sizeof(bool), MPI_BYTE, &type);
+    MPI_Type_commit(&type);
+    return type;
+  }
+}
+
+/// Support for bool. There is no corresponding MPI_BOOL.
+/// INTERNAL ONLY
+template<>
+inline MPI_Datatype get_mpi_datatype<bool>(const bool&)
+{
+  static MPI_Datatype type = detail::build_mpi_datatype_for_bool();
+  return type;
+}
+
+/// INTERNAL ONLY
+template<>
+struct is_mpi_datatype<bool>
+  : boost::mpl::bool_<true>
+{};
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_MPI_DATATYPE_HPP

include/boost/mpi/datatype_fwd.hpp

@@ -0,0 +1,32 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file datatype_fwd.hpp
+ *
+ *  This header provides forward declarations for the contents of the
+ *  header @c datatype.hpp. It is expected to be used primarily by
+ *  user-defined C++ classes that need to specialize @c
+ *  is_mpi_datatype.
+ */
+#ifndef BOOST_PARALLEL_MPI_DATATYPE_FWD_HPP
+#define BOOST_PARALLEL_MPI_DATATYPE_FWD_HPP
+
+#include <mpi.h>
+
+namespace boost { namespace parallel { namespace mpi {
+
+template<typename T> struct is_mpi_builtin_datatype;
+template<typename T> struct is_mpi_integer_datatype;
+template<typename T> struct is_mpi_floating_point_datatype;
+template<typename T> struct is_mpi_logical_datatype;
+template<typename T> struct is_mpi_complex_datatype;
+template<typename T> struct is_mpi_byte_datatype;
+template<typename T> struct is_mpi_datatype;
+template<typename T> MPI_Datatype get_mpi_datatype(const T& x = T());
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_MPI_DATATYPE_FWD_HPP

include/boost/mpi/detail/broadcast_sc.hpp

@@ -0,0 +1,41 @@
+// Copyright (C) 2005, 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Allows broadcast of skeletons via proxy.
+
+// This header may only be included after both the broadcast.hpp and
+// and skeleton_and_content.hpp headers have been included.
+#ifndef BOOST_PARALLEL_MPI_BROADCAST_SC_HPP
+#define BOOST_PARALLEL_MPI_BROADCAST_SC_HPP
+
+namespace boost { namespace parallel { namespace mpi {
+
+template<typename T>
+inline void
+broadcast(const communicator& comm, skeleton_proxy<T>& proxy, int root)
+{
+  const skeleton_proxy<T>& const_proxy(proxy);
+  broadcast(comm, const_proxy, root);
+}
+
+template<typename T>
+void
+broadcast(const communicator& comm, const skeleton_proxy<T>& proxy, int root)
+{
+  if (comm.rank() == root) {
+    packed_skeleton_oarchive oa(comm);
+    oa << proxy.object;
+    broadcast(comm, oa, root);
+  } else {
+    packed_skeleton_iarchive ia(comm);
+    broadcast(comm, ia, root);
+    ia >> proxy.object;
+  }
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_BROADCAST_SC_HPP

include/boost/mpi/detail/communicator_sc.hpp

@@ -0,0 +1,96 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Skeleton and content support for communicators
+
+// This header should be included only after both communicator.hpp and
+// skeleton_and_content.hpp have been included.
+#ifndef BOOST_PARALLEL_MPI_COMMUNICATOR_SC_HPP
+#define BOOST_PARALLEL_MPI_COMMUNICATOR_SC_HPP
+
+namespace boost { namespace parallel { namespace mpi {
+
+template<typename T>
+void
+communicator::send(int dest, int tag, const skeleton_proxy<T>& proxy) const
+{
+  packed_skeleton_oarchive ar(*this);
+  ar << proxy.object;
+  send(dest, tag, ar);
+}
+
+template<typename T>
+status
+communicator::recv(int source, int tag, const skeleton_proxy<T>& proxy) const
+{
+  packed_skeleton_iarchive ar(*this);
+  status result = recv(source, tag, ar);
+  ar >> proxy.object;
+  return result;
+}
+
+template<typename T>
+status communicator::recv(int source, int tag, skeleton_proxy<T>& proxy) const
+{
+  packed_skeleton_iarchive ar(*this);
+  status result = recv(source, tag, ar);
+  ar >> proxy.object;
+  return result;
+}
+
+template<typename T>
+request
+communicator::isend(int dest, int tag, const skeleton_proxy<T>& proxy) const
+{
+  shared_ptr<packed_skeleton_oarchive> 
+    archive(new packed_skeleton_oarchive(*this));
+
+  *archive << proxy.object;
+  request result = isend(dest, tag, *archive);
+  result.m_data = archive;
+  return result;
+}
+
+namespace detail {
+  template<typename T>
+  struct serialized_irecv_data<const skeleton_proxy<T> >
+  {
+    serialized_irecv_data(const communicator& comm, int source, int tag, 
+                          skeleton_proxy<T> proxy)
+      : comm(comm), source(source), tag(tag), isa(comm), 
+        ia(isa.get_skeleton()), proxy(proxy) { }
+
+    void deserialize(status& stat) 
+    { 
+      isa >> proxy.object;
+      stat.m_count = 1;
+    }
+
+    communicator comm;
+    int source;
+    int tag;
+    std::size_t count;
+    packed_skeleton_iarchive isa;
+    packed_iarchive& ia;
+    skeleton_proxy<T> proxy;
+  };
+
+  template<typename T>
+  struct serialized_irecv_data<skeleton_proxy<T> >
+    : public serialized_irecv_data<const skeleton_proxy<T> >
+  {
+    typedef serialized_irecv_data<const skeleton_proxy<T> > inherited;
+
+    serialized_irecv_data(const communicator& comm, int source, int tag, 
+                          const skeleton_proxy<T>& proxy)
+      : inherited(comm, source, tag, proxy) { }
+  };
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_COMMUNICATOR_SC_HPP
+

include/boost/mpi/detail/computation_tree.hpp

@@ -0,0 +1,86 @@
+// Copyright (C) 2005 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Compute parents, children, levels, etc. to effect a parallel
+// computation tree.
+#ifndef BOOST_PARALLEL_MPI_COMPUTATION_TREE_HPP
+#define BOOST_PARALLEL_MPI_COMPUTATION_TREE_HPP
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/**
+ * @brief Aids tree-based parallel collective algorithms.
+ *
+ * Objects of this type
+ */
+class computation_tree
+{
+ public:
+  computation_tree(int rank, int size, int root, int branching_factor = -1);
+
+  /// Returns the branching factor of the tree.
+  int branching_factor() const { return branching_factor_; }
+
+  /// Returns the level in the tree on which this process resides.
+  int level() const { return level_; }
+
+  /**
+   * Returns the index corresponding to the n^th level of the tree.
+   *
+   * @param n The level in the tree whose index will be returned.
+   */
+  int level_index(int n) const;
+
+  /**
+   *  @brief Returns the parent of this process.
+   *
+   *  @returns If this process is the root, returns itself. Otherwise,
+   *  returns the process number that is the parent in the computation
+   *  tree.
+   */
+  int parent() const;
+
+  /// Returns the index for the first child of this process.
+  int child_begin() const;
+
+  /**
+   * @brief The default branching factor within the computation tree.
+   *
+   * This is the default branching factor for the computation tree, to
+   * be used by any computation tree that does not fix the branching
+   * factor itself. The default is initialized to 3, but may be
+   * changed by the application so long as all processes have the same
+   * branching factor.
+   */
+  static int default_branching_factor;
+
+ protected:
+  /// The rank of this process in the computation tree.
+  int rank;
+
+  /// The number of processes participating in the computation tree.
+  int size;
+
+  /// The process number that is acting as the root in the computation
+  /// tree.
+  int root;
+
+  /**
+   * @brief The branching factor within the computation tree.
+   *
+   * This is the default number of children that each node in a
+   * computation tree will have. This value will be used for
+   * collective operations that use tree-based algorithms.
+   */
+  int branching_factor_;
+
+  /// The level in the tree at which this process resides.
+  int level_;
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_COMPUTATION_TREE_HPP

include/boost/mpi/detail/content_oarchive.hpp

@@ -0,0 +1,61 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_CONTENT_OARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_CONTENT_OARCHIVE_HPP
+
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/basic_archive.hpp>
+#include <boost/parallel/mpi/detail/ignore_skeleton_oarchive.hpp>
+#include <boost/parallel/mpi/detail/mpi_datatype_primitive.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+namespace detail {
+  // an archive wrapper that stores only the data members but not the
+  // special types defined by the serialization library
+  // to define the data skeletons (classes, pointers, container sizes, ...)
+
+  class content_oarchive
+    : public mpi_datatype_primitive,
+      public ignore_skeleton_oarchive<content_oarchive>
+  {
+  public:
+      content_oarchive()
+       : committed(false)
+          {}
+
+      content get_content()
+      {
+        if (!committed)
+        {
+          // create the content holder only once
+          c=this->get_mpi_datatype();
+          committed=true;
+        }
+        return c;
+      }
+
+  private:
+    bool committed;
+    content c;
+  };
+} // end namespace detail
+
+template <class T>
+const content get_content(const T& x)
+{
+  detail::content_oarchive ar;
+  ar << x;
+  return ar.get_content();
+}
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_CONTENT_OARCHIVE_HPP

include/boost/mpi/detail/forward_iprimitive.hpp

@@ -0,0 +1,72 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#include <boost/serialization/array.hpp>
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_FORWARD_IPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_FORWARD_IPRIMITIVE_HPP
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/// @brief a minimal input archive, which forwards reading to another archive
+///
+/// This class template is designed to use the loading facilities of another
+/// input archive (the "implementation archive", whose type is specified by 
+/// the template argument, to handle serialization of primitive types, 
+/// while serialization for specific types can be overriden independently 
+/// of that archive.
+
+template <class ImplementationArchive>
+class forward_iprimitive
+{
+public:
+
+    /// the type of the archive to which the loading of primitive types will be forwarded
+    typedef ImplementationArchive implementation_archive_type;
+	
+	/// the constructor takes a reference to the implementation archive used for loading primitve types
+    forward_iprimitive(implementation_archive_type& ar)
+	 : implementation_archive(ar)
+    {}
+
+	/// binary loading is forwarded to the implementation archive
+    void load_binary(void * address, std::size_t count )
+	{
+	  implementation_archive.load_binary(address,count);
+	}
+	
+	/// loading of arrays is forwarded to the implementation archive
+    template<class T>
+    void load_array(serialization::array<T> & x, unsigned int file_version )
+    {
+	  implementation_archive.load_array(x,file_version);
+    }
+
+    typedef typename ImplementationArchive::use_array_optimization use_array_optimization;	
+
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+    friend class archive::load_access;
+protected:
+#else
+public:
+#endif
+
+    ///  loading of primitives is forwarded to the implementation archive
+    template<class T>
+    void load(T & t)
+    {
+	  implementation_archive >> t;
+    }
+	
+private:
+    implementation_archive_type& implementation_archive;
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_FORWARD_IPRIMITIVE_HPP

include/boost/mpi/detail/forward_oprimitive.hpp

@@ -0,0 +1,73 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_FORWARD_OPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_FORWARD_OPRIMITIVE_HPP
+
+#include <boost/config.hpp>
+#include <boost/serialization/array.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/// @brief a minimal output archive, which forwards saving to another archive
+///
+/// This class template is designed to use the saving facilities of another
+/// output archive (the "implementation archive", whose type is specified by 
+/// the template argument, to handle serialization of primitive types, 
+/// while serialization for specific types can be overriden independently 
+/// of that archive.
+
+template <class ImplementationArchive>
+class forward_oprimitive
+{
+public:
+
+    /// the type of the archive to which the saving of primitive types will be forwarded
+    typedef ImplementationArchive implementation_archive_type;
+	
+	/// the constructor takes a reference to the implementation archive used for saving primitve types
+    forward_oprimitive(implementation_archive_type& ar)
+	 : implementation_archive(ar)
+    {}
+
+	/// binary saving is forwarded to the implementation archive
+    void save_binary(const void * address, std::size_t count)
+	{
+	  implementation_archive.save_binary(address,count);
+	}
+	
+	/// saving of arrays is forwarded to the implementation archive
+    template<class T>
+    void save_array(serialization::array<T> const& x, unsigned int file_version )
+    {
+	  implementation_archive.save_array(x,file_version);
+    }
+
+    typedef typename ImplementationArchive::use_array_optimization use_array_optimization;	
+
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+    friend class archive::save_access;
+protected:
+#else
+public:
+#endif
+
+    ///  saving of primitives is forwarded to the implementation archive
+    template<class T>
+    void save(const T & t)
+    {
+	  implementation_archive << t;
+    }
+	
+private:
+    implementation_archive_type& implementation_archive;
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_FORWARD_OPRIMITIVE_HPP

include/boost/mpi/detail/forward_skeleton_iarchive.hpp

@@ -0,0 +1,78 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_FORWARD_SKELETON_IARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_FORWARD_SKELETON_IARCHIVE_HPP
+
+#include <boost/pfto.hpp>
+
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/detail/iserializer.hpp>
+#include <boost/archive/detail/interface_iarchive.hpp>
+#include <boost/archive/array/iarchive.hpp>
+#include <boost/serialization/collection_size_type.hpp>
+#include <boost/archive/array/iarchive.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+template<class Archive, class ImplementationArchive>
+class forward_skeleton_iarchive 
+  : public archive::array::iarchive<Archive>
+{
+public:
+
+    typedef ImplementationArchive implementation_archive_type;
+	
+    forward_skeleton_iarchive(implementation_archive_type& ar) 
+	  : archive::array::iarchive<Archive>(archive::no_header),
+	    implementation_archive(ar)
+	{
+	}
+		
+#ifdef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+public:
+#else
+    friend class archive::detail::interface_iarchive<Archive>;
+    friend class archive::load_access;
+protected:
+#endif
+
+    // intermediate level to support override of operators
+    // for templates in the absence of partial function 
+    // template ordering
+    template<class T>
+    void load_override(T & t, BOOST_PFTO int)
+    {
+        archive::load(* this->This(), t);
+    }
+	
+#define BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(T) \
+    void load_override(T & t , int)             \
+	{                                           \
+	  implementation_archive >> t;              \
+	}
+
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_id_optional_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::version_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_id_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_id_reference_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::object_id_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::tracking_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_name_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(serialization::collection_size_type)
+
+#undef BOOST_ARCHIVE_FORWARD_IMPLEMENTATION
+protected:
+    /// the actual archive used to serialize the information we actually want to store
+    implementation_archive_type& implementation_archive;
+};
+
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_FORWARD_STRUCTURE_IARCHIVE_HPP

include/boost/mpi/detail/forward_skeleton_oarchive.hpp

@@ -0,0 +1,79 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_FORWARD_SKELETON_OARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_FORWARD_SKELETON_OARCHIVE_HPP
+
+#include <boost/pfto.hpp>
+
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/detail/oserializer.hpp>
+#include <boost/archive/detail/interface_oarchive.hpp>
+#include <boost/archive/array/oarchive.hpp>
+#include <boost/serialization/collection_size_type.hpp>
+#include <boost/archive/array/oarchive.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+template<class Archive, class ImplementationArchive>
+class forward_skeleton_oarchive 
+  : public archive::array::oarchive<Archive>
+{
+public:
+
+    typedef ImplementationArchive implementation_archive_type;
+	
+    forward_skeleton_oarchive(implementation_archive_type& ar) 
+	  : archive::array::oarchive<Archive>(archive::no_header),
+		implementation_archive(ar)
+	{
+	}
+		
+#ifdef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+public:
+#else
+    friend class archive::detail::interface_oarchive<Archive>;
+    friend class archive::save_access;
+protected:
+#endif
+
+    // intermediate level to support override of operators
+    // for templates in the absence of partial function 
+    // template ordering
+    template<class T>
+    void save_override(T const& t, BOOST_PFTO int)
+    {
+        archive::save(* this->This(), t);
+    }
+	
+#define BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(T) \
+    void save_override(T const & t , int)       \
+	{                                           \
+	  implementation_archive << t;              \
+	}
+
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_id_optional_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::version_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_id_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_id_reference_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::object_id_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::object_reference_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::tracking_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(archive::class_name_type)
+BOOST_ARCHIVE_FORWARD_IMPLEMENTATION(serialization::collection_size_type)
+
+
+#undef BOOST_ARCHIVE_FORWARD_IMPLEMENTATION
+protected:
+    /// the actual archive used to serialize the information we actually want to store
+    implementation_archive_type& implementation_archive;
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_FORWARD_STRUCTURE_OARCHIVE_HPP

include/boost/mpi/detail/ignore_iprimitive.hpp

@@ -0,0 +1,61 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_IGNORE_IPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_IGNORE_IPRIMITIVE_HPP
+
+#include <boost/config.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/serialization/array.hpp>
+
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/// @brief a minimal input archive, which ignores any load
+///
+/// This class implements a minimal input archive, probably an input archive
+/// archetype, doing nothing at any load. It's use, besides acting as an
+/// archetype is as a base class to implement special archives that ignore
+/// loading of most types
+
+class ignore_iprimitive
+{
+public:
+    /// a trivial default constructor
+    ignore_iprimitive()
+    {}
+
+
+        /// don't do anything when loading binary data
+    void load_binary(void *, std::size_t )
+        {}
+
+        /// don't do anything when loading arrays
+    template<class T>
+    void load_array(serialization::array<T> &, unsigned int )
+    {}
+
+    typedef is_mpi_datatype<mpl::_1> use_array_optimization;
+
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+    friend class archive::load_access;
+protected:
+#else
+public:
+#endif
+
+        /// don't do anything when loading primitive types
+    template<class T>
+    void load(T & t)
+        {
+    }
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_IGNORE_IPRIMITIVE_HPP

include/boost/mpi/detail/ignore_oprimitive.hpp

@@ -0,0 +1,62 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_IGNORE_OPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_IGNORE_OPRIMITIVE_HPP
+
+#include <boost/config.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/serialization/array.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/// @brief a minimal output archive, which ignores any save
+///
+/// This class implements a minimal output archive, probably an output archive
+/// archetype, doing nothing at any save. It's use, besides acting as an
+/// archetype is as a base class to implement special archives that ignore
+/// saving of most types
+
+class ignore_oprimitive
+{
+public:
+    /// a trivial default constructor
+    ignore_oprimitive()
+    {}
+
+        /// don't do anything when saving binary data
+    void save_binary(const void *, std::size_t )
+        {
+        }
+
+        /// don't do anything when saving arrays
+    template<class T>
+    void save_array(serialization::array<T> const&, unsigned int )
+    {
+    }
+
+    typedef is_mpi_datatype<mpl::_1> use_array_optimization;
+
+
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+    friend class archive::save_access;
+protected:
+#else
+public:
+#endif
+
+        /// don't do anything when saving primitive types
+    template<class T>
+    void save(const T & t)
+    {
+    }
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_IGNORE_OPRIMITIVE_HPP

include/boost/mpi/detail/ignore_skeleton_oarchive.hpp

@@ -0,0 +1,70 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_IGNORE_SKELETON_OARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_IGNORE_SKELETON_OARCHIVE_HPP
+
+#include <boost/pfto.hpp>
+
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/array/oarchive.hpp>
+#include <boost/archive/basic_archive.hpp>
+#include <boost/archive/detail/oserializer.hpp>
+#include <boost/serialization/collection_size_type.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+template<class Archive>
+class ignore_skeleton_oarchive 
+  : public archive::array::oarchive<Archive>
+{
+public:
+	
+    ignore_skeleton_oarchive()
+	  : archive::array::oarchive<Archive>(archive::no_header)
+	{
+	}
+		
+#ifdef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+public:
+#else
+    friend class archive::detail::interface_oarchive<Archive>;
+    friend class archive::save_access;
+protected:
+#endif
+
+    // intermediate level to support override of operators
+    // for templates in the absence of partial function 
+    // template ordering
+    template<class T>
+    void save_override(T const& t, BOOST_PFTO int)
+    {
+        archive::save(* this->This(), t);
+    }
+	
+#define BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(T) \
+    void save_override(T const & t , int)      \
+	{}                                         \
+
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::class_id_optional_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::version_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::class_id_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::class_id_reference_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::object_id_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::object_reference_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::tracking_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(archive::class_name_type)
+BOOST_ARCHIVE_IGNORE_IMPLEMENTATION(serialization::collection_size_type)
+
+#undef BOOST_ARCHIVE_IGNORE_IMPLEMENTATION
+};
+
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_IGNORE_SKELETON_OARCHIVE_HPP

include/boost/mpi/detail/mpi_datatype_cache.hpp

@@ -0,0 +1,98 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_TYPE_MPI_DATATYPE_CACHE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_TYPE_MPI_DATATYPE_CACHE_HPP
+
+#include <boost/parallel/mpi/datatype_fwd.hpp>
+#include <boost/parallel/mpi/detail/mpi_datatype_oarchive.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/utility/enable_if.hpp>
+#include <boost/mpl/assert.hpp>
+#include <boost/noncopyable.hpp>
+#include <map>
+#include <typeinfo>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/// @brief comparison function object for two std::type_info pointers
+///
+/// is implemented using the before() member function of the std::type_info
+/// class
+
+struct type_info_compare
+{
+  bool operator()(std::type_info const* lhs, std::type_info const* rhs) const
+  {
+    return lhs->before(*rhs);
+  }
+};
+
+
+/// @brief a map of MPI data types, indexed by their type_info
+///
+///
+class mpi_datatype_map
+ : private std::map<std::type_info const*,MPI_Datatype,type_info_compare>,
+   public boost::noncopyable
+{
+public:
+  mpi_datatype_map()
+  {}
+
+  ~mpi_datatype_map()
+  {
+    // do not free after call to MPI_FInalize
+    int finalized=0;
+    BOOST_MPI_CHECK_RESULT(MPI_Finalized,(&finalized));
+    if (!finalized)
+      free();
+  }
+
+  template <class T>
+  MPI_Datatype datatype(const T& = T(), typename boost::enable_if<is_mpi_builtin_datatype<T> >::type* =0)
+  {
+    return get_mpi_datatype<T>();
+  }
+
+  template <class T>
+  MPI_Datatype datatype(const T& x=T(), typename boost::disable_if<is_mpi_builtin_datatype<T> >::type* =0 )
+  {
+    BOOST_MPL_ASSERT((is_mpi_datatype<T>));
+
+    // check whether the type already exists
+    std::type_info const* t = &typeid(T);
+    const_iterator it = find(t);
+    if(it ==end())
+    {
+      // need to create a type
+      mpi_datatype_oarchive ar(x);
+      insert(std::make_pair(t,ar.get_mpi_datatype()));
+      it = find(t);
+    }
+
+  return it->second;
+  }
+
+private:
+  // free all MPI data types
+  void free()
+  {
+    // ignore errors in the destructor
+    for (iterator it=begin(); it !=end(); ++it)
+          MPI_Type_free(&(it->second));
+  }
+
+};
+
+extern mpi_datatype_map mpi_datatype_cache;
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_TYPE_MPI_DATATYPE_CACHE_HPP

include/boost/mpi/detail/mpi_datatype_oarchive.hpp

@@ -0,0 +1,43 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_MPI_DATATYPE_OARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_MPI_DATATYPE_OARCHIVE_HPP
+
+#include <boost/archive/detail/oserializer.hpp>
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/basic_archive.hpp>
+#include <boost/parallel/mpi/detail/ignore_skeleton_oarchive.hpp>
+#include <boost/parallel/mpi/detail/mpi_datatype_primitive.hpp>
+#include <boost/parallel/mpi/datatype_fwd.hpp>
+#include <boost/mpl/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+
+// an archive wrapper that stores only the data members but not the
+// special types defined by the serialization library
+// to define the data skeletons (classes, pointers, container sizes, ...)
+
+class mpi_datatype_oarchive
+  : public mpi_datatype_primitive,
+    public ignore_skeleton_oarchive<mpi_datatype_oarchive>
+{
+public:
+    template <class T>
+    mpi_datatype_oarchive(const T& x)
+         :  mpi_datatype_primitive(&x) // register address
+        {
+      BOOST_MPL_ASSERT((is_mpi_datatype<T>));
+          *this << x;                   // serialize the object
+        }
+};
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_MPI_DATATYPE_OARCHIVE_HPP

include/boost/mpi/detail/mpi_datatype_primitive.hpp

@@ -0,0 +1,128 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_DETAIL_MPI_DATATYPE_OPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_MPI_DATATYPE_OPRIMITIVE_HPP
+
+#include <mpi.h>
+#include <cstddef> // size_t
+
+#include <boost/config.hpp>
+#if defined(BOOST_NO_STDC_NAMESPACE)
+namespace std{
+    using ::size_t;
+} // namespace std
+#endif
+
+#include <boost/parallel/mpi/datatype_fwd.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/throw_exception.hpp>
+#include <boost/assert.hpp>
+#include <boost/mpl/placeholders.hpp>
+#include <boost/serialization/array.hpp>
+#include <boost/serialization/detail/get_data.hpp>
+#include <stdexcept>
+#include <iostream>
+#include <vector>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/////////////////////////////////////////////////////////////////////////
+// class mpi_data_type_oprimitive - creation of custom MPI data types
+
+class mpi_datatype_primitive
+{
+public:
+
+    // trivial default constructor
+    mpi_datatype_primitive()
+     : is_committed(false),
+       origin(0)
+    {}
+
+    mpi_datatype_primitive(void const* orig)
+     : is_committed(false),
+       origin()
+    {
+      BOOST_MPI_CHECK_RESULT(MPI_Address,(const_cast<void*>(orig), &origin));
+    }
+
+    void save_binary(void const *address, std::size_t count)
+    {
+      save_impl(address,MPI_BYTE,count);
+    }
+
+    // fast saving of arrays of MPI types
+    template<class T>
+    void save_array(serialization::array<T> const& x, unsigned int /* version */)
+    {
+      if (x.count())
+        save_impl(x.address(), boost::parallel::mpi::get_mpi_datatype(*x.address()), x.count());
+    }
+
+    typedef is_mpi_datatype<mpl::_1> use_array_optimization;
+
+    // create and return the custom MPI data type
+    MPI_Datatype get_mpi_datatype()
+    {
+      if (!is_committed)
+      {
+        BOOST_MPI_CHECK_RESULT(MPI_Type_struct,
+                    (
+                      addresses.size(),
+                      boost::serialization::detail::get_data(lengths),
+                      boost::serialization::detail::get_data(addresses),
+                      boost::serialization::detail::get_data(types),
+                      &datatype_
+                    ));
+
+        BOOST_MPI_CHECK_RESULT(MPI_Type_commit,(&datatype_));
+
+        is_committed = true;
+      }
+
+      return datatype_;
+    }
+
+    // default saving of primitives.
+    template<class T>
+    void save(const T & t)
+    {
+        save_impl(&t, boost::parallel::mpi::get_mpi_datatype(t), 1);
+    }
+
+private:
+
+    void save_impl(void const * p, MPI_Datatype t, int l)
+    {
+      BOOST_ASSERT ( !is_committed );
+
+      // store address, type and length
+
+      MPI_Aint a;
+      BOOST_MPI_CHECK_RESULT(MPI_Address,(const_cast<void*>(p), &a));
+
+      addresses.push_back(a-origin);
+      types.push_back(t);
+      lengths.push_back(l);
+    }
+
+    std::vector<MPI_Aint> addresses;
+    std::vector<MPI_Datatype> types;
+    std::vector<int> lengths;
+
+    bool is_committed;
+    MPI_Datatype datatype_;
+    MPI_Aint origin;
+};
+
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_MPI_DATATYPE_OPRIMITIVE_HPP

include/boost/mpi/detail/packed_iprimitive.hpp

@@ -0,0 +1,119 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_PACKED_IPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_PACKED_IPRIMITIVE_HPP
+
+#include <mpi.h>
+#include <iostream>
+#include <cstddef> // size_t
+#include <boost/config.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/assert.hpp>
+#include <boost/serialization/array.hpp>
+#include <boost/serialization/detail/get_data.hpp>
+#include <vector>
+#include <boost/parallel/mpi/allocator.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/// deserialization using MPI_Unpack
+
+class packed_iprimitive
+{
+public:
+    /// the type of the buffer from which the data is unpacked upon deserialization
+    typedef std::vector<char, allocator<char> > buffer_type;
+
+    packed_iprimitive(buffer_type & b, MPI_Comm const & comm, int position = 0)
+         : buffer_(b),
+           comm(comm),
+           position(position)
+        {
+        }
+
+    void* address ()
+    {
+      return &buffer_[0];
+    }
+
+    void const* address () const
+    {
+      return &buffer_[0];
+    }
+
+    const std::size_t& size() const
+    {
+      return size_ = buffer_.size();
+    }
+
+    void resize(std::size_t s)
+    {
+      buffer_.resize(s);
+    }
+
+    void load_binary(void *address, std::size_t count)
+        {
+          load_impl(address,MPI_BYTE,count);
+        }
+
+    // fast saving of arrays of fundamental types
+    template<class T>
+    void load_array(serialization::array<T> & x, unsigned int /* file_version */)
+    {
+      if (x.count())
+        load_impl(x.address(), get_mpi_datatype(*x.address()), x.count());
+    }
+
+    typedef is_mpi_datatype<mpl::_1> use_array_optimization;
+
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+    friend class archive::load_access;
+protected:
+#else
+public:
+#endif
+
+    // default saving of primitives.
+    template<class T>
+    void load( T & t)
+    {
+        load_impl(&t, get_mpi_datatype(t), 1);
+    }
+
+    void load( std::string & s)
+    {
+       unsigned int l;
+        load(l);
+        // borland de-allocator fixup
+        #if BOOST_WORKAROUND(_RWSTD_VER, BOOST_TESTED_AT(20101))
+        if(NULL != s.data())
+        #endif
+        s.resize(l);
+        // note breaking a rule here - could be a problem on some platform
+        load_impl(const_cast<char *>(s.data()),MPI_CHAR,l);
+        }
+
+private:
+
+    void load_impl(void * p, MPI_Datatype t, int l)
+        {
+        BOOST_MPI_CHECK_RESULT(MPI_Unpack,
+        (const_cast<char*>(boost::serialization::detail::get_data(buffer_)), buffer_.size(), &position, p, l, t, comm));
+        }
+
+        buffer_type & buffer_;
+        mutable std::size_t size_;
+        MPI_Comm comm;
+        int position;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_PACKED_IPRIMITIVE_HPP

include/boost/mpi/detail/packed_oprimitive.hpp

@@ -0,0 +1,115 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_PACKED_OPRIMITIVE_HPP
+#define BOOST_PARALLEL_MPI_PACKED_OPRIMITIVE_HPP
+
+#include <mpi.h>
+#include <iostream>
+#include <cstddef> // size_t
+#include <boost/config.hpp>
+
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <boost/serialization/detail/get_data.hpp>
+#include <boost/serialization/array.hpp>
+#include <boost/assert.hpp>
+#include <vector>
+#include <boost/parallel/mpi/allocator.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/// serialization using MPI::Pack
+
+class packed_oprimitive
+{
+public:
+    /// the type of the buffer into which the data is packed upon serialization
+    typedef std::vector<char, allocator<char> > buffer_type;
+
+    packed_oprimitive(buffer_type & b, MPI_Comm const & comm)
+         : buffer_(b),
+           comm(comm)
+        {
+        }
+
+    void const * address() const
+    {
+      return &buffer_[0];
+    }
+
+    const std::size_t& size() const
+    {
+      return size_ = buffer_.size();
+    }
+
+    void save_binary(void const *address, std::size_t count)
+        {
+          save_impl(address,MPI_BYTE,count);
+        }
+
+    // fast saving of arrays
+    template<class T>
+    void save_array(serialization::array<T> const& x, unsigned int /* file_version */)
+    {
+        if (x.count())
+          save_impl(x.address(), get_mpi_datatype(*x.address()), x.count());
+    }
+
+    typedef is_mpi_datatype<mpl::_1> use_array_optimization;
+
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+    friend class archive::save_access;
+protected:
+#else
+public:
+#endif
+
+    // default saving of primitives.
+    template<class T>
+    void save(const T & t)
+    {
+        save_impl(&t, get_mpi_datatype<T>(), 1);
+    }
+
+        void save(const std::string &s)
+        {
+      unsigned int l = static_cast<unsigned int>(s.size());
+      save(l);
+      save_impl(s.data(),MPI_CHAR,s.size());
+        }
+
+private:
+
+    void save_impl(void const * p, MPI_Datatype t, int l)
+        {
+          // allocate enough memory
+      int memory_needed;
+      BOOST_MPI_CHECK_RESULT(MPI_Pack_size,(l,t,comm,&memory_needed));
+
+         int position = buffer_.size();
+          buffer_.resize(position + memory_needed);
+
+          // pack the data into the buffer
+          BOOST_MPI_CHECK_RESULT(MPI_Pack,
+        (const_cast<void*>(p), l, t, boost::serialization::detail::get_data(buffer_), buffer_.size(), &position, comm));
+
+          // reduce the buffer size if needed
+          BOOST_ASSERT(std::size_t(position) <= buffer_.size());
+          if (std::size_t(position) < buffer_.size())
+            buffer_.resize(position);
+        }
+
+  buffer_type& buffer_;
+  mutable std::size_t size_;
+  MPI_Comm comm;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_PACKED_OPRIMITIVE_HPP

include/boost/mpi/detail/point_to_point.hpp

@@ -0,0 +1,51 @@
+// Copyright 2005 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 3. MPI Point-to-point
+#ifndef BOOST_PARALLEL_MPI_DETAIL_POINT_TO_POINT_HPP
+#define BOOST_PARALLEL_MPI_DETAIL_POINT_TO_POINT_HPP
+
+// For (de-)serializing sends and receives
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+/** Sends a packed archive using MPI_Send. */
+void
+packed_archive_send(MPI_Comm comm, int dest, int tag,
+                    const packed_oarchive& ar);
+
+/** Sends a packed archive using MPI_Isend.
+ *
+ * This routine may split sends into multiple packets. The MPI_Request
+ * for each packet will be placed into the out_requests array, up to
+ * num_out_requests packets. The number of packets sent will be
+ * returned from the function.
+ *
+ * @pre num_out_requests >= 2
+ */
+int
+packed_archive_isend(MPI_Comm comm, int dest, int tag,
+                     const packed_oarchive& ar,
+                     MPI_Request* out_requests, int num_out_requests);
+
+/**
+ * \overload
+ */
+int
+packed_archive_isend(MPI_Comm comm, int dest, int tag,
+                     const packed_iarchive& ar,
+                     MPI_Request* out_requests, int num_out_requests);
+
+/** Receives a packed archive using MPI_Recv. */
+void
+packed_archive_recv(MPI_Comm comm, int source, int tag, packed_iarchive& ar,
+                    MPI_Status& status);
+
+} } } } // end namespace boost::parallel::mpi::detail
+
+#endif // BOOST_PARALLEL_MPI_DETAIL_POINT_TO_POINT_HPP

include/boost/mpi/detail/text_skeleton_oarchive.hpp

@@ -0,0 +1,39 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#ifndef BOOST_PARALLEL_MPI_TEXT_SKELETON_OARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_TEXT_SKELETON_OARCHIVE_HPP
+
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/text_oarchive.hpp>
+#include <boost/parallel/mpi/detail/forward_skeleton_oarchive.hpp>
+#include <boost/parallel/mpi/detail/ignore_oprimitive.hpp>
+#include <boost/archive/array/oarchive.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+// an archive that writes a text skeleton into a stream
+
+class text_skeleton_oarchive 
+  : public detail::ignore_oprimitive,
+    public detail::forward_skeleton_oarchive<text_skeleton_oarchive,boost::archive::text_oarchive>
+{
+public:
+    text_skeleton_oarchive(std::ostream & s, unsigned int flags = 0) 
+	 : detail::forward_skeleton_oarchive<text_skeleton_oarchive,boost::archive::text_oarchive>(skeleton_archive_)
+     , skeleton_archive_(s,flags)
+	{}
+
+private:
+    boost::archive::text_oarchive skeleton_archive_;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+
+#endif // BOOST_PARALLEL_MPI_TEXT_SKELETON_OARCHIVE_HPP

include/boost/mpi/environment.hpp

@@ -0,0 +1,183 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file environment.hpp
+ *
+ *  This header provides the @c environment class, which provides
+ *  routines to initialize, finalization, and query the status of the
+ *  Boost MPI environment.
+ */
+#ifndef BOOST_PARALLEL_MPI_ENVIRONMENT_HPP
+#define BOOST_PARALLEL_MPI_ENVIRONMENT_HPP
+
+#include <boost/noncopyable.hpp>
+#include <boost/optional.hpp>
+#include <string>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/** @brief Initialize, finalize, and query the MPI environment.
+ *
+ *  The @c environment class is used to initialize, finalize, and
+ *  query the MPI environment. It will typically be used in the @c
+ *  main() function of a program, which will create a single instance
+ *  of @c environment initialized with the arguments passed to the
+ *  program:
+ *
+ *  @code
+ *  int main(int argc, char* argv[])
+ *  {
+ *    mpi::environment env(argc, argv);
+ *  }
+ *  @endcode
+ *
+ *  The instance of @c environment will initialize MPI (by calling @c
+ *  MPI_Init) in its constructor and finalize MPI (by calling @c
+ *  MPI_Finalize for normal termination or @c MPI_Abort for an
+ *  uncaught exception) in its destructor.
+ *
+ *  The use of @c environment is not mandatory. Users may choose to
+ *  invoke @c MPI_Init and @c MPI_Finalize manually. In this case, no
+ *  @c environment object is needed. If one is created, however, it
+ *  will do nothing on either construction or destruction.
+ */
+class environment : noncopyable {
+public:
+  /** Initialize the MPI environment.
+   *
+   *  If the MPI environment has not already been initialized,
+   *  initializes MPI with a call to @c MPI_Init.
+   *
+   *  @param argc The number of arguments provided in @p argv, as
+   *  passed into the program's @c main function.
+   *
+   *  @param argv The array of argument strings passed to the program
+   *  via @c main.
+   *
+   *  @param abort_on_exception When true, this object will abort the
+   *  program if it is destructed due to an uncaught exception.
+   */
+  environment(int& argc, char** &argv, bool abort_on_exception = true);
+
+  /** Shuts down the MPI environment.
+   *
+   *  If this @c environment object was used to initialize the MPI
+   *  environment, and the MPI environment has not already been shut
+   *  down (finalized), this destructor will shut down the MPI
+   *  environment. Under normal circumstances, this only involves
+   *  invoking @c MPI_Finalize. However, if destruction is the result
+   *  of an uncaught exception and the @c abort_on_exception parameter
+   *  of the constructor had the value @c true, this destructor will
+   *  invoke @c MPI_Abort with @c MPI_COMM_WORLD to abort the entire
+   *  MPI program with a result code of -1.
+   */
+  ~environment();
+
+  /** Abort all MPI processes.
+   *
+   *  Aborts all MPI processes and returns to the environment. The
+   *  precise behavior will be defined by the underlying MPI
+   *  implementation. This is equivalent to a call to @c MPI_Abort
+   *  with @c MPI_COMM_WORLD.
+   *
+   *  @param errcode The error code to return to the environment.
+   *  @returns Will not return.
+   */
+  static void abort(int errcode);
+
+  /** Determine if the MPI environment has already been initialized.
+   *
+   *  This routine is equivalent to a call to @c MPI_Initialized.
+   *
+   *  @returns @c true if the MPI environment has been initialized.
+   */
+  static bool initialized();
+
+  /** Determine if the MPI environment has already been finalized.
+   *
+   *  The routine is equivalent to a call to @c MPI_Finalized.
+   *
+   *  @returns @c true if the MPI environment has been finalized.
+   */
+  static bool finalized();
+
+  /** Retrieves the maximum tag value.
+   *
+   *  Returns the maximum value that may be used for the @c tag
+   *  parameter of send/receive operations. This value will be
+   *  somewhat smaller than the value of @c MPI_TAG_UB, because the
+   *  Boost.MPI implementation reserves some tags for collective
+   *  operations.
+   *
+   *  @returns the maximum tag value.
+   */
+  static int max_tag();
+
+  /** The tag value used for collective operations.
+   *
+   *  Returns the reserved tag value used by the Boost.MPI
+   *  implementation for collective operations. Although users are not
+   *  permitted to use this tag to send or receive messages, it may be
+   *  useful when monitoring communication patterns.
+   *
+   * @returns the tag value used for collective operations.
+   */
+  static int collectives_tag();
+
+  /** Retrieves the rank of the host process, if one exists.
+   *
+   *  If there is a host process, this routine returns the rank of
+   *  that process. Otherwise, it returns an empty @c
+   *  optional<int>. MPI does not define the meaning of a "host"
+   *  process: consult the documentation for the MPI
+   *  implementation. This routine examines the @c MPI_HOST attribute
+   *  of @c MPI_COMM_WORLD.
+   *
+   *  @returns The rank of the host process, if one exists.
+   */
+  static optional<int> host_rank();
+
+  /** Retrieves the rank of a process that can perform input/output.
+   *
+   *  This routine returns the rank of a process that can perform
+   *  input/output via the standard C and C++ I/O facilities. If every
+   *  process can perform I/O using the standard facilities, this
+   *  routine will return @c any_source; if no process can perform
+   *  I/O, this routine will return no value (an empty @c
+   *  optional). This routine examines the @c MPI_IO attribute of @c
+   *  MPI_COMM_WORLD.
+   *
+   *  @returns the rank of the process that can perform I/O, @c
+   *  any_source if every process can perform I/O, or no value if no
+   *  process can perform I/O.
+   */
+  static optional<int> io_rank();
+
+  /** Retrieve the name of this processor.
+   *
+   *  This routine returns the name of this processor. The actual form
+   *  of the name is unspecified, but may be documented by the
+   *  underlying MPI implementation. This routine is implemented as a
+   *  call to @c MPI_Get_processor_name.
+   *
+   *  @returns the name of this processor.
+   */
+  static std::string processor_name();
+
+private:
+  /// Whether this environment object called MPI_Init
+  bool i_initialized;
+
+  /// Whether we should abort if the destructor is
+  bool abort_on_exception;
+
+  /// The number of reserved tags.
+  static const int num_reserved_tags = 1;
+};
+
+} } } /// end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_ENVIRONMENT_HPP

include/boost/mpi/exception.hpp

@@ -0,0 +1,89 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file exception.hpp
+ *
+ *  This header provides exception classes that report MPI errors to
+ *  the user and macros that translate MPI error codes into Boost.MPI
+ *  exceptions.
+ */
+#ifndef BOOST_PARALLEL_MPI_EXCEPTION_HPP
+#define BOOST_PARALLEL_MPI_EXCEPTION_HPP
+
+#include <mpi.h>
+#include <exception>
+#include <boost/config.hpp>
+#include <boost/throw_exception.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/** @brief Catch-all exception class for MPI errors.
+ *
+ * Instances of this class will be thrown when an MPI error
+ * occurs. MPI failures that trigger these exceptions may or may not
+ * be recoverable, depending on the underlying MPI
+ * implementation. Consult the documentation for your MPI
+ * implementation to determine the effect of MPI errors.
+ */
+class exception : public std::exception
+{
+ public:
+  /**
+   * Build a new @c exception exception.
+   *
+   *   @param routine The MPI routine in which the error
+   *   occurred. This should be a pointer to a string constant: it
+   *   will not be copied.
+   *
+   *   @param result_code The result code returned from the MPI
+   *   routine that aborted with an error.
+   */
+  exception(const char* routine, int result_code)
+    : routine_(routine), result_code_(result_code) { }
+
+  /**
+   * A description of the error that occured. At present, this refers
+   * only to the name of the MPI routine that failed.
+   */
+  virtual const char * what () const throw ()
+  {
+    return routine_;
+  }
+
+  /** Retrieve the name of the MPI routine that reported the error. */
+  const char* routine() const { return routine_; }
+
+  /**
+   * Retrieve the result code returned from the MPI routine that
+   * reported the error.
+   */
+  int result_code() const { return result_code_; }
+
+ protected:
+  /// The MPI routine that triggered the error
+  const char* routine_;
+
+  /// The failed result code reported by the MPI implementation.
+  int result_code_;
+};
+
+/**
+ * Call the MPI routine MPIFunc with arguments Args (surrounded by
+ * parentheses). If the result is not MPI_SUCCESS, use
+ * boost::throw_exception to throw an exception or abort, depending on
+ * BOOST_NO_EXCEPTIONS.
+ */
+#define BOOST_MPI_CHECK_RESULT( MPIFunc, Args )                         \
+ {                                                                      \
+   int _check_result = MPIFunc Args;                                    \
+   if (_check_result != MPI_SUCCESS)                                    \
+     boost::throw_exception(boost::parallel::mpi::exception(#MPIFunc,   \
+                                                             _check_result)); \
+ }
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_EXCEPTION_HPP

include/boost/mpi/nonblocking.hpp

@@ -0,0 +1,732 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file nonblocking.hpp
+ *
+ *  This header defines operations for completing non-blocking
+ *  communication requests.
+ */
+#ifndef BOOST_PARALLEL_MPI_NONBLOCKING_HPP
+#define BOOST_PARALLEL_MPI_NONBLOCKING_HPP
+
+#include <mpi.h>
+#include <vector>
+#include <iterator> // for std::iterator_traits
+#include <boost/optional.hpp>
+#include <utility> // for std::pair
+#include <algorithm> // for iter_swap, reverse
+#include <boost/static_assert.hpp>
+#include <boost/parallel/mpi/request.hpp>
+#include <boost/parallel/mpi/status.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/** 
+ *  @brief Wait until any non-blocking request has completed.
+ *
+ *  This routine takes in a set of requests stored in the iterator
+ *  range @c [first,last) and waits until any of these requests has
+ *  been completed. It provides functionality equivalent to 
+ *  @c MPI_Waitany.
+ *
+ *  @param first The iterator that denotes the beginning of the
+ *  sequence of request objects.
+ *
+ *  @param last The iterator that denotes the end of the sequence of
+ *  request objects. This may not be equal to @c first.
+ *
+ *  @returns A pair containing the status object that corresponds to
+ *  the completed operation and the iterator referencing the completed
+ *  request.
+ */
+template<typename ForwardIterator>
+std::pair<status, ForwardIterator> 
+wait_any(ForwardIterator first, ForwardIterator last)
+{
+  using std::advance;
+
+  BOOST_ASSERT(first != last);
+  
+  typedef typename std::iterator_traits<ForwardIterator>::difference_type
+    difference_type;
+
+  bool all_trivial_requests = true;
+  difference_type n = 0;
+  ForwardIterator current = first;
+  while (true) {
+    // Check if we have found a completed request. If so, return it.
+    if (optional<status> result = current->test())
+      return std::make_pair(*result, current);
+
+    // Check if this request (and all others before it) are "trivial"
+    // requests, e.g., they can be represented with a single
+    // MPI_Request.
+    all_trivial_requests = 
+      all_trivial_requests
+      && !current->m_handler 
+      && current->m_requests[1] == MPI_REQUEST_NULL;
+
+    // Move to the next request.
+    ++n;
+    if (++current == last) {
+      // We have reached the end of the list. If all requests thus far
+      // have been trivial, we can call MPI_Waitany directly, because
+      // it may be more efficient than our busy-wait semantics.
+      if (all_trivial_requests) {
+        std::vector<MPI_Request> requests;
+        requests.reserve(n);
+        for (current = first; current != last; ++current)
+          requests.push_back(current->m_requests[0]);
+
+        // Let MPI wait until one of these operations completes.
+        int index;
+        status stat;
+        BOOST_MPI_CHECK_RESULT(MPI_Waitany, 
+                               (n, &requests[0], &index, &stat.m_status));
+
+        // We don't have a notion of empty requests or status objects,
+        // so this is an error.
+        if (index == MPI_UNDEFINED)
+          boost::throw_exception(exception("MPI_Waitany", MPI_ERR_REQUEST));
+
+        // Find the iterator corresponding to the completed request.
+        current = first;
+        advance(current, index);
+        current->m_requests[0] = requests[index];
+        return std::make_pair(stat, current);
+      }
+
+      // There are some nontrivial requests, so we must continue our
+      // busy waiting loop.
+      n = 0;
+      current = first;
+      all_trivial_requests = true;
+    }
+  }
+
+  // We cannot ever get here
+  BOOST_ASSERT(false);
+}
+
+/** 
+ *  @brief Test whether any non-blocking request has completed.
+ *
+ *  This routine takes in a set of requests stored in the iterator
+ *  range @c [first,last) and tests whether any of these requests has
+ *  been completed. This routine is similar to @c wait_any, but will
+ *  not block waiting for requests to completed. It provides
+ *  functionality equivalent to @c MPI_Testany.
+ *
+ *  @param first The iterator that denotes the beginning of the
+ *  sequence of request objects.
+ *
+ *  @param last The iterator that denotes the end of the sequence of
+ *  request objects. 
+ *
+ *  @returns If any outstanding requests have completed, a pair
+ *  containing the status object that corresponds to the completed
+ *  operation and the iterator referencing the completed
+ *  request. Otherwise, an empty @c optional<>.
+ */
+template<typename ForwardIterator>
+optional<std::pair<status, ForwardIterator> >
+test_any(ForwardIterator first, ForwardIterator last)
+{
+  for (ForwardIterator current = first; first != last; ++first) {
+    // Check if we have found a completed request. If so, return it.
+    if (optional<status> result = current->test())
+      return std::make_pair(*result, current);
+  }
+
+  // We found nothing
+  return optional<std::pair<status, ForwardIterator> >();
+}
+
+/** 
+ *  @brief Wait until all non-blocking requests have completed.
+ *
+ *  This routine takes in a set of requests stored in the iterator
+ *  range @c [first,last) and waits until all of these requests have
+ *  been completed. It provides functionality equivalent to 
+ *  @c MPI_Waitall.
+ *
+ *  @param first The iterator that denotes the beginning of the
+ *  sequence of request objects.
+ *
+ *  @param last The iterator that denotes the end of the sequence of
+ *  request objects. 
+ *
+ *  @param out If provided, an output iterator through which the
+ *  status of each request will be emitted. The @c status objects are
+ *  emitted in the same order as the requests are retrieved from 
+ *  @c [first,last).
+ *
+ *  @returns If an @p out parameter was provided, the value @c out
+ *  after all of the @c status objects have been emitted.
+ */
+template<typename ForwardIterator, typename OutputIterator>
+OutputIterator 
+wait_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
+{
+  typedef typename std::iterator_traits<ForwardIterator>::difference_type
+    difference_type;
+
+  using std::distance;
+
+  difference_type num_outstanding_requests = distance(first, last);
+
+  std::vector<status> results(num_outstanding_requests);
+  std::vector<bool> completed(num_outstanding_requests);
+
+  while (num_outstanding_requests > 0) {
+    bool all_trivial_requests = true;
+    difference_type idx = 0;
+    for (ForwardIterator current = first; current != last; ++current, ++idx) {
+      if (!completed[idx]) {
+        if (optional<status> stat = current->test()) {
+          // This outstanding request has been completed. We're done.
+          results[idx] = *stat;
+          completed[idx] = true;
+          --num_outstanding_requests;
+          all_trivial_requests = false;
+        } else {
+          // Check if this request (and all others before it) are "trivial"
+          // requests, e.g., they can be represented with a single
+          // MPI_Request.
+          all_trivial_requests = 
+            all_trivial_requests
+            && !current->m_handler 
+            && current->m_requests[1] == MPI_REQUEST_NULL;          
+        }
+      }
+    }
+
+    // If we have yet to fulfill any requests and all of the requests
+    // are trivial (i.e., require only a single MPI_Request to be
+    // fulfilled), call MPI_Waitall directly.
+    if (all_trivial_requests 
+        && num_outstanding_requests == (difference_type)results.size()) {
+      std::vector<MPI_Request> requests;
+      requests.reserve(num_outstanding_requests);
+      for (ForwardIterator current = first; current != last; ++current)
+        requests.push_back(current->m_requests[0]);
+
+      // Let MPI wait until all of these operations completes.
+      std::vector<MPI_Status> stats(num_outstanding_requests);
+      BOOST_MPI_CHECK_RESULT(MPI_Waitall, 
+                             (num_outstanding_requests, &requests[0], 
+                              &stats[0]));
+
+      for (std::vector<MPI_Status>::iterator i = stats.begin(); 
+           i != stats.end(); ++i, ++out) {
+        status stat;
+        stat.m_status = *i;
+        *out = stat;
+      }
+
+      return out;
+    }
+
+    all_trivial_requests = false;
+  }
+
+  return std::copy(results.begin(), results.end(), out);
+}
+
+/**
+ * \overload
+ */
+template<typename ForwardIterator>
+void
+wait_all(ForwardIterator first, ForwardIterator last)
+{
+  typedef typename std::iterator_traits<ForwardIterator>::difference_type
+    difference_type;
+
+  using std::distance;
+
+  difference_type num_outstanding_requests = distance(first, last);
+
+  std::vector<bool> completed(num_outstanding_requests);
+
+  while (num_outstanding_requests > 0) {
+    bool all_trivial_requests = true;
+
+    difference_type idx = 0;
+    for (ForwardIterator current = first; current != last; ++current, ++idx) {
+      if (!completed[idx]) {
+        if (optional<status> stat = current->test()) {
+          // This outstanding request has been completed.
+          completed[idx] = true;
+          --num_outstanding_requests;
+          all_trivial_requests = false;
+        } else {
+          // Check if this request (and all others before it) are "trivial"
+          // requests, e.g., they can be represented with a single
+          // MPI_Request.
+          all_trivial_requests = 
+            all_trivial_requests
+            && !current->m_handler 
+            && current->m_requests[1] == MPI_REQUEST_NULL;          
+        }
+      }
+    }
+
+    // If we have yet to fulfill any requests and all of the requests
+    // are trivial (i.e., require only a single MPI_Request to be
+    // fulfilled), call MPI_Waitall directly.
+    if (all_trivial_requests 
+        && num_outstanding_requests == (difference_type)completed.size()) {
+      std::vector<MPI_Request> requests;
+      requests.reserve(num_outstanding_requests);
+      for (ForwardIterator current = first; current != last; ++current)
+        requests.push_back(current->m_requests[0]);
+
+      // Let MPI wait until all of these operations completes.
+      BOOST_MPI_CHECK_RESULT(MPI_Waitall, 
+                             (num_outstanding_requests, &requests[0], 
+                              MPI_STATUSES_IGNORE));
+
+      // Signal completion
+      num_outstanding_requests = 0;
+    }
+  }
+}
+
+/** 
+ *  @brief Tests whether all non-blocking requests have completed.
+ *
+ *  This routine takes in a set of requests stored in the iterator
+ *  range @c [first,last) and determines whether all of these requests
+ *  have been completed. However, due to limitations of the underlying
+ *  MPI implementation, if any of the requests refers to a
+ *  non-blocking send or receive of a serialized data type, @c
+ *  test_all will always return the equivalent of @c false (i.e., the
+ *  requests cannot all be finished at this time). This routine
+ *  performs the same functionality as @c wait_all, except that this
+ *  routine will not block. This routine provides functionality
+ *  equivalent to @c MPI_Testall.
+ *
+ *  @param first The iterator that denotes the beginning of the
+ *  sequence of request objects.
+ *
+ *  @param last The iterator that denotes the end of the sequence of
+ *  request objects. 
+ *
+ *  @param out If provided and all requests hav been completed, an
+ *  output iterator through which the status of each request will be
+ *  emitted. The @c status objects are emitted in the same order as
+ *  the requests are retrieved from @c [first,last).
+ *
+ *  @returns If an @p out parameter was provided, the value @c out
+ *  after all of the @c status objects have been emitted (if all
+ *  requests were completed) or an empty @c optional<>. If no @p out
+ *  parameter was provided, returns @c true if all requests have
+ *  completed or @c false otherwise.
+ */
+template<typename ForwardIterator, typename OutputIterator>
+optional<OutputIterator>
+test_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
+{
+  std::vector<MPI_Request> requests;
+  for (; first != last; ++first) {
+    // If we have a non-trivial request, then no requests can be
+    // completed.
+    if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
+      return optional<OutputIterator>();
+
+    requests.push_back(first->m_requests[0]);
+  }
+
+  int flag = 0;
+  int n = requests.size();
+  std::vector<MPI_Status> stats(n);
+  BOOST_MPI_CHECK_RESULT(MPI_Testall, (n, &requests[0], &flag, &stats[0]));
+  if (flag) {
+    for (int i = 0; i < n; ++i, ++out) {
+      status stat;
+      stat.m_status = stats[i];
+      *out = stat;
+    }
+    return out;
+  } else {
+    return optional<OutputIterator>();
+  }
+}
+
+/**
+ *  \overload
+ */
+template<typename ForwardIterator>
+bool
+test_all(ForwardIterator first, ForwardIterator last)
+{
+  std::vector<MPI_Request> requests;
+  for (; first != last; ++first) {
+    // If we have a non-trivial request, then no requests can be
+    // completed.
+    if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
+      return false;
+
+    requests.push_back(first->m_requests[0]);
+  }
+
+  int flag = 0;
+  int n = requests.size();
+  BOOST_MPI_CHECK_RESULT(MPI_Testall, 
+                         (n, &requests[0], &flag, MPI_STATUSES_IGNORE));
+  return flag != 0;
+}
+
+/** 
+ *  @brief Wait until some non-blocking requests have completed.
+ *
+ *  This routine takes in a set of requests stored in the iterator
+ *  range @c [first,last) and waits until at least one of the requests
+ *  has completed. It then completes all of the requests it can,
+ *  partitioning the input sequence into pending requests followed by
+ *  completed requests. If an output iterator is provided, @c status
+ *  objects will be emitted for each of the completed requests. This
+ *  routine provides functionality equivalent to @c MPI_Waitsome.
+ *
+ *  @param first The iterator that denotes the beginning of the
+ *  sequence of request objects.
+ *
+ *  @param last The iterator that denotes the end of the sequence of
+ *  request objects. This may not be equal to @c first.
+ *
+ *  @param out If provided, the @c status objects corresponding to
+ *  completed requests will be emitted through this output iterator.
+
+ *  @returns If the @p out parameter was provided, a pair containing
+ *  the output iterator @p out after all of the @c status objects have
+ *  been written through it and an iterator referencing the first
+ *  completed request. If no @p out parameter was provided, only the
+ *  iterator referencing the first completed request will be emitted.
+ */
+template<typename BidirectionalIterator, typename OutputIterator>
+std::pair<OutputIterator, BidirectionalIterator> 
+wait_some(BidirectionalIterator first, BidirectionalIterator last,
+          OutputIterator out)
+{
+  using std::advance;
+
+  if (first == last)
+    return std::make_pair(out, first);
+  
+  typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
+    difference_type;
+
+  bool all_trivial_requests = true;
+  difference_type n = 0;
+  BidirectionalIterator current = first;
+  BidirectionalIterator start_of_completed = last;
+  while (true) {
+    // Check if we have found a completed request. 
+    if (optional<status> result = current->test()) {
+      using std::iter_swap;
+
+      // Emit the resulting status object
+      *out++ = *result;
+
+      // We're expanding the set of completed requests
+      --start_of_completed;
+
+      if (current == start_of_completed) {
+        // If we have hit the end of the list of pending
+        // requests. Finish up by fixing the order of the completed
+        // set to match the order in which we emitted status objects,
+        // then return.
+        std::reverse(start_of_completed, last);
+        return std::make_pair(out, start_of_completed);
+      }
+
+      // Swap the request we just completed with the last request that
+      // has not yet been tested.
+      iter_swap(current, start_of_completed);
+
+      continue;
+    }
+
+    // Check if this request (and all others before it) are "trivial"
+    // requests, e.g., they can be represented with a single
+    // MPI_Request.
+    all_trivial_requests = 
+      all_trivial_requests
+      && !current->m_handler 
+      && current->m_requests[1] == MPI_REQUEST_NULL;
+
+    // Move to the next request.
+    ++n;
+    if (++current == start_of_completed) {
+      if (start_of_completed != last) {
+        // We have satisfied some requests. Make the order of the
+        // completed requests match that of the status objects we've
+        // already emitted and we're done.
+        std::reverse(start_of_completed, last);
+        return std::make_pair(out, start_of_completed);
+      }
+
+      // We have reached the end of the list. If all requests thus far
+      // have been trivial, we can call MPI_Waitsome directly, because
+      // it may be more efficient than our busy-wait semantics.
+      if (all_trivial_requests) {
+        std::vector<MPI_Request> requests;
+        std::vector<int> indices(n);
+        std::vector<MPI_Status> stats(n);
+        requests.reserve(n);
+        for (current = first; current != last; ++current)
+          requests.push_back(current->m_requests[0]);
+
+        // Let MPI wait until some of these operations complete.
+        int num_completed;
+        BOOST_MPI_CHECK_RESULT(MPI_Waitsome, 
+                               (n, &requests[0], &num_completed, &indices[0],
+                                &stats[0]));
+
+        // Translate the index-based result of MPI_Waitsome into a
+        // partitioning on the requests.
+        int current_offset = 0;
+        current = first;
+        for (int index = 0; index < num_completed; ++index, ++out) {
+          using std::iter_swap;
+
+          // Move "current" to the request object at this index
+          advance(current, indices[index] - current_offset);
+          current_offset = indices[index];
+
+          // Emit the status object
+          status stat;
+          stat.m_status = stats[index];
+          *out = stat;
+
+          // Finish up the request and swap it into the "completed
+          // requests" partition.
+          current->m_requests[0] = requests[indices[index]];
+          --start_of_completed;
+          iter_swap(current, start_of_completed);
+        }
+
+        // We have satisfied some requests. Make the order of the
+        // completed requests match that of the status objects we've
+        // already emitted and we're done.
+        std::reverse(start_of_completed, last);
+        return std::make_pair(out, start_of_completed);
+      }
+
+      // There are some nontrivial requests, so we must continue our
+      // busy waiting loop.
+      n = 0;
+      current = first;
+    }
+  }
+
+  // We cannot ever get here
+  BOOST_ASSERT(false);
+}
+
+/**
+ *  \overload
+ */
+template<typename BidirectionalIterator>
+BidirectionalIterator
+wait_some(BidirectionalIterator first, BidirectionalIterator last)
+{
+  using std::advance;
+
+  if (first == last)
+    return first;
+  
+  typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
+    difference_type;
+
+  bool all_trivial_requests = true;
+  difference_type n = 0;
+  BidirectionalIterator current = first;
+  BidirectionalIterator start_of_completed = last;
+  while (true) {
+    // Check if we have found a completed request. 
+    if (optional<status> result = current->test()) {
+      using std::iter_swap;
+
+      // We're expanding the set of completed requests
+      --start_of_completed;
+
+      // If we have hit the end of the list of pending requests, we're
+      // done.
+      if (current == start_of_completed)
+        return start_of_completed;
+
+      // Swap the request we just completed with the last request that
+      // has not yet been tested.
+      iter_swap(current, start_of_completed);
+
+      continue;
+    }
+
+    // Check if this request (and all others before it) are "trivial"
+    // requests, e.g., they can be represented with a single
+    // MPI_Request.
+    all_trivial_requests = 
+      all_trivial_requests
+      && !current->m_handler 
+      && current->m_requests[1] == MPI_REQUEST_NULL;
+
+    // Move to the next request.
+    ++n;
+    if (++current == start_of_completed) {
+        // If we have satisfied some requests, we're done.
+      if (start_of_completed != last)
+        return start_of_completed;
+
+      // We have reached the end of the list. If all requests thus far
+      // have been trivial, we can call MPI_Waitsome directly, because
+      // it may be more efficient than our busy-wait semantics.
+      if (all_trivial_requests) {
+        std::vector<MPI_Request> requests;
+        std::vector<int> indices(n);
+        requests.reserve(n);
+        for (current = first; current != last; ++current)
+          requests.push_back(current->m_requests[0]);
+
+        // Let MPI wait until some of these operations complete.
+        int num_completed;
+        BOOST_MPI_CHECK_RESULT(MPI_Waitsome, 
+                               (n, &requests[0], &num_completed, &indices[0],
+                                MPI_STATUSES_IGNORE));
+
+        // Translate the index-based result of MPI_Waitsome into a
+        // partitioning on the requests.
+        int current_offset = 0;
+        current = first;
+        for (int index = 0; index < num_completed; ++index) {
+          using std::iter_swap;
+
+          // Move "current" to the request object at this index
+          advance(current, indices[index] - current_offset);
+          current_offset = indices[index];
+
+          // Finish up the request and swap it into the "completed
+          // requests" partition.
+          current->m_requests[0] = requests[indices[index]];
+          --start_of_completed;
+          iter_swap(current, start_of_completed);
+        }
+
+        // We have satisfied some requests, so we are done.
+        return start_of_completed;
+      }
+
+      // There are some nontrivial requests, so we must continue our
+      // busy waiting loop.
+      n = 0;
+      current = first;
+    }
+  }
+
+  // We cannot ever get here
+  BOOST_ASSERT(false);
+}
+
+/** 
+ *  @brief Test whether some non-blocking requests have completed.
+ *
+ *  This routine takes in a set of requests stored in the iterator
+ *  range @c [first,last) and tests to see if any of the requests has
+ *  completed. It completes all of the requests it can, partitioning
+ *  the input sequence into pending requests followed by completed
+ *  requests. If an output iterator is provided, @c status objects
+ *  will be emitted for each of the completed requests. This routine
+ *  is similar to @c wait_some, but does not wait until any requests
+ *  have completed. This routine provides functionality equivalent to
+ *  @c MPI_Testsome.
+ *
+ *  @param first The iterator that denotes the beginning of the
+ *  sequence of request objects.
+ *
+ *  @param last The iterator that denotes the end of the sequence of
+ *  request objects. This may not be equal to @c first.
+ *
+ *  @param out If provided, the @c status objects corresponding to
+ *  completed requests will be emitted through this output iterator.
+
+ *  @returns If the @p out parameter was provided, a pair containing
+ *  the output iterator @p out after all of the @c status objects have
+ *  been written through it and an iterator referencing the first
+ *  completed request. If no @p out parameter was provided, only the
+ *  iterator referencing the first completed request will be emitted.
+ */
+template<typename BidirectionalIterator, typename OutputIterator>
+std::pair<OutputIterator, BidirectionalIterator> 
+test_some(BidirectionalIterator first, BidirectionalIterator last,
+          OutputIterator out)
+{
+  BidirectionalIterator current = first;
+  BidirectionalIterator start_of_completed = last;
+  while (current != start_of_completed) {
+    // Check if we have found a completed request. 
+    if (optional<status> result = current->test()) {
+      using std::iter_swap;
+
+      // Emit the resulting status object
+      *out++ = *result;
+
+      // We're expanding the set of completed requests
+      --start_of_completed;
+
+      // Swap the request we just completed with the last request that
+      // has not yet been tested.
+      iter_swap(current, start_of_completed);
+
+      continue;
+    }
+
+    // Move to the next request.
+    ++current;
+  }
+
+  // Finish up by fixing the order of the completed set to match the
+  // order in which we emitted status objects, then return.
+  std::reverse(start_of_completed, last);
+  return std::make_pair(out, start_of_completed);
+}
+
+/**
+ *  \overload
+ */
+template<typename BidirectionalIterator>
+BidirectionalIterator
+test_some(BidirectionalIterator first, BidirectionalIterator last)
+{
+  BidirectionalIterator current = first;
+  BidirectionalIterator start_of_completed = last;
+  while (current != start_of_completed) {
+    // Check if we have found a completed request. 
+    if (optional<status> result = current->test()) {
+      using std::iter_swap;
+
+      // We're expanding the set of completed requests
+      --start_of_completed;
+
+      // Swap the request we just completed with the last request that
+      // has not yet been tested.
+      iter_swap(current, start_of_completed);
+
+      continue;
+    }
+
+    // Move to the next request.
+    ++current;
+  }
+
+  return start_of_completed;
+}
+
+} } } // end namespace boost::parallel::mpi
+
+
+#endif // BOOST_PARALLEL_MPI_NONBLOCKING_HPP

include/boost/mpi/operations.hpp

@@ -0,0 +1,322 @@
+// Copyright (C) 2004 The Trustees of Indiana University.
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+//           Andrew Lumsdaine
+
+/** @file operations.hpp
+ *
+ *  This header provides a mapping from function objects to @c MPI_Op
+ *  constants used in MPI collective operations. It also provides
+ *  several new function object types not present in the standard @c
+ *  <functional> header that have direct mappings to @c MPI_Op.
+ */
+#ifndef BOOST_PARALLEL_MPI_IS_MPI_OP_HPP
+#define BOOST_PARALLEL_MPI_IS_MPI_OP_HPP
+
+#include <mpi.h>
+#include <boost/mpl/bool.hpp>
+#include <boost/mpl/if.hpp>
+#include <boost/mpl/and.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/utility/enable_if.hpp>
+#include <functional>
+
+namespace boost { namespace parallel { namespace mpi {
+
+template<typename Op, typename T> struct is_mpi_op;
+
+/**
+ * @brief Determine if a function object type is commutative.
+ *
+ * This trait determines if an operation @c Op is commutative when
+ * applied to values of type @c T. Parallel operations such as @c
+ * reduce and @c prefix_sum can be implemented more efficiently with
+ * commutative operations. To mark an operation as commutative, users
+ * should specialize @c is_commutative and derive from the class @c
+ * mpl::true_.
+ */
+template<typename Op, typename T>
+struct is_commutative : public mpl::false_ { };
+
+/**************************************************************************
+ * Function objects for MPI operations not in <functional> header         *
+ **************************************************************************/
+
+/**
+ *  @brief Compute the maximum of two values.
+ *
+ *  This binary function object computes the maximum of the two values
+ *  it is given. When used with MPI and a type @c T that has an
+ *  associated, built-in MPI data type, translates to @c MPI_MAX.
+ */
+template<typename T>
+struct maximum : public std::binary_function<T, T, T>
+{
+  /** @returns the maximum of x and y. */
+  const T& operator()(const T& x, const T& y) const
+  {
+    return x < y? y : x;
+  }
+};
+
+/**
+ *  @brief Compute the minimum of two values.
+ *
+ *  This binary function object computes the minimum of the two values
+ *  it is given. When used with MPI and a type @c T that has an
+ *  associated, built-in MPI data type, translates to @c MPI_MIN.
+ */
+template<typename T>
+struct minimum : public std::binary_function<T, T, T>
+{
+  /** @returns the minimum of x and y. */
+  const T& operator()(const T& x, const T& y) const
+  {
+    return x < y? x : y;
+  }
+};
+
+
+/**
+ *  @brief Compute the bitwise AND of two integral values.
+ *
+ *  This binary function object computes the bitwise AND of the two
+ *  values it is given. When used with MPI and a type @c T that has an
+ *  associated, built-in MPI data type, translates to @c MPI_BAND.
+ */
+template<typename T>
+struct bitwise_and : public std::binary_function<T, T, T>
+{
+  /** @returns @c x & y. */
+  T operator()(const T& x, const T& y) const
+  {
+    return x & y;
+  }
+};
+
+/**
+ *  @brief Compute the bitwise OR of two integral values.
+ *
+ *  This binary function object computes the bitwise OR of the two
+ *  values it is given. When used with MPI and a type @c T that has an
+ *  associated, built-in MPI data type, translates to @c MPI_BOR.
+ */
+template<typename T>
+struct bitwise_or : public std::binary_function<T, T, T>
+{
+  /** @returns the @c x | y. */
+  T operator()(const T& x, const T& y) const
+  {
+    return x | y;
+  }
+};
+
+/**
+ *  @brief Compute the logical exclusive OR of two integral values.
+ *
+ *  This binary function object computes the logical exclusive of the
+ *  two values it is given. When used with MPI and a type @c T that has
+ *  an associated, built-in MPI data type, translates to @c MPI_LXOR.
+ */
+template<typename T>
+struct logical_xor : public std::binary_function<T, T, T>
+{
+  /** @returns the logical exclusive OR of x and y. */
+  T operator()(const T& x, const T& y) const
+  {
+    return (x || y) && !(x && y);
+  }
+};
+
+/**
+ *  @brief Compute the bitwise exclusive OR of two integral values.
+ *
+ *  This binary function object computes the bitwise exclusive OR of
+ *  the two values it is given. When used with MPI and a type @c T that
+ *  has an associated, built-in MPI data type, translates to @c
+ *  MPI_BXOR.
+ */
+template<typename T>
+struct bitwise_xor : public std::binary_function<T, T, T>
+{
+  /** @returns @c x ^ y. */
+  T operator()(const T& x, const T& y) const
+  {
+    return x ^ y;
+  }
+};
+
+/**************************************************************************
+ * MPI_Op queries                                                         *
+ **************************************************************************/
+
+/**
+ *  @brief Determine if a function object has an associated @c MPI_Op.
+ *
+ *  This trait determines if a function object type @c Op, when used
+ *  with argument type @c T, has an associated @c MPI_Op. If so, @c
+ *  is_mpi_op<Op,T> will derive from @c mpl::false_ and will
+ *  contain a static member function @c op that takes no arguments but
+ *  returns the associated @c MPI_Op value. For instance, @c
+ *  is_mpi_op<std::plus<int>,int>::op() returns @c MPI_SUM.
+ *
+ *  Users may specialize @c is_mpi_op for any other class templates
+ *  that map onto operations that have @c MPI_Op equivalences, such as
+ *  bitwise OR, logical and, or maximum. However, users are encouraged
+ *  to use the standard function objects in the @c functional and @c
+ *  boost/parallel/mpi/operations.hpp headers whenever possible. For
+ *  function objects that are class templates with a single template
+ *  parameter, it may be easier to specialize @c is_builtin_mpi_op.
+ */
+template<typename Op, typename T>
+struct is_mpi_op : public mpl::false_ { };
+
+/// INTERNAL ONLY
+template<typename T>
+struct is_mpi_op<maximum<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_floating_point_datatype<T> >
+{
+  static MPI_Op op() { return MPI_MAX; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+struct is_mpi_op<minimum<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_floating_point_datatype<T> >
+{
+  static MPI_Op op() { return MPI_MIN; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<std::plus<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_floating_point_datatype<T>,
+                           is_mpi_complex_datatype<T> >
+{
+  static MPI_Op op() { return MPI_SUM; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<std::multiplies<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_floating_point_datatype<T>,
+                           is_mpi_complex_datatype<T> >
+{
+  static MPI_Op op() { return MPI_PROD; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<std::logical_and<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_logical_datatype<T> >
+{
+  static MPI_Op op() { return MPI_LAND; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<std::logical_or<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_logical_datatype<T> >
+{
+  static MPI_Op op() { return MPI_LOR; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<logical_xor<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_logical_datatype<T> >
+{
+  static MPI_Op op() { return MPI_LXOR; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<bitwise_and<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_byte_datatype<T> >
+{
+  static MPI_Op op() { return MPI_BAND; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<bitwise_or<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_byte_datatype<T> >
+{
+  static MPI_Op op() { return MPI_BOR; }
+};
+
+/// INTERNAL ONLY
+template<typename T>
+ struct is_mpi_op<bitwise_xor<T>, T>
+  : public boost::mpl::or_<is_mpi_integer_datatype<T>,
+                           is_mpi_byte_datatype<T> >
+{
+  static MPI_Op op() { return MPI_BXOR; }
+};
+
+namespace detail {
+  // A helper class used to create user-defined MPI_Ops
+  template<typename Op, typename T>
+  class user_op
+  {
+  public:
+    explicit user_op(Op& op)
+    {
+      BOOST_MPI_CHECK_RESULT(MPI_Op_create,
+                             (&user_op<Op, T>::perform,
+                              is_commutative<Op, T>::value,
+                              &mpi_op));
+
+      op_ptr = &op;
+    }
+
+    ~user_op()
+    {
+      if (std::uncaught_exception()) {
+        // Ignore failure cases: there are obviously other problems
+        // already, and we don't want to cause program termination if
+        // MPI_Op_free fails.
+        MPI_Op_free(&mpi_op);
+      } else {
+        BOOST_MPI_CHECK_RESULT(MPI_Op_free, (&mpi_op));
+      }
+    }
+
+    MPI_Op& get_mpi_op()
+    {
+      return mpi_op;
+    }
+
+  private:
+    MPI_Op mpi_op;
+    static Op* op_ptr;
+
+    static void perform(void* vinvec, void* voutvec, int* plen, MPI_Datatype*)
+    {
+      T* invec = static_cast<T*>(vinvec);
+      T* outvec = static_cast<T*>(voutvec);
+      std::transform(invec, invec + *plen, outvec, outvec, *op_ptr);
+    }
+  };
+
+  template<typename Op, typename T> Op* user_op<Op, T>::op_ptr = 0;
+
+} // end namespace detail
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_GET_MPI_OP_HPP

include/boost/mpi/packed_iarchive.hpp

@@ -0,0 +1,92 @@
+// (C) Copyright 2005 Matthias Troyer
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+//           Douglas Gregor
+
+/** @file packed_iarchive.hpp
+ *
+ *  This header provides the facilities for packing Serializable data
+ *  types into a buffer using @c MPI_Pack. The buffers can then be
+ *  transmitted via MPI and then be unpacked either via the facilities
+ *  in @c packed_oarchive.hpp or @c MPI_Unpack.
+ */
+#ifndef BOOST_PARALLEL_MPI_PACKED_IARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_PACKED_IARCHIVE_HPP
+
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/basic_binary_iarchive.hpp>
+#include <boost/parallel/mpi/detail/packed_iprimitive.hpp>
+#include <boost/assert.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/** @brief An archive that packs binary data into an MPI buffer.
+ *
+ *  The @c packed_iarchive class is an Archiver (as in the
+ *  Boost.Serialization library) that packs binary data into a buffer
+ *  for transmission via MPI. It can operate on any Serializable data
+ *  type and will use the @c MPI_Pack function of the underlying MPI
+ *  implementation to perform serialization.
+ */
+class packed_iarchive
+  : public packed_iprimitive,
+    public archive::basic_binary_iarchive<packed_iarchive>
+{
+public:
+  /**
+   *  Construct a @c packed_iarchive for transmission over the given
+   *  MPI communicator and with an initial buffer.
+   *
+   *  @param comm The communicator over which this archive will be
+   *  sent.
+   *
+   *  @param b A user-defined buffer that will be filled with the
+   *  binary representation of serialized objects.
+   *
+   *  @param flags Control the serialization of the data types. Refer
+   *  to the Boost.Serialization documentation before changing the
+   *  default flags.
+   *
+   *  @param position Set the offset into buffer @p b at which
+   *  deserialization will begin.
+   */
+  packed_iarchive(MPI_Comm const & comm, buffer_type & b, unsigned int flags = boost::archive::no_header, int position = 0)
+        : packed_iprimitive(b,comm,position),
+          archive::basic_binary_iarchive<packed_iarchive>(flags)
+        {}
+
+  /**
+   *  Construct a @c packed_iarchive for transmission over the given
+   *  MPI communicator.
+   *
+   *  @param comm The communicator over which this archive will be
+   *  sent.
+   *
+   *  @param flags Control the serialization of the data types. Refer
+   *  to the Boost.Serialization documentation before changing the
+   *  default flags.
+   */
+  packed_iarchive
+          ( MPI_Comm const & comm , unsigned int flags = boost::archive::no_header)
+         : packed_iprimitive(internal_buffer_,comm),
+           archive::basic_binary_iarchive<packed_iarchive>(flags)
+        {}
+
+private:
+  /// An internal buffer to be used when the user does not supply his
+  /// own buffer.
+  buffer_type internal_buffer_;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+
+BOOST_BROKEN_COMPILER_TYPE_TRAITS_SPECIALIZATION(parallel::mpi::packed_iarchive)
+
+#endif // BOOST_PARALLEL_MPI_PACKED_IARCHIVE_HPP

include/boost/mpi/packed_oarchive.hpp

@@ -0,0 +1,85 @@
+// (C) Copyright 2005 Matthias Troyer
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+//           Douglas Gregor
+
+/** @file packed_oarchive.hpp
+ *
+ *  This header provides the facilities for unpacking Serializable
+ *  data types from a buffer using @c MPI_Unpack. The buffers are
+ *  typically received via MPI and have been packed either by via the
+ *  facilities in @c packed_iarchive.hpp or @c MPI_Pack.
+ */
+#ifndef BOOST_PARALLEL_MPI_PACKED_OARCHIVE_HPP
+#define BOOST_PARALLEL_MPI_PACKED_OARCHIVE_HPP
+
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/archive/basic_binary_oarchive.hpp>
+#include <boost/parallel/mpi/detail/packed_oprimitive.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/** @brief An archive that unpacks binary data from an MPI buffer.
+ *
+ *  The @c packed_oarchive class is an Archiver (as in the
+ *  Boost.Serialization library) that unpacks binary data from a
+ *  buffer received via MPI. It can operate on any Serializable data
+ *  type and will use the @c MPI_Unpack function of the underlying MPI
+ *  implementation to perform deserialization.
+ */
+class packed_oarchive
+  : public packed_oprimitive,
+    public archive::basic_binary_oarchive<packed_oarchive>
+{
+public:
+  /**
+   *  Construct a @c packed_oarchive to receive data over the given
+   *  MPI communicator and with an initial buffer.
+   *
+   *  @param comm The communicator over which this archive will be
+   *  received.
+   *
+   *  @param b A user-defined buffer that contains the binary
+   *  representation of serialized objects.
+   *
+   *  @param flags Control the serialization of the data types. Refer
+   *  to the Boost.Serialization documentation before changing the
+   *  default flags.
+   */
+  packed_oarchive( MPI_Comm const & comm, buffer_type & b, unsigned int flags = boost::archive::no_header)
+         : packed_oprimitive(b,comm),
+           archive::basic_binary_oarchive<packed_oarchive>(flags)
+        {}
+
+  /**
+   *  Construct a @c packed_oarchive to receive data over the given
+   *  MPI communicator.
+   *
+   *  @param comm The communicator over which this archive will be
+   *  received.
+   *
+   *  @param flags Control the serialization of the data types. Refer
+   *  to the Boost.Serialization documentation before changing the
+   *  default flags.
+   */
+  packed_oarchive ( MPI_Comm const & comm, unsigned int flags =  boost::archive::no_header)
+         : packed_oprimitive(internal_buffer_,comm),
+           archive::basic_binary_oarchive<packed_oarchive>(flags)
+        {}
+
+
+private:
+  /// An internal buffer to be used when the user does not supply his
+  /// own buffer.
+  buffer_type internal_buffer_;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_PACKED_OARCHIVE_HPP

include/boost/mpi/python.hpp

@@ -0,0 +1,79 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+#ifndef BOOST_PARALLEL_MPI_PYTHON_HPP
+#define BOOST_PARALLEL_MPI_PYTHON_HPP
+
+#include <boost/python/object.hpp>
+
+/** @file python.hpp
+ *
+ *  This header interacts with the Python bindings for Boost.MPI. The
+ *  routines in this header can be used to register user-defined and
+ *  library-defined data types with Boost.MPI for efficient
+ *  (de-)serialization and separate transmission of skeletons and
+ *  content.
+ *
+ */
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+/**
+ * @brief Register the type T for direct serialization within Boost.MPI
+ *
+ * The @c register_serialized function registers a C++ type for direct
+ * serialization within Boost.MPI. Direct serialization elides the use
+ * of the Python @c pickle package when serializing Python objects
+ * that represent C++ values. Direct serialization can be beneficial
+ * both to improve serialization performance (Python pickling can be
+ * very inefficient) and to permit serialization for Python-wrapped
+ * C++ objects that do not support pickling.
+ *
+ *  @param value A sample value of the type @c T. This may be used
+ *  to compute the Python type associated with the C++ type @c T.
+ *
+ *  @param type The Python type associated with the C++ type @c
+ *  T. If not provided, it will be computed from the same value @p
+ *  value.
+ */
+template<typename T>
+void
+register_serialized(const T& value = T(), PyTypeObject* type = 0);
+
+/**
+ * @brief Registers a type for use with the skeleton/content mechanism
+ * in Python.
+ *
+ * The skeleton/content mechanism can only be used from Python with
+ * C++ types that have previously been registered via a call to this
+ * function. Both the sender and the transmitter must register the
+ * type. It is permitted to call this function multiple times for the
+ * same type @c T, but only one call per process per type is
+ * required. The type @c T must be Serializable.
+ *
+ *  @param value A sample object of type T that will be used to
+ *  determine the Python type associated with T, if @p type is not
+ *  specified.
+ *
+ *  @param type The Python type associated with the C++ type @c
+ *  T. If not provided, it will be computed from the same value @p
+ *  value.
+ */
+template<typename T>
+void 
+register_skeleton_and_content(const T& value = T(), PyTypeObject* type = 0);
+
+} } } } // end namespace boost::parallel::mpi::python
+
+#ifndef BOOST_PARALLEL_MPI_PYTHON_FORWARD_ONLY
+#  include <boost/parallel/mpi/python/serialize.hpp>
+#  include <boost/parallel/mpi/python/skeleton_and_content.hpp>
+#else
+#  undef BOOST_PARALLEL_MPI_PYTHON_FORWARD_ONLY
+#endif
+
+#endif // BOOST_PARALLEL_MPI_PYTHON_HPP

include/boost/mpi/python/serialize.hpp

@@ -0,0 +1,533 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file serialize.hpp
+ *
+ *  This file provides Boost.Serialization support for Python objects
+ *  within Boost.MPI. Python objects can be serialized in one of two
+ *  ways. The default serialization method involves using the Python
+ *  "pickle" module to pickle the Python objects, transmits the
+ *  pickled representation, and unpickles the result when
+ *  received. For C++ types that have been exposed to Python and
+ *  registered with register_serialized(), objects are directly
+ *  serialized for transmissing, skipping the pickling step.
+ */
+#ifndef BOOST_PARALLEL_MPI_PYTHON_SERIALIZE_HPP
+#define BOOST_PARALLEL_MPI_PYTHON_SERIALIZE_HPP
+
+#include <boost/python/object.hpp>
+#include <boost/python/str.hpp>
+#include <boost/python/extract.hpp>
+
+#include <memory>
+#include <map>
+
+#include <boost/function/function3.hpp>
+
+#include <boost/mpl/bool.hpp>
+#include <boost/mpl/if.hpp>
+
+#include <boost/serialization/split_free.hpp>
+#include <boost/serialization/array.hpp>
+
+#include <boost/assert.hpp>
+
+#include <boost/type_traits/is_fundamental.hpp>
+
+#define BOOST_PARALLEL_MPI_PYTHON_FORWARD_ONLY
+#include <boost/parallel/mpi/python.hpp>
+
+/************************************************************************
+ * Boost.Python Serialization Section                                   *
+ ************************************************************************/
+#if !defined(BOOST_NO_SFINAE) && !defined(BOOST_NO_IS_CONVERTIBLE)
+/**
+ * @brief Declare IArchive and OArchive as a Boost.Serialization
+ * archives that can be used for Python objects.
+ *
+ * This macro can only be expanded from the global namespace. It only
+ * requires that Archiver be forward-declared. IArchiver and OArchiver
+ * will only support Serialization of Python objects by pickling
+ * them. If the Archiver type should also support "direct"
+ * serialization (for C++ types), use
+ * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE instead.
+ */
+#  define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)        \
+namespace boost { namespace python { namespace api {    \
+  template<typename R, typename T>                      \
+  struct enable_binary< IArchiver , R, T> {};           \
+                                                        \
+  template<typename R, typename T>                      \
+  struct enable_binary< OArchiver , R, T> {};           \
+} } } 
+# else
+#  define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)
+#endif
+
+/**
+ * @brief Declare IArchiver and OArchiver as a Boost.Serialization
+ * archives that can be used for Python objects and C++ objects
+ * wrapped in Python.
+ *
+ * This macro can only be expanded from the global namespace. It only
+ * requires that IArchiver and OArchiver be forward-declared. However,
+ * note that you will also need to write
+ * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver,
+ * OArchiver) in one of your translation units.
+
+DPG PICK UP HERE
+ */
+#define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \
+BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)                \
+namespace boost { namespace python { namespace detail {                 \
+template<>                                                              \
+direct_serialization_table< IArchiver , OArchiver >&                    \
+ get_direct_serialization_table< IArchiver , OArchiver >();             \
+}                                                                       \
+                                                                        \
+template<>                                                              \
+struct has_direct_serialization< IArchiver , OArchiver> : mpl::true_ { }; \
+                                                                        \
+template<>                                                              \
+struct output_archiver< IArchiver > { typedef OArchiver type; };        \
+                                                                        \
+template<>                                                              \
+struct input_archiver< OArchiver > { typedef IArchiver type; };         \
+} }
+
+/**
+ * @brief Define the implementation for Boost.Serialization archivers
+ * that can be used for Python objects and C++ objects wrapped in
+ * Python.
+ *
+ * This macro can only be expanded from the global namespace. It only
+ * requires that IArchiver and OArchiver be forward-declared. Before
+ * using this macro, you will need to declare IArchiver and OArchiver
+ * as direct serialization archives with
+ * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver).
+ */
+#define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver, OArchiver) \
+namespace boost { namespace python { namespace detail {                 \
+template<>                                                              \
+ direct_serialization_table< IArchiver , OArchiver >&                   \
+ get_direct_serialization_table< IArchiver , OArchiver >( )             \
+{                                                                       \
+  static direct_serialization_table< IArchiver, OArchiver > table;      \
+  return table;                                                         \
+}                                                                       \
+} } }
+
+namespace boost { namespace python {
+
+/**
+ * INTERNAL ONLY
+ *
+ * Provides access to the Python "pickle" module from within C++.
+ */
+class pickle {
+  struct data_t;
+
+public:
+  static str dumps(object obj, int protocol = -1);
+  static object loads(str s);
+  
+private:
+  static void initialize_data();
+
+  static data_t* data;
+};
+
+/**
+ * @brief Whether the input/output archiver pair has "direct"
+ * serialization for C++ objects exposed in Python.
+ *
+ * Users do not typically need to specialize this trait, as it will be
+ * specialized as part of the macro
+ * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
+ */
+template<typename IArchiver, typename OArchiver>
+struct has_direct_serialization : mpl::false_ { };
+
+/**
+ *  @brief A metafunction that determines the output archiver for the
+ *  given input archiver.
+ *
+ * Users do not typically need to specialize this trait, as it will be
+ * specialized as part of the macro
+ * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
+ */
+template<typename IArchiver> struct output_archiver { };
+
+/**
+ *  @brief A metafunction that determines the input archiver for the
+ *  given output archiver.
+ *
+ * Users do not typically need to specialize this trait, as it will be
+ * specialized as part of the macro
+ * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
+ *
+ */
+template<typename OArchiver> struct input_archiver { };
+
+namespace detail {
+
+  /**
+   * INTERNAL ONLY
+   *
+   * This class contains the direct-serialization code for the given
+   * IArchiver/OArchiver pair. It is intended to be used as a
+   * singleton class, and will be accessed when (de-)serializing a
+   * Boost.Python object with an archiver that supports direct
+   * serializations. Do not create instances of this class directly:
+   * instead, use get_direct_serialization_table.
+   */
+  template<typename IArchiver, typename OArchiver>
+  class direct_serialization_table
+  {
+  public:
+    typedef boost::function3<void, OArchiver&, const object&, const unsigned int>
+      saver_t;
+    typedef boost::function3<void, IArchiver&, object&, const unsigned int>
+      loader_t;
+
+    typedef std::map<PyTypeObject*, std::pair<int, saver_t> > savers_t;
+    typedef std::map<int, loader_t> loaders_t;
+
+    /**
+     * Retrieve the saver (serializer) associated with the Python
+     * object @p obj.
+     *
+     *   @param obj The object we want to save. Only its (Python) type
+     *   is important.
+     *
+     *   @param descriptor The value of the descriptor associated to
+     *   the returned saver. Will be set to zero if no saver was found
+     *   for @p obj.
+     *
+     *   @returns a function object that can be used to serialize this
+     *   object (and other objects of the same type), if possible. If
+     *   no saver can be found, returns an empty function object..
+     */
+    saver_t saver(const object& obj, int& descriptor)
+    {
+      typename savers_t::iterator pos = savers.find(obj.ptr()->ob_type);
+      if (pos != savers.end()) {
+        descriptor = pos->second.first;
+        return pos->second.second;
+      }
+      else {
+        descriptor = 0;
+        return saver_t();
+      }
+    }
+
+    /**
+     * Retrieve the loader (deserializer) associated with the given
+     * descriptor.
+     *
+     *  @param descriptor The descriptor number provided by saver()
+     *  when determining the saver for this type.
+     *
+     *  @returns a function object that can be used to deserialize an
+     *  object whose type is the same as that corresponding to the
+     *  descriptor. If the descriptor is unknown, the return value
+     *  will be an empty function object.
+     */
+    loader_t loader(int descriptor)
+    {
+      typename loaders_t::iterator pos = loaders.find(descriptor);
+      if (pos != loaders.end())
+        return pos->second;
+      else
+        return loader_t();
+    }
+
+    /**
+     * Register the type T for direct serialization.
+     *
+     *  @param value A sample value of the type @c T. This may be used
+     *  to compute the Python type associated with the C++ type @c T.
+     *
+     *  @param type The Python type associated with the C++ type @c
+     *  T. If not provided, it will be computed from the same value @p
+     *  value.
+     */
+    template<typename T>
+    void register_type(const T& value = T(), PyTypeObject* type = 0)
+    {
+      // If the user did not provide us with a Python type, figure it
+      // out for ourselves.
+      if (!type) {
+        object obj(value);
+        type = obj.ptr()->ob_type;
+      }
+
+      register_type(default_saver<T>(), default_loader<T>(type), value, type);
+    }
+
+    /**
+     * Register the type T for direct serialization.
+     *
+     *  @param saver A function object that will serialize a
+     *  Boost.Python object (that represents a C++ object of type @c
+     *  T) to an @c OArchive.
+     *
+     *  @param loader A function object that will deserialize from an
+     *  @c IArchive into a Boost.Python object that represents a C++
+     *  object of type @c T.
+     *
+     *  @param value A sample value of the type @c T. This may be used
+     *  to compute the Python type associated with the C++ type @c T.
+     *
+     *  @param type The Python type associated with the C++ type @c
+     *  T. If not provided, it will be computed from the same value @p
+     *  value.
+     */
+    template<typename T>
+    void register_type(const saver_t& saver, const loader_t& loader, 
+                       const T& value = T(), PyTypeObject* type = 0)
+    {
+      // If the user did not provide us with a Python type, figure it
+      // out for ourselves.
+      if (!type) {
+        object obj(value);
+        type = obj.ptr()->ob_type;
+      }
+
+      int descriptor = savers.size() + 1;
+      if (savers.find(type) != savers.end())
+        return;
+
+      savers[type] = std::make_pair(descriptor, saver);
+      loaders[descriptor] = loader;
+    }
+
+  protected:
+    template<typename T>
+    struct default_saver {
+      void operator()(OArchiver& ar, const object& obj, const unsigned int) {
+        typedef typename mpl::if_<is_fundamental<T>, T, T&>::type Type;
+        ar << extract<Type>(obj)();
+      }
+    };
+
+    template<typename T>
+    struct default_loader {
+      default_loader(PyTypeObject* type) : type(type) { }
+
+      void operator()(IArchiver& ar, object& obj, const unsigned int) {
+        // If we can, extract the object in place.
+        if (!is_fundamental<T>::value && obj && obj.ptr()->ob_type == type) {
+          ar >> extract<T&>(obj)();
+        } else {
+          T value;
+          ar >> value;
+          obj = object(value);
+        }
+      }
+
+    private:
+      PyTypeObject* type;
+    };
+
+    savers_t savers;
+    loaders_t loaders;
+  };
+
+  /**
+   * @brief Retrieve the direct-serialization table for an
+   * IArchiver/OArchiver pair.
+   *
+   * This function is responsible for returning a reference to the
+   * singleton direct-serialization table. Its primary template is
+   * left undefined, to force the use of an explicit specialization
+   * with a definition in a single translation unit. Use the macro
+   * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL to define this
+   * explicit specialization.
+   */
+  template<typename IArchiver, typename OArchiver>
+  direct_serialization_table<IArchiver, OArchiver>&
+  get_direct_serialization_table();
+} // end namespace detail 
+
+/**
+ * @brief Register the type T for direct serialization.
+ *
+ * The @c register_serialized function registers a C++ type for direct
+ * serialization with the given @c IArchiver/@c OArchiver pair. Direct
+ * serialization elides the use of the Python @c pickle package when
+ * serializing Python objects that represent C++ values. Direct
+ * serialization can be beneficial both to improve serialization
+ * performance (Python pickling can be very inefficient) and to permit
+ * serialization for Python-wrapped C++ objects that do not support
+ * pickling.
+ *
+ *  @param value A sample value of the type @c T. This may be used
+ *  to compute the Python type associated with the C++ type @c T.
+ *
+ *  @param type The Python type associated with the C++ type @c
+ *  T. If not provided, it will be computed from the same value @p
+ *  value.
+ */
+template<typename IArchiver, typename OArchiver, typename T>
+void
+register_serialized(const T& value = T(), PyTypeObject* type = 0)
+{
+  detail::direct_serialization_table<IArchiver, OArchiver>& table = 
+    detail::get_direct_serialization_table<IArchiver, OArchiver>();
+  table.register_type(value, type);
+}
+
+namespace detail {
+
+/// Save a Python object by pickling it.
+template<typename Archiver>
+void 
+save_impl(Archiver& ar, const boost::python::object& obj, 
+          const unsigned int /*version*/,
+          mpl::false_ /*has_direct_serialization*/)
+{
+  boost::python::str py_string = boost::python::pickle::dumps(obj);
+  int len = boost::python::extract<int>(py_string.attr("__len__")());
+  const char* string = boost::python::extract<const char*>(py_string);
+  ar << len << boost::serialization::make_array(string, len);
+}
+
+/// Try to save a Python object by directly serializing it; fall back
+/// on pickling if required.
+template<typename Archiver>
+void 
+save_impl(Archiver& ar, const boost::python::object& obj, 
+          const unsigned int version,
+          mpl::true_ /*has_direct_serialization*/)
+{
+  typedef Archiver OArchiver;
+  typedef typename input_archiver<OArchiver>::type IArchiver;
+  typedef typename direct_serialization_table<IArchiver, OArchiver>::saver_t
+    saver_t;
+
+  direct_serialization_table<IArchiver, OArchiver>& table = 
+    get_direct_serialization_table<IArchiver, OArchiver>();
+
+  int descriptor = 0;
+  if (saver_t saver = table.saver(obj, descriptor)) {
+    ar << descriptor;
+    saver(ar, obj, version);
+  } else {
+    // Pickle it
+    ar << descriptor;
+    detail::save_impl(ar, obj, version, mpl::false_());
+  }
+}
+
+/// Load a Python object by unpickling it
+template<typename Archiver>
+void 
+load_impl(Archiver& ar, boost::python::object& obj, 
+          const unsigned int /*version*/, 
+          mpl::false_ /*has_direct_serialization*/)
+{
+  int len;
+  ar >> len;
+
+  std::auto_ptr<char> string(new char[len]);
+  ar >> boost::serialization::make_array(string.get(), len);
+  boost::python::str py_string(string.get(), len);
+  obj = boost::python::pickle::loads(py_string);
+}
+
+/// Try to load a Python object by directly deserializing it; fall back
+/// on unpickling if required.
+template<typename Archiver>
+void 
+load_impl(Archiver& ar, boost::python::object& obj, 
+          const unsigned int version,
+          mpl::true_ /*has_direct_serialization*/)
+{
+  typedef Archiver IArchiver;
+  typedef typename output_archiver<IArchiver>::type OArchiver;
+  typedef typename direct_serialization_table<IArchiver, OArchiver>::loader_t
+    loader_t;
+
+  direct_serialization_table<IArchiver, OArchiver>& table = 
+    get_direct_serialization_table<IArchiver, OArchiver>();
+
+  int descriptor;
+  ar >> descriptor;
+
+  if (descriptor) {
+    loader_t loader = table.loader(descriptor);
+    BOOST_ASSERT(loader);
+
+    loader(ar, obj, version);
+  } else {
+    // Unpickle it
+    detail::load_impl(ar, obj, version, mpl::false_());
+  }
+}
+
+} // end namespace detail
+
+template<typename Archiver>
+void 
+save(Archiver& ar, const boost::python::object& obj, 
+     const unsigned int version)
+{
+  typedef Archiver OArchiver;
+  typedef typename input_archiver<OArchiver>::type IArchiver;
+
+  detail::save_impl(ar, obj, version, 
+                    has_direct_serialization<IArchiver, OArchiver>());
+}
+
+template<typename Archiver>
+void 
+load(Archiver& ar, boost::python::object& obj, 
+     const unsigned int version)
+{
+  typedef Archiver IArchiver;
+  typedef typename output_archiver<IArchiver>::type OArchiver;
+
+  detail::load_impl(ar, obj, version, 
+                    has_direct_serialization<IArchiver, OArchiver>());
+}
+
+template<typename Archive>
+inline void 
+serialize(Archive& ar, boost::python::object& obj, const unsigned int version)
+{
+  boost::serialization::split_free(ar, obj, version);
+}
+
+} } // end namespace boost::python
+
+/************************************************************************
+ * Boost.MPI-Specific Section                                           *
+ ************************************************************************/
+namespace boost { namespace parallel { namespace mpi {
+ class packed_iarchive;
+ class packed_oarchive;
+} } } // end namespace boost::parallel::mpi
+
+BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(
+  ::boost::parallel::mpi::packed_iarchive,
+  ::boost::parallel::mpi::packed_oarchive)
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+template<typename T>
+void
+register_serialized(const T& value, PyTypeObject* type)
+{
+  using boost::python::register_serialized;
+  register_serialized<packed_iarchive, packed_oarchive>(value, type);
+}
+
+} } } } // end namespace boost::parallel::mpi::python
+
+#endif // BOOST_PARALLEL_MPI_PYTHON_SERIALIZE_HPP

include/boost/mpi/python/skeleton_and_content.hpp

@@ -0,0 +1,209 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+#ifndef BOOST_PARALLEL_MPI_PYTHON_SKELETON_AND_CONTENT_HPP
+#define BOOST_PARALLEL_MPI_PYTHON_SKELETON_AND_CONTENT_HPP
+
+/** @file skeleton_and_content.hpp
+ *
+ *  This file reflects the skeleton/content facilities into Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+#include <boost/function/function1.hpp>
+#define BOOST_PARALLEL_MPI_PYTHON_FORWARD_ONLY
+#include <boost/parallel/mpi/python.hpp>
+#include <boost/parallel/mpi/python/serialize.hpp>
+
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+/**
+ * INTERNAL ONLY
+ *
+ * This @c content class is a wrapper around the C++ "content"
+ * retrieved from get_content. This wrapper is only needed to store a
+ * copy of the Python object on which get_content() was called.
+ */
+class content : public boost::parallel::mpi::content
+{
+  typedef boost::parallel::mpi::content inherited;
+
+ public:
+  content(const inherited& base, boost::python::object object) 
+    : inherited(base), object(object) { }
+
+  inherited&       base()       { return *this; }
+  const inherited& base() const { return *this; }
+
+  boost::python::object object;
+};
+
+/**
+ * INTERNAL ONLY
+ *
+ * A class specific to the Python bindings that mimics the behavior of
+ * the skeleton_proxy<T> template. In the case of Python skeletons, we
+ * only need to know the object (and its type) to transmit the
+ * skeleton. This is the only user-visible skeleton proxy type,
+ * although instantiations of its derived classes (@c
+ * skeleton_proxy<T>) will be returned from the Python skeleton()
+ * function.
+ */
+class skeleton_proxy_base 
+{
+public:
+  skeleton_proxy_base(const boost::python::object& object) : object(object) { }
+
+  boost::python::object object;
+};
+
+/**
+ * INTERNAL ONLY
+ *
+ * The templated @c skeleton_proxy class represents a skeleton proxy
+ * in Python. The only data is stored in the @c skeleton_proxy_base
+ * class (which is the type actually exposed as @c skeleton_proxy in
+ * Python). However, the type of @c skeleton_proxy<T> is important for
+ * (de-)serialization of @c skeleton_proxy<T>'s for transmission.
+ */
+template<typename T>
+class skeleton_proxy : public skeleton_proxy_base
+{
+ public:
+  skeleton_proxy(const boost::python::object& object) 
+    : skeleton_proxy_base(object) { }
+};
+
+namespace detail {
+  using boost::python::object;
+  using boost::python::extract;
+   
+  extern boost::python::object skeleton_proxy_base_type;
+
+  template<typename T>
+  struct skeleton_saver
+  {
+    void 
+    operator()(packed_oarchive& ar, const object& obj, const unsigned int)
+    {
+      packed_skeleton_oarchive pso(ar);
+      pso << extract<T&>(obj.attr("object"))();
+    }
+  };
+
+  template<typename T> 
+  struct skeleton_loader
+  {
+    void 
+    operator()(packed_iarchive& ar, object& obj, const unsigned int)
+    {
+      packed_skeleton_iarchive psi(ar);
+      extract<skeleton_proxy<T>&> proxy(obj);
+      if (!proxy.check())
+        obj = object(skeleton_proxy<T>(object(T())));
+
+      psi >> extract<T&>(obj.attr("object"))();
+    }
+  };
+
+  /**
+   * The @c skeleton_content_handler structure contains all of the
+   * information required to extract a skeleton and content from a
+   * Python object with a certain C++ type.
+   */
+  struct skeleton_content_handler {
+    function1<object, const object&> get_skeleton_proxy;
+    function1<content, const object&> get_content;
+  };
+
+  /**
+   * A function object that extracts the skeleton from of a Python
+   * object, which is actually a wrapped C++ object of type T.
+   */
+  template<typename T>
+  struct do_get_skeleton_proxy
+  {
+    object operator()(object value) {
+      return object(skeleton_proxy<T>(value));
+    }
+  };
+
+  /**
+   * A function object that extracts the content of a Python object,
+   * which is actually a wrapped C++ object of type T.
+   */
+  template<typename T>
+  struct do_get_content
+  {
+    content operator()(object value_obj) {
+      T& value = extract<T&>(value_obj)();
+      return content(boost::parallel::mpi::get_content(value), value_obj);
+    }
+  };
+
+  /**
+   * Determine if a skeleton and content handler for @p type has
+   * already been registered.
+   */
+  bool
+  skeleton_and_content_handler_registered(PyTypeObject* type);
+ 
+  /**
+   * Register a skeleton/content handler with a particular Python type
+   * (which actually wraps a C++ type).
+   */
+  void 
+  register_skeleton_and_content_handler(PyTypeObject*, 
+                                        const skeleton_content_handler&);
+} // end namespace detail
+
+template<typename T>
+void register_skeleton_and_content(const T& value, PyTypeObject* type)
+{
+  using boost::python::detail::direct_serialization_table;
+  using boost::python::detail::get_direct_serialization_table;
+  using namespace boost::python;
+
+  // Determine the type
+  if (!type)
+    type = object(value).ptr()->ob_type;
+
+  // Don't re-register the same type.
+  if (detail::skeleton_and_content_handler_registered(type))
+    return;
+
+  // Register the skeleton proxy type
+  {
+    boost::python::scope proxy_scope(detail::skeleton_proxy_base_type);
+    std::string name("skeleton_proxy<");
+    name += typeid(T).name();
+    name += ">";
+    class_<skeleton_proxy<T>, bases<skeleton_proxy_base> >(name.c_str(), 
+                                                           no_init);
+  }
+
+  // Register the saver and loader for the associated skeleton and
+  // proxy, to allow (de-)serialization of skeletons via the proxy.
+  direct_serialization_table<packed_iarchive, packed_oarchive>& table = 
+    get_direct_serialization_table<packed_iarchive, packed_oarchive>();
+  table.register_type(detail::skeleton_saver<T>(), 
+                      detail::skeleton_loader<T>(), 
+                      skeleton_proxy<T>(object(value)));
+
+  // Register the rest of the skeleton/content mechanism, including
+  // handlers that extract a skeleton proxy from a Python object and
+  // extract the content from a Python object.
+  detail::skeleton_content_handler handler;
+  handler.get_skeleton_proxy = detail::do_get_skeleton_proxy<T>();
+  handler.get_content = detail::do_get_content<T>();
+  detail::register_skeleton_and_content_handler(type, handler);
+}
+
+} } } } // end namespace boost::parallel::mpi::python
+
+#endif // BOOST_PARALLEL_MPI_PYTHON_SKELETON_AND_CONTENT_HPP

include/boost/mpi/request.hpp

@@ -0,0 +1,102 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file request.hpp
+ *
+ *  This header defines the class @c request, which contains a request
+ *  for non-blocking communication.
+ */
+#ifndef BOOST_PARALLEL_MPI_REQUEST_HPP
+#define BOOST_PARALLEL_MPI_REQUEST_HPP
+
+#include <mpi.h>
+#include <boost/optional.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+class status;
+class communicator;
+
+/**
+ *  @brief A request for a non-blocking send or receive.
+ *
+ *  This structure contains information about a non-blocking send or
+ *  receive and will be returned from @c isend or @c irecv,
+ *  respectively.
+ */
+class request 
+{
+ public:
+  /**
+   *  Constructs a NULL request.
+   */
+  request();
+
+  /**
+   *  Wait until the communication associated with this request has
+   *  completed, then return a @c status object describing the
+   *  communication.
+   */
+  status wait();
+
+  /**
+   *  Determine whether the communication associated with this request
+   *  has completed successfully. If so, returns the @c status object
+   *  describing the communication. Otherwise, returns an empty @c
+   *  optional<> to indicate that the communication has not completed
+   *  yet. Note that once @c test() returns a @c status object, the
+   *  request has completed and @c wait() should not be called.
+   */
+  optional<status> test();
+
+  /**
+   *  Cancel a pending communication, assuming it has not already been
+   *  completed.
+   */
+  void cancel();
+
+ private:
+  enum request_action { ra_wait, ra_test, ra_cancel };
+  typedef optional<status> (*handler_type)(request* self, 
+                                           request_action action);
+
+  /**
+   * INTERNAL ONLY
+   *
+   * Handles the non-blocking receive of a serialized value.
+   */
+  template<typename T>
+  static optional<status> 
+  handle_serialized_irecv(request* self, request_action action);
+
+  /**
+   * INTERNAL ONLY
+   *
+   * Handles the non-blocking receive of an array of  serialized values.
+   */
+  template<typename T>
+  static optional<status> 
+  handle_serialized_array_irecv(request* self, request_action action);
+
+ public: // template friends are not portable
+
+  /// INTERNAL ONLY
+  MPI_Request m_requests[2];
+
+  /// INTERNAL ONLY
+  handler_type m_handler;
+
+  /// INTERNAL ONLY
+  shared_ptr<void> m_data;
+
+  friend class communicator;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_REQUEST_HPP

include/boost/mpi/skeleton_and_content.hpp

@@ -0,0 +1,376 @@
+// (C) Copyright 2005 Matthias Troyer
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+//           Douglas Gregor
+
+/** @file skeleton_and_content.hpp
+ *
+ *  This header provides facilities that allow the structure of data
+ *  types (called the "skeleton") to be transmitted and received
+ *  separately from the content stored in those data types. These
+ *  facilities are useful when the data in a stable data structure
+ *  (e.g., a mesh or a graph) will need to be transmitted
+ *  repeatedly. In this case, transmitting the skeleton only once
+ *  saves both communication effort (it need not be sent again) and
+ *  local computation (serialization need only be performed once for
+ *  the content).
+ */
+#ifndef BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_HPP
+#define BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_HPP
+
+#include <mpi.h>
+#include <boost/archive/detail/auto_link_archive.hpp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+#include <boost/parallel/mpi/detail/forward_skeleton_iarchive.hpp>
+#include <boost/parallel/mpi/detail/forward_skeleton_oarchive.hpp>
+#include <boost/parallel/mpi/detail/ignore_iprimitive.hpp>
+#include <boost/parallel/mpi/detail/ignore_oprimitive.hpp>
+#include <boost/shared_ptr.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/**
+ *  @brief A proxy that requests that the skeleton of an object be
+ *  transmitted.
+ *
+ *  The @c skeleton_proxy is a lightweight proxy object used to
+ *  indicate that the skeleton of an object, not the object itself,
+ *  should be transmitted. It can be used with the @c send and @c recv
+ *  operations of communicators or the @c broadcast collective. When a
+ *  @c skeleton_proxy is sent, Boost.MPI generates a description
+ *  containing the structure of the stored object. When that skeleton
+ *  is received, the receiving object is reshaped to match the
+ *  structure. Once the skeleton of an object as been transmitted, its
+ *  @c content can be transmitted separately (often several times)
+ *  without changing the structure of the object.
+ */
+template <class T>
+struct skeleton_proxy
+{
+  /**
+   *  Constructs a @c skeleton_proxy that references object @p x.
+   *
+   *  @param x the object whose structure will be transmitted or
+   *  altered.
+   */
+  skeleton_proxy(T& x)
+   : object(x)
+  {}
+
+  T& object;
+};
+
+/**
+ *  @brief Create a skeleton proxy object.
+ *
+ *  This routine creates an instance of the skeleton_proxy class. It
+ *  will typically be used when calling @c send, @c recv, or @c
+ *  broadcast, to indicate that only the skeleton (structure) of an
+ *  object should be transmitted and not its contents.
+ *
+ *  @param x the object whose structure will be transmitted.
+ *
+ *  @returns a skeleton_proxy object referencing @p x
+ */
+template <class T>
+inline const skeleton_proxy<T> skeleton(T& x)
+{
+  return skeleton_proxy<T>(x);
+}
+
+namespace detail {
+  /// @brief a class holding an MPI datatype
+  /// INTERNAL ONLY
+  /// the type is freed upon destruction
+  class mpi_datatype_holder : public boost::noncopyable
+  {
+  public:
+    mpi_datatype_holder()
+     : is_committed(false)
+    {}
+
+    mpi_datatype_holder(MPI_Datatype t, bool committed = true)
+     : d(t)
+     , is_committed(committed)
+    {}
+
+    void commit()
+    {
+      BOOST_MPI_CHECK_RESULT(MPI_Type_commit,(&d));
+      is_committed=true;
+    }
+
+    MPI_Datatype get_mpi_datatype() const
+    {
+      return d;
+    }
+
+    ~mpi_datatype_holder()
+    {
+      int finalized=0;
+      BOOST_MPI_CHECK_RESULT(MPI_Finalized,(&finalized));
+      if (!finalized && is_committed)
+        BOOST_MPI_CHECK_RESULT(MPI_Type_free,(&d));
+    }
+
+  private:
+    MPI_Datatype d;
+    bool is_committed;
+  };
+} // end namespace detail
+
+/** @brief A proxy object that transfers the content of an object
+ *  without its structure.
+ *
+ *  The @c content class indicates that Boost.MPI should transmit or
+ *  receive the content of an object, but without any information
+ *  about the structure of the object. It is only meaningful to
+ *  transmit the content of an object after the receiver has already
+ *  received the skeleton for the same object.
+ *
+ *  Most users will not use @c content objects directly. Rather, they
+ *  will invoke @c send, @c recv, or @c broadcast operations using @c
+ *  get_content().
+ */
+class content
+{
+public:
+  /**
+   *  Constructs an empty @c content object. This object will not be
+   *  useful for any Boost.MPI operations until it is reassigned.
+   */
+  content() {}
+
+  /**
+   *  This routine initializes the @c content object with an MPI data
+   *  type that refers to the content of an object without its structure.
+   *
+   *  @param d the MPI data type referring to the content of the object.
+   *
+   *  @param committed @c true indicates that @c MPI_Type_commit has
+   *  already been excuted for the data type @p d.
+   */
+  content(MPI_Datatype d, bool committed=true)
+   : holder(new detail::mpi_datatype_holder(d,committed))
+  {}
+
+  /**
+   *  Replace the MPI data type referencing the content of an object.
+   *
+   *  @param d the new MPI data type referring to the content of the
+   *  object.
+   *
+   *  @returns *this
+   */
+  const content& operator=(MPI_Datatype d)
+  {
+    holder.reset(new detail::mpi_datatype_holder(d));
+    return *this;
+  }
+
+  /**
+   * Retrieve the MPI data type that refers to the content of the
+   * object.
+   *
+   * @returns the MPI data type, which should only be transmitted or
+   * received using @c MPI_BOTTOM as the address.
+   */
+  MPI_Datatype get_mpi_datatype() const
+  {
+    return holder->get_mpi_datatype();
+  }
+
+  /**
+   *  Commit the MPI data type referring to the content of the
+   *  object.
+   */
+  void commit()
+  {
+    holder->commit();
+  }
+
+private:
+  boost::shared_ptr<detail::mpi_datatype_holder> holder;
+};
+
+/** @brief Returns the content of an object, suitable for transmission
+ *   via Boost.MPI.
+ *
+ *  The function creates an absolute MPI datatype for the object,
+ *  where all offsets are counted from the address 0 (a.k.a. @c
+ *  MPI_BOTTOM) instead of the address @c &x of the object. This
+ *  allows the creation of MPI data types for complex data structures
+ *  containing pointers, such as linked lists or trees.
+ *
+ *  The disadvantage, compared to relative MPI data types is that for
+ *  each object a new MPI data type has to be created.
+ *
+ *  The contents of an object can only be transmitted when the
+ *  receiver already has an object with the same structure or shape as
+ *  the sender. To accomplish this, first transmit the skeleton of the
+ *  object using, e.g., @c skeleton() or @c skeleton_proxy.
+ *
+ *  The type @c T has to allow creation of an absolute MPI data type
+ *  (content).
+ *
+ *  @param x the object for which the content will be transmitted.
+ *
+ *  @returns the content of the object @p x, which can be used for
+ *  transmission via @c send, @c recv, or @c broadcast.
+ */
+template <class T> const content get_content(const T& x);
+
+/** @brief An archiver that reconstructs a data structure based on the
+ *  binary skeleton stored in a buffer.
+ *
+ *  The @c packed_skeleton_iarchive class is an Archiver (as in the
+ *  Boost.Serialization library) that can construct the the shape of a
+ *  data structure based on a binary skeleton stored in a buffer. The
+ *  @c packed_skeleton_iarchive is typically used by the receiver of a
+ *  skeleton, to prepare a data structure that will eventually receive
+ *  content separately.
+ *
+ *  Users will not generally need to use @c packed_skeleton_iarchive
+ *  directly. Instead, use @c skeleton or @c get_skeleton.
+ */
+class packed_skeleton_iarchive
+  : public detail::ignore_iprimitive,
+    public detail::forward_skeleton_iarchive<packed_skeleton_iarchive,packed_iarchive>
+{
+public:
+  /**
+   *  Construct a @c packed_skeleton_iarchive for the given
+   *  communicator.
+   *
+   *  @param comm The communicator over which this archive will be
+   *  transmitted.
+   *
+   *  @param flags Control the serialization of the skeleton. Refer to
+   *  the Boost.Serialization documentation before changing the
+   *  default flags.
+   */
+  packed_skeleton_iarchive(MPI_Comm const & comm,
+                           unsigned int flags =  boost::archive::no_header)
+         : detail::forward_skeleton_iarchive<packed_skeleton_iarchive,packed_iarchive>(skeleton_archive_)
+         , skeleton_archive_(comm,flags)
+        {}
+
+  /**
+   *  Construct a @c packed_skeleton_iarchive that unpacks a skeleton
+   *  from the given @p archive.
+   *
+   *  @param archive the archive from which the skeleton will be
+   *  unpacked.
+   *
+   */
+  explicit packed_skeleton_iarchive(packed_iarchive & archive)
+         : detail::forward_skeleton_iarchive<packed_skeleton_iarchive,packed_iarchive>(archive)
+         , skeleton_archive_(MPI_COMM_WORLD, boost::archive::no_header)
+        {}
+
+  /**
+   *  Retrieve the archive corresponding to this skeleton.
+   */
+  const packed_iarchive& get_skeleton() const
+  {
+    return this->implementation_archive;
+  }
+
+  /**
+   *  Retrieve the archive corresponding to this skeleton.
+   */
+  packed_iarchive& get_skeleton()
+  {
+    return this->implementation_archive;
+  }
+
+private:
+  /// Store the actual archive that holds the structure, unless the
+  /// user overrides this with their own archive.
+  packed_iarchive skeleton_archive_;
+};
+
+/** @brief An archiver that records the binary skeleton of a data
+ * structure into a buffer.
+ *
+ *  The @c packed_skeleton_oarchive class is an Archiver (as in the
+ *  Boost.Serialization library) that can record the shape of a data
+ *  structure (called the "skeleton") into a binary representation
+ *  stored in a buffer. The @c packed_skeleton_oarchive is typically
+ *  used by the send of a skeleton, to pack the skeleton of a data
+ *  structure for transmission separately from the content.
+ *
+ *  Users will not generally need to use @c packed_skeleton_oarchive
+ *  directly. Instead, use @c skeleton or @c get_skeleton.
+ */
+class packed_skeleton_oarchive
+  : public detail::ignore_oprimitive,
+    public detail::forward_skeleton_oarchive<packed_skeleton_oarchive,packed_oarchive>
+{
+public:
+  /**
+   *  Construct a @c packed_skeleton_oarchive for the given
+   *  communicator.
+   *
+   *  @param comm The communicator over which this archive will be
+   *  transmitted.
+   *
+   *  @param flags Control the serialization of the skeleton. Refer to
+   *  the Boost.Serialization documentation before changing the
+   *  default flags.
+   */
+  packed_skeleton_oarchive(MPI_Comm const & comm,
+                           unsigned int flags =  boost::archive::no_header)
+         : detail::forward_skeleton_oarchive<packed_skeleton_oarchive,packed_oarchive>(skeleton_archive_)
+         , skeleton_archive_(comm,flags)
+        {}
+
+  /**
+   *  Construct a @c packed_skeleton_oarchive that packs a skeleton
+   *  into the given @p archive.
+   *
+   *  @param archive the archive to which the skeleton will be packed.
+   *
+   */
+  explicit packed_skeleton_oarchive(packed_oarchive & archive)
+         : detail::forward_skeleton_oarchive<packed_skeleton_oarchive,packed_oarchive>(archive)
+         , skeleton_archive_(MPI_COMM_WORLD, boost::archive::no_header)
+        {}
+
+  /**
+   *  Retrieve the archive corresponding to this skeleton.
+   */
+  const packed_oarchive& get_skeleton() const
+  {
+    return this->implementation_archive;
+  }
+
+private:
+  /// Store the actual archive that holds the structure.
+  packed_oarchive skeleton_archive_;
+};
+
+} } } // end namespace boost::parallel::mpi
+
+#include <boost/parallel/mpi/detail/content_oarchive.hpp>
+
+// For any headers that have provided declarations based on forward
+// declarations of the contents of this header, include definitions
+// for those declarations. This means that the inclusion of
+// skeleton_and_content.hpp enables the use of skeleton/content
+// transmission throughout the library.
+#ifdef BOOST_PARALLEL_MPI_BROADCAST_HPP
+#  include <boost/parallel/mpi/detail/broadcast_sc.hpp>
+#endif
+
+#ifdef BOOST_PARALLEL_MPI_COMMUNICATOR_HPP
+#  include <boost/parallel/mpi/detail/communicator_sc.hpp>
+#endif
+
+#endif // BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_HPP

include/boost/mpi/skeleton_and_content_fwd.hpp

@@ -0,0 +1,31 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file skeleton_and_content_fwd.hpp
+ *
+ *  This header contains all of the forward declarations required to
+ *  use transmit skeletons of data structures and the content of data
+ *  structures separately. To actually transmit skeletons or content,
+ *  include the header @c boost/parallel/mpi/skeleton_and_content.hpp.
+ */
+
+#ifndef BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_FWD_HPP
+#define BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_FWD_HPP
+
+namespace boost { namespace parallel { namespace mpi {
+
+template <class T> struct skeleton_proxy;
+template <class T> const skeleton_proxy<T> skeleton(T& x);
+class content;
+template <class T> const content get_content(const T& x);
+class packed_skeleton_iarchive;
+class packed_skeleton_oarchive;
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_SKELETON_AND_CONTENT_FWD_HPP

include/boost/mpi/status.hpp

@@ -0,0 +1,105 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file status.hpp
+ *
+ *  This header defines the class @c status, which reports on the
+ *  results of point-to-point communication.
+ */
+#ifndef BOOST_PARALLEL_MPI_STATUS_HPP
+#define BOOST_PARALLEL_MPI_STATUS_HPP
+
+#include <mpi.h>
+#include <boost/optional.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+class request;
+class communicator;
+
+/** @brief Contains information about a message that has been or can
+ *  be received.
+ *
+ *  This structure contains status information about messages that
+ *  have been received (with @c communicator::recv) or can be received
+ *  (returned from @c communicator::probe or @c
+ *  communicator::iprobe). It permits access to the source of the
+ *  message, message tag, error code (rarely used), or the number of
+ *  elements that have been transmitted.
+ */
+class status
+{
+ public:
+  status() : m_count(-1) { }
+
+  /**
+   * Retrieve the source of the message.
+   */
+  int source() const { return m_status.MPI_SOURCE; }
+
+  /**
+   * Retrieve the message tag.
+   */
+  int tag() const { return m_status.MPI_TAG; }
+
+  /**
+   * Retrieve the error code.
+   */
+  int error() const { return m_status.MPI_ERROR; }
+
+  /**
+   * Determine whether the communication associated with this object
+   * has been successfully cancelled.
+  */
+  bool cancelled();
+
+  /**
+   * Determines the number of elements of type @c T contained in the
+   * message. The type @c T must have an associated data type, i.e.,
+   * @c is_mpi_datatype<T> must derive @c mpl::true_. In cases where
+   * the type @c T does not match the transmitted type, this routine
+   * will return an empty @c optional<int>.
+   *
+   * @returns the number of @c T elements in the message, if it can be
+   * determined.
+   */
+  template<typename T> optional<int> count();
+
+  /**
+   * References the underlying @c MPI_Status
+   */
+  operator       MPI_Status&()       { return m_status; }
+
+  /**
+   * References the underlying @c MPI_Status
+   */
+  operator const MPI_Status&() const { return m_status; }
+
+ private:
+  /**
+   * INTERNAL ONLY
+   */
+  template<typename T> optional<int> count_impl(mpl::true_);
+
+  /**
+   * INTERNAL ONLY
+   */
+  template<typename T> optional<int> count_impl(mpl::false_);
+
+ public: // friend templates are not portable
+
+  /// INTERNAL ONLY
+  MPI_Status m_status;
+  int m_count;
+
+  friend class communicator;
+  friend class request;
+};
+
+
+} } } // end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_STATUS_HPP

include/boost/mpi/timer.hpp

@@ -0,0 +1,91 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file timer.hpp
+ *
+ *  This header provides the @c timer class, which provides access to
+ *  the MPI timers.
+ */
+#ifndef BOOST_PARALLEL_MPI_TIMER_HPP
+#define BOOST_PARALLEL_MPI_TIMER_HPP
+
+#include <mpi.h>
+#include <boost/limits.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/** @brief A simple timer that provides access to the MPI timing
+ * facilities.
+ *
+ *  The @c timer class is a simple wrapper around the MPI timing
+ *  facilities that mimics the interface of the Boost Timer library.
+ */
+class timer {
+public:
+  /** Initializes the timer
+   *
+   * @post @c elapsed() == 0
+   */
+  timer();
+
+  /** Restart the timer.
+   *
+   * @post @c elapsed() == 0
+   */
+  void restart();
+
+  /** Return the amount of time that has elapsed since the last
+   *  construction or reset, in seconds.
+   */
+  double elapsed() const;
+
+  /** Return an estimate of the maximum possible value of
+   *  elapsed(). Note that this routine may return too high a value on
+   *  some systems. 
+   */
+  double elapsed_max() const;  
+
+  /** Returns the minimum non-zero value that @c elapsed() may
+   *  return. This is the resolution of the timer.
+   */
+  double elapsed_min() const;
+
+  /** Determines whether the elapsed time values are global times or
+      local processor times. */
+  static bool time_is_global();
+
+private:
+  double start_time;
+}; // timer
+
+inline timer::timer() 
+{ 
+  restart(); 
+}
+
+inline void timer::restart()
+{
+  start_time = MPI_Wtime();
+}
+
+inline double timer::elapsed() const
+{
+  return MPI_Wtime() - start_time;
+}
+
+inline double timer::elapsed_max() const
+{
+  return (std::numeric_limits<double>::max)();
+}
+
+inline double timer::elapsed_min() const
+{
+  return MPI_Wtick();
+}
+
+} } } /// end namespace boost::parallel::mpi
+
+#endif // BOOST_PARALLEL_MPI_TIMER_HPP

src/mpi/broadcast.cpp

@@ -0,0 +1,151 @@
+// Copyright 2005 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 4.4. Broadcast
+
+#include <boost/parallel/mpi/collectives/broadcast.hpp>
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+template<>
+void
+broadcast<const packed_oarchive>(const communicator& comm,
+                                 const packed_oarchive& oa,
+                                 int root)
+{
+  // Only the root can broadcast the packed_oarchive
+  assert(comm.rank() == root);
+
+  int size = comm.size();
+  if (size < 2) return;
+
+  // Determine maximum tag value
+  int tag = environment::collectives_tag();
+
+  // Broadcast data to all nodes
+  std::vector<MPI_Request> requests(size * 2);
+  int num_requests = 0;
+  for (int dest = 0; dest < size; ++dest) {
+    if (dest != root) {
+      // Build up send requests for each child send.
+      num_requests += detail::packed_archive_isend(comm, dest, tag, oa,
+                                                   &requests[num_requests], 2);
+    }
+  }
+
+  // Complete all of the sends
+  BOOST_MPI_CHECK_RESULT(MPI_Waitall,
+                         (num_requests, &requests[0], MPI_STATUSES_IGNORE));
+}
+
+template<>
+void
+broadcast<packed_oarchive>(const communicator& comm, packed_oarchive& oa,
+                           int root)
+{
+  broadcast(comm, const_cast<const packed_oarchive&>(oa), root);
+}
+
+template<>
+void
+broadcast<packed_iarchive>(const communicator& comm, packed_iarchive& ia,
+                           int root)
+{
+  int size = comm.size();
+  if (size < 2) return;
+
+  // Determine maximum tag value
+  int tag = environment::collectives_tag();
+
+  // Receive data from the root.
+  if (comm.rank() != root) {
+    MPI_Status status;
+    detail::packed_archive_recv(comm, root, tag, ia, status);
+  } else {
+    // Broadcast data to all nodes
+    std::vector<MPI_Request> requests(size * 2);
+    int num_requests = 0;
+    for (int dest = 0; dest < size; ++dest) {
+      if (dest != root) {
+        // Build up send requests for each child send.
+        num_requests += detail::packed_archive_isend(comm, dest, tag, ia,
+                                                     &requests[num_requests],
+                                                     2);
+      }
+    }
+
+    // Complete all of the sends
+    BOOST_MPI_CHECK_RESULT(MPI_Waitall,
+                           (num_requests, &requests[0], MPI_STATUSES_IGNORE));
+  }
+}
+
+template<>
+void
+broadcast<const packed_skeleton_oarchive>(const communicator& comm,
+                                          const packed_skeleton_oarchive& oa,
+                                          int root)
+{
+  broadcast(comm, oa.get_skeleton(), root);
+}
+
+template<>
+void
+broadcast<packed_skeleton_oarchive>(const communicator& comm,
+                                    packed_skeleton_oarchive& oa, int root)
+{
+  broadcast(comm, oa.get_skeleton(), root);
+}
+
+template<>
+void
+broadcast<packed_skeleton_iarchive>(const communicator& comm,
+                                    packed_skeleton_iarchive& ia, int root)
+{
+  broadcast(comm, ia.get_skeleton(), root);
+}
+
+template<>
+void broadcast<content>(const communicator& comm, content& c, int root)
+{
+  broadcast(comm, const_cast<const content&>(c), root);
+}
+
+template<>
+void broadcast<const content>(const communicator& comm, const content& c,
+                              int root)
+{
+#ifdef LAM_MPI
+  if (comm.size() < 2)
+    return;
+
+  // Some versions of LAM/MPI behave badly when broadcasting using
+  // MPI_BOTTOM, so we'll instead use manual send/recv operations.
+  if (comm.rank() == root) {
+    for (int p = 0; p < comm.size(); ++p) {
+      if (p != root) {
+        BOOST_MPI_CHECK_RESULT(MPI_Send,
+                               (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                                p, environment::collectives_tag(), comm));
+      }
+    }
+  } else {
+    BOOST_MPI_CHECK_RESULT(MPI_Recv,
+                           (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                            root, environment::collectives_tag(),
+                            comm, MPI_STATUS_IGNORE));
+  }
+#else
+  BOOST_MPI_CHECK_RESULT(MPI_Bcast,
+                         (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                          root, comm));
+#endif
+}
+
+} } } // end namespace boost::parallel::mpi

src/mpi/communicator.cpp

@@ -0,0 +1,267 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/***************************************************************************
+ * status                                                                  *
+ ***************************************************************************/
+bool status::cancelled()
+{
+  int flag = 0;
+  BOOST_MPI_CHECK_RESULT(MPI_Test_cancelled, (&m_status, &flag));
+  return flag != 0;
+}
+
+/***************************************************************************
+ * communicator                                                            *
+ ***************************************************************************/
+
+communicator::communicator()
+{
+  comm_ptr.reset(new MPI_Comm(MPI_COMM_WORLD));
+}
+
+communicator::communicator(const MPI_Comm& comm, comm_create_kind kind)
+{
+  if (comm == MPI_COMM_NULL)
+    /* MPI_COMM_NULL indicates that the communicator is not usable. */
+    return;
+
+  switch (kind) {
+  case comm_duplicate:
+    {
+      MPI_Comm newcomm;
+      BOOST_MPI_CHECK_RESULT(MPI_Comm_dup, (comm, &newcomm));
+      comm_ptr.reset(new MPI_Comm(newcomm), comm_free());
+      MPI_Errhandler_set(newcomm, MPI_ERRORS_RETURN);
+      break;
+    }
+
+  case comm_take_ownership:
+    comm_ptr.reset(new MPI_Comm(comm), comm_free());
+    break;
+
+  case comm_attach:
+    comm_ptr.reset(new MPI_Comm(comm));
+    break;
+  }
+ }
+
+int communicator::size() const
+{
+  int size_;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_size, (MPI_Comm(*this), &size_));
+  return size_;
+}
+
+int communicator::rank() const
+{
+  int rank_;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_rank, (MPI_Comm(*this), &rank_));
+  return rank_;
+}
+
+void communicator::send(int dest, int tag) const
+{
+  BOOST_MPI_CHECK_RESULT(MPI_Send,
+                         (MPI_BOTTOM, 0, MPI_PACKED,
+                          dest, tag, MPI_Comm(*this)));
+}
+
+status communicator::recv(int source, int tag) const
+{
+  status stat;
+  BOOST_MPI_CHECK_RESULT(MPI_Recv,
+                         (MPI_BOTTOM, 0, MPI_PACKED,
+                          source, tag, MPI_Comm(*this), &stat.m_status));
+  return stat;
+}
+
+optional<status> communicator::iprobe(int source, int tag) const
+{
+  typedef optional<status> result_type;
+
+  status stat;
+  int flag;
+  BOOST_MPI_CHECK_RESULT(MPI_Iprobe,
+                         (source, tag, MPI_Comm(*this), &flag,
+                          &stat.m_status));
+  if (flag) return stat;
+  else return result_type();
+}
+
+status communicator::probe(int source, int tag) const
+{
+  typedef optional<status> result_type;
+
+  status stat;
+  BOOST_MPI_CHECK_RESULT(MPI_Probe,
+                         (source, tag, MPI_Comm(*this), &stat.m_status));
+  return stat;
+}
+
+void (communicator::barrier)() const
+{
+  BOOST_MPI_CHECK_RESULT(MPI_Barrier, (MPI_Comm(*this)));
+}
+
+
+communicator::operator MPI_Comm() const
+{
+  if (comm_ptr) return *comm_ptr;
+  else return MPI_COMM_NULL;
+}
+
+communicator communicator::split(int color) const
+{
+  return split(color, rank());
+}
+
+communicator communicator::split(int color, int key) const
+{
+  MPI_Comm newcomm;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_split,
+                         (MPI_Comm(*this), color, key, &newcomm));
+  return communicator(newcomm, comm_take_ownership);
+}
+
+void communicator::abort(int errcode) const
+{
+  BOOST_MPI_CHECK_RESULT(MPI_Abort, (MPI_Comm(*this), errcode));
+}
+
+/*************************************************************
+ * archived send/recv                                     *
+ *************************************************************/
+template<>
+void
+communicator::send<packed_oarchive>(int dest, int tag,
+                                    const packed_oarchive& ar) const
+{
+  detail::packed_archive_send(MPI_Comm(*this), dest, tag, ar);
+}
+
+template<>
+void
+communicator::send<packed_skeleton_oarchive>
+  (int dest, int tag, const packed_skeleton_oarchive& ar) const
+{
+  this->send(dest, tag, ar.get_skeleton());
+}
+
+template<>
+void communicator::send<content>(int dest, int tag, const content& c) const
+{
+  BOOST_MPI_CHECK_RESULT(MPI_Send,
+                         (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                          dest, tag, MPI_Comm(*this)));
+}
+
+template<>
+status
+communicator::recv<packed_iarchive>(int source, int tag,
+                                    packed_iarchive& ar) const
+{
+  status stat;
+  detail::packed_archive_recv(MPI_Comm(*this), source, tag, ar,
+                              stat.m_status);
+  return stat;
+}
+
+template<>
+status
+communicator::recv<packed_skeleton_iarchive>
+  (int source, int tag, packed_skeleton_iarchive& ar) const
+{
+  return this->recv(source, tag, ar.get_skeleton());
+}
+
+template<>
+status
+communicator::recv<const content>(int source, int tag, const content& c) const
+{
+  status stat;
+  BOOST_MPI_CHECK_RESULT(MPI_Recv,
+                         (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                          source, tag, MPI_Comm(*this), &stat.m_status));
+  return stat;
+}
+
+/*************************************************************
+ * non-blocking send/recv                                    *
+ *************************************************************/
+template<>
+request
+communicator::isend<packed_oarchive>(int dest, int tag,
+                                     const packed_oarchive& ar) const
+{
+  request req;
+  detail::packed_archive_isend(MPI_Comm(*this), dest, tag, ar,
+                               &req.m_requests[0] ,2);
+  return req;
+}
+
+template<>
+request
+communicator::isend<packed_skeleton_oarchive>
+  (int dest, int tag, const packed_skeleton_oarchive& ar) const
+{
+  return this->isend(dest, tag, ar.get_skeleton());
+}
+
+template<>
+request communicator::isend<content>(int dest, int tag, const content& c) const
+{
+  request req;
+  BOOST_MPI_CHECK_RESULT(MPI_Isend,
+                         (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                          dest, tag, MPI_Comm(*this), &req.m_requests[0]));
+  return req;
+}
+
+request communicator::isend(int dest, int tag) const
+{
+  request req;
+  BOOST_MPI_CHECK_RESULT(MPI_Isend,
+                         (MPI_BOTTOM, 0, MPI_PACKED,
+                          dest, tag, MPI_Comm(*this), &req.m_requests[0]));
+  return req;
+}
+
+template<>
+request
+communicator::irecv<packed_skeleton_iarchive>
+  (int source, int tag, packed_skeleton_iarchive& ar) const
+{
+  return this->irecv(source, tag, ar.get_skeleton());
+}
+
+template<>
+request
+communicator::irecv<const content>(int source, int tag,
+                                   const content& c) const
+{
+  request req;
+  BOOST_MPI_CHECK_RESULT(MPI_Irecv,
+                         (MPI_BOTTOM, 1, c.get_mpi_datatype(),
+                          source, tag, MPI_Comm(*this), &req.m_requests[0]));
+  return req;
+}
+
+request communicator::irecv(int source, int tag) const
+{
+  request req;
+  BOOST_MPI_CHECK_RESULT(MPI_Irecv,
+                         (MPI_BOTTOM, 0, MPI_PACKED,
+                          source, tag, MPI_Comm(*this), &req.m_requests[0]));
+  return req;
+}
+
+} } } // end namespace boost::parallel::mpi

src/mpi/computation_tree.cpp

@@ -0,0 +1,72 @@
+// Copyright (C) 2005 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Compute parents, children, levels, etc. to effect a parallel
+// computation tree.
+
+#include <boost/parallel/mpi/detail/computation_tree.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+int computation_tree::default_branching_factor = 3;
+
+computation_tree
+::computation_tree(int rank, int size, int root, int branching_factor)
+  : rank(rank), size(size), root(root),
+    branching_factor_(branching_factor > 1? branching_factor
+                      /* default */: default_branching_factor),
+    level_(0)
+{
+  // The position in the tree, once we've adjusted for non-zero
+  // roots.
+  int n = (rank + size - root) % size;
+  int sum = 0;
+  int term = 1;
+
+  /* The level is the smallest value of k such that
+
+  f^0 + f^1 + ... + f^k > n
+
+  for branching factor f and index n in the tree. */
+  while (sum <= n) {
+    ++level_;
+    term *= branching_factor_;
+    sum += term;
+  }
+}
+
+int computation_tree::level_index(int n) const
+{
+  int sum = 0;
+  int term = 1;
+  while (n--) {
+    sum += term;
+    term *= branching_factor_;
+  }
+  return sum;
+}
+
+int computation_tree::parent() const
+{
+  if (rank == root) return rank;
+  int n = rank + size - 1 - root;
+  return ((n % size / branching_factor_) + root) % size ;
+}
+
+int computation_tree::child_begin() const
+{
+  // Zero-based index of this node
+  int n = (rank + size - root) % size;
+
+  // Compute the index of the child (in a zero-based tree)
+  int child_index = level_index(level_ + 1)
+    + branching_factor_ * (n - level_index(level_));
+
+  if (child_index >= size) return root;
+  else return (child_index + root) % size;
+}
+
+} } } } // end namespace boost::parallel::mpi::detail

src/mpi/content_oarchive.cpp

@@ -0,0 +1,18 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+
+namespace boost { namespace archive { namespace detail {
+// explicitly instantiate all required template functions
+
+template class archive_pointer_oserializer<parallel::mpi::detail::content_oarchive> ;
+
+} } }

src/mpi/environment.cpp

@@ -0,0 +1,109 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- 7.1.1. Environmental Inquiries
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <cassert>
+#include <exception>
+#include <stdexcept>
+
+namespace boost { namespace parallel { namespace mpi {
+
+environment::environment(int& argc, char** &argv, bool abort_on_exception)
+  : i_initialized(false),
+    abort_on_exception(abort_on_exception)
+{
+  if (!initialized()) {
+    BOOST_MPI_CHECK_RESULT(MPI_Init, (&argc, &argv));
+    i_initialized = true;
+  }
+
+  MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
+}
+
+environment::~environment()
+{
+  if (i_initialized) {
+    if (std::uncaught_exception() && abort_on_exception) {
+      abort(-1);
+    } else if (!finalized()) {
+      BOOST_MPI_CHECK_RESULT(MPI_Finalize, ());
+    }
+  }
+}
+
+void environment::abort(int errcode)
+{
+  BOOST_MPI_CHECK_RESULT(MPI_Abort, (MPI_COMM_WORLD, errcode));
+}
+
+bool environment::initialized()
+{
+  int flag;
+  BOOST_MPI_CHECK_RESULT(MPI_Initialized, (&flag));
+  return flag != 0;
+}
+
+bool environment::finalized()
+{
+  int flag;
+  BOOST_MPI_CHECK_RESULT(MPI_Finalized, (&flag));
+  return flag != 0;
+}
+
+int environment::max_tag()
+{
+  int* max_tag_value;
+  int found = 0;
+
+  BOOST_MPI_CHECK_RESULT(MPI_Attr_get,
+                         (MPI_COMM_WORLD, MPI_TAG_UB, &max_tag_value, &found));
+  assert(found != 0);
+  return *max_tag_value - num_reserved_tags;
+}
+
+int environment::collectives_tag()
+{
+  return max_tag() + 1;
+}
+
+optional<int> environment::host_rank()
+{
+  int* host;
+  int found = 0;
+
+  BOOST_MPI_CHECK_RESULT(MPI_Attr_get,
+                         (MPI_COMM_WORLD, MPI_HOST, &host, &found));
+  if (!found || *host == MPI_PROC_NULL)
+    return optional<int>();
+  else
+    return *host;
+}
+
+optional<int> environment::io_rank()
+{
+  int* io;
+  int found = 0;
+
+  BOOST_MPI_CHECK_RESULT(MPI_Attr_get,
+                         (MPI_COMM_WORLD, MPI_IO, &io, &found));
+  if (!found || *io == MPI_PROC_NULL)
+    return optional<int>();
+  else
+    return *io;
+}
+
+std::string environment::processor_name()
+{
+  char name[MPI_MAX_PROCESSOR_NAME];
+  int len;
+
+  BOOST_MPI_CHECK_RESULT(MPI_Get_processor_name, (name, &len));
+  return std::string(name, len);
+}
+
+} } } // end namespace boost::parallel::mpi

src/mpi/mpi_datatype_cache.cpp

@@ -0,0 +1,16 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+#include <boost/parallel/mpi/detail/mpi_datatype_cache.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+mpi_datatype_map mpi_datatype_cache;
+  
+}}}}

src/mpi/mpi_datatype_oarchive.cpp

@@ -0,0 +1,18 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+#include <boost/parallel/mpi/detail/mpi_datatype_oarchive.hpp>
+
+namespace boost { namespace archive { namespace detail {
+// explicitly instantiate all required template functions
+
+template class archive_pointer_oserializer<parallel::mpi::detail::mpi_datatype_oarchive> ;
+
+} } } 

src/mpi/packed_iarchive.cpp

@@ -0,0 +1,26 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/archive/impl/archive_pointer_iserializer.ipp>
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+#include <boost/parallel/mpi/packed_iarchive.hpp>
+
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/archive/impl/basic_binary_iprimitive.ipp>
+#include <boost/archive/impl/basic_binary_iarchive.ipp>
+
+namespace boost { namespace archive {
+
+// explicitly instantiate all required templates
+
+template class basic_binary_iarchive<parallel::mpi::packed_iarchive> ;
+template class detail::archive_pointer_iserializer<parallel::mpi::packed_iarchive> ;
+//template class binary_iarchive_impl<mpi_packed_iarchive> ;
+
+} } // end namespace boost::archive

src/mpi/packed_oarchive.cpp

@@ -0,0 +1,24 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/parallel/mpi/packed_oarchive.hpp>
+
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/impl/basic_binary_oprimitive.ipp>
+#include <boost/archive/impl/basic_binary_oarchive.ipp>
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+
+namespace boost { namespace archive {
+// explicitly instantiate all required templates
+
+template class detail::archive_pointer_oserializer<parallel::mpi::packed_oarchive> ;
+template class basic_binary_oarchive<parallel::mpi::packed_oarchive> ;
+//template class binary_oarchive_impl<mpi_packed_oarchive> ;
+
+} } // end namespace boost::archive

src/mpi/packed_skeleton_iarchive.cpp

@@ -0,0 +1,26 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/archive/impl/archive_pointer_iserializer.ipp>
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/archive/impl/basic_binary_iprimitive.ipp>
+#include <boost/archive/impl/basic_binary_iarchive.ipp>
+
+namespace boost { namespace archive {
+
+// explicitly instantiate all required templates
+
+template class basic_binary_iarchive<parallel::mpi::packed_skeleton_iarchive> ;
+template class detail::archive_pointer_iserializer<parallel::mpi::packed_skeleton_iarchive> ;
+//template class binary_iarchive_impl<packed_skeleton_iarchive> ;
+
+} } // end namespace boost::archive

src/mpi/packed_skeleton_oarchive.cpp

@@ -0,0 +1,24 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/impl/basic_binary_oprimitive.ipp>
+#include <boost/archive/impl/basic_binary_oarchive.ipp>
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+
+namespace boost { namespace archive {
+// explicitly instantiate all required templates
+
+template class detail::archive_pointer_oserializer<parallel::mpi::packed_skeleton_oarchive> ;
+template class basic_binary_oarchive<parallel::mpi::packed_skeleton_oarchive> ;
+//template class binary_oarchive_impl<mpi_packed_oarchive> ;
+
+} } // end namespace boost::archive

src/mpi/point_to_point.cpp

@@ -0,0 +1,97 @@
+// Copyright 2005 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// Message Passing Interface 1.1 -- Section 3. MPI Point-to-point
+
+/* There is the potential for optimization here. We could keep around
+   a "small message" buffer of size N that we just receive into by
+   default. If the message is N - sizeof(int) bytes or smaller, it can
+   just be sent with that buffer. If it's larger, we send the first N
+   - sizeof(int) bytes in the first packet followed by another
+   packet. The size of the second packet will be stored in an integer
+   at the end of the first packet.
+
+   We will introduce this optimization later, when we have more
+   performance test cases and have met our functionality goals. */
+
+#include <boost/parallel/mpi/detail/point_to_point.hpp>
+#include <boost/parallel/mpi/datatype.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <cassert>
+
+namespace boost { namespace parallel { namespace mpi { namespace detail {
+
+void
+packed_archive_send(MPI_Comm comm, int dest, int tag,
+                    const packed_oarchive& ar)
+{
+  const void* size = &ar.size();
+  BOOST_MPI_CHECK_RESULT(MPI_Send,
+                         (const_cast<void*>(size), 1, 
+                          get_mpi_datatype<std::size_t>(), dest, tag, comm));
+  BOOST_MPI_CHECK_RESULT(MPI_Send,
+                         (const_cast<void*>(ar.address()), ar.size(),
+                          MPI_PACKED,
+                          dest, tag, comm));
+}
+
+int
+packed_archive_isend(MPI_Comm comm, int dest, int tag,
+                     const packed_oarchive& ar,
+                     MPI_Request* out_requests, int num_out_requests)
+{
+  assert(num_out_requests >= 2);
+  const void* size = &ar.size();
+  BOOST_MPI_CHECK_RESULT(MPI_Isend,
+                         (const_cast<void*>(size), 1, 
+                          get_mpi_datatype<std::size_t>(), 
+                          dest, tag, comm, out_requests));
+  BOOST_MPI_CHECK_RESULT(MPI_Isend,
+                         (const_cast<void*>(ar.address()), ar.size(),
+                          MPI_PACKED,
+                          dest, tag, comm, out_requests + 1));
+
+  return 2;
+}
+
+int
+packed_archive_isend(MPI_Comm comm, int dest, int tag,
+                     const packed_iarchive& ar,
+                     MPI_Request* out_requests, int num_out_requests)
+{
+  assert(num_out_requests >= 2);
+
+  const void* size = &ar.size();
+  BOOST_MPI_CHECK_RESULT(MPI_Isend,
+                         (const_cast<void*>(size), 1, 
+                          get_mpi_datatype<std::size_t>(), 
+                          dest, tag, comm, out_requests));
+  BOOST_MPI_CHECK_RESULT(MPI_Isend,
+                         (const_cast<void*>(ar.address()), ar.size(),
+                          MPI_PACKED,
+                          dest, tag, comm, out_requests + 1));
+
+  return 2;
+}
+
+void
+packed_archive_recv(MPI_Comm comm, int source, int tag, packed_iarchive& ar,
+                    MPI_Status& status)
+{
+  std::size_t count;
+  BOOST_MPI_CHECK_RESULT(MPI_Recv,
+                         (&count, 1, get_mpi_datatype<std::size_t>(),
+                          source, tag, comm, &status));
+
+  // Prepare input buffer and receive the message
+  ar.resize(count);
+  BOOST_MPI_CHECK_RESULT(MPI_Recv,
+                         (ar.address(), ar.size(), MPI_PACKED,
+                          status.MPI_SOURCE, status.MPI_TAG,
+                          comm, &status));
+}
+
+} } } } // end namespace boost::parallel::mpi::detail

src/mpi/python/collectives.cpp

@@ -0,0 +1,144 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file status.cpp
+ *
+ *  This file reflects the Boost.MPI @c status class into
+ *  Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+#include <boost/parallel/mpi/python/serialize.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* all_gather_docstring;
+extern const char* all_reduce_docstring;
+extern const char* all_to_all_docstring;
+extern const char* broadcast_docstring;
+extern const char* gather_docstring;
+extern const char* reduce_docstring;
+extern const char* scan_docstring;
+extern const char* scatter_docstring;
+
+object all_gather(const communicator& comm, object value)
+{
+  std::vector<object> values;
+  boost::parallel::mpi::all_gather(comm, value, values);
+
+  boost::python::list l;
+  for (int i = 0; i < comm.size(); ++i)
+    l.append(values[i]);
+  return boost::python::tuple(l);
+}
+
+object all_to_all(const communicator& comm, object in_values)
+{
+  // Build input values
+  std::vector<object> in_values_vec(comm.size());
+  object iterator = object(handle<>(PyObject_GetIter(in_values.ptr())));
+  for (int i = 0; i < comm.size(); ++i)
+    in_values_vec[i] = object(handle<>(PyIter_Next(iterator.ptr())));
+
+  std::vector<object> out_values_vec(comm.size());
+  boost::parallel::mpi::all_to_all(comm, in_values_vec, out_values_vec);
+
+  boost::python::list l;
+  for (int i = 0; i < comm.size(); ++i)
+    l.append(out_values_vec[i]);
+  return boost::python::tuple(l);
+}
+
+object broadcast(const communicator& comm, object value, int root)
+{
+  boost::parallel::mpi::broadcast(comm, value, root);
+  return value;
+}
+
+object gather(const communicator& comm, object value, int root)
+{
+  if (comm.rank() == root) {
+    std::vector<object> values;
+    boost::parallel::mpi::gather(comm, value, values, root);
+
+    boost::python::list l;
+    for (int i = 0; i < comm.size(); ++i)
+      l.append(values[i]);
+    return boost::python::tuple(l);
+  } else {
+    boost::parallel::mpi::gather(comm, value, root);
+    return object();
+  }
+}
+
+object reduce(const communicator& comm, object value, object op, int root)
+{
+  if (comm.rank() == root) {
+    object out_value;
+    boost::parallel::mpi::reduce(comm, value, out_value, op, root);
+    return out_value;
+  } else {
+    boost::parallel::mpi::reduce(comm, value, op, root);
+    return object();
+  }
+}
+
+object scatter(const communicator& comm, object values, int root)
+{
+  object result;
+
+  if (comm.rank() == root) {
+    std::vector<object> values_vec(comm.size());
+    object iterator = object(handle<>(PyObject_GetIter(values.ptr())));
+    for (int i = 0; i < comm.size(); ++i)
+      values_vec[i] = object(handle<>(PyIter_Next(iterator.ptr())));
+
+    boost::parallel::mpi::scatter(comm, values_vec, result, root);
+  } else {
+    boost::parallel::mpi::scatter(comm, result, root);
+  }
+  return result;
+}
+
+void export_collectives()
+{
+  using boost::python::arg;
+
+  def("all_reduce", 
+      (object (*)(const communicator&, const object&, object))&all_reduce,
+      (arg("comm") = communicator(), arg("value"), arg("op")),
+      all_reduce_docstring);
+  def("all_gather", &all_gather, 
+      (arg("comm") = communicator(), arg("value") = object()),
+      all_gather_docstring);
+  def("all_to_all", &all_to_all,
+      (arg("comm") = communicator(), arg("values") = object()),
+      all_to_all_docstring);
+  def("broadcast", &broadcast, 
+      (arg("comm") = communicator(), arg("value") = object(), arg("root")),
+      broadcast_docstring);
+  def("gather", &gather, 
+      (arg("comm") = communicator(), arg("value") = object(), arg("root")),
+      gather_docstring);
+  def("reduce", &reduce, 
+      (arg("comm") = communicator(), arg("value"), arg("op"), 
+       arg("root")),
+      reduce_docstring);
+  def("scan", 
+      (object (*)(const communicator&, const object&, object))&scan,
+      (arg("comm") = communicator(), arg("value"), arg("op")),
+      scan_docstring);
+  def("scatter", &scatter, 
+      (arg("comm") = communicator(), arg("values") = object(), arg("root")),
+      scatter_docstring);
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/communicator.cpp

@@ -0,0 +1,134 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file communicator.cpp
+ *
+ *  This file reflects the Boost.MPI @c communicator class into
+ *  Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+#include <boost/parallel/mpi/python/serialize.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* communicator_docstring;
+extern const char* communicator_default_constructor_docstring;
+extern const char* communicator_rank_docstring;
+extern const char* communicator_size_docstring;
+extern const char* communicator_send_docstring;
+extern const char* communicator_recv_docstring;
+extern const char* communicator_isend_docstring;
+extern const char* communicator_irecv_docstring;
+extern const char* communicator_probe_docstring;
+extern const char* communicator_iprobe_docstring;
+extern const char* communicator_barrier_docstring;
+extern const char* communicator_split_docstring;
+extern const char* communicator_split_key_docstring;
+extern const char* communicator_abort_docstring;
+
+object 
+communicator_recv(const communicator& comm, int source, int tag,
+                  bool return_status)
+{
+  using boost::python::make_tuple;
+
+  object result;
+  status stat = comm.recv(source, tag, result);
+  if (return_status)
+    return make_tuple(result, stat);
+  else
+    return result;
+}
+
+object 
+communicator_irecv(const communicator& comm, int source, int tag)
+{
+  using boost::python::make_tuple;
+
+  object result;
+  object req(comm.irecv(source, tag, result));
+  req.attr("value") = result;
+  return req;
+}
+
+object
+communicator_iprobe(const communicator& comm, int source, int tag)
+{
+  if (boost::optional<status> result = comm.iprobe(source, tag))
+    return object(*result);
+  else
+    return object();
+}
+
+extern void export_skeleton_and_content(class_<communicator>&);
+
+void export_communicator()
+{
+  using boost::python::arg;
+  using boost::python::object;
+  
+  class_<communicator> comm("communicator", communicator_docstring);
+  comm
+    .def(init<>())
+    .add_property("rank", &communicator::rank, communicator_rank_docstring)
+    .add_property("size", &communicator::size, communicator_size_docstring)
+    .def("send", 
+         (void (communicator::*)(int, int, const object&) const)
+           &communicator::send<object>, 
+         (arg("dest"), arg("tag") = 0, arg("value") = object()),
+         communicator_send_docstring)
+    .def("recv", &communicator_recv, 
+         (arg("source") = any_source, arg("tag") = any_tag,
+          arg("return_status") = false),
+         communicator_recv_docstring)
+    .def("isend", 
+         (request (communicator::*)(int, int, const object&) const)
+           &communicator::isend<object>, 
+         (arg("dest"), arg("tag") = 0, arg("value") = object()),
+         communicator_isend_docstring)
+    .def("irecv", &communicator_irecv, 
+         (arg("source") = any_source, arg("tag") = any_tag),
+         communicator_irecv_docstring)
+    .def("probe", &communicator::probe, 
+         (arg("source") = any_source, arg("tag") = any_tag),
+         communicator_probe_docstring)
+    .def("iprobe", &communicator_iprobe, 
+         (arg("source") = any_source, arg("tag") = any_tag),
+         communicator_iprobe_docstring)
+    .def("barrier", &communicator::barrier, communicator_barrier_docstring)
+    .def("__nonzero__", &communicator::operator bool)
+    .def("split", 
+         (communicator (communicator::*)(int) const)&communicator::split,
+         (arg("color")), communicator_split_docstring)
+    .def("split", 
+         (communicator (communicator::*)(int, int) const)&communicator::split,
+         (arg("color"), arg("key")))
+    .def("abort", &communicator::abort, arg("errcode"), 
+         communicator_abort_docstring)
+       ;
+
+  // Module-level attributes
+  scope().attr("any_source") = any_source;
+  scope().attr("any_tag") = any_tag;
+
+  {
+    communicator world;
+    scope().attr("world") = world;
+    scope().attr("rank") = world.rank();
+    scope().attr("size") = world.size();
+  }
+
+  // Export skeleton and content
+  export_skeleton_and_content(comm);
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/datatypes.cpp

@@ -0,0 +1,25 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file datatypes.cpp
+ *
+ *  This file provides datatypes support for Boost.MPI in Python.
+ */
+#include <boost/parallel/mpi/python/serialize.hpp>
+#include <boost/parallel/mpi.hpp>
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+void export_datatypes()
+{
+  register_serialized(long(0), &PyInt_Type);
+  register_serialized(false, &PyBool_Type);
+  register_serialized(double(0.0), &PyFloat_Type);
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/documentation.cpp

@@ -0,0 +1,581 @@
+// (C) Copyright 2005 The Trustees of Indiana University.
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file documentation.cpp
+ *
+ *  This file contains all of the documentation strings for the
+ *  Boost.MPI Python bindings.
+ */
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+const char* module_docstring = 
+  "The boost.parallel.mpi module contains Python wrappers for Boost.MPI.\n"
+  "Boost.MPI is a C++ interface to the Message Passing Interface 1.1,\n"
+  "a high-performance message passing library for parallel programming.\n"
+  "\n"
+  "This module supports the most commonly used subset of MPI 1.1. All\n"
+  "communication operations can transmit any Python object that can be\n"
+  "pickled and unpickled, along with C++-serialized data types and\n"
+  "separation of the structure of a data type from its content.\n"
+  "Collectives that have a user-supplied functions,\n"
+  "such as reduce() or scan(), accept arbitrary Python functions, and\n"
+  "all collectives can operate on any serializable or picklable data type.\n"
+  "\n"
+  "IMPORTANT MODULE DATA\n"
+  "  any_source       This constant may be used for the source parameter of\n"
+  "                   receive and probe operations to indicate that a\n"
+  "                   message may be received from any source.\n"
+  "\n"
+  "  any_tag          This constant may be used for the tag parameter of\n"
+  "                   receive or probe operations to indicate that a send\n"
+  "                   with any tag will be matched.\n"
+  "\n"
+  "  collectives_tag  Returns the reserved tag value used by the Boost.MPI\n"
+  "                   implementation for collective operations. Although\n"
+  "                   users are not permitted to use this tag to send or\n"
+  "                   receive messages with this tag, it may be useful when\n"
+  "                   monitoring communication patterns.\n"
+  "\n"
+  "  host_rank        If there is a host process, this is the rank of that\n"
+  "                   that process. Otherwise, this value will be None. MPI\n"
+  "                   does not define the meaning of a \"host\" process: \n"
+  "                   consult the documentation for your MPI implementation.\n"
+  "\n"
+  "  io_rank          The rank of a process that can perform input/output\n"
+  "                   via the standard facilities. If every process can\n"
+  "                   perform I/O using the standard facilities, this value\n"
+  "                   will be the same as any_source. If no process can\n"
+  "                   perform I/O, this value will be None.\n"
+  "\n"
+  "  max_tag          The maximum value that may be used for the tag\n"
+  "                   parameter of send/receive operations. This value will\n"
+  "                   be somewhat smaller than the value of MPI_TAG_UB,\n"
+  "                   because the Boost.MPI implementation reserves some\n"
+  "                   tags for collective operations.\n"
+  "\n"
+  "  processor_name   The name of this processor. The actual form of the\n"
+  "                   of the name is unspecified, but may be documented by\n"
+  "                   the underlying MPI implementation.\n"
+  "\n"
+  "  rank             The rank of this process in the \"world\" communicator.\n"
+  "\n"
+  "  size             The number of processes in the \"world\" communicator.\n"
+  "                   that process. Otherwise, this value will be None. MPI\n"
+  "                   does not define the meaning of a \"host\" process: \n"
+  "\n"
+  "  world            The \"world\" communicator from which all other\n"
+  "                   communicators will be derived. This is the equivalent\n"
+  "                   of MPI_COMM_WORLD.\n"
+  "\n"
+  "TRANSMITTING USER-DEFINED DATA\n"
+  "  Boost.MPI can transmit user-defined data in several different ways.\n"
+  "  Most importantly, it can transmit arbitrary Python objects by pickling\n"
+  "  them at the sender and unpickling them at the receiver, allowing\n"
+  "  arbitrarily complex Python data structures to interoperate with MPI.\n"
+  "\n"
+  "  Boost.MPI also supports efficient serialization and transmission of\n"
+  "  C++ objects (that have been exposed to Python) through its C++\n"
+  "  interface. Any C++ type that provides (de-)serialization routines that\n"
+  "  meet the requirements of the Boost.Serialization library is eligible\n"
+  "  for this optimization, but the type must be registered in advance. To\n"
+  "  register a C++ type, invoke the C++ function:\n"
+  "    boost::parallel::mpi::python::register_serialized\n"
+  "\n"
+  "  Finally, Boost.MPI supports separation of the structure of an object\n"
+  "  from the data it stores, allowing the two pieces to be transmitted\n"
+  "  separately. This \"skeleton/content\" mechanism, described in more\n"
+  "  detail in a later section, is a communication optimization suitable\n"
+  "  for problems with fixed data structures whose internal data changes\n"
+  "  frequently.\n"
+  "\n"
+  "COLLECTIVES\n"
+  "  Boost.MPI supports all of the MPI collectives (scatter, reduce, scan,\n"
+  "  broadcast, etc.) for any type of data that can be transmitted with the\n"
+  "  point-to-point communication operations. For the MPI collectives that\n"
+  "  require a user-specified operation (e.g., reduce and scan), the\n"
+  "  operation can be an arbitrary Python function. For instance, one could\n"
+  "  concatenate strings with all_reduce:\n\n"
+  "    mpi.all_reduce(my_string, lambda x,y: x + y)\n\n"
+  "  The following module-level functions implement MPI collectives:\n"
+  "    all_gather    Gather the values from all processes.\n"
+  "    all_reduce    Combine the results from all processes.\n"
+  "    all_to_all    Every process sends data to every other process.\n"
+  "    broadcast     Broadcast data from one process to all other processes.\n"
+  "    gather        Gather the values from all processes to the root.\n"
+  "    reduce        Combine the results from all processes to the root.\n"
+  "    scan          Prefix reduction of the values from all processes.\n"
+  "    scatter       Scatter the values stored at the root to all processes.\n"
+  "\n"
+  "SKELETON/CONTENT MECHANISM\n"
+  "  Boost.MPI provides a skeleton/content mechanism that allows the\n"
+  "  transfer of large data structures to be split into two separate stages,\n"
+  "  with the `skeleton' (or, `shape') of the data structure sent first and\n"
+  "  the content (or, `data') of the data structure sent later, potentially\n"
+  "  several times, so long as the structure has not changed since the\n"
+  "  skeleton was transferred. The skeleton/content mechanism can improve\n"
+  "  performance when the data structure is large and its shape is fixed,\n"
+  "  because while the skeleton requires serialization (it has an unknown\n"
+  "  size), the content transfer is fixed-size and can be done without\n"
+  "  extra copies.\n"
+  "\n"
+  "  To use the skeleton/content mechanism from Python, you must first\n"
+  "  register the type of your data structure with the skeleton/content\n"
+  "  mechanism *from C++*. The registration function is\n"
+  "    boost::parallel::mpi::python::register_skeleton_and_content\n"
+  "  and resides in the <boost/parallel/mpi/python.hpp> header.\n"
+  "\n"
+  "  Once you have registered your C++ data structures, you can extract\n"
+  "  the skeleton for an instance of that data structure with skeleton().\n"
+  "  The resulting skeleton_proxy can be transmitted via the normal send\n"
+  "  routine, e.g.,\n\n"
+  "    mpi.world.send(1, 0, skeleton(my_data_structure))\n\n"
+  "  skeleton_proxy objects can be received on the other end via recv(),\n"
+  "  which stores a newly-created instance of your data structure with the\n"
+  "  same `shape' as the sender in its `object' attribute:\n\n"
+  "    shape = mpi.world.recv(0, 0)\n"
+  "    my_data_structure = shape.object\n\n"
+  "  Once the skeleton has been transmitted, the content (accessed via \n"
+  "  get_content) can be transmitted in much the same way. Note, however,\n"
+  "  that the receiver also specifies get_content(my_data_structure) in its\n"
+  "  call to receive:\n\n"
+  "    if mpi.rank == 0:\n"
+  "      mpi.world.send(1, 0, get_content(my_data_structure))\n"
+  "    else:\n"
+  "      mpi.world.recv(0, 0, get_content(my_data_structure))\n\n"
+  "  Of course, this transmission of content can occur repeatedly, if the\n"
+  "  values in the data structure--but not its shape--changes.\n"
+  "\n"
+  "  The skeleton/content mechanism is a structured way to exploit the\n"
+  "  interaction between custom-built MPI datatypes and MPI_BOTTOM, to\n"
+  "  eliminate extra buffer copies.\n"
+  "\n"
+  "C++/PYTHON MPI COMPATIBILITY\n"
+  "  Boost.MPI is a C++ library whose facilities have been exposed to Python\n"
+  "  via the Boost.Python library. Since the Boost.MPI Python bindings are\n"
+  "  build directly on top of the C++ library, and nearly every feature of\n"
+  "  C++ library is available in Python, hybrid C++/Python programs using\n"
+  "  Boost.MPI can interact, e.g., sending a value from Python but receiving\n"
+  "  that value in C++ (or vice versa). However, doing so requires some\n"
+  "  care. Because Python objects are dynamically typed, Boost.MPI transfers\n"
+  "  type information along with the serialized form of the object, so that\n"
+  "  the object can be received even when its type is not known. This\n"
+  "  mechanism differs from its C++ counterpart, where the static types of\n"
+  "  transmitted values are always known.\n"
+  "\n"
+  "  The only way to communicate between the C++ and Python views on \n"
+  "  Boost.MPI is to traffic entirely in Python objects. For Python, this is\n"
+  "  the normal state of affairs, so nothing will change. For C++, this\n"
+  "  means sending and receiving values of type boost::python::object, from\n"
+  "  the Boost.Python library. For instance, say we want to transmit an\n"
+  "  integer value from Python:\n\n"
+  "    comm.send(1, 0, 17)\n\n"
+  "  In C++, we would receive that value into a Python object and then\n"
+  "  `extract' an integer value:\n\n"
+  "    boost::python::object value;\n"
+  "    comm.recv(0, 0, value);\n"
+  "    int int_value = boost::python::extract<int>(value);\n\n"
+  "  In the future, Boost.MPI will be extended to allow improved\n"
+  "  interoperability with the C++ Boost.MPI and the C MPI bindings.\n"
+  ;
+
+/***********************************************************
+ * environment documentation                               *
+ ***********************************************************/
+const char* environment_init_docstring = 
+  "Initialize the MPI environment. Users should not need to call\n"
+  "this function directly, because the MPI environment will be\n"
+  "automatically initialized when the Boost.MPI module is loaded.\n";
+
+const char* environment_finalize_docstring = 
+  "Finalize (shut down) the MPI environment. Users only need to\n"
+  "invoke this function if MPI should be shut down before program\n"
+  "termination. Boost.MPI will automatically finalize the MPI\n"
+  "environment when the program exits.\n";
+ 
+const char* environment_abort_docstring = 
+  "Aborts all MPI processes and returns to the environment. The\n"
+  "precise behavior will be defined by the underlying MPI\n"
+  "implementation. This is equivalent to a call to MPI_Abort with\n"
+  "MPI_COMM_WORLD.\n"
+  "errcode is the error code to return from aborted processes.\n";
+
+const char* environment_initialized_docstring = 
+ "Determine if the MPI environment has already been initialized.\n";
+
+const char* environment_finalized_docstring = 
+  "Determine if the MPI environment has already been finalized.\n";
+
+/***********************************************************
+ * exception documentation                                 *
+ ***********************************************************/
+const char* exception_docstring = 
+  "Instances of this class will be thrown when an MPI error\n"
+  "occurs. MPI failures that trigger these exceptions may or may not\n"
+  "be recoverable, depending on the underlying MPI implementation.\n"
+  "Consult the documentation for your MPI implementation to determine\n"
+  "the effect of MPI errors.\n";
+
+const char* exception_what_docstring = 
+  "A description of the error that occured. At present, this refers\n"
+  "only to the name of the MPI routine that failed.\n";
+
+const char* exception_routine_docstring = 
+  "The name of the MPI routine that reported the error.\n";
+
+const char* exception_result_code_docstring = 
+  "The result code returned from the MPI routine that reported the\n"
+  "error.\n";
+
+/***********************************************************
+ * collectives documentation                               *
+ ***********************************************************/
+const char* all_gather_docstring =
+  "all_gather is a collective algorithm that collects the values\n"
+  "stored at each process into a tuple of values indexed by the\n"
+  "process number they came from. all_gather is (semantically) a\n" 
+  "gather followed by a broadcast. The same tuple of values is\n"
+  "returned to all processes.\n";
+
+const char* all_reduce_docstring =
+  "all_reduce is a collective algorithm that combines the values\n"
+  "stored by each process into a single value. The values can be\n"
+  "combined arbitrarily, specified via any function. The values\n"
+  "a1, a2, .., ap provided by p processors will be combined by the\n"
+  "binary function op into the result\n"
+  "         op(a1, op(a2, ... op(ap-1,ap)))\n"
+  "that will be returned to all processes. This function is the\n"
+  "equivalent of calling all_gather() and then applying the built-in\n"
+  "reduce() function to the returned sequence. op is assumed to be\n"
+  "associative.\n";
+
+const char* all_to_all_docstring =
+  "all_to_all is a collective algorithm that transmits values from\n"
+  "every process to every other process. On process i, the jth value\n"
+  "of the values sequence is sent to process j and placed in the ith\n"
+  "position of the tuple that will be returned from all_to_all.\n";
+
+const char* broadcast_docstring =
+  "broadcast is a collective algorithm that transfers a value from an\n"
+  "arbitrary root process to every other process that is part of the\n"
+  "given communicator (comm). The root parameter must be the same for\n"
+  "every process. The value parameter need only be specified at the root\n"
+  "root. broadcast() returns the same broadcasted value to every process.\n";
+
+const char* gather_docstring =
+  "gather is a collective algorithm that collects the values\n"
+  "stored at each process into a tuple of values at the root\n"
+  "process. This tuple is indexed by the process number that the\n"
+  "value came from, and will be returned only by the root process.\n"
+  "All other processes return None.\n";
+
+const char* reduce_docstring =
+  "reduce is a collective algorithm that combines the values\n"
+  "stored by each process into a single value at the root. The\n"
+  "values can be combined arbitrarily, specified via any function.\n"
+  "The values a1, a2, .., ap provided by p processors will be\n"
+  "combined by the binary function op into the result\n"
+  "     op(a1, op(a2, ... op(ap-1,ap)))\n"
+  "that will be returned on the root process. This function is the\n"
+  "equivalent of calling gather() to the root and then applying the\n"
+  "built-in reduce() function to the returned sequence. All non-root\n"
+  "processes return None. op is assumed to be associative.\n";
+
+const char* scan_docstring =
+  "@c scan computes a prefix reduction of values from all processes.\n"
+  "It is a collective algorithm that combines the values stored by\n"
+  "each process with the values of all processes with a smaller rank.\n"
+  "The values can be arbitrarily combined, specified via a binary\n"
+  "function op. If each process i provides the value ai, then scan\n"
+  "returns op(a1, op(a2, ... op(ai-1, ai))) to the ith process. op is\n"
+  "assumed to be associative. This routine is the equivalent of an\n"
+  "all_gather(), followed by a built-in reduce() on the first i+1\n"
+  "values in the resulting sequence on processor i. op is assumed\n"
+  "to be associative.\n";
+
+const char* scatter_docstring =
+  "scatter is a collective algorithm that scatters the values stored\n"
+  "in the root process (as a container with comm.size elements) to\n"
+  "all of the processes in the communicator. The values parameter \n"
+  "(only significant at the root) is indexed by the process number to\n"
+  "which the corresponding value will be sent. The value received by \n"
+  "each process is returned from scatter.\n";
+
+/***********************************************************
+ * communicator documentation                              *
+ ***********************************************************/
+const char* communicator_docstring =
+ "The communicator class abstracts a set of communicating\n"
+ "processes in MPI. All of the processes that belong to a certain\n"
+ "communicator can determine the size of the communicator, their rank\n"
+ "within the communicator, and communicate with any other processes\n"
+ "in the communicator.\n";
+
+const char* communicator_default_constructor_docstring =
+  "Build a new Boost.MPI communicator for MPI_COMM_WORLD.\n";
+
+const char* communicator_rank_docstring =
+  "Returns the rank of the process in the communicator, which will be a\n"
+  "value in [0, size).\n";
+
+const char* communicator_size_docstring =
+  "Returns the number of processes in the communicator.\n";
+
+const char* communicator_send_docstring =
+  "This routine executes a potentially blocking send with the given\n"
+  "tag to the process with rank dest. It can be received by the\n"
+  "destination process with a matching recv call. The value will be\n"
+  "transmitted in one of several ways:\n"
+  "\n"
+  "  - For C++ objects registered via register_serialized(), the value\n"
+  "    will be serialized and transmitted.\n"
+  "\n"
+  "  - For skeleton_proxy objects, the skeleton of the object will be\n"
+  "    serialized and transmitted.\n"
+  "\n"
+  "  - For content objects, the content will be transmitted directly.\n"
+  "    This content can be received by a matching recv/irecv call that\n"
+  "    provides a suitable `buffer' argument.\n"
+  "\n"
+  "  - For all other Python objects, the value will be pickled and\n"
+  "    transmitted.\n";
+
+const char* communicator_recv_docstring =
+  "This routine blocks until it receives a message from the process\n"
+  "source with the given tag. If the source parameter is not specified,\n"
+  "the message can be received from any process. Likewise, if the tag\n"
+  "parameter is not specified, a message with any tag can be received.\n"
+  "If return_status is True, returns a tuple containing the received\n"
+  "object followed by a status object describing the communication.\n"
+  "Otherwise, recv() returns just the received object.\n"
+  "\n"
+  "When receiving the content of a data type that has been sent separately\n"
+  "from its skeleton, user code must provide a value for the `buffer'\n"
+  "argument. This value should be the content object returned from\n"
+  "get_content().\n";
+
+const char* communicator_isend_docstring =
+  "This routine executes a nonblocking send with the given\n"
+  "tag to the process with rank dest. It can be received by the\n"
+  "destination process with a matching recv call. The value will be\n"
+  "transmitted in the same way as with send().\n"
+  "This routine returns a request object, which can be used to query\n"
+  "when the transmission has completed, wait for its completion, or\n"
+  "cancel the transmission.\n";
+
+const char* communicator_irecv_docstring =
+  "This routine initiates a non-blocking receive from the process\n"
+  "source with the given tag. If the source parameter is not specified,\n"
+  "the message can be received from any process. Likewise, if the tag\n"
+  "parameter is not specified, a message with any tag can be received.\n"
+  "This routine returns a request object, which can be used to query\n"
+  "when the transmission has completed, wait for its completion, or\n"
+  "cancel the transmission. The received value be accessible\n"
+  "through the `value' attribute of the request object once transmission\n"
+  "has completed.\n"
+  "\n"
+  "As with the recv() routine, when receiving the content of a data type\n"
+  "that has been sent separately from its skeleton, user code must provide\n"
+  "a value for the `buffer' argument. This value should be the content\n"
+  "object returned from get_content().\n"; 
+
+ const char* communicator_probe_docstring =
+  "This operation waits until a message matching (source, tag)\n"
+  "is available to be received. It then returns information about\n"
+  "that message. If source is omitted, a message from any process\n"
+  "will match. If tag is omitted, a message with any tag will match.\n"
+  "The actual source and tag can be retrieved from the returned status\n"
+  "object. To check if a message is available without blocking, use\n"
+  "iprobe.\n";
+
+const char* communicator_iprobe_docstring =
+  "This operation determines if a message matching (source, tag) is\n"
+  "available to be received. If so, it returns information about that\n"
+  "message; otherwise, it returns None. If source is omitted, a message\n"
+  "from any process will match. If tag is omitted, a message with any\n"
+  "tag will match. The actual source and tag can be retrieved from the\n"
+  "returned status object. To wait for a message to become available, use\n"
+  "probe.\n";
+
+const char* communicator_barrier_docstring = 
+  "Wait for all processes within a communicator to reach the\n"
+  "barrier.\n";
+
+const char* communicator_split_docstring = 
+  "Split the communicator into multiple, disjoint communicators\n"
+  "each of which is based on a particular color. This is a\n"
+  "collective operation that returns a new communicator that is a\n"
+  "subgroup of this. This routine is functionally equivalent to\n"
+  "MPI_Comm_split.\n\n"
+  "color is the color of this process. All processes with the\n"
+  "same color value will be placed into the same group.\n\n"
+  "If provided, key is a key value that will be used to determine\n"
+  "the ordering of processes with the same color in the resulting\n"
+  "communicator. If omitted, the key will default to the rank of\n"
+  "the process in the current communicator.\n\n"
+  "Returns a new communicator containing all of the processes in\n"
+  "this communicator that have the same color.\n";
+
+const char* communicator_abort_docstring = 
+  "Makes a \"best attempt\" to abort all of the tasks in the group of\n"
+  "this communicator. Depending on the underlying MPI\n"
+  "implementation, this may either abort the entire program (and\n"
+  "possibly return errcode to the environment) or only abort\n"
+  "some processes, allowing the others to continue. Consult the\n"
+  "documentation for your MPI implementation. This is equivalent to\n"
+  "a call to MPI_Abort\n\n"
+  "errcode is the error code to return from aborted processes.\n";
+
+/***********************************************************
+ * request documentation                                   *
+ ***********************************************************/
+const char* request_docstring = 
+  "The request class contains information about a non-blocking send\n"
+  "or receive and will be returned from isend or irecv, respectively.\n"
+  "When a request object represents a completed irecv, the `value' \n"
+  "attribute will contain the received value.\n";
+
+const char* request_wait_docstring =
+  "Wait until the communication associated with this request has\n"
+  "completed. For a request that is associated with an isend(), returns\n"
+  "a status object describing the communication. For an irecv()\n"
+  "operation, returns the received value by default. However, when\n"
+  "return_status=True, a (value, status) pair is returned by a.\n"
+  "completed irecv request.\n";
+
+const char* request_test_docstring =
+  "Determine whether the communication associated with this request\n"
+  "has completed successfully. If so, returns the status object\n"
+  "describing the communication (for an isend request) or a tuple\n"
+  "containing the received value and a status object (for an irecv\n"
+  "request). Note that once test() returns a status object, the\n"
+  "request has completed and wait() should not be called.\n";
+
+const char* request_cancel_docstring =
+  "Cancel a pending communication, assuming it has not already been\n"
+  "completed.\n";
+
+/***********************************************************
+ * skeleton/content documentation                          *
+ ***********************************************************/
+const char* object_without_skeleton_docstring = 
+  "The object_without_skeleton class is an exception class used only\n"
+  "when the skeleton() or get_content() function is called with an\n"
+  "object that is not supported by the skeleton/content mechanism.\n"
+  "All C++ types for which skeletons and content can be transmitted\n"
+  "must be registered with the C++ routine:\n"
+  "  boost::parallel::mpi::python::register_skeleton_and_content\n";
+
+const char* object_without_skeleton_object_docstring = 
+  "The object on which skeleton() or get_content() was invoked.\n";
+
+const char* skeleton_proxy_docstring = 
+  "The skeleton_proxy class is used to represent the skeleton of an\n"
+  "object. The skeleton_proxy can be used as the value parameter of\n"
+  "send() or isend() operations, but instead of transmitting the\n"
+  "entire object, only its skeleton (\"shape\") will be sent, without\n"
+  "the actual data. Its content can then be transmitted, separately.\n"
+  "\n"
+  "User code cannot generate skeleton_proxy instances directly. To\n"
+  "refer to the skeleton of an object, use skeleton(object). Skeletons\n"
+  "can also be received with the recv() and irecv() methods.\n"
+  "\n"
+  "Note that the skeleton/content mechanism can only be used with C++\n"
+  "types that have been explicitly registered.\n";
+
+const char* skeleton_proxy_object_docstring = 
+  "The actual object whose skeleton is represented by this proxy object.\n";
+
+const char* content_docstring = 
+  "The content is a proxy class that represents the content of an object,\n"
+  "which can be separately sent or received from its skeleton.\n"
+  "\n"
+  "User code cannot generate content instances directly. Call the\n"
+  "get_content() routine to retrieve the content proxy for a particular\n"
+  "object. The content instance can be used with any of the send() or\n"
+  "recv() variants. Note that get_content() can only be used with C++\n"
+  "data types that have been explicitly registered with the Python\n"
+  "skeleton/content mechanism.\n";
+
+const char* skeleton_docstring = 
+  "The skeleton function retrieves the skeleton_proxy for its object\n"
+  "parameter, allowing the transmission of the skeleton (or \"shape\")\n"
+  "of the object separately from its data. The skeleton/content mechanism\n"
+  "is useful when a large data structure remains structurally the same\n"
+  "throughout a computation, but its content (i.e., the values in the\n"
+  "structure) changes several times. Tranmission of the content part does\n"
+  "not require any serialization or unnecessary buffer copies, so it is\n"
+  "very efficient for large data structures.\n"
+  "\n"
+  "Only C++ types that have been explicitly registered with the Boost.MPI\n"
+  "Python library can be used with the skeleton/content mechanism. Use:\b"
+  "  boost::parallel::mpi::python::register_skeleton_and_content\n";
+
+const char* get_content_docstring = 
+  "The get_content function retrieves the content for its object parameter,\n"
+  "allowing the transmission of the data in a data structure separately\n"
+  "from its skeleton (or \"shape\"). The skeleton/content mechanism\n"
+  "is useful when a large data structure remains structurally the same\n"
+  "throughout a computation, but its content (i.e., the values in the\n"
+  "structure) changes several times. Tranmission of the content part does\n"
+  "not require any serialization or unnecessary buffer copies, so it is\n"
+  "very efficient for large data structures.\n"
+  "\n"
+  "Only C++ types that have been explicitly registered with the Boost.MPI\n"
+  "Python library can be used with the skeleton/content mechanism. Use:\b"
+  "  boost::parallel::mpi::python::register_skeleton_and_content\n";
+
+/***********************************************************
+ * status documentation                                    *
+ ***********************************************************/
+const char* status_docstring = 
+  "The status class stores information about a given message, including\n"
+  "its source, tag, and whether the message transmission was cancelled\n"
+  "or resulted in an error.\n";
+
+const char* status_source_docstring =
+  "The source of the incoming message.\n";
+
+const char* status_tag_docstring =
+  "The tag of the incoming message.\n";
+
+const char* status_error_docstring =
+  "The error code associated with this transmission.\n";
+
+const char* status_cancelled_docstring =
+  "Whether this transmission was cancelled.\n";
+
+/***********************************************************
+ * timer documentation                                     *
+ ***********************************************************/
+const char* timer_docstring =
+  "The timer class is a simple wrapper around the MPI timing facilities.\n";
+
+const char* timer_default_constructor_docstring =
+  "Initializes the timer. After this call, elapsed == 0.\n";
+
+const char* timer_restart_docstring =
+  "Restart the timer, after which elapsed == 0.\n";
+
+const char* timer_elapsed_docstring =
+  "The time elapsed since initialization or the last restart(),\n"
+  "whichever is more recent.\n";
+
+const char* timer_elapsed_min_docstring =
+  "Returns the minimum non-zero value that elapsed may return\n"
+  "This is the resolution of the timer.\n";
+
+const char* timer_elapsed_max_docstring =
+  "Return an estimate of the maximum possible value of elapsed. Note\n"
+  "that this routine may return too high a value on some systems.\n";
+
+const char* timer_time_is_global_docstring = 
+  "Determines whether the elapsed time values are global times or\n"
+  "local processor times.\n";
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/environment.cpp

@@ -0,0 +1,111 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file environment.cpp
+ *
+ *  This file reflects the Boost.MPI "environment" class into Python
+ *  methods at module level.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* environment_init_docstring;
+extern const char* environment_finalize_docstring;
+extern const char* environment_abort_docstring;
+extern const char* environment_initialized_docstring;
+extern const char* environment_finalized_docstring;
+
+/**
+ * The environment used by the Boost.MPI Python module. This will be
+ * zero-initialized before it is used. 
+ */
+static environment* env; 
+
+bool mpi_init(list python_argv, bool abort_on_exception)
+{
+  // If MPI is already initialized, do nothing.
+  if (environment::initialized())
+    return false;
+
+  // Convert Python argv into C-style argc/argv. 
+  int my_argc = extract<int>(python_argv.attr("__len__")());
+  char** my_argv = new char*[my_argc];
+  for (int arg = 0; arg < my_argc; ++arg)
+    my_argv[arg] = strdup(extract<const char*>(python_argv[arg]));
+
+  // Initialize MPI
+  int mpi_argc = my_argc;
+  char** mpi_argv = my_argv;
+  env = new environment(mpi_argc, mpi_argv, abort_on_exception);
+
+  // If anything changed, convert C-style argc/argv into Python argv
+  if (mpi_argv != my_argv)
+    PySys_SetArgv(mpi_argc, mpi_argv);
+
+  for (int arg = 0; arg < my_argc; ++arg)
+    free(my_argv[arg]);
+  delete [] my_argv;
+
+  return true;
+}
+
+void mpi_finalize()
+{
+  if (env) {
+    delete env;
+    env = 0;
+  }
+}
+
+void export_environment()
+{
+  using boost::python::arg;
+
+  def("init", mpi_init, (arg("argv"), arg("abort_on_exception") = true),
+      environment_init_docstring);
+  def("finalize", mpi_finalize, environment_finalize_docstring);
+
+  // Setup initialization and finalization code
+  if (!environment::initialized()) {
+    // MPI_Init from sys.argv
+    object sys = object(handle<>(PyImport_ImportModule("sys")));
+    mpi_init(extract<list>(sys.attr("argv")), true);
+
+    // Setup MPI_Finalize call when the program exits
+    object atexit = object(handle<>(PyImport_ImportModule("atexit")));
+    object finalize = scope().attr("finalize");
+    atexit.attr("register")(finalize);
+  }
+
+  def("abort", &environment::abort, arg("errcode"),
+      environment_abort_docstring);
+  def("initialized", &environment::initialized, 
+      environment_initialized_docstring);
+  def("finalized", &environment::finalized,
+      environment_finalized_docstring);
+  scope().attr("max_tag") = environment::max_tag();
+  scope().attr("collectives_tag") = environment::collectives_tag();
+  scope().attr("processor_name") = environment::processor_name();
+
+  if (optional<int> host_rank = environment::host_rank())
+    scope().attr("host_rank") = *host_rank;
+  else
+    scope().attr("host_rank") = object();
+  
+  if (optional<int> io_rank = environment::io_rank())
+    scope().attr("io_rank") = *io_rank;
+  else
+    scope().attr("io_rank") = object();
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/exception.cpp

@@ -0,0 +1,55 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+// Copyright (C) 2005 The Trustees of Indiana University.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file exception.cpp
+ *
+ *  This file reflects the Boost.MPI @c mpi_error class into
+ *  Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+#include <string>
+#include <boost/lexical_cast.hpp>
+#include "utility.hpp"
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* exception_docstring;
+extern const char* exception_what_docstring;
+extern const char* exception_routine_docstring;
+extern const char* exception_result_code_docstring;
+
+str exception_str(const exception& e)
+{
+  return str("MPI routine `"  + std::string(e.routine()) + 
+             "' returned error code " + 
+             lexical_cast<std::string>(e.result_code()));
+}
+
+void export_exception()
+{
+  using boost::python::arg;
+  using boost::python::object;
+  
+  object type = 
+    class_<exception>
+      ("exception", exception_docstring, no_init)
+      .add_property("what", &exception::what, exception_what_docstring)
+      .add_property("routine", &exception::what, exception_routine_docstring)
+      .add_property("result_code", &exception::what, 
+                    exception_result_code_docstring)
+      .def("__str__", &exception_str)
+    ;
+  translate_exception<exception>::declare(type);
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/module.cpp

@@ -0,0 +1,53 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file module.cpp
+ *
+ *  This file provides the top-level module for the Boost.MPI Python
+ *  bindings.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern void export_environment();
+extern void export_exception();
+extern void export_collectives();
+extern void export_communicator();
+extern void export_datatypes();
+extern void export_request();
+extern void export_status();
+extern void export_timer();
+
+extern const char* module_docstring;
+
+BOOST_PYTHON_MODULE(mpi)
+{
+  // Setup module documentation
+  scope().attr("__doc__") = module_docstring;
+  scope().attr("__author__") = "Douglas Gregor <doug.gregor@gmail.com>";
+  scope().attr("__date__") = "$LastChangedDate: 2006-07-16 15:25:47 -0400 (Sun, 16 Jul 2006) $";
+  scope().attr("__version__") = "$Revision$";
+  scope().attr("__copyright__") = "Copyright (C) 2006 Douglas Gregor";
+  scope().attr("__license__") = "http://www.boost.org/LICENSE_1_0.txt";
+
+  export_environment();
+  export_exception();
+  export_communicator();
+  export_collectives();
+  export_datatypes();
+  export_request();
+  export_status();
+  export_timer();
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/request.cpp

@@ -0,0 +1,64 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file request.cpp
+ *
+ *  This file reflects the Boost.MPI @c request class into
+ *  Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* request_docstring;
+extern const char* request_wait_docstring;
+extern const char* request_test_docstring;
+extern const char* request_cancel_docstring;
+
+object request_wait(object req_obj)
+{
+  request& req = extract<request&>(req_obj)();
+  status stat = req.wait();
+  if (PyObject_HasAttrString(req_obj.ptr(), "value"))
+    return boost::python::make_tuple(stat, req_obj.attr("value"));
+  else
+    return object(stat);  
+}
+
+object request_test(object req_obj)
+{
+  request& req = extract<request&>(req_obj)();
+
+  if (optional<status> stat = req.test())
+    {
+      if (PyObject_HasAttrString(req_obj.ptr(), "value"))
+        return boost::python::make_tuple(stat, req_obj.attr("value"));
+      else
+        return object(stat);
+    }
+  else
+    return object();
+}
+
+void export_request()
+{
+  using boost::python::arg;
+  using boost::python::object;
+  
+  class_<request>("request", request_docstring, no_init)
+    .def("wait", &request_wait, request_wait_docstring)
+    .def("test", &request_test, request_test_docstring)
+    .def("cancel", &request::cancel, request_cancel_docstring)
+    ;
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/serialize.cpp

@@ -0,0 +1,78 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file serialize.cpp
+ *
+ *  This file provides Boost.Serialization support for Python objects.
+ */
+#include <boost/parallel/mpi/python/serialize.hpp>
+#include <boost/parallel/mpi/python/skeleton_and_content.hpp>
+#include <boost/parallel/mpi.hpp>
+
+namespace boost { namespace python {
+
+struct pickle::data_t {
+  object module;
+  object dumps;
+  object loads;
+};
+
+
+/// Data used for communicating with the Python `pickle' module.
+pickle::data_t* pickle::data;
+
+str pickle::dumps(object obj, int protocol)
+{
+  if (!data) initialize_data();
+  return extract<str>((data->dumps)(obj, protocol));
+}
+
+object pickle::loads(str s)
+{
+  if (!data) initialize_data();
+  return ((data->loads)(s));
+}
+
+void pickle::initialize_data()
+{
+  data = new data_t;
+  data->module = object(handle<>(PyImport_ImportModule("pickle")));
+  data->dumps = data->module.attr("dumps");
+  data->loads = data->module.attr("loads");
+}
+
+} } // end namespace boost::python
+
+BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(
+  ::boost::parallel::mpi::packed_iarchive,
+  ::boost::parallel::mpi::packed_oarchive)
+
+namespace boost { namespace parallel { namespace mpi { namespace python { namespace detail {
+
+  boost::python::object skeleton_proxy_base_type; 
+
+  // A map from Python type objects to skeleton/content handlers
+  typedef std::map<PyTypeObject*, skeleton_content_handler>
+    skeleton_content_handlers_type;
+  skeleton_content_handlers_type skeleton_content_handlers;
+
+  bool
+  skeleton_and_content_handler_registered(PyTypeObject* type)
+  {
+    return 
+      skeleton_content_handlers.find(type) != skeleton_content_handlers.end();
+  }
+
+  void 
+  register_skeleton_and_content_handler(PyTypeObject* type, 
+                                        const skeleton_content_handler& handler)
+  {
+    skeleton_content_handlers[type] = handler;
+  }
+
+} } } } } // end namespace boost::parallel::mpi::python::detail

src/mpi/python/skeleton_and_content.cpp

@@ -0,0 +1,164 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file skeleton_and_content.cpp
+ *
+ *  This file reflects the skeleton/content facilities into Python.
+ */
+#include <boost/parallel/mpi/python/skeleton_and_content.hpp>
+#include <boost/parallel/mpi/python/serialize.hpp>
+#include <boost/python/list.hpp>
+#include <typeinfo>
+#include <list>
+#include "utility.hpp"
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+namespace detail {
+  typedef std::map<PyTypeObject*, skeleton_content_handler>
+    skeleton_content_handlers_type;
+  extern skeleton_content_handlers_type skeleton_content_handlers;
+}
+
+/**
+ * An exception that will be thrown when the object passed to the
+ * Python version of skeleton() does not have a skeleton.
+ */
+struct object_without_skeleton : public std::exception {
+  explicit object_without_skeleton(object value) : value(value) { }
+  virtual ~object_without_skeleton() throw() { }
+
+  object value;
+};
+
+str object_without_skeleton_str(const object_without_skeleton& e)
+{
+  return str("\nThe skeleton() or get_content() function was invoked for a Python\n"
+             "object that is not supported by the Boost.MPI skeleton/content\n"
+             "mechanism. To transfer objects via skeleton/content, you must\n"
+             "register the C++ type of this object with the C++ function:\n"
+             "  boost::parallel::mpi::python::register_skeleton_and_content()\n"
+             "Object: " + str(e.value) + "\n");
+}
+
+/**
+ * Extract the "skeleton" from a Python object. In truth, all we're
+ * doing at this point is verifying that the object is a C++ type that
+ * has been registered for the skeleton/content mechanism.
+ */
+object skeleton(object value)
+{
+  PyTypeObject* type = value.ptr()->ob_type;
+  detail::skeleton_content_handlers_type::iterator pos = 
+    detail::skeleton_content_handlers.find(type);
+  if (pos == detail::skeleton_content_handlers.end())
+    throw object_without_skeleton(value);
+  else
+    return pos->second.get_skeleton_proxy(value);
+}
+
+/**
+ * Extract the "content" from a Python object, which must be a C++
+ * type that has been registered for the skeleton/content mechanism.
+ */
+content get_content(object value)
+{
+  PyTypeObject* type = value.ptr()->ob_type;
+  detail::skeleton_content_handlers_type::iterator pos = 
+    detail::skeleton_content_handlers.find(type);
+  if (pos == detail::skeleton_content_handlers.end())
+    throw object_without_skeleton(value);
+  else
+    return pos->second.get_content(value);
+}
+
+/// Send the content part of a Python object.
+void 
+communicator_send_content(const communicator& comm, int dest, int tag, 
+                          const content& c)
+{
+  comm.send(dest, tag, c.base());
+}
+
+/// Receive the content of a Python object. We return the object
+/// received, not the content wrapper.
+object 
+communicator_recv_content(const communicator& comm, int source, int tag,
+                          const content& c, bool return_status)
+{
+  using boost::python::make_tuple;
+
+  status stat = comm.recv(source, tag, c.base());
+  if (return_status)
+    return make_tuple(c.object, stat);
+  else
+    return c.object;
+}
+
+/// Receive the content of a Python object. The request object's value
+/// attribute will reference the object whose content is being
+/// received, not the content wrapper.
+object 
+communicator_irecv_content(const communicator& comm, int source, int tag,
+                           const content& c)
+{
+  using boost::python::make_tuple;
+
+  object req(comm.irecv(source, tag, c.base()));
+  req.attr("value") = c.object;
+  return req;
+}
+
+extern const char* object_without_skeleton_docstring;
+extern const char* object_without_skeleton_object_docstring;
+extern const char* skeleton_proxy_docstring;
+extern const char* skeleton_proxy_object_docstring;
+extern const char* content_docstring;
+extern const char* skeleton_docstring;
+extern const char* get_content_docstring;
+
+void export_skeleton_and_content(class_<communicator>& comm)
+{
+  using boost::python::arg;
+
+  // Expose the object_without_skeleton exception
+  object type = 
+    class_<object_without_skeleton>
+      ("object_without_skeleton", object_without_skeleton_docstring, no_init)
+      .def_readonly("object", &object_without_skeleton::value,
+                    object_without_skeleton_object_docstring)
+      .def("__str__", &object_without_skeleton_str)
+    ;
+  translate_exception<object_without_skeleton>::declare(type);
+
+  // Expose the Python variants of "skeleton_proxy" and "content", and
+  // their generator functions.
+  detail::skeleton_proxy_base_type = 
+    class_<skeleton_proxy_base>("skeleton_proxy", skeleton_proxy_docstring, 
+                                no_init)
+      .def_readonly("object", &skeleton_proxy_base::object,
+                    skeleton_proxy_object_docstring);
+  class_<content>("content", content_docstring, no_init);
+  def("skeleton", &skeleton, arg("object"), skeleton_docstring);
+  def("get_content", &get_content, arg("object"), get_content_docstring);
+
+  // Expose communicator send/recv operations for content.
+  comm
+    .def("send", communicator_send_content,
+         (arg("dest"), arg("tag") = 0, arg("value")))
+    .def("recv", communicator_recv_content,
+         (arg("source") = any_source, arg("tag") = any_tag, arg("buffer"), 
+          arg("return_status") = false))
+    .def("irecv", communicator_irecv_content,
+         (arg("source") = any_source, arg("tag") = any_tag, arg("buffer")));
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/status.cpp

@@ -0,0 +1,41 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file status.cpp
+ *
+ *  This file reflects the Boost.MPI @c status class into
+ *  Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* status_docstring;
+extern const char* status_source_docstring;
+extern const char* status_tag_docstring;
+extern const char* status_error_docstring;
+extern const char* status_cancelled_docstring;
+
+void export_status()
+{
+  using boost::python::arg;
+  using boost::python::object;
+  
+  class_<status>("status", status_docstring, no_init)
+    .add_property("source", &status::source, status_source_docstring)
+    .add_property("tag", &status::tag, status_tag_docstring)
+    .add_property("error", &status::error, status_error_docstring)
+    .add_property("cancelled", &status::cancelled, status_cancelled_docstring)
+    ;
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/python/timer.cpp

@@ -0,0 +1,48 @@
+// (C) Copyright 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Douglas Gregor
+
+/** @file timer.cpp
+ *
+ *  This file reflects the Boost.MPI @c timer class into
+ *  Python.
+ */
+#include <boost/python.hpp>
+#include <boost/parallel/mpi/timer.hpp>
+
+using namespace boost::python;
+using namespace boost::parallel::mpi;
+
+namespace boost { namespace parallel { namespace mpi { namespace python {
+
+extern const char* timer_docstring;
+extern const char* timer_default_constructor_docstring;
+extern const char* timer_restart_docstring;
+extern const char* timer_elapsed_docstring;
+extern const char* timer_elapsed_min_docstring;
+extern const char* timer_elapsed_max_docstring;
+extern const char* timer_time_is_global_docstring;
+
+void export_timer()
+{
+  using boost::python::arg;
+  using boost::python::object;
+  
+  class_<timer>("timer", timer_docstring)
+    .def(init<>())
+    .def("restart", &timer::restart, timer_restart_docstring)
+    .add_property("elapsed", &timer::elapsed, timer_elapsed_docstring)
+    .add_property("elapsed_min", &timer::elapsed_min, 
+                  timer_elapsed_min_docstring)
+    .add_property("elapsed_max", &timer::elapsed_max, 
+                  timer_elapsed_max_docstring)
+    .add_property("time_is_global", &timer::time_is_global, 
+                  timer_time_is_global_docstring)
+    ;
+}
+
+} } } } // end namespace boost::parallel::mpi::python

src/mpi/request.cpp

@@ -0,0 +1,120 @@
+// Copyright (C) 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+#include <boost/parallel/mpi/request.hpp>
+#include <boost/parallel/mpi/status.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+/***************************************************************************
+ * request                                                                 *
+ ***************************************************************************/
+request::request()
+  : m_handler(0), m_data()
+{
+  m_requests[0] = MPI_REQUEST_NULL;
+  m_requests[1] = MPI_REQUEST_NULL;
+}
+
+status request::wait()
+{
+  if (m_handler) {
+    // This request is a receive for a serialized type. Use the
+    // handler to wait for completion.
+    return *m_handler(this, ra_wait);
+  } else if (m_requests[1] == MPI_REQUEST_NULL) {
+    // This request is either a send or a receive for a type with an
+    // associated MPI datatype, or a serialized datatype that has been
+    // packed into a single message. Just wait on the one receive/send
+    // and return the status to the user.
+    status result;
+    BOOST_MPI_CHECK_RESULT(MPI_Wait, (&m_requests[0], &result.m_status));
+    return result;
+  } else {
+    // This request is a send of a serialized type, broken into two
+    // separate messages. Complete both sends at once.
+    MPI_Status stats[2];
+    int error_code = MPI_Waitall(2, m_requests, stats);
+    if (error_code == MPI_ERR_IN_STATUS) {
+      // Dig out which status structure has the error, and use that
+      // one when throwing the exception.
+      if (stats[0].MPI_ERROR == MPI_SUCCESS 
+          || stats[0].MPI_ERROR == MPI_ERR_PENDING)
+        boost::throw_exception(exception("MPI_Waitall", stats[1].MPI_ERROR));
+      else
+        boost::throw_exception(exception("MPI_Waitall", stats[0].MPI_ERROR));
+    } else if (error_code != MPI_SUCCESS) {
+      // There was an error somewhere in the MPI_Waitall call; throw
+      // an exception for it.
+      boost::throw_exception(exception("MPI_Waitall", error_code));
+    } 
+
+    // No errors. Returns the first status structure.
+    status result;
+    result.m_status = stats[0];
+    return result;
+  }
+}
+
+optional<status> request::test()
+{
+  if (m_handler) {
+    // This request is a receive for a serialized type. Use the
+    // handler to test for completion.
+    return m_handler(this, ra_test);
+  } else if (m_requests[1] == MPI_REQUEST_NULL) {
+    // This request is either a send or a receive for a type with an
+    // associated MPI datatype, or a serialized datatype that has been
+    // packed into a single message. Just test the one receive/send
+    // and return the status to the user if it has completed.
+    status result;
+    int flag = 0;
+    BOOST_MPI_CHECK_RESULT(MPI_Test, 
+                           (&m_requests[0], &flag, &result.m_status));
+    return flag != 0? optional<status>(result) : optional<status>();
+  } else {
+    // This request is a send of a serialized type, broken into two
+    // separate messages. We only get a result if both complete.
+    MPI_Status stats[2];
+    int flag = 0;
+    int error_code = MPI_Testall(2, m_requests, &flag, stats);
+    if (error_code == MPI_ERR_IN_STATUS) {
+      // Dig out which status structure has the error, and use that
+      // one when throwing the exception.
+      if (stats[0].MPI_ERROR == MPI_SUCCESS 
+          || stats[0].MPI_ERROR == MPI_ERR_PENDING)
+        boost::throw_exception(exception("MPI_Testall", stats[1].MPI_ERROR));
+      else
+        boost::throw_exception(exception("MPI_Testall", stats[0].MPI_ERROR));
+    } else if (error_code != MPI_SUCCESS) {
+      // There was an error somewhere in the MPI_Testall call; throw
+      // an exception for it.
+      boost::throw_exception(exception("MPI_Testall", error_code));
+    }
+
+    // No errors. Returns the second status structure if the send has
+    // completed.
+    if (flag != 0) {
+      status result;
+      result.m_status = stats[1];
+      return result;
+    } else {
+      return optional<status>();
+    }
+  }
+}
+
+void request::cancel()
+{
+  if (m_handler) {
+    m_handler(this, ra_cancel);
+  } else {
+    BOOST_MPI_CHECK_RESULT(MPI_Cancel, (&m_requests[0]));
+    if (m_requests[1] != MPI_REQUEST_NULL)
+      BOOST_MPI_CHECK_RESULT(MPI_Cancel, (&m_requests[1]));
+  }
+}
+
+} } } // end namespace boost::parallel::mpi

src/mpi/text_skeleton_iarchive.cpp

@@ -0,0 +1,24 @@
+// (C) Copyright 2005 Matthias Troyer 
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/archive/impl/archive_pointer_iserializer.ipp>
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+#include <boost/parallel/mpi/text_skeleton_iarchive.hpp>
+
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/archive/impl/basic_binary_iprimitive.ipp>
+#include <boost/archive/impl/basic_binary_iarchive.ipp>
+
+namespace boost { namespace archive {
+
+// explicitly instantiate all required templates
+
+template class detail::archive_pointer_iserializer<parallel::mpi::text_skeleton_iarchive> ;
+
+} } // end namespace boost::archive

src/mpi/text_skeleton_oarchive.cpp

@@ -0,0 +1,22 @@
+// (C) Copyright 2005 Matthias Troyer
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  Authors: Matthias Troyer
+
+#define BOOST_ARCHIVE_SOURCE
+#include <boost/parallel/mpi/detail/text_skeleton_oarchive.hpp>
+
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/impl/basic_binary_oprimitive.ipp>
+#include <boost/archive/impl/basic_binary_oarchive.ipp>
+#include <boost/archive/impl/archive_pointer_oserializer.ipp>
+
+namespace boost { namespace archive {
+// explicitly instantiate all required templates
+
+template class detail::archive_pointer_oserializer<parallel::mpi::text_skeleton_oarchive> ;
+
+} } // end namespace boost::archive

src/mpi/timer.cpp

@@ -0,0 +1,25 @@
+// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+#include <boost/parallel/mpi/timer.hpp>
+#include <boost/parallel/mpi/exception.hpp>
+
+namespace boost { namespace parallel { namespace mpi {
+
+bool timer::time_is_global()
+{
+  int* is_global;
+  int found = 0;
+
+  BOOST_MPI_CHECK_RESULT(MPI_Attr_get,
+                         (MPI_COMM_WORLD, MPI_WTIME_IS_GLOBAL, &is_global,
+                          &found));
+  if (!found)
+    return false;
+  else
+    return *is_global != 0;
+}
+
+} } } /// end namespace boost::parallel::mpi

test/all_gather_test.cpp

@@ -0,0 +1,108 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the all_gather() collective.
+#include <boost/parallel/mpi/collectives/all_gather.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/test/minimal.hpp>
+#include <algorithm>
+#include "gps_position.hpp"
+#include <boost/serialization/string.hpp>
+#include <boost/serialization/list.hpp>
+#include <boost/iterator/counting_iterator.hpp>
+#include <boost/lexical_cast.hpp>
+
+using boost::parallel::mpi::communicator;
+
+using boost::parallel::mpi::packed_skeleton_iarchive;
+using boost::parallel::mpi::packed_skeleton_oarchive;
+
+template<typename Generator>
+void
+all_gather_test(const communicator& comm, Generator generator,
+                const char* kind)
+{
+  typedef typename Generator::result_type value_type;
+  value_type value = generator(comm.rank());
+
+  using boost::parallel::mpi::all_gather;
+
+  std::vector<value_type> values;
+  if (comm.rank() == 0) {
+    std::cout << "Gathering " << kind << "...";
+    std::cout.flush();
+  }
+
+  all_gather(comm, value, values);
+
+  std::vector<value_type> expected_values;
+  for (int p = 0; p < comm.size(); ++p)
+    expected_values.push_back(generator(p));
+  BOOST_CHECK(values == expected_values);
+  if (comm.rank() == 0 && values == expected_values)
+    std::cout << "OK." << std::endl;
+
+  (comm.barrier)();
+}
+
+// Generates integers to test with gather()
+struct int_generator
+{
+  typedef int result_type;
+
+  int operator()(int p) const { return 17 + p; }
+};
+
+// Generates GPS positions to test with gather()
+struct gps_generator
+{
+  typedef gps_position result_type;
+
+  gps_position operator()(int p) const
+  {
+    return gps_position(39 + p, 16, 20.2799);
+  }
+};
+
+struct string_generator
+{
+  typedef std::string result_type;
+
+  std::string operator()(int p) const
+  {
+    std::string result = boost::lexical_cast<std::string>(p);
+    result += " rosebud";
+    if (p != 1) result += 's';
+    return result;
+  }
+};
+
+struct string_list_generator
+{
+  typedef std::list<std::string> result_type;
+
+  std::list<std::string> operator()(int p) const
+  {
+    std::list<std::string> result;
+    for (int i = 0; i <= p; ++i) {
+      std::string value = boost::lexical_cast<std::string>(i);
+      result.push_back(value);
+    }
+    return result;
+  }
+};
+
+int test_main(int argc, char* argv[])
+{
+  boost::parallel::mpi::environment env(argc, argv);
+  communicator comm;
+  all_gather_test(comm, int_generator(), "integers");
+  all_gather_test(comm, gps_generator(), "GPS positions");
+  all_gather_test(comm, string_generator(), "string");
+  all_gather_test(comm, string_list_generator(), "list of strings");
+  return 0;
+}

test/all_reduce_test.cpp

@@ -0,0 +1,228 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the all_reduce() collective.
+#include <boost/parallel/mpi/collectives/all_reduce.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/test/minimal.hpp>
+#include <algorithm>
+#include <boost/serialization/string.hpp>
+#include <boost/iterator/counting_iterator.hpp>
+#include <boost/lexical_cast.hpp>
+#include <numeric>
+
+using boost::parallel::mpi::communicator;
+
+// A simple point class that we can build, add, compare, and
+// serialize.
+struct point
+{
+  point() : x(0), y(0), z(0) { }
+  point(int x, int y, int z) : x(x), y(y), z(z) { }
+
+  int x;
+  int y;
+  int z;
+
+ private:
+  template<typename Archiver>
+  void serialize(Archiver& ar, unsigned int /*version*/)
+  {
+    ar & x & y & z;
+  }
+
+  friend class boost::serialization::access;
+};
+
+std::ostream& operator<<(std::ostream& out, const point& p)
+{
+  return out << p.x << ' ' << p.y << ' ' << p.z;
+}
+
+bool operator==(const point& p1, const point& p2)
+{
+  return p1.x == p2.x && p1.y == p2.y && p1.z == p2.z;
+}
+
+bool operator!=(const point& p1, const point& p2)
+{
+  return !(p1 == p2);
+}
+
+point operator+(const point& p1, const point& p2)
+{
+  return point(p1.x + p2.x, p1.y + p2.y, p1.z + p2.z);
+}
+
+namespace boost { namespace parallel { namespace mpi {
+
+  template <>
+  struct is_mpi_datatype<point> : public mpl::true_ { };
+
+} } } // end namespace boost::parallel::mpi
+
+template<typename Generator, typename Op>
+void
+all_reduce_test(const communicator& comm, Generator generator,
+                const char* type_kind, Op op, const char* op_kind,
+                typename Generator::result_type init)
+{
+  typedef typename Generator::result_type value_type;
+  value_type value = generator(comm.rank());
+
+  using boost::parallel::mpi::all_reduce;
+
+  if (comm.rank() == 0) {
+    std::cout << "Reducing to " << op_kind << " of " << type_kind << "...";
+    std::cout.flush();
+  }
+
+  value_type result_value = all_reduce(comm, value, op);
+
+  // Compute expected result
+  std::vector<value_type> generated_values;
+  for (int p = 0; p < comm.size(); ++p)
+    generated_values.push_back(generator(p));
+  value_type expected_result = std::accumulate(generated_values.begin(),
+                                               generated_values.end(),
+                                               init, op);
+  BOOST_CHECK(result_value == expected_result);
+  if (result_value == expected_result && comm.rank() == 0)
+    std::cout << "OK." << std::endl;
+
+  (comm.barrier)();
+}
+
+// Generates integers to test with all_reduce()
+struct int_generator
+{
+  typedef int result_type;
+
+  int_generator(int base = 1) : base(base) { }
+
+  int operator()(int p) const { return base + p; }
+
+ private:
+  int base;
+};
+
+// Generate points to test with all_reduce()
+struct point_generator
+{
+  typedef point result_type;
+
+  point_generator(point origin) : origin(origin) { }
+
+  point operator()(int p) const
+  {
+    return point(origin.x + 1, origin.y + 1, origin.z + 1);
+  }
+
+ private:
+  point origin;
+};
+
+struct string_generator
+{
+  typedef std::string result_type;
+
+  std::string operator()(int p) const
+  {
+    std::string result = boost::lexical_cast<std::string>(p);
+    result += " rosebud";
+    if (p != 1) result += 's';
+    return result;
+  }
+};
+
+struct secret_int_bit_and
+{
+  int operator()(int x, int y) const { return x & y; }
+};
+
+struct wrapped_int
+{
+  wrapped_int() : value(0) { }
+  explicit wrapped_int(int value) : value(value) { }
+
+  template<typename Archive>
+  void serialize(Archive& ar, unsigned int /* version */)
+  {
+    ar & value;
+  }
+
+  int value;
+};
+
+wrapped_int operator+(const wrapped_int& x, const wrapped_int& y)
+{
+  return wrapped_int(x.value + y.value);
+}
+
+bool operator==(const wrapped_int& x, const wrapped_int& y)
+{
+  return x.value == y.value;
+}
+
+// Generates wrapped_its to test with all_reduce()
+struct wrapped_int_generator
+{
+  typedef wrapped_int result_type;
+
+  wrapped_int_generator(int base = 1) : base(base) { }
+
+  wrapped_int operator()(int p) const { return wrapped_int(base + p); }
+
+ private:
+  int base;
+};
+
+namespace boost { namespace parallel { namespace mpi {
+
+// Make std::plus<wrapped_int> commutative.
+template<>
+struct is_commutative<std::plus<wrapped_int>, wrapped_int>
+  : mpl::true_ { };
+
+} } } // end namespace boost::parallel::mpi
+
+int test_main(int argc, char* argv[])
+{
+  using namespace boost::parallel::mpi;
+  environment env(argc, argv);
+
+  communicator comm;
+
+  // Built-in MPI datatypes with built-in MPI operations
+  all_reduce_test(comm, int_generator(), "integers", std::plus<int>(), "sum",
+                  0);
+  all_reduce_test(comm, int_generator(), "integers", std::multiplies<int>(),
+                  "product", 1);
+  all_reduce_test(comm, int_generator(), "integers", maximum<int>(),
+                  "maximum", 0);
+  all_reduce_test(comm, int_generator(), "integers", minimum<int>(),
+                  "minimum", 2);
+
+  // User-defined MPI datatypes with operations that have the
+  // same name as built-in operations.
+  all_reduce_test(comm, point_generator(point(0,0,0)), "points",
+                  std::plus<point>(), "sum", point());
+
+  // Built-in MPI datatypes with user-defined operations
+  all_reduce_test(comm, int_generator(17), "integers", secret_int_bit_and(),
+                  "bitwise and", -1);
+
+    // Arbitrary types with user-defined, commutative operations.
+  all_reduce_test(comm, wrapped_int_generator(17), "wrapped integers",
+                  std::plus<wrapped_int>(), "sum", wrapped_int(0));
+
+  // Arbitrary types with (non-commutative) user-defined operations
+  all_reduce_test(comm, string_generator(), "strings",
+                  std::plus<std::string>(), "concatenation", std::string());
+
+  return 0;
+}

test/all_to_all_test.cpp

@@ -0,0 +1,105 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the all_to_all() collective.
+#include <boost/parallel/mpi/collectives/all_to_all.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/test/minimal.hpp>
+#include <algorithm>
+#include "gps_position.hpp"
+#include <boost/serialization/string.hpp>
+#include <boost/serialization/list.hpp>
+#include <boost/iterator/counting_iterator.hpp>
+#include <boost/lexical_cast.hpp>
+
+using boost::parallel::mpi::communicator;
+
+using boost::parallel::mpi::packed_skeleton_iarchive;
+using boost::parallel::mpi::packed_skeleton_oarchive;
+
+template<typename Generator>
+void
+all_to_all_test(const communicator& comm, Generator generator,
+                const char* kind)
+{
+  typedef typename Generator::result_type value_type;
+
+  using boost::parallel::mpi::all_to_all;
+
+  std::vector<value_type> in_values;
+  for (int p = 0; p < comm.size(); ++p)
+    in_values.push_back(generator((p + 1) * (comm.rank() + 1)));
+
+  std::vector<value_type> out_values;
+  all_to_all(comm, in_values, out_values);
+
+  for (int p = 0; p < comm.size(); ++p) {
+    BOOST_CHECK(out_values[p] == generator((p + 1) * (comm.rank() + 1)));
+  }
+
+  (comm.barrier)();
+}
+
+// Generates integers to test with all_to_all()
+struct int_generator
+{
+  typedef int result_type;
+
+  int operator()(int p) const { return 17 + p; }
+};
+
+// Generates GPS positions to test with all_to_all()
+struct gps_generator
+{
+  typedef gps_position result_type;
+
+  gps_position operator()(int p) const
+  {
+    return gps_position(39 + p, 16, 20.2799);
+  }
+};
+
+struct string_generator
+{
+  typedef std::string result_type;
+
+  std::string operator()(int p) const
+  {
+    std::string result = boost::lexical_cast<std::string>(p);
+    result += " rosebud";
+    if (p != 1) result += 's';
+    return result;
+  }
+};
+
+struct string_list_generator
+{
+  typedef std::list<std::string> result_type;
+
+  std::list<std::string> operator()(int p) const
+  {
+    std::list<std::string> result;
+    for (int i = 0; i <= p; ++i) {
+      std::string value = boost::lexical_cast<std::string>(i);
+      result.push_back(value);
+    }
+    return result;
+  }
+};
+
+int test_main(int argc, char* argv[])
+{
+  boost::parallel::mpi::environment env(argc, argv);
+
+  communicator comm;
+  all_to_all_test(comm, int_generator(), "integers");
+  all_to_all_test(comm, gps_generator(), "GPS positions");
+  all_to_all_test(comm, string_generator(), "string");
+  all_to_all_test(comm, string_list_generator(), "list of strings");
+
+  return 0;
+}

test/broadcast_test.cpp

@@ -0,0 +1,155 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the broadcast() collective.
+#include <boost/parallel/mpi/collectives/broadcast.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/test/minimal.hpp>
+#include <algorithm>
+#include "gps_position.hpp"
+#include <boost/serialization/string.hpp>
+#include <boost/serialization/list.hpp>
+#include <boost/parallel/mpi/skeleton_and_content.hpp>
+#include <boost/iterator/counting_iterator.hpp>
+
+using boost::parallel::mpi::communicator;
+
+using boost::parallel::mpi::packed_skeleton_iarchive;
+using boost::parallel::mpi::packed_skeleton_oarchive;
+
+template<typename T>
+void
+broadcast_test(const communicator& comm, const T& bc_value,
+               const char* kind, int root = -1)
+{
+  if (root == -1) {
+    for (root = 0; root < comm.size(); ++root)
+      broadcast_test(comm, bc_value, kind, root);
+  } else {
+    using boost::parallel::mpi::broadcast;
+
+    T value;
+    if (comm.rank() == root) {
+      value = bc_value;
+      std::cout << "Broadcasting " << kind << " from root " << root << "...";
+      std::cout.flush();
+    }
+
+    broadcast(comm, value, root);
+    BOOST_CHECK(value == bc_value);
+    if (comm.rank() == root && value == bc_value)
+      std::cout << "OK." << std::endl;
+  }
+
+  (comm.barrier)();
+}
+
+void
+test_skeleton_and_content(const communicator& comm, int root = 0)
+{
+  using boost::parallel::mpi::content;
+  using boost::parallel::mpi::get_content;
+  using boost::make_counting_iterator;
+  using boost::parallel::mpi::broadcast;
+  using boost::parallel::mpi::content;
+  using boost::parallel::mpi::get_content;
+
+  typedef std::list<int>::iterator iterator;
+
+  int list_size = comm.size() + 7;
+  if (comm.rank() == root) {
+    // Fill in the seed data
+    std::list<int> original_list;
+    for (int i = 0; i < list_size; ++i)
+      original_list.push_back(i);
+
+    // Build up the skeleton
+    packed_skeleton_oarchive oa(comm);
+    oa << original_list;
+
+    // Broadcast the skeleton
+    std::cout << "Broadcasting integer list skeleton from root " << root
+              << "...";
+    broadcast(comm, oa, root);
+    std::cout << "OK." << std::endl;
+
+    // Broadcast the content
+    std::cout << "Broadcasting integer list content from root " << root
+              << "...";
+    {
+      content c = get_content(original_list);
+      broadcast(comm, c, root);
+    }
+    std::cout << "OK." << std::endl;
+
+    // Reverse the list, broadcast the content again
+    std::reverse(original_list.begin(), original_list.end());
+    std::cout << "Broadcasting reversed integer list content from root "
+              << root << "...";
+    {
+      content c = get_content(original_list);
+      broadcast(comm, c, root);
+    }
+    std::cout << "OK." << std::endl;
+  } else {
+    // Allocate some useless data, to try to get the addresses of the
+    // list<int>'s used later to be different across processes.
+    std::list<int> junk_list(comm.rank() * 3 + 1, 17);
+
+    // Receive the skeleton
+    packed_skeleton_iarchive ia(comm);
+    broadcast(comm, ia, root);
+
+    // Build up a list to match the skeleton, and make sure it has the
+    // right structure (we have no idea what the data will be).
+    std::list<int> transferred_list;
+    ia >> transferred_list;
+    BOOST_CHECK((int)transferred_list.size() == list_size);
+
+    // Receive the content and check it
+    broadcast(comm, get_content(transferred_list), root);
+    BOOST_CHECK(std::equal(make_counting_iterator(0),
+                           make_counting_iterator(list_size),
+                           transferred_list.begin()));
+
+    // Receive the reversed content and check it
+    broadcast(comm, get_content(transferred_list), root);
+    BOOST_CHECK(std::equal(make_counting_iterator(0),
+                           make_counting_iterator(list_size),
+                           transferred_list.rbegin()));
+  }
+
+  (comm.barrier)();
+}
+
+int test_main(int argc, char* argv[])
+{
+  boost::parallel::mpi::environment env(argc, argv);
+
+  communicator comm;
+  if (comm.size() == 1) {
+    std::cerr << "ERROR: Must run the broadcast test with more than one "
+              << "process." << std::endl;
+    MPI_Abort(comm, -1);
+  }
+
+  // Check transfer of individual objects
+  broadcast_test(comm, 17, "integers");
+  broadcast_test(comm, gps_position(39,16,20.2799), "GPS positions");
+  broadcast_test(comm, gps_position(26,25,30.0), "GPS positions");
+  broadcast_test(comm, std::string("Rosie"), "string");
+
+  std::list<std::string> strings;
+  strings.push_back("Hello");
+  strings.push_back("MPI");
+  strings.push_back("World");
+  broadcast_test(comm, strings, "list of strings");
+
+  test_skeleton_and_content(comm, 0);
+  test_skeleton_and_content(comm, 1);
+  return 0;
+}

test/gather_test.cpp

@@ -0,0 +1,119 @@
+// Copyright (C) 2005, 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the gather() collective.
+#include <boost/parallel/mpi/collectives/gather.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/test/minimal.hpp>
+#include <algorithm>
+#include "gps_position.hpp"
+#include <boost/serialization/string.hpp>
+#include <boost/serialization/list.hpp>
+#include <boost/iterator/counting_iterator.hpp>
+#include <boost/lexical_cast.hpp>
+
+using boost::parallel::mpi::communicator;
+
+using boost::parallel::mpi::packed_skeleton_iarchive;
+using boost::parallel::mpi::packed_skeleton_oarchive;
+
+template<typename Generator>
+void
+gather_test(const communicator& comm, Generator generator,
+            const char* kind, int root = -1)
+{
+  typedef typename Generator::result_type value_type;
+  value_type value = generator(comm.rank());
+
+  if (root == -1) {
+    for (root = 0; root < comm.size(); ++root)
+      gather_test(comm, generator, kind, root);
+  } else {
+    using boost::parallel::mpi::gather;
+
+    std::vector<value_type> values;
+    if (comm.rank() == root) {
+      std::cout << "Gathering " << kind << " from root " << root << "...";
+      std::cout.flush();
+    }
+
+    gather(comm, value, values, root);
+
+    if (comm.rank() == root) {
+      std::vector<value_type> expected_values;
+      for (int p = 0; p < comm.size(); ++p)
+        expected_values.push_back(generator(p));
+      BOOST_CHECK(values == expected_values);
+      if (values == expected_values)
+        std::cout << "OK." << std::endl;
+    } else {
+      BOOST_CHECK(values.empty());
+    }
+  }
+
+  (comm.barrier)();
+}
+
+// Generates integers to test with gather()
+struct int_generator
+{
+  typedef int result_type;
+
+  int operator()(int p) const { return 17 + p; }
+};
+
+// Generates GPS positions to test with gather()
+struct gps_generator
+{
+  typedef gps_position result_type;
+
+  gps_position operator()(int p) const
+  {
+    return gps_position(39 + p, 16, 20.2799);
+  }
+};
+
+struct string_generator
+{
+  typedef std::string result_type;
+
+  std::string operator()(int p) const
+  {
+    std::string result = boost::lexical_cast<std::string>(p);
+    result += " rosebud";
+    if (p != 1) result += 's';
+    return result;
+  }
+};
+
+struct string_list_generator
+{
+  typedef std::list<std::string> result_type;
+
+  std::list<std::string> operator()(int p) const
+  {
+    std::list<std::string> result;
+    for (int i = 0; i <= p; ++i) {
+      std::string value = boost::lexical_cast<std::string>(i);
+      result.push_back(value);
+    }
+    return result;
+  }
+};
+
+int test_main(int argc, char* argv[])
+{
+  boost::parallel::mpi::environment env(argc, argv);
+
+  communicator comm;
+  gather_test(comm, int_generator(), "integers");
+  gather_test(comm, gps_generator(), "GPS positions");
+  gather_test(comm, string_generator(), "string");
+  gather_test(comm, string_list_generator(), "list of strings");
+
+  return 0;
+}

test/is_mpi_op_test.cpp

@@ -0,0 +1,34 @@
+// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor@gmail.com>
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the is_mpi_op functionality.
+#include <boost/parallel/mpi/operations.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/type_traits/is_base_and_derived.hpp>
+#include <boost/test/minimal.hpp>
+
+using namespace boost::parallel::mpi;
+using namespace std;
+using boost::is_base_and_derived;
+
+int test_main(int argc, char* argv[])
+{
+  boost::parallel::mpi::environment env(argc, argv);
+
+  // Check each predefined MPI_Op type that we support directly.
+  BOOST_CHECK((is_mpi_op<maximum<int>, int>::op() == MPI_MAX));
+  BOOST_CHECK((is_mpi_op<minimum<float>, float>::op() == MPI_MIN));
+  BOOST_CHECK((is_mpi_op<plus<double>, double>::op() == MPI_SUM));
+  BOOST_CHECK((is_mpi_op<multiplies<long>, long>::op() == MPI_PROD));
+  BOOST_CHECK((is_mpi_op<logical_and<int>, int>::op() == MPI_LAND));
+  BOOST_CHECK((is_mpi_op<bitwise_and<int>, int>::op() == MPI_BAND));
+  BOOST_CHECK((is_mpi_op<logical_or<int>, int>::op() == MPI_LOR));
+  BOOST_CHECK((is_mpi_op<bitwise_or<int>, int>::op() == MPI_BOR));
+  BOOST_CHECK((is_mpi_op<logical_xor<int>, int>::op() == MPI_LXOR));
+  BOOST_CHECK((is_mpi_op<bitwise_xor<int>, int>::op() == MPI_BXOR));
+
+  return 0;
+}

test/nonblocking_test.cpp

@@ -0,0 +1,247 @@
+// Copyright (C) 2006 Douglas Gregor.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// A test of the nonblocking point-to-point operations.
+#include <boost/parallel/mpi/nonblocking.hpp>
+#include <boost/parallel/mpi/communicator.hpp>
+#include <boost/parallel/mpi/environment.hpp>
+#include <boost/test/minimal.hpp>
+#include "gps_position.hpp"
+#include <boost/lexical_cast.hpp>
+#include <boost/serialization/string.hpp>
+#include <boost/serialization/list.hpp>
+#include <iterator>
+#include <algorithm>
+
+using boost::parallel::mpi::communicator;
+using boost::parallel::mpi::request;
+using boost::parallel::mpi::status;
+
+enum method_kind { 
+  mk_wait_any, mk_test_any, mk_wait_all, mk_wait_all_keep, 
+  mk_test_all, mk_test_all_keep, mk_wait_some, mk_wait_some_keep,
+  mk_test_some, mk_test_some_keep,
+  mk_all,                // use to run all of the different methods
+  mk_all_except_test_all // use for serialized types
+};
+
+static char* method_kind_names[mk_all] = {
+  "wait_any",
+  "test_any",
+  "wait_all",
+  "wait_all (keep results)",
+  "test_all",
+  "test_all (keep results)",
+  "wait_some",
+  "wait_some (keep results)",
+  "test_some",
+  "test_some (keep results)"
+};
+
+
+template<typename T>
+void
+nonblocking_test(const communicator& comm, const T* values, int num_values, 
+                 const char* kind, method_kind method = mk_all)
+{
+  using boost::parallel::mpi::wait_any;
+  using boost::parallel::mpi::test_any;
+  using boost::parallel::mpi::wait_all;
+  using boost::parallel::mpi::test_all;
+  using boost::parallel::mpi::wait_some;
+  using boost::parallel::mpi::test_some;
+
+  if (method == mk_all || method == mk_all_except_test_all) {
+    nonblocking_test(comm, values, num_values, kind, mk_wait_any);
+    nonblocking_test(comm, values, num_values, kind, mk_test_any);
+    nonblocking_test(comm, values, num_values, kind, mk_wait_all);
+    nonblocking_test(comm, values, num_values, kind, mk_wait_all_keep);
+    if (method == mk_all) {
+      nonblocking_test(comm, values, num_values, kind, mk_test_all);
+      nonblocking_test(comm, values, num_values, kind, mk_test_all_keep);
+    }
+    nonblocking_test(comm, values, num_values, kind, mk_wait_some);
+    nonblocking_test(comm, values, num_values, kind, mk_wait_some_keep);
+    nonblocking_test(comm, values, num_values, kind, mk_test_some);
+    nonblocking_test(comm, values, num_values, kind, mk_test_some_keep);
+  } else {
+    if (comm.rank() == 0) {
+      std::cout << "Testing " << method_kind_names[method] 
+                << " with " << kind << "...";
+      std::cout.flush();
+    }
+
+    typedef std::pair<status, std::vector<request>::iterator> 
+      status_iterator_pair;
+
+    T incoming_value;
+    std::vector<T> incoming_values(num_values);
+
+    std::vector<request> reqs;
+    // Send/receive the first value
+    reqs.push_back(comm.isend((comm.rank() + 1) % comm.size(), 0, values[0]));
+    reqs.push_back(comm.irecv((comm.rank() + comm.size() - 1) % comm.size(),
+                              0, incoming_value));
+
+    if (method != mk_wait_any && method != mk_test_any) {
+#ifndef LAM_MPI
+      // We've run into problems here (with 0-length messages) with
+      // LAM/MPI on Mac OS X and x86-86 Linux. Will investigate
+      // further at a later time, but the problem only seems to occur
+      // when using shared memory, not TCP.
+
+      // Send/receive an empty message
+      reqs.push_back(comm.isend((comm.rank() + 1) % comm.size(), 1));
+      reqs.push_back(comm.irecv((comm.rank() + comm.size() - 1) % comm.size(),
+                                1));
+#endif
+
+      // Send/receive an array
+      reqs.push_back(comm.isend((comm.rank() + 1) % comm.size(), 2, values,
+                                num_values));
+      reqs.push_back(comm.irecv((comm.rank() + comm.size() - 1) % comm.size(),
+                                2, &incoming_values.front(), num_values));
+    }
+
+    switch (method) {
+    case mk_wait_any:
+      if (wait_any(reqs.begin(), reqs.end()).second == reqs.begin())
+        reqs[1].wait();
+      else
+        reqs[0].wait();
+      break;
+
+    case mk_test_any:
+      {
+        boost::optional<status_iterator_pair> result;
+        do {
+          result = test_any(reqs.begin(), reqs.end());
+        } while (!result);
+        if (result->second == reqs.begin())
+          reqs[1].wait();
+        else
+          reqs[0].wait();
+        break;
+      }
+
+    case mk_wait_all:
+      wait_all(reqs.begin(), reqs.end());
+      break;
+
+    case mk_wait_all_keep:
+      {
+        std::vector<status> stats;
+        wait_all(reqs.begin(), reqs.end(), std::back_inserter(stats));
+      }
+      break;
+
+    case mk_test_all:
+      while (!test_all(reqs.begin(), reqs.end())) { /* Busy wait */ }
+      break;
+
+    case mk_test_all_keep:
+      {
+        std::vector<status> stats;
+        while (!test_all(reqs.begin(), reqs.end(), std::back_inserter(stats)))
+          /* Busy wait */;
+      }
+      break;
+
+    case mk_wait_some:
+      {
+        std::vector<request>::iterator pos = reqs.end();
+        do {
+          pos = wait_some(reqs.begin(), pos);
+        } while (pos != reqs.begin());
+      }
+      break;
+
+    case mk_wait_some_keep:
+      {
+        std::vector<status> stats;
+        std::vector<request>::iterator pos = reqs.end();
+        do {
+          pos = wait_some(reqs.begin(), pos, std::back_inserter(stats)).second;
+        } while (pos != reqs.begin());
+      }
+      break;
+
+    case mk_test_some:
+      {
+        std::vector<request>::iterator pos = reqs.end();
+        do {
+          pos = test_some(reqs.begin(), pos);
+        } while (pos != reqs.begin());
+      }
+      break;
+
+    case mk_test_some_keep:
+      {
+        std::vector<status> stats;
+        std::vector<request>::iterator pos = reqs.end();
+        do {
+          pos = test_some(reqs.begin(), pos, std::back_inserter(stats)).second;
+        } while (pos != reqs.begin());
+      }
+      break;
+
+    default:
+      BOOST_CHECK(false);
+    }
+
+    if (comm.rank() == 0) {
+      bool okay = true;
+
+      if (!((incoming_value == values[0])))
+        okay = false;
+
+      if (method != mk_wait_any && method != mk_test_any
+          && !std::equal(incoming_values.begin(), incoming_values.end(),
+                         values))
+        okay = false;
+
+      if (okay)
+        std::cout << "OK." << std::endl;
+      else
+        std::cerr << "ERROR!" << std::endl;
+    }
+
+    BOOST_CHECK(incoming_value == values[0]);
+
+    if (method != mk_wait_any && method != mk_test_any)
+      BOOST_CHECK(std::equal(incoming_values.begin(), incoming_values.end(),
+                             values));
+  }
+}
+
+int test_main(int argc, char* argv[])
+{
+  boost::parallel::mpi::environment env(argc, argv);
+
+  communicator comm;
+
+  int int_array[3] = {17, 42, 256};
+  nonblocking_test(comm, int_array, 3, "integers");
+
+  gps_position gps_array[2] = {
+    gps_position(17, 42, .06),
+    gps_position(42, 17, .06)
+  };
+  nonblocking_test(comm, gps_array, 2, "gps positions");
+
+  std::string string_array[2] = { "Hello", "World" };
+  nonblocking_test(comm, string_array, 2, "strings", 
+                   mk_all_except_test_all);
+
+  std::list<std::string> lst_of_strings;
+  for (int i = 0; i < comm.size(); ++i)
+    lst_of_strings.push_back(boost::lexical_cast<std::string>(i));
+
+  nonblocking_test(comm, &lst_of_strings, 1, "list of strings", 
+                   mk_all_except_test_all);
+
+  return 0;
+}