Merge branch 'issue-266' into 'master'

Issue 266

Closes #266

See merge request !46

src/Sample/CMakeLists.txt

@@ -3,6 +3,7 @@ set (_SRCS
     RNGStream.cpp
     SampleCmd.cpp
     Sampler.cpp
+    SamplingMethod.cpp
   )
 
 include_directories (

src/Sample/FromFile.h

@@ -27,12 +27,52 @@ public:
         : n_m(0)
         , counter_m(0)
         , mod_m(modulo)
+        , filename_m(filename)
+        , dvarName_m(dvarName)
     {
-        std::ifstream in(filename);
+        // we need to count the number of lines
+        std::ifstream in(filename_m);
 
         if ( !in.is_open() ) {
             throw OpalException("FromFile()",
-                                "Couldn't open file \"" + filename + "\".");
+                                "Couldn't open file \"" + filename_m + "\".");
+        }
+
+        int nLines = std::count(std::istreambuf_iterator<char>(in),
+                                std::istreambuf_iterator<char>(), '\n');
+
+        // make sure we do not count empty lines at end
+        in.seekg(-1, std::ios_base::end);
+        std::size_t pos =  in.tellg();
+
+        std::string line;
+        std::getline(in, line);
+
+        while ( line.empty() ) {
+            --nLines;
+            --pos;
+            in.seekg(pos, std::ios_base::beg);
+            std::getline(in, line);
+        }
+
+        if ( nLines < 0 )
+            throw OpalException("FromFile()", "Empty file \"" + filename_m + "\".");
+
+        globalSize_m = nLines;
+
+        in.close();
+    }
+
+    void create(boost::shared_ptr<SampleIndividual>& ind, size_t i) {
+        ind->genes[i] = getNext();
+    }
+
+    void allocate(const CmdArguments_t& args, const Comm::Bundle_t& comm) {
+        std::ifstream in(filename_m);
+
+        if ( !in.is_open() ) {
+            throw OpalException("FromFile()",
+                                "Couldn't open file \"" + filename_m + "\".");
         }
 
         std::string header;
@@ -42,18 +82,45 @@ public:
                                         std::istream_iterator<std::string>{}});
         size_t j = 0;
         for (const std::string str: dvars) {
-            if (str == dvarName) break;
+            if (str == dvarName_m) break;
             ++ j;
         }
 
         if (j == dvars.size()) {
             throw OpalException("FromFile()",
-                                "Couldn't find the dvar '" + dvarName + "' in the file '" + filename + "'");
+                                "Couldn't find the dvar '" + dvarName_m + "' in the file '" + filename_m + "'");
+        }
+
+        int nSamples = args->getArg<int>("nsamples", true);
+        int nMasters = args->getArg<int>("num-masters", true);
+
+        int nLocSamples = nSamples / nMasters;
+        int rest = nSamples - nMasters * nLocSamples;
+
+        int id = comm.island_id;
+        if ( id < rest )
+            nLocSamples++;
+
+        int skip = 0;
+
+        if ( rest == 0 )
+            skip = nLocSamples * id;
+        else {
+            if ( id < rest ) {
+                skip = nLocSamples * id;
+            } else {
+                skip = (nLocSamples + 1) * rest + (id - rest) * nLocSamples;
+            }
+        }
+
+        while ( skip > 0 ) {
+            skip--;
+            in.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
         }
 
         std::string line;
         std::getline(in, line);
-        while (in.good() && !in.eof()) {
+        while (nLocSamples-- > 0) {
             std::istringstream iss(line);
             std::vector<std::string> numbers({std::istream_iterator<std::string>{iss},
                                               std::istream_iterator<std::string>{}});
@@ -62,10 +129,7 @@ public:
 
             std::getline(in, line);
         }
-    }
-
-    void create(boost::shared_ptr<SampleIndividual>& ind, size_t i) {
-        ind->genes[i] = getNext();
+        in.close();
     }
 
     double getNext() {
@@ -76,7 +140,7 @@ public:
     }
 
     unsigned int getSize() const {
-        return chain_m.size();
+        return globalSize_m;
     }
 
 private:
@@ -84,6 +148,10 @@ private:
     unsigned int n_m;
     size_t counter_m;
     size_t mod_m;
+    std::string filename_m;
+    std::string dvarName_m;
+
+    unsigned int globalSize_m;
 
     void incrementCounter() {
         ++ counter_m;

src/Sample/LatinHyperCube.h

@@ -18,21 +18,22 @@ public:
         , upper_m(upper)
         , lower_m(lower)
         , dist_m(0.0, 1.0)
-    {
-        RNGInstance_m = RNGStream::getInstance();
-    }
+        , RNGInstance_m(RNGStream::getInstance())
+        , seed_m(RNGStream::getGlobalSeed())
+    {}
     
     LatinHyperCube(double lower, double upper, int seed)
         : binsize_m(0.0)
         , upper_m(upper)
         , lower_m(lower)
         , dist_m(0.0, 1.0)
-    {
-        RNGInstance_m = RNGStream::getInstance(seed);
-    }
+        , RNGInstance_m(nullptr)
+        , seed_m(seed)
+    {}
 
     ~LatinHyperCube() {
-        RNGStream::deleteInstance(RNGInstance_m);
+        if ( RNGInstance_m )
+            RNGStream::deleteInstance(RNGInstance_m);
     }
 
     void create(boost::shared_ptr<SampleIndividual>& ind, std::size_t i) {
@@ -42,11 +43,36 @@ public:
         ind->genes[i] = map2domain_m(RNGInstance_m->getNext(dist_m));
     }
     
-    void allocate(std::size_t n) {
+    void allocate(const CmdArguments_t& args, const Comm::Bundle_t& comm) {
+        int id = comm.island_id;
+        
+        if ( !RNGInstance_m )
+            RNGInstance_m = RNGStream::getInstance(seed_m + id);
+        
+        int nSamples = args->getArg<int>("nsamples", true);
+        int nMasters = args->getArg<int>("num-masters", true);
+        
+        int nLocSamples = nSamples / nMasters;
+        int rest = nSamples - nMasters * nLocSamples;
+        
+        if ( id < rest )
+            nLocSamples++;
+        
+        int startBin = 0;
+        
+        if ( rest == 0 )
+            startBin = nLocSamples * id;
+        else {
+            if ( id < rest ) {
+                startBin = nLocSamples * id;
+            } else {
+                startBin = (nLocSamples + 1) * rest + (id - rest) * nLocSamples;
+            }
+        }
         
-        binsize_m = ( upper_m - lower_m ) / double(n);
+        binsize_m = ( upper_m - lower_m ) / double(nSamples);
         
-        this->fillBins_m(n);
+        this->fillBins_m(nSamples, nLocSamples, startBin, seed_m);
     }
     
 private:
@@ -67,18 +93,24 @@ private:
         return  binsize_m * (val + bin) + lower_m;
     }
     
-    void fillBins_m(std::size_t n) {
-        bin_m.resize(n);
-        std::iota(bin_m.begin(), bin_m.end(), 0);
-        
-        std::random_device rd;
-        std::mt19937_64 eng(rd());
-        std::shuffle(bin_m.begin(), bin_m.end(), eng);
+    void fillBins_m(std::size_t nTotal, std::size_t nLocal, int startBin,
+                    std::size_t seed)
+    {
+        std::deque<std::size_t> tmp;
+        tmp.resize(nTotal);
+        std::iota(tmp.begin(), tmp.end(), 0);
+
+        // all masters need to shuffle the same way
+        std::mt19937_64 eng(seed);
+        std::shuffle(tmp.begin(), tmp.end(), eng);
+
+        // each master takes its bins
+        std::copy(tmp.begin()+startBin,
+                  tmp.begin()+startBin+nLocal,
+                  std::back_inserter(bin_m));
     }
 
 private:
-    RNGStream *RNGInstance_m;
-    
     std::deque<std::size_t> bin_m;
     double binsize_m;
     
@@ -86,6 +118,10 @@ private:
     double lower_m;
     
     dist_t dist_m;
+    
+    RNGStream *RNGInstance_m;
+    
+    std::size_t seed_m;
 };
 
 #endif

src/Sample/Normal.h

@@ -15,30 +15,36 @@ public:
 
     Normal(double lower, double upper)
         : dist_m(0.5 * (lower + upper), (upper - lower) / 10)
+        , RNGInstance_m(RNGStream::getInstance())
+        , seed_m(RNGStream::getGlobalSeed())
+    {}
 
-    {
-        RNGInstance_m = RNGStream::getInstance();
-    }
-
-    Normal(double lower, double upper, double seed)
+    Normal(double lower, double upper, std::size_t seed)
         : dist_m(0.5 * (lower + upper), (upper - lower) / 10)
-
-    {
-        RNGInstance_m = RNGStream::getInstance(seed);
-    }
+        , RNGInstance_m(nullptr)
+        , seed_m(seed)
+    {}
 
     ~Normal() {
-        RNGStream::deleteInstance(RNGInstance_m);
+        if ( RNGInstance_m)
+            RNGStream::deleteInstance(RNGInstance_m);
     }
 
     void create(boost::shared_ptr<SampleIndividual>& ind, size_t i) {
         ind->genes[i] = RNGInstance_m->getNext(dist_m);
     }
+    
+    void allocate(const CmdArguments_t& args, const Comm::Bundle_t& comm) {
+        if ( !RNGInstance_m )
+            RNGInstance_m = RNGStream::getInstance(seed_m + comm.island_id);
+    }
 
 private:
-    RNGStream *RNGInstance_m;
-
     dist_t dist_m;
+
+    RNGStream *RNGInstance_m;
+    
+    std::size_t seed_m;
 };
 
 #endif
\ No newline at end of file

src/Sample/RNGStream.cpp

@@ -40,6 +40,10 @@ void RNGStream::setGlobalSeed(unsigned int seed) {
         globalInstance_sm->RNGenerator_m.seed(seed);
 }
 
+unsigned int RNGStream::getGlobalSeed() {
+    return globalSeed_sm;
+}
+
 std::mt19937_64 & RNGStream::getGenerator() {
     return RNGenerator_m;
 }
\ No newline at end of file

src/Sample/RNGStream.h

@@ -12,12 +12,15 @@ public:
 
     static void setGlobalSeed(unsigned int seed);
 
+    static unsigned int getGlobalSeed();
+    
     std::mt19937_64 & getGenerator();
 
     template <class DISTR>
     typename DISTR::result_type getNext(DISTR & RNGDist) {
         return RNGDist(RNGenerator_m);
     }
+    
 private:
     RNGStream():
         RNGenerator_m(globalSeed_sm),

src/Sample/SampleCmd.cpp

@@ -438,7 +438,7 @@ void SampleCmd::execute() {
             for (sampleMethods_t::iterator it = sampleMethods.begin();
                  it != sampleMethods.end(); ++it)
             {
-                it->second->allocate(nSample);
+                it->second->allocate(args, comm->getBundle());
             }
         }
 

src/Sample/SampleGaussianSequence.h

@@ -22,10 +22,8 @@ class SampleGaussianSequence : public SamplingMethod
 public:
 
     SampleGaussianSequence(double lower, double upper, size_t modulo, int nSample)
-        : sampleNr_m(0)
-        , numSamples_m(nSample)
+        : numSamples_m(nSample)
         , volumeLowerDimensions_m(modulo)
-        , individualCounter_m(0)
     {
         double mean = 0.5 * (lower + upper);
         double sigma = (upper - lower) / 10; // +- 5 sigma
@@ -39,13 +37,13 @@ public:
     }
 
     void create(boost::shared_ptr<SampleIndividual>& ind, size_t i) {
-        ind->genes[i] = getNext();
+        ind->genes[i] = getNext(ind->id);
     }
 
-    double getNext() {
-        double sample = sampleChain_m[sampleNr_m];
-        incrementCounter();
-
+    double getNext(unsigned int id) {
+        int bin = int(id / volumeLowerDimensions_m) % numSamples_m;
+        
+        double sample = sampleChain_m[bin];
         return sample;
     }
 
@@ -54,18 +52,8 @@ private:
     FRIEND_TEST(GaussianSampleTest, ChainTest);
 #endif
     std::vector<double> sampleChain_m;
-    unsigned int sampleNr_m;
     unsigned int numSamples_m; // size of this "dimension"
     size_t volumeLowerDimensions_m; // the "volume" of the sampling space of the lower "dimensions"
-    size_t individualCounter_m; // counts how many "individuals" have been created
-
-    void incrementCounter() {
-        ++ individualCounter_m;
-        if (individualCounter_m % volumeLowerDimensions_m == 0)
-            ++ sampleNr_m;
-
-        sampleNr_m = sampleNr_m % numSamples_m;
-    }
 };
 
 #endif
\ No newline at end of file

src/Sample/SampleSequence.h

@@ -17,32 +17,24 @@ public:
     SampleSequence(T lower, T upper, size_t modulo, int nSample)
         : lowerLimit_m(lower)
         , stepSize_m( (upper - lower) / double(nSample - 1) )
-        , sampleNr_m(0)
         , numSamples_m(nSample)
         , volumeLowerDimensions_m(modulo)
-        , individualCounter_m(0)
     { }
 
     void create(boost::shared_ptr<SampleIndividual>& ind, size_t i) {
-        ind->genes[i] = static_cast<T>(lowerLimit_m + stepSize_m * sampleNr_m);
-        incrementCounter();
+        
+        unsigned int id = ind->id;
+        
+        int bin = int(id / volumeLowerDimensions_m) % numSamples_m;
+        
+        ind->genes[i] = static_cast<T>(lowerLimit_m + stepSize_m * bin);
     }
-
+    
 private:
-    void incrementCounter() {
-        ++ individualCounter_m;
-        if (individualCounter_m % volumeLowerDimensions_m == 0)
-            ++ sampleNr_m;
-
-        sampleNr_m = sampleNr_m % numSamples_m;
-    }
-
     T lowerLimit_m;
     double stepSize_m;
-    unsigned int sampleNr_m;
     unsigned int numSamples_m; // size of this "dimension"
     size_t volumeLowerDimensions_m; // the "volume" of the sampling space of the lower "dimensions"
-    size_t individualCounter_m; // counts how many "individuals" have been created
 };
 
 #endif
\ No newline at end of file

src/Sample/Sampler.cpp

@@ -75,9 +75,8 @@ void Sampler::initialize() {
     if ( nMasters > nSamples_m )
         throw OptPilotException("Sampler::initialize",
                                 "More masters than samples.");
-
     
-    // unique job id, FIXME does not work with more than 1 sampler
+    // unique job id
     int nLocSamples = nSamples_m / nMasters;
     int rest = nSamples_m - nMasters * nLocSamples;
     
@@ -174,14 +173,14 @@ void Sampler::createNewIndividual_m() {
 
     boost::shared_ptr<Individual_t> ind = boost::shared_ptr<Individual_t>( new Individual_t(dNames));
 
+    ind->id = gid++;
+    
     for(itr = dvars_m.begin(); itr != dvars_m.end(); itr++) {
         std::string dName = boost::get<VAR_NAME>(itr->second);
         int i = ind->getIndex(dName);
         sampleMethods_m[dName]->create(ind, i);
     }
 
-    // FIXME does not work with more than 1 master
-    ind->id = gid++;
 
     individuals_m.push(ind);
 }

src/Sample/SamplingMethod.cpp

+#include "SamplingMethod.h"
+
+int SamplingMethod::nSequenceSamplers = 0;

src/Sample/SamplingMethod.h

@@ -6,6 +6,9 @@
 
 #include <boost/smart_ptr.hpp>
 
+#include "Comm/types.h"
+#include "Util/CmdArguments.h"
+
 class SamplingMethod
 {
 
@@ -19,14 +22,18 @@ public:
      * This function is used to reduce memory since only the
      * sampler ranks need these sampling methods.
      * 
-     * @param n number of samples
+     * @param args samler arguments
+     * @param comm sampler communicator
      */
-    virtual void allocate(std::size_t n) {
+    virtual void allocate(const CmdArguments_t& args, const Comm::Bundle_t& comm) {
         /* Some sampling methods require a container.
          * In order to reduce memory only samplers should allocate
          * the memory
          */
     }
+    
+    // number of sequence sampling methods (important for multi-master case)
+    static int nSequenceSamplers;
 };
 
 #endif
\ No newline at end of file

src/Sample/Uniform.h

@@ -19,29 +19,36 @@ public:
 
     Uniform(T lower, T upper)
         : dist_m(lower, upper)
-    {
-        RNGInstance_m = RNGStream::getInstance();
-    }
+        , RNGInstance_m(RNGStream::getInstance())
+        , seed_m(RNGStream::getGlobalSeed())
+    {}
 
-    Uniform(T lower, T upper, int seed)
+    Uniform(T lower, T upper, std::size_t seed)
         : dist_m(lower, upper)
-
-    {
-        RNGInstance_m = RNGStream::getInstance(seed);
-    }
+        , RNGInstance_m(nullptr)
+        , seed_m(seed)
+    {}
 
     ~Uniform() {
-        RNGStream::deleteInstance(RNGInstance_m);
+        if ( RNGInstance_m )
+            RNGStream::deleteInstance(RNGInstance_m);
     }
 
     void create(boost::shared_ptr<SampleIndividual>& ind, size_t i) {
         ind->genes[i] = RNGInstance_m->getNext(dist_m);
     }
+    
+    void allocate(const CmdArguments_t& args, const Comm::Bundle_t& comm) {
+        if ( !RNGInstance_m )
+            RNGInstance_m = RNGStream::getInstance(seed_m + comm.island_id);
+    }
 
 private:
-    RNGStream *RNGInstance_m;
-
     dist_t dist_m;
+    
+    RNGStream *RNGInstance_m;
+    
+    std::size_t seed_m;
 };
 
 #endif
\ No newline at end of file