Commit 0fe87fd1 authored by kraus's avatar kraus
Browse files

adding objectives to the sampler. Implements #249

parent 9fff414e
......@@ -670,7 +670,6 @@ protected:
postPoll();
}
std::cout << std::boolalpha << pending_opt_request << ", " << pending_worker_request << ", " << pending_pilot_request << std::endl;
if(pending_opt_request) MPI_Cancel( &opt_request );
if(pending_worker_request) MPI_Cancel( &worker_request );
if(pending_pilot_request) MPI_Cancel( &pilot_request );
......
......@@ -105,4 +105,4 @@ protected:
}
};
#endif
#endif
\ No newline at end of file
......@@ -53,7 +53,8 @@ public:
Expressions::Named_t constraints,
std::string simName,
Comm::Bundle_t comms,
CmdArguments_t args)
CmdArguments_t args,
bool isOptimizer = true)
: Poller(comms.worker)
, cmd_args_(args)
{
......@@ -65,9 +66,11 @@ public:
coworker_comm_ = comms.coworkers;
leader_pid_ = 0;
MPI_Comm_size(coworker_comm_, &num_coworkers_);
if (!isOptimizer) return;
int my_local_pid = 0;
MPI_Comm_rank(coworker_comm_, &my_local_pid);
MPI_Comm_size(coworker_comm_, &num_coworkers_);
// distinction between leader and coworkers
if(my_local_pid == leader_pid_)
......@@ -175,7 +178,6 @@ protected:
{}
void onStop() {
if(num_coworkers_ > 1)
notifyCoWorkers(MPI_STOP_TAG);
}
......
......@@ -359,7 +359,7 @@ void OpalSimulation::collectResults() {
Expressions::Named_t::iterator namedIt;
try {
for(namedIt=objectives_.begin(); namedIt!=objectives_.end(); ++namedIt) {
if (namedIt->first == "dummy") continue;
Expressions::Expr_t *objective = namedIt->second;
// find out which variables we need in order to evaluate the
......
......@@ -4,11 +4,13 @@
#include "Sample/RNGStream.h"
#include "Optimize/DVar.h"
#include "Optimize/Objective.h"
#include "Optimize/OpalSimulation.h"
#include "Attributes/Attributes.h"
#include "AbstractObjects/OpalData.h"
#include "Utilities/OpalException.h"
#include "Utilities/Util.h"
#include "Utility/IpplInfo.h"
#include "Utility/IpplTimings.h"
......@@ -25,6 +27,14 @@
#include "Comm/MasterGraph/SocialNetworkGraph.h"
#include "Expression/Parser/function.hpp"
#include "Expression/FromFile.h"
#include "Expression/SumErrSq.h"
#include "Expression/SDDSVariable.h"
#include "Expression/RadialPeak.h"
#include "Expression/MaxNormRadialPeak.h"
#include "Expression/NumberOfPeaks.h"
#include "Expression/SumErrSqRadialPeak.h"
#include "Expression/ProbeVariable.h"
#include <boost/filesystem.hpp>
......@@ -40,6 +50,7 @@ namespace {
INPUT,
OUTPUT,
OUTDIR,
OBJECTIVES,
DVARS,
SAMPLINGS,
NUMMASTERS,
......@@ -62,6 +73,8 @@ SampleCmd::SampleCmd():
("OUTPUT", "Name used in output file sample");
itsAttr[OUTDIR] = Attributes::makeString
("OUTDIR", "Name of directory used to run and store sample output files");
itsAttr[OBJECTIVES] = Attributes::makeStringArray
("OBJECTIVES", "List of expressions to evaluate and store");
itsAttr[DVARS] = Attributes::makeStringArray
("DVARS", "List of sampling variables to be used");
itsAttr[SAMPLINGS] = Attributes::makeStringArray
......@@ -105,7 +118,9 @@ void SampleCmd::execute() {
unsigned int seed = Attributes::getReal(itsAttr[SEED]);
RNGStream::setGlobalSeed(seed);
std::vector<std::string> objectivesstr = Attributes::getStringArray(itsAttr[OBJECTIVES]);
std::vector<std::string> dvarsstr = Attributes::getStringArray(itsAttr[DVARS]);
Expressions::Named_t objectives;
DVarContainer_t dvars;
std::vector<std::string> sampling = Attributes::getStringArray(itsAttr[SAMPLINGS]);
......@@ -119,7 +134,8 @@ void SampleCmd::execute() {
std::map<std::string, std::pair<double, double> > vars;
for (std::string& name : dvarsstr) {
for (std::string &name : dvarsstr) {
name = Util::toUpper(name);
Object *obj = opal->find(name);
DVar* dvar = dynamic_cast<DVar*>(obj);
if (dvar == nullptr) {
......@@ -131,7 +147,7 @@ void SampleCmd::execute() {
double lowerbound = dvar->getLowerBound();
double upperbound = dvar->getUpperBound();
auto ret = vars.insert(std::make_pair(var,std::make_pair(lowerbound, upperbound)));
auto ret = vars.insert(std::make_pair(var, std::make_pair(lowerbound, upperbound)));
if (ret.second == false) {
throw OpalException("SampleCmd::execute",
"There is already a design variable with the variable " + var + " defined");
......@@ -141,6 +157,69 @@ void SampleCmd::execute() {
dvars.insert(namedDVar_t(name, tmp));
}
//////////////////////////////////////////////////////////////////////////
functionDictionary_t funcs;
client::function::type ff;
ff = FromFile();
funcs.insert(std::pair<std::string, client::function::type>
("fromFile", ff));
ff = SumErrSq();
funcs.insert(std::pair<std::string, client::function::type>
("sumErrSq", ff));
ff = SDDSVariable();
funcs.insert(std::pair<std::string, client::function::type>
("sddsVariableAt", ff));
ff = RadialPeak();
funcs.insert(std::pair<std::string, client::function::type>
("radialPeak", ff));
ff = MaxNormRadialPeak();
funcs.insert(std::pair<std::string, client::function::type>
("maxNormRadialPeak", ff));
ff = NumberOfPeaks();
funcs.insert(std::pair<std::string, client::function::type>
("numberOfPeaks", ff));
ff = SumErrSqRadialPeak();
funcs.insert(std::pair<std::string, client::function::type>
("sumErrSqRadialPeak", ff));
ff = ProbeVariable();
funcs.insert(std::pair<std::string, client::function::type>
("probVariableWithID", ff));
std::string fname = inputfile.stem().native();
ff = sameSDDSVariable(fname);
funcs.insert(std::pair<std::string, client::function::type>
("statVariableAt", ff));
//////////////////////////////////////////////////////////////////////////
std::set<std::string> objExpressions; // check if all unique objective expressions
for (std::string name: objectivesstr) {
name = Util::toUpper(name);
Object *obj = opal->find(name);
Objective* objective = dynamic_cast<Objective*>(obj);
if (objective == nullptr) {
throw OpalException("OptimizeCmd::execute",
"The objective " + name + " is not known");
}
std::string expr = objective->getExpression();
objectives.insert(Expressions::SingleNamed_t(
name, new Expressions::Expr_t(expr, funcs)));
auto ret = objExpressions.insert(expr);
if (ret.second == false) {
throw OpalException("OptimizeCmd::execute",
"There is already a objective with the expression " + expr + " defined");
}
}
bool raster = Attributes::getBool(itsAttr[RASTER]);
size_t modulo = 1;
unsigned int nSample = std::numeric_limits<unsigned int>::max();
......@@ -154,7 +233,7 @@ void SampleCmd::execute() {
"Sampling method not found.");
}
std::string name = s->getVariable();
std::string name = Util::toUpper(s->getVariable());
if ( vars.find(name) == vars.end() ) {
throw OpalException("SampleCmd::execute",
......@@ -278,7 +357,7 @@ void SampleCmd::execute() {
setenv("FIELDMAPS", dir.c_str(), 1);
}
if (Attributes::getString(itsAttr[DISTDIR]) != "") {
fs::path dir(Attributes::getString(itsAttr[DISTDIR]));
if (dir.is_relative()) {
......@@ -304,7 +383,7 @@ void SampleCmd::execute() {
CmdArguments_t args(new CmdArguments(argv.size(), &argv[0]));
boost::shared_ptr<Comm_t> comm(new Comm_t(args, MPI_COMM_WORLD));
boost::scoped_ptr<pilot_t> pi(new pilot_t(args, comm, dvars, sampleMethods));
boost::scoped_ptr<pilot_t> pi(new pilot_t(args, comm, funcs, dvars, objectives, sampleMethods));
} catch (OptPilotException &e) {
std::cout << "Exception caught: " << e.what() << std::endl;
......
......@@ -28,6 +28,8 @@ public:
typedef std::vector<double> genes_t;
/// gene names
typedef std::vector<std::string> names_t;
/// objectives array
typedef std::vector<double> objectives_t;
SampleIndividual()
{}
......@@ -42,11 +44,14 @@ public:
template<class Archive>
void serialize(Archive & ar, const unsigned int version) {
ar & genes;
ar & objectives;
ar & id;
}
/// genes of an individual
genes_t genes;
/// values of objectives of an individual
objectives_t objectives;
/// id
unsigned int id = 0;
......
......@@ -3,6 +3,7 @@
#include "Pilot/Pilot.h"
#include "Sample/SampleWorker.h"
#include "Expression/Parser/function.hpp"
......@@ -44,7 +45,9 @@ class SamplePilot : protected Pilot<Input_t,
public:
SamplePilot(CmdArguments_t args, boost::shared_ptr<Comm_t> comm,
functionDictionary_t known_expr_funcs,
const DVarContainer_t &dvar,
const Expressions::Named_t &obj,
const std::map< std::string,
std::shared_ptr<SamplingMethod>
>& sampleMethods)
......@@ -54,15 +57,22 @@ public:
SolPropagationGraph_t,
Comm_t>(args,
comm,
dvar)
known_expr_funcs,
dvar,
obj,
Expressions::Named_t(),
{},
false)
, sampleMethods_m(sampleMethods)
{
if (obj.size() == 0) {
// create a dummy objective, base class requires at least 1 objective
this->objectives_ = {
{"dummy", new Expressions::Expr_t("dummy")}
};
this->objectives_ = {
{"dummy", new Expressions::Expr_t("dummy")}
};
}
setup();
this->setup(known_expr_funcs);
}
~SamplePilot()
......@@ -79,10 +89,11 @@ protected:
virtual
void setup() {
void setup(functionDictionary_t known_expr_funcs) {
this->global_rank_ = this->comm_->globalRank();
this->parseInputFile(functionDictionary_t());
this->parseInputFile(known_expr_funcs, false);
MPI_Barrier(MPI_COMM_WORLD);
// here the control flow starts to diverge
......@@ -95,16 +106,16 @@ protected:
void startSampler() {
std::ostringstream os;
os << "\033[01;35m" << " " << this->global_rank_ << " ▶ Sampler"
os << "\033[01;35m" << " " << this->global_rank_ << " (PID: " << getpid() << ") ▶ Sampler"
<< "\e[0m" << std::endl;
std::cout << os.str() << std::flush;
boost::scoped_ptr<Opt_t> opt(
new Opt_t(sampleMethods_m, this->dvars_,
new Opt_t(sampleMethods_m, this->objectives_, this->dvars_,
this->comm_->getBundle(), this->cmd_args_));
opt->initialize();
std::cout << "Stop Opt.." << std::endl;
std::cout << "Stop Sampler.." << std::endl;
}
......@@ -112,7 +123,7 @@ protected:
void startWorker() /*override*/ {
std::ostringstream os;
os << "\033[01;35m" << " " << this->global_rank_ << " ▶ Worker"
os << "\033[01;35m" << " " << this->global_rank_ << " (PID: " << getpid() << ") ▶ Worker"
<< "\e[0m" << std::endl;
std::cout << os.str() << std::flush;
......@@ -124,7 +135,7 @@ protected:
std::string simName = tmplfile.substr(0,pos);
boost::scoped_ptr< SampleWorker<Sim_t> > w(
new SampleWorker<Sim_t>(this->constraints_, simName,
new SampleWorker<Sim_t>(this->objectives_, this->constraints_, simName,
this->comm_->getBundle(), this->cmd_args_));
std::cout << "Stop Worker.." << std::endl;
......@@ -144,10 +155,10 @@ protected:
if (i == this->my_rank_in_worker_comm_) continue;
all_worker_idle = all_worker_idle && this->is_worker_idle_[i];
if (this->is_worker_idle_[i] && request_queue_.size() > 0)
sendNewJobToWorker(i);
all_worker_idle = all_worker_idle && this->is_worker_idle_[i];
}
// when all workers have been notified we can stop polling
......@@ -203,6 +214,9 @@ protected:
MPI_Send(&dummy, 1, MPI_UNSIGNED_LONG, status.MPI_SOURCE,
MPI_WORKER_FINISHED_ACK_TAG, this->worker_comm_);
reqVarContainer_t res;
MPI_Recv_reqvars(res, status.MPI_SOURCE, this->worker_comm_);
running_job_list_.erase(job_id);
this->is_worker_idle_[status.MPI_SOURCE] = true;
......@@ -218,6 +232,8 @@ protected:
MPI_Send(&job_id, 1, MPI_UNSIGNED_LONG, opt_master_rank,
MPI_OPT_JOB_FINISHED_TAG, this->opt_comm_);
MPI_Send_reqvars(res, opt_master_rank, this->opt_comm_);
// we keep worker busy _after_ results have been sent to sampler
if (request_queue_.size() > 0)
sendNewJobToWorker(status.MPI_SOURCE);
......
......@@ -21,18 +21,17 @@ class SampleWorker : protected Worker<Sim_t> {
public:
SampleWorker(Expressions::Named_t constraints,
std::string simName, Comm::Bundle_t comms, CmdArguments_t args)
: Worker<Sim_t>(constraints, simName, comms, args)
SampleWorker(Expressions::Named_t objectives,
Expressions::Named_t constraints,
std::string simName,
Comm::Bundle_t comms,
CmdArguments_t args)
: Worker<Sim_t>(objectives, constraints, simName, comms, args, false)
{
// simulation pointer requires at least 1 objective --> provide dummy
this->objectives_ = {
{"dummy", new Expressions::Expr_t("dummy") }
};
int my_local_pid = 0;
MPI_Comm_rank(this->coworker_comm_, &my_local_pid);
// distinction between leader and coworkers
if(my_local_pid == this->leader_pid_)
this->run();
......@@ -44,7 +43,7 @@ public:
{}
protected:
/// notify coworkers of incoming broadcast
void notifyCoWorkers(size_t job_id, int tag) {
......@@ -55,7 +54,7 @@ protected:
MPI_Send(&job_id, 1, MPI_UNSIGNED_LONG, i, tag, this->coworker_comm_);
}
}
/// coworkers simply wait on a job broadcast from the leader and then
/// start a simulation..
void runSlave() {
......@@ -94,9 +93,9 @@ protected:
this->cmd_args_));
sim->setFilename(job_id);
sim->run();
} catch(OptPilotException &ex) {
std::cout << "Exception while running simulation: "
<< ex.what() << std::endl;
......@@ -113,9 +112,9 @@ protected:
}
}
}
bool onMessage(MPI_Status status, size_t recv_value) override {
if(status.MPI_TAG == MPI_WORK_JOBID_TAG) {
this->is_idle_ = false;
......@@ -131,6 +130,7 @@ protected:
MPI_Bcast_params(params, this->leader_pid_, this->coworker_comm_);
}
reqVarContainer_t requested_results;
try {
typename Worker<Sim_t>::SimPtr_t sim(new Sim_t(this->objectives_,
this->constraints_,
......@@ -138,12 +138,14 @@ protected:
this->simulation_name_,
this->coworker_comm_,
this->cmd_args_));
sim->setFilename(job_id);
// run simulation in a "blocking" fashion
sim->run();
sim->collectResults();
requested_results = sim->getResults();
} catch(OptPilotException &ex) {
std::cout << "Exception while running simulation: "
<< ex.what() << std::endl;
......@@ -155,7 +157,9 @@ protected:
size_t dummy = 0;
MPI_Recv(&dummy, 1, MPI_UNSIGNED_LONG, this->pilot_rank_,
MPI_WORKER_FINISHED_ACK_TAG, this->comm_m, &status);
MPI_Send_reqvars(requested_results, (size_t)this->pilot_rank_, this->comm_m);
this->is_idle_ = true;
return true;
......@@ -168,4 +172,4 @@ protected:
}
};
#endif
#endif
\ No newline at end of file
#include <iostream>
#include <string>
#include <limits>
#include "Sample/Sampler.h"
......@@ -26,12 +27,15 @@ Sampler::Sampler(Expressions::Named_t objectives,
Sampler::Sampler(const std::map<std::string,
std::shared_ptr<SamplingMethod>
>& sampleMethods,
DVarContainer_t dvars, Comm::Bundle_t comms,
Expressions::Named_t objectives,
DVarContainer_t dvars,
Comm::Bundle_t comms,
CmdArguments_t args)
: Optimizer(comms.opt)
, sampleMethods_m(sampleMethods)
, comms_(comms)
, dvars_m(dvars)
, objectives_m(objectives)
, args_(args)
{
my_local_pid_ = 0;
......@@ -93,6 +97,25 @@ bool Sampler::onMessage(MPI_Status status, size_t length) {
boost::shared_ptr<Individual_t> ind = it->second;
reqVarContainer_t res;
MPI_Recv_reqvars(res, status.MPI_SOURCE, comms_.opt);
ind->objectives.clear();
reqVarContainer_t::iterator itr = res.begin();
for(; itr != res.end(); ++itr) {
// mark invalid if expression could not be evaluated or constraint does not hold
if(!itr->second.is_valid || (itr->second.value.size() > 1 && !itr->second.value[0])) {
ind->objectives.push_back(std::numeric_limits<double>::infinity());
} else {
// update objective value for valid objective
if(itr->second.value.size() == 1)
ind->objectives.push_back(itr->second.value[0]);
}
}
addIndividualToJSON_m(ind);
jobmapping_m.erase(it);
done_sample_m++;
......@@ -139,8 +162,6 @@ void Sampler::createNewIndividual_m() {
// FIXME does not work with more than 1 master
ind->id = gid++;
addIndividualToJSON_m(ind);
individuals_m.push(ind);
}
......@@ -183,6 +204,14 @@ void Sampler::addIndividualToJSON_m(const boost::shared_ptr<Individual_t>& ind)
sample.put("dvar." + name, ind->genes[i]);
}
Expressions::Named_t::iterator expr_it;
expr_it = objectives_m.begin();
for(size_t i=0; i < ind->objectives.size(); i++, expr_it++) {
std::string name = expr_it->first;
sample.put("obj." + name, ind->objectives[i]);
}
samples_m.push_back(std::make_pair("", sample));
}
......
......@@ -68,7 +68,9 @@ public:
Sampler(const std::map< std::string,
std::shared_ptr<SamplingMethod>
>& sampleMethods,
DVarContainer_t dvars, Comm::Bundle_t comms,
Expressions::Named_t objectives,
DVarContainer_t dvars,
Comm::Bundle_t comms,
CmdArguments_t args);
/// Starting selection algorithm and variator PISA state machine
......@@ -118,6 +120,8 @@ private:
/// design variables
DVarContainer_t dvars_m;
/// objectives
Expressions::Named_t objectives_m;
int nSamples_m;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment