Merge pull request #259 from jtwhite79/feat_romou

experimenting with robust opt in mou, optimized timeouts in the serial run mgr

src/libs/common/config_os.h

@@ -2,7 +2,7 @@
 #define CONFIG_OS_H_
 
 
-#define PESTPP_VERSION "5.2.4";
+#define PESTPP_VERSION "5.2.5";
 
 #if defined(_WIN32) || defined(_WIN64)
 #define OS_WIN

src/libs/common/system_variables.cpp

@@ -46,8 +46,6 @@ const std::string OperSys::DIR_SEP = "/";
 const std::string OperSys::COMMAND_LINE_APPEND = " & ";
 #endif
 
-const int OperSys::thread_sleep_milli_secs=1000;
-
 using namespace std;
 
 void OperSys::string2pathname(string &s)

src/libs/common/system_variables.h

@@ -25,8 +25,6 @@
 class OperSys
 {
 public:
-	//const static int thread_sleep_secs = 1;
-	static const int thread_sleep_milli_secs;
 	static const std::string DIR_SEP;
 	static const std::string COMMAND_LINE_APPEND;
 	void string2pathname(std::string &s);

src/libs/pestpp_common/Ensemble.cpp

@@ -4299,14 +4299,86 @@ void FixedParInfo::add_realization(string rname, Eigen::VectorXd& rvals, vector<
 	map<string, double> v;
 	for (int i = 0; i < rvals.size(); i++)
 		v[pnames[i]] = rvals[i];
+	map<string,double>::iterator  end = v.end();
 	for (auto& name : fixed_names)
 	{
-		if (v.find(name) == v.end())
+		if (v.find(name) == end)
 			throw runtime_error("FixedParInfo::add_realization(): fixed name '" + name + "' not in pnames");
 		fixed_info.at(name)[rname] = v.at(name);
 	}
 }
 
+vector<string> FixedParInfo::get_real_names()
+{
+    if (!initialized)
+    {
+        return vector<string>();
+    }
+    if (fixed_names.size() == 0)
+    {
+        return vector<string>();
+    }
+    vector<string> rnames;
+    for (auto& fi : fixed_info)
+    {
+        for (auto& ri : fi.second)
+        {
+            rnames.push_back(ri.first);
+        }
+    }
+    return rnames;
+
+}
+
+void FixedParInfo::add_realizations(map<string,map<string,double>>& other_fixed_info)
+{
+    if (!initialized)
+    {
+        throw runtime_error("FixedParInfo::update_realizations: not initialized");
+    }
+    if (fixed_names.size() == 0)
+    {
+        //this needs to be error checked..
+        fixed_info = other_fixed_info;
+        return;
+    }
+    map<string,map<string,double>>::iterator end = fixed_info.end();
+    for (auto& ofi : other_fixed_info)
+    {
+        if (fixed_info.find(ofi.first) == end)
+        {
+            throw runtime_error("FixedParInfo::add_realizations() error: pname "+ofi.first+" not in fixed_info");
+        }
+        for (auto& ori : ofi.second)
+        {
+            //probably should error check this also to make sure other_fixed_info isnt replacing things...
+            fixed_info.at(ofi.first)[ori.first] = ori.second;
+        }
+    }
+}
+
+
+void FixedParInfo::add_realization(string rname, map<string, double>& rvals)
+{
+    if (!initialized)
+    {
+        throw runtime_error("FixedParInfo::add_realization(): not initialized");
+    }
+    if (fixed_names.size() == 0)
+    {
+        return;
+    }
+    map<string,double>::iterator  end = rvals.end();
+    for (auto& name : fixed_names)
+    {
+        if (rvals.find(name) == end)
+            throw runtime_error("FixedParInfo::add_realization(): fixed name '" + name + "' not in pnames");
+        fixed_info.at(name)[rname] = rvals.at(name);
+    }
+
+
+}
+
 void FixedParInfo::keep_realizations(const vector<string>& keep)
 {
 	if (!initialized)
@@ -4342,6 +4414,7 @@ void FixedParInfo::keep_realizations(const vector<string>& keep)
 	}
 }
 
+
 void FixedParInfo::update_realizations(const vector<string>& other_var_names, const vector<string>& other_real_names, const Eigen::MatrixXd& other_mat)
 {
 	if (!initialized)

src/libs/pestpp_common/Ensemble.h

@@ -144,12 +144,18 @@ class FixedParInfo
 	vector<double> get_real_fixed_values(const string& rname, vector<string>& pnames);
 	map<string, double> get_real_fixed_values(const string& rname);
 	void add_realization(string rname, Eigen::VectorXd& rvals, vector<string>& pnames);
+    void add_realization(string rname, map<string, double>& rvals);
 	void keep_realizations(const vector<string>& keep);
 	void update_realizations(const vector<string>& other_var_names, const vector<string>& other_real_names, const Eigen::MatrixXd& other_mat);
+	void add_realizations(map<string,map<string,double>>& other_fixed_info);
 	void update_par_values(const map<string, double>& pval_map);
 	void clear() { fixed_info.clear(); fixed_names.clear(); }
 	void fill_fixed(map<string, double>& fixed_map, vector<string>& rnames);
 	int get_map_size() {return fixed_info.size();}
+	vector<string> get_real_names();
+    vector<string> get_fixed_names() {return fixed_names;}
+    map<string,map<string,double>> get_fixed_info_map() {return fixed_info;}
+
 private:
 	bool initialized;
 	vector<string> fixed_names;

src/libs/pestpp_common/EnsembleMethodUtils.cpp

@@ -581,11 +581,12 @@ void EnsembleSolver::initialize_for_mm_solve()
     }
 
     mat = base_oe.get_eigen(oe_real_names, obs_names);
+    mat.transposeInPlace();
     Observations ctl_obs = pest_scenario.get_ctl_observations();
-    Eigen::VectorXd ovals = ctl_obs.get_data_eigen_vec(obs_names);
+    Eigen::ArrayXd ovals = ctl_obs.get_data_eigen_vec(obs_names).array();
     for (int i = 0; i < oe_real_names.size(); i++)
     {
-        obs_err_map[oe_real_names[i]] = mat.row(i).array() - ovals.array();
+        obs_err_map[oe_real_names[i]] = mat.col(i).array() - ovals;
     }
 
     mat = ph.get_par_resid_subset(pe,pe_real_names);
@@ -5470,10 +5471,11 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
 
 	if (ppo->get_ies_use_mda())
 	{
+	    message(1, "using multiple-data-assimilation algorithm");
 		int noptmax = pest_scenario.get_control_info().noptmax;
 		if (noptmax > 0)
 		{
-			message(0, "using multiple-data-assimilation algorithm");
+
             if (ppo->get_ies_no_noise())
             {
                 throw_em_error("'no noise'is not compatible with 'use_mda' as this solution relies on noise draws");
@@ -5482,7 +5484,7 @@ void EnsembleMethod::initialize(int cycle, bool run, bool use_existing)
 	}
 	else
 	{
-		message(0, "using glm algorithm");
+		message(1, "using glm algorithm");
 	}
 
 	verbose_level = pest_scenario.get_pestpp_options_ptr()->get_ies_verbose_level();
@@ -6347,7 +6349,7 @@ void EnsembleMethod::check_and_fill_phi_factors(map<string,vector<string>>& grou
     }
     ss.str("");
     ss << "checking phi factors in file " << fname;
-    message(0,ss.str());
+    message(1,ss.str());
     ;
     if (pest_scenario.get_pestpp_options().get_ies_phi_factors_by_real())
     {
@@ -9155,7 +9157,7 @@ vector<string> EnsembleMethod::detect_prior_data_conflict(bool save)
             in_conflict.push_back(oname);
             dist = max((smin - omax), (omin - smax));
 
-            pdccsv << oname << "," << omn << "," << ostd << "," << omin << "," << omax << "," << omin_stat << ","
+            pdccsv << pest_utils::lower_cp(oname) << "," << omn << "," << ostd << "," << omin << "," << omax << "," << omin_stat << ","
                    << omax_stat;
             pdccsv << "," << smn << "," << sstd << "," << smin << "," << smax << "," << smin_stat << ","
                    << smax_stat << "," << dist << endl;