64#include <boost/program_options.hpp>
66#cmakedefine NEKTAR_TEST_FORCEMPIEXEC 1
72namespace po = boost::program_options;
74int main(
int argc,
char *argv[])
80 po::options_description desc(
"Available options");
82 (
"help,h",
"Produce this help message.")
83 (
"verbose,v",
"Turn on verbosity.")
84 (
"generate-metric,g", po::value<vector<int>>(),
85 "Generate a single metric.")
86 (
"generate-all-metrics,a",
"Generate all metrics.")
87 (
"executable,e", po::value<string>(),
88 "Use specified executable.");
90 po::options_description hidden(
"Hidden options");
91 hidden.add_options()(
"input-file", po::value<string>(),
"Input filename");
93 po::options_description cmdline_options(
"Command-line options");
94 cmdline_options.add(hidden).add(desc);
96 po::options_description visible(
"Allowed options");
99 po::positional_options_description
p;
100 p.add(
"input-file", -1);
102 po::variables_map vm;
106 po::store(po::command_line_parser(argc, argv)
107 .options(cmdline_options)
113 catch (
const exception &e)
115 cerr << e.what() << endl;
120 if (vm.count(
"help") || vm.count(
"input-file") != 1)
122 cerr <<
"Usage: Tester [options] input-file.tst" << endl;
127 bool verbose = vm.count(
"verbose");
130 vector<int> metricGenVec;
131 if (vm.count(
"generate-metric"))
133 metricGenVec = vm[
"generate-metric"].as<vector<int>>();
135 set<int> metricGen(metricGenVec.begin(), metricGenVec.end());
138 const fs::path specFile(vm[
"input-file"].as<string>());
141 fs::path specPath = specFile.parent_path();
143 if (specPath.empty())
145 specPath = fs::current_path();
148 string specFileStem = specFile.stem().string();
152 const fs::path masterDir =
156 const fs::path startDir = fs::current_path();
162 cerr <<
"Reading test file definition: " << specFile << endl;
170 cerr <<
"Creating metrics:" << endl;
174 vector<MetricSharedPtr> metrics;
177 set<int>::iterator it = metricGen.find(file.
GetMetricId(i));
179 it != metricGen.end() || (vm.count(
"generate-all-metrics") > 0);
186 cerr <<
" - ID " << metrics.back()->GetID() <<
": "
187 << metrics.back()->GetType() << endl;
190 if (it != metricGen.end())
196 if (metricGen.size() != 0)
198 string s = metricGen.size() == 1 ?
"s" :
"";
199 set<int>::iterator it;
200 cerr <<
"Unable to find metric" + s +
" with ID" + s +
" ";
201 for (it = metricGen.begin(); it != metricGen.end(); ++it)
210 if (fs::exists(masterDir))
212 fs::remove_all(masterDir);
217 cerr <<
"Creating master directory: " << masterDir << endl;
221 fs::create_directory(masterDir);
224 fs::current_path(masterDir);
228 fstream masterOut(
"master.out", ios::out | ios::in | ios::trunc);
229 fstream masterErr(
"master.err", ios::out | ios::in | ios::trunc);
231 if (masterOut.bad() || masterErr.bad())
233 cerr <<
"One or more master output files are unreadable." << endl;
238 vector<fs::path> tmpWorkingDirs;
241 for (
unsigned int i = 0; i < file.
GetNumRuns(); ++i)
247 cerr <<
"Starting run " << i <<
"." << endl;
251 const fs::path tmpDir =
252 masterDir / fs::path(
"run" + std::to_string(i));
253 tmpWorkingDirs.push_back(tmpDir);
257 cerr <<
"Creating working directory: " << tmpDir << endl;
261 fs::create_directory(tmpDir);
264 fs::current_path(tmpDir);
268 cerr <<
"Copying required files: " << endl;
277 fs::path source = specPath / source_file;
278 fs::path dest = tmpDir / source_file.filename();
281 cerr <<
" - " << source <<
" -> " << dest << endl;
284 if (fs::is_directory(source))
286 fs::create_directory(dest);
289 for (
const auto &dirEnt :
290 fs::recursive_directory_iterator{source})
292 fs::path newdest = dest / dirEnt.path().filename();
293 fs::copy_file(dirEnt.path(), newdest);
298 fs::copy_file(source, dest);
303 fs::path source_file(
"test.opt");
304 fs::path source = specPath / source_file;
305 bool HaveOptFile =
false;
306 if (fs::exists(source))
308 fs::path dest = tmpDir / source_file.filename();
311 cerr <<
" - " << source <<
" -> " << dest << endl;
314 if (fs::is_directory(source))
316 fs::create_directory(dest);
319 for (
const auto &dirEnt :
320 fs::recursive_directory_iterator{source})
322 fs::path newdest = dest / dirEnt.path().filename();
323 fs::copy_file(dirEnt.path(), newdest);
328 fs::copy_file(source, dest);
339 bool pythonAdded =
false, mpiAdded =
false;
346 command =
"PYTHONPATH=\"@CMAKE_BINARY_DIR@\" " + command;
350#ifdef NEKTAR_TEST_FORCEMPIEXEC
360 command +=
"\"@MPIEXEC@\" ";
361 if (std::string(
"@NEKTAR_TEST_USE_HOSTFILE@") ==
"ON")
363 command +=
"-hostfile hostfile ";
364 if (system(
"echo 'localhost slots=12' > hostfile"))
366 cerr <<
"Unable to write 'hostfile' in path '"
367 << fs::current_path() << endl;
374 command +=
"--tag-output ";
396 command +=
"@MPIEXEC_NUMPROC_FLAG@ ";
402 if (!fs::exists(execPath))
412 command +=
"@PYTHON_EXECUTABLE@ ";
418 command +=
" --use-opt-file test.opt ";
423 command +=
" 1>output.out 2>output.err";
430 cerr <<
"Running command: " << command << endl;
434 if (system(command.c_str()))
436 cerr <<
"Error occurred running test:" << endl;
437 cerr <<
"Command: " << command << endl;
442 if (!(fs::exists(
"output.out") && fs::exists(
"output.err")))
444 cerr <<
"One or more test output files are missing." << endl;
449 ifstream vStdout(
"output.out");
450 ifstream vStderr(
"output.err");
451 if (vStdout.bad() || vStderr.bad())
453 cerr <<
"One or more test output files are unreadable." << endl;
460 cerr <<
"Appending run " << i <<
" output and error to master."
464 while (getline(vStdout, line))
466 masterOut << line << endl;
469 while (getline(vStderr, line))
471 masterErr << line << endl;
479 for (
int i = 0; i < metrics.size(); ++i)
481 if (!metrics[i]->SupportsAverage() && file.
GetNumRuns() > 1)
483 cerr <<
"WARNING: Metric " << metrics[i]->GetType()
484 <<
" does not support multiple runs. Test may yield "
485 "unexpected results."
493 if (verbose && metrics.size())
495 cerr <<
"Checking metrics:" << endl;
498 for (
int i = 0; i < metrics.size(); ++i)
501 metricGen.find(metrics[i]->GetID()) != metricGen.end() ||
502 (vm.count(
"generate-all-metrics") > 0);
506 masterOut.seekg(0, ios::beg);
507 masterErr.seekg(0, ios::beg);
511 cerr <<
" - " << (gen ?
"generating" :
"checking")
512 <<
" metric " << metrics[i]->GetID() <<
" ("
513 << metrics[i]->GetType() <<
")... ";
516 if (!metrics[i]->Test(masterOut, masterErr))
521 cerr <<
"failed!" << endl;
526 cerr <<
"passed" << endl;
533 cerr << endl << endl;
537 if (status == 1 || verbose)
541 masterOut.seekg(0, ios::beg);
542 masterErr.seekg(0, ios::beg);
544 cout <<
"=== Output ===" << endl;
545 while (masterOut.good())
547 getline(masterOut, line);
548 cout << line << endl;
550 cout <<
"=== Errors ===" << endl;
551 while (masterErr.good())
553 getline(masterErr, line);
554 cout << line << endl;
563 fs::current_path(startDir);
567 cerr <<
"Removing working directory" << endl;
580 fs::remove_all(masterDir);
583 catch (
const fs::filesystem_error &e)
585 using namespace std::chrono_literals;
586 std::this_thread::sleep_for(1ms);
590 cout <<
"Locked files encountered. "
591 <<
"Retrying after 1ms..." << endl;
603 if (vm.count(
"generate-metric") > 0 ||
604 vm.count(
"generate-all-metrics") > 0)
612 catch (
const fs::filesystem_error &e)
614 cerr <<
"Filesystem operation error occurred:" << endl;
615 cerr <<
" " << e.what() << endl;
616 cerr <<
" Files left in " << masterDir.string() << endl;
620 cerr <<
"Error occurred during test:" << endl;
621 cerr <<
" " << e.what() << endl;
622 cerr <<
" Files left in " << masterDir.string() << endl;
624 catch (
const std::exception &e)
626 cerr <<
"Unhandled exception during test:" << endl;
627 cerr <<
" " << e.what() << endl;
628 cerr <<
" Files left in " << masterDir.string() << endl;
632 cerr <<
"Unknown error during test" << endl;
633 cerr <<
" Files left in " << masterDir.string() << endl;
#define ASSERTL0(condition, msg)
int main(int argc, char *argv[])
MetricSharedPtr CreateInstance(std::string key, TiXmlElement *elmt, bool generate)
The TestData class is responsible for parsing a test XML file and storing the data.
DependentFile GetDependentFile(unsigned int pId) const
unsigned int GetMetricId(unsigned int pId)
Returns the ID of the metric for a given metric ID.
unsigned int GetNumDependentFiles() const
Returns the number of dependent files required for the test.
unsigned int GetNumMetrics() const
Returns the number of metrics to be collected for the test.
unsigned int GetNumRuns() const
Returns the number of runs to be performed for the test.
TiXmlElement * GetMetric(unsigned int pId)
Returns a pointer to the TiXmlElement object representing the metric for a given metric ID.
unsigned int GetNumCommands() const
std::string GetMetricType(unsigned int pId) const
Returns the type of metric to be collected for a given metric ID.
const Command & GetCommand(unsigned int pId) const
static std::string PortablePath(const fs::path &path)
create portable path on different platforms for std::filesystem path.
static fs::path UniquePath(std::string specFileStem)
Create a unique (random) path, based on an input stem string. The returned string is a filename or di...
MetricFactory & GetMetricFactory()
Subclass of std::runtime_error to handle exceptions raised by Tester.