75{
76 int status = 0;
77 string command;
78
79
80 po::options_description desc("Available options");
81 desc.add_options()
82 ("help,h", "Produce this help message.")
83 ("verbose,v", "Turn on verbosity.")
84 ("generate-metric,g", po::value<vector<int>>(),
85 "Generate a single metric.")
86 ("generate-all-metrics,a", "Generate all metrics.")
87 ("executable,e", po::value<string>(),
88 "Use specified executable.");
89
90 po::options_description hidden("Hidden options");
91 hidden.add_options()("input-file", po::value<string>(), "Input filename");
92
93 po::options_description cmdline_options("Command-line options");
94 cmdline_options.add(hidden).add(desc);
95
96 po::options_description visible("Allowed options");
97 visible.add(desc);
98
99 po::positional_options_description
p;
100 p.add(
"input-file", -1);
101
102 po::variables_map vm;
103
104 try
105 {
106 po::store(po::command_line_parser(argc, argv)
107 .options(cmdline_options)
110 vm);
111 po::notify(vm);
112 }
113 catch (const exception &e)
114 {
115 cerr << e.what() << endl;
116 cerr << desc;
117 return 1;
118 }
119
120 if (vm.count("help") || vm.count("input-file") != 1)
121 {
122 cerr << "Usage: Tester [options] input-file.tst" << endl;
123 cout << desc;
124 return 1;
125 }
126
127 bool verbose = vm.count("verbose");
128
129
130 vector<int> metricGenVec;
131 if (vm.count("generate-metric"))
132 {
133 metricGenVec = vm["generate-metric"].as<vector<int>>();
134 }
135 set<int> metricGen(metricGenVec.begin(), metricGenVec.end());
136
137
138 const fs::path specFile(vm["input-file"].as<string>());
139
140
141 fs::path specPath = specFile.parent_path();
142
143 if (specPath.empty())
144 {
145 specPath = fs::current_path();
146 }
147
148 string specFileStem = specFile.stem().string();
149
150
151
152 const fs::path masterDir =
154
155
156 const fs::path startDir = fs::current_path();
157
158 try
159 {
160 if (verbose)
161 {
162 cerr << "Reading test file definition: " << specFile << endl;
163 }
164
165
167
168 if (verbose && file.GetNumMetrics() > 0)
169 {
170 cerr << "Creating metrics:" << endl;
171 }
172
173
174 vector<MetricSharedPtr> metrics;
175 for (unsigned int i = 0; i < file.GetNumMetrics(); ++i)
176 {
177 set<int>::iterator it = metricGen.find(file.GetMetricId(i));
178 bool genMetric =
179 it != metricGen.end() || (vm.count("generate-all-metrics") > 0);
180
182 file.GetMetricType(i), file.GetMetric(i), genMetric));
183
184 if (verbose)
185 {
186 cerr << " - ID " << metrics.back()->GetID() << ": "
187 << metrics.back()->GetType() << endl;
188 }
189
190 if (it != metricGen.end())
191 {
192 metricGen.erase(it);
193 }
194 }
195
196 if (metricGen.size() != 0)
197 {
198 string s = metricGen.size() == 1 ? "s" : "";
199 set<int>::iterator it;
200 cerr << "Unable to find metric" + s + " with ID" + s + " ";
201 for (it = metricGen.begin(); it != metricGen.end(); ++it)
202 {
203 cerr << *it << " ";
204 }
205 cerr << endl;
206 return 1;
207 }
208
209
210 if (fs::exists(masterDir))
211 {
212 fs::remove_all(masterDir);
213 }
214
215 if (verbose)
216 {
217 cerr << "Creating master directory: " << masterDir << endl;
218 }
219
220
221 fs::create_directory(masterDir);
222
223
224 fs::current_path(masterDir);
225
226
227
228 fstream masterOut("master.out", ios::out | ios::in | ios::trunc);
229 fstream masterErr("master.err", ios::out | ios::in | ios::trunc);
230
231 if (masterOut.bad() || masterErr.bad())
232 {
233 cerr << "One or more master output files are unreadable." << endl;
234 throw 1;
235 }
236
237
238 vector<fs::path> tmpWorkingDirs;
239 string line;
240
241 for (unsigned int i = 0; i < file.GetNumRuns(); ++i)
242 {
243 command = "";
244
245 if (verbose)
246 {
247 cerr << "Starting run " << i << "." << endl;
248 }
249
250
251 const fs::path tmpDir =
252 masterDir / fs::path("run" + std::to_string(i));
253 tmpWorkingDirs.push_back(tmpDir);
254
255 if (verbose)
256 {
257 cerr << "Creating working directory: " << tmpDir << endl;
258 }
259
260
261 fs::create_directory(tmpDir);
262
263
264 fs::current_path(tmpDir);
265
266 if (verbose && file.GetNumDependentFiles())
267 {
268 cerr << "Copying required files: " << endl;
269 }
270
271
272
273 for (unsigned int j = 0; j < file.GetNumDependentFiles(); ++j)
274 {
275 fs::path source_file(file.GetDependentFile(j).m_filename);
276
277 fs::path source = specPath / source_file;
278 fs::path dest = tmpDir / source_file.filename();
279 if (verbose)
280 {
281 cerr << " - " << source << " -> " << dest << endl;
282 }
283
284 if (fs::is_directory(source))
285 {
286 fs::create_directory(dest);
287
288
289 for (const auto &dirEnt :
290 fs::recursive_directory_iterator{source})
291 {
292 fs::path newdest = dest / dirEnt.path().filename();
293 fs::copy_file(dirEnt.path(), newdest);
294 }
295 }
296 else
297 {
298 fs::copy_file(source, dest);
299 }
300 }
301
302
303 fs::path source_file("test.opt");
304 fs::path source = specPath / source_file;
305 bool HaveOptFile = false;
306 if (fs::exists(source))
307 {
308 fs::path dest = tmpDir / source_file.filename();
309 if (verbose)
310 {
311 cerr << " - " << source << " -> " << dest << endl;
312 }
313
314 if (fs::is_directory(source))
315 {
316 fs::create_directory(dest);
317
318
319 for (const auto &dirEnt :
320 fs::recursive_directory_iterator{source})
321 {
322 fs::path newdest = dest / dirEnt.path().filename();
323 fs::copy_file(dirEnt.path(), newdest);
324 }
325 }
326 else
327 {
328 fs::copy_file(source, dest);
329 }
330
331 HaveOptFile = true;
332 }
333
334
335
336
337
338
339 bool pythonAdded = false, mpiAdded = false;
340 for (unsigned int j = 0; j < file.GetNumCommands(); ++j)
341 {
342 Command cmd = file.GetCommand(j);
344 {
345
346 command = "PYTHONPATH=\"@CMAKE_BINARY_DIR@\" " + command;
347 pythonAdded = true;
348 }
349
350#ifdef NEKTAR_TEST_FORCEMPIEXEC
351#else
352 if (cmd.
m_processes > 1 || file.GetNumCommands() > 1)
353#endif
354 {
355 if (mpiAdded)
356 {
357 continue;
358 }
359
360 command += "\"@MPIEXEC@\" ";
361 if (std::string("@NEKTAR_TEST_USE_HOSTFILE@") == "ON")
362 {
363 command += "-hostfile hostfile ";
364 if (system("echo 'localhost slots=12' > hostfile"))
365 {
366 cerr << "Unable to write 'hostfile' in path '"
367 << fs::current_path() << endl;
368 status = 1;
369 }
370 }
371
372 if (file.GetNumCommands() > 1)
373 {
374 command += "--tag-output ";
375 }
376
377 mpiAdded = true;
378 }
379 }
380
381
382 for (unsigned int j = 0; j < file.GetNumCommands(); ++j)
383 {
384 Command cmd = file.GetCommand(j);
385
386
387
388 if (j > 0)
389 {
390 command += " : ";
391 }
392
393
394 if (file.GetNumCommands() > 1 || cmd.
m_processes > 1)
395 {
396 command += "@MPIEXEC_NUMPROC_FLAG@ ";
398 }
399
400
402 if (!fs::exists(execPath))
403 {
406 }
407
408
409
411 {
412 command += "@PYTHON_EXECUTABLE@ ";
413 }
414
416 if (HaveOptFile)
417 {
418 command += " --use-opt-file test.opt ";
419 }
420
421 command += " ";
423 command += " 1>output.out 2>output.err";
424 }
425
426 status = 0;
427
428 if (verbose)
429 {
430 cerr << "Running command: " << command << endl;
431 }
432
433
434 if (system(command.c_str()))
435 {
436 cerr << "Error occurred running test:" << endl;
437 cerr << "Command: " << command << endl;
438 status = 1;
439 }
440
441
442 if (!(fs::exists("output.out") && fs::exists("output.err")))
443 {
444 cerr << "One or more test output files are missing." << endl;
445 throw 1;
446 }
447
448
449 ifstream vStdout("output.out");
450 ifstream vStderr("output.err");
451 if (vStdout.bad() || vStderr.bad())
452 {
453 cerr << "One or more test output files are unreadable." << endl;
454 throw 1;
455 }
456
457
458 if (verbose)
459 {
460 cerr << "Appending run " << i << " output and error to master."
461 << endl;
462 }
463
464 while (getline(vStdout, line))
465 {
466 masterOut << line << endl;
467 }
468
469 while (getline(vStderr, line))
470 {
471 masterErr << line << endl;
472 }
473
474 vStdout.close();
475 vStderr.close();
476 }
477
478
479 for (int i = 0; i < metrics.size(); ++i)
480 {
481 if (!metrics[i]->SupportsAverage() && file.GetNumRuns() > 1)
482 {
483 cerr << "WARNING: Metric " << metrics[i]->GetType()
484 << " does not support multiple runs. Test may yield "
485 "unexpected results."
486 << endl;
487 }
488 }
489
490
491 if (status == 0)
492 {
493 if (verbose && metrics.size())
494 {
495 cerr << "Checking metrics:" << endl;
496 }
497
498 for (int i = 0; i < metrics.size(); ++i)
499 {
500 bool gen =
501 metricGen.find(metrics[i]->GetID()) != metricGen.end() ||
502 (vm.count("generate-all-metrics") > 0);
503
504 masterOut.clear();
505 masterErr.clear();
506 masterOut.seekg(0, ios::beg);
507 masterErr.seekg(0, ios::beg);
508
509 if (verbose)
510 {
511 cerr << " - " << (gen ? "generating" : "checking")
512 << " metric " << metrics[i]->GetID() << " ("
513 << metrics[i]->GetType() << ")... ";
514 }
515
516 if (!metrics[i]->Test(masterOut, masterErr))
517 {
518 status = 1;
519 if (verbose)
520 {
521 cerr << "failed!" << endl;
522 }
523 }
524 else if (verbose)
525 {
526 cerr << "passed" << endl;
527 }
528 }
529 }
530
531 if (verbose)
532 {
533 cerr << endl << endl;
534 }
535
536
537 if (status == 1 || verbose)
538 {
539 masterOut.clear();
540 masterErr.clear();
541 masterOut.seekg(0, ios::beg);
542 masterErr.seekg(0, ios::beg);
543
544 cout << "=== Output ===" << endl;
545 while (masterOut.good())
546 {
547 getline(masterOut, line);
548 cout << line << endl;
549 }
550 cout << "=== Errors ===" << endl;
551 while (masterErr.good())
552 {
553 getline(masterErr, line);
554 cout << line << endl;
555 }
556 }
557
558
559 masterOut.close();
560 masterErr.close();
561
562
563 fs::current_path(startDir);
564
565 if (verbose)
566 {
567 cerr << "Removing working directory" << endl;
568 }
569
570
571
572
573
574 int i = 1000;
575 while (i > 0)
576 {
577 try
578 {
579
580 fs::remove_all(masterDir);
581 break;
582 }
583 catch (const fs::filesystem_error &e)
584 {
585 using namespace std::chrono_literals;
586 std::this_thread::sleep_for(1ms);
587 i--;
588 if (i > 0)
589 {
590 cout << "Locked files encountered. "
591 << "Retrying after 1ms..." << endl;
592 }
593 else
594 {
595
596
597 throw e;
598 }
599 }
600 }
601
602
603 if (vm.count("generate-metric") > 0 ||
604 vm.count("generate-all-metrics") > 0)
605 {
606 file.SaveFile();
607 }
608
609
610 return status;
611 }
612 catch (const fs::filesystem_error &e)
613 {
614 cerr << "Filesystem operation error occurred:" << endl;
615 cerr << " " << e.what() << endl;
616 cerr << " Files left in " << masterDir.string() << endl;
617 }
619 {
620 cerr << "Error occurred during test:" << endl;
621 cerr << " " << e.what() << endl;
622 cerr << " Files left in " << masterDir.string() << endl;
623 }
624 catch (const std::exception &e)
625 {
626 cerr << "Unhandled exception during test:" << endl;
627 cerr << " " << e.what() << endl;
628 cerr << " Files left in " << masterDir.string() << endl;
629 }
630 catch (...)
631 {
632 cerr << "Unknown error during test" << endl;
633 cerr << " Files left in " << masterDir.string() << endl;
634 }
635
636
637 return 2;
638}
#define ASSERTL0(condition, msg)
MetricSharedPtr CreateInstance(std::string key, TiXmlElement *elmt, bool generate)
The TestData class is responsible for parsing a test XML file and storing the data.
static std::string PortablePath(const fs::path &path)
create portable path on different platforms for std::filesystem path.
static fs::path UniquePath(std::string specFileStem)
Create a unique (random) path, based on an input stem string. The returned string is a filename or di...
MetricFactory & GetMetricFactory()
Subclass of std::runtime_error to handle exceptions raised by Tester.