Tester.cpp.in File Reference

This file contains the main function for the Tester program, which is a tool for testing Nektar++ executables. More...

#include <algorithm>
#include <chrono>
#include <fstream>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include <Metric.h>
#include <TestData.h>
#include <LibUtilities/BasicUtils/Filesystem.hpp>
#include <boost/program_options.hpp>

Go to the source code of this file.

Macros
#define	NEKTAR_TEST_FORCEMPIEXEC   1

Functions
int	main (int argc, char *argv[])

Detailed Description

This file contains the main function for the Tester program, which is a tool for testing Nektar++ executables.

The main function reads command line options and parses the provided test (.tst) file. Using information provided in this file, the Tester program generates test metrics, and creates temporary subdirectories in which to run the executable. All test outputs are appended to a single master.out file, and errors are appended to master.err. These files are sent to all of the metrics for analysis. If the test fails, the output and error files are dumped to the terminal for debugging purposes.

See also

Metric

Metric::Test:

Definition in file Tester.cpp.in.

Macro Definition Documentation

NEKTAR_TEST_FORCEMPIEXEC

#define NEKTAR_TEST_FORCEMPIEXEC   1

Definition at line 67 of file Tester.cpp.in.

Function Documentation

main()

int main	(	int	argc,
		char *	argv[]
	)

Definition at line 91 of file Tester.cpp.in.

{
    int status = 0;
    string command;
 
    // Set up command line options.
    po::options_description desc("Available options");
    desc.add_options()("help,h", "Produce this help message.")(
        "verbose,v", "Turn on verbosity.")("generate-metric,g",
                                           po::value<vector<int>>(),
                                           "Generate a single metric.")(
        "generate-all-metrics,a", "Generate all metrics.")(
        "executable,e", po::value<string>(), "Use specified executable.");
 
    po::options_description hidden("Hidden options");
    hidden.add_options()("input-file", po::value<string>(), "Input filename");
 
    po::options_description cmdline_options("Command-line options");
    cmdline_options.add(hidden).add(desc);
 
    po::options_description visible("Allowed options");
    visible.add(desc);
 
    po::positional_options_description p;
    p.add("input-file", -1);
 
    po::variables_map vm;
 
    try
    {
        po::store(po::command_line_parser(argc, argv)
                      .options(cmdline_options)
                      .positional(p)
                      .run(),
                  vm);
        po::notify(vm);
    }
    catch (const exception &e)
    {
        cerr << e.what() << endl;
        cerr << desc;
        return 1;
    }
 
    if (vm.count("help") || vm.count("input-file") != 1)
    {
        cerr << "Usage: Tester [options] input-file.tst" << endl;
        cout << desc;
        return 1;
    }
 
    bool verbose = vm.count("verbose");
 
    // Set up set containing metrics to be generated.
    vector<int> metricGenVec;
    if (vm.count("generate-metric"))
    {
        metricGenVec = vm["generate-metric"].as<vector<int>>();
    }
    set<int> metricGen(metricGenVec.begin(), metricGenVec.end());
 
    // Path to test definition file
    const fs::path specFile(vm["input-file"].as<string>());
 
    // Parent path of test definition file containing dependent files
    fs::path specPath = specFile.parent_path();
 
    if (specPath.empty())
    {
        specPath = fs::current_path();
    }
 
    string specFileStem = specFile.stem().string();
 
    // Temporary master directory to create which holds master output and error
    // files, and the working directories for each run
    const fs::path masterDir =
        fs::current_path() / LibUtilities::UniquePath(specFileStem);
 
    // The current directory
    const fs::path startDir = fs::current_path();
 
    try
    {
        if (verbose)
        {
            cerr << "Reading test file definition: " << specFile << endl;
        }
 
        // Parse the test file
        TestData file(specFile, vm);
 
        if (verbose && file.GetNumMetrics() > 0)
        {
            cerr << "Creating metrics:" << endl;
        }
 
        // Generate the metric objects
        vector<MetricSharedPtr> metrics;
        for (unsigned int i = 0; i < file.GetNumMetrics(); ++i)
        {
            set<int>::iterator it = metricGen.find(file.GetMetricId(i));
            bool genMetric =
                it != metricGen.end() || (vm.count("generate-all-metrics") > 0);
 
            metrics.push_back(GetMetricFactory().CreateInstance(
                file.GetMetricType(i), file.GetMetric(i), genMetric));
 
            if (verbose)
            {
                cerr << "  - ID " << metrics.back()->GetID() << ": "
                     << metrics.back()->GetType() << endl;
            }
 
            if (it != metricGen.end())
            {
                metricGen.erase(it);
            }
        }
 
        if (metricGen.size() != 0)
        {
            string s = metricGen.size() == 1 ? "s" : "";
            set<int>::iterator it;
            cerr << "Unable to find metric" + s + " with ID" + s + " ";
            for (it = metricGen.begin(); it != metricGen.end(); ++it)
            {
                cerr << *it << " ";
            }
            cerr << endl;
            return 1;
        }
 
        // Remove the master directory if left from a previous test
        if (fs::exists(masterDir))
        {
            fs::remove_all(masterDir);
        }
 
        if (verbose)
        {
            cerr << "Creating master directory: " << masterDir << endl;
        }
 
        // Create the master directory
        fs::create_directory(masterDir);
 
        // Change working directory to the master directory
        fs::current_path(masterDir);
 
        // Create a master output and error file. Output and error files from
        // all runs will be appended to these files.
        fstream masterOut("master.out", ios::out | ios::in | ios::trunc);
        fstream masterErr("master.err", ios::out | ios::in | ios::trunc);
 
        if (masterOut.bad() || masterErr.bad())
        {
            cerr << "One or more master output files are unreadable." << endl;
            throw 1;
        }
 
        // Vector of temporary subdirectories to create and conduct tests in
        vector<fs::path> tmpWorkingDirs;
        string line;
 
        for (unsigned int i = 0; i < file.GetNumRuns(); ++i)
        {
            command = "";
 
            if (verbose)
            {
                cerr << "Starting run " << i << "." << endl;
            }
 
            // Temporary directory to create and in which to hold the run
            const fs::path tmpDir =
                masterDir / fs::path("run" + std::to_string(i));
            tmpWorkingDirs.push_back(tmpDir);
 
            if (verbose)
            {
                cerr << "Creating working directory: " << tmpDir << endl;
            }
 
            // Create temporary directory
            fs::create_directory(tmpDir);
 
            // Change working directory to the temporary directory
            fs::current_path(tmpDir);
 
            if (verbose && file.GetNumDependentFiles())
            {
                cerr << "Copying required files: " << endl;
            }
 
            // Copy required files for this test from the test definition
            // directory to the temporary directory.
            for (unsigned int j = 0; j < file.GetNumDependentFiles(); ++j)
            {
                fs::path source_file(file.GetDependentFile(j).m_filename);
 
                fs::path source = specPath / source_file;
                fs::path dest   = tmpDir / source_file.filename();
                if (verbose)
                {
                    cerr << "  - " << source << " -> " << dest << endl;
                }
 
                if (fs::is_directory(source))
                {
                    fs::create_directory(dest);
                    // If source is a directory, then only directory name is
                    // created, so call copy again to copy files.
                    for (const auto &dirEnt :
                         fs::recursive_directory_iterator{source})
                    {
                        fs::path newdest = dest / dirEnt.path().filename();
                        fs::copy_file(dirEnt.path(), newdest);
                    }
                }
                else
                {
                    fs::copy_file(source, dest);
                }
            }
 
            // Copy opt file if exists to  to the temporary directory.
            fs::path source_file("test.opt");
            fs::path source  = specPath / source_file;
            bool HaveOptFile = false;
            if (fs::exists(source))
            {
                fs::path dest = tmpDir / source_file.filename();
                if (verbose)
                {
                    cerr << "  - " << source << " -> " << dest << endl;
                }
 
                if (fs::is_directory(source))
                {
                    fs::create_directory(dest);
                    // If source is a directory, then only directory name is
                    // created, so call copy again to copy files.
                    for (const auto &dirEnt :
                         fs::recursive_directory_iterator{source})
                    {
                        fs::path newdest = dest / dirEnt.path().filename();
                        fs::copy_file(dirEnt.path(), newdest);
                    }
                }
                else
                {
                    fs::copy_file(source, dest);
                }
 
                HaveOptFile = true;
            }
 
            // If we're Python, copy script too.
 
            // Set PYTHONPATH environment variable in case Python is run inside
            // any of our tests.  For non-Python tests this will do nothing.
            setenv("PYTHONPATH", "@NEKPY_BASE_DIR@", true);
 
            // Construct test command to run. Output from stdout and stderr are
            // directed to the files output.out and output.err, respectively.
 
            bool mpiAdded = false;
            for (unsigned int j = 0; j < file.GetNumCommands(); ++j)
            {
                Command cmd = file.GetCommand(j);
 
#ifdef NEKTAR_TEST_FORCEMPIEXEC
#else
                if (cmd.m_processes > 1 || (file.GetNumCommands() > 1 &&
                                            cmd.m_commandType == eParallel))
#endif
                {
                    if (mpiAdded)
                    {
                        continue;
                    }
 
                    command += "\"@MPIEXEC@\" ";
                    if (std::string("@NEKTAR_TEST_USE_HOSTFILE@") == "ON")
                    {
                        command += "-hostfile hostfile ";
#if (NEKTAR_MPI_TYPE == 1) // MPICH
                        if (system("echo 'localhost:12' > hostfile"))
#else
                        if (system("echo 'localhost slots=12' > hostfile"))
#endif
                        {
                            cerr << "Unable to write 'hostfile' in path '"
                                 << fs::current_path() << endl;
                            status = 1;
                        }
                    }
 
                    if (file.GetNumCommands() > 1)
                    {
#if (NEKTAR_MPI_TYPE == 1)
                        // MPICH prepends the rank to each cout, causing the
                        // rank annotation to appear in the middle of the output
                        // and not just at the beginning of lines.
                        // This causes tests not to pass as the tester fails to
                        // parse the output correctly.
//                        command += "--prepend-rank ";
#else
                        command += "--tag-output ";
#endif
                    }
 
                    mpiAdded = true;
                }
            }
 
            // Parse commands.
            for (unsigned int j = 0; j < file.GetNumCommands(); ++j)
            {
                Command cmd = file.GetCommand(j);
 
                // If running with multiple commands simultaneously, separate
                // with colon.
                if (j > 0 && cmd.m_commandType == eParallel)
                {
                    command += " : ";
                }
                else if (j > 0 && cmd.m_commandType == eSequential)
                {
                    command += " && ";
                    if (cmd.m_processes > 1)
                    {
                        command += "\"@MPIEXEC@\" ";
                        if (std::string("@NEKTAR_TEST_USE_HOSTFILE@") == "ON")
                        {
                            command += "-hostfile hostfile ";
                        }
                    }
                }
 
                // Add -n where appropriate.
                if (cmd.m_processes > 1 || (file.GetNumCommands() > 1 &&
                                            cmd.m_commandType == eParallel))
                {
                    command += "@MPIEXEC_NUMPROC_FLAG@ ";
                    command += std::to_string(cmd.m_processes) + " ";
                }
 
                // Look for executable or Python script.
                fs::path execPath = startDir / cmd.m_executable;
                if (!fs::exists(execPath))
                {
                    ASSERTL0(!cmd.m_pythonTest, "Python script not found.");
                    execPath = cmd.m_executable;
                }
 
                // Prepend script name with Python executable path if this is a
                // Python test.
                if (cmd.m_pythonTest)
                {
                    command += "@Python3_EXECUTABLE@ ";
                }
 
                std::string pathString = LibUtilities::PortablePath(execPath);
                command += pathString;
                if (HaveOptFile)
                {
                    command += " --use-opt-file test.opt ";
                }
 
                command += " ";
                command += cmd.m_parameters;
                command += " 1>output.out 2>output.err";
            }
 
            status = 0;
 
            if (verbose)
            {
                cerr << "Running command: " << command << endl;
            }
 
            // Run executable to perform test.
            if (system(command.c_str()))
            {
                cerr << "Error occurred running test:" << endl;
                cerr << "Command: " << command << endl;
                status = 1;
            }
 
            // Check output files exist
            if (!(fs::exists("output.out") && fs::exists("output.err")))
            {
                cerr << "One or more test output files are missing." << endl;
                throw 1;
            }
 
            // Open output files and check they are readable
            ifstream vStdout("output.out");
            ifstream vStderr("output.err");
            if (vStdout.bad() || vStderr.bad())
            {
                cerr << "One or more test output files are unreadable." << endl;
                throw 1;
            }
 
            // Append output to the master output and error files.
            if (verbose)
            {
                cerr << "Appending run " << i << " output and error to master."
                     << endl;
            }
 
            while (getline(vStdout, line))
            {
                masterOut << line << endl;
            }
 
            while (getline(vStderr, line))
            {
                masterErr << line << endl;
            }
 
            vStdout.close();
            vStderr.close();
        }
 
        // Warn user if any metrics don't support multiple runs.
        for (int i = 0; i < metrics.size(); ++i)
        {
            if (!metrics[i]->SupportsAverage() && file.GetNumRuns() > 1)
            {
                cerr << "WARNING: Metric " << metrics[i]->GetType()
                     << " does not support multiple runs. Test may yield "
                        "unexpected results."
                     << endl;
            }
        }
 
        // Test against all metrics
        if (status == 0)
        {
            if (verbose && metrics.size())
            {
                cerr << "Checking metrics:" << endl;
            }
 
            for (int i = 0; i < metrics.size(); ++i)
            {
                bool gen =
                    metricGen.find(metrics[i]->GetID()) != metricGen.end() ||
                    (vm.count("generate-all-metrics") > 0);
 
                masterOut.clear();
                masterErr.clear();
                masterOut.seekg(0, ios::beg);
                masterErr.seekg(0, ios::beg);
 
                if (verbose)
                {
                    cerr << "  - " << (gen ? "generating" : "checking")
                         << " metric " << metrics[i]->GetID() << " ("
                         << metrics[i]->GetType() << ")... ";
                }
 
                if (!metrics[i]->Test(masterOut, masterErr))
                {
                    status = 1;
                    if (verbose)
                    {
                        cerr << "failed!" << endl;
                    }
                }
                else if (verbose)
                {
                    cerr << "passed" << endl;
                }
            }
        }
 
        if (verbose)
        {
            cerr << endl << endl;
        }
 
        // Dump output files to terminal for debugging purposes on fail.
        if (status == 1 || verbose)
        {
            masterOut.clear();
            masterErr.clear();
            masterOut.seekg(0, ios::beg);
            masterErr.seekg(0, ios::beg);
 
            cout << "=== Output ===" << endl;
            while (masterOut.good())
            {
                getline(masterOut, line);
                cout << line << endl;
            }
            cout << "=== Errors ===" << endl;
            while (masterErr.good())
            {
                getline(masterErr, line);
                cout << line << endl;
            }
        }
 
        // Close output files.
        masterOut.close();
        masterErr.close();
 
        // Change back to the original path and delete temporary directory.
        fs::current_path(startDir);
 
        if (verbose)
        {
            cerr << "Removing working directory" << endl;
        }
 
        // Repeatedly try deleting directory with sleep for filesystems which
        // work asynchronously. This allows time for the filesystem to register
        // the output files are closed so they can be deleted and not cause a
        // filesystem failure. Attempts made for 1 second.
        int i = 1000;
        while (i > 0)
        {
            try
            {
                // If delete successful, stop trying.
                fs::remove_all(masterDir);
                break;
            }
            catch (const fs::filesystem_error &e)
            {
                using namespace std::chrono_literals;
                std::this_thread::sleep_for(1ms);
                i--;
                if (i > 0)
                {
                    cout << "Locked files encountered. "
                         << "Retrying after 1ms..." << endl;
                }
                else
                {
                    // If still failing after 1sec, we consider it a permanent
                    // filesystem error and abort.
                    throw e;
                }
            }
        }
 
        // Save any changes.
        if (vm.count("generate-metric") > 0 ||
            vm.count("generate-all-metrics") > 0)
        {
            file.SaveFile();
        }
 
        // Return status of test. 0 = PASS, 1 = FAIL
        return status;
    }
    catch (const fs::filesystem_error &e)
    {
        cerr << "Filesystem operation error occurred:" << endl;
        cerr << "  " << e.what() << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
    catch (const TesterException &e)
    {
        cerr << "Error occurred during test:" << endl;
        cerr << "  " << e.what() << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
    catch (const std::exception &e)
    {
        cerr << "Unhandled exception during test:" << endl;
        cerr << "  " << e.what() << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
    catch (...)
    {
        cerr << "Unknown error during test" << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
 
    // If a system error, return 2
    return 2;
}

References ASSERTL0, Nektar::MetricFactory::CreateInstance(), Nektar::eParallel, Nektar::eSequential, Nektar::TestData::GetCommand(), Nektar::TestData::GetDependentFile(), Nektar::TestData::GetMetric(), Nektar::GetMetricFactory(), Nektar::TestData::GetMetricId(), Nektar::TestData::GetMetricType(), Nektar::TestData::GetNumCommands(), Nektar::TestData::GetNumDependentFiles(), Nektar::TestData::GetNumMetrics(), Nektar::TestData::GetNumRuns(), Nektar::Command::m_commandType, Nektar::Command::m_executable, Nektar::DependentFile::m_filename, Nektar::Command::m_parameters, Nektar::Command::m_processes, Nektar::Command::m_pythonTest, CellMLToNektar.cellml_metadata::p, Nektar::LibUtilities::PortablePath(), CellMLToNektar.translators::run(), Nektar::TestData::SaveFile(), and Nektar::LibUtilities::UniquePath().