Tester.cpp.in

Go to the documentation of this file.

///////////////////////////////////////////////////////////////////////////////
//
// File: Tester.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: Tester executable.
//
///////////////////////////////////////////////////////////////////////////////
 
/**
 * @file Tester.cpp.in
 * @brief This file contains the main function for the Tester program, which is
 * a tool for testing Nektar++ executables.
 *
 * The main function reads command line options and parses the provided test
 * (.tst) file. Using information provided in this file, the Tester program
 * generates test metrics, and creates temporary subdirectories in which to run
 * the executable. All test outputs are appended to a single @p master.out file,
 * and errors are appended to @p master.err. These files are sent to all of the
 * metrics for analysis. If the test fails, the output and error files are
 * dumped to the terminal for debugging purposes.
 *
 * @see Metric
 * @see Metric#Test:
 */
 
#include <algorithm>
#include <chrono>
#include <fstream>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
 
#include <Metric.h>
#include <TestData.h>
 
#include <LibUtilities/BasicUtils/Filesystem.hpp>
 
#include <boost/program_options.hpp>
 
#cmakedefine NEKTAR_TEST_FORCEMPIEXEC 1
 
using namespace std;
using namespace Nektar;
 
// Define some namespace aliases
namespace po = boost::program_options;
 
#ifdef _WIN32
// Define a setenv function for Windows
int setenv(const char *name, const char *value, int overwrite)
{
    int errcode = 0;
    if (!overwrite)
    {
        size_t envsize = 0;
        errcode        = getenv_s(&envsize, NULL, 0, name);
        if (errcode || envsize)
            return errcode;
    }
    return _putenv_s(name, value);
}
#endif
 
int main(int argc, char *argv[])
{
    int status = 0;
    string command;
 
    // Set up command line options.
    po::options_description desc("Available options");
    desc.add_options()("help,h", "Produce this help message.")(
        "verbose,v", "Turn on verbosity.")("generate-metric,g",
                                           po::value<vector<int>>(),
                                           "Generate a single metric.")(
        "generate-all-metrics,a", "Generate all metrics.")(
        "executable,e", po::value<string>(), "Use specified executable.");
 
    po::options_description hidden("Hidden options");
    hidden.add_options()("input-file", po::value<string>(), "Input filename");
 
    po::options_description cmdline_options("Command-line options");
    cmdline_options.add(hidden).add(desc);
 
    po::options_description visible("Allowed options");
    visible.add(desc);
 
    po::positional_options_description p;
    p.add("input-file", -1);
 
    po::variables_map vm;
 
    try
    {
        po::store(po::command_line_parser(argc, argv)
                      .options(cmdline_options)
                      .positional(p)
                      .run(),
                  vm);
        po::notify(vm);
    }
    catch (const exception &e)
    {
        cerr << e.what() << endl;
        cerr << desc;
        return 1;
    }
 
    if (vm.count("help") || vm.count("input-file") != 1)
    {
        cerr << "Usage: Tester [options] input-file.tst" << endl;
        cout << desc;
        return 1;
    }
 
    bool verbose = vm.count("verbose");
 
    // Set up set containing metrics to be generated.
    vector<int> metricGenVec;
    if (vm.count("generate-metric"))
    {
        metricGenVec = vm["generate-metric"].as<vector<int>>();
    }
    set<int> metricGen(metricGenVec.begin(), metricGenVec.end());
 
    // Path to test definition file
    const fs::path specFile(vm["input-file"].as<string>());
 
    // Parent path of test definition file containing dependent files
    fs::path specPath = specFile.parent_path();
 
    if (specPath.empty())
    {
        specPath = fs::current_path();
    }
 
    string specFileStem = specFile.stem().string();
 
    // Temporary master directory to create which holds master output and error
    // files, and the working directories for each run
    const fs::path masterDir =
        fs::current_path() / LibUtilities::UniquePath(specFileStem);
 
    // The current directory
    const fs::path startDir = fs::current_path();
 
    try
    {
        if (verbose)
        {
            cerr << "Reading test file definition: " << specFile << endl;
        }
 
        // Parse the test file
        TestData file(specFile, vm);
 
        if (verbose && file.GetNumMetrics() > 0)
        {
            cerr << "Creating metrics:" << endl;
        }
 
        // Generate the metric objects
        vector<MetricSharedPtr> metrics;
        for (unsigned int i = 0; i < file.GetNumMetrics(); ++i)
        {
            set<int>::iterator it = metricGen.find(file.GetMetricId(i));
            bool genMetric =
                it != metricGen.end() || (vm.count("generate-all-metrics") > 0);
 
            metrics.push_back(GetMetricFactory().CreateInstance(
                file.GetMetricType(i), file.GetMetric(i), genMetric));
 
            if (verbose)
            {
                cerr << "  - ID " << metrics.back()->GetID() << ": "
                     << metrics.back()->GetType() << endl;
            }
 
            if (it != metricGen.end())
            {
                metricGen.erase(it);
            }
        }
 
        if (metricGen.size() != 0)
        {
            string s = metricGen.size() == 1 ? "s" : "";
            set<int>::iterator it;
            cerr << "Unable to find metric" + s + " with ID" + s + " ";
            for (it = metricGen.begin(); it != metricGen.end(); ++it)
            {
                cerr << *it << " ";
            }
            cerr << endl;
            return 1;
        }
 
        // Remove the master directory if left from a previous test
        if (fs::exists(masterDir))
        {
            fs::remove_all(masterDir);
        }
 
        if (verbose)
        {
            cerr << "Creating master directory: " << masterDir << endl;
        }
 
        // Create the master directory
        fs::create_directory(masterDir);
 
        // Change working directory to the master directory
        fs::current_path(masterDir);
 
        // Create a master output and error file. Output and error files from
        // all runs will be appended to these files.
        fstream masterOut("master.out", ios::out | ios::in | ios::trunc);
        fstream masterErr("master.err", ios::out | ios::in | ios::trunc);
 
        if (masterOut.bad() || masterErr.bad())
        {
            cerr << "One or more master output files are unreadable." << endl;
            throw 1;
        }
 
        // Vector of temporary subdirectories to create and conduct tests in
        vector<fs::path> tmpWorkingDirs;
        string line;
 
        for (unsigned int i = 0; i < file.GetNumRuns(); ++i)
        {
            command = "";
 
            if (verbose)
            {
                cerr << "Starting run " << i << "." << endl;
            }
 
            // Temporary directory to create and in which to hold the run
            const fs::path tmpDir =
                masterDir / fs::path("run" + std::to_string(i));
            tmpWorkingDirs.push_back(tmpDir);
 
            if (verbose)
            {
                cerr << "Creating working directory: " << tmpDir << endl;
            }
 
            // Create temporary directory
            fs::create_directory(tmpDir);
 
            // Change working directory to the temporary directory
            fs::current_path(tmpDir);
 
            if (verbose && file.GetNumDependentFiles())
            {
                cerr << "Copying required files: " << endl;
            }
 
            // Copy required files for this test from the test definition
            // directory to the temporary directory.
            for (unsigned int j = 0; j < file.GetNumDependentFiles(); ++j)
            {
                fs::path source_file(file.GetDependentFile(j).m_filename);
 
                fs::path source = specPath / source_file;
                fs::path dest   = tmpDir / source_file.filename();
                if (verbose)
                {
                    cerr << "  - " << source << " -> " << dest << endl;
                }
 
                if (fs::is_directory(source))
                {
                    fs::create_directory(dest);
                    // If source is a directory, then only directory name is
                    // created, so call copy again to copy files.
                    for (const auto &dirEnt :
                         fs::recursive_directory_iterator{source})
                    {
                        fs::path newdest = dest / dirEnt.path().filename();
                        fs::copy_file(dirEnt.path(), newdest);
                    }
                }
                else
                {
                    fs::copy_file(source, dest);
                }
            }
 
            // Copy opt file if exists to  to the temporary directory.
            fs::path source_file("test.opt");
            fs::path source  = specPath / source_file;
            bool HaveOptFile = false;
            if (fs::exists(source))
            {
                fs::path dest = tmpDir / source_file.filename();
                if (verbose)
                {
                    cerr << "  - " << source << " -> " << dest << endl;
                }
 
                if (fs::is_directory(source))
                {
                    fs::create_directory(dest);
                    // If source is a directory, then only directory name is
                    // created, so call copy again to copy files.
                    for (const auto &dirEnt :
                         fs::recursive_directory_iterator{source})
                    {
                        fs::path newdest = dest / dirEnt.path().filename();
                        fs::copy_file(dirEnt.path(), newdest);
                    }
                }
                else
                {
                    fs::copy_file(source, dest);
                }
 
                HaveOptFile = true;
            }
 
            // If we're Python, copy script too.
 
            // Set PYTHONPATH environment variable in case Python is run inside
            // any of our tests.  For non-Python tests this will do nothing.
            setenv("PYTHONPATH", "@NEKPY_BASE_DIR@", true);
 
            // Construct test command to run. Output from stdout and stderr are
            // directed to the files output.out and output.err, respectively.
 
            bool mpiAdded = false;
            for (unsigned int j = 0; j < file.GetNumCommands(); ++j)
            {
                Command cmd = file.GetCommand(j);
 
#ifdef NEKTAR_TEST_FORCEMPIEXEC
#else
                if (cmd.m_processes > 1 || (file.GetNumCommands() > 1 &&
                                            cmd.m_commandType == eParallel))
#endif
                {
                    if (mpiAdded)
                    {
                        continue;
                    }
 
                    command += "\"@MPIEXEC@\" ";
                    if (std::string("@NEKTAR_TEST_USE_HOSTFILE@") == "ON")
                    {
                        command += "-hostfile hostfile ";
#if (NEKTAR_MPI_TYPE == 1) // MPICH
                        if (system("echo 'localhost:12' > hostfile"))
#else
                        if (system("echo 'localhost slots=12' > hostfile"))
#endif
                        {
                            cerr << "Unable to write 'hostfile' in path '"
                                 << fs::current_path() << endl;
                            status = 1;
                        }
                    }
 
                    if (file.GetNumCommands() > 1)
                    {
#if (NEKTAR_MPI_TYPE == 1)
                        // MPICH prepends the rank to each cout, causing the
                        // rank annotation to appear in the middle of the output
                        // and not just at the beginning of lines.
                        // This causes tests not to pass as the tester fails to
                        // parse the output correctly.
//                        command += "--prepend-rank ";
#else
                        command += "--tag-output ";
#endif
                    }
 
                    mpiAdded = true;
                }
            }
 
            // Parse commands.
            for (unsigned int j = 0; j < file.GetNumCommands(); ++j)
            {
                Command cmd = file.GetCommand(j);
 
                // If running with multiple commands simultaneously, separate
                // with colon.
                if (j > 0 && cmd.m_commandType == eParallel)
                {
                    command += " : ";
                }
                else if (j > 0 && cmd.m_commandType == eSequential)
                {
                    command += " && ";
                    if (cmd.m_processes > 1)
                    {
                        command += "\"@MPIEXEC@\" ";
                        if (std::string("@NEKTAR_TEST_USE_HOSTFILE@") == "ON")
                        {
                            command += "-hostfile hostfile ";
                        }
                    }
                }
 
                // Add -n where appropriate.
                if (cmd.m_processes > 1 || (file.GetNumCommands() > 1 &&
                                            cmd.m_commandType == eParallel))
                {
                    command += "@MPIEXEC_NUMPROC_FLAG@ ";
                    command += std::to_string(cmd.m_processes) + " ";
                }
 
                // Look for executable or Python script.
                fs::path execPath = startDir / cmd.m_executable;
                if (!fs::exists(execPath))
                {
                    ASSERTL0(!cmd.m_pythonTest, "Python script not found.");
                    execPath = cmd.m_executable;
                }
 
                // Prepend script name with Python executable path if this is a
                // Python test.
                if (cmd.m_pythonTest)
                {
                    command += "@Python3_EXECUTABLE@ ";
                }
 
                std::string pathString = LibUtilities::PortablePath(execPath);
                command += pathString;
                if (HaveOptFile)
                {
                    command += " --use-opt-file test.opt ";
                }
 
                command += " ";
                command += cmd.m_parameters;
                command += " 1>output.out 2>output.err";
            }
 
            status = 0;
 
            if (verbose)
            {
                cerr << "Running command: " << command << endl;
            }
 
            // Run executable to perform test.
            if (system(command.c_str()))
            {
                cerr << "Error occurred running test:" << endl;
                cerr << "Command: " << command << endl;
                status = 1;
            }
 
            // Check output files exist
            if (!(fs::exists("output.out") && fs::exists("output.err")))
            {
                cerr << "One or more test output files are missing." << endl;
                throw 1;
            }
 
            // Open output files and check they are readable
            ifstream vStdout("output.out");
            ifstream vStderr("output.err");
            if (vStdout.bad() || vStderr.bad())
            {
                cerr << "One or more test output files are unreadable." << endl;
                throw 1;
            }
 
            // Append output to the master output and error files.
            if (verbose)
            {
                cerr << "Appending run " << i << " output and error to master."
                     << endl;
            }
 
            while (getline(vStdout, line))
            {
                masterOut << line << endl;
            }
 
            while (getline(vStderr, line))
            {
                masterErr << line << endl;
            }
 
            vStdout.close();
            vStderr.close();
        }
 
        // Warn user if any metrics don't support multiple runs.
        for (int i = 0; i < metrics.size(); ++i)
        {
            if (!metrics[i]->SupportsAverage() && file.GetNumRuns() > 1)
            {
                cerr << "WARNING: Metric " << metrics[i]->GetType()
                     << " does not support multiple runs. Test may yield "
                        "unexpected results."
                     << endl;
            }
        }
 
        // Test against all metrics
        if (status == 0)
        {
            if (verbose && metrics.size())
            {
                cerr << "Checking metrics:" << endl;
            }
 
            for (int i = 0; i < metrics.size(); ++i)
            {
                bool gen =
                    metricGen.find(metrics[i]->GetID()) != metricGen.end() ||
                    (vm.count("generate-all-metrics") > 0);
 
                masterOut.clear();
                masterErr.clear();
                masterOut.seekg(0, ios::beg);
                masterErr.seekg(0, ios::beg);
 
                if (verbose)
                {
                    cerr << "  - " << (gen ? "generating" : "checking")
                         << " metric " << metrics[i]->GetID() << " ("
                         << metrics[i]->GetType() << ")... ";
                }
 
                if (!metrics[i]->Test(masterOut, masterErr))
                {
                    status = 1;
                    if (verbose)
                    {
                        cerr << "failed!" << endl;
                    }
                }
                else if (verbose)
                {
                    cerr << "passed" << endl;
                }
            }
        }
 
        if (verbose)
        {
            cerr << endl << endl;
        }
 
        // Dump output files to terminal for debugging purposes on fail.
        if (status == 1 || verbose)
        {
            masterOut.clear();
            masterErr.clear();
            masterOut.seekg(0, ios::beg);
            masterErr.seekg(0, ios::beg);
 
            cout << "=== Output ===" << endl;
            while (masterOut.good())
            {
                getline(masterOut, line);
                cout << line << endl;
            }
            cout << "=== Errors ===" << endl;
            while (masterErr.good())
            {
                getline(masterErr, line);
                cout << line << endl;
            }
        }
 
        // Close output files.
        masterOut.close();
        masterErr.close();
 
        // Change back to the original path and delete temporary directory.
        fs::current_path(startDir);
 
        if (verbose)
        {
            cerr << "Removing working directory" << endl;
        }
 
        // Repeatedly try deleting directory with sleep for filesystems which
        // work asynchronously. This allows time for the filesystem to register
        // the output files are closed so they can be deleted and not cause a
        // filesystem failure. Attempts made for 1 second.
        int i = 1000;
        while (i > 0)
        {
            try
            {
                // If delete successful, stop trying.
                fs::remove_all(masterDir);
                break;
            }
            catch (const fs::filesystem_error &e)
            {
                using namespace std::chrono_literals;
                std::this_thread::sleep_for(1ms);
                i--;
                if (i > 0)
                {
                    cout << "Locked files encountered. "
                         << "Retrying after 1ms..." << endl;
                }
                else
                {
                    // If still failing after 1sec, we consider it a permanent
                    // filesystem error and abort.
                    throw e;
                }
            }
        }
 
        // Save any changes.
        if (vm.count("generate-metric") > 0 ||
            vm.count("generate-all-metrics") > 0)
        {
            file.SaveFile();
        }
 
        // Return status of test. 0 = PASS, 1 = FAIL
        return status;
    }
    catch (const fs::filesystem_error &e)
    {
        cerr << "Filesystem operation error occurred:" << endl;
        cerr << "  " << e.what() << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
    catch (const TesterException &e)
    {
        cerr << "Error occurred during test:" << endl;
        cerr << "  " << e.what() << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
    catch (const std::exception &e)
    {
        cerr << "Unhandled exception during test:" << endl;
        cerr << "  " << e.what() << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
    catch (...)
    {
        cerr << "Unknown error during test" << endl;
        cerr << "  Files left in " << masterDir.string() << endl;
    }
 
    // If a system error, return 2
    return 2;
}