/**
 * Class ITimingTest: Used to measure the time for a specific operation to complete.
 *
 * @package Test Framework
 * @category Testing
 *
 * @author Alan Liu
 * @task Initial Test Framework
 * @author Esther Tong
 * @task TestFrameworkLite
 * @author David McCusker, Christoper Miller, Carol Widra, and Kwansook Nam
 * @task Many of the other Contributors to TestFramework
 *
 * @copyright
 *      IBM Open Class Library
 *      (C) Copyright International Business Machines Corporation 1997
 *      Licensed Material - Program-Property of IBM - All Rights Reserved.
 *
 */
// Revision: 34 1.8.1.4 source/core/testfw/itimetst.hpp, testfw, ioc.v400, 980918 

#ifndef _ITIMETST_
#define _ITIMETST_

#include "itest.hpp"

#if __IBMCPP__ >= 400
#pragma namemangling(compat)
#endif

class IStatisticalSet;
class ISimpleClock;

#pragma enum(4)
#pragma pack(push,4)

/**
 * An Abstract base class that represents the tests that measure the median and time
 * required to execute a well-defined operation. It is the main class used to measure
 * CPU performance.
 *
 * Instead of the usual setup, test, and cleanup framework methods, ITimingTest has
 * timingSetup, timingTest, and timingCleanup framework methods.  These methods
 * behave somewhat differently from their namesakes.  Subclasses place the code
 * to be timed in timingTest.  timingSetup and timingCleanup allow subclasses to do
 * any setup and cleanup for the timingTest method; code in these methods is NOT
 * timed.
 *
 * The ITimingTest base class enables you to write tests that measure the time
 * taken by a specific operation to complete.  The derived class
 * IStartStopTimingTest allows you more control in timing events.
 *
 * ITimingTest expects the operation to show some variation in execution time
 * due to multitasking, varying CPU load, and other variable conditions.
 * Consequently, the class takes multiple measurements and reports the median.
 *
 * ITimingTest measures the time needed to execute the TimingTest member
 * function.  It does this by taking some number of samples, the "sample count".
 * Each sample consists of a call to TimingSetup, some number of calls to
 * TimingTest, and a call to TimingCleanup.
 *
 * The calls to TimingTest happen in a tight loop, the "timing loop".  Derived
 * classes, or the caller through a RunTest option, can control how many
 * iterations comprise the timing loop by changing the "timing count".  The
 * total time taken by the timing loop is divided by the number of iterations
 * to get a single "sample time".  The median of all sample times is taken as
 * the estimate of the time to execute the TimingTest member function.
 *
 *<PRE>
 * To create a timed test:
 *     1. Derive your test class from ITimingTest.
 *     2. Override TimingSetup, if necessary.
 *     3. Override TimingTest.
 *     4. Override TimingCleanup, if necessary.
 *
 * Default settings:
 *
 *      kDefaultTimingCount = 500,
 *      kDefaultSampleCount = 10,
 *      kDefaultMaximumSampleWindows = 10
 *
 *      kDefaultAutomaticSamplingTolerance = 0.1
 *      kDefaultMinimumCorrelation = .4
 *</PRE>
 *
 * @see IStartStopTimingTest
 *
 */
class ITimingTest : public ITest {
public:

//-------------------------------------------------------------------------
/** @group Destruction and Assignment */

/**
 * Destructor.
 */
virtual ~ITimingTest ();


/**
 * Standard copy constructor.
 */
ITimingTest (const ITimingTest& source);


/**
 * Standard assignment operator.
 */
ITimingTest& operator= (const ITimingTest& rhs);


//-------------------------------------------------------------------------
// ITest Overrides
//-------------------------------------------------------------------------

/**
 * Adds information about input syntax and a timing test description.
 * Can be overridden to provide metainformation.
 */
virtual void copyInfo () ;


//-------------------------------------------------------------------------
// Timing Methods
//-------------------------------------------------------------------------

/**
 * Controls whether timing operations are done for the timingTest method.
 *
 * @param timingOn A bool.  If true, multiple samples are taken, each sample
 * consisting of a call to timingSetup, many calls to timingTest, and a call
 * to timingCleanup. If false, each run makes a single call to timingSetup,
 * timingTest, and timingCleanup.  In addition, echoing to the console is
 * suppressed.
 *
 *@see timingOn
 */
void setTimingOn (bool timingOn);


/**
 * Returns true if timing operations are done for the timingTest.
 * Otherwise returns false.
 *
 * @return bool
 *
 * @see setTimingOn
 */
bool timingOn () const;


/**
 * Sets the number of times the timingTest method will be called in a
 * loop during a single sample.
 *
 * Note: This timing count also applies to the calibration method.
 *
 * @param timingCount    The number of calls to make within a single sample
 *
 * @see timingCount
 */
void setTimingCount (unsigned long timingCount);


/**
 * Returns the number of times timingTest will be called in a loop
 * during a single sample.
 *
 * @return long     Number of times timingTest will be called
 *
 * @see timing setTimingCount
 */
unsigned long timingCount () const;


/**
 * Sets the number of samples that will be taken for timing and calibration.
 * This is also the number of samples that will be taken (in a moving window)
 * for automatic timing.
 *
 * @param sampleCount     The number of samples to take
 *
 * @see sampleCount
 */
void setSampleCount (unsigned long sampleCount);


/**
 * Returns the number of samples that will be taken for timing and calibration.
 *
 * @return long     Number of samples that will be taken
 *
 * @see setSampleCount
 */
unsigned long sampleCount () const;


/**
 * Returns the number of samples actually taken for a full test.
 *
 * @returns long     The number of samples actually taken
 */
unsigned long samplesTaken () const;


/**
 * Returns the number of samples discarded in the case that automatic timing is on.
 *
 * @return long     The number of samples discarded
 */
unsigned long samplesDiscarded () const;


/**
 * This method sets a pointer to point at an ordered array of each sample value.
 *
 * Note: The caller owns its storage for the elements in the array.
 *
 * @param count        The number of elements of the array pointed to
 * @param double *&    the array pointer to set
 */
void copySamples(long& count, double *&) const;  //caller owns storage


/**
 * Controls whether calibrate is called for each sample.
 *
 * Continual calibration is off by default under the assumption that
 * the empty method time is relatively constant.  If the client or
 * subclass is varying the load, then continual calibration may provide
 * more accurate timing results.
 *
 * @param continuallyCalibrate     bool; If true, timing is set to be continually calibrated
 *
 * @see continualCalibration
 */
void setContinualCalibration (bool continuallyCalibrate);


/**
 * Returns true if calibrate is called for each sample; false otherwise.
 *
 * @return bool     True if calibrate is called for each sample; false otherwise
 *
 * @see setContinualCalibration
 */
bool continualCalibration () const;


/**
 * Sets the time limit for this timing test.  If the final calibrated
 * median time exceeds the specified time limit, then the test is
 * considered a failure and setSuccess(false) is called.  If
 * setMaximumAllowedMedianMicroseconds is never called (the default
 * case) then no time limit will be set, and all runs will be reported
 * to be passing tests.
 *
 * @param timeLimit     Maximum allowed time
 *
 * @see maximumAllowedMedianMicroseconds
 */
void setMaximumAllowedMedianMicroseconds (double timeLimit);


/**
 * Returns the time limit for this timing test.
 *
 * @return double     The time limit for this timing test.
 *
 * @see setMaximumAllowedMedianSeconds
 */
double maximumAllowedMedianMicroseconds () const;


/**
 * Returns the estimated median time for the timingTest method to
 * execute minus the empty method time, in microseconds.
 *
 * @return The estimated median time for the timingTest method
 */
double calibratedMedianMicroseconds ();


/**
 * Returns the mean of all sampled times.
 *
 * @return double     The mean of all sampled times
 */
double calibratedMeanMicroseconds ();


/**
 * Returns the ratio of the 95% confidence interval for the mean
 * (as returned by errorMicroseconds() ) to the mean itself.
 *
 * @return double     The ratio of the 95% confidence interval
 */
double relativeErrorMicroseconds () const;


/**
 * Returns the correlation between samples.
 *
 * @return double     The correlation between samples
 */
double correlationMicroseconds () const;


/**
 * Returns the 95% confidence interval for the test.
 *
 * @return double     The 95% confidence interval for the test
 */
double errorMicroseconds () const;


/**
 * Returns true if the relative error from relativeErrorMicroseconds is
 * greater than the tolerance specified by the tester on the command line
 * or by calling setAutomaticSampleTolerance.
 *
 * @return bool
 */
bool errorTooLarge () const;


/**
 * Returns true if the correlation between samples exceeds the minimum
 * correlation specified by the user (using setMinimumSampleCorrelation).
 *
 * @return bool
 *
 * @see setMinimumSampleCorrelation
 */
bool correlationTooLarge () const;


/**
 * Measures the empty method time.
 */
void calibrate ();


/**
 * Returns the overhead required to read the clock.  This is a measured
 * value used to compute other error tolerances.
 *
 * @return double    Overhead required to read the clock
 */
double clockNowOverhead () const;

//-------------------------------------------------------------------------
/** @group Automatic Sampling Methods */


/**
 * Controls whether automatic sampling is done.
 *
 * @param doAutomaticSampling   If true, samples are teken until the error
 * on the measured mean reaches a desired percentage, the tolerance.  This
 * allows the operation being timed to "settle" if the first measurements
 * being taken are too high or low for some reason.  Samples taken prior to
 * settling are discarded.  If false, then the first sampleCount samples are
 * always taken and accepted.
 *
 * @see automaticSampling
 */
void setAutomaticSampling (bool doAutomaticSampling);


/**
 * Returns true if automatic sampling is done, false otherwise.
 *
 * @return bool     State of automatic sampling setting
 *
 * @see setAutomaticSampling
 */
bool automaticSampling () const;


/**
 * Sets the threshold value for automatic sampling.  Samples will continue
 * to be taken until the error on the measured mean is below this threshold
 * value.  For example, if the Tolerance is 0.05, then samples will continue
 * to be taken until the error on the mean is less than or equal to 5 percent.
 *
 * @param tolerance     The threshold value for automatic sampling
 *
 * @see automaticSampleTolerance
 */
void setAutomaticSampleTolerance (double tolerance);


/**
 * Returns the threshold value for automatic sampling.
 *
 * @return double   The current threshold value for automatic sampling
 *
 * @see setAutomaticSampleTolerance
 */
double automaticSampleTolerance () const;


/**
 * Sets the maximum number of sample windows that are taken.
 *
 * @param maxWindows     Number of sample windows to take
 *
 * @see maximumSampleWindows
 */
void setMaximumSampleWindows (unsigned long maxWindows);


/**
 * Returns the maximum number of sample windows.
 *
 * @return The maximum number of sample windows
 *
 * @see setMaximumSampleWindows
 */
unsigned long maximumSampleWindows () const;


/**
 * Sets the minimum acceptable correlation between samples.  If
 * the correlation between samples exceeds this number, a message
 * will be printed and the actual correlation will be posted.
 *
 * @param minSampleCorr    The minimum correlation:  0 < minSampleCorr < 1
 *
 * @see minimumSampleCorrelation
 */
void setMinimumSampleCorrelation (double minSampleCorr);


/**
 * Returns the minimum acceptable correlation between samples.
 *
 * @return The minimum acceptable correlation between samples
 *
 * @see setMinimumSampleCorrelation
 */
double minimumSampleCorrelation () const;


//-------------------------------------------------------------------------
// Default Constants
//-------------------------------------------------------------------------
enum
{
        kDefaultTimingCount = 500,
        kDefaultSampleCount = 10,
        kDefaultMaximumSampleWindows = 10
};

static const double  kDefaultAutomaticSamplingTolerance; // = 0.1
static const double  kDefaultMinimumCorrelation; // = .4

protected:
//------------------------------------------------------------------------
/** @group Constructors */


/**
 * This default constructor sets the timing count and sample count to default
 * values, with no maximum allowed median time.
 *
 * Constructs an ITimingTest specifying a timing count, sample count,
 * maximum sample count, minimum correlation, and an automatic sample
 * tolerance.  The default values are:
 *
 *<PRE>
 *      kDefaultTimingCount = 500,
 *      kDefaultSampleCount = 10,
 *      kDefaultMaximumSampleWindows = 10
 *      kDefaultAutomaticSamplingTolerance = 0.1
 *      kDefaultMinimumCorrelation = .4
 *      automatic sampling ON
 *      no maximum allowed median time
 *</PRE>
 */
ITimingTest ();


/**
 * Constructor that explicitly sets timing count and sample count.
 */
ITimingTest (unsigned long timingCount, unsigned long sampleCount);


//-------------------------------------------------------------------------
/** @group Timing Framework Methods */


/**
 * Performs any required setup for n consecutive calls to timingTest, where n
 * is the timing count.  Called by the framework once per sample.  Subclasses
 * must override timingSetup if any setup is necessary before calls to
 * timingTest.
 *
 * @override sometimes
 */
 virtual void timingSetup ();


/**
 * A framework method for timing tests.  Subclasses must override timingTest and
 * place the code to be timed in the overridden method.  If this code requires
 * any setup and/or cleanup, subclasses should override timingSetup and
 * timingCleanup and place the appropriate code in each.
 *
 * For each timing sample, ITimingTest calls timingSetup once, followed by
 * timingCount calls to timingTest, followed by a call to timingCleanup.  The
 * loop of calls to timingTest is timed, and this time is divided by timingCount
 * to get the uncalibrated sample time.  The median empty time is subtracted
 * from this to get the calibrated sample time.
 *
 * @override always
 */
virtual void timingTest () = 0;     // Subclass must override


/**
 * Performs any required cleanup for n consecutive calls to timingTest, where n
 * is the timing count.  Called by the framework once per sample.  Subclasses
 * must override timingCleanup if any cleanup is necessary after calls to
 * timingTest.
 *
 * @override sometimes
 */
virtual void timingCleanup ();


//-------------------------------------------------------------------------
// ITest Overrides: Subclasses May Override These
//-------------------------------------------------------------------------

/**
 * A framework setup method.  It is called once before the test is run.
 * The test should override this function to execute any setup code that
 * only needs to be called once.
 *
 * It is also a good place to parse command line arguments.  If this
 * method is overridden it must call the parent ITimingTest::setup
 * method.
 *
 * @override sometimes
 * @callup yes
 */
virtual void setup ();


/**
 * This is a framework cleanup method.  It is called once after the test
 * is run and should contain whatever code is necessary to cleanup after
 * setup.  It also must call the parent ITimingTest::cleanup method.
 *
 * @override sometimes
 * @callup yes
 */
virtual void cleanup ();

//-------------------------------------------------------------------------
// ITest Overrides: Subclasses Should NOT Override These
//-------------------------------------------------------------------------

/**
 * A framework test method. ITimingTest overrides the ITest::test method;
 * subtests of ITimingTest should not override it.  ITimingTest::test first
 * calls calibrate if this has not been done before. It then takes sampleCount
 * timing samples if automatic sampling is off, or automatically takes
 * enough samples to achieve the desired tolerance, as returned by
 * automaticSamplingTolerance, up to a maximym of maximymSampleCount samples.
 *
 * Note: Do NOT override this method!
 */
virtual void test ();               //Do not override this method


/**
 * Prints information about the object to the given tiered text buffer.
 *
 * @param out     The buffer where information about the test is printed.
 */
virtual void print (ITieredTextBuffer& out);


//-------------------------------------------------------------------------
// Advanced Subclass Specialization Methods: Authorized Personnel Only
//-------------------------------------------------------------------------

/** @group Advanced Subclass Specialization Methods */


/**
 * Determine the time it takes to run one sample of the timingTest.
 *
 * @return theTime    The time to run the one sample
 */
virtual double oneSample ();


/**
 * Run the timingTest without performing the timing.  (e.g., execute only
 * timingSetup, timingTest,  and timingCleanup). Useful to debug the
 * timingTest without concern for the timing aspects.
 */
virtual void   doUntimedTest ();


/**
 * Runs an empty sample.  Useful to determine the amount of time necessary
 * for the overhead of running timingTestt, the function call to an empty
 * test function (e.g., timingTest return 0.0), and timingCleanup.
 *
 * @return theTime     The time to run the empty sample
 */
virtual double oneEmptySample ();


/**
 * Returns the sampling error.
 *
 * @return double     The sampling error
 */
virtual double samplingErrorMicroseconds () const;


//-------------------------------------------------------------------------

private:
//-------------------------------------------------------------------------
// ITimingTest private functions
//-------------------------------------------------------------------------
bool   parseArguments ();
double calibration ();
double emptyTimingTest ();
void   copyDataMembers (const ITimingTest& other);

//-------------------------------------------------------------------------
// test variables
//-------------------------------------------------------------------------
static const double fgDouble;
static const double *fgDoublePointer;

unsigned long fTimingCount;
unsigned long fSampleCount;
unsigned long fMaximumSampleWindows;
unsigned long fDiscardedSamples;
unsigned long fSamplesTaken;
unsigned long fCurrentTimingCount;
double fCalibratedMedianMicroseconds;
double fCalibratedMeanMicroseconds;
double fClockNowOverhead;
double fEmptyMedianTime;
double fAutomaticSamplingTolerance;
double fMinimumSampleCorrelation;
double fMaxAllowedCalibratedMedianTime;
bool fContinuallyCalibrate;
bool fIsCalibrated;
bool fDoAutomaticSampling;
bool fTimingIsOn;

ISimpleClock* fClock;
IStatisticalSet* fCalibrationSamples;
IStatisticalSet* fSamples;

friend class IAutoSamplingTest;
};

#pragma pack(pop)
#pragma enum(pop)

#if __IBMCPP__ >= 400
#pragma namemangling()
#endif

#endif   // _ITIMETST_
