mirror of
https://github.com/beefytech/Beef.git
synced 2025-06-08 19:48:20 +02:00
Test improvements (continue after fail, console output, error location)
This commit is contained in:
parent
cadd1f809f
commit
78ae79b802
7 changed files with 356 additions and 68 deletions
|
@ -376,6 +376,7 @@ namespace System
|
||||||
public bool ShouldFail;
|
public bool ShouldFail;
|
||||||
public bool Ignore;
|
public bool Ignore;
|
||||||
public bool Profile;
|
public bool Profile;
|
||||||
|
public String Tag;
|
||||||
}
|
}
|
||||||
|
|
||||||
public struct ImportAttribute : Attribute
|
public struct ImportAttribute : Attribute
|
||||||
|
|
|
@ -68,14 +68,18 @@ namespace System
|
||||||
{
|
{
|
||||||
if (outStreamWriter == null)
|
if (outStreamWriter == null)
|
||||||
{
|
{
|
||||||
|
Stream stream;
|
||||||
|
#if BF_TEST_BUILD
|
||||||
|
stream = new Test.TestStream();
|
||||||
|
#else
|
||||||
FileStream fileStream = new .();
|
FileStream fileStream = new .();
|
||||||
Stream stream = fileStream;
|
stream = fileStream;
|
||||||
if (fileStream.OpenStd(stdKind) case .Err)
|
if (fileStream.OpenStd(stdKind) case .Err)
|
||||||
{
|
{
|
||||||
DeleteAndNullify!(fileStream);
|
DeleteAndNullify!(fileStream);
|
||||||
stream = new NullStream();
|
stream = new NullStream();
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
StreamWriter newStreamWriter = new StreamWriter(stream, InputEncoding, 4096, true);
|
StreamWriter newStreamWriter = new StreamWriter(stream, InputEncoding, 4096, true);
|
||||||
newStreamWriter.AutoFlush = true;
|
newStreamWriter.AutoFlush = true;
|
||||||
|
|
||||||
|
|
|
@ -108,6 +108,10 @@ namespace System
|
||||||
[CallingConvention(.Cdecl)]
|
[CallingConvention(.Cdecl)]
|
||||||
static extern void Test_Init(char8* testData);
|
static extern void Test_Init(char8* testData);
|
||||||
[CallingConvention(.Cdecl)]
|
[CallingConvention(.Cdecl)]
|
||||||
|
static extern void Test_Error(char8* error);
|
||||||
|
[CallingConvention(.Cdecl)]
|
||||||
|
static extern void Test_Write(char8* str);
|
||||||
|
[CallingConvention(.Cdecl)]
|
||||||
static extern int32 Test_Query();
|
static extern int32 Test_Query();
|
||||||
[CallingConvention(.Cdecl)]
|
[CallingConvention(.Cdecl)]
|
||||||
static extern void Test_Finish();
|
static extern void Test_Finish();
|
||||||
|
|
|
@ -1,23 +1,78 @@
|
||||||
|
using System.IO;
|
||||||
namespace System
|
namespace System
|
||||||
{
|
{
|
||||||
class Test
|
class Test
|
||||||
{
|
{
|
||||||
[NoReturn]
|
public class TestStream : Stream
|
||||||
public static void FatalError(String msg = "Test fatal error encountered")
|
|
||||||
{
|
{
|
||||||
Internal.FatalError(msg, 1);
|
public override int64 Position
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
Runtime.FatalError();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void Assert(bool condition)
|
set
|
||||||
{
|
{
|
||||||
if (!condition)
|
}
|
||||||
Internal.FatalError("Test Assert failed", 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void Assert(bool condition, String error)
|
public override int64 Length
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
Runtime.FatalError();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override bool CanRead
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override bool CanWrite
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override Result<int> TryRead(Span<uint8> data)
|
||||||
|
{
|
||||||
|
return default;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override Result<int> TryWrite(Span<uint8> data)
|
||||||
|
{
|
||||||
|
String str = scope String();
|
||||||
|
str.Append((char8*)data.Ptr, data.Length);
|
||||||
|
Internal.[Friend]Test_Write(str.CStr());
|
||||||
|
return .Ok(data.Length);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Close()
|
||||||
|
{
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void FatalError(String msg = "Test fatal error encountered", String filePath = Compiler.CallerFilePath, int line = Compiler.CallerLineNum)
|
||||||
|
{
|
||||||
|
String failStr = scope .()..AppendF("{} at line {} in {}", msg, line, filePath);
|
||||||
|
Internal.[Friend]Test_Error(failStr);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void Assert(bool condition, String error = Compiler.CallerExpression[0], String filePath = Compiler.CallerFilePath, int line = Compiler.CallerLineNum)
|
||||||
{
|
{
|
||||||
if (!condition)
|
if (!condition)
|
||||||
Internal.FatalError(error, 2);
|
{
|
||||||
|
String failStr = scope .()..AppendF("Assert failed: {} at line {} in {}", error, line, filePath);
|
||||||
|
Internal.[Friend]Test_Error(failStr);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,6 +55,13 @@ static Beefy::StringT<0> gCmdLineString;
|
||||||
bf::System::Runtime::BfRtCallbacks gBfRtCallbacks;
|
bf::System::Runtime::BfRtCallbacks gBfRtCallbacks;
|
||||||
BfRtFlags gBfRtFlags = (BfRtFlags)0;
|
BfRtFlags gBfRtFlags = (BfRtFlags)0;
|
||||||
|
|
||||||
|
static int gTestMethodIdx = -1;
|
||||||
|
static uint32 gTestStartTick = 0;
|
||||||
|
static bool gTestBreakOnFailure = false;
|
||||||
|
|
||||||
|
static BfpFile* gClientPipe = NULL;
|
||||||
|
static Beefy::String gTestInBuffer;
|
||||||
|
|
||||||
namespace bf
|
namespace bf
|
||||||
{
|
{
|
||||||
namespace System
|
namespace System
|
||||||
|
@ -101,6 +108,8 @@ namespace bf
|
||||||
|
|
||||||
private:
|
private:
|
||||||
BFRT_EXPORT static void Test_Init(char* testData);
|
BFRT_EXPORT static void Test_Init(char* testData);
|
||||||
|
BFRT_EXPORT static void Test_Error(char* error);
|
||||||
|
BFRT_EXPORT static void Test_Write(char* str);
|
||||||
BFRT_EXPORT static int32 Test_Query();
|
BFRT_EXPORT static int32 Test_Query();
|
||||||
BFRT_EXPORT static void Test_Finish();
|
BFRT_EXPORT static void Test_Finish();
|
||||||
};
|
};
|
||||||
|
@ -201,6 +210,24 @@ bool IsDebuggerPresent()
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
static void TestString(const StringImpl& str);
|
||||||
|
static void TestReadCmd(Beefy::String& str);
|
||||||
|
|
||||||
|
static void Internal_FatalError(const char* error)
|
||||||
|
{
|
||||||
|
if (gClientPipe != NULL)
|
||||||
|
{
|
||||||
|
Beefy::String str = ":TestFatal\t";
|
||||||
|
str += error;
|
||||||
|
str += "\n";
|
||||||
|
TestString(str);
|
||||||
|
|
||||||
|
Beefy::String result;
|
||||||
|
TestReadCmd(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
BfpSystem_FatalError(error, "BEEF FATAL ERROR");
|
||||||
|
}
|
||||||
|
|
||||||
extern "C" BFRT_EXPORT int BF_CALLTYPE ftoa(float val, char* str)
|
extern "C" BFRT_EXPORT int BF_CALLTYPE ftoa(float val, char* str)
|
||||||
{
|
{
|
||||||
|
@ -265,7 +292,7 @@ void bf::System::Runtime::Init(int version, int flags, BfRtCallbacks* callbacks)
|
||||||
|
|
||||||
if (gBfRtCallbacks.Alloc != NULL)
|
if (gBfRtCallbacks.Alloc != NULL)
|
||||||
{
|
{
|
||||||
BfpSystem_FatalError(StrFormat("BeefRT already initialized. Multiple executable modules in the same process cannot dynamically link to the Beef runtime.").c_str(), "BEEF FATAL ERROR");
|
Internal_FatalError(StrFormat("BeefRT already initialized. Multiple executable modules in the same process cannot dynamically link to the Beef runtime.").c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (version != BFRT_VERSION)
|
if (version != BFRT_VERSION)
|
||||||
|
@ -394,7 +421,7 @@ void Internal::ThrowIndexOutOfRange(intptr stackOffset)
|
||||||
BF_DEBUG_BREAK();
|
BF_DEBUG_BREAK();
|
||||||
}
|
}
|
||||||
|
|
||||||
BfpSystem_FatalError("Index out of range", "FATAL ERROR");
|
Internal_FatalError("Index out of range");
|
||||||
}
|
}
|
||||||
|
|
||||||
void Internal::FatalError(bf::System::String* error, intptr stackOffset)
|
void Internal::FatalError(bf::System::String* error, intptr stackOffset)
|
||||||
|
@ -405,7 +432,7 @@ void Internal::FatalError(bf::System::String* error, intptr stackOffset)
|
||||||
BF_DEBUG_BREAK();
|
BF_DEBUG_BREAK();
|
||||||
}
|
}
|
||||||
|
|
||||||
BfpSystem_FatalError(error->CStr(), "FATAL ERROR");
|
Internal_FatalError(error->CStr());
|
||||||
}
|
}
|
||||||
|
|
||||||
void Internal::MemCpy(void* dest, void* src, intptr length)
|
void Internal::MemCpy(void* dest, void* src, intptr length)
|
||||||
|
@ -568,12 +595,6 @@ void Internal::ReportMemory()
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
static int gTestMethodIdx = -1;
|
|
||||||
static uint32 gTestStartTick = 0;
|
|
||||||
|
|
||||||
static BfpFile* gClientPipe = NULL;
|
|
||||||
static Beefy::String gTestInBuffer;
|
|
||||||
|
|
||||||
static void TestString(const StringImpl& str)
|
static void TestString(const StringImpl& str)
|
||||||
{
|
{
|
||||||
BfpFileResult fileResult;
|
BfpFileResult fileResult;
|
||||||
|
@ -626,6 +647,40 @@ void Internal::Test_Init(char* testData)
|
||||||
TestString(outStr);
|
TestString(outStr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Internal::Test_Error(char* error)
|
||||||
|
{
|
||||||
|
if (gTestBreakOnFailure)
|
||||||
|
{
|
||||||
|
SETUP_ERROR(error, 3);
|
||||||
|
BF_DEBUG_BREAK();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (gClientPipe != NULL)
|
||||||
|
{
|
||||||
|
Beefy::String str = ":TestFail\t";
|
||||||
|
str += error;
|
||||||
|
str += "\n";
|
||||||
|
TestString(str);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void Internal::Test_Write(char* strPtr)
|
||||||
|
{
|
||||||
|
if (gClientPipe != NULL)
|
||||||
|
{
|
||||||
|
Beefy::String str = ":TestWrite\t";
|
||||||
|
str += strPtr;
|
||||||
|
for (char& c : str)
|
||||||
|
{
|
||||||
|
if (c == '\n')
|
||||||
|
c = '\r';
|
||||||
|
}
|
||||||
|
|
||||||
|
str += "\n";
|
||||||
|
TestString(str);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
int32 Internal::Test_Query()
|
int32 Internal::Test_Query()
|
||||||
{
|
{
|
||||||
if (gTestMethodIdx != -1)
|
if (gTestMethodIdx != -1)
|
||||||
|
@ -650,7 +705,16 @@ int32 Internal::Test_Query()
|
||||||
if (result == ":TestRun")
|
if (result == ":TestRun")
|
||||||
{
|
{
|
||||||
gTestStartTick = BfpSystem_TickCount();
|
gTestStartTick = BfpSystem_TickCount();
|
||||||
|
Beefy::String options;
|
||||||
|
int tabPos = (int)param.IndexOf('\t');
|
||||||
|
if (tabPos != -1)
|
||||||
|
{
|
||||||
|
options = param.Substring(tabPos + 1);
|
||||||
|
param.RemoveToEnd(tabPos);
|
||||||
|
}
|
||||||
|
|
||||||
gTestMethodIdx = atoi(param.c_str());
|
gTestMethodIdx = atoi(param.c_str());
|
||||||
|
gTestBreakOnFailure = options.Contains("FailBreak");
|
||||||
return gTestMethodIdx;
|
return gTestMethodIdx;
|
||||||
}
|
}
|
||||||
else if (result == ":TestFinish")
|
else if (result == ":TestFinish")
|
||||||
|
@ -791,7 +855,7 @@ void Contract::ReportFailure(Contract::ContractFailureKind failureKind, char* us
|
||||||
gBfRtCallbacks.DebugMessageData_Fatal();
|
gBfRtCallbacks.DebugMessageData_Fatal();
|
||||||
}
|
}
|
||||||
|
|
||||||
BfpSystem_FatalError(errorMsg.c_str(), "CONTRACT ERROR");
|
Internal_FatalError(errorMsg.c_str());
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -324,6 +324,7 @@ namespace IDE
|
||||||
public bool mEnableGCCollect = true;
|
public bool mEnableGCCollect = true;
|
||||||
public bool mDbgFastUpdate;
|
public bool mDbgFastUpdate;
|
||||||
public bool mTestEnableConsole = false;
|
public bool mTestEnableConsole = false;
|
||||||
|
public bool mTestBreakOnFailure = true;
|
||||||
public ProfileInstance mLongUpdateProfileId;
|
public ProfileInstance mLongUpdateProfileId;
|
||||||
public uint32 mLastLongUpdateCheck;
|
public uint32 mLastLongUpdateCheck;
|
||||||
public uint32 mLastLongUpdateCheckError;
|
public uint32 mLastLongUpdateCheckError;
|
||||||
|
@ -5418,9 +5419,15 @@ namespace IDE
|
||||||
AddMenuItem(testRunMenu, "&Normal Tests", "Run Normal Tests");
|
AddMenuItem(testRunMenu, "&Normal Tests", "Run Normal Tests");
|
||||||
AddMenuItem(testRunMenu, "&All Tests", "Run All Tests");
|
AddMenuItem(testRunMenu, "&All Tests", "Run All Tests");
|
||||||
|
|
||||||
|
|
||||||
var testDebugMenu = testMenu.AddMenuItem("&Debug", null, null, new => UpdateMenuItem_DebugStopped_HasWorkspace);
|
var testDebugMenu = testMenu.AddMenuItem("&Debug", null, null, new => UpdateMenuItem_DebugStopped_HasWorkspace);
|
||||||
AddMenuItem(testDebugMenu, "&Normal Tests", "Debug Normal Tests");
|
AddMenuItem(testDebugMenu, "&Normal Tests", "Debug Normal Tests");
|
||||||
AddMenuItem(testDebugMenu, "&All Tests", "Debug All Tests");
|
AddMenuItem(testDebugMenu, "&All Tests", "Debug All Tests");
|
||||||
|
testDebugMenu.AddMenuItem(null);
|
||||||
|
testDebugMenu.AddMenuItem("Break on Failure", null, new (menu) =>
|
||||||
|
{
|
||||||
|
ToggleCheck(menu, ref mTestBreakOnFailure);
|
||||||
|
}, null, null, true, mTestBreakOnFailure ? 1 : 0);
|
||||||
|
|
||||||
AddMenuItem(testMenu, "Enable Console", "Test Enable Console", null, null, true, mTestEnableConsole ? 1 : 0);
|
AddMenuItem(testMenu, "Enable Console", "Test Enable Console", null, null, true, mTestEnableConsole ? 1 : 0);
|
||||||
|
|
||||||
|
@ -12173,6 +12180,14 @@ namespace IDE
|
||||||
{
|
{
|
||||||
if (isFirstMsg)
|
if (isFirstMsg)
|
||||||
{
|
{
|
||||||
|
if (mTestManager != null)
|
||||||
|
{
|
||||||
|
// Give test manager time to flush
|
||||||
|
Thread.Sleep(100);
|
||||||
|
mTestManager.Update();
|
||||||
|
mOutputPanel.Update();
|
||||||
|
}
|
||||||
|
|
||||||
OutputLineSmart(scope String("ERROR: ", errorMsg));
|
OutputLineSmart(scope String("ERROR: ", errorMsg));
|
||||||
if (gApp.mRunningTestScript)
|
if (gApp.mRunningTestScript)
|
||||||
{
|
{
|
||||||
|
@ -13302,7 +13317,7 @@ namespace IDE
|
||||||
if (mTestManager != null)
|
if (mTestManager != null)
|
||||||
{
|
{
|
||||||
mTestManager.Update();
|
mTestManager.Update();
|
||||||
if (mTestManager.mIsDone)
|
if ((mTestManager.mIsDone) && (mTestManager.mQueuedOutput.IsEmpty))
|
||||||
{
|
{
|
||||||
if (mMainFrame != null)
|
if (mMainFrame != null)
|
||||||
mMainFrame.mStatusBar.SelectConfig(mTestManager.mPrevConfigName);
|
mMainFrame.mStatusBar.SelectConfig(mTestManager.mPrevConfigName);
|
||||||
|
|
|
@ -12,6 +12,10 @@ namespace IDE
|
||||||
{
|
{
|
||||||
public Project mProject;
|
public Project mProject;
|
||||||
public String mTestExePath ~ delete _;
|
public String mTestExePath ~ delete _;
|
||||||
|
public int32 mTestCount;
|
||||||
|
public int32 mExecutedCount;
|
||||||
|
public int32 mSkipCount;
|
||||||
|
public int32 mFailedCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
public class TestEntry
|
public class TestEntry
|
||||||
|
@ -23,6 +27,8 @@ namespace IDE
|
||||||
public bool mShouldFail;
|
public bool mShouldFail;
|
||||||
public bool mProfile;
|
public bool mProfile;
|
||||||
public bool mIgnore;
|
public bool mIgnore;
|
||||||
|
public bool mFailed;
|
||||||
|
public bool mExecuted;
|
||||||
}
|
}
|
||||||
|
|
||||||
public class TestInstance
|
public class TestInstance
|
||||||
|
@ -35,7 +41,7 @@ namespace IDE
|
||||||
public String mArgs ~ delete _;
|
public String mArgs ~ delete _;
|
||||||
public String mWorkingDir ~ delete _;
|
public String mWorkingDir ~ delete _;
|
||||||
public NamedPipe mPipeServer ~ delete _;
|
public NamedPipe mPipeServer ~ delete _;
|
||||||
public int mShouldFailIdx = -1;
|
public int32 mCurTestIdx = -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool mIsDone;
|
public bool mIsDone;
|
||||||
|
@ -147,15 +153,73 @@ namespace IDE
|
||||||
|
|
||||||
String clientStr = scope String();
|
String clientStr = scope String();
|
||||||
|
|
||||||
int curTestIdx = -1;
|
|
||||||
int curTestRunCount = 0;
|
int curTestRunCount = 0;
|
||||||
bool testsFinished = false;
|
bool testsFinished = false;
|
||||||
bool failed = false;
|
bool failed = false;
|
||||||
|
int testFailCount = 0;
|
||||||
|
bool testHadOutput = false;
|
||||||
|
|
||||||
int exitCode = 0;
|
int exitCode = 0;
|
||||||
|
|
||||||
|
String queuedOutText = scope .();
|
||||||
|
|
||||||
|
Stopwatch testTimer = scope .();
|
||||||
|
|
||||||
|
void FlushTestStart()
|
||||||
|
{
|
||||||
|
if (testHadOutput)
|
||||||
|
return;
|
||||||
|
|
||||||
|
if ((testInstance.mCurTestIdx >= 0) && (testInstance.mCurTestIdx < testInstance.mTestEntries.Count))
|
||||||
|
{
|
||||||
|
testHadOutput = true;
|
||||||
|
String outputLine = scope String();
|
||||||
|
let testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx];
|
||||||
|
outputLine.AppendF($"Testing '{testEntry.mName}'");
|
||||||
|
QueueOutputLine(outputLine);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void FlushOutText(bool force)
|
||||||
|
{
|
||||||
|
if (queuedOutText.IsEmpty)
|
||||||
|
return;
|
||||||
|
if ((!queuedOutText.EndsWith('\n')) && (force))
|
||||||
|
queuedOutText.Append('\n');
|
||||||
|
int lastCrPos = -1;
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
|
int crPos = queuedOutText.IndexOf('\n', lastCrPos + 1);
|
||||||
|
if (crPos == -1)
|
||||||
|
break;
|
||||||
|
|
||||||
|
FlushTestStart();
|
||||||
|
|
||||||
|
String outputLine = scope String();
|
||||||
|
outputLine.Append(" >");
|
||||||
|
if (crPos - lastCrPos - 2 > 0)
|
||||||
|
outputLine.Append(StringView(queuedOutText, lastCrPos + 1, crPos - lastCrPos - 2));
|
||||||
|
|
||||||
|
QueueOutputLine(outputLine);
|
||||||
|
lastCrPos = crPos;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lastCrPos != -1)
|
||||||
|
queuedOutText.Remove(0, lastCrPos + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
if ((mDebug) && (gApp.mDebugger.IsPaused()))
|
||||||
|
{
|
||||||
|
FlushTestStart();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (testTimer.ElapsedMilliseconds > 1000)
|
||||||
|
{
|
||||||
|
FlushTestStart();
|
||||||
|
}
|
||||||
|
|
||||||
int doneCount = 0;
|
int doneCount = 0;
|
||||||
|
|
||||||
for (int itr < 2)
|
for (int itr < 2)
|
||||||
|
@ -190,34 +254,34 @@ namespace IDE
|
||||||
outStr.AppendF("CMD: {0}", cmd);
|
outStr.AppendF("CMD: {0}", cmd);
|
||||||
QueueOutput(outStr);*/
|
QueueOutput(outStr);*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
List<StringView> cmdParts = scope .(cmd.Split('\t'));
|
List<StringView> cmdParts = scope .(cmd.Split('\t'));
|
||||||
switch (cmdParts[0])
|
switch (cmdParts[0])
|
||||||
{
|
{
|
||||||
case ":TestInit":
|
case ":TestInit":
|
||||||
case ":TestBegin":
|
case ":TestBegin":
|
||||||
case ":TestQuery":
|
case ":TestQuery":
|
||||||
if ((curTestIdx == -1) || (curTestRunCount > 0))
|
testTimer.Stop();
|
||||||
|
testTimer.Start();
|
||||||
|
FlushOutText(true);
|
||||||
|
testHadOutput = false;
|
||||||
|
if ((testInstance.mCurTestIdx == -1) || (curTestRunCount > 0))
|
||||||
{
|
{
|
||||||
curTestIdx++;
|
testInstance.mCurTestIdx++;
|
||||||
curTestRunCount = 0;
|
curTestRunCount = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
curProjectInfo.mTestCount = (.)testInstance.mTestEntries.Count;
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
if (curTestIdx < testInstance.mTestEntries.Count)
|
if (testInstance.mCurTestIdx < testInstance.mTestEntries.Count)
|
||||||
{
|
{
|
||||||
curTestRunCount++;
|
curTestRunCount++;
|
||||||
bool skipEntry = false;
|
bool skipEntry = false;
|
||||||
|
|
||||||
let testEntry = testInstance.mTestEntries[curTestIdx];
|
let testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx];
|
||||||
if (testEntry.mShouldFail)
|
|
||||||
{
|
|
||||||
skipEntry = testInstance.mShouldFailIdx != curTestIdx;
|
|
||||||
}
|
|
||||||
else if (testInstance.mShouldFailIdx != -1)
|
|
||||||
{
|
|
||||||
skipEntry = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((!skipEntry) && (testEntry.mIgnore) && (!mIncludeIgnored))
|
if ((!skipEntry) && (testEntry.mIgnore) && (!mIncludeIgnored))
|
||||||
{
|
{
|
||||||
|
@ -227,13 +291,20 @@ namespace IDE
|
||||||
|
|
||||||
if (skipEntry)
|
if (skipEntry)
|
||||||
{
|
{
|
||||||
curTestIdx++;
|
curProjectInfo.mSkipCount++;
|
||||||
|
testInstance.mCurTestIdx++;
|
||||||
curTestRunCount = 0;
|
curTestRunCount = 0;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
var clientCmd = scope String();
|
curProjectInfo.mExecutedCount++;
|
||||||
clientCmd.AppendF(":TestRun\t{0}\n", curTestIdx);
|
testEntry.mExecuted = true;
|
||||||
|
|
||||||
|
String clientCmd = scope $":TestRun\t{testInstance.mCurTestIdx}";
|
||||||
|
if ((gApp.mTestBreakOnFailure) && (mDebug))
|
||||||
|
clientCmd.Append("\tFailBreak");
|
||||||
|
clientCmd.Append("\n");
|
||||||
|
|
||||||
if (testInstance.mPipeServer.Write(clientCmd) case .Err)
|
if (testInstance.mPipeServer.Write(clientCmd) case .Err)
|
||||||
failed = true;
|
failed = true;
|
||||||
}
|
}
|
||||||
|
@ -245,15 +316,55 @@ namespace IDE
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case ":TestResult":
|
case ":TestResult":
|
||||||
|
testTimer.Stop();
|
||||||
int timeMS = int32.Parse(cmdParts[1]).Get();
|
int timeMS = int32.Parse(cmdParts[1]).Get();
|
||||||
var testEntry = testInstance.mTestEntries[curTestIdx];
|
var testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx];
|
||||||
|
|
||||||
if (testEntry.mShouldFail)
|
if (testEntry.mShouldFail)
|
||||||
{
|
{
|
||||||
QueueOutputLine("ERROR: Test should have failed but didn't: {0} Time: {1}ms", testEntry.mName, timeMS);
|
if (testEntry.mFailed)
|
||||||
failed = true;
|
{
|
||||||
|
QueueOutputLine("Test expectedly failed: {0} Time: {1}ms", testEntry.mName, timeMS);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
QueueOutputLine("Test completed: {0} Time: {1}ms", testEntry.mName, timeMS);
|
{
|
||||||
|
curProjectInfo.mFailedCount++;
|
||||||
|
QueueOutputLine("ERROR: Test should have failed but didn't: '{0}' defined at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (testHadOutput)
|
||||||
|
QueueOutputLine(" Test Time: {1}ms", testEntry.mName, timeMS);
|
||||||
|
else
|
||||||
|
QueueOutputLine("Test '{0}' Time: {1}ms", testEntry.mName, timeMS);
|
||||||
|
}
|
||||||
|
case ":TestFail",
|
||||||
|
":TestFatal":
|
||||||
|
var testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx];
|
||||||
|
if (!testEntry.mFailed)
|
||||||
|
{
|
||||||
|
testFailCount++;
|
||||||
|
testEntry.mFailed = true;
|
||||||
|
|
||||||
|
if (!testEntry.mShouldFail)
|
||||||
|
{
|
||||||
|
curProjectInfo.mFailedCount++;
|
||||||
|
FlushTestStart();
|
||||||
|
QueueOutputLine("ERROR: Test failed. {}", cmd.Substring(cmdParts[0].Length + 1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cmdParts[0] == ":TestFatal")
|
||||||
|
{
|
||||||
|
if (testInstance.mPipeServer.Write(":TestContinue\n") case .Err)
|
||||||
|
failed = true;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case ":TestWrite":
|
||||||
|
String str = scope String()..Append(cmd.Substring(cmdParts[0].Length + 1));
|
||||||
|
str.Replace('\r', '\n');
|
||||||
|
queuedOutText.Append(str);
|
||||||
|
FlushOutText(false);
|
||||||
case ":TestFinish":
|
case ":TestFinish":
|
||||||
testsFinished = true;
|
testsFinished = true;
|
||||||
default:
|
default:
|
||||||
|
@ -318,6 +429,14 @@ namespace IDE
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FlushOutText(true);
|
||||||
|
|
||||||
|
/*if ((testFailCount > 0) && (!failed))
|
||||||
|
{
|
||||||
|
failed = true;
|
||||||
|
TestFailed();
|
||||||
|
}*/
|
||||||
|
|
||||||
if (mWantsStop)
|
if (mWantsStop)
|
||||||
{
|
{
|
||||||
QueueOutputLine("Tests aborted");
|
QueueOutputLine("Tests aborted");
|
||||||
|
@ -325,26 +444,42 @@ namespace IDE
|
||||||
else if (!testsFinished)
|
else if (!testsFinished)
|
||||||
{
|
{
|
||||||
var str = scope String();
|
var str = scope String();
|
||||||
if (curTestIdx == -1)
|
if (testInstance.mCurTestIdx == -1)
|
||||||
{
|
{
|
||||||
str.AppendF("Failed to start tests");
|
str.AppendF("Failed to start tests");
|
||||||
}
|
}
|
||||||
else if (curTestIdx < testInstance.mTestEntries.Count)
|
else if (testInstance.mCurTestIdx < testInstance.mTestEntries.Count)
|
||||||
{
|
{
|
||||||
var testEntry = testInstance.mTestEntries[curTestIdx];
|
var testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx];
|
||||||
if (testInstance.mShouldFailIdx == curTestIdx)
|
if (testEntry.mShouldFail)
|
||||||
|
{
|
||||||
|
// Success
|
||||||
|
testEntry.mFailed = true;
|
||||||
|
QueueOutputLine("Test expectedly failed: {0}", testEntry.mName);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (!testEntry.mFailed)
|
||||||
|
{
|
||||||
|
curProjectInfo.mFailedCount++;
|
||||||
|
testEntry.mFailed = true;
|
||||||
|
testFailCount++;
|
||||||
|
QueueOutputLine("Failed test '{0}' defined at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/*if (testInstance.mShouldFailIdx == testInstance.mCurTestIdx)
|
||||||
{
|
{
|
||||||
// Success
|
// Success
|
||||||
QueueOutputLine("Test expectedly failed: {0}", testEntry.mName);
|
QueueOutputLine("Test expectedly failed: {0}", testEntry.mName);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
str.AppendF("ERROR: Failed test '{0}' at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1);
|
str.AppendF("Failed test '{0}' defined at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1);
|
||||||
}
|
}*/
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
str.AppendF("ERROR: Failed to finish tests");
|
str.AppendF("Failed to finish tests");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (str.Length > 0)
|
if (str.Length > 0)
|
||||||
|
@ -363,8 +498,9 @@ namespace IDE
|
||||||
else if (testInstance.mTestEntries.IsEmpty)
|
else if (testInstance.mTestEntries.IsEmpty)
|
||||||
{
|
{
|
||||||
QueueOutputLine(
|
QueueOutputLine(
|
||||||
"""
|
$"""
|
||||||
WARNING: No test methods defined. Consider adding a [Test] attribute to a static method in a project whose build type is set to 'Test'.
|
WARNING: No test methods defined in project '{mProjectInfos[testInstance.mProjectIdx].mProject.mProjectName}'.
|
||||||
|
Consider adding a [Test] attribute to a static method in a project whose build type is set to 'Test'.
|
||||||
If you do have test methods defined, make sure the Workspace properties has that project's 'Test' configuration selected.
|
If you do have test methods defined, make sure the Workspace properties has that project's 'Test' configuration selected.
|
||||||
""");
|
""");
|
||||||
}
|
}
|
||||||
|
@ -391,7 +527,8 @@ namespace IDE
|
||||||
gApp.mDebugger.Terminate();
|
gApp.mDebugger.Terminate();
|
||||||
}
|
}
|
||||||
|
|
||||||
int nextShouldFailIdx = -1;
|
int32 nextTestIdx = -1;
|
||||||
|
|
||||||
bool doNext = true;
|
bool doNext = true;
|
||||||
var curProjectInfo = GetCurProjectInfo();
|
var curProjectInfo = GetCurProjectInfo();
|
||||||
if (curProjectInfo != null)
|
if (curProjectInfo != null)
|
||||||
|
@ -400,14 +537,9 @@ namespace IDE
|
||||||
{
|
{
|
||||||
if (mTestInstance.mThread.Join(0))
|
if (mTestInstance.mThread.Join(0))
|
||||||
{
|
{
|
||||||
for (int entryIdx = mTestInstance.mShouldFailIdx + 1; entryIdx < mTestInstance.mTestEntries.Count; entryIdx++)
|
if (mTestInstance.mCurTestIdx < mTestInstance.mTestEntries.Count - 1)
|
||||||
{
|
{
|
||||||
let testEntry = mTestInstance.mTestEntries[entryIdx];
|
nextTestIdx = mTestInstance.mCurTestIdx + 1;
|
||||||
if (testEntry.mShouldFail)
|
|
||||||
{
|
|
||||||
nextShouldFailIdx = entryIdx;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
DeleteAndNullify!(mTestInstance);
|
DeleteAndNullify!(mTestInstance);
|
||||||
|
@ -431,8 +563,9 @@ namespace IDE
|
||||||
|
|
||||||
Debug.Assert(mTestInstance == null);
|
Debug.Assert(mTestInstance == null);
|
||||||
|
|
||||||
if (nextShouldFailIdx == -1)
|
if (nextTestIdx == -1)
|
||||||
{
|
{
|
||||||
|
PrintProjectSummary();
|
||||||
mProjectInfoIdx++;
|
mProjectInfoIdx++;
|
||||||
if (mProjectInfoIdx >= mProjectInfos.Count)
|
if (mProjectInfoIdx >= mProjectInfos.Count)
|
||||||
{
|
{
|
||||||
|
@ -443,12 +576,9 @@ namespace IDE
|
||||||
|
|
||||||
mTestInstance = new TestInstance();
|
mTestInstance = new TestInstance();
|
||||||
mTestInstance.mProjectIdx = mProjectInfoIdx;
|
mTestInstance.mProjectIdx = mProjectInfoIdx;
|
||||||
mTestInstance.mShouldFailIdx = nextShouldFailIdx;
|
mTestInstance.mCurTestIdx = nextTestIdx;
|
||||||
|
|
||||||
curProjectInfo = GetCurProjectInfo();
|
curProjectInfo = GetCurProjectInfo();
|
||||||
if (mTestInstance.mShouldFailIdx != -1)
|
|
||||||
gApp.OutputLineSmart("Starting should-fail testing on {0}...", curProjectInfo.mProject.mProjectName);
|
|
||||||
else
|
|
||||||
gApp.OutputLineSmart("Starting testing on {0}...", curProjectInfo.mProject.mProjectName);
|
gApp.OutputLineSmart("Starting testing on {0}...", curProjectInfo.mProject.mProjectName);
|
||||||
|
|
||||||
mTestInstance.mThread = new Thread(new () => { TestProc(mTestInstance); } );
|
mTestInstance.mThread = new Thread(new () => { TestProc(mTestInstance); } );
|
||||||
|
@ -501,6 +631,21 @@ namespace IDE
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void PrintProjectSummary()
|
||||||
|
{
|
||||||
|
var curProjectInfo = GetCurProjectInfo();
|
||||||
|
if (curProjectInfo == null)
|
||||||
|
return;
|
||||||
|
|
||||||
|
String completeStr = scope $"Completed {curProjectInfo.mExecutedCount} of {curProjectInfo.mTestCount} tests for '{curProjectInfo.mProject.mProjectName}'";
|
||||||
|
if (curProjectInfo.mFailedCount > 0)
|
||||||
|
{
|
||||||
|
completeStr.AppendF($" with {curProjectInfo.mFailedCount} failure{((curProjectInfo.mFailedCount != 1) ? "s" : "")}");
|
||||||
|
}
|
||||||
|
completeStr.Append("\n");
|
||||||
|
QueueOutputLine(completeStr);
|
||||||
|
}
|
||||||
|
|
||||||
public void TestFailed()
|
public void TestFailed()
|
||||||
{
|
{
|
||||||
gApp.TestFailed();
|
gApp.TestFailed();
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue