diff --git a/BeefLibs/corlib/src/Attribute.bf b/BeefLibs/corlib/src/Attribute.bf index fa03996b..ffa71bcb 100644 --- a/BeefLibs/corlib/src/Attribute.bf +++ b/BeefLibs/corlib/src/Attribute.bf @@ -376,6 +376,7 @@ namespace System public bool ShouldFail; public bool Ignore; public bool Profile; + public String Tag; } public struct ImportAttribute : Attribute diff --git a/BeefLibs/corlib/src/Console.bf b/BeefLibs/corlib/src/Console.bf index 2c993d96..b48277e8 100644 --- a/BeefLibs/corlib/src/Console.bf +++ b/BeefLibs/corlib/src/Console.bf @@ -68,14 +68,18 @@ namespace System { if (outStreamWriter == null) { + Stream stream; +#if BF_TEST_BUILD + stream = new Test.TestStream(); +#else FileStream fileStream = new .(); - Stream stream = fileStream; + stream = fileStream; if (fileStream.OpenStd(stdKind) case .Err) { DeleteAndNullify!(fileStream); stream = new NullStream(); } - +#endif StreamWriter newStreamWriter = new StreamWriter(stream, InputEncoding, 4096, true); newStreamWriter.AutoFlush = true; diff --git a/BeefLibs/corlib/src/Internal.bf b/BeefLibs/corlib/src/Internal.bf index 8255a3e3..1fa1a978 100644 --- a/BeefLibs/corlib/src/Internal.bf +++ b/BeefLibs/corlib/src/Internal.bf @@ -108,6 +108,10 @@ namespace System [CallingConvention(.Cdecl)] static extern void Test_Init(char8* testData); [CallingConvention(.Cdecl)] + static extern void Test_Error(char8* error); + [CallingConvention(.Cdecl)] + static extern void Test_Write(char8* str); + [CallingConvention(.Cdecl)] static extern int32 Test_Query(); [CallingConvention(.Cdecl)] static extern void Test_Finish(); diff --git a/BeefLibs/corlib/src/Test.bf b/BeefLibs/corlib/src/Test.bf index bf9abe03..81ba4d76 100644 --- a/BeefLibs/corlib/src/Test.bf +++ b/BeefLibs/corlib/src/Test.bf @@ -1,23 +1,78 @@ +using System.IO; namespace System { class Test { - [NoReturn] - public static void FatalError(String msg = "Test fatal error encountered") + public class TestStream : Stream { - Internal.FatalError(msg, 1); + public override int64 Position + { + get + { + Runtime.FatalError(); + } + + set + { + } + } + + public override int64 Length + { + get + { + Runtime.FatalError(); + } + } + + public override bool CanRead + { + get + { + return false; + } + } + + public override bool CanWrite + { + get + { + return true; + } + } + + public override Result TryRead(Span data) + { + return default; + } + + public override Result TryWrite(Span data) + { + String str = scope String(); + str.Append((char8*)data.Ptr, data.Length); + Internal.[Friend]Test_Write(str.CStr()); + return .Ok(data.Length); + } + + public override void Close() + { + + } } - public static void Assert(bool condition) + public static void FatalError(String msg = "Test fatal error encountered", String filePath = Compiler.CallerFilePath, int line = Compiler.CallerLineNum) { - if (!condition) - Internal.FatalError("Test Assert failed", 1); + String failStr = scope .()..AppendF("{} at line {} in {}", msg, line, filePath); + Internal.[Friend]Test_Error(failStr); } - public static void Assert(bool condition, String error) + public static void Assert(bool condition, String error = Compiler.CallerExpression[0], String filePath = Compiler.CallerFilePath, int line = Compiler.CallerLineNum) { if (!condition) - Internal.FatalError(error, 2); + { + String failStr = scope .()..AppendF("Assert failed: {} at line {} in {}", error, line, filePath); + Internal.[Friend]Test_Error(failStr); + } } } } diff --git a/BeefRT/rt/Internal.cpp b/BeefRT/rt/Internal.cpp index 3d269199..d391e642 100644 --- a/BeefRT/rt/Internal.cpp +++ b/BeefRT/rt/Internal.cpp @@ -55,6 +55,13 @@ static Beefy::StringT<0> gCmdLineString; bf::System::Runtime::BfRtCallbacks gBfRtCallbacks; BfRtFlags gBfRtFlags = (BfRtFlags)0; +static int gTestMethodIdx = -1; +static uint32 gTestStartTick = 0; +static bool gTestBreakOnFailure = false; + +static BfpFile* gClientPipe = NULL; +static Beefy::String gTestInBuffer; + namespace bf { namespace System @@ -101,6 +108,8 @@ namespace bf private: BFRT_EXPORT static void Test_Init(char* testData); + BFRT_EXPORT static void Test_Error(char* error); + BFRT_EXPORT static void Test_Write(char* str); BFRT_EXPORT static int32 Test_Query(); BFRT_EXPORT static void Test_Finish(); }; @@ -201,6 +210,24 @@ bool IsDebuggerPresent() #endif +static void TestString(const StringImpl& str); +static void TestReadCmd(Beefy::String& str); + +static void Internal_FatalError(const char* error) +{ + if (gClientPipe != NULL) + { + Beefy::String str = ":TestFatal\t"; + str += error; + str += "\n"; + TestString(str); + + Beefy::String result; + TestReadCmd(result); + } + + BfpSystem_FatalError(error, "BEEF FATAL ERROR"); +} extern "C" BFRT_EXPORT int BF_CALLTYPE ftoa(float val, char* str) { @@ -265,7 +292,7 @@ void bf::System::Runtime::Init(int version, int flags, BfRtCallbacks* callbacks) if (gBfRtCallbacks.Alloc != NULL) { - BfpSystem_FatalError(StrFormat("BeefRT already initialized. Multiple executable modules in the same process cannot dynamically link to the Beef runtime.").c_str(), "BEEF FATAL ERROR"); + Internal_FatalError(StrFormat("BeefRT already initialized. Multiple executable modules in the same process cannot dynamically link to the Beef runtime.").c_str()); } if (version != BFRT_VERSION) @@ -394,7 +421,7 @@ void Internal::ThrowIndexOutOfRange(intptr stackOffset) BF_DEBUG_BREAK(); } - BfpSystem_FatalError("Index out of range", "FATAL ERROR"); + Internal_FatalError("Index out of range"); } void Internal::FatalError(bf::System::String* error, intptr stackOffset) @@ -405,7 +432,7 @@ void Internal::FatalError(bf::System::String* error, intptr stackOffset) BF_DEBUG_BREAK(); } - BfpSystem_FatalError(error->CStr(), "FATAL ERROR"); + Internal_FatalError(error->CStr()); } void Internal::MemCpy(void* dest, void* src, intptr length) @@ -568,12 +595,6 @@ void Internal::ReportMemory() #endif } -static int gTestMethodIdx = -1; -static uint32 gTestStartTick = 0; - -static BfpFile* gClientPipe = NULL; -static Beefy::String gTestInBuffer; - static void TestString(const StringImpl& str) { BfpFileResult fileResult; @@ -608,7 +629,7 @@ static void TestReadCmd(Beefy::String& str) } void Internal::Test_Init(char* testData) -{ +{ BfpSystem_SetCrashReportKind(BfpCrashReportKind_None); Beefy::String args = GetCommandLineArgs(); @@ -626,6 +647,40 @@ void Internal::Test_Init(char* testData) TestString(outStr); } +void Internal::Test_Error(char* error) +{ + if (gTestBreakOnFailure) + { + SETUP_ERROR(error, 3); + BF_DEBUG_BREAK(); + } + + if (gClientPipe != NULL) + { + Beefy::String str = ":TestFail\t"; + str += error; + str += "\n"; + TestString(str); + } +} + +void Internal::Test_Write(char* strPtr) +{ + if (gClientPipe != NULL) + { + Beefy::String str = ":TestWrite\t"; + str += strPtr; + for (char& c : str) + { + if (c == '\n') + c = '\r'; + } + + str += "\n"; + TestString(str); + } +} + int32 Internal::Test_Query() { if (gTestMethodIdx != -1) @@ -650,7 +705,16 @@ int32 Internal::Test_Query() if (result == ":TestRun") { gTestStartTick = BfpSystem_TickCount(); + Beefy::String options; + int tabPos = (int)param.IndexOf('\t'); + if (tabPos != -1) + { + options = param.Substring(tabPos + 1); + param.RemoveToEnd(tabPos); + } + gTestMethodIdx = atoi(param.c_str()); + gTestBreakOnFailure = options.Contains("FailBreak"); return gTestMethodIdx; } else if (result == ":TestFinish") @@ -791,7 +855,7 @@ void Contract::ReportFailure(Contract::ContractFailureKind failureKind, char* us gBfRtCallbacks.DebugMessageData_Fatal(); } - BfpSystem_FatalError(errorMsg.c_str(), "CONTRACT ERROR"); + Internal_FatalError(errorMsg.c_str()); return; } diff --git a/IDE/src/IDEApp.bf b/IDE/src/IDEApp.bf index 6d09f877..a0ea597f 100644 --- a/IDE/src/IDEApp.bf +++ b/IDE/src/IDEApp.bf @@ -324,6 +324,7 @@ namespace IDE public bool mEnableGCCollect = true; public bool mDbgFastUpdate; public bool mTestEnableConsole = false; + public bool mTestBreakOnFailure = true; public ProfileInstance mLongUpdateProfileId; public uint32 mLastLongUpdateCheck; public uint32 mLastLongUpdateCheckError; @@ -5417,10 +5418,16 @@ namespace IDE var testRunMenu = testMenu.AddMenuItem("&Run", null, null, new => UpdateMenuItem_DebugStopped_HasWorkspace); AddMenuItem(testRunMenu, "&Normal Tests", "Run Normal Tests"); AddMenuItem(testRunMenu, "&All Tests", "Run All Tests"); + var testDebugMenu = testMenu.AddMenuItem("&Debug", null, null, new => UpdateMenuItem_DebugStopped_HasWorkspace); AddMenuItem(testDebugMenu, "&Normal Tests", "Debug Normal Tests"); AddMenuItem(testDebugMenu, "&All Tests", "Debug All Tests"); + testDebugMenu.AddMenuItem(null); + testDebugMenu.AddMenuItem("Break on Failure", null, new (menu) => + { + ToggleCheck(menu, ref mTestBreakOnFailure); + }, null, null, true, mTestBreakOnFailure ? 1 : 0); AddMenuItem(testMenu, "Enable Console", "Test Enable Console", null, null, true, mTestEnableConsole ? 1 : 0); @@ -12173,6 +12180,14 @@ namespace IDE { if (isFirstMsg) { + if (mTestManager != null) + { + // Give test manager time to flush + Thread.Sleep(100); + mTestManager.Update(); + mOutputPanel.Update(); + } + OutputLineSmart(scope String("ERROR: ", errorMsg)); if (gApp.mRunningTestScript) { @@ -13302,7 +13317,7 @@ namespace IDE if (mTestManager != null) { mTestManager.Update(); - if (mTestManager.mIsDone) + if ((mTestManager.mIsDone) && (mTestManager.mQueuedOutput.IsEmpty)) { if (mMainFrame != null) mMainFrame.mStatusBar.SelectConfig(mTestManager.mPrevConfigName); diff --git a/IDE/src/TestManager.bf b/IDE/src/TestManager.bf index 3af1db36..011b56de 100644 --- a/IDE/src/TestManager.bf +++ b/IDE/src/TestManager.bf @@ -12,6 +12,10 @@ namespace IDE { public Project mProject; public String mTestExePath ~ delete _; + public int32 mTestCount; + public int32 mExecutedCount; + public int32 mSkipCount; + public int32 mFailedCount; } public class TestEntry @@ -23,6 +27,8 @@ namespace IDE public bool mShouldFail; public bool mProfile; public bool mIgnore; + public bool mFailed; + public bool mExecuted; } public class TestInstance @@ -35,7 +41,7 @@ namespace IDE public String mArgs ~ delete _; public String mWorkingDir ~ delete _; public NamedPipe mPipeServer ~ delete _; - public int mShouldFailIdx = -1; + public int32 mCurTestIdx = -1; } public bool mIsDone; @@ -147,15 +153,73 @@ namespace IDE String clientStr = scope String(); - int curTestIdx = -1; int curTestRunCount = 0; bool testsFinished = false; bool failed = false; + int testFailCount = 0; + bool testHadOutput = false; int exitCode = 0; + String queuedOutText = scope .(); + + Stopwatch testTimer = scope .(); + + void FlushTestStart() + { + if (testHadOutput) + return; + + if ((testInstance.mCurTestIdx >= 0) && (testInstance.mCurTestIdx < testInstance.mTestEntries.Count)) + { + testHadOutput = true; + String outputLine = scope String(); + let testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx]; + outputLine.AppendF($"Testing '{testEntry.mName}'"); + QueueOutputLine(outputLine); + } + } + + void FlushOutText(bool force) + { + if (queuedOutText.IsEmpty) + return; + if ((!queuedOutText.EndsWith('\n')) && (force)) + queuedOutText.Append('\n'); + int lastCrPos = -1; + while (true) + { + int crPos = queuedOutText.IndexOf('\n', lastCrPos + 1); + if (crPos == -1) + break; + + FlushTestStart(); + + String outputLine = scope String(); + outputLine.Append(" >"); + if (crPos - lastCrPos - 2 > 0) + outputLine.Append(StringView(queuedOutText, lastCrPos + 1, crPos - lastCrPos - 2)); + + QueueOutputLine(outputLine); + lastCrPos = crPos; + } + + if (lastCrPos != -1) + queuedOutText.Remove(0, lastCrPos + 1); + } + while (true) { + if ((mDebug) && (gApp.mDebugger.IsPaused())) + { + FlushTestStart(); + } + + if (testTimer.ElapsedMilliseconds > 1000) + { + FlushTestStart(); + } + int doneCount = 0; for (int itr < 2) @@ -190,35 +254,35 @@ namespace IDE outStr.AppendF("CMD: {0}", cmd); QueueOutput(outStr);*/ + + List cmdParts = scope .(cmd.Split('\t')); switch (cmdParts[0]) { case ":TestInit": case ":TestBegin": case ":TestQuery": - if ((curTestIdx == -1) || (curTestRunCount > 0)) + testTimer.Stop(); + testTimer.Start(); + FlushOutText(true); + testHadOutput = false; + if ((testInstance.mCurTestIdx == -1) || (curTestRunCount > 0)) { - curTestIdx++; + testInstance.mCurTestIdx++; curTestRunCount = 0; } + curProjectInfo.mTestCount = (.)testInstance.mTestEntries.Count; + while (true) { - if (curTestIdx < testInstance.mTestEntries.Count) + if (testInstance.mCurTestIdx < testInstance.mTestEntries.Count) { curTestRunCount++; bool skipEntry = false; - let testEntry = testInstance.mTestEntries[curTestIdx]; - if (testEntry.mShouldFail) - { - skipEntry = testInstance.mShouldFailIdx != curTestIdx; - } - else if (testInstance.mShouldFailIdx != -1) - { - skipEntry = true; - } - + let testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx]; + if ((!skipEntry) && (testEntry.mIgnore) && (!mIncludeIgnored)) { QueueOutputLine("Test Ignored: {0}", testEntry.mName); @@ -227,13 +291,20 @@ namespace IDE if (skipEntry) { - curTestIdx++; + curProjectInfo.mSkipCount++; + testInstance.mCurTestIdx++; curTestRunCount = 0; continue; } - var clientCmd = scope String(); - clientCmd.AppendF(":TestRun\t{0}\n", curTestIdx); + curProjectInfo.mExecutedCount++; + testEntry.mExecuted = true; + + String clientCmd = scope $":TestRun\t{testInstance.mCurTestIdx}"; + if ((gApp.mTestBreakOnFailure) && (mDebug)) + clientCmd.Append("\tFailBreak"); + clientCmd.Append("\n"); + if (testInstance.mPipeServer.Write(clientCmd) case .Err) failed = true; } @@ -245,15 +316,55 @@ namespace IDE break; } case ":TestResult": + testTimer.Stop(); int timeMS = int32.Parse(cmdParts[1]).Get(); - var testEntry = testInstance.mTestEntries[curTestIdx]; + var testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx]; + if (testEntry.mShouldFail) { - QueueOutputLine("ERROR: Test should have failed but didn't: {0} Time: {1}ms", testEntry.mName, timeMS); - failed = true; + if (testEntry.mFailed) + { + QueueOutputLine("Test expectedly failed: {0} Time: {1}ms", testEntry.mName, timeMS); + } + else + { + curProjectInfo.mFailedCount++; + QueueOutputLine("ERROR: Test should have failed but didn't: '{0}' defined at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1); + } } else - QueueOutputLine("Test completed: {0} Time: {1}ms", testEntry.mName, timeMS); + { + if (testHadOutput) + QueueOutputLine(" Test Time: {1}ms", testEntry.mName, timeMS); + else + QueueOutputLine("Test '{0}' Time: {1}ms", testEntry.mName, timeMS); + } + case ":TestFail", + ":TestFatal": + var testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx]; + if (!testEntry.mFailed) + { + testFailCount++; + testEntry.mFailed = true; + + if (!testEntry.mShouldFail) + { + curProjectInfo.mFailedCount++; + FlushTestStart(); + QueueOutputLine("ERROR: Test failed. {}", cmd.Substring(cmdParts[0].Length + 1)); + } + } + if (cmdParts[0] == ":TestFatal") + { + if (testInstance.mPipeServer.Write(":TestContinue\n") case .Err) + failed = true; + } + break; + case ":TestWrite": + String str = scope String()..Append(cmd.Substring(cmdParts[0].Length + 1)); + str.Replace('\r', '\n'); + queuedOutText.Append(str); + FlushOutText(false); case ":TestFinish": testsFinished = true; default: @@ -318,6 +429,14 @@ namespace IDE } } + FlushOutText(true); + + /*if ((testFailCount > 0) && (!failed)) + { + failed = true; + TestFailed(); + }*/ + if (mWantsStop) { QueueOutputLine("Tests aborted"); @@ -325,26 +444,42 @@ namespace IDE else if (!testsFinished) { var str = scope String(); - if (curTestIdx == -1) + if (testInstance.mCurTestIdx == -1) { str.AppendF("Failed to start tests"); } - else if (curTestIdx < testInstance.mTestEntries.Count) + else if (testInstance.mCurTestIdx < testInstance.mTestEntries.Count) { - var testEntry = testInstance.mTestEntries[curTestIdx]; - if (testInstance.mShouldFailIdx == curTestIdx) + var testEntry = testInstance.mTestEntries[testInstance.mCurTestIdx]; + if (testEntry.mShouldFail) + { + // Success + testEntry.mFailed = true; + QueueOutputLine("Test expectedly failed: {0}", testEntry.mName); + } + else + { + if (!testEntry.mFailed) + { + curProjectInfo.mFailedCount++; + testEntry.mFailed = true; + testFailCount++; + QueueOutputLine("Failed test '{0}' defined at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1); + } + } + /*if (testInstance.mShouldFailIdx == testInstance.mCurTestIdx) { // Success QueueOutputLine("Test expectedly failed: {0}", testEntry.mName); } else { - str.AppendF("ERROR: Failed test '{0}' at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1); - } + str.AppendF("Failed test '{0}' defined at line {2}:{3} in {1}", testEntry.mName, testEntry.mFilePath, testEntry.mLine + 1, testEntry.mColumn + 1); + }*/ } else { - str.AppendF("ERROR: Failed to finish tests"); + str.AppendF("Failed to finish tests"); } if (str.Length > 0) @@ -363,8 +498,9 @@ namespace IDE else if (testInstance.mTestEntries.IsEmpty) { QueueOutputLine( - """ - WARNING: No test methods defined. Consider adding a [Test] attribute to a static method in a project whose build type is set to 'Test'. + $""" + WARNING: No test methods defined in project '{mProjectInfos[testInstance.mProjectIdx].mProject.mProjectName}'. + Consider adding a [Test] attribute to a static method in a project whose build type is set to 'Test'. If you do have test methods defined, make sure the Workspace properties has that project's 'Test' configuration selected. """); } @@ -391,7 +527,8 @@ namespace IDE gApp.mDebugger.Terminate(); } - int nextShouldFailIdx = -1; + int32 nextTestIdx = -1; + bool doNext = true; var curProjectInfo = GetCurProjectInfo(); if (curProjectInfo != null) @@ -400,14 +537,9 @@ namespace IDE { if (mTestInstance.mThread.Join(0)) { - for (int entryIdx = mTestInstance.mShouldFailIdx + 1; entryIdx < mTestInstance.mTestEntries.Count; entryIdx++) + if (mTestInstance.mCurTestIdx < mTestInstance.mTestEntries.Count - 1) { - let testEntry = mTestInstance.mTestEntries[entryIdx]; - if (testEntry.mShouldFail) - { - nextShouldFailIdx = entryIdx; - break; - } + nextTestIdx = mTestInstance.mCurTestIdx + 1; } DeleteAndNullify!(mTestInstance); @@ -431,25 +563,23 @@ namespace IDE Debug.Assert(mTestInstance == null); - if (nextShouldFailIdx == -1) + if (nextTestIdx == -1) { + PrintProjectSummary(); mProjectInfoIdx++; if (mProjectInfoIdx >= mProjectInfos.Count) { mIsDone = true; return; } - } + } mTestInstance = new TestInstance(); mTestInstance.mProjectIdx = mProjectInfoIdx; - mTestInstance.mShouldFailIdx = nextShouldFailIdx; + mTestInstance.mCurTestIdx = nextTestIdx; curProjectInfo = GetCurProjectInfo(); - if (mTestInstance.mShouldFailIdx != -1) - gApp.OutputLineSmart("Starting should-fail testing on {0}...", curProjectInfo.mProject.mProjectName); - else - gApp.OutputLineSmart("Starting testing on {0}...", curProjectInfo.mProject.mProjectName); + gApp.OutputLineSmart("Starting testing on {0}...", curProjectInfo.mProject.mProjectName); mTestInstance.mThread = new Thread(new () => { TestProc(mTestInstance); } ); @@ -501,6 +631,21 @@ namespace IDE } } + public void PrintProjectSummary() + { + var curProjectInfo = GetCurProjectInfo(); + if (curProjectInfo == null) + return; + + String completeStr = scope $"Completed {curProjectInfo.mExecutedCount} of {curProjectInfo.mTestCount} tests for '{curProjectInfo.mProject.mProjectName}'"; + if (curProjectInfo.mFailedCount > 0) + { + completeStr.AppendF($" with {curProjectInfo.mFailedCount} failure{((curProjectInfo.mFailedCount != 1) ? "s" : "")}"); + } + completeStr.Append("\n"); + QueueOutputLine(completeStr); + } + public void TestFailed() { gApp.TestFailed();