diff --git a/BeefLibs/corlib/src/String.bf b/BeefLibs/corlib/src/String.bf index 586e72df..7e035abb 100644 --- a/BeefLibs/corlib/src/String.bf +++ b/BeefLibs/corlib/src/String.bf @@ -2436,29 +2436,43 @@ namespace System public mixin ToScopedNativeWChar() { int encodedLen = UTF16.GetEncodedLen(this); - char16* buf; + c_wchar* buf; if (encodedLen < 128) { - buf = scope:mixin char16[encodedLen]* ( ? ); + buf = scope:mixin c_wchar[encodedLen]* ( ? ); } else { - buf = new char16[encodedLen]* ( ? ); + buf = new c_wchar[encodedLen]* ( ? ); defer:mixin delete buf; } - UTF16.Encode(this, buf, encodedLen); + if (sizeof(c_wchar) == 2) + UTF16.Encode(this, (.)buf, encodedLen); + else + UTF32.Encode(this, (.)buf, encodedLen); buf } [Comptime(ConstEval=true)] public Span ToConstNativeW() { - int encodedLen = UTF16.GetEncodedLen(this); - var buf = new char16[encodedLen + 1]* ( ? ); - UTF16.Encode(this, buf, encodedLen); - buf[encodedLen] = 0; - return .(buf, encodedLen + 1); + if (sizeof(c_wchar) == 2) + { + int encodedLen = UTF16.GetEncodedLen(this); + var buf = new char16[encodedLen + 1]* ( ? ); + UTF16.Encode(this, buf, encodedLen); + buf[encodedLen] = 0; + return .((.)buf, encodedLen + 1); + } + else + { + int encodedLen = UTF32.GetEncodedLen(this); + var buf = new char32[encodedLen + 1]* ( ? ); + UTF32.Encode(this, buf, encodedLen); + buf[encodedLen] = 0; + return .((.)buf, encodedLen + 1); + } } public static bool Equals(char8* str1, char8* str2) diff --git a/BeefLibs/corlib/src/Text/UTF32.bf b/BeefLibs/corlib/src/Text/UTF32.bf new file mode 100644 index 00000000..8a537bfa --- /dev/null +++ b/BeefLibs/corlib/src/Text/UTF32.bf @@ -0,0 +1,153 @@ +using System.Diagnostics; +namespace System.Text +{ + public static class UTF32 + { + public enum EncodeError + { + case Overflow(int len); + } + + public static void Decode(char32* utf32Str, String outStr) + { + int utf8Len = GetLengthAsUTF8(utf32Str); + outStr.Reserve(outStr.Length + utf8Len); + + char32* utf32Ptr = utf32Str; + + while (true) + { + char32 c32 = *(utf32Ptr++); + if (c32 == 0) + break; + outStr.Append(c32); + } + } + + public static void Decode(Span utf32Str, String outStr) + { + int utf8Len = GetLengthAsUTF8(utf32Str); + outStr.Reserve(outStr.Length + utf8Len); + + char32* utf32Ptr = utf32Str.Ptr; + char32* utf32End = utf32Str.EndPtr; + while (utf32Ptr < utf32End) + { + char32 c32 = *(utf32Ptr++); + outStr.Append(c32); + } + } + + public static (char32 c, int8 cSize) Decode(char32* buf, int lenLeft = 0) + { + char32 c = buf[0]; + return (c, 1); + } + + public static int GetLengthAsUTF8(char32* utf32Str) + { + int utf8len = 0; + char32* utf32Ptr = utf32Str; + while (true) + { + char32 c32 = *(utf32Ptr++); + if (c32 == 0) + return utf8len; + utf8len += UTF8.GetEncodedLength(c32); + } + } + + public static int GetLengthAsUTF8(Span utf32Str) + { + int utf8len = 0; + char32* c16Ptr = utf32Str.Ptr; + int lenLeft = utf32Str.Length; + while (lenLeft > 0) + { + let (c, encLen) = Decode(c16Ptr, lenLeft); + c16Ptr += encLen; + lenLeft -= encLen; + utf8len += UTF8.GetEncodedLength(c); + } + return utf8len; + } + + public static bool Equals(char32* utf32Str, String str) + { + int strIdx = 0; + char32* c16Ptr = utf32Str; + while (true) + { + let (cA, encLenA) = Decode(c16Ptr); + if (strIdx == str.Length) + return cA == 0; + let (cB, encLenB) = str.GetChar32(strIdx); + if (cA != cB) + return false; + c16Ptr += encLenA; + strIdx += encLenB; + } + } + + public static int GetMaxEncodedLen(int utf8Len) + { + return utf8Len; + } + + public static int GetEncodedLength(char32 c) + { + return 1; + } + + public static int GetEncodedLen(StringView str) + { + int len = 0; + for (var c in str.DecodedChars) + { + len++; + } + len++; // null terminator + return len; + } + + public static int Encode(char32 c, Span dest) + { + if (dest.Length >= 2) + *((char32*)dest.Ptr) = (char32)c; + return 2; + } + + public static Result Encode(StringView str, char32* oututf32Buf, int bufLen) + { + char32* buf = oututf32Buf; + int bufLeft = bufLen; + + void EncodeChar(char32 c) + { + if (buf != null) + *(buf++) = (char32)c; + if (--bufLeft == 0) + buf = null; + } + + for (var c in str.DecodedChars) + { + + EncodeChar((char32)c); + } + EncodeChar(0); + + int encodedLen = bufLen - bufLeft; + if (bufLeft < 0) + return .Err(.Overflow(encodedLen)); + return .Ok(encodedLen); + } + + public static int CStrLen(char32* str) + { + for (int i = 0; true; i++) + if (str[i] == 0) + return i; + } + } +} diff --git a/IDEHelper/Tests/src/Strings.bf b/IDEHelper/Tests/src/Strings.bf index 42f6c2e9..7cb111be 100644 --- a/IDEHelper/Tests/src/Strings.bf +++ b/IDEHelper/Tests/src/Strings.bf @@ -1,4 +1,5 @@ using System; +using System.Interop; namespace Tests { @@ -19,6 +20,14 @@ namespace Tests var str2 = scope String(); FormatString(str2, $"\a{200+300}B{200+300:X}"); Test.Assert(str2 == "\a500B1F4"); + + static c_wchar[?] cWStr = "Test".ToConstNativeW(); + c_wchar* wStr = &cWStr; + Test.Assert(wStr[0] == 'T'); + Test.Assert(wStr[1] == 'e'); + Test.Assert(wStr[2] == 's'); + Test.Assert(wStr[3] == 't'); + Test.Assert(wStr[4] == '\0'); } } }