From 29f01ea414465d53bb972fccced82d1ed3d99893 Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Mon, 3 Apr 2023 20:05:38 +0200 Subject: [PATCH 1/7] Update to .NET 6 and update packages NetCoreServer 5.1.0 -> 6.7.0 BenchmarkDotNet 0.13.1 -> 0.13.5 ENet-CSharp 2.4.7 -> 2.4.8 --- NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj | 8 ++++---- NetworkBenchmarkDotNet/Program.cs | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj index e7cd1f6..e74d074 100644 --- a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj +++ b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj @@ -12,7 +12,7 @@ Debug;Release AnyCPU 9 - net5.0 + net6.0 true true true @@ -30,11 +30,11 @@ - - + + - + diff --git a/NetworkBenchmarkDotNet/Program.cs b/NetworkBenchmarkDotNet/Program.cs index c60995f..06c82aa 100644 --- a/NetworkBenchmarkDotNet/Program.cs +++ b/NetworkBenchmarkDotNet/Program.cs @@ -12,6 +12,7 @@ using System.CommandLine; using System.CommandLine.Invocation; using System.Linq; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; namespace NetworkBenchmark @@ -112,7 +113,7 @@ private static void RunBenchmark() "The following benchmarks are failed to build: " + string.Join(", ", summary.Reports.Where(r => !r.BuildResult.IsBuildSuccess).Select(r => r.BenchmarkCase.DisplayInfo))); - Assert(summary.Reports.All(r => r.ExecuteResults.Any(er => er.FoundExecutable && er.Data.Any())), + Assert(summary.Reports.All(r => r.ExecuteResults.Any(er => er.FoundExecutable && er.Results.Any())), "All reports should have at least one \"ExecuteResult\" with \"FoundExecutable\" = true and at least one \"Data\" item"); Assert(summary.Reports.All(report => report.AllMeasurements.Any()), From b0834d6ea3f3e1940bbd3f45eee2ca7d3e93bac0 Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Mon, 3 Apr 2023 21:14:57 +0200 Subject: [PATCH 2/7] Updat kcp2k to v1.34 Sha d9210476c7b8bfe70c28c451c9d3c6f2339ca2dd from https://github.com/vis2k/kcp2k --- Kcp2k/kcp2k/KCP.asmdef | 4 +- Kcp2k/kcp2k/{LICENSE => LICENSE.txt} | 0 Kcp2k/kcp2k/VERSION | 96 --- Kcp2k/kcp2k/VERSION.txt | 222 ++++++ Kcp2k/kcp2k/empty/KcpClientConnection.cs | 1 + .../empty/KcpClientConnectionNonAlloc.cs | 1 + Kcp2k/kcp2k/empty/KcpClientNonAlloc.cs | 1 + .../empty/KcpServerConnectionNonAlloc.cs | 1 + Kcp2k/kcp2k/empty/KcpServerNonAlloc.cs | 1 + Kcp2k/kcp2k/highlevel/Common.cs | 49 ++ Kcp2k/kcp2k/highlevel/ErrorCode.cs | 15 + Kcp2k/kcp2k/highlevel/Extensions.cs | 162 ++++ Kcp2k/kcp2k/highlevel/KcpChannel.cs | 4 +- Kcp2k/kcp2k/highlevel/KcpClient.cs | 211 +++-- Kcp2k/kcp2k/highlevel/KcpClientConnection.cs | 109 --- Kcp2k/kcp2k/highlevel/KcpConfig.cs | 97 +++ Kcp2k/kcp2k/highlevel/KcpConnection.cs | 674 ---------------- Kcp2k/kcp2k/highlevel/KcpHeader.cs | 10 +- Kcp2k/kcp2k/highlevel/KcpPeer.cs | 737 ++++++++++++++++++ Kcp2k/kcp2k/highlevel/KcpServer.cs | 490 ++++++------ Kcp2k/kcp2k/highlevel/KcpServerConnection.cs | 24 +- Kcp2k/kcp2k/highlevel/Log.cs | 4 +- .../NonAlloc/KcpClientConnectionNonAlloc.cs | 24 - .../highlevel/NonAlloc/KcpClientNonAlloc.cs | 17 - .../NonAlloc/KcpServerConnectionNonAlloc.cs | 25 - .../highlevel/NonAlloc/KcpServerNonAlloc.cs | 51 -- Kcp2k/kcp2k/kcp/Kcp.cs | 124 ++- Kcp2k/kcp2k/kcp/Segment.cs | 17 +- Kcp2k/kcp2k/where-allocation/LICENSE | 21 - .../where-allocation/Scripts/AssemblyInfo.cs | 3 - .../where-allocation/Scripts/Extensions.cs | 58 -- .../Scripts/IPEndPointNonAlloc.cs | 208 ----- .../Scripts/where-allocations.asmdef | 13 - Kcp2k/kcp2k/where-allocation/VERSION | 2 - .../Libraries/Kcp2k/EchoClient.cs | 30 +- .../Libraries/Kcp2k/EchoServer.cs | 18 +- 36 files changed, 1856 insertions(+), 1668 deletions(-) rename Kcp2k/kcp2k/{LICENSE => LICENSE.txt} (100%) delete mode 100644 Kcp2k/kcp2k/VERSION create mode 100644 Kcp2k/kcp2k/VERSION.txt create mode 100644 Kcp2k/kcp2k/empty/KcpClientConnection.cs create mode 100644 Kcp2k/kcp2k/empty/KcpClientConnectionNonAlloc.cs create mode 100644 Kcp2k/kcp2k/empty/KcpClientNonAlloc.cs create mode 100644 Kcp2k/kcp2k/empty/KcpServerConnectionNonAlloc.cs create mode 100644 Kcp2k/kcp2k/empty/KcpServerNonAlloc.cs create mode 100644 Kcp2k/kcp2k/highlevel/Common.cs create mode 100644 Kcp2k/kcp2k/highlevel/ErrorCode.cs create mode 100644 Kcp2k/kcp2k/highlevel/Extensions.cs delete mode 100644 Kcp2k/kcp2k/highlevel/KcpClientConnection.cs create mode 100644 Kcp2k/kcp2k/highlevel/KcpConfig.cs delete mode 100644 Kcp2k/kcp2k/highlevel/KcpConnection.cs create mode 100644 Kcp2k/kcp2k/highlevel/KcpPeer.cs delete mode 100644 Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientConnectionNonAlloc.cs delete mode 100644 Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientNonAlloc.cs delete mode 100644 Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerConnectionNonAlloc.cs delete mode 100644 Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerNonAlloc.cs delete mode 100644 Kcp2k/kcp2k/where-allocation/LICENSE delete mode 100644 Kcp2k/kcp2k/where-allocation/Scripts/AssemblyInfo.cs delete mode 100644 Kcp2k/kcp2k/where-allocation/Scripts/Extensions.cs delete mode 100644 Kcp2k/kcp2k/where-allocation/Scripts/IPEndPointNonAlloc.cs delete mode 100644 Kcp2k/kcp2k/where-allocation/Scripts/where-allocations.asmdef delete mode 100644 Kcp2k/kcp2k/where-allocation/VERSION diff --git a/Kcp2k/kcp2k/KCP.asmdef b/Kcp2k/kcp2k/KCP.asmdef index b786144..c10bf83 100644 --- a/Kcp2k/kcp2k/KCP.asmdef +++ b/Kcp2k/kcp2k/KCP.asmdef @@ -1,9 +1,7 @@ { "name": "kcp", "rootNamespace": "", - "references": [ - "GUID:63c380d6dae6946209ed0832388a657c" - ], + "references": [], "includePlatforms": [], "excludePlatforms": [], "allowUnsafeCode": true, diff --git a/Kcp2k/kcp2k/LICENSE b/Kcp2k/kcp2k/LICENSE.txt similarity index 100% rename from Kcp2k/kcp2k/LICENSE rename to Kcp2k/kcp2k/LICENSE.txt diff --git a/Kcp2k/kcp2k/VERSION b/Kcp2k/kcp2k/VERSION deleted file mode 100644 index 1d4b93b..0000000 --- a/Kcp2k/kcp2k/VERSION +++ /dev/null @@ -1,96 +0,0 @@ -V1.12 [2021-07-16] -- Tests: don't depend on Unity anymore -- fix: #26 - Kcp now catches exception if host couldn't be resolved, and calls - OnDisconnected to let the user now. -- fix: KcpServer.DualMode is now configurable in the constructor instead of - using #if UNITY_SWITCH. makes it run on all other non dual mode platforms too. -- fix: where-allocation made optional via virtuals and inheriting - KcpServer/Client/Connection NonAlloc classes. fixes a bug where some platforms - might not support where-allocation. - -V1.11 rollback [2021-06-01] -- perf: Segment MemoryStream initial capacity set to MTU to avoid early runtime - resizing/allocations - -V1.10 [2021-05-28] -- feature: configurable Timeout -- allocations explained with comments (C# ReceiveFrom / IPEndPoint.GetHashCode) -- fix: #17 KcpConnection.ReceiveNextReliable now assigns message default so it - works in .net too -- fix: Segment pool is not static anymore. Each kcp instance now has it's own - Pool. fixes #18 concurrency issues - -V1.9 [2021-03-02] -- Tick() split into TickIncoming()/TickOutgoing() to use in Mirror's new update - functions. allows to minimize latency. - => original Tick() is still supported for convenience. simply processes both! - -V1.8 [2021-02-14] -- fix: Unity IPv6 errors on Nintendo Switch -- fix: KcpConnection now disconnects if data message was received without content. - previously it would call OnData with an empty ArraySegment, causing all kinds of - weird behaviour in Mirror/DOTSNET. Added tests too. -- fix: KcpConnection.SendData: don't allow sending empty messages anymore. disconnect - and log a warning to make it completely obvious. - -V1.7 [2021-01-13] -- fix: unreliable messages reset timeout now too -- perf: KcpConnection OnCheckEnabled callback changed to a simple 'paused' boolean. - This is faster than invoking a Func every time and allows us to fix #8 more - easily later by calling .Pause/.Unpause from OnEnable/OnDisable in MirrorTransport. -- fix #8: Unpause now resets timeout to fix a bug where Mirror would pause kcp, - change the scene which took >10s, then unpause and kcp would detect the lack of - any messages for >10s as timeout. Added test to make sure it never happens again. -- MirrorTransport: statistics logging for headless servers -- Mirror Transport: Send/Receive window size increased once more from 2048 to 4096. - -V1.6 [2021-01-10] -- Unreliable channel added! -- perf: KcpHeader byte added to every kcp message to indicate - Handshake/Data/Ping/Disconnect instead of scanning each message for Hello/Byte/Ping - content via SegmentEquals. It's a lot cleaner, should be faster and should avoid - edge cases where a message content would equal Hello/Ping/Bye sequence accidentally. -- Kcp.Input: offset moved to parameters for cases where it's needed -- Kcp.SetMtu from original Kcp.c - -V1.5 [2021-01-07] -- KcpConnection.MaxSend/ReceiveRate calculation based on the article -- MirrorTransport: large send/recv window size defaults to avoid high latencies caused - by packets not being processed fast enough -- MirrorTransport: show MaxSend/ReceiveRate in debug gui -- MirrorTransport: don't Log.Info to console in headless mode if debug log is disabled - -V1.4 [2020-11-27] -- fix: OnCheckEnabled added. KcpConnection message processing while loop can now - be interrupted immediately. fixes Mirror Transport scene changes which need to stop - processing any messages immediately after a scene message) -- perf: Mirror KcpTransport: FastResend enabled by default. turbo mode according to: - https://github.com/skywind3000/kcp/blob/master/README.en.md#protocol-configuration -- perf: Mirror KcpTransport: CongestionControl disabled by default (turbo mode) - -V1.3 [2020-11-17] -- Log.Info/Warning/Error so logging doesn't depend on UnityEngine anymore -- fix: Server.Tick catches SocketException which happens if Android client is killed -- MirrorTransport: debugLog option added that can be checked in Unity Inspector -- Utils.Clamp so Kcp.cs doesn't depend on UnityEngine -- Utils.SegmentsEqual: use Linq SequenceEqual so it doesn't depend on UnityEngine -=> kcp2k can now be used in any C# project even without Unity - -V1.2 [2020-11-10] -- more tests added -- fix: raw receive buffers are now all of MTU size -- fix: raw receive detects error where buffer was too small for msgLength and - result in excess data being dropped silently -- KcpConnection.MaxMessageSize added for use in high level -- KcpConnection.MaxMessageSize increased from 1200 bytes to to maximum allowed - message size of 145KB for kcp (based on mtu, overhead, wnd_rcv) - -V1.1 [2020-10-30] -- high level cleanup, fixes, improvements - -V1.0 [2020-10-22] -- Kcp.cs now mirrors original Kcp.c behaviour - (this fixes dozens of bugs) - -V0.1 -- initial kcp-csharp based version \ No newline at end of file diff --git a/Kcp2k/kcp2k/VERSION.txt b/Kcp2k/kcp2k/VERSION.txt new file mode 100644 index 0000000..58399ad --- /dev/null +++ b/Kcp2k/kcp2k/VERSION.txt @@ -0,0 +1,222 @@ +V1.34 [2023-03-15] +- Send/SendTo/Receive/ReceiveFrom NonBlocking extensions. + to encapsulate WouldBlock allocations, exceptions, etc. + allows for reuse when overwriting KcpServer/Client (i.e. for relays). + +V1.33 [2023-03-14] +- perf: KcpServer/Client RawReceive now call socket.Poll to avoid non-blocking + socket's allocating a new SocketException in case they WouldBlock. + fixes https://github.com/MirrorNetworking/Mirror/issues/3413 +- perf: KcpServer/Client RawSend now call socket.Poll to avoid non-blocking + socket's allocating a new SocketException in case they WouldBlock. + fixes https://github.com/MirrorNetworking/Mirror/issues/3413 + +V1.32 [2023-03-12] +- fix: KcpPeer RawInput now doesn't disconnect in case of random internet noise + +V1.31 [2023-03-05] +- KcpClient: Tick/Incoming/Outgoing can now be overwritten (virtual) +- breaking: KcpClient now takes KcpConfig in constructor instead of in Connect. + cleaner, and prepares for KcpConfig.MTU setting. +- KcpConfig now includes MTU; KcpPeer now works with KcpConfig's MTU, KcpServer/Client + buffers are now created with config's MTU. + +V1.30 [2023-02-20] +- fix: set send/recv buffer sizes directly instead of iterating to find the limit. + fixes: https://github.com/MirrorNetworking/Mirror/issues/3390 +- fix: server & client sockets are now always non-blocking to ensure main thread never + blocks on socket.recv/send. Send() now also handles WouldBlock. +- fix: socket.Receive/From directly with non-blocking sockets and handle WouldBlock, + instead of socket.Poll. faster, more obvious, and fixes Poll() looping forever while + socket is in error state. fixes: https://github.com/MirrorNetworking/Mirror/issues/2733 + +V1.29 [2023-01-28] +- fix: KcpServer.CreateServerSocket now handles NotSupportedException when setting DualMode + https://github.com/MirrorNetworking/Mirror/issues/3358 + +V1.28 [2023-01-28] +- fix: KcpClient.Connect now resolves hostname before creating peer + https://github.com/MirrorNetworking/Mirror/issues/3361 + +V1.27 [2023-01-08] +- KcpClient.Connect: invoke own events directly instead of going through peer, + which calls our own events anyway +- fix: KcpPeer/Client/Server callbacks are readonly and assigned in constructor + to ensure they are safe to use at all times. + fixes https://github.com/MirrorNetworking/Mirror/issues/3337 + +V1.26 [2022-12-22] +- KcpPeer.RawInput: fix compile error in old Unity Mono versions +- fix: KcpServer sets up a new connection's OnError immediately. + fixes KcpPeer throwing NullReferenceException when attempting to call OnError + after authentication errors. +- improved log messages + +V1.25 [2022-12-14] +- breaking: removed where-allocation. use IL2CPP on servers instead. +- breaking: KcpConfig to simplify configuration +- high level cleanups + +V1.24 [2022-12-14] +- KcpClient: fixed NullReferenceException when connection without a server. + added test coverage to ensure this never happens again. + +V1.23 [2022-12-07] +- KcpClient: rawReceiveBuffer exposed +- fix: KcpServer RawSend uses connection.remoteEndPoint instead of the helper + 'newClientEP'. fixes clients receiving the wrong messages meant for others. + https://github.com/MirrorNetworking/Mirror/issues/3296 + +V1.22 [2022-11-30] +- high level refactor, part two. + +V1.21 [2022-11-24] +- high level refactor, part one. + - KcpPeer instead of KcpConnection, KcpClientConnection, KcpServerConnection + - RawSend/Receive can now easily be overwritten in KcpClient/Server. + for non-alloc, relays, etc. + +V1.20 [2022-11-22] +- perf: KcpClient receive allocation was removed entirely. + reduces Mirror benchmark client sided allocations from 4.9 KB / 1.7 KB (non-alloc) to 0B. +- fix: KcpConnection.Disconnect does not check socket.Connected anymore. + UDP sockets don't have a connection. + fixes Disconnects not being sent to clients in netcore. +- KcpConnection.SendReliable: added OnError instead of logs + +V1.19 [2022-05-12] +- feature: OnError ErrorCodes + +V1.18 [2022-05-08] +- feature: OnError to allow higher level to show popups etc. +- feature: KcpServer.GetClientAddress is now GetClientEndPoint in order to + expose more details +- ResolveHostname: include exception in log for easier debugging +- fix: KcpClientConnection.RawReceive now logs the SocketException even if + it was expected. makes debugging easier. +- fix: KcpServer.TickIncoming now logs the SocketException even if it was + expected. makes debugging easier. +- fix: KcpClientConnection.RawReceive now calls Disconnect() if the other end + has closed the connection. better than just remaining in a state with unusable + sockets. + +V1.17 [2022-01-09] +- perf: server/client MaximizeSendReceiveBuffersToOSLimit option to set send/recv + buffer sizes to OS limit. avoids drops due to small buffers under heavy load. + +V1.16 [2022-01-06] +- fix: SendUnreliable respects ArraySegment.Offset +- fix: potential bug with negative length (see PR #2) +- breaking: removed pause handling because it's not necessary for Mirror anymore + +V1.15 [2021-12-11] +- feature: feature: MaxRetransmits aka dead_link now configurable +- dead_link disconnect message improved to show exact retransmit count + +V1.14 [2021-11-30] +- fix: Send() now throws an exception for messages which require > 255 fragments +- fix: ReliableMaxMessageSize is now limited to messages which require <= 255 fragments + +V1.13 [2021-11-28] +- fix: perf: uncork max message size from 144 KB to as much as we want based on + receive window size. + fixes https://github.com/vis2k/kcp2k/issues/22 + fixes https://github.com/skywind3000/kcp/pull/291 +- feature: OnData now includes channel it was received on + +V1.12 [2021-07-16] +- Tests: don't depend on Unity anymore +- fix: #26 - Kcp now catches exception if host couldn't be resolved, and calls + OnDisconnected to let the user now. +- fix: KcpServer.DualMode is now configurable in the constructor instead of + using #if UNITY_SWITCH. makes it run on all other non dual mode platforms too. +- fix: where-allocation made optional via virtuals and inheriting + KcpServer/Client/Connection NonAlloc classes. fixes a bug where some platforms + might not support where-allocation. + +V1.11 rollback [2021-06-01] +- perf: Segment MemoryStream initial capacity set to MTU to avoid early runtime + resizing/allocations + +V1.10 [2021-05-28] +- feature: configurable Timeout +- allocations explained with comments (C# ReceiveFrom / IPEndPoint.GetHashCode) +- fix: #17 KcpConnection.ReceiveNextReliable now assigns message default so it + works in .net too +- fix: Segment pool is not static anymore. Each kcp instance now has it's own + Pool. fixes #18 concurrency issues + +V1.9 [2021-03-02] +- Tick() split into TickIncoming()/TickOutgoing() to use in Mirror's new update + functions. allows to minimize latency. + => original Tick() is still supported for convenience. simply processes both! + +V1.8 [2021-02-14] +- fix: Unity IPv6 errors on Nintendo Switch +- fix: KcpConnection now disconnects if data message was received without content. + previously it would call OnData with an empty ArraySegment, causing all kinds of + weird behaviour in Mirror/DOTSNET. Added tests too. +- fix: KcpConnection.SendData: don't allow sending empty messages anymore. disconnect + and log a warning to make it completely obvious. + +V1.7 [2021-01-13] +- fix: unreliable messages reset timeout now too +- perf: KcpConnection OnCheckEnabled callback changed to a simple 'paused' boolean. + This is faster than invoking a Func every time and allows us to fix #8 more + easily later by calling .Pause/.Unpause from OnEnable/OnDisable in MirrorTransport. +- fix #8: Unpause now resets timeout to fix a bug where Mirror would pause kcp, + change the scene which took >10s, then unpause and kcp would detect the lack of + any messages for >10s as timeout. Added test to make sure it never happens again. +- MirrorTransport: statistics logging for headless servers +- Mirror Transport: Send/Receive window size increased once more from 2048 to 4096. + +V1.6 [2021-01-10] +- Unreliable channel added! +- perf: KcpHeader byte added to every kcp message to indicate + Handshake/Data/Ping/Disconnect instead of scanning each message for Hello/Byte/Ping + content via SegmentEquals. It's a lot cleaner, should be faster and should avoid + edge cases where a message content would equal Hello/Ping/Bye sequence accidentally. +- Kcp.Input: offset moved to parameters for cases where it's needed +- Kcp.SetMtu from original Kcp.c + +V1.5 [2021-01-07] +- KcpConnection.MaxSend/ReceiveRate calculation based on the article +- MirrorTransport: large send/recv window size defaults to avoid high latencies caused + by packets not being processed fast enough +- MirrorTransport: show MaxSend/ReceiveRate in debug gui +- MirrorTransport: don't Log.Info to console in headless mode if debug log is disabled + +V1.4 [2020-11-27] +- fix: OnCheckEnabled added. KcpConnection message processing while loop can now + be interrupted immediately. fixes Mirror Transport scene changes which need to stop + processing any messages immediately after a scene message) +- perf: Mirror KcpTransport: FastResend enabled by default. turbo mode according to: + https://github.com/skywind3000/kcp/blob/master/README.en.md#protocol-configuration +- perf: Mirror KcpTransport: CongestionControl disabled by default (turbo mode) + +V1.3 [2020-11-17] +- Log.Info/Warning/Error so logging doesn't depend on UnityEngine anymore +- fix: Server.Tick catches SocketException which happens if Android client is killed +- MirrorTransport: debugLog option added that can be checked in Unity Inspector +- Utils.Clamp so Kcp.cs doesn't depend on UnityEngine +- Utils.SegmentsEqual: use Linq SequenceEqual so it doesn't depend on UnityEngine +=> kcp2k can now be used in any C# project even without Unity + +V1.2 [2020-11-10] +- more tests added +- fix: raw receive buffers are now all of MTU size +- fix: raw receive detects error where buffer was too small for msgLength and + result in excess data being dropped silently +- KcpConnection.MaxMessageSize added for use in high level +- KcpConnection.MaxMessageSize increased from 1200 bytes to to maximum allowed + message size of 145KB for kcp (based on mtu, overhead, wnd_rcv) + +V1.1 [2020-10-30] +- high level cleanup, fixes, improvements + +V1.0 [2020-10-22] +- Kcp.cs now mirrors original Kcp.c behaviour + (this fixes dozens of bugs) + +V0.1 +- initial kcp-csharp based version \ No newline at end of file diff --git a/Kcp2k/kcp2k/empty/KcpClientConnection.cs b/Kcp2k/kcp2k/empty/KcpClientConnection.cs new file mode 100644 index 0000000..7926893 --- /dev/null +++ b/Kcp2k/kcp2k/empty/KcpClientConnection.cs @@ -0,0 +1 @@ +// removed 2022-11-23 \ No newline at end of file diff --git a/Kcp2k/kcp2k/empty/KcpClientConnectionNonAlloc.cs b/Kcp2k/kcp2k/empty/KcpClientConnectionNonAlloc.cs new file mode 100644 index 0000000..8c3864d --- /dev/null +++ b/Kcp2k/kcp2k/empty/KcpClientConnectionNonAlloc.cs @@ -0,0 +1 @@ +// removed 2022-11-22 \ No newline at end of file diff --git a/Kcp2k/kcp2k/empty/KcpClientNonAlloc.cs b/Kcp2k/kcp2k/empty/KcpClientNonAlloc.cs new file mode 100644 index 0000000..8c3864d --- /dev/null +++ b/Kcp2k/kcp2k/empty/KcpClientNonAlloc.cs @@ -0,0 +1 @@ +// removed 2022-11-22 \ No newline at end of file diff --git a/Kcp2k/kcp2k/empty/KcpServerConnectionNonAlloc.cs b/Kcp2k/kcp2k/empty/KcpServerConnectionNonAlloc.cs new file mode 100644 index 0000000..7926893 --- /dev/null +++ b/Kcp2k/kcp2k/empty/KcpServerConnectionNonAlloc.cs @@ -0,0 +1 @@ +// removed 2022-11-23 \ No newline at end of file diff --git a/Kcp2k/kcp2k/empty/KcpServerNonAlloc.cs b/Kcp2k/kcp2k/empty/KcpServerNonAlloc.cs new file mode 100644 index 0000000..4623b53 --- /dev/null +++ b/Kcp2k/kcp2k/empty/KcpServerNonAlloc.cs @@ -0,0 +1 @@ +// removed 2022-12-13 \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/Common.cs b/Kcp2k/kcp2k/highlevel/Common.cs new file mode 100644 index 0000000..cc310a0 --- /dev/null +++ b/Kcp2k/kcp2k/highlevel/Common.cs @@ -0,0 +1,49 @@ +using System.Net; +using System.Net.Sockets; + +namespace kcp2k +{ + public static class Common + { + // helper function to resolve host to IPAddress + public static bool ResolveHostname(string hostname, out IPAddress[] addresses) + { + try + { + // NOTE: dns lookup is blocking. this can take a second. + addresses = Dns.GetHostAddresses(hostname); + return addresses.Length >= 1; + } + catch (SocketException exception) + { + Log.Info($"Failed to resolve host: {hostname} reason: {exception}"); + addresses = null; + return false; + } + } + + // if connections drop under heavy load, increase to OS limit. + // if still not enough, increase the OS limit. + public static void ConfigureSocketBuffers(Socket socket, int recvBufferSize, int sendBufferSize) + { + // log initial size for comparison. + // remember initial size for log comparison + int initialReceive = socket.ReceiveBufferSize; + int initialSend = socket.SendBufferSize; + + // set to configured size + try + { + socket.ReceiveBufferSize = recvBufferSize; + socket.SendBufferSize = sendBufferSize; + } + catch (SocketException) + { + Log.Warning($"Kcp: failed to set Socket RecvBufSize = {recvBufferSize} SendBufSize = {sendBufferSize}"); + } + + + Log.Info($"Kcp: RecvBuf = {initialReceive}=>{socket.ReceiveBufferSize} ({socket.ReceiveBufferSize/initialReceive}x) SendBuf = {initialSend}=>{socket.SendBufferSize} ({socket.SendBufferSize/initialSend}x)"); + } + } +} diff --git a/Kcp2k/kcp2k/highlevel/ErrorCode.cs b/Kcp2k/kcp2k/highlevel/ErrorCode.cs new file mode 100644 index 0000000..15b872f --- /dev/null +++ b/Kcp2k/kcp2k/highlevel/ErrorCode.cs @@ -0,0 +1,15 @@ +// kcp specific error codes to allow for error switching, localization, +// translation to Mirror errors, etc. +namespace kcp2k +{ + public enum ErrorCode : byte + { + DnsResolve, // failed to resolve a host name + Timeout, // ping timeout or dead link + Congestion, // more messages than transport / network can process + InvalidReceive, // recv invalid packet (possibly intentional attack) + InvalidSend, // user tried to send invalid data + ConnectionClosed, // connection closed voluntarily or lost involuntarily + Unexpected // unexpected error / exception, requires fix. + } +} \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/Extensions.cs b/Kcp2k/kcp2k/highlevel/Extensions.cs new file mode 100644 index 0000000..490a7fd --- /dev/null +++ b/Kcp2k/kcp2k/highlevel/Extensions.cs @@ -0,0 +1,162 @@ +using System; +using System.Net; +using System.Net.Sockets; + +namespace kcp2k +{ + public static class Extensions + { + // non-blocking UDP send. + // allows for reuse when overwriting KcpServer/Client (i.e. for relays). + // => wrapped with Poll to avoid WouldBlock allocating new SocketException. + // => wrapped with try-catch to ignore WouldBlock exception. + // make sure to set socket.Blocking = false before using this! + public static bool SendToNonBlocking(this Socket socket, ArraySegment data, EndPoint remoteEP) + { + try + { + // when using non-blocking sockets, SendTo may return WouldBlock. + // in C#, WouldBlock throws a SocketException, which is expected. + // unfortunately, creating the SocketException allocates in C#. + // let's poll first to avoid the WouldBlock allocation. + // note that this entirely to avoid allocations. + // non-blocking UDP doesn't need Poll in other languages. + // and the code still works without the Poll call. + if (!socket.Poll(0, SelectMode.SelectWrite)) return false; + + // send to the the endpoint. + // do not send to 'newClientEP', as that's always reused. + // fixes https://github.com/MirrorNetworking/Mirror/issues/3296 + socket.SendTo(data.Array, data.Offset, data.Count, SocketFlags.None, remoteEP); + return true; + } + catch (SocketException e) + { + // for non-blocking sockets, SendTo may throw WouldBlock. + // in that case, simply drop the message. it's UDP, it's fine. + if (e.SocketErrorCode == SocketError.WouldBlock) return false; + + // otherwise it's a real socket error. throw it. + throw; + } + } + + // non-blocking UDP send. + // allows for reuse when overwriting KcpServer/Client (i.e. for relays). + // => wrapped with Poll to avoid WouldBlock allocating new SocketException. + // => wrapped with try-catch to ignore WouldBlock exception. + // make sure to set socket.Blocking = false before using this! + public static bool SendNonBlocking(this Socket socket, ArraySegment data) + { + try + { + // when using non-blocking sockets, SendTo may return WouldBlock. + // in C#, WouldBlock throws a SocketException, which is expected. + // unfortunately, creating the SocketException allocates in C#. + // let's poll first to avoid the WouldBlock allocation. + // note that this entirely to avoid allocations. + // non-blocking UDP doesn't need Poll in other languages. + // and the code still works without the Poll call. + if (!socket.Poll(0, SelectMode.SelectWrite)) return false; + + // SendTo allocates. we used bound Send. + socket.Send(data.Array, data.Offset, data.Count, SocketFlags.None); + return true; + } + catch (SocketException e) + { + // for non-blocking sockets, SendTo may throw WouldBlock. + // in that case, simply drop the message. it's UDP, it's fine. + if (e.SocketErrorCode == SocketError.WouldBlock) return false; + + // otherwise it's a real socket error. throw it. + throw; + } + } + + // non-blocking UDP receive. + // allows for reuse when overwriting KcpServer/Client (i.e. for relays). + // => wrapped with Poll to avoid WouldBlock allocating new SocketException. + // => wrapped with try-catch to ignore WouldBlock exception. + // make sure to set socket.Blocking = false before using this! + public static bool ReceiveFromNonBlocking(this Socket socket, byte[] recvBuffer, out ArraySegment data, ref EndPoint remoteEP) + { + data = default; + + try + { + // when using non-blocking sockets, ReceiveFrom may return WouldBlock. + // in C#, WouldBlock throws a SocketException, which is expected. + // unfortunately, creating the SocketException allocates in C#. + // let's poll first to avoid the WouldBlock allocation. + // note that this entirely to avoid allocations. + // non-blocking UDP doesn't need Poll in other languages. + // and the code still works without the Poll call. + if (!socket.Poll(0, SelectMode.SelectRead)) return false; + + // NOTE: ReceiveFrom allocates. + // we pass our IPEndPoint to ReceiveFrom. + // receive from calls newClientEP.Create(socketAddr). + // IPEndPoint.Create always returns a new IPEndPoint. + // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761 + // + // throws SocketException if datagram was larger than buffer. + // https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0 + int size = socket.ReceiveFrom(recvBuffer, 0, recvBuffer.Length, SocketFlags.None, ref remoteEP); + data = new ArraySegment(recvBuffer, 0, size); + return true; + } + catch (SocketException e) + { + // for non-blocking sockets, Receive throws WouldBlock if there is + // no message to read. that's okay. only log for other errors. + if (e.SocketErrorCode == SocketError.WouldBlock) return false; + + // otherwise it's a real socket error. throw it. + throw; + } + } + + // non-blocking UDP receive. + // allows for reuse when overwriting KcpServer/Client (i.e. for relays). + // => wrapped with Poll to avoid WouldBlock allocating new SocketException. + // => wrapped with try-catch to ignore WouldBlock exception. + // make sure to set socket.Blocking = false before using this! + public static bool ReceiveNonBlocking(this Socket socket, byte[] recvBuffer, out ArraySegment data) + { + data = default; + + try + { + // when using non-blocking sockets, ReceiveFrom may return WouldBlock. + // in C#, WouldBlock throws a SocketException, which is expected. + // unfortunately, creating the SocketException allocates in C#. + // let's poll first to avoid the WouldBlock allocation. + // note that this entirely to avoid allocations. + // non-blocking UDP doesn't need Poll in other languages. + // and the code still works without the Poll call. + if (!socket.Poll(0, SelectMode.SelectRead)) return false; + + // ReceiveFrom allocates. we used bound Receive. + // returns amount of bytes written into buffer. + // throws SocketException if datagram was larger than buffer. + // https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0 + // + // throws SocketException if datagram was larger than buffer. + // https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.receive?view=net-6.0 + int size = socket.Receive(recvBuffer, 0, recvBuffer.Length, SocketFlags.None); + data = new ArraySegment(recvBuffer, 0, size); + return true; + } + catch (SocketException e) + { + // for non-blocking sockets, Receive throws WouldBlock if there is + // no message to read. that's okay. only log for other errors. + if (e.SocketErrorCode == SocketError.WouldBlock) return false; + + // otherwise it's a real socket error. throw it. + throw; + } + } + } +} \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/KcpChannel.cs b/Kcp2k/kcp2k/highlevel/KcpChannel.cs index ccb19ba..c085e17 100644 --- a/Kcp2k/kcp2k/highlevel/KcpChannel.cs +++ b/Kcp2k/kcp2k/highlevel/KcpChannel.cs @@ -4,7 +4,7 @@ namespace kcp2k public enum KcpChannel : byte { // don't react on 0x00. might help to filter out random noise. - Reliable = 0x01, - Unreliable = 0x02 + Reliable = 1, + Unreliable = 2 } } \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/KcpClient.cs b/Kcp2k/kcp2k/highlevel/KcpClient.cs index 64a005a..4c9cd3b 100644 --- a/Kcp2k/kcp2k/highlevel/KcpClient.cs +++ b/Kcp2k/kcp2k/highlevel/KcpClient.cs @@ -1,74 +1,176 @@ // kcp client logic abstracted into a class. // for use in Mirror, DOTSNET, testing, etc. using System; +using System.Net; +using System.Net.Sockets; namespace kcp2k { public class KcpClient { - // events - public Action OnConnected; - public Action> OnData; - public Action OnDisconnected; + // kcp + // public so that bandwidth statistics can be accessed from the outside + public KcpPeer peer; + + // IO + protected Socket socket; + public EndPoint remoteEndPoint; + + // config + readonly KcpConfig config; + + // raw receive buffer always needs to be of 'MTU' size, even if + // MaxMessageSize is larger. kcp always sends in MTU segments and having + // a buffer smaller than MTU would silently drop excess data. + // => we need the MTU to fit channel + message! + // => protected because someone may overwrite RawReceive but still wants + // to reuse the buffer. + protected readonly byte[] rawReceiveBuffer; + + // callbacks + // even for errors, to allow liraries to show popups etc. + // instead of logging directly. + // (string instead of Exception for ease of use and to avoid user panic) + // + // events are readonly, set in constructor. + // this ensures they are always initialized when used. + // fixes https://github.com/MirrorNetworking/Mirror/issues/3337 and more + readonly Action OnConnected; + readonly Action, KcpChannel> OnData; + readonly Action OnDisconnected; + readonly Action OnError; // state - public KcpClientConnection connection; public bool connected; - public KcpClient(Action OnConnected, Action> OnData, Action OnDisconnected) + public KcpClient(Action OnConnected, + Action, KcpChannel> OnData, + Action OnDisconnected, + Action OnError, + KcpConfig config) { + // initialize callbacks first to ensure they can be used safely. this.OnConnected = OnConnected; this.OnData = OnData; this.OnDisconnected = OnDisconnected; - } + this.OnError = OnError; + this.config = config; - // CreateConnection can be overwritten for where-allocation: - // https://github.com/vis2k/where-allocation - protected virtual KcpClientConnection CreateConnection() => - new KcpClientConnection(); + // create mtu sized receive buffer + rawReceiveBuffer = new byte[config.Mtu]; + } - public void Connect(string address, ushort port, bool noDelay, uint interval, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = KcpConnection.DEFAULT_TIMEOUT) + public void Connect(string address, ushort port) { if (connected) { - Log.Warning("KCP: client already connected!"); + Log.Warning("KcpClient: already connected!"); return; } - // create connection - connection = CreateConnection(); + // resolve host name before creating peer. + // fixes: https://github.com/MirrorNetworking/Mirror/issues/3361 + if (!Common.ResolveHostname(address, out IPAddress[] addresses)) + { + // pass error to user callback. no need to log it manually. + OnError(ErrorCode.DnsResolve, $"Failed to resolve host: {address}"); + OnDisconnected(); + return; + } - // setup events - connection.OnAuthenticated = () => + // create fresh peer for each new session + peer = new KcpPeer(RawSend, OnAuthenticatedWrap, OnData, OnDisconnectedWrap, OnError, config); + + // some callbacks need to wrapped with some extra logic + void OnAuthenticatedWrap() { - Log.Info($"KCP: OnClientConnected"); + Log.Info($"KcpClient: OnConnected"); connected = true; - OnConnected.Invoke(); - }; - connection.OnData = (message) => - { - //Log.Debug($"KCP: OnClientData({BitConverter.ToString(message.Array, message.Offset, message.Count)})"); - OnData.Invoke(message); - }; - connection.OnDisconnected = () => + OnConnected(); + } + void OnDisconnectedWrap() { - Log.Info($"KCP: OnClientDisconnected"); + Log.Info($"KcpClient: OnDisconnected"); connected = false; - connection = null; - OnDisconnected.Invoke(); - }; + peer = null; + socket?.Close(); + socket = null; + remoteEndPoint = null; + OnDisconnected(); + } + + Log.Info($"KcpClient: connect to {address}:{port}"); + + // create socket + remoteEndPoint = new IPEndPoint(addresses[0], port); + socket = new Socket(remoteEndPoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + + // recv & send are called from main thread. + // need to ensure this never blocks. + // even a 1ms block per connection would stop us from scaling. + socket.Blocking = false; + + // configure buffer sizes + Common.ConfigureSocketBuffers(socket, config.RecvBufferSize, config.SendBufferSize); + + // bind to endpoint so we can use send/recv instead of sendto/recvfrom. + socket.Connect(remoteEndPoint); - // connect - connection.Connect(address, port, noDelay, interval, fastResend, congestionWindow, sendWindowSize, receiveWindowSize, timeout); + // client should send handshake to server as very first message + peer.SendHandshake(); + } + + // io - input. + // virtual so it may be modified for relays, etc. + // call this while it returns true, to process all messages this tick. + // returned ArraySegment is valid until next call to RawReceive. + protected virtual bool RawReceive(out ArraySegment segment) + { + segment = default; + if (socket == null) return false; + + try + { + return socket.ReceiveNonBlocking(rawReceiveBuffer, out segment); + } + // for non-blocking sockets, Receive throws WouldBlock if there is + // no message to read. that's okay. only log for other errors. + catch (SocketException e) + { + // the other end closing the connection is not an 'error'. + // but connections should never just end silently. + // at least log a message for easier debugging. + // for example, his can happen when connecting without a server. + // see test: ConnectWithoutServer(). + Log.Info($"KcpClient: looks like the other end has closed the connection. This is fine: {e}"); + peer.Disconnect(); + return false; + } + } + + // io - output. + // virtual so it may be modified for relays, etc. + protected virtual void RawSend(ArraySegment data) + { + try + { + socket.SendNonBlocking(data); + } + catch (SocketException e) + { + Log.Error($"KcpClient: Send failed: {e}"); + } } public void Send(ArraySegment segment, KcpChannel channel) { - if (connected) + if (!connected) { - connection.SendData(segment, channel); + Log.Warning("KcpClient: can't send because not connected!"); + return; } - else Log.Warning("KCP: can't send because client not connected!"); + + peer.SendData(segment, channel); } public void Disconnect() @@ -76,45 +178,48 @@ public void Disconnect() // only if connected // otherwise we end up in a deadlock because of an open Mirror bug: // https://github.com/vis2k/Mirror/issues/2353 - if (connected) - { - // call Disconnect and let the connection handle it. - // DO NOT set it to null yet. it needs to be updated a few more - // times first. let the connection handle it! - connection?.Disconnect(); - } + if (!connected) return; + + // call Disconnect and let the connection handle it. + // DO NOT set it to null yet. it needs to be updated a few more + // times first. let the connection handle it! + peer?.Disconnect(); } // process incoming messages. should be called before updating the world. - public void TickIncoming() + // virtual because relay may need to inject their own ping or similar. + public virtual void TickIncoming() { // recv on socket first, then process incoming // (even if we didn't receive anything. need to tick ping etc.) // (connection is null if not active) - connection?.RawReceive(); - connection?.TickIncoming(); + if (peer != null) + { + + while (RawReceive(out ArraySegment segment)) + peer.RawInput(segment); + } + + // RawReceive may have disconnected peer. null check again. + peer?.TickIncoming(); } // process outgoing messages. should be called after updating the world. - public void TickOutgoing() + // virtual because relay may need to inject their own ping or similar. + public virtual void TickOutgoing() { // process outgoing // (connection is null if not active) - connection?.TickOutgoing(); + peer?.TickOutgoing(); } // process incoming and outgoing for convenience // => ideally call ProcessIncoming() before updating the world and // ProcessOutgoing() after updating the world for minimum latency - public void Tick() + public virtual void Tick() { TickIncoming(); TickOutgoing(); } - - // pause/unpause to safely support mirror scene handling and to - // immediately pause the receive while loop if needed. - public void Pause() => connection?.Pause(); - public void Unpause() => connection?.Unpause(); } } diff --git a/Kcp2k/kcp2k/highlevel/KcpClientConnection.cs b/Kcp2k/kcp2k/highlevel/KcpClientConnection.cs deleted file mode 100644 index 9bde4af..0000000 --- a/Kcp2k/kcp2k/highlevel/KcpClientConnection.cs +++ /dev/null @@ -1,109 +0,0 @@ -using System.Net; -using System.Net.Sockets; - -namespace kcp2k -{ - public class KcpClientConnection : KcpConnection - { - // IMPORTANT: raw receive buffer always needs to be of 'MTU' size, even - // if MaxMessageSize is larger. kcp always sends in MTU - // segments and having a buffer smaller than MTU would - // silently drop excess data. - // => we need the MTU to fit channel + message! - readonly byte[] rawReceiveBuffer = new byte[Kcp.MTU_DEF]; - - // helper function to resolve host to IPAddress - public static bool ResolveHostname(string hostname, out IPAddress[] addresses) - { - try - { - addresses = Dns.GetHostAddresses(hostname); - return addresses.Length >= 1; - } - catch (SocketException) - { - Log.Info($"Failed to resolve host: {hostname}"); - addresses = null; - return false; - } - } - - // EndPoint & Receive functions can be overwritten for where-allocation: - // https://github.com/vis2k/where-allocation - // NOTE: Client's SendTo doesn't allocate, don't need a virtual. - protected virtual void CreateRemoteEndPoint(IPAddress[] addresses, ushort port) => - remoteEndPoint = new IPEndPoint(addresses[0], port); - - protected virtual int ReceiveFrom(byte[] buffer) => - socket.ReceiveFrom(buffer, ref remoteEndPoint); - - public void Connect(string host, ushort port, bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT) - { - Log.Info($"KcpClient: connect to {host}:{port}"); - - // try resolve host name - if (ResolveHostname(host, out IPAddress[] addresses)) - { - // create remote endpoint - CreateRemoteEndPoint(addresses, port); - - // create socket - socket = new Socket(remoteEndPoint.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - socket.Connect(remoteEndPoint); - - // set up kcp - SetupKcp(noDelay, interval, fastResend, congestionWindow, sendWindowSize, receiveWindowSize, timeout); - - // client should send handshake to server as very first message - SendHandshake(); - - RawReceive(); - } - // otherwise call OnDisconnected to let the user know. - else OnDisconnected(); - } - - - // call from transport update - public void RawReceive() - { - try - { - if (socket != null) - { - while (socket.Poll(0, SelectMode.SelectRead)) - { - int msgLength = ReceiveFrom(rawReceiveBuffer); - // IMPORTANT: detect if buffer was too small for the - // received msgLength. otherwise the excess - // data would be silently lost. - // (see ReceiveFrom documentation) - if (msgLength <= rawReceiveBuffer.Length) - { - //Log.Debug($"KCP: client raw recv {msgLength} bytes = {BitConverter.ToString(buffer, 0, msgLength)}"); - RawInput(rawReceiveBuffer, msgLength); - } - else - { - Log.Error($"KCP ClientConnection: message of size {msgLength} does not fit into buffer of size {rawReceiveBuffer.Length}. The excess was silently dropped. Disconnecting."); - Disconnect(); - } - } - } - } - // this is fine, the socket might have been closed in the other end - catch (SocketException) {} - } - - protected override void Dispose() - { - socket.Close(); - socket = null; - } - - protected override void RawSend(byte[] data, int length) - { - socket.Send(data, length, SocketFlags.None); - } - } -} diff --git a/Kcp2k/kcp2k/highlevel/KcpConfig.cs b/Kcp2k/kcp2k/highlevel/KcpConfig.cs new file mode 100644 index 0000000..382d06b --- /dev/null +++ b/Kcp2k/kcp2k/highlevel/KcpConfig.cs @@ -0,0 +1,97 @@ +// common config struct, instead of passing 10 parameters manually every time. +using System; + +namespace kcp2k +{ + // [Serializable] to show it in Unity inspector. + // 'class' so we can set defaults easily. + [Serializable] + public class KcpConfig + { + // socket configuration //////////////////////////////////////////////// + // DualMode uses both IPv6 and IPv4. not all platforms support it. + // (Nintendo Switch, etc.) + public bool DualMode; + + // UDP servers use only one socket. + // maximize buffer to handle as many connections as possible. + // + // M1 mac pro: + // recv buffer default: 786896 (771 KB) + // send buffer default: 9216 (9 KB) + // max configurable: ~7 MB + public int RecvBufferSize; + public int SendBufferSize; + + // kcp configuration /////////////////////////////////////////////////// + // configurable MTU in case kcp sits on top of other abstractions like + // encrypted transports, relays, etc. + public int Mtu; + + // NoDelay is recommended to reduce latency. This also scales better + // without buffers getting full. + public bool NoDelay; + + // KCP internal update interval. 100ms is KCP default, but a lower + // interval is recommended to minimize latency and to scale to more + // networked entities. + public uint Interval; + + // KCP fastresend parameter. Faster resend for the cost of higher + // bandwidth. + public int FastResend; + + // KCP congestion window heavily limits messages flushed per update. + // congestion window may actually be broken in kcp: + // - sending max sized message @ M1 mac flushes 2-3 messages per update + // - even with super large send/recv window, it requires thousands of + // update calls + // best to leave this disabled, as it may significantly increase latency. + public bool CongestionWindow; + + // KCP window size can be modified to support higher loads. + // for example, Mirror Benchmark requires: + // 128, 128 for 4k monsters + // 512, 512 for 10k monsters + // 8192, 8192 for 20k monsters + public uint SendWindowSize; + public uint ReceiveWindowSize; + + // timeout in milliseconds + public int Timeout; + + // maximum retransmission attempts until dead_link + public uint MaxRetransmits; + + // constructor ///////////////////////////////////////////////////////// + // constructor with defaults for convenience. + // makes it easy to define "new KcpConfig(DualMode=false)" etc. + public KcpConfig( + bool DualMode = true, + int RecvBufferSize = 1024 * 1024 * 7, + int SendBufferSize = 1024 * 1024 * 7, + int Mtu = Kcp.MTU_DEF, + bool NoDelay = true, + uint Interval = 10, + int FastResend = 0, + bool CongestionWindow = false, + uint SendWindowSize = Kcp.WND_SND, + uint ReceiveWindowSize = Kcp.WND_RCV, + int Timeout = KcpPeer.DEFAULT_TIMEOUT, + uint MaxRetransmits = Kcp.DEADLINK) + { + this.DualMode = DualMode; + this.RecvBufferSize = RecvBufferSize; + this.SendBufferSize = SendBufferSize; + this.Mtu = Mtu; + this.NoDelay = NoDelay; + this.Interval = Interval; + this.FastResend = FastResend; + this.CongestionWindow = CongestionWindow; + this.SendWindowSize = SendWindowSize; + this.ReceiveWindowSize = ReceiveWindowSize; + this.Timeout = Timeout; + this.MaxRetransmits = MaxRetransmits; + } + } +} diff --git a/Kcp2k/kcp2k/highlevel/KcpConnection.cs b/Kcp2k/kcp2k/highlevel/KcpConnection.cs deleted file mode 100644 index ecfe562..0000000 --- a/Kcp2k/kcp2k/highlevel/KcpConnection.cs +++ /dev/null @@ -1,674 +0,0 @@ -using System; -using System.Diagnostics; -using System.Net; -using System.Net.Sockets; - -namespace kcp2k -{ - enum KcpState { Connected, Authenticated, Disconnected } - - public abstract class KcpConnection - { - protected Socket socket; - protected EndPoint remoteEndPoint; - internal Kcp kcp; - - // kcp can have several different states, let's use a state machine - KcpState state = KcpState.Disconnected; - - public Action OnAuthenticated; - public Action> OnData; - public Action OnDisconnected; - - // Mirror needs a way to stop the kcp message processing while loop - // immediately after a scene change message. Mirror can't process any - // other messages during a scene change. - // (could be useful for others too) - bool paused; - - // If we don't receive anything these many milliseconds - // then consider us disconnected - public const int DEFAULT_TIMEOUT = 10000; - public int timeout = DEFAULT_TIMEOUT; - uint lastReceiveTime; - - // internal time. - // StopWatch offers ElapsedMilliSeconds and should be more precise than - // Unity's time.deltaTime over long periods. - readonly Stopwatch refTime = new Stopwatch(); - - // we need to subtract the channel byte from every MaxMessageSize - // calculation. - // we also need to tell kcp to use MTU-1 to leave space for the byte. - const int CHANNEL_HEADER_SIZE = 1; - - // reliable channel (= kcp) MaxMessageSize so the outside knows largest - // allowed message to send the calculation in Send() is not obvious at - // all, so let's provide the helper here. - // - // kcp does fragmentation, so max message is way larger than MTU. - // - // -> runtime MTU changes are disabled: mss is always MTU_DEF-OVERHEAD - // -> Send() checks if fragment count < WND_RCV, so we use WND_RCV - 1. - // note that Send() checks WND_RCV instead of wnd_rcv which may or - // may not be a bug in original kcp. but since it uses the define, we - // can use that here too. - // -> we add 1 byte KcpHeader enum to each message, so -1 - // - // IMPORTANT: max message is MTU * WND_RCV, in other words it completely - // fills the receive window! due to head of line blocking, - // all other messages have to wait while a maxed size message - // is being delivered. - // => in other words, DO NOT use max size all the time like - // for batching. - // => sending UNRELIABLE max message size most of the time is - // best for performance (use that one for batching!) - public const int ReliableMaxMessageSize = (Kcp.MTU_DEF - Kcp.OVERHEAD - CHANNEL_HEADER_SIZE) * (Kcp.WND_RCV - 1) - 1; - - // unreliable max message size is simply MTU - channel header size - public const int UnreliableMaxMessageSize = Kcp.MTU_DEF - CHANNEL_HEADER_SIZE; - - // buffer to receive kcp's processed messages (avoids allocations). - // IMPORTANT: this is for KCP messages. so it needs to be of size: - // 1 byte header + MaxMessageSize content - byte[] kcpMessageBuffer = new byte[1 + ReliableMaxMessageSize]; - - // send buffer for handing user messages to kcp for processing. - // (avoids allocations). - // IMPORTANT: needs to be of size: - // 1 byte header + MaxMessageSize content - byte[] kcpSendBuffer = new byte[1 + ReliableMaxMessageSize]; - - // raw send buffer is exactly MTU. - byte[] rawSendBuffer = new byte[Kcp.MTU_DEF]; - - // send a ping occasionally so we don't time out on the other end. - // for example, creating a character in an MMO could easily take a - // minute of no data being sent. which doesn't mean we want to time out. - // same goes for slow paced card games etc. - public const int PING_INTERVAL = 1000; - uint lastPingTime; - - // if we send more than kcp can handle, we will get ever growing - // send/recv buffers and queues and minutes of latency. - // => if a connection can't keep up, it should be disconnected instead - // to protect the server under heavy load, and because there is no - // point in growing to gigabytes of memory or minutes of latency! - // => 2k isn't enough. we reach 2k when spawning 4k monsters at once - // easily, but it does recover over time. - // => 10k seems safe. - // - // note: we have a ChokeConnectionAutoDisconnects test for this too! - internal const int QueueDisconnectThreshold = 10000; - - // getters for queue and buffer counts, used for debug info - public int SendQueueCount => kcp.snd_queue.Count; - public int ReceiveQueueCount => kcp.rcv_queue.Count; - public int SendBufferCount => kcp.snd_buf.Count; - public int ReceiveBufferCount => kcp.rcv_buf.Count; - - // maximum send rate per second can be calculated from kcp parameters - // source: https://translate.google.com/translate?sl=auto&tl=en&u=https://wetest.qq.com/lab/view/391.html - // - // KCP can send/receive a maximum of WND*MTU per interval. - // multiple by 1000ms / interval to get the per-second rate. - // - // example: - // WND(32) * MTU(1400) = 43.75KB - // => 43.75KB * 1000 / INTERVAL(10) = 4375KB/s - // - // returns bytes/second! - public uint MaxSendRate => - kcp.snd_wnd * kcp.mtu * 1000 / kcp.interval; - - public uint MaxReceiveRate => - kcp.rcv_wnd * kcp.mtu * 1000 / kcp.interval; - - // SetupKcp creates and configures a new KCP instance. - // => useful to start from a fresh state every time the client connects - // => NoDelay, interval, wnd size are the most important configurations. - // let's force require the parameters so we don't forget it anywhere. - protected void SetupKcp(bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT) - { - // set up kcp over reliable channel (that's what kcp is for) - kcp = new Kcp(0, RawSendReliable); - // set nodelay. - // note that kcp uses 'nocwnd' internally so we negate the parameter - kcp.SetNoDelay(noDelay ? 1u : 0u, interval, fastResend, !congestionWindow); - kcp.SetWindowSize(sendWindowSize, receiveWindowSize); - - // IMPORTANT: high level needs to add 1 channel byte to each raw - // message. so while Kcp.MTU_DEF is perfect, we actually need to - // tell kcp to use MTU-1 so we can still put the header into the - // message afterwards. - kcp.SetMtu(Kcp.MTU_DEF - CHANNEL_HEADER_SIZE); - - this.timeout = timeout; - state = KcpState.Connected; - - refTime.Start(); - } - - void HandleTimeout(uint time) - { - // note: we are also sending a ping regularly, so timeout should - // only ever happen if the connection is truly gone. - if (time >= lastReceiveTime + timeout) - { - Log.Warning($"KCP: Connection timed out after not receiving any message for {timeout}ms. Disconnecting."); - Disconnect(); - } - } - - void HandleDeadLink() - { - // kcp has 'dead_link' detection. might as well use it. - if (kcp.state == -1) - { - Log.Warning("KCP Connection dead_link detected. Disconnecting."); - Disconnect(); - } - } - - // send a ping occasionally in order to not time out on the other end. - void HandlePing(uint time) - { - // enough time elapsed since last ping? - if (time >= lastPingTime + PING_INTERVAL) - { - // ping again and reset time - //Log.Debug("KCP: sending ping..."); - SendPing(); - lastPingTime = time; - } - } - - void HandleChoked() - { - // disconnect connections that can't process the load. - // see QueueSizeDisconnect comments. - // => include all of kcp's buffers and the unreliable queue! - int total = kcp.rcv_queue.Count + kcp.snd_queue.Count + - kcp.rcv_buf.Count + kcp.snd_buf.Count; - if (total >= QueueDisconnectThreshold) - { - Log.Warning($"KCP: disconnecting connection because it can't process data fast enough.\n" + - $"Queue total {total}>{QueueDisconnectThreshold}. rcv_queue={kcp.rcv_queue.Count} snd_queue={kcp.snd_queue.Count} rcv_buf={kcp.rcv_buf.Count} snd_buf={kcp.snd_buf.Count}\n" + - $"* Try to Enable NoDelay, decrease INTERVAL, disable Congestion Window (= enable NOCWND!), increase SEND/RECV WINDOW or compress data.\n" + - $"* Or perhaps the network is simply too slow on our end, or on the other end.\n"); - - // let's clear all pending sends before disconnting with 'Bye'. - // otherwise a single Flush in Disconnect() won't be enough to - // flush thousands of messages to finally deliver 'Bye'. - // this is just faster and more robust. - kcp.snd_queue.Clear(); - - Disconnect(); - } - } - - // reads the next reliable message type & content from kcp. - // -> to avoid buffering, unreliable messages call OnData directly. - bool ReceiveNextReliable(out KcpHeader header, out ArraySegment message) - { - int msgSize = kcp.PeekSize(); - if (msgSize > 0) - { - // only allow receiving up to buffer sized messages. - // otherwise we would get BlockCopy ArgumentException anyway. - if (msgSize <= kcpMessageBuffer.Length) - { - // receive from kcp - int received = kcp.Receive(kcpMessageBuffer, msgSize); - if (received >= 0) - { - // extract header & content without header - header = (KcpHeader)kcpMessageBuffer[0]; - message = new ArraySegment(kcpMessageBuffer, 1, msgSize - 1); - lastReceiveTime = (uint)refTime.ElapsedMilliseconds; - return true; - } - else - { - // if receive failed, close everything - Log.Warning($"Receive failed with error={received}. closing connection."); - Disconnect(); - } - } - // we don't allow sending messages > Max, so this must be an - // attacker. let's disconnect to avoid allocation attacks etc. - else - { - Log.Warning($"KCP: possible allocation attack for msgSize {msgSize} > buffer {kcpMessageBuffer.Length}. Disconnecting the connection."); - Disconnect(); - } - } - - message = default; - header = KcpHeader.Disconnect; - return false; - } - - void TickIncoming_Connected(uint time) - { - // detect common events & ping - HandleTimeout(time); - HandleDeadLink(); - HandlePing(time); - HandleChoked(); - - // any reliable kcp message received? - if (ReceiveNextReliable(out KcpHeader header, out ArraySegment message)) - { - // message type FSM. no default so we never miss a case. - switch (header) - { - case KcpHeader.Handshake: - { - // we were waiting for a handshake. - // it proves that the other end speaks our protocol. - Log.Info("KCP: received handshake"); - state = KcpState.Authenticated; - OnAuthenticated?.Invoke(); - break; - } - case KcpHeader.Ping: - { - // ping keeps kcp from timing out. do nothing. - break; - } - case KcpHeader.Data: - case KcpHeader.Disconnect: - { - // everything else is not allowed during handshake! - Log.Warning($"KCP: received invalid header {header} while Connected. Disconnecting the connection."); - Disconnect(); - break; - } - } - } - } - - void TickIncoming_Authenticated(uint time) - { - // detect common events & ping - HandleTimeout(time); - HandleDeadLink(); - HandlePing(time); - HandleChoked(); - - // process all received messages - // - // Mirror scene changing requires transports to immediately stop - // processing any more messages after a scene message was - // received. and since we are in a while loop here, we need this - // extra check. - // - // note while that this is mainly for Mirror, but might be - // useful in other applications too. - // - // note that we check it BEFORE ever calling ReceiveNext. otherwise - // we would silently eat the received message and never process it. - while (!paused && - ReceiveNextReliable(out KcpHeader header, out ArraySegment message)) - { - // message type FSM. no default so we never miss a case. - switch (header) - { - case KcpHeader.Handshake: - { - // should never receive another handshake after auth - Log.Warning($"KCP: received invalid header {header} while Authenticated. Disconnecting the connection."); - Disconnect(); - break; - } - case KcpHeader.Data: - { - // call OnData IF the message contained actual data - if (message.Count > 0) - { - //Log.Warning($"Kcp recv msg: {BitConverter.ToString(message.Array, message.Offset, message.Count)}"); - OnData?.Invoke(message); - } - // empty data = attacker, or something went wrong - else - { - Log.Warning("KCP: received empty Data message while Authenticated. Disconnecting the connection."); - Disconnect(); - } - break; - } - case KcpHeader.Ping: - { - // ping keeps kcp from timing out. do nothing. - break; - } - case KcpHeader.Disconnect: - { - // disconnect might happen - Log.Info("KCP: received disconnect message"); - Disconnect(); - break; - } - } - } - } - - public void TickIncoming() - { - uint time = (uint)refTime.ElapsedMilliseconds; - - try - { - switch (state) - { - case KcpState.Connected: - { - TickIncoming_Connected(time); - break; - } - case KcpState.Authenticated: - { - TickIncoming_Authenticated(time); - break; - } - case KcpState.Disconnected: - { - // do nothing while disconnected - break; - } - } - } - catch (SocketException exception) - { - // this is ok, the connection was closed - Log.Info($"KCP Connection: Disconnecting because {exception}. This is fine."); - Disconnect(); - } - catch (ObjectDisposedException exception) - { - // fine, socket was closed - Log.Info($"KCP Connection: Disconnecting because {exception}. This is fine."); - Disconnect(); - } - catch (Exception ex) - { - // unexpected - Log.Error(ex.ToString()); - Disconnect(); - } - } - - public void TickOutgoing() - { - uint time = (uint)refTime.ElapsedMilliseconds; - - try - { - switch (state) - { - case KcpState.Connected: - case KcpState.Authenticated: - { - // update flushes out messages - kcp.Update(time); - break; - } - case KcpState.Disconnected: - { - // do nothing while disconnected - break; - } - } - } - catch (SocketException exception) - { - // this is ok, the connection was closed - Log.Info($"KCP Connection: Disconnecting because {exception}. This is fine."); - Disconnect(); - } - catch (ObjectDisposedException exception) - { - // fine, socket was closed - Log.Info($"KCP Connection: Disconnecting because {exception}. This is fine."); - Disconnect(); - } - catch (Exception ex) - { - // unexpected - Log.Error(ex.ToString()); - Disconnect(); - } - } - - public void RawInput(byte[] buffer, int msgLength) - { - // parse channel - if (msgLength > 0) - { - byte channel = buffer[0]; - switch (channel) - { - case (byte)KcpChannel.Reliable: - { - // input into kcp, but skip channel byte - int input = kcp.Input(buffer, 1, msgLength - 1); - if (input != 0) - { - Log.Warning($"Input failed with error={input} for buffer with length={msgLength - 1}"); - } - break; - } - case (byte)KcpChannel.Unreliable: - { - // ideally we would queue all unreliable messages and - // then process them in ReceiveNext() together with the - // reliable messages, but: - // -> queues/allocations/pools are slow and complex. - // -> DOTSNET 10k is actually slower if we use pooled - // unreliable messages for transform messages. - // - // DOTSNET 10k benchmark: - // reliable-only: 170 FPS - // unreliable queued: 130-150 FPS - // unreliable direct: 183 FPS(!) - // - // DOTSNET 50k benchmark: - // reliable-only: FAILS (queues keep growing) - // unreliable direct: 18-22 FPS(!) - // - // -> all unreliable messages are DATA messages anyway. - // -> let's skip the magic and call OnData directly if - // the current state allows it. - if (state == KcpState.Authenticated) - { - // only process messages while not paused for Mirror - // scene switching etc. - // -> if an unreliable message comes in while - // paused, simply drop it. it's unreliable! - if (!paused) - { - ArraySegment message = new ArraySegment(buffer, 1, msgLength - 1); - OnData?.Invoke(message); - } - - // set last receive time to avoid timeout. - // -> we do this in ANY case even if not enabled. - // a message is a message. - // -> we set last receive time for both reliable and - // unreliable messages. both count. - // otherwise a connection might time out even - // though unreliable were received, but no - // reliable was received. - lastReceiveTime = (uint)refTime.ElapsedMilliseconds; - } - else - { - // should never - Log.Warning($"KCP: received unreliable message in state {state}. Disconnecting the connection."); - Disconnect(); - } - break; - } - default: - { - // not a valid channel. random data or attacks. - Log.Info($"Disconnecting connection because of invalid channel header: {channel}"); - Disconnect(); - break; - } - } - } - } - - // raw send puts the data into the socket - protected abstract void RawSend(byte[] data, int length); - - // raw send called by kcp - void RawSendReliable(byte[] data, int length) - { - // copy channel header, data into raw send buffer, then send - rawSendBuffer[0] = (byte)KcpChannel.Reliable; - Buffer.BlockCopy(data, 0, rawSendBuffer, 1, length); - RawSend(rawSendBuffer, length + 1); - } - - void SendReliable(KcpHeader header, ArraySegment content) - { - // 1 byte header + content needs to fit into send buffer - if (1 + content.Count <= kcpSendBuffer.Length) // TODO - { - // copy header, content (if any) into send buffer - kcpSendBuffer[0] = (byte)header; - if (content.Count > 0) - Buffer.BlockCopy(content.Array, content.Offset, kcpSendBuffer, 1, content.Count); - - // send to kcp for processing - int sent = kcp.Send(kcpSendBuffer, 0, 1 + content.Count); - if (sent < 0) - { - Log.Warning($"Send failed with error={sent} for content with length={content.Count}"); - } - } - // otherwise content is larger than MaxMessageSize. let user know! - else Log.Error($"Failed to send reliable message of size {content.Count} because it's larger than ReliableMaxMessageSize={ReliableMaxMessageSize}"); - } - - void SendUnreliable(ArraySegment message) - { - // message size needs to be <= unreliable max size - if (message.Count <= UnreliableMaxMessageSize) - { - // copy channel header, data into raw send buffer, then send - rawSendBuffer[0] = (byte)KcpChannel.Unreliable; - Buffer.BlockCopy(message.Array, 0, rawSendBuffer, 1, message.Count); - RawSend(rawSendBuffer, message.Count + 1); - } - // otherwise content is larger than MaxMessageSize. let user know! - else Log.Error($"Failed to send unreliable message of size {message.Count} because it's larger than UnreliableMaxMessageSize={UnreliableMaxMessageSize}"); - } - - // server & client need to send handshake at different times, so we need - // to expose the function. - // * client should send it immediately. - // * server should send it as reply to client's handshake, not before - // (server should not reply to random internet messages with handshake) - // => handshake info needs to be delivered, so it goes over reliable. - public void SendHandshake() - { - Log.Info("KcpConnection: sending Handshake to other end!"); - SendReliable(KcpHeader.Handshake, default); - } - - public void SendData(ArraySegment data, KcpChannel channel) - { - // sending empty segments is not allowed. - // nobody should ever try to send empty data. - // it means that something went wrong, e.g. in Mirror/DOTSNET. - // let's make it obvious so it's easy to debug. - if (data.Count == 0) - { - Log.Warning("KcpConnection: tried sending empty message. This should never happen. Disconnecting."); - Disconnect(); - return; - } - - switch (channel) - { - case KcpChannel.Reliable: - SendReliable(KcpHeader.Data, data); - break; - case KcpChannel.Unreliable: - SendUnreliable(data); - break; - } - } - - // ping goes through kcp to keep it from timing out, so it goes over the - // reliable channel. - void SendPing() => SendReliable(KcpHeader.Ping, default); - - // disconnect info needs to be delivered, so it goes over reliable - void SendDisconnect() => SendReliable(KcpHeader.Disconnect, default); - - protected virtual void Dispose() {} - - // disconnect this connection - public void Disconnect() - { - // only if not disconnected yet - if (state == KcpState.Disconnected) - return; - - // send a disconnect message - if (socket.Connected) - { - try - { - SendDisconnect(); - kcp.Flush(); - } - catch (SocketException) - { - // this is ok, the connection was already closed - } - catch (ObjectDisposedException) - { - // this is normal when we stop the server - // the socket is stopped so we can't send anything anymore - // to the clients - - // the clients will eventually timeout and realize they - // were disconnected - } - } - - // set as Disconnected, call event - Log.Info("KCP Connection: Disconnected."); - state = KcpState.Disconnected; - OnDisconnected?.Invoke(); - } - - // get remote endpoint - public EndPoint GetRemoteEndPoint() => remoteEndPoint; - - // pause/unpause to safely support mirror scene handling and to - // immediately pause the receive while loop if needed. - public void Pause() => paused = true; - public void Unpause() - { - // unpause - paused = false; - - // reset the timeout. - // we have likely been paused for > timeout seconds, but that - // doesn't mean we should disconnect. for example, Mirror pauses - // kcp during scene changes which could easily take > 10s timeout: - // see also: https://github.com/vis2k/kcp2k/issues/8 - // => Unpause completely resets the timeout instead of restoring the - // time difference when we started pausing. it's more simple and - // it's a good idea to start counting from 0 after we unpaused! - lastReceiveTime = (uint)refTime.ElapsedMilliseconds; - } - } -} diff --git a/Kcp2k/kcp2k/highlevel/KcpHeader.cs b/Kcp2k/kcp2k/highlevel/KcpHeader.cs index bc4b047..7817796 100644 --- a/Kcp2k/kcp2k/highlevel/KcpHeader.cs +++ b/Kcp2k/kcp2k/highlevel/KcpHeader.cs @@ -7,13 +7,13 @@ namespace kcp2k public enum KcpHeader : byte { // don't react on 0x00. might help to filter out random noise. - Handshake = 0x01, - // ping goes over reliable & KcpHeader for now. could go over reliable + Handshake = 1, + // ping goes over reliable & KcpHeader for now. could go over unreliable // too. there is no real difference except that this is easier because // we already have a KcpHeader for reliable messages. // ping is only used to keep it alive, so latency doesn't matter. - Ping = 0x02, - Data = 0x03, - Disconnect = 0x04 + Ping = 2, + Data = 3, + Disconnect = 4 } } \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/KcpPeer.cs b/Kcp2k/kcp2k/highlevel/KcpPeer.cs new file mode 100644 index 0000000..22bd980 --- /dev/null +++ b/Kcp2k/kcp2k/highlevel/KcpPeer.cs @@ -0,0 +1,737 @@ +// Kcp Peer, similar to UDP Peer but wrapped with reliability, channels, +// timeouts, authentication, state, etc. +// +// still IO agnostic to work with udp, nonalloc, relays, native, etc. +using System; +using System.Diagnostics; +using System.Net.Sockets; + +namespace kcp2k +{ + enum KcpState { Connected, Authenticated, Disconnected } + + public class KcpPeer + { + // kcp reliability algorithm + internal Kcp kcp; + + // IO agnostic + readonly Action> RawSend; + + // state: connected as soon as we create the peer. + // leftover from KcpConnection. remove it after refactoring later. + KcpState state = KcpState.Connected; + + // events are readonly, set in constructor. + // this ensures they are always initialized when used. + // fixes https://github.com/MirrorNetworking/Mirror/issues/3337 and more + readonly Action OnAuthenticated; + readonly Action, KcpChannel> OnData; + readonly Action OnDisconnected; + // error callback instead of logging. + // allows libraries to show popups etc. + // (string instead of Exception for ease of use and to avoid user panic) + readonly Action OnError; + + // If we don't receive anything these many milliseconds + // then consider us disconnected + public const int DEFAULT_TIMEOUT = 10000; + public int timeout; + uint lastReceiveTime; + + // internal time. + // StopWatch offers ElapsedMilliSeconds and should be more precise than + // Unity's time.deltaTime over long periods. + readonly Stopwatch watch = new Stopwatch(); + + // we need to subtract the channel byte from every MaxMessageSize + // calculation. + // we also need to tell kcp to use MTU-1 to leave space for the byte. + const int CHANNEL_HEADER_SIZE = 1; + + // reliable channel (= kcp) MaxMessageSize so the outside knows largest + // allowed message to send. the calculation in Send() is not obvious at + // all, so let's provide the helper here. + // + // kcp does fragmentation, so max message is way larger than MTU. + // + // -> runtime MTU changes are disabled: mss is always MTU_DEF-OVERHEAD + // -> Send() checks if fragment count < rcv_wnd, so we use rcv_wnd - 1. + // NOTE that original kcp has a bug where WND_RCV default is used + // instead of configured rcv_wnd, limiting max message size to 144 KB + // https://github.com/skywind3000/kcp/pull/291 + // we fixed this in kcp2k. + // -> we add 1 byte KcpHeader enum to each message, so -1 + // + // IMPORTANT: max message is MTU * rcv_wnd, in other words it completely + // fills the receive window! due to head of line blocking, + // all other messages have to wait while a maxed size message + // is being delivered. + // => in other words, DO NOT use max size all the time like + // for batching. + // => sending UNRELIABLE max message size most of the time is + // best for performance (use that one for batching!) + static int ReliableMaxMessageSize_Unconstrained(int mtu, uint rcv_wnd) => + (mtu - Kcp.OVERHEAD - CHANNEL_HEADER_SIZE) * ((int)rcv_wnd - 1) - 1; + + // kcp encodes 'frg' as 1 byte. + // max message size can only ever allow up to 255 fragments. + // WND_RCV gives 127 fragments. + // WND_RCV * 2 gives 255 fragments. + // so we can limit max message size by limiting rcv_wnd parameter. + public static int ReliableMaxMessageSize(int mtu, uint rcv_wnd) => + ReliableMaxMessageSize_Unconstrained(mtu, Math.Min(rcv_wnd, Kcp.FRG_MAX)); + + // unreliable max message size is simply MTU - channel header size + public static int UnreliableMaxMessageSize(int mtu) => + mtu - CHANNEL_HEADER_SIZE; + + // buffer to receive kcp's processed messages (avoids allocations). + // IMPORTANT: this is for KCP messages. so it needs to be of size: + // 1 byte header + MaxMessageSize content + readonly byte[] kcpMessageBuffer;// = new byte[1 + ReliableMaxMessageSize]; + + // send buffer for handing user messages to kcp for processing. + // (avoids allocations). + // IMPORTANT: needs to be of size: + // 1 byte header + MaxMessageSize content + readonly byte[] kcpSendBuffer;// = new byte[1 + ReliableMaxMessageSize]; + + // raw send buffer is exactly MTU. + readonly byte[] rawSendBuffer; + + // send a ping occasionally so we don't time out on the other end. + // for example, creating a character in an MMO could easily take a + // minute of no data being sent. which doesn't mean we want to time out. + // same goes for slow paced card games etc. + public const int PING_INTERVAL = 1000; + uint lastPingTime; + + // if we send more than kcp can handle, we will get ever growing + // send/recv buffers and queues and minutes of latency. + // => if a connection can't keep up, it should be disconnected instead + // to protect the server under heavy load, and because there is no + // point in growing to gigabytes of memory or minutes of latency! + // => 2k isn't enough. we reach 2k when spawning 4k monsters at once + // easily, but it does recover over time. + // => 10k seems safe. + // + // note: we have a ChokeConnectionAutoDisconnects test for this too! + internal const int QueueDisconnectThreshold = 10000; + + // getters for queue and buffer counts, used for debug info + public int SendQueueCount => kcp.snd_queue.Count; + public int ReceiveQueueCount => kcp.rcv_queue.Count; + public int SendBufferCount => kcp.snd_buf.Count; + public int ReceiveBufferCount => kcp.rcv_buf.Count; + + // maximum send rate per second can be calculated from kcp parameters + // source: https://translate.google.com/translate?sl=auto&tl=en&u=https://wetest.qq.com/lab/view/391.html + // + // KCP can send/receive a maximum of WND*MTU per interval. + // multiple by 1000ms / interval to get the per-second rate. + // + // example: + // WND(32) * MTU(1400) = 43.75KB + // => 43.75KB * 1000 / INTERVAL(10) = 4375KB/s + // + // returns bytes/second! + public uint MaxSendRate => kcp.snd_wnd * kcp.mtu * 1000 / kcp.interval; + public uint MaxReceiveRate => kcp.rcv_wnd * kcp.mtu * 1000 / kcp.interval; + + // calculate max message sizes based on mtu and wnd only once + public readonly int unreliableMax; + public readonly int reliableMax; + + // SetupKcp creates and configures a new KCP instance. + // => useful to start from a fresh state every time the client connects + // => NoDelay, interval, wnd size are the most important configurations. + // let's force require the parameters so we don't forget it anywhere. + public KcpPeer( + Action> output, + Action OnAuthenticated, + Action, KcpChannel> OnData, + Action OnDisconnected, + Action OnError, + KcpConfig config) + { + // initialize callbacks first to ensure they can be used safely. + this.OnAuthenticated = OnAuthenticated; + this.OnData = OnData; + this.OnDisconnected = OnDisconnected; + this.OnError = OnError; + this.RawSend = output; + + // set up kcp over reliable channel (that's what kcp is for) + kcp = new Kcp(0, RawSendReliable); + + // set nodelay. + // note that kcp uses 'nocwnd' internally so we negate the parameter + kcp.SetNoDelay(config.NoDelay ? 1u : 0u, config.Interval, config.FastResend, !config.CongestionWindow); + kcp.SetWindowSize(config.SendWindowSize, config.ReceiveWindowSize); + + // IMPORTANT: high level needs to add 1 channel byte to each raw + // message. so while Kcp.MTU_DEF is perfect, we actually need to + // tell kcp to use MTU-1 so we can still put the header into the + // message afterwards. + kcp.SetMtu((uint)config.Mtu - CHANNEL_HEADER_SIZE); + + // create mtu sized send buffer + rawSendBuffer = new byte[config.Mtu]; + + // calculate max message sizes once + unreliableMax = UnreliableMaxMessageSize(config.Mtu); + reliableMax = ReliableMaxMessageSize(config.Mtu, config.ReceiveWindowSize); + + // set maximum retransmits (aka dead_link) + kcp.dead_link = config.MaxRetransmits; + + // create message buffers AFTER window size is set + // see comments on buffer definition for the "+1" part + kcpMessageBuffer = new byte[1 + reliableMax]; + kcpSendBuffer = new byte[1 + reliableMax]; + + timeout = config.Timeout; + + watch.Start(); + } + + void HandleTimeout(uint time) + { + // note: we are also sending a ping regularly, so timeout should + // only ever happen if the connection is truly gone. + if (time >= lastReceiveTime + timeout) + { + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.Timeout, $"KcpPeer: Connection timed out after not receiving any message for {timeout}ms. Disconnecting."); + Disconnect(); + } + } + + void HandleDeadLink() + { + // kcp has 'dead_link' detection. might as well use it. + if (kcp.state == -1) + { + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.Timeout, $"KcpPeer: dead_link detected: a message was retransmitted {kcp.dead_link} times without ack. Disconnecting."); + Disconnect(); + } + } + + // send a ping occasionally in order to not time out on the other end. + void HandlePing(uint time) + { + // enough time elapsed since last ping? + if (time >= lastPingTime + PING_INTERVAL) + { + // ping again and reset time + //Log.Debug("KCP: sending ping..."); + SendPing(); + lastPingTime = time; + } + } + + void HandleChoked() + { + // disconnect connections that can't process the load. + // see QueueSizeDisconnect comments. + // => include all of kcp's buffers and the unreliable queue! + int total = kcp.rcv_queue.Count + kcp.snd_queue.Count + + kcp.rcv_buf.Count + kcp.snd_buf.Count; + if (total >= QueueDisconnectThreshold) + { + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.Congestion, + $"KcpPeer: disconnecting connection because it can't process data fast enough.\n" + + $"Queue total {total}>{QueueDisconnectThreshold}. rcv_queue={kcp.rcv_queue.Count} snd_queue={kcp.snd_queue.Count} rcv_buf={kcp.rcv_buf.Count} snd_buf={kcp.snd_buf.Count}\n" + + $"* Try to Enable NoDelay, decrease INTERVAL, disable Congestion Window (= enable NOCWND!), increase SEND/RECV WINDOW or compress data.\n" + + $"* Or perhaps the network is simply too slow on our end, or on the other end."); + + // let's clear all pending sends before disconnting with 'Bye'. + // otherwise a single Flush in Disconnect() won't be enough to + // flush thousands of messages to finally deliver 'Bye'. + // this is just faster and more robust. + kcp.snd_queue.Clear(); + + Disconnect(); + } + } + + // reads the next reliable message type & content from kcp. + // -> to avoid buffering, unreliable messages call OnData directly. + bool ReceiveNextReliable(out KcpHeader header, out ArraySegment message) + { + message = default; + header = KcpHeader.Disconnect; + + int msgSize = kcp.PeekSize(); + if (msgSize <= 0) return false; + + // only allow receiving up to buffer sized messages. + // otherwise we would get BlockCopy ArgumentException anyway. + if (msgSize > kcpMessageBuffer.Length) + { + // we don't allow sending messages > Max, so this must be an + // attacker. let's disconnect to avoid allocation attacks etc. + // pass error to user callback. no need to log it manually. + OnError(ErrorCode.InvalidReceive, $"KcpPeer: possible allocation attack for msgSize {msgSize} > buffer {kcpMessageBuffer.Length}. Disconnecting the connection."); + Disconnect(); + return false; + } + + // receive from kcp + int received = kcp.Receive(kcpMessageBuffer, msgSize); + if (received < 0) + { + // if receive failed, close everything + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.InvalidReceive, $"KcpPeer: Receive failed with error={received}. closing connection."); + Disconnect(); + return false; + } + + // extract header & content without header + header = (KcpHeader)kcpMessageBuffer[0]; + message = new ArraySegment(kcpMessageBuffer, 1, msgSize - 1); + lastReceiveTime = (uint)watch.ElapsedMilliseconds; + return true; + } + + void TickIncoming_Connected(uint time) + { + // detect common events & ping + HandleTimeout(time); + HandleDeadLink(); + HandlePing(time); + HandleChoked(); + + // any reliable kcp message received? + if (ReceiveNextReliable(out KcpHeader header, out ArraySegment message)) + { + // message type FSM. no default so we never miss a case. + switch (header) + { + case KcpHeader.Handshake: + { + // we were waiting for a handshake. + // it proves that the other end speaks our protocol. + // GetType() shows Server/ClientConn instead of just Connection. + Log.Info($"KcpPeer: received handshake"); + state = KcpState.Authenticated; + OnAuthenticated?.Invoke(); + break; + } + case KcpHeader.Ping: + { + // ping keeps kcp from timing out. do nothing. + break; + } + case KcpHeader.Data: + case KcpHeader.Disconnect: + { + // everything else is not allowed during handshake! + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.InvalidReceive, $"KcpPeer: received invalid header {header} while Connected. Disconnecting the connection."); + Disconnect(); + break; + } + } + } + } + + void TickIncoming_Authenticated(uint time) + { + // detect common events & ping + HandleTimeout(time); + HandleDeadLink(); + HandlePing(time); + HandleChoked(); + + // process all received messages + while (ReceiveNextReliable(out KcpHeader header, out ArraySegment message)) + { + // message type FSM. no default so we never miss a case. + switch (header) + { + case KcpHeader.Handshake: + { + // should never receive another handshake after auth + // GetType() shows Server/ClientConn instead of just Connection. + Log.Warning($"KcpPeer: received invalid header {header} while Authenticated. Disconnecting the connection."); + Disconnect(); + break; + } + case KcpHeader.Data: + { + // call OnData IF the message contained actual data + if (message.Count > 0) + { + //Log.Warning($"Kcp recv msg: {BitConverter.ToString(message.Array, message.Offset, message.Count)}"); + OnData?.Invoke(message, KcpChannel.Reliable); + } + // empty data = attacker, or something went wrong + else + { + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.InvalidReceive, $"KcpPeer: received empty Data message while Authenticated. Disconnecting the connection."); + Disconnect(); + } + break; + } + case KcpHeader.Ping: + { + // ping keeps kcp from timing out. do nothing. + break; + } + case KcpHeader.Disconnect: + { + // disconnect might happen + // GetType() shows Server/ClientConn instead of just Connection. + Log.Info($"KcpPeer: received disconnect message"); + Disconnect(); + break; + } + } + } + } + + public void TickIncoming() + { + uint time = (uint)watch.ElapsedMilliseconds; + + try + { + switch (state) + { + case KcpState.Connected: + { + TickIncoming_Connected(time); + break; + } + case KcpState.Authenticated: + { + TickIncoming_Authenticated(time); + break; + } + case KcpState.Disconnected: + { + // do nothing while disconnected + break; + } + } + } + // TODO KcpConnection is IO agnostic. move this to outside later. + catch (SocketException exception) + { + // this is ok, the connection was closed + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.ConnectionClosed, $"KcpPeer: Disconnecting because {exception}. This is fine."); + Disconnect(); + } + catch (ObjectDisposedException exception) + { + // fine, socket was closed + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.ConnectionClosed, $"KcpPeer: Disconnecting because {exception}. This is fine."); + Disconnect(); + } + catch (Exception exception) + { + // unexpected + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.Unexpected, $"KcpPeer: unexpected Exception: {exception}"); + Disconnect(); + } + } + + public void TickOutgoing() + { + uint time = (uint)watch.ElapsedMilliseconds; + + try + { + switch (state) + { + case KcpState.Connected: + case KcpState.Authenticated: + { + // update flushes out messages + kcp.Update(time); + break; + } + case KcpState.Disconnected: + { + // do nothing while disconnected + break; + } + } + } + // TODO KcpConnection is IO agnostic. move this to outside later. + catch (SocketException exception) + { + // this is ok, the connection was closed + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.ConnectionClosed, $"KcpPeer: Disconnecting because {exception}. This is fine."); + Disconnect(); + } + catch (ObjectDisposedException exception) + { + // fine, socket was closed + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.ConnectionClosed, $"KcpPeer: Disconnecting because {exception}. This is fine."); + Disconnect(); + } + catch (Exception exception) + { + // unexpected + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.Unexpected, $"KcpPeer: unexpected exception: {exception}"); + Disconnect(); + } + } + + void OnRawInputReliable(ArraySegment message) + { + // input into kcp, but skip channel byte + int input = kcp.Input(message.Array, message.Offset, message.Count); + if (input != 0) + { + // GetType() shows Server/ClientConn instead of just Connection. + Log.Warning($"KcpPeer: Input failed with error={input} for buffer with length={message.Count - 1}"); + } + } + + void OnRawInputUnreliable(ArraySegment message) + { + // ideally we would queue all unreliable messages and + // then process them in ReceiveNext() together with the + // reliable messages, but: + // -> queues/allocations/pools are slow and complex. + // -> DOTSNET 10k is actually slower if we use pooled + // unreliable messages for transform messages. + // + // DOTSNET 10k benchmark: + // reliable-only: 170 FPS + // unreliable queued: 130-150 FPS + // unreliable direct: 183 FPS(!) + // + // DOTSNET 50k benchmark: + // reliable-only: FAILS (queues keep growing) + // unreliable direct: 18-22 FPS(!) + // + // -> all unreliable messages are DATA messages anyway. + // -> let's skip the magic and call OnData directly if + // the current state allows it. + if (state == KcpState.Authenticated) + { + OnData?.Invoke(message, KcpChannel.Unreliable); + + // set last receive time to avoid timeout. + // -> we do this in ANY case even if not enabled. + // a message is a message. + // -> we set last receive time for both reliable and + // unreliable messages. both count. + // otherwise a connection might time out even + // though unreliable were received, but no + // reliable was received. + lastReceiveTime = (uint)watch.ElapsedMilliseconds; + } + else + { + // invalid unreliable messages may be random internet noise. + // show a warning, but don't disconnect. + // otherwise attackers could disconnect someone with random noise. + Log.Warning($"KcpPeer: received unreliable message while not authenticated. Disconnecting the connection."); + } + } + + // insert raw IO. usually from socket.Receive. + // offset is useful for relays, where we may parse a header and then + // feed the rest to kcp. + public void RawInput(ArraySegment segment) + { + // ensure valid size: at least 1 byte for channel + if (segment.Count <= 0) return; + + // parse channel + // byte channel = segment[0]; ArraySegment[i] isn't supported in some older Unity Mono versions + byte channel = segment.Array[segment.Offset + 0]; + + // parse message + ArraySegment message = new ArraySegment(segment.Array, segment.Offset + 1, segment.Count - 1); + + switch (channel) + { + case (byte)KcpChannel.Reliable: + { + OnRawInputReliable(message); + break; + } + case (byte)KcpChannel.Unreliable: + { + OnRawInputUnreliable(message); + break; + } + default: + { + // invalid channel indicates random internet noise. + // servers may receive random UDP data. + // just ignore it, but log for easier debugging. + Log.Warning($"KcpPeer: invalid channel header: {channel}, likely internet noise"); + break; + } + } + } + + // raw send called by kcp + void RawSendReliable(byte[] data, int length) + { + // copy channel header, data into raw send buffer, then send + rawSendBuffer[0] = (byte)KcpChannel.Reliable; + Buffer.BlockCopy(data, 0, rawSendBuffer, 1, length); + + // IO send + ArraySegment segment = new ArraySegment(rawSendBuffer, 0, length + 1); + RawSend(segment); + } + + void SendReliable(KcpHeader header, ArraySegment content) + { + // 1 byte header + content needs to fit into send buffer + if (1 + content.Count > kcpSendBuffer.Length) // TODO + { + // otherwise content is larger than MaxMessageSize. let user know! + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.InvalidSend, $"KcpPeer: Failed to send reliable message of size {content.Count} because it's larger than ReliableMaxMessageSize={reliableMax}"); + return; + } + + // copy header, content (if any) into send buffer + kcpSendBuffer[0] = (byte)header; + if (content.Count > 0) + Buffer.BlockCopy(content.Array, content.Offset, kcpSendBuffer, 1, content.Count); + + // send to kcp for processing + int sent = kcp.Send(kcpSendBuffer, 0, 1 + content.Count); + if (sent < 0) + { + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.InvalidSend, $"KcpPeer: Send failed with error={sent} for content with length={content.Count}"); + } + } + + void SendUnreliable(ArraySegment message) + { + // message size needs to be <= unreliable max size + if (message.Count > unreliableMax) + { + // otherwise content is larger than MaxMessageSize. let user know! + // GetType() shows Server/ClientConn instead of just Connection. + Log.Error($"KcpPeer: Failed to send unreliable message of size {message.Count} because it's larger than UnreliableMaxMessageSize={unreliableMax}"); + return; + } + + // copy channel header, data into raw send buffer, then send + rawSendBuffer[0] = (byte)KcpChannel.Unreliable; + Buffer.BlockCopy(message.Array, message.Offset, rawSendBuffer, 1, message.Count); + + // IO send + ArraySegment segment = new ArraySegment(rawSendBuffer, 0, message.Count + 1); + RawSend(segment); + } + + // server & client need to send handshake at different times, so we need + // to expose the function. + // * client should send it immediately. + // * server should send it as reply to client's handshake, not before + // (server should not reply to random internet messages with handshake) + // => handshake info needs to be delivered, so it goes over reliable. + public void SendHandshake() + { + // GetType() shows Server/ClientConn instead of just Connection. + Log.Info($"KcpPeer: sending Handshake to other end!"); + SendReliable(KcpHeader.Handshake, default); + } + + public void SendData(ArraySegment data, KcpChannel channel) + { + // sending empty segments is not allowed. + // nobody should ever try to send empty data. + // it means that something went wrong, e.g. in Mirror/DOTSNET. + // let's make it obvious so it's easy to debug. + if (data.Count == 0) + { + // pass error to user callback. no need to log it manually. + // GetType() shows Server/ClientConn instead of just Connection. + OnError(ErrorCode.InvalidSend, $"KcpPeer: tried sending empty message. This should never happen. Disconnecting."); + Disconnect(); + return; + } + + switch (channel) + { + case KcpChannel.Reliable: + SendReliable(KcpHeader.Data, data); + break; + case KcpChannel.Unreliable: + SendUnreliable(data); + break; + } + } + + // ping goes through kcp to keep it from timing out, so it goes over the + // reliable channel. + void SendPing() => SendReliable(KcpHeader.Ping, default); + + // disconnect info needs to be delivered, so it goes over reliable + void SendDisconnect() => SendReliable(KcpHeader.Disconnect, default); + + // disconnect this connection + public void Disconnect() + { + // only if not disconnected yet + if (state == KcpState.Disconnected) + return; + + // send a disconnect message + try + { + SendDisconnect(); + kcp.Flush(); + } + // TODO KcpConnection is IO agnostic. move this to outside later. + catch (SocketException) + { + // this is ok, the connection was already closed + } + catch (ObjectDisposedException) + { + // this is normal when we stop the server + // the socket is stopped so we can't send anything anymore + // to the clients + + // the clients will eventually timeout and realize they + // were disconnected + } + + // set as Disconnected, call event + // GetType() shows Server/ClientConn instead of just Connection. + Log.Info($"KcpPeer: Disconnected."); + state = KcpState.Disconnected; + OnDisconnected?.Invoke(); + } + } +} diff --git a/Kcp2k/kcp2k/highlevel/KcpServer.cs b/Kcp2k/kcp2k/highlevel/KcpServer.cs index c9847df..62c54ab 100644 --- a/Kcp2k/kcp2k/highlevel/KcpServer.cs +++ b/Kcp2k/kcp2k/highlevel/KcpServer.cs @@ -9,114 +9,116 @@ namespace kcp2k { public class KcpServer { - // events - public Action OnConnected; - public Action> OnData; - public Action OnDisconnected; + // callbacks + // even for errors, to allow liraries to show popups etc. + // instead of logging directly. + // (string instead of Exception for ease of use and to avoid user panic) + // + // events are readonly, set in constructor. + // this ensures they are always initialized when used. + // fixes https://github.com/MirrorNetworking/Mirror/issues/3337 and more + readonly Action OnConnected; + readonly Action, KcpChannel> OnData; + readonly Action OnDisconnected; + readonly Action OnError; // configuration - // DualMode uses both IPv6 and IPv4. not all platforms support it. - // (Nintendo Switch, etc.) - public bool DualMode; - // NoDelay is recommended to reduce latency. This also scales better - // without buffers getting full. - public bool NoDelay; - // KCP internal update interval. 100ms is KCP default, but a lower - // interval is recommended to minimize latency and to scale to more - // networked entities. - public uint Interval; - // KCP fastresend parameter. Faster resend for the cost of higher - // bandwidth. - public int FastResend; - // KCP 'NoCongestionWindow' is false by default. here we negate it for - // ease of use. This can be disabled for high scale games if connections - // choke regularly. - public bool CongestionWindow; - // KCP window size can be modified to support higher loads. - // for example, Mirror Benchmark requires: - // 128, 128 for 4k monsters - // 512, 512 for 10k monsters - // 8192, 8192 for 20k monsters - public uint SendWindowSize; - public uint ReceiveWindowSize; - // timeout in milliseconds - public int Timeout; + readonly KcpConfig config; // state protected Socket socket; EndPoint newClientEP; - // IMPORTANT: raw receive buffer always needs to be of 'MTU' size, even - // if MaxMessageSize is larger. kcp always sends in MTU - // segments and having a buffer smaller than MTU would - // silently drop excess data. - // => we need the mtu to fit channel + message! - readonly byte[] rawReceiveBuffer = new byte[Kcp.MTU_DEF]; + // raw receive buffer always needs to be of 'MTU' size, even if + // MaxMessageSize is larger. kcp always sends in MTU segments and having + // a buffer smaller than MTU would silently drop excess data. + // => we need the mtu to fit channel + message! + protected readonly byte[] rawReceiveBuffer; // connections where connectionId is EndPoint.GetHashCode - public Dictionary connections = new Dictionary(); + public Dictionary connections = + new Dictionary(); public KcpServer(Action OnConnected, - Action> OnData, + Action, KcpChannel> OnData, Action OnDisconnected, - bool DualMode, - bool NoDelay, - uint Interval, - int FastResend = 0, - bool CongestionWindow = true, - uint SendWindowSize = Kcp.WND_SND, - uint ReceiveWindowSize = Kcp.WND_RCV, - int Timeout = KcpConnection.DEFAULT_TIMEOUT) + Action OnError, + KcpConfig config) { + // initialize callbacks first to ensure they can be used safely. this.OnConnected = OnConnected; this.OnData = OnData; this.OnDisconnected = OnDisconnected; - this.DualMode = DualMode; - this.NoDelay = NoDelay; - this.Interval = Interval; - this.FastResend = FastResend; - this.CongestionWindow = CongestionWindow; - this.SendWindowSize = SendWindowSize; - this.ReceiveWindowSize = ReceiveWindowSize; - this.Timeout = Timeout; + this.OnError = OnError; + this.config = config; + + // create mtu sized receive buffer + rawReceiveBuffer = new byte[config.Mtu]; // create newClientEP either IPv4 or IPv6 - newClientEP = DualMode + newClientEP = config.DualMode ? new IPEndPoint(IPAddress.IPv6Any, 0) - : new IPEndPoint(IPAddress.Any, 0); + : new IPEndPoint(IPAddress.Any, 0); } - public bool IsActive() => socket != null; + public virtual bool IsActive() => socket != null; - public void Start(ushort port) + static Socket CreateServerSocket(bool DualMode, ushort port) { - // only start once - if (socket != null) - { - Log.Warning("KCP: server already started!"); - } - - // listen if (DualMode) { - // IPv6 socket with DualMode - socket = new Socket(AddressFamily.InterNetworkV6, SocketType.Dgram, ProtocolType.Udp); - socket.DualMode = true; + // IPv6 socket with DualMode @ "::" : port + Socket socket = new Socket(AddressFamily.InterNetworkV6, SocketType.Dgram, ProtocolType.Udp); + // settings DualMode may throw: + // https://learn.microsoft.com/en-us/dotnet/api/System.Net.Sockets.Socket.DualMode?view=net-7.0 + // attempt it, otherwise log but continue + // fixes: https://github.com/MirrorNetworking/Mirror/issues/3358 + try + { + socket.DualMode = true; + } + catch (NotSupportedException e) + { + Log.Warning($"Failed to set Dual Mode, continuing with IPv6 without Dual Mode. Error: {e}"); + } socket.Bind(new IPEndPoint(IPAddress.IPv6Any, port)); + return socket; } else { - // IPv4 socket - socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + // IPv4 socket @ "0.0.0.0" : port + Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); socket.Bind(new IPEndPoint(IPAddress.Any, port)); + return socket; + } + } + + public virtual void Start(ushort port) + { + // only start once + if (socket != null) + { + Log.Warning("KcpServer: already started!"); + return; } + + // listen + socket = CreateServerSocket(config.DualMode, port); + + // recv & send are called from main thread. + // need to ensure this never blocks. + // even a 1ms block per connection would stop us from scaling. + socket.Blocking = false; + + // configure buffer sizes + Common.ConfigureSocketBuffers(socket, config.RecvBufferSize, config.SendBufferSize); } public void Send(int connectionId, ArraySegment segment, KcpChannel channel) { if (connections.TryGetValue(connectionId, out KcpServerConnection connection)) { - connection.SendData(segment, channel); + connection.peer.SendData(segment, channel); } } @@ -124,165 +126,216 @@ public void Disconnect(int connectionId) { if (connections.TryGetValue(connectionId, out KcpServerConnection connection)) { - connection.Disconnect(); + connection.peer.Disconnect(); } } - public string GetClientAddress(int connectionId) + // expose the whole IPEndPoint, not just the IP address. some need it. + public IPEndPoint GetClientEndPoint(int connectionId) { if (connections.TryGetValue(connectionId, out KcpServerConnection connection)) { - return (connection.GetRemoteEndPoint() as IPEndPoint).Address.ToString(); + return connection.remoteEndPoint as IPEndPoint; } - return ""; + return null; } - // EndPoint & Receive functions can be overwritten for where-allocation: + // io - input. + // virtual so it may be modified for relays, nonalloc workaround, etc. // https://github.com/vis2k/where-allocation - protected virtual int ReceiveFrom(byte[] buffer, out int connectionHash) + // bool return because not all receives may be valid. + // for example, relay may expect a certain header. + protected virtual bool RawReceiveFrom(out ArraySegment segment, out int connectionId) { - // NOTE: ReceiveFrom allocates. - // we pass our IPEndPoint to ReceiveFrom. - // receive from calls newClientEP.Create(socketAddr). - // IPEndPoint.Create always returns a new IPEndPoint. - // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761 - int read = socket.ReceiveFrom(buffer, 0, buffer.Length, SocketFlags.None, ref newClientEP); - - // calculate connectionHash from endpoint - // NOTE: IPEndPoint.GetHashCode() allocates. - // it calls m_Address.GetHashCode(). - // m_Address is an IPAddress. - // GetHashCode() allocates for IPv6: - // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPAddress.cs#L699 - // - // => using only newClientEP.Port wouldn't work, because - // different connections can have the same port. - connectionHash = newClientEP.GetHashCode(); - return read; + segment = default; + connectionId = 0; + if (socket == null) return false; + + try + { + if (socket.ReceiveFromNonBlocking(rawReceiveBuffer, out segment, ref newClientEP)) + { + // set connectionId to hash from endpoint + // NOTE: IPEndPoint.GetHashCode() allocates. + // it calls m_Address.GetHashCode(). + // m_Address is an IPAddress. + // GetHashCode() allocates for IPv6: + // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPAddress.cs#L699 + // + // => using only newClientEP.Port wouldn't work, because + // different connections can have the same port. + connectionId = newClientEP.GetHashCode(); + return true; + } + } + catch (SocketException e) + { + // NOTE: SocketException is not a subclass of IOException. + // the other end closing the connection is not an 'error'. + // but connections should never just end silently. + // at least log a message for easier debugging. + Log.Info($"KcpServer: ReceiveFrom failed: {e}"); + } + + return false; } - protected virtual KcpServerConnection CreateConnection() => - new KcpServerConnection(socket, newClientEP, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout); + // io - out. + // virtual so it may be modified for relays, nonalloc workaround, etc. + // relays may need to prefix connId (and remoteEndPoint would be same for all) + protected virtual void RawSend(int connectionId, ArraySegment data) + { + // get the connection's endpoint + if (!connections.TryGetValue(connectionId, out KcpServerConnection connection)) + { + return; + } + + try + { + socket.SendToNonBlocking(data, connection.remoteEndPoint); + } + catch (SocketException e) + { + Log.Error($"KcpServer: SendTo failed: {e}"); + } + } + + protected virtual KcpServerConnection CreateConnection(int connectionId) + { + // events need to be wrapped with connectionIds + Action> RawSendWrap = + data => RawSend(connectionId, data); + + // create empty connection without peer first. + // we need it to set up peer callbacks. + // afterwards we assign the peer. + KcpServerConnection connection = new KcpServerConnection(newClientEP); + + // set up peer with callbacks + KcpPeer peer = new KcpPeer(RawSendWrap, OnAuthenticatedWrap, OnDataWrap, OnDisconnectedWrap, OnErrorWrap, config); + + // assign peer to connection + connection.peer = peer; + return connection; + + // setup authenticated event that also adds to connections + void OnAuthenticatedWrap() + { + // only send handshake to client AFTER we received his + // handshake in OnAuthenticated. + // we don't want to reply to random internet messages + // with handshakes each time. + connection.peer.SendHandshake(); + + // add to connections dict after being authenticated. + connections.Add(connectionId, connection); + Log.Info($"KcpServer: added connection({connectionId})"); + + // setup Data + Disconnected events only AFTER the + // handshake. we don't want to fire OnServerDisconnected + // every time we receive invalid random data from the + // internet. + + // setup data event + + + // finally, call mirror OnConnected event + Log.Info($"KcpServer: OnConnected({connectionId})"); + OnConnected(connectionId); + } + + void OnDataWrap(ArraySegment message, KcpChannel channel) + { + // call mirror event + //Log.Info($"KCP: OnServerDataReceived({connectionId}, {BitConverter.ToString(message.Array, message.Offset, message.Count)})"); + OnData(connectionId, message, channel); + } + + void OnDisconnectedWrap() + { + // flag for removal + // (can't remove directly because connection is updated + // and event is called while iterating all connections) + connectionsToRemove.Add(connectionId); + + // call mirror event + Log.Info($"KcpServer: OnDisconnected({connectionId})"); + OnDisconnected(connectionId); + } + + void OnErrorWrap(ErrorCode error, string reason) + { + OnError(connectionId, error, reason); + } + } + + // receive + add + process once. + // best to call this as long as there is more data to receive. + void ProcessMessage(ArraySegment segment, int connectionId) + { + //Log.Info($"KCP: server raw recv {msgLength} bytes = {BitConverter.ToString(buffer, 0, msgLength)}"); + + // is this a new connection? + if (!connections.TryGetValue(connectionId, out KcpServerConnection connection)) + { + // create a new KcpConnection based on last received + // EndPoint. can be overwritten for where-allocation. + connection = CreateConnection(connectionId); + + // DO NOT add to connections yet. only if the first message + // is actually the kcp handshake. otherwise it's either: + // * random data from the internet + // * or from a client connection that we just disconnected + // but that hasn't realized it yet, still sending data + // from last session that we should absolutely ignore. + // + // + // TODO this allocates a new KcpConnection for each new + // internet connection. not ideal, but C# UDP Receive + // already allocated anyway. + // + // expecting a MAGIC byte[] would work, but sending the raw + // UDP message without kcp's reliability will have low + // probability of being received. + // + // for now, this is fine. + + + // now input the message & process received ones + // connected event was set up. + // tick will process the first message and adds the + // connection if it was the handshake. + connection.peer.RawInput(segment); + connection.peer.TickIncoming(); + + // again, do not add to connections. + // if the first message wasn't the kcp handshake then + // connection will simply be garbage collected. + } + // existing connection: simply input the message into kcp + else + { + connection.peer.RawInput(segment); + } + } // process incoming messages. should be called before updating the world. - HashSet connectionsToRemove = new HashSet(); - public void TickIncoming() + // virtual because relay may need to inject their own ping or similar. + readonly HashSet connectionsToRemove = new HashSet(); + public virtual void TickIncoming() { - while (socket != null && socket.Poll(0, SelectMode.SelectRead)) + // input all received messages into kcp + while (RawReceiveFrom(out ArraySegment segment, out int connectionId)) { - try - { - // receive - int msgLength = ReceiveFrom(rawReceiveBuffer, out int connectionId); - //Log.Info($"KCP: server raw recv {msgLength} bytes = {BitConverter.ToString(buffer, 0, msgLength)}"); - - // IMPORTANT: detect if buffer was too small for the received - // msgLength. otherwise the excess data would be - // silently lost. - // (see ReceiveFrom documentation) - if (msgLength <= rawReceiveBuffer.Length) - { - // is this a new connection? - if (!connections.TryGetValue(connectionId, out KcpServerConnection connection)) - { - // create a new KcpConnection based on last received - // EndPoint. can be overwritten for where-allocation. - connection = CreateConnection(); - - // DO NOT add to connections yet. only if the first message - // is actually the kcp handshake. otherwise it's either: - // * random data from the internet - // * or from a client connection that we just disconnected - // but that hasn't realized it yet, still sending data - // from last session that we should absolutely ignore. - // - // - // TODO this allocates a new KcpConnection for each new - // internet connection. not ideal, but C# UDP Receive - // already allocated anyway. - // - // expecting a MAGIC byte[] would work, but sending the raw - // UDP message without kcp's reliability will have low - // probability of being received. - // - // for now, this is fine. - - // setup authenticated event that also adds to connections - connection.OnAuthenticated = () => - { - // only send handshake to client AFTER we received his - // handshake in OnAuthenticated. - // we don't want to reply to random internet messages - // with handshakes each time. - connection.SendHandshake(); - - // add to connections dict after being authenticated. - connections.Add(connectionId, connection); - Log.Info($"KCP: server added connection({connectionId})"); - - // setup Data + Disconnected events only AFTER the - // handshake. we don't want to fire OnServerDisconnected - // every time we receive invalid random data from the - // internet. - - // setup data event - connection.OnData = (message) => - { - // call mirror event - //Log.Info($"KCP: OnServerDataReceived({connectionId}, {BitConverter.ToString(message.Array, message.Offset, message.Count)})"); - OnData.Invoke(connectionId, message); - }; - - // setup disconnected event - connection.OnDisconnected = () => - { - // flag for removal - // (can't remove directly because connection is updated - // and event is called while iterating all connections) - connectionsToRemove.Add(connectionId); - - // call mirror event - Log.Info($"KCP: OnServerDisconnected({connectionId})"); - OnDisconnected.Invoke(connectionId); - }; - - // finally, call mirror OnConnected event - Log.Info($"KCP: OnServerConnected({connectionId})"); - OnConnected.Invoke(connectionId); - }; - - // now input the message & process received ones - // connected event was set up. - // tick will process the first message and adds the - // connection if it was the handshake. - connection.RawInput(rawReceiveBuffer, msgLength); - connection.TickIncoming(); - - // again, do not add to connections. - // if the first message wasn't the kcp handshake then - // connection will simply be garbage collected. - } - // existing connection: simply input the message into kcp - else - { - connection.RawInput(rawReceiveBuffer, msgLength); - } - } - else - { - Log.Error($"KCP Server: message of size {msgLength} does not fit into buffer of size {rawReceiveBuffer.Length}. The excess was silently dropped. Disconnecting connectionId={connectionId}."); - Disconnect(connectionId); - } - } - // this is fine, the socket might have been closed in the other end - catch (SocketException) {} + ProcessMessage(segment, connectionId); } // process inputs for all server connections // (even if we didn't receive anything. need to tick ping etc.) foreach (KcpServerConnection connection in connections.Values) { - connection.TickIncoming(); + connection.peer.TickIncoming(); } // remove disconnected connections @@ -296,42 +349,29 @@ public void TickIncoming() } // process outgoing messages. should be called after updating the world. - public void TickOutgoing() + // virtual because relay may need to inject their own ping or similar. + public virtual void TickOutgoing() { // flush all server connections foreach (KcpServerConnection connection in connections.Values) { - connection.TickOutgoing(); + connection.peer.TickOutgoing(); } } // process incoming and outgoing for convenience. // => ideally call ProcessIncoming() before updating the world and // ProcessOutgoing() after updating the world for minimum latency - public void Tick() + public virtual void Tick() { TickIncoming(); TickOutgoing(); } - public void Stop() + public virtual void Stop() { socket?.Close(); socket = null; } - - // pause/unpause to safely support mirror scene handling and to - // immediately pause the receive while loop if needed. - public void Pause() - { - foreach (KcpServerConnection connection in connections.Values) - connection.Pause(); - } - - public void Unpause() - { - foreach (KcpServerConnection connection in connections.Values) - connection.Unpause(); - } } } diff --git a/Kcp2k/kcp2k/highlevel/KcpServerConnection.cs b/Kcp2k/kcp2k/highlevel/KcpServerConnection.cs index 450a4da..5b9c888 100644 --- a/Kcp2k/kcp2k/highlevel/KcpServerConnection.cs +++ b/Kcp2k/kcp2k/highlevel/KcpServerConnection.cs @@ -1,22 +1,22 @@ +// server needs to store a separate KcpPeer for each connection. +// as well as remoteEndPoint so we know where to send data to. using System.Net; -using System.Net.Sockets; namespace kcp2k { - public class KcpServerConnection : KcpConnection + // struct to avoid memory indirection + public struct KcpServerConnection { - // Constructor & Send functions can be overwritten for where-allocation: - // https://github.com/vis2k/where-allocation - public KcpServerConnection(Socket socket, EndPoint remoteEndPoint, bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT) - { - this.socket = socket; - this.remoteEndPoint = remoteEndPoint; - SetupKcp(noDelay, interval, fastResend, congestionWindow, sendWindowSize, receiveWindowSize, timeout); - } + // peer can't be set from constructor at the moment. + // because peer callbacks need to know 'connection'. + // see KcpServer.CreateConnection. + public KcpPeer peer; + public readonly EndPoint remoteEndPoint; - protected override void RawSend(byte[] data, int length) + public KcpServerConnection(EndPoint remoteEndPoint) { - socket.SendTo(data, 0, length, SocketFlags.None, remoteEndPoint); + peer = null; + this.remoteEndPoint = remoteEndPoint; } } } diff --git a/Kcp2k/kcp2k/highlevel/Log.cs b/Kcp2k/kcp2k/highlevel/Log.cs index 939dae7..c28d8b8 100644 --- a/Kcp2k/kcp2k/highlevel/Log.cs +++ b/Kcp2k/kcp2k/highlevel/Log.cs @@ -7,8 +7,8 @@ namespace kcp2k { public static class Log { - public static Action Info = Console.WriteLine; + public static Action Info = Console.WriteLine; public static Action Warning = Console.WriteLine; - public static Action Error = Console.Error.WriteLine; + public static Action Error = Console.Error.WriteLine; } } diff --git a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientConnectionNonAlloc.cs b/Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientConnectionNonAlloc.cs deleted file mode 100644 index b3e1b27..0000000 --- a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientConnectionNonAlloc.cs +++ /dev/null @@ -1,24 +0,0 @@ -// where-allocation version of KcpClientConnection. -// may not be wanted on all platforms, so it's an extra optional class. -using System.Net; -using WhereAllocation; - -namespace kcp2k -{ - public class KcpClientConnectionNonAlloc : KcpClientConnection - { - IPEndPointNonAlloc reusableEP; - - protected override void CreateRemoteEndPoint(IPAddress[] addresses, ushort port) - { - // create reusableEP with same address family as remoteEndPoint. - // otherwise ReceiveFrom_NonAlloc couldn't use it. - reusableEP = new IPEndPointNonAlloc(addresses[0], port); - base.CreateRemoteEndPoint(addresses, port); - } - - // where-allocation nonalloc recv - protected override int ReceiveFrom(byte[] buffer) => - socket.ReceiveFrom_NonAlloc(buffer, reusableEP); - } -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientNonAlloc.cs b/Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientNonAlloc.cs deleted file mode 100644 index acd8e6b..0000000 --- a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpClientNonAlloc.cs +++ /dev/null @@ -1,17 +0,0 @@ -// where-allocation version of KcpClientConnectionNonAlloc. -// may not be wanted on all platforms, so it's an extra optional class. -using System; - -namespace kcp2k -{ - public class KcpClientNonAlloc : KcpClient - { - public KcpClientNonAlloc(Action OnConnected, Action> OnData, Action OnDisconnected) - : base(OnConnected, OnData, OnDisconnected) - { - } - - protected override KcpClientConnection CreateConnection() => - new KcpClientConnectionNonAlloc(); - } -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerConnectionNonAlloc.cs b/Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerConnectionNonAlloc.cs deleted file mode 100644 index fe2e154..0000000 --- a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerConnectionNonAlloc.cs +++ /dev/null @@ -1,25 +0,0 @@ -// where-allocation version of KcpServerConnection. -// may not be wanted on all platforms, so it's an extra optional class. -using System.Net; -using System.Net.Sockets; -using WhereAllocation; - -namespace kcp2k -{ - public class KcpServerConnectionNonAlloc : KcpServerConnection - { - IPEndPointNonAlloc reusableSendEndPoint; - - public KcpServerConnectionNonAlloc(Socket socket, EndPoint remoteEndpoint, IPEndPointNonAlloc reusableSendEndPoint, bool noDelay, uint interval = Kcp.INTERVAL, int fastResend = 0, bool congestionWindow = true, uint sendWindowSize = Kcp.WND_SND, uint receiveWindowSize = Kcp.WND_RCV, int timeout = DEFAULT_TIMEOUT) - : base(socket, remoteEndpoint, noDelay, interval, fastResend, congestionWindow, sendWindowSize, receiveWindowSize, timeout) - { - this.reusableSendEndPoint = reusableSendEndPoint; - } - - protected override void RawSend(byte[] data, int length) - { - // where-allocation nonalloc send - socket.SendTo_NonAlloc(data, 0, length, SocketFlags.None, reusableSendEndPoint); - } - } -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerNonAlloc.cs b/Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerNonAlloc.cs deleted file mode 100644 index ec571b5..0000000 --- a/Kcp2k/kcp2k/highlevel/NonAlloc/KcpServerNonAlloc.cs +++ /dev/null @@ -1,51 +0,0 @@ -// where-allocation version of KcpServer. -// may not be wanted on all platforms, so it's an extra optional class. -using System; -using System.Net; -using System.Net.Sockets; -using WhereAllocation; - -namespace kcp2k -{ - public class KcpServerNonAlloc : KcpServer - { - IPEndPointNonAlloc reusableClientEP; - - public KcpServerNonAlloc(Action OnConnected, Action> OnData, Action OnDisconnected, bool DualMode, bool NoDelay, uint Interval, int FastResend = 0, bool CongestionWindow = true, uint SendWindowSize = Kcp.WND_SND, uint ReceiveWindowSize = Kcp.WND_RCV, int Timeout = KcpConnection.DEFAULT_TIMEOUT) - : base(OnConnected, OnData, OnDisconnected, DualMode, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout) - { - // create reusableClientEP either IPv4 or IPv6 - reusableClientEP = DualMode - ? new IPEndPointNonAlloc(IPAddress.IPv6Any, 0) - : new IPEndPointNonAlloc(IPAddress.Any, 0); - } - - protected override int ReceiveFrom(byte[] buffer, out int connectionHash) - { - // where-allocation nonalloc ReceiveFrom. - int read = socket.ReceiveFrom_NonAlloc(buffer, 0, buffer.Length, SocketFlags.None, reusableClientEP); - SocketAddress remoteAddress = reusableClientEP.temp; - - // where-allocation nonalloc GetHashCode - connectionHash = remoteAddress.GetHashCode(); - return read; - } - - protected override KcpServerConnection CreateConnection() - { - // IPEndPointNonAlloc is reused all the time. - // we can't store that as the connection's endpoint. - // we need a new copy! - IPEndPoint newClientEP = reusableClientEP.DeepCopyIPEndPoint(); - - // for allocation free sending, we also need another - // IPEndPointNonAlloc... - IPEndPointNonAlloc reusableSendEP = new IPEndPointNonAlloc(newClientEP.Address, newClientEP.Port); - - // create a new KcpConnection NonAlloc version - // -> where-allocation IPEndPointNonAlloc is reused. - // need to create a new one from the temp address. - return new KcpServerConnectionNonAlloc(socket, newClientEP, reusableSendEP, NoDelay, Interval, FastResend, CongestionWindow, SendWindowSize, ReceiveWindowSize, Timeout); - } - } -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/kcp/Kcp.cs b/Kcp2k/kcp2k/kcp/Kcp.cs index 253757a..36ec23e 100644 --- a/Kcp2k/kcp2k/kcp/Kcp.cs +++ b/Kcp2k/kcp2k/kcp/Kcp.cs @@ -26,7 +26,8 @@ public class Kcp public const int ACK_FAST = 3; public const int INTERVAL = 100; public const int OVERHEAD = 24; - public const int DEADLINK = 20; + public const int FRG_MAX = byte.MaxValue; // kcp encodes 'frg' as byte. so we can only ever send up to 255 fragments. + public const int DEADLINK = 20; // default maximum amount of 'xmit' retransmissions until a segment is considered lost public const int THRESH_INIT = 2; public const int THRESH_MIN = 2; public const int PROBE_INIT = 7000; // 7 secs to probe window size @@ -58,19 +59,19 @@ internal struct AckItem internal uint cwnd; // congestion window internal uint probe; internal uint interval; - internal uint ts_flush; + internal uint ts_flush; // last flush timestamp in milliseconds internal uint xmit; internal uint nodelay; // not a bool. original Kcp has '<2 else' check. internal bool updated; - internal uint ts_probe; // timestamp probe + internal uint ts_probe; // probe timestamp internal uint probe_wait; - internal uint dead_link; + internal uint dead_link; // maximum amount of 'xmit' retransmissions until a segment is considered lost internal uint incr; internal uint current; // current time (milliseconds). set by Update. internal int fastresend; internal int fastlimit; - internal bool nocwnd; // no congestion window + internal bool nocwnd; // congestion control, negated. heavily restricts send/recv window sizes. internal readonly Queue snd_queue = new Queue(16); // send queue internal readonly Queue rcv_queue = new Queue(16); // receive queue // snd_buffer needs index removals. @@ -103,18 +104,18 @@ internal struct AckItem // from the same connection. public Kcp(uint conv, Action output) { - this.conv = conv; + this.conv = conv; this.output = output; snd_wnd = WND_SND; rcv_wnd = WND_RCV; rmt_wnd = WND_RCV; mtu = MTU_DEF; mss = mtu - OVERHEAD; - rx_rto = RTO_DEF; + rx_rto = RTO_DEF; rx_minrto = RTO_MIN; - interval = INTERVAL; - ts_flush = INTERVAL; - ssthresh = THRESH_INIT; + interval = INTERVAL; + ts_flush = INTERVAL; + ssthresh = THRESH_INIT; fastlimit = FASTACK_LIMIT; dead_link = DEADLINK; buffer = new byte[(mtu + OVERHEAD) * 3]; @@ -247,7 +248,7 @@ public int PeekSize() } // ikcp_send - // sends byte[] to the other end. + // splits message into MTU sized fragments, adds them to snd_queue. public int Send(byte[] buffer, int offset, int len) { // fragment count @@ -262,10 +263,19 @@ public int Send(byte[] buffer, int offset, int len) if (len <= mss) count = 1; else count = (int)((len + mss - 1) / mss); - // original kcp uses WND_RCV const even though rcv_wnd is the - // runtime variable. may or may not be correct, see also: - // see also: https://github.com/skywind3000/kcp/pull/291/files - if (count >= WND_RCV) return -2; + // IMPORTANT kcp encodes 'frg' as 1 byte. + // so we can only support up to 255 fragments. + // (which limits max message size to around 288 KB) + // this is really nasty to debug. let's make this 100% obvious. + if (count > FRG_MAX) + throw new Exception($"Send len={len} requires {count} fragments, but kcp can only handle up to {FRG_MAX} fragments."); + + // original kcp uses WND_RCV const instead of rcv_wnd runtime: + // https://github.com/skywind3000/kcp/pull/291/files + // which always limits max message size to 144 KB: + //if (count >= WND_RCV) return -2; + // using configured rcv_wnd uncorks max message size to 'any': + if (count >= rcv_wnd) return -2; if (count == 0) count = 1; @@ -303,7 +313,7 @@ void UpdateAck(int rtt) // round trip time int delta = rtt - rx_srtt; if (delta < 0) delta = -delta; rx_rttval = (3 * rx_rttval + delta) / 4; - rx_srtt = (7 * rx_srtt + rtt) / 8; + rx_srtt = (7 * rx_srtt + rtt) / 8; if (rx_srtt < 1) rx_srtt = 1; } int rto = rx_srtt + Math.Max((int)interval, 4 * rx_rttval); @@ -511,6 +521,9 @@ public int Input(byte[] data, int offset, int size) if (conv_ != conv) return -1; offset += Utils.Decode8u(data, offset, ref cmd); + // IMPORTANT kcp encodes 'frg' as 1 byte. + // so we can only support up to 255 fragments. + // (which limits max message size to around 288 KB) offset += Utils.Decode8u(data, offset, ref frg); offset += Utils.Decode16U(data, offset, ref wnd); offset += Utils.Decode32U(data, offset, ref ts); @@ -522,7 +535,8 @@ public int Input(byte[] data, int offset, int size) size -= OVERHEAD; // enough remaining to read 'len' bytes of the actual payload? - if (size < len || len < 0) return -2; + // note: original kcp casts uint len to int for <0 check. + if (size < len || (int)len < 0) return -2; if (cmd != CMD_PUSH && cmd != CMD_ACK && cmd != CMD_WASK && cmd != CMD_WINS) @@ -575,8 +589,8 @@ public int Input(byte[] data, int offset, int size) seg.cmd = cmd; seg.frg = frg; seg.wnd = wnd; - seg.ts = ts; - seg.sn = sn; + seg.ts = ts; + seg.sn = sn; seg.una = una; if (len > 0) { @@ -649,27 +663,32 @@ uint WndUnused() } // ikcp_flush - // flush remain ack segments + // flush remain ack segments. + // flush may output multiple <= MTU messages from MakeSpace / FlushBuffer. + // the amount of messages depends on the sliding window. + // configured by send/receive window sizes + congestion control. + // with congestion control, the window will be extremely small(!). public void Flush() { - int offset = 0; // buffer ptr in original C + int size = 0; // amount of bytes to flush. 'buffer ptr' in C. bool lost = false; // lost segments // helper functions void MakeSpace(int space) { - if (offset + space > mtu) + if (size + space > mtu) { - output(buffer, offset); - offset = 0; + output(buffer, size); + size = 0; } } void FlushBuffer() { - if (offset > 0) + // flush buffer up to 'offset' (<= MTU) + if (size > 0) { - output(buffer, offset); + output(buffer, size); } } @@ -694,7 +713,7 @@ void FlushBuffer() // ikcp_ack_get assigns ack[i] to seg.sn, seg.ts seg.sn = ack.serialNumber; seg.ts = ack.timestamp; - offset += seg.Encode(buffer, offset); + size += seg.Encode(buffer, size); } acklist.Clear(); @@ -732,7 +751,7 @@ void FlushBuffer() { seg.cmd = CMD_WASK; MakeSpace(OVERHEAD); - offset += seg.Encode(buffer, offset); + size += seg.Encode(buffer, size); } // flush window probing commands @@ -740,22 +759,28 @@ void FlushBuffer() { seg.cmd = CMD_WINS; MakeSpace(OVERHEAD); - offset += seg.Encode(buffer, offset); + size += seg.Encode(buffer, size); } probe = 0; - // calculate window size + // calculate the window size which is currently safe to send. + // it's send window, or remote window, whatever is smaller. + // for our max uint cwnd_ = Math.Min(snd_wnd, rmt_wnd); - // if congestion window: - if (!nocwnd) cwnd_ = Math.Min(cwnd, cwnd_); - // move data from snd_queue to snd_buf - // sliding window, controlled by snd_nxt && sna_una+cwnd + // double negative: if congestion window is enabled: + // limit window size to cwnd. // - // ELI5: 'snd_nxt' is what we want to send. - // 'snd_una' is what hasn't been acked yet. - // copy up to 'cwnd_' difference between them (sliding window) + // note this may heavily limit window sizes. + // for our max message size test with super large windows of 32k, + // 'congestion window' limits it down from 32.000 to 2. + if (!nocwnd) cwnd_ = Math.Min(cwnd, cwnd_); + + // move cwnd_ 'window size' messages from snd_queue to snd_buf + // 'snd_nxt' is what we want to send. + // 'snd_una' is what hasn't been acked yet. + // copy up to 'cwnd_' difference between them (sliding window) while (Utils.TimeDiff(snd_nxt, snd_una + cwnd_) < 0) { if (snd_queue.Count == 0) break; @@ -832,14 +857,16 @@ void FlushBuffer() int need = OVERHEAD + (int)segment.data.Position; MakeSpace(need); - offset += segment.Encode(buffer, offset); + size += segment.Encode(buffer, size); if (segment.data.Position > 0) { - Buffer.BlockCopy(segment.data.GetBuffer(), 0, buffer, offset, (int)segment.data.Position); - offset += (int)segment.data.Position; + Buffer.BlockCopy(segment.data.GetBuffer(), 0, buffer, size, (int)segment.data.Position); + size += (int)segment.data.Position; } + // dead link happens if a message was resent N times, but an + // ack was still not received. if (segment.xmit >= dead_link) { state = -1; @@ -891,6 +918,9 @@ void FlushBuffer() // // 'current' - current timestamp in millisec. pass it to Kcp so that // Kcp doesn't have to do any stopwatch/deltaTime/etc. code + // + // time as uint, likely to minimize bandwidth. + // uint.max = 4294967295 ms = 1193 hours = 49 days public void Update(uint currentTimeMilliSeconds) { current = currentTimeMilliSeconds; @@ -901,18 +931,26 @@ public void Update(uint currentTimeMilliSeconds) ts_flush = current; } + // slap is time since last flush in milliseconds int slap = Utils.TimeDiff(current, ts_flush); + // hard limit: if 10s elapsed, always flush no matter what if (slap >= 10000 || slap < -10000) { ts_flush = current; slap = 0; } + // last flush is increased by 'interval' each time. + // so slap >= is a strange way to check if interval has elapsed yet. if (slap >= 0) { + // increase last flush time by one interval ts_flush += interval; - if (Utils.TimeDiff(current, ts_flush) >= 0) + + // if last flush is still behind, increase it to current + interval + // if (Utils.TimeDiff(current, ts_flush) >= 0) // original kcp.c + if (current >= ts_flush) // less confusing { ts_flush = current + interval; } @@ -984,8 +1022,8 @@ public void SetMtu(uint mtu) // ikcp_interval public void SetInterval(uint interval) { - if (interval > 5000) interval = 5000; - else if (interval < 10) interval = 10; + if (interval > 5000) interval = 5000; + else if (interval < 10) interval = 10; this.interval = interval; } diff --git a/Kcp2k/kcp2k/kcp/Segment.cs b/Kcp2k/kcp2k/kcp/Segment.cs index b5c9dcf..759a075 100644 --- a/Kcp2k/kcp2k/kcp/Segment.cs +++ b/Kcp2k/kcp2k/kcp/Segment.cs @@ -7,20 +7,20 @@ internal class Segment { internal uint conv; // conversation internal uint cmd; // command, e.g. Kcp.CMD_ACK etc. - internal uint frg; // fragment + internal uint frg; // fragment (sent as 1 byte) internal uint wnd; // window size that the receive can currently receive internal uint ts; // timestamp internal uint sn; // serial number internal uint una; internal uint resendts; // resend timestamp - internal int rto; + internal int rto; internal uint fastack; - internal uint xmit; + internal uint xmit; // retransmit count // we need an auto scaling byte[] with a WriteBytes function. // MemoryStream does that perfectly, no need to reinvent the wheel. // note: no need to pool it, because Segment is already pooled. - // -> MTU as initial capacity to avoid most runtime resizing/allocations + // -> default MTU as initial capacity to avoid most runtime resizing/allocations internal MemoryStream data = new MemoryStream(Kcp.MTU_DEF); // ikcp_encode_seg @@ -30,6 +30,9 @@ internal int Encode(byte[] ptr, int offset) int offset_ = offset; offset += Utils.Encode32U(ptr, offset, conv); offset += Utils.Encode8u(ptr, offset, (byte)cmd); + // IMPORTANT kcp encodes 'frg' as 1 byte. + // so we can only support up to 255 fragments. + // (which limits max message size to around 288 KB) offset += Utils.Encode8u(ptr, offset, (byte)frg); offset += Utils.Encode16U(ptr, offset, (ushort)wnd); offset += Utils.Encode32U(ptr, offset, ts); @@ -47,13 +50,13 @@ internal void Reset() cmd = 0; frg = 0; wnd = 0; - ts = 0; - sn = 0; + ts = 0; + sn = 0; una = 0; rto = 0; xmit = 0; resendts = 0; - fastack = 0; + fastack = 0; // keep buffer for next pool usage, but reset length (= bytes written) data.SetLength(0); diff --git a/Kcp2k/kcp2k/where-allocation/LICENSE b/Kcp2k/kcp2k/where-allocation/LICENSE deleted file mode 100644 index 0330370..0000000 --- a/Kcp2k/kcp2k/where-allocation/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Mirror Networking (vis2k, FakeByte) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/Kcp2k/kcp2k/where-allocation/Scripts/AssemblyInfo.cs b/Kcp2k/kcp2k/where-allocation/Scripts/AssemblyInfo.cs deleted file mode 100644 index 246a5d1..0000000 --- a/Kcp2k/kcp2k/where-allocation/Scripts/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("where-allocations.Tests")] \ No newline at end of file diff --git a/Kcp2k/kcp2k/where-allocation/Scripts/Extensions.cs b/Kcp2k/kcp2k/where-allocation/Scripts/Extensions.cs deleted file mode 100644 index fcf18f6..0000000 --- a/Kcp2k/kcp2k/where-allocation/Scripts/Extensions.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Net; -using System.Net.Sockets; - -namespace WhereAllocation -{ - public static class Extensions - { - // always pass the same IPEndPointNonAlloc instead of allocating a new - // one each time. - // - // use IPEndPointNonAlloc.temp to get the latest SocketAdddress written - // by ReceiveFrom_Internal! - // - // IMPORTANT: .temp will be overwritten in next call! - // hash or manually copy it if you need to store it, e.g. - // when adding a new connection. - public static int ReceiveFrom_NonAlloc( - this Socket socket, - byte[] buffer, - int offset, - int size, - SocketFlags socketFlags, - IPEndPointNonAlloc remoteEndPoint) - { - // call ReceiveFrom with IPEndPointNonAlloc. - // need to wrap this in ReceiveFrom_NonAlloc because it's not - // obvious that IPEndPointNonAlloc.Create does NOT create a new - // IPEndPoint. it saves the result in IPEndPointNonAlloc.temp! - EndPoint casted = remoteEndPoint; - return socket.ReceiveFrom(buffer, offset, size, socketFlags, ref casted); - } - - // same as above, different parameters - public static int ReceiveFrom_NonAlloc(this Socket socket, byte[] buffer, IPEndPointNonAlloc remoteEndPoint) - { - EndPoint casted = remoteEndPoint; - return socket.ReceiveFrom(buffer, ref casted); - } - - // SendTo allocates too: - // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L2240 - // -> the allocation is in EndPoint.Serialize() - // NOTE: technically this function isn't necessary. - // could just pass IPEndPointNonAlloc. - // still good for strong typing. - public static int SendTo_NonAlloc( - this Socket socket, - byte[] buffer, - int offset, - int size, - SocketFlags socketFlags, - IPEndPointNonAlloc remoteEndPoint) - { - EndPoint casted = remoteEndPoint; - return socket.SendTo(buffer, offset, size, socketFlags, casted); - } - } -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/where-allocation/Scripts/IPEndPointNonAlloc.cs b/Kcp2k/kcp2k/where-allocation/Scripts/IPEndPointNonAlloc.cs deleted file mode 100644 index 65eb453..0000000 --- a/Kcp2k/kcp2k/where-allocation/Scripts/IPEndPointNonAlloc.cs +++ /dev/null @@ -1,208 +0,0 @@ -using System; -using System.Net; -using System.Net.Sockets; - -namespace WhereAllocation -{ - public class IPEndPointNonAlloc : IPEndPoint - { - // Two steps to remove allocations in ReceiveFrom_Internal: - // - // 1.) remoteEndPoint.Serialize(): - // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1733 - // -> creates an EndPoint for ReceiveFrom_Internal to write into - // -> it's never read from: - // ReceiveFrom_Internal passes it to native: - // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1885 - // native recv populates 'sockaddr* from' with the remote address: - // https://docs.microsoft.com/en-us/windows/win32/api/winsock/nf-winsock-recvfrom - // -> can NOT be null. bricks both Unity and Unity Hub otherwise. - // -> it seems as if Serialize() is only called to avoid allocating - // a 'new SocketAddress' in ReceiveFrom. it's up to the EndPoint. - // - // 2.) EndPoint.Create(SocketAddress): - // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1761 - // -> SocketAddress is the remote's address that we want to return - // -> to avoid 'new EndPoint(SocketAddress), it seems up to the user - // to decide how to create a new EndPoint via .Create - // -> SocketAddress is the object that was returned by Serialize() - // - // in other words, all we need is an extra SocketAddress field that we - // can pass to ReceiveFrom_Internal to write the result into. - // => callers can then get the result from the extra field! - // => no allocations - // - // IMPORTANT: remember that IPEndPointNonAlloc is always the same object - // and never changes. only the helper field is changed. - public SocketAddress temp; - - // constructors simply create the field once by calling the base method. - // (our overwritten method would create anything new) - public IPEndPointNonAlloc(long address, int port) : base(address, port) - { - temp = base.Serialize(); - } - public IPEndPointNonAlloc(IPAddress address, int port) : base(address, port) - { - temp = base.Serialize(); - } - - // Serialize simply returns it - public override SocketAddress Serialize() => temp; - - // Create doesn't need to create anything. - // SocketAddress object is already the one we returned in Serialize(). - // ReceiveFrom_Internal simply wrote into it. - public override EndPoint Create(SocketAddress socketAddress) - { - // original IPEndPoint.Create validates: - if (socketAddress.Family != AddressFamily) - throw new ArgumentException($"Unsupported socketAddress.AddressFamily: {socketAddress.Family}. Expected: {AddressFamily}"); - if (socketAddress.Size < 8) - throw new ArgumentException($"Unsupported socketAddress.Size: {socketAddress.Size}. Expected: <8"); - - // double check to guarantee that ReceiveFrom actually did write - // into our 'temp' field. just in case that's ever changed. - if (socketAddress != temp) - { - // well this is fun. - // in the latest mono from the above github links, - // the result of Serialize() is passed as 'ref' so ReceiveFrom - // does in fact write into it. - // - // in Unity 2019 LTS's mono version, it does create a new one - // each time. this is from ILSpy Receive_From: - // - // SocketPal.CheckDualModeReceiveSupport(this); - // ValidateBlockingMode(); - // if (NetEventSource.IsEnabled) - // { - // NetEventSource.Info(this, $"SRC{LocalEndPoint} size:{size} remoteEP:{remoteEP}", "ReceiveFrom"); - // } - // EndPoint remoteEP2 = remoteEP; - // System.Net.Internals.SocketAddress socketAddress = SnapshotAndSerialize(ref remoteEP2); - // System.Net.Internals.SocketAddress socketAddress2 = IPEndPointExtensions.Serialize(remoteEP2); - // int bytesTransferred; - // SocketError socketError = SocketPal.ReceiveFrom(_handle, buffer, offset, size, socketFlags, socketAddress.Buffer, ref socketAddress.InternalSize, out bytesTransferred); - // SocketException ex = null; - // if (socketError != 0) - // { - // ex = new SocketException((int)socketError); - // UpdateStatusAfterSocketError(ex); - // if (NetEventSource.IsEnabled) - // { - // NetEventSource.Error(this, ex, "ReceiveFrom"); - // } - // if (ex.SocketErrorCode != SocketError.MessageSize) - // { - // throw ex; - // } - // } - // if (!socketAddress2.Equals(socketAddress)) - // { - // try - // { - // remoteEP = remoteEP2.Create(socketAddress); - // } - // catch - // { - // } - // if (_rightEndPoint == null) - // { - // _rightEndPoint = remoteEP2; - // } - // } - // if (ex != null) - // { - // throw ex; - // } - // if (NetEventSource.IsEnabled) - // { - // NetEventSource.DumpBuffer(this, buffer, offset, size, "ReceiveFrom"); - // NetEventSource.Exit(this, bytesTransferred, "ReceiveFrom"); - // } - // return bytesTransferred; - // - - // so until they upgrade their mono version, we are stuck with - // some allocations. - // - // for now, let's pass the newly created on to our temp so at - // least we reuse it next time. - temp = socketAddress; - - // SocketAddress.GetHashCode() depends on SocketAddress.m_changed. - // ReceiveFrom only sets the buffer, it does not seem to set m_changed. - // we need to reset m_changed for two reasons: - // * if m_changed is false, GetHashCode() returns the cahced m_hash - // which is '0'. that would be a problem. - // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/SocketAddress.cs#L262 - // * if we have a cached m_hash, but ReceiveFrom modified the buffer - // then the GetHashCode() should change too. so we need to reset - // either way. - // - // the only way to do that is by _actually_ modifying the buffer: - // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/SocketAddress.cs#L99 - // so let's do that. - // -> unchecked in case it's byte.Max - unchecked - { - temp[0] += 1; - temp[0] -= 1; - } - - // make sure this worked. - // at least throw an Exception to make it obvious if the trick does - // not work anymore, in case ReceiveFrom is ever changed. - if (temp.GetHashCode() == 0) - throw new Exception($"SocketAddress GetHashCode() is 0 after ReceiveFrom. Does the m_changed trick not work anymore?"); - - // in the future, enable this again: - //throw new Exception($"Socket.ReceiveFrom(): passed SocketAddress={socketAddress} but expected {temp}. This should never happen. Did ReceiveFrom() change?"); - } - - // ReceiveFrom sets seed_endpoint to the result of Create(): - // https://github.com/mono/mono/blob/f74eed4b09790a0929889ad7fc2cf96c9b6e3757/mcs/class/System/System.Net.Sockets/Socket.cs#L1764 - // so let's return ourselves at least. - // (seed_endpoint only seems to matter for BeginSend etc.) - return this; - } - - // we need to overwrite GetHashCode() for two reasons. - // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPEndPoint.cs#L160 - // * it uses m_Address. but our true SocketAddress is in m_temp. - // m_Address might not be set at all. - // * m_Address.GetHashCode() allocates: - // https://github.com/mono/mono/blob/bdd772531d379b4e78593587d15113c37edd4a64/mcs/class/referencesource/System/net/System/Net/IPAddress.cs#L699 - public override int GetHashCode() => temp.GetHashCode(); - - // helper function to create an ACTUAL new IPEndPoint from this. - // server needs it to store new connections as unique IPEndPoints. - public IPEndPoint DeepCopyIPEndPoint() - { - // we need to create a new IPEndPoint from 'temp' SocketAddress. - // there is no 'new IPEndPoint(SocketAddress) constructor. - // so we need to be a bit creative... - - // allocate a placeholder IPAddress to copy - // our SocketAddress into. - // -> needs to be the same address family. - IPAddress ipAddress; - if (temp.Family == AddressFamily.InterNetworkV6) - ipAddress = IPAddress.IPv6Any; - else if (temp.Family == AddressFamily.InterNetwork) - ipAddress = IPAddress.Any; - else - throw new Exception($"Unexpected SocketAddress family: {temp.Family}"); - - // allocate a placeholder IPEndPoint - // with the needed size form IPAddress. - // (the real class. not NonAlloc) - IPEndPoint placeholder = new IPEndPoint(ipAddress, 0); - - // the real IPEndPoint's .Create function can create a new IPEndPoint - // copy from a SocketAddress. - return (IPEndPoint)placeholder.Create(temp); - } - } -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/where-allocation/Scripts/where-allocations.asmdef b/Kcp2k/kcp2k/where-allocation/Scripts/where-allocations.asmdef deleted file mode 100644 index a185c2b..0000000 --- a/Kcp2k/kcp2k/where-allocation/Scripts/where-allocations.asmdef +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "where-allocations", - "references": [], - "includePlatforms": [], - "excludePlatforms": [], - "allowUnsafeCode": false, - "overrideReferences": false, - "precompiledReferences": [], - "autoReferenced": true, - "defineConstraints": [], - "versionDefines": [], - "noEngineReferences": false -} \ No newline at end of file diff --git a/Kcp2k/kcp2k/where-allocation/VERSION b/Kcp2k/kcp2k/where-allocation/VERSION deleted file mode 100644 index 8341d28..0000000 --- a/Kcp2k/kcp2k/where-allocation/VERSION +++ /dev/null @@ -1,2 +0,0 @@ -V0.1 [2021-06-01] -- initial release \ No newline at end of file diff --git a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs index 59078d4..6a4e43e 100644 --- a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs @@ -27,7 +27,7 @@ internal class EchoClient : AClient private readonly BenchmarkStatistics benchmarkStatistics; private readonly Thread tickThread; - private readonly KcpClientConnection client; + private readonly KcpClient client; private readonly KcpChannel communicationChannel; private readonly bool noDelay; @@ -39,11 +39,12 @@ public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkSta noDelay = true; communicationChannel = Kcp2kBenchmark.GetChannel(config.Transmission); - client = new KcpClientConnection(); + KcpConfig kcpConfig = new KcpConfig(); + var interval = (uint) Utilities.CalculateTimeout(config.ClientTickRate); + kcpConfig.Interval = interval; + kcpConfig.NoDelay = noDelay; - client.OnAuthenticated = OnPeerConnected; - client.OnData = OnNetworkReceive; - client.OnDisconnected = OnPeerDisconnected; + client = new KcpClient(OnPeerConnected, OnNetworkReceive, OnPeerDisconnected, OnError, kcpConfig); tickThread = new Thread(TickLoop); tickThread.Name = $"Kcp2k Client {id}"; @@ -62,8 +63,7 @@ public override void StartClient() private void TickLoop() { - var interval = (uint) Utilities.CalculateTimeout(config.ClientTickRate); - client.Connect(config.Address, (ushort) config.Port, noDelay, interval); + client.Connect(config.Address, (ushort) config.Port); while (Listen) { @@ -74,9 +74,7 @@ private void TickLoop() private void Tick() { - client.RawReceive(); - client.TickIncoming(); - client.TickOutgoing(); + client.Tick(); } public override void StartBenchmark() @@ -126,7 +124,7 @@ private void Send(ArraySegment buffer, KcpChannel channel) return; } - client.SendData(buffer, channel); + client.Send(buffer, channel); Interlocked.Increment(ref benchmarkStatistics.MessagesClientSent); } @@ -136,7 +134,7 @@ private void OnPeerConnected() isConnected = true; } - private void OnNetworkReceive(ArraySegment arraySegment) + private void OnNetworkReceive(ArraySegment arraySegment, KcpChannel kcpChannel) { if (BenchmarkRunning) { @@ -157,5 +155,13 @@ private void OnPeerDisconnected() isConnected = false; } + + private void OnError(ErrorCode errorCode, string message) + { + if (BenchmarkRunning) + { + Utilities.WriteVerboseLine($"Client {id} error {errorCode}: {message}"); + } + } } } diff --git a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs index 288d99c..5cf116f 100644 --- a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs @@ -34,7 +34,11 @@ public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) var interval = (uint) Utilities.CalculateTimeout(config.ServerTickRate); - server = new KcpServer(OnConnected, OnReceiveMessage, OnDisconnected, DualMode, NoDelay, interval); + KcpConfig kcpConfig = new KcpConfig(); + kcpConfig.DualMode = DualMode; + kcpConfig.NoDelay = NoDelay; + kcpConfig.Interval = interval; + server = new KcpServer(OnConnected, OnReceiveMessage, OnDisconnected, OnError, kcpConfig); serverThread = new Thread(TickLoop); serverThread.Name = "Kcp2k Server"; @@ -87,7 +91,7 @@ private void OnConnected(int connectionId) } } - private void OnReceiveMessage(int connectionId, ArraySegment arraySegment) + private void OnReceiveMessage(int connectionId, ArraySegment arraySegment, KcpChannel arg3) { if (benchmarkRunning) { @@ -108,6 +112,14 @@ private void OnDisconnected(int connectionId) } } + private void OnError(int clientId, ErrorCode errorCode, string message) + { + if (benchmarkPreparing || benchmarkRunning) + { + Utilities.WriteVerboseLine($"Server error for client {clientId} {errorCode}: {message}"); + } + } + private void Send(int connectionId, ArraySegment message, KcpChannel channel) { server.Send(connectionId, message, channel); @@ -118,7 +130,7 @@ private void Broadcast(ArraySegment message, KcpChannel channel) { foreach (var connection in server.connections.Values) { - connection.SendData(message, channel); + connection.peer.SendData(message, channel); } var messagesSent = server.connections.Count; From a63e20eb350cab45cb70f28e8e6bf69291c40063 Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Mon, 3 Apr 2023 21:30:54 +0200 Subject: [PATCH 3/7] Add option to for native sockets commandline arguments --- NetworkBenchmarkDotNet/Configuration/Configuration.cs | 8 ++++++++ NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs | 8 +++----- NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs | 4 +--- NetworkBenchmarkDotNet/Utils/CommandLineUtilities.cs | 6 +++++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/NetworkBenchmarkDotNet/Configuration/Configuration.cs b/NetworkBenchmarkDotNet/Configuration/Configuration.cs index e3750c1..abec7b5 100644 --- a/NetworkBenchmarkDotNet/Configuration/Configuration.cs +++ b/NetworkBenchmarkDotNet/Configuration/Configuration.cs @@ -99,6 +99,12 @@ public class Configuration /// public int ServerTickRate { get; set; } + /// + /// Special feature of LiteNetLib for using faster socket implementation + /// This is currently not supported for all platforms + /// + public bool UseNativeSockets { get; set; } + public byte[] Message { get; private set; } public void PrepareForNewBenchmark() @@ -210,6 +216,7 @@ public void AppendCommandlineInstruction(StringBuilder sb) sb.Append($" --message-payload {MessagePayload}"); sb.Append($" --client-tick-rate {ClientTickRate}"); sb.Append($" --server-tick-rate {ServerTickRate}"); + sb.Append($" --use-native-sockets {UseNativeSockets}"); } public static void ApplyPredefinedBenchmarkConfiguration(Configuration config) @@ -225,6 +232,7 @@ public static void ApplyPredefinedBenchmarkConfiguration(Configuration config) config.MessageByteSize = 32; config.ServerTickRate = 60; config.ClientTickRate = 60; + config.UseNativeSockets = true; } } } diff --git a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs index 06a4d62..dcb5871 100644 --- a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs @@ -28,7 +28,6 @@ internal class EchoClient : AClient, IClient, INetEventListener private readonly BenchmarkStatistics benchmarkStatistics; private readonly NetManager netManager; - private readonly DeliveryMethod deliveryMethod; private NetPeer peer; public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) @@ -36,7 +35,6 @@ public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkSta this.id = id; this.config = config; this.benchmarkStatistics = benchmarkStatistics; - deliveryMethod = LiteNetLibBenchmark.GetDeliveryMethod(config.Transmission); netManager = new NetManager(this); if (!config.Address.Contains(':')) @@ -44,7 +42,7 @@ public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkSta netManager.IPv6Mode = IPv6Mode.Disabled; } - //netManager.UseNativeSockets = true; + netManager.UseNativeSockets = config.UseNativeSockets; netManager.UpdateTime = Utilities.CalculateTimeout(config.ClientTickRate); netManager.UnsyncedEvents = true; netManager.DisconnectTimeout = 10000; @@ -138,14 +136,14 @@ void INetEventListener.OnPeerDisconnected(NetPeer peer, DisconnectInfo disconnec isConnected = false; } - void INetEventListener.OnNetworkReceive(NetPeer peer, NetPacketReader reader, byte channelNumber, DeliveryMethod deliverymethod) + void INetEventListener.OnNetworkReceive(NetPeer peer, NetPacketReader reader, byte channelNumber, DeliveryMethod deliveryMethod) { if (BenchmarkRunning) { Interlocked.Increment(ref benchmarkStatistics.MessagesClientReceived); if (!ManualMode) { - Send(Message, deliverymethod); + Send(Message, deliveryMethod); netManager.TriggerUpdate(); } } diff --git a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs index f17b749..94c20d9 100644 --- a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs @@ -23,17 +23,15 @@ internal class EchoServer : AServer, INetEventListener private readonly Configuration config; private readonly BenchmarkStatistics benchmarkStatistics; private readonly NetManager netManager; - private readonly DeliveryMethod deliveryMethod; public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.config = config; this.benchmarkStatistics = benchmarkStatistics; - deliveryMethod = LiteNetLibBenchmark.GetDeliveryMethod(config.Transmission); netManager = new NetManager(this); netManager.UpdateTime = Utilities.CalculateTimeout(config.ServerTickRate); - //netManager.UseNativeSockets = true; + netManager.UseNativeSockets = config.UseNativeSockets; if (!config.Address.Contains(':')) { netManager.IPv6Mode = IPv6Mode.Disabled; diff --git a/NetworkBenchmarkDotNet/Utils/CommandLineUtilities.cs b/NetworkBenchmarkDotNet/Utils/CommandLineUtilities.cs index 4ab9382..06cc9bb 100644 --- a/NetworkBenchmarkDotNet/Utils/CommandLineUtilities.cs +++ b/NetworkBenchmarkDotNet/Utils/CommandLineUtilities.cs @@ -77,7 +77,11 @@ public static RootCommand GenerateRootCommand() new Option( "--server-tick-rate", getDefaultValue: () => 60, - "Server ticks per second if supported") + "Server ticks per second if supported"), + new Option( + "--use-native-sockets", + getDefaultValue: () => true, + "Use native Sockets (LiteNetLib only)"), }; rootCommand.Name = "NetworkBenchmarkDotNet"; From 13a50437834653b8d292c5a279fe0b35e008558e Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Mon, 3 Apr 2023 21:46:35 +0200 Subject: [PATCH 4/7] Change benchmark scripts to .NET 6 --- Kcp2k/Kcp2k.csproj | 2 +- .../Config/PerformanceBenchmarkConfig.cs | 5 ++--- .../PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs | 5 ++--- .../PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs | 5 ++--- linux-benchmark.sh | 2 +- linux-build.sh | 2 +- win-benchmark.bat | 2 +- win-build.bat | 2 +- 8 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Kcp2k/Kcp2k.csproj b/Kcp2k/Kcp2k.csproj index aca0ec6..cc10006 100644 --- a/Kcp2k/Kcp2k.csproj +++ b/Kcp2k/Kcp2k.csproj @@ -1,7 +1,7 @@ - net5.0;netcoreapp3.1 + net6.0 diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs index 0061a89..fe06b96 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs @@ -26,10 +26,9 @@ public PerformanceBenchmarkConfig() .WithIterationCount(10) .WithGcServer(true) .WithGcConcurrent(true) - .WithGcForce(true) - .WithPlatform(Platform.X64); + .WithGcForce(true); - AddJob(baseJob.WithRuntime(CoreRuntime.Core50)); + AddJob(baseJob.WithRuntime(CoreRuntime.Core60)); ConfigHelper.AddDefaultColumns(this); AddColumn(new MessagesPerSecondColumn()); diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs index 0e2148e..cd6b849 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs @@ -28,11 +28,10 @@ public QuickBenchmarkConfig() .WithIterationCount(5) .WithGcServer(true) .WithGcConcurrent(true) - .WithGcForce(true) - .WithPlatform(Platform.X64); + .WithGcForce(true); // Here you can test different runtimes - AddJob(baseJob.WithRuntime(CoreRuntime.Core50)); + AddJob(baseJob.WithRuntime(CoreRuntime.Core60)); ConfigHelper.AddDefaultColumns(this); AddColumn(new MessagesPerSecondColumn()); diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs index 473bf8f..2cc7273 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs @@ -30,10 +30,9 @@ public SamplingBenchmarkConfig() .WithIterationCount(1) .WithGcServer(true) .WithGcConcurrent(true) - .WithGcForce(true) - .WithPlatform(Platform.X64); + .WithGcForce(true); - AddJob(baseJob.WithRuntime(CoreRuntime.Core50)); + AddJob(baseJob.WithRuntime(CoreRuntime.Core60)); ConfigHelper.AddDefaultColumns(this); diff --git a/linux-benchmark.sh b/linux-benchmark.sh index 812bbe0..f16b45b 100644 --- a/linux-benchmark.sh +++ b/linux-benchmark.sh @@ -4,7 +4,7 @@ # Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build # Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog -dotnet build --configuration Release --framework net5.0 --output ./bin/NetworkBenchmarkDotNet-Linux/ +dotnet build --configuration Release --framework net6.0 --output ./bin/NetworkBenchmarkDotNet-Linux/ if [ -z "$1" ] then diff --git a/linux-build.sh b/linux-build.sh index 2fb1bb7..d5eb0e1 100644 --- a/linux-build.sh +++ b/linux-build.sh @@ -5,4 +5,4 @@ rm -rf ./bin/NetworkBenchmarkDotNet-Linux/ # Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build # Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog -dotnet build --configuration Release --framework net5.0 --output ./bin/NetworkBenchmarkDotNet-Linux/ +dotnet build --configuration Release --framework net6.0 --output ./bin/NetworkBenchmarkDotNet-Linux/ diff --git a/win-benchmark.bat b/win-benchmark.bat index 1792040..791580f 100644 --- a/win-benchmark.bat +++ b/win-benchmark.bat @@ -16,7 +16,7 @@ set /p benchmark=Which benchmark do you want to run [Quick/Performance/Sampling/ echo on :: Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build :: Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog -dotnet build --configuration Release --framework net5.0 --output .\bin\NetworkBenchmarkDotNet-Windows\ +dotnet build --configuration Release --framework net6.0 --output .\bin\NetworkBenchmarkDotNet-Windows\ .\bin\NetworkBenchmarkDotNet-Windows\NetworkBenchmarkDotNet -b %benchmark% echo off diff --git a/win-build.bat b/win-build.bat index 402fee1..4c1b82a 100644 --- a/win-build.bat +++ b/win-build.bat @@ -4,5 +4,5 @@ if exist .\bin\NetworkBenchmarkDotNet-Windows\ rmdir .\bin\NetworkBenchmarkDotNe :: Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build :: Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog -dotnet build --configuration Release --framework net5.0 --output .\bin\NetworkBenchmarkDotNet-Windows\ +dotnet build --configuration Release --framework net6.0 --output .\bin\NetworkBenchmarkDotNet-Windows\ PAUSE From b8e45474dd378671b582bf7854d4a89166932abb Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Tue, 4 Apr 2023 21:14:27 +0200 Subject: [PATCH 5/7] Fix manual config setup Minimal config is now used as the base, therefore no columns are defined twice --- .../PredefinedBenchmarks/Config/ConfigHelper.cs | 2 +- .../PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs | 2 -- .../PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs | 2 -- .../PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs | 2 -- NetworkBenchmarkDotNet/Program.cs | 4 +++- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs index 2b2ff93..b49eb51 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs @@ -26,7 +26,7 @@ public static class ConfigHelper /// * units stay the same /// * No units in cell data (Always numbers) /// - public static readonly SummaryStyle CsvStyle = new SummaryStyle(CultureInfo.InvariantCulture, false, SizeUnit.KB, TimeUnit.Millisecond, + private static readonly SummaryStyle CsvStyle = new SummaryStyle(CultureInfo.InvariantCulture, false, SizeUnit.KB, TimeUnit.Millisecond, false, true, 100); public static void AddDefaultColumns(ManualConfig config) diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs index fe06b96..ad0d6f4 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs @@ -18,8 +18,6 @@ public class PerformanceBenchmarkConfig : ManualConfig { public PerformanceBenchmarkConfig() { - Add(DefaultConfig.Instance); - Job baseJob = Job.Default .WithLaunchCount(1) .WithWarmupCount(1) diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs index cd6b849..34d8f94 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs @@ -19,8 +19,6 @@ public class QuickBenchmarkConfig : ManualConfig { public QuickBenchmarkConfig() { - Add(DefaultConfig.Instance); - Job baseJob = Job.Default .WithStrategy(RunStrategy.Monitoring) .WithLaunchCount(1) diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs index 2cc7273..7557aed 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs @@ -22,8 +22,6 @@ public class SamplingBenchmarkConfig : ManualConfig { public SamplingBenchmarkConfig() { - Add(DefaultConfig.Instance); - Job baseJob = Job.Default .WithLaunchCount(1) .WithWarmupCount(1) diff --git a/NetworkBenchmarkDotNet/Program.cs b/NetworkBenchmarkDotNet/Program.cs index 06c82aa..8d07a66 100644 --- a/NetworkBenchmarkDotNet/Program.cs +++ b/NetworkBenchmarkDotNet/Program.cs @@ -12,6 +12,7 @@ using System.CommandLine; using System.CommandLine.Invocation; using System.Linq; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; @@ -103,7 +104,8 @@ private static void RunCustomBenchmark() /// Type of the benchmark to run private static void RunBenchmark() { - var summary = BenchmarkRunner.Run(); + ManualConfig config = ManualConfig.CreateMinimumViable(); + var summary = BenchmarkRunner.Run(config); Assert(!summary.HasCriticalValidationErrors, "The \"Summary\" should have NOT \"HasCriticalValidationErrors\""); From 484a725414b4c9e821946b26b36021a02cf7b33b Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Tue, 4 Apr 2023 21:42:51 +0200 Subject: [PATCH 6/7] Update to version 1.1.0 --- NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj index e74d074..d90c9ea 100644 --- a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj +++ b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj @@ -16,7 +16,7 @@ true true true - 1.0.1 + 1.1.0 @@ -60,7 +60,7 @@ true NetworkBenchmark.Program - 0.5.0 + 1.1.0 NNB is a benchmark for low level networking libraries using UDP and can be used with Unity and for .Net 5 standalone server applications. The benchmark focuses on latency, performance and scalability. https://github.com/JohannesDeml/NetworkBenchmarkDotNet sockets, UDP, benchmark, network, Unity, network-benchmark From dc23b3f63d135b388b761e5ff1c01d621c0e01ec Mon Sep 17 00:00:00 2001 From: Johannes Deml Date: Tue, 4 Apr 2023 22:53:02 +0200 Subject: [PATCH 7/7] Update readme with updated libaries and .NET version --- README.md | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b6a25fb..8af8ed0 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Network Benchmark .NET -*Low Level .NET 5 Networking libraries benchmarked for UDP socket performance* +*Low Level .NET 6 Networking libraries benchmarked for UDP socket performance* ![Screenshot](./Docs/screenshot.png) -[![Releases](https://img.shields.io/github/release/JohannesDeml/NetworkBenchmarkDotNet/all.svg)](../../releases) [![.NET 5.0](https://img.shields.io/badge/.NET-5.0-blueviolet.svg)](https://dotnet.microsoft.com/download/dotnet/5.0) +[![Releases](https://img.shields.io/github/release/JohannesDeml/NetworkBenchmarkDotNet/all.svg)](../../releases) [![.NET 6.0](https://img.shields.io/badge/.NET-6.0-blueviolet.svg)](https://dotnet.microsoft.com/download/dotnet/6.0) ## Table of Contents @@ -23,20 +23,20 @@ NBN is a benchmark for low level networking libraries using UDP and can be used ### Supported Libraries -* [ENet-CSharp](https://github.com/nxrighthere/ENet-CSharp) (v 2.4.7) +* [ENet-CSharp](https://github.com/nxrighthere/ENet-CSharp) (v 2.4.8) * Wrapper for [ENet](https://github.com/lsalzman/enet), building a reliable sequenced protocol on top of UDP * Max concurrent connections are limited to 4095 due to the protocol * Packetsize overhead: 10 bytes * [Unity Client Example](https://github.com/JohannesDeml/ENetUnityMobile) -* [LiteNetLib](https://github.com/RevenantX/LiteNetLib) (v 0.9.5.2) +* [LiteNetLib](https://github.com/RevenantX/LiteNetLib) (v 1.0.1) * Very feature-rich library * Packetsize overhead: 1 byte for unreliable, 4 bytes for reliable * [Unity Client Example](https://github.com/RevenantX/NetGameExample) -* [Kcp2k](https://github.com/vis2k/kcp2k) (v 1.12.0) +* [Kcp2k](https://github.com/vis2k/kcp2k) (v 1.34.0) * Port of KCP with 100% C# Code, Future Technology for [Mirror-NG](https://github.com/MirrorNG/MirrorNG) * Packetsize overhead: 24 byte * [Unity Example](https://github.com/vis2k/kcp2k) -* [NetCoreServer](https://github.com/chronoxor/NetCoreServer) (v 5.1.0) +* [NetCoreServer](https://github.com/chronoxor/NetCoreServer) (v 6.7.0) * Pure C# / .Net library for TCP/UDP/SSL with no additional protocols on top * Packetsize overhead: 0 bytes, but you have to invent the wheel yourself * [Unity Client Example](https://github.com/JohannesDeml/Unity-Net-Core-Networking-Sockets) @@ -48,17 +48,29 @@ NBN is a benchmark for low level networking libraries using UDP and can be used * Ubuntu VPS * Virtual private server with dedicated CPU's running - [Hardware](https://www.netcup.eu/bestellen/produkt.php?produkt=2624) - * Ubuntu 20.04.3 LTS x86-64 Kernel 5.14.0-051400-generic - + * Ubuntu 22.04.2 LTS + ``` + $> hostnamectl + Chassis: vm + Virtualization: kvm + Operating System: Ubuntu 22.04.2 LTS + Kernel: Linux 5.15.0-48-generic + Architecture: x86-64 + Hardware Vendor: netcup + Hardware Model: KVM Server + ``` + + + * Ubuntu Desktop / Windows Desktop * Desktop PC from 2020 - [Hardware](https://pcpartpicker.com/user/JohannesDeml/saved/zz7yK8) - * Windows 10 Pro x86-64 Build 19043.1266 (21H1/May2021Update) - * Ubuntu 20.04.3 LTS x86-64 Kernel 5.11.0-37-generic + * xxxxWindows 10 Pro x86-64 Build 19043.1266 (21H1/May2021Update) + * xxxxUbuntu 20.04.3 LTS x86-64 Kernel 5.11.0-37-generic ### Software -* [.NET](https://dotnet.microsoft.com/download/dotnet) 5.0.11 (5.0.1121.47308) -* [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) 0.13.1 +* [.NET](https://dotnet.microsoft.com/download/dotnet) 6.0.407 (`dotnet --version`) +* [BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet) 0.13.5 ### Procedure For the two desktop setups, the benchmarks are run on a restarted system with 5 minutes idle time before starting the benchmark. They are run with admin privileges and all unnecessary other processes are killed before running the benchmarks. For Ubuntu VPS, the benchmarks are run through continuous integration on a typical indie server setup with other processes running as well. After the benchmarks are run, a list of all running processes to make them more reproducible. To reproduce the benchmarks, run `sudo sh linux-benchmark.sh` or `win-benchmark.bat` . If you want to execute directly from the compiled program, run `./NetworkBenchmarkDotNet -b Essential`. @@ -107,7 +119,7 @@ This is a comparison between all tests with their message throughput (higher is ## Installation -Make sure you have [.Net 5 SDK](https://dotnet.microsoft.com/download) installed. +Make sure you have [.Net 6 SDK](https://dotnet.microsoft.com/download) installed. Then just open the solution file with Visual Studio/Rider/Visual Studio Code and build it. Note that results of the benchmarks can be very different with a different operating system and hardware. @@ -187,7 +199,7 @@ Your favorite library is missing, or you feel like the benchmarks are not testin Your new proposed library ... * works with Unity as a Client -* works with .NET 5 for the server +* works with .NET 6 for the server * uses UDP (additional RUDP would be nice) * is Open Source (can still be commercial) * is stable enough not to break in the benchmarks