diff --git a/NetworkBenchmarkDotNet/BenchmarkCoordinator.cs b/NetworkBenchmarkDotNet/BenchmarkCoordinator.cs index 6d890b6..d023426 100644 --- a/NetworkBenchmarkDotNet/BenchmarkCoordinator.cs +++ b/NetworkBenchmarkDotNet/BenchmarkCoordinator.cs @@ -65,12 +65,29 @@ public static void PrepareBenchmark(INetworkBenchmark networkBenchmark) Utilities.WriteVerboseLine(" Done"); } + public static void RunBenchmark(INetworkBenchmark networkBenchmark) + { + if (Config.Test == TestType.Manual) + { + RunManualMode(networkBenchmark); + return; + } + + if (Config.Duration < 0) + { + RunIndefinitely(networkBenchmark); + return; + } + + RunTimedBenchmark(networkBenchmark); + } + /// /// Run the benchmark for a specific duration /// The benchmark needs to be prepared once before running it. /// /// Library to run - public static void RunTimedBenchmark(INetworkBenchmark networkBenchmark) + private static void RunTimedBenchmark(INetworkBenchmark networkBenchmark) { Utilities.WriteVerbose($"-> Run Benchmark {Config.Library}..."); StartBenchmark(networkBenchmark); @@ -85,7 +102,7 @@ public static void RunTimedBenchmark(INetworkBenchmark networkBenchmark) /// Runs until the user stops the process /// /// - public static void RunIndefinitely(INetworkBenchmark networkBenchmark) + private static void RunIndefinitely(INetworkBenchmark networkBenchmark) { Utilities.WriteVerbose($"-> Run indefinitely {Config.Library}... (press enter to stop)"); StartBenchmark(networkBenchmark); @@ -96,6 +113,95 @@ public static void RunIndefinitely(INetworkBenchmark networkBenchmark) Utilities.WriteVerboseLine(" Done"); } + /// + /// Enables to enter defined commands + /// Runs until the user stops the process + /// + /// + private static void RunManualMode(INetworkBenchmark networkBenchmark) + { + Utilities.WriteVerbose($"-> Run Manual Mode {Config.Library}\n"); + StartBenchmark(networkBenchmark); + + bool running = true; + + while (running) + { + var input = Console.ReadLine(); + if (input == null) + { + PrintInvalidManualInput(); + continue; + } + var parts = input.ToLower().Split(' '); + if (parts.Length == 0 || parts[0].Length == 0) + { + PrintInvalidManualInput(); + continue; + } + + running = ProcessManualInput(networkBenchmark, parts); + } + + StopBenchmark(networkBenchmark); + Utilities.WriteVerboseLine(" Done"); + } + + private static bool ProcessManualInput(INetworkBenchmark networkBenchmark, string[] parts) + { + var target = parts[0][0]; + + if (target == 'q') + { + return false; + } + + if (parts.Length < 2 || !int.TryParse(parts[1], out int value)) + { + PrintInvalidManualInput(); + return true; + } + + var transmissionMode = Config.Transmission; + if (parts.Length >= 3) + { + var transmission = parts[2][0]; + if (transmission == 'r') + { + transmissionMode = TransmissionType.Reliable; + } + else if (transmission == 'u') + { + transmissionMode = TransmissionType.Unreliable; + } + } + + switch (target) + { + case 'c': + var clients = networkBenchmark.Clients; + for (int i = 0; i < clients.Count; i++) + { + clients[i].SendMessages(value, transmissionMode); + } + + break; + case 's': + networkBenchmark.Server.SendMessages(value, transmissionMode); + break; + default: + PrintInvalidManualInput(); + break; + } + + return true; + } + + private static void PrintInvalidManualInput() + { + Utilities.WriteVerbose($"Invalid - Enter a command, q|c|s (0-9)* [r|u] e.g. 'c 1' or 's 4 u', to quit enter q"); + } + public static void StartBenchmark(INetworkBenchmark networkBenchmark) { BenchmarkStatistics.Reset(); diff --git a/NetworkBenchmarkDotNet/BenchmarkStatistics.cs b/NetworkBenchmarkDotNet/BenchmarkStatistics.cs index 9e27a0f..636b5a4 100644 --- a/NetworkBenchmarkDotNet/BenchmarkStatistics.cs +++ b/NetworkBenchmarkDotNet/BenchmarkStatistics.cs @@ -1,5 +1,5 @@ // -------------------------------------------------------------------------------------------------------------------- -// +// // Copyright (c) 2020 Johannes Deml. All rights reserved. // // @@ -83,6 +83,7 @@ public string PrintStatistics(Configuration config) { sb.AppendLine($"Client Round Trip Time: {clientRtt.ToString()}"); } + sb.AppendLine("```"); sb.AppendLine(); diff --git a/NetworkBenchmarkDotNet/Configuration/BenchmarkMode.cs b/NetworkBenchmarkDotNet/Configuration/BenchmarkMode.cs index 95ac9e7..cdf63c3 100644 --- a/NetworkBenchmarkDotNet/Configuration/BenchmarkMode.cs +++ b/NetworkBenchmarkDotNet/Configuration/BenchmarkMode.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -28,11 +28,11 @@ public enum BenchmarkMode Performance = 1 << 0, /// - /// Run Benchmark Garbage - /// Benchmark which collects GC information - /// Runtime: ~1 minute + /// Run Benchmark Sampling + /// Benchmark which collects GC information and CPU time samples + /// Runtime: ~30 seconds /// - Garbage = 1 << 1, + Sampling = 1 << 1, /// /// Run all essential benchmarks (Performance, Garbage) diff --git a/NetworkBenchmarkDotNet/Configuration/Configuration.cs b/NetworkBenchmarkDotNet/Configuration/Configuration.cs index db9d7be..e3750c1 100644 --- a/NetworkBenchmarkDotNet/Configuration/Configuration.cs +++ b/NetworkBenchmarkDotNet/Configuration/Configuration.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -157,7 +157,7 @@ public string ToFormattedString() { sb.AppendLine($"* Number of clients: {Clients}"); } - + sb.AppendLine($"* Parallel messages: {ParallelMessages:n0}, Size: {MessageByteSize} bytes, Payload: {MessagePayload}"); sb.AppendLine($"* TickRate per second: Client: {ClientTickRate}, Server: {ServerTickRate}"); sb.AppendLine($"* Reproduce: `"); @@ -169,7 +169,8 @@ public string ToFormattedString() return sb.ToString(); } - private string GetDurationString() { + private string GetDurationString() + { if (Duration < 0) { return "indefinite time"; diff --git a/NetworkBenchmarkDotNet/Configuration/ExecutionMode.cs b/NetworkBenchmarkDotNet/Configuration/ExecutionMode.cs index 9d4d7b7..0885f16 100644 --- a/NetworkBenchmarkDotNet/Configuration/ExecutionMode.cs +++ b/NetworkBenchmarkDotNet/Configuration/ExecutionMode.cs @@ -24,13 +24,15 @@ public enum ExecutionMode /// Run the clients /// Client = 1 << 0, + /// /// Run the server /// Server = 1 << 1, + /// /// Run both clients and server /// - Complete = (1 << 2) -1 + Complete = (1 << 2) - 1 } } diff --git a/NetworkBenchmarkDotNet/Configuration/MessagePayload.cs b/NetworkBenchmarkDotNet/Configuration/MessagePayload.cs index 06f6a53..8b4aa38 100644 --- a/NetworkBenchmarkDotNet/Configuration/MessagePayload.cs +++ b/NetworkBenchmarkDotNet/Configuration/MessagePayload.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/Configuration/TestType.cs b/NetworkBenchmarkDotNet/Configuration/TestType.cs index 031dce7..3a42fbb 100644 --- a/NetworkBenchmarkDotNet/Configuration/TestType.cs +++ b/NetworkBenchmarkDotNet/Configuration/TestType.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -18,6 +18,20 @@ public enum TestType /// 3. Client again sends each message it receives back to the server and so on. /// Also known as Echo. Great for testing round trip if just one parallel message is used. /// - PingPong + PingPong, + + /// + /// Messages are sent manually by clients or server + /// Helps to understand data by sniffing the traffic or to debug libraries + /// Can also be used in conjunction with a ping-pong server process to get direct responses from the server + /// + /// + /// c 1 // sends one message for each client + /// s 5 // send five messages to each client the server is connected to + /// q // quit + /// + /// + /// + Manual } } diff --git a/NetworkBenchmarkDotNet/Libraries/AClient.cs b/NetworkBenchmarkDotNet/Libraries/AClient.cs index e0bf4e2..d2987da 100644 --- a/NetworkBenchmarkDotNet/Libraries/AClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/AClient.cs @@ -8,6 +8,8 @@ // // -------------------------------------------------------------------------------------------------------------------- +using System; + namespace NetworkBenchmark { public abstract class AClient : IClient @@ -25,15 +27,33 @@ public virtual bool IsStopped /// Benchmark is preparing to be run /// protected volatile bool BenchmarkPreparing; + /// /// Client should listen for incoming messages /// protected volatile bool Listen; + /// /// Is a benchmark running (and therefore messages should be counted in the statistics) /// protected volatile bool BenchmarkRunning; + /// + /// Manual Mode stops the default behavior and waits for user input to execute tasks + /// + protected readonly bool ManualMode; + + protected readonly byte[] Message; + + protected AClient(Configuration config) + { + ManualMode = config.Test == TestType.Manual; + + // Use Pinned Object Heap to reduce GC pressure + Message = GC.AllocateArray(config.MessageByteSize, true); + config.Message.CopyTo(Message, 0); + } + public virtual void StartClient() { Listen = true; @@ -59,5 +79,11 @@ public virtual void StopClient() public abstract void DisconnectClient(); public abstract void Dispose(); + + #region ManualMode + + public abstract void SendMessages(int messageCount, TransmissionType transmissionType); + + #endregion } } diff --git a/NetworkBenchmarkDotNet/Libraries/ANetworkBenchmark.cs b/NetworkBenchmarkDotNet/Libraries/ANetworkBenchmark.cs index fb6c641..bb51d14 100644 --- a/NetworkBenchmarkDotNet/Libraries/ANetworkBenchmark.cs +++ b/NetworkBenchmarkDotNet/Libraries/ANetworkBenchmark.cs @@ -15,33 +15,37 @@ namespace NetworkBenchmark { public abstract class ANetworkBenchmark : INetworkBenchmark { + public IServer Server => server; + public List Clients => clients; + private Configuration config; private BenchmarkStatistics benchmarkStatistics; - private IServer echoServer; - private List echoClients; + private IServer server; + private List clients; + public virtual void Initialize(Configuration config, BenchmarkStatistics benchmarkStatistics) { this.config = config; this.benchmarkStatistics = benchmarkStatistics; - echoServer = CreateNewServer(config, benchmarkStatistics); - echoClients = new List(); + server = CreateNewServer(config, benchmarkStatistics); + clients = new List(); } protected abstract IServer CreateNewServer(Configuration config, BenchmarkStatistics statistics); public Task StartServer() { - echoServer.StartServer(); - return Utilities.WaitForServerToStart(echoServer); + server.StartServer(); + return Utilities.WaitForServerToStart(server); } public Task StartClients() { for (int i = 0; i < config.Clients; i++) { - echoClients.Add(CreateNewClient(i, config, benchmarkStatistics)); + clients.Add(CreateNewClient(i, config, benchmarkStatistics)); } return Task.CompletedTask; @@ -53,71 +57,71 @@ public Task ConnectClients() { for (int i = 0; i < config.Clients; i++) { - echoClients[i].StartClient(); + clients[i].StartClient(); } - return Utilities.WaitForClientsToConnect(echoClients); + return Utilities.WaitForClientsToConnect(clients); } public void StartBenchmark() { - echoServer.StartBenchmark(); + server.StartBenchmark(); - for (int i = 0; i < echoClients.Count; i++) + for (int i = 0; i < clients.Count; i++) { - echoClients[i].StartBenchmark(); + clients[i].StartBenchmark(); } } public void StopBenchmark() { - echoServer.StopBenchmark(); + server.StopBenchmark(); - for (int i = 0; i < echoClients.Count; i++) + for (int i = 0; i < clients.Count; i++) { - echoClients[i].StopBenchmark(); + clients[i].StopBenchmark(); } } public Task DisconnectClients() { - for (int i = 0; i < echoClients.Count; i++) + for (int i = 0; i < clients.Count; i++) { - echoClients[i].DisconnectClient(); + clients[i].DisconnectClient(); } - return Utilities.WaitForClientsToDisconnect(echoClients); + return Utilities.WaitForClientsToDisconnect(clients); } public Task StopServer() { - echoServer.StopServer(); - return Utilities.WaitForServerToStop(echoServer); + server.StopServer(); + return Utilities.WaitForServerToStop(server); } public Task StopClients() { - for (int i = 0; i < echoClients.Count; i++) + for (int i = 0; i < clients.Count; i++) { - echoClients[i].StopClient(); + clients[i].StopClient(); } - return Utilities.WaitForClientsToStop(echoClients); + return Utilities.WaitForClientsToStop(clients); } public Task DisposeClients() { - for (int i = 0; i < echoClients.Count; i++) + for (int i = 0; i < clients.Count; i++) { - echoClients[i].Dispose(); + clients[i].Dispose(); } - return Utilities.WaitForClientsToDispose(echoClients); + return Utilities.WaitForClientsToDispose(clients); } public Task DisposeServer() { - echoServer.Dispose(); + server.Dispose(); return Task.CompletedTask; } diff --git a/NetworkBenchmarkDotNet/Libraries/AServer.cs b/NetworkBenchmarkDotNet/Libraries/AServer.cs index a87c600..8a978e4 100644 --- a/NetworkBenchmarkDotNet/Libraries/AServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/AServer.cs @@ -8,6 +8,8 @@ // // -------------------------------------------------------------------------------------------------------------------- +using System; + namespace NetworkBenchmark { public abstract class AServer : IServer @@ -18,6 +20,21 @@ public abstract class AServer : IServer protected volatile bool listen; protected volatile bool benchmarkRunning; + /// + /// Manual Mode stops the default behavior and waits for user input to execute tasks + /// + protected readonly bool ManualMode; + + protected readonly byte[] MessageBuffer; + + protected AServer(Configuration config) + { + ManualMode = config.Test == TestType.Manual; + + // Use Pinned Object Heap to reduce GC pressure + MessageBuffer = GC.AllocateArray(config.MessageByteSize, true); + config.Message.CopyTo(MessageBuffer, 0); + } public virtual void StartServer() { @@ -42,5 +59,11 @@ public virtual void StopServer() } public abstract void Dispose(); + + #region ManualMode + + public abstract void SendMessages(int messageCount, TransmissionType transmissionType); + + #endregion } } diff --git a/NetworkBenchmarkDotNet/Libraries/Enet/ENetBenchmark.cs b/NetworkBenchmarkDotNet/Libraries/Enet/ENetBenchmark.cs index b0ff64e..6128cb4 100644 --- a/NetworkBenchmarkDotNet/Libraries/Enet/ENetBenchmark.cs +++ b/NetworkBenchmarkDotNet/Libraries/Enet/ENetBenchmark.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -8,7 +8,6 @@ // // -------------------------------------------------------------------------------------------------------------------- - using System; using ENet; diff --git a/NetworkBenchmarkDotNet/Libraries/Enet/EchoClient.cs b/NetworkBenchmarkDotNet/Libraries/Enet/EchoClient.cs index dfeb955..4078f97 100644 --- a/NetworkBenchmarkDotNet/Libraries/Enet/EchoClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/Enet/EchoClient.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -8,7 +8,6 @@ // // -------------------------------------------------------------------------------------------------------------------- -using System; using System.Threading; using ENet; @@ -25,7 +24,6 @@ internal class EchoClient : AClient private readonly Configuration config; private readonly BenchmarkStatistics benchmarkStatistics; - private readonly byte[] message; private readonly int timeout; private readonly PacketFlags packetFlags; private readonly Host host; @@ -33,12 +31,11 @@ internal class EchoClient : AClient private readonly Thread listenThread; private Peer peer; - public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) + public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.id = id; this.config = config; this.benchmarkStatistics = benchmarkStatistics; - message = config.Message; timeout = Utilities.CalculateTimeout(this.config.ClientTickRate); packetFlags = ENetBenchmark.GetPacketFlags(config.Transmission); @@ -62,11 +59,9 @@ public override void StartClient() public override void StartBenchmark() { base.StartBenchmark(); - var parallelMessagesPerClient = config.ParallelMessages; - - for (int i = 0; i < parallelMessagesPerClient; i++) + if (!ManualMode) { - Send(message, 0, peer); + SendMessages(config.ParallelMessages, config.Transmission); } } @@ -87,6 +82,22 @@ public override void Dispose() isDisposed = true; } + #region ManualMode + + public override void SendMessages(int messageCount, TransmissionType transmissionType) + { + var flags = ENetBenchmark.GetPacketFlags(transmissionType); + + // Don't do this in a real-world application, ENet is not thread safe + // send should only be called in the thread that also calls host.Service + for (int i = 0; i < messageCount; i++) + { + Send(Message, 0, peer, flags); + } + } + + #endregion + private void ListenLoop() { host.Create(); @@ -127,6 +138,7 @@ private void HandleNetEvent(Event netEvent) { Utilities.WriteVerboseLine($"Client {id} timed out while benchmark is running."); } + break; case EventType.Disconnect: @@ -134,13 +146,17 @@ private void HandleNetEvent(Event netEvent) { Utilities.WriteVerboseLine($"Client {id} disconnected while benchmark is running."); } + break; case EventType.Receive: if (BenchmarkRunning) { Interlocked.Increment(ref benchmarkStatistics.MessagesClientReceived); - OnReceiveMessage(netEvent); + if (!ManualMode) + { + OnReceiveMessage(netEvent); + } } netEvent.Packet.Dispose(); @@ -150,15 +166,15 @@ private void HandleNetEvent(Event netEvent) private void OnReceiveMessage(Event netEvent) { - netEvent.Packet.CopyTo(message); - Send(message, 0, peer); + netEvent.Packet.CopyTo(Message); + Send(Message, 0, peer, packetFlags); } - private void Send(byte[] data, byte channelID, Peer peer) + private void Send(byte[] data, byte channelID, Peer peer, PacketFlags flags) { Packet packet = default(Packet); - packet.Create(data, data.Length, packetFlags); + packet.Create(data, data.Length, flags); peer.Send(channelID, ref packet); Interlocked.Increment(ref benchmarkStatistics.MessagesClientSent); } diff --git a/NetworkBenchmarkDotNet/Libraries/Enet/EchoServer.cs b/NetworkBenchmarkDotNet/Libraries/Enet/EchoServer.cs index 2e2890c..9f47041 100644 --- a/NetworkBenchmarkDotNet/Libraries/Enet/EchoServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/Enet/EchoServer.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -8,7 +8,6 @@ // // -------------------------------------------------------------------------------------------------------------------- -using System; using System.Threading; using ENet; @@ -23,11 +22,10 @@ internal class EchoServer : AServer private readonly Thread serverThread; private readonly Host host; private readonly Address address; - private readonly byte[] message; private readonly int timeout; private readonly PacketFlags packetFlags; - public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) + public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.config = config; this.benchmarkStatistics = benchmarkStatistics; @@ -36,7 +34,6 @@ public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) host = new Host(); address = new Address(); - message = new byte[config.MessageByteSize]; address.Port = (ushort) config.Port; address.SetHost(config.Address); @@ -88,7 +85,10 @@ private void HandleNetEvent(Event netEvent) if (benchmarkRunning) { Interlocked.Increment(ref benchmarkStatistics.MessagesServerReceived); - OnReceiveMessage(netEvent); + if (!ManualMode) + { + OnReceiveMessage(netEvent); + } } netEvent.Packet.Dispose(); @@ -106,8 +106,8 @@ private void HandleNetEvent(Event netEvent) private void OnReceiveMessage(Event netEvent) { - netEvent.Packet.CopyTo(message); - Send(message, 0, netEvent.Peer); + netEvent.Packet.CopyTo(MessageBuffer); + Send(MessageBuffer, 0, netEvent.Peer); } public override void Dispose() @@ -116,6 +116,20 @@ public override void Dispose() host.Dispose(); } + #region ManualMode + + public override void SendMessages(int messageCount, TransmissionType transmissionType) + { + // Don't do this in a real-world application, ENet is not thread safe + // send should only be called in the thread that also calls host.Service + for (int i = 0; i < messageCount; i++) + { + Broadcast(MessageBuffer, 0, transmissionType); + } + } + + #endregion + private void Send(byte[] data, byte channelId, Peer peer) { Packet packet = default(Packet); @@ -124,5 +138,16 @@ private void Send(byte[] data, byte channelId, Peer peer) peer.Send(channelId, ref packet); Interlocked.Increment(ref benchmarkStatistics.MessagesServerSent); } + + private void Broadcast(byte[] data, byte channelId, TransmissionType transmissionType) + { + Packet packet = default(Packet); + var flags = ENetBenchmark.GetPacketFlags(transmissionType); + + packet.Create(data, data.Length, flags); + host.Broadcast(channelId, ref packet); + var messagesSent = host.PeersCount; + Interlocked.Add(ref benchmarkStatistics.MessagesServerSent, messagesSent); + } } } diff --git a/NetworkBenchmarkDotNet/Libraries/IClient.cs b/NetworkBenchmarkDotNet/Libraries/IClient.cs index 98b2896..6acadfc 100644 --- a/NetworkBenchmarkDotNet/Libraries/IClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/IClient.cs @@ -58,5 +58,11 @@ public interface IClient : IDisposable /// Stop the client and its listening activity /// public void StopClient(); + + #region ManualMode + + public void SendMessages(int messageCount, TransmissionType transmissionType); + + #endregion } } diff --git a/NetworkBenchmarkDotNet/Libraries/INetworkBenchmark.cs b/NetworkBenchmarkDotNet/Libraries/INetworkBenchmark.cs index 5a5f90a..b6a2954 100644 --- a/NetworkBenchmarkDotNet/Libraries/INetworkBenchmark.cs +++ b/NetworkBenchmarkDotNet/Libraries/INetworkBenchmark.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -9,6 +9,7 @@ // -------------------------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Threading.Tasks; using NetworkBenchmark.Enet; using NetworkBenchmark.Kcp2k; @@ -19,6 +20,9 @@ namespace NetworkBenchmark { public interface INetworkBenchmark { + public IServer Server { get; } + public List Clients { get; } + public static INetworkBenchmark CreateNetworkBenchmark(NetworkLibrary library) { switch (library) diff --git a/NetworkBenchmarkDotNet/Libraries/IServer.cs b/NetworkBenchmarkDotNet/Libraries/IServer.cs index e32f8f7..1b52c72 100644 --- a/NetworkBenchmarkDotNet/Libraries/IServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/IServer.cs @@ -39,5 +39,11 @@ public interface IServer : IDisposable /// Stop the server and its listening activity /// public void StopServer(); + + #region ManualMode + + public void SendMessages(int messageCount, TransmissionType transmissionType); + + #endregion } } diff --git a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs index 1fab36b..25bc348 100644 --- a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoClient.cs @@ -27,17 +27,15 @@ internal class EchoClient : AClient private readonly BenchmarkStatistics benchmarkStatistics; private readonly Thread tickThread; - private readonly byte[] messageArray; private readonly KcpClientConnection client; private readonly KcpChannel communicationChannel; private readonly bool noDelay; - public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) + public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.id = id; this.config = config; this.benchmarkStatistics = benchmarkStatistics; - messageArray = config.Message; noDelay = true; communicationChannel = Kcp2kBenchmark.GetChannel(config.Transmission); @@ -83,14 +81,10 @@ private void Tick() public override void StartBenchmark() { base.StartBenchmark(); - var parallelMessagesPerClient = config.ParallelMessages; - - for (int i = 0; i < parallelMessagesPerClient; i++) + if (!ManualMode) { - Send(messageArray, communicationChannel); + SendMessages(config.ParallelMessages, config.Transmission); } - - Tick(); } public override void DisconnectClient() @@ -109,14 +103,29 @@ public override void Dispose() isDisposed = true; } - private void Send(ArraySegment message, KcpChannel channel) + #region ManualMode + + public override void SendMessages(int messageCount, TransmissionType transmissionType) + { + var channel = Kcp2kBenchmark.GetChannel(transmissionType); + + for (int i = 0; i < messageCount; i++) + { + Send(Message, channel); + } + Tick(); + } + + #endregion + + private void Send(ArraySegment buffer, KcpChannel channel) { if (!IsConnected) { return; } - client.SendData(message, channel); + client.SendData(buffer, channel); Interlocked.Increment(ref benchmarkStatistics.MessagesClientSent); } @@ -131,7 +140,10 @@ private void OnNetworkReceive(ArraySegment arraySegment) if (BenchmarkRunning) { Interlocked.Increment(ref benchmarkStatistics.MessagesClientReceived); - Send(messageArray, communicationChannel); + if (!ManualMode) + { + Send(Message, communicationChannel); + } } } diff --git a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs index 58aa0fb..37ee0f8 100644 --- a/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/Kcp2k/EchoServer.cs @@ -25,9 +25,7 @@ internal class EchoServer : AServer private readonly KcpChannel communicationChannel; private readonly bool noDelay; - private readonly byte[] message; - - public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) + public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.config = config; this.benchmarkStatistics = benchmarkStatistics; @@ -38,9 +36,6 @@ public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) var interval = (uint) Utilities.CalculateTimeout(config.ServerTickRate); server = new KcpServer(OnConnected, OnReceiveMessage, OnDisconnected, noDelay, interval); - - message = new byte[config.MessageByteSize]; - serverThread = new Thread(TickLoop); serverThread.Name = "Kcp2k Server"; serverThread.Priority = ThreadPriority.AboveNormal; @@ -52,6 +47,25 @@ public override void StartServer() serverThread.Start(); } + public override void Dispose() + { + // Server already stopped, maybe there is the need to dispose something else? + } + + #region ManualMode + + public override void SendMessages(int messageCount, TransmissionType transmissionType) + { + var channel = Kcp2kBenchmark.GetChannel(transmissionType); + + for (int i = 0; i < messageCount; i++) + { + Broadcast(MessageBuffer, channel); + } + } + + #endregion + private void TickLoop() { server.Start((ushort) config.Port); @@ -78,8 +92,11 @@ private void OnReceiveMessage(int connectionId, ArraySegment arraySegment) if (benchmarkRunning) { Interlocked.Increment(ref benchmarkStatistics.MessagesServerReceived); - Array.Copy(arraySegment.Array, arraySegment.Offset, message, 0, arraySegment.Count); - Send(connectionId, message, communicationChannel); + if (!ManualMode) + { + Array.Copy(arraySegment.Array, arraySegment.Offset, MessageBuffer, 0, arraySegment.Count); + Send(connectionId, MessageBuffer, communicationChannel); + } } } @@ -97,9 +114,15 @@ private void Send(int connectionId, ArraySegment message, KcpChannel chann Interlocked.Increment(ref benchmarkStatistics.MessagesServerSent); } - public override void Dispose() + private void Broadcast(ArraySegment message, KcpChannel channel) { - // Server already stopped, maybe there is the need to dispose something else? + foreach (var connection in server.connections.Values) + { + connection.SendData(message, channel); + } + + var messagesSent = server.connections.Count; + Interlocked.Add(ref benchmarkStatistics.MessagesServerSent, messagesSent); } } } diff --git a/NetworkBenchmarkDotNet/Libraries/Kcp2k/Kcp2kBenchmark.cs b/NetworkBenchmarkDotNet/Libraries/Kcp2k/Kcp2kBenchmark.cs index db6cb1a..8bb09d9 100644 --- a/NetworkBenchmarkDotNet/Libraries/Kcp2k/Kcp2kBenchmark.cs +++ b/NetworkBenchmarkDotNet/Libraries/Kcp2k/Kcp2kBenchmark.cs @@ -32,7 +32,7 @@ public static KcpChannel GetChannel(TransmissionType transmissionType) case TransmissionType.Reliable: return KcpChannel.Reliable; case TransmissionType.Unreliable: - return KcpChannel.Reliable; + return KcpChannel.Unreliable; default: throw new ArgumentOutOfRangeException(nameof(transmissionType), $"Transmission Type {transmissionType} not supported"); } diff --git a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs index ff36ded..ef1ab1f 100644 --- a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoClient.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -8,7 +8,6 @@ // // -------------------------------------------------------------------------------------------------------------------- -using System; using System.Net; using System.Net.Sockets; using System.Threading; @@ -28,18 +27,16 @@ internal class EchoClient : AClient, IClient private readonly Configuration config; private readonly BenchmarkStatistics benchmarkStatistics; - private readonly byte[] message; private readonly EventBasedNetListener listener; private readonly NetManager netManager; private readonly DeliveryMethod deliveryMethod; private NetPeer peer; - public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) + public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.id = id; this.config = config; this.benchmarkStatistics = benchmarkStatistics; - message = config.Message; deliveryMethod = LiteNetLibBenchmark.GetDeliveryMethod(config.Transmission); listener = new EventBasedNetListener(); @@ -77,14 +74,10 @@ public override void StartClient() public override void StartBenchmark() { base.StartBenchmark(); - var parallelMessagesPerClient = config.ParallelMessages; - - for (int i = 0; i < parallelMessagesPerClient; i++) + if (!ManualMode) { - Send(message); + SendMessages(config.ParallelMessages, config.Transmission); } - - netManager.TriggerUpdate(); } public override void DisconnectClient() @@ -116,14 +109,30 @@ public override void Dispose() isDisposed = true; } - private void Send(byte[] bytes) + #region ManualMode + + public override void SendMessages(int messageCount, TransmissionType transmissionType) + { + var delivery = LiteNetLibBenchmark.GetDeliveryMethod(transmissionType); + + for (int i = 0; i < messageCount; i++) + { + Send(Message, delivery); + } + + netManager.TriggerUpdate(); + } + + #endregion + + private void Send(byte[] bytes, DeliveryMethod delivery) { if (!IsConnected) { return; } - peer.Send(bytes, deliveryMethod); + peer.Send(bytes, delivery); Interlocked.Increment(ref benchmarkStatistics.MessagesClientSent); } @@ -149,8 +158,11 @@ private void OnNetworkReceive(NetPeer peer, NetPacketReader reader, DeliveryMeth if (BenchmarkRunning) { Interlocked.Increment(ref benchmarkStatistics.MessagesClientReceived); - Send(message); - netManager.TriggerUpdate(); + if (!ManualMode) + { + Send(Message, deliverymethod); + netManager.TriggerUpdate(); + } } reader.Recycle(); diff --git a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs index 6f387cf..49a514e 100644 --- a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/EchoServer.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -24,10 +24,9 @@ internal class EchoServer : AServer private readonly BenchmarkStatistics benchmarkStatistics; private readonly EventBasedNetListener listener; private readonly NetManager netManager; - private readonly byte[] message; private readonly DeliveryMethod deliveryMethod; - public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) + public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config) { this.config = config; this.benchmarkStatistics = benchmarkStatistics; @@ -47,8 +46,6 @@ public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) netManager.UnsyncedEvents = true; - message = new byte[config.MessageByteSize]; - listener.ConnectionRequestEvent += OnConnectionRequest; listener.NetworkReceiveEvent += OnNetworkReceive; listener.NetworkErrorEvent += OnNetworkError; @@ -75,6 +72,21 @@ public override void Dispose() listener.PeerDisconnectedEvent -= OnPeerDisconnected; } + #region ManualMode + + public override void SendMessages(int messageCount, TransmissionType transmissionType) + { + var delivery = LiteNetLibBenchmark.GetDeliveryMethod(transmissionType); + + for (int i = 0; i < messageCount; i++) + { + Broadcast(MessageBuffer, delivery); + } + netManager.TriggerUpdate(); + } + + #endregion + private void OnConnectionRequest(ConnectionRequest request) { if (netManager.ConnectedPeerList.Count > config.Clients) @@ -89,19 +101,28 @@ private void OnConnectionRequest(ConnectionRequest request) private void OnNetworkReceive(NetPeer peer, NetPacketReader reader, DeliveryMethod clientDeliveryMethod) { - Interlocked.Increment(ref benchmarkStatistics.MessagesServerReceived); - if (benchmarkRunning) { - Buffer.BlockCopy(reader.RawData, reader.UserDataOffset, message, 0, reader.UserDataSize); - peer.Send(message, deliveryMethod); - Interlocked.Increment(ref benchmarkStatistics.MessagesServerSent); - netManager.TriggerUpdate(); + Interlocked.Increment(ref benchmarkStatistics.MessagesServerReceived); + if (!ManualMode) + { + Buffer.BlockCopy(reader.RawData, reader.UserDataOffset, MessageBuffer, 0, reader.UserDataSize); + peer.Send(MessageBuffer, deliveryMethod); + Interlocked.Increment(ref benchmarkStatistics.MessagesServerSent); + netManager.TriggerUpdate(); + } } reader.Recycle(); } + private void Broadcast(byte[] data, DeliveryMethod delivery) + { + netManager.SendToAll(data, delivery); + var messagesSent = netManager.ConnectedPeersCount; + Interlocked.Add(ref benchmarkStatistics.MessagesServerSent, messagesSent); + } + private void OnNetworkError(IPEndPoint endpoint, SocketError socketerror) { Interlocked.Increment(ref benchmarkStatistics.Errors); diff --git a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/LiteNetLibBenchmark.cs b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/LiteNetLibBenchmark.cs index 969df66..ef16082 100644 --- a/NetworkBenchmarkDotNet/Libraries/LiteNetLib/LiteNetLibBenchmark.cs +++ b/NetworkBenchmarkDotNet/Libraries/LiteNetLib/LiteNetLibBenchmark.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoClient.cs b/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoClient.cs index a2ac517..d333676 100644 --- a/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoClient.cs +++ b/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoClient.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -8,6 +8,7 @@ // // -------------------------------------------------------------------------------------------------------------------- +using System; using System.Net; using System.Net.Sockets; using System.Threading; @@ -24,17 +25,22 @@ internal class EchoClient : UdpClient, IClient private volatile bool benchmarkRunning; private readonly int id; - private readonly byte[] message; - private readonly int initialMessages; + private readonly Configuration config; private readonly BenchmarkStatistics benchmarkStatistics; + private readonly bool manualMode; + private readonly byte[] message; public EchoClient(int id, Configuration config, BenchmarkStatistics benchmarkStatistics) : base(config.Address, config.Port) { this.id = id; + this.config = config; NetCoreServerBenchmark.ProcessTransmissionType(config.Transmission); - message = config.Message; - initialMessages = config.ParallelMessages; + manualMode = config.Test == TestType.Manual; + // Use Pinned Object Heap to reduce GC pressure + message = GC.AllocateArray(config.MessageByteSize, true); + config.Message.CopyTo(message, 0); + this.benchmarkStatistics = benchmarkStatistics; } @@ -50,18 +56,12 @@ public void StartBenchmark() benchmarkPreparing = false; benchmarkRunning = true; - for (int i = 0; i < initialMessages; i++) + if (manualMode) { - SendMessage(); + SendMessages(config.ParallelMessages, config.Transmission); } } - private void SendMessage() - { - Send(message); - benchmarkStatistics.MessagesClientSent++; - } - public void StopBenchmark() { benchmarkRunning = false; @@ -77,6 +77,20 @@ public void DisconnectClient() Disconnect(); } + #region ManualMode + + public void SendMessages(int messageCount, TransmissionType transmissionType) + { + NetCoreServerBenchmark.ProcessTransmissionType(transmissionType); + + for (int i = 0; i < messageCount; i++) + { + SendMessage(); + } + } + + #endregion + protected override void OnConnected() { // Start receive datagrams @@ -98,8 +112,11 @@ protected override void OnReceived(EndPoint endpoint, byte[] buffer, long offset { if (benchmarkRunning) { - benchmarkStatistics.MessagesClientReceived++; - SendMessage(); + Interlocked.Increment(ref benchmarkStatistics.MessagesClientReceived); + if (!manualMode) + { + SendMessage(); + } } if (listen) @@ -118,5 +135,11 @@ protected override void OnError(SocketError error) Interlocked.Increment(ref benchmarkStatistics.Errors); } } + + private void SendMessage() + { + Send(message); + Interlocked.Increment(ref benchmarkStatistics.MessagesClientSent); + } } } diff --git a/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoServer.cs b/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoServer.cs index d1c1672..e33fb96 100644 --- a/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoServer.cs +++ b/NetworkBenchmarkDotNet/Libraries/NetCoreServer/EchoServer.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -8,6 +8,7 @@ // // -------------------------------------------------------------------------------------------------------------------- +using System; using System.Net; using System.Net.Sockets; using System.Threading; @@ -20,9 +21,16 @@ internal class EchoServer : UdpServer, IServer private volatile bool listen; private volatile bool benchmarkRunning; private readonly BenchmarkStatistics benchmarkStatistics; + private readonly bool ManualMode; + private readonly byte[] message; public EchoServer(Configuration config, BenchmarkStatistics benchmarkStatistics) : base(IPAddress.Parse(config.Address), config.Port) { + ManualMode = config.Test == TestType.Manual; + // Use Pinned Object Heap to reduce GC pressure + message = GC.AllocateArray(config.MessageByteSize, true); + config.Message.CopyTo(message, 0); + NetCoreServerBenchmark.ProcessTransmissionType(config.Transmission); this.benchmarkStatistics = benchmarkStatistics; } @@ -49,6 +57,20 @@ public void StopServer() listen = true; } + #region ManualMode + + public void SendMessages(int messageCount, TransmissionType transmissionType) + { + NetCoreServerBenchmark.ProcessTransmissionType(transmissionType); + + for (int i = 0; i < messageCount; i++) + { + Broadcast(message); + } + } + + #endregion + protected override void OnStarted() { // Start receive datagrams @@ -60,9 +82,12 @@ protected override void OnReceived(EndPoint endpoint, byte[] buffer, long offset if (benchmarkRunning) { benchmarkStatistics.MessagesServerReceived++; - // Echo the message back to the sender - SendAsync(endpoint, buffer, offset, size); - return; + if (!ManualMode) + { + // Echo the message back to the sender + SendAsync(endpoint, buffer, offset, size); + return; + } } // Keep listening for next possible benchmark @@ -92,5 +117,10 @@ protected override void OnError(SocketError error) benchmarkStatistics.Errors++; } } + + private void Broadcast(byte[] bytes) + { + throw new NotImplementedException(); + } } } diff --git a/NetworkBenchmarkDotNet/Libraries/NetCoreServer/NetCoreServerBenchmark.cs b/NetworkBenchmarkDotNet/Libraries/NetCoreServer/NetCoreServerBenchmark.cs index 33d6e8b..41da7e7 100644 --- a/NetworkBenchmarkDotNet/Libraries/NetCoreServer/NetCoreServerBenchmark.cs +++ b/NetworkBenchmarkDotNet/Libraries/NetCoreServer/NetCoreServerBenchmark.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/Libraries/NetworkLibrary.cs b/NetworkBenchmarkDotNet/Libraries/NetworkLibrary.cs index d491178..bc4df3c 100644 --- a/NetworkBenchmarkDotNet/Libraries/NetworkLibrary.cs +++ b/NetworkBenchmarkDotNet/Libraries/NetworkLibrary.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj index febde5c..1835058 100644 --- a/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj +++ b/NetworkBenchmarkDotNet/NetworkBenchmarkDotNet.csproj @@ -7,7 +7,7 @@ Johannes Deml NetworkingBenchmark 1.0.0 - 0.9.0 + 1.0.0 en-US Debug;Release AnyCPU @@ -16,7 +16,7 @@ true true true - 0.9.0 + 1.0.0 @@ -61,7 +61,7 @@ true NetworkBenchmark.Program 0.5.0 - NCNB is a benchmark for low level networking libraries using UDP and can be used with Unity and for .Net Core standalone server applications. The benchmark focuses on latency, performance and scalability. + NNB is a benchmark for low level networking libraries using UDP and can be used with Unity and for .Net 5 standalone server applications. The benchmark focuses on latency, performance and scalability. https://github.com/JohannesDeml/NetworkBenchmarkDotNet sockets, UDP, benchmark, network, Unity, network-benchmark LICENSE diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigConstants.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs similarity index 56% rename from NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigConstants.cs rename to NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs index 99774bf..2b2ff93 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigConstants.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/ConfigHelper.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -10,12 +10,15 @@ using System.Globalization; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Reports; using Perfolizer.Horology; namespace NetworkBenchmark { - public static class ConfigConstants + public static class ConfigHelper { /// /// A summary style that makes processing of data more accessible. @@ -25,5 +28,16 @@ public static class ConfigConstants /// public static readonly SummaryStyle CsvStyle = new SummaryStyle(CultureInfo.InvariantCulture, false, SizeUnit.KB, TimeUnit.Millisecond, false, true, 100); + + public static void AddDefaultColumns(ManualConfig config) + { + config.AddColumn(FixedColumn.VersionColumn); + config.AddColumn(FixedColumn.OperatingSystemColumn); + config.AddColumn(FixedColumn.DateTimeColumn); + config.AddColumn(new EnvironmentVariableColumn("SystemTag", "SYSTEM_TAG")); + + config.AddExporter(MarkdownExporter.GitHub); + config.AddExporter(new CsvExporter(CsvSeparator.Comma, ConfigHelper.CsvStyle)); + } } } diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/EnvironmentVariableColumn.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/EnvironmentVariableColumn.cs new file mode 100644 index 0000000..9d10713 --- /dev/null +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/EnvironmentVariableColumn.cs @@ -0,0 +1,63 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. +// +// +// Johannes Deml +// public@deml.io +// +// -------------------------------------------------------------------------------------------------------------------- + +using System; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; + +namespace NetworkBenchmark +{ + /// + /// Static column presenting the value of an environment variable of the system. + /// Helpful for tagging machines with specific names. + /// + public class EnvironmentVariableColumn : IColumn + { + public string Id { get; } + public string ColumnName { get; } + + private readonly string environmentVariableKey; + private readonly string cellValue; + + public EnvironmentVariableColumn(string columnName, string environmentVariableKey, string defaultValue = "Not set") + { + this.environmentVariableKey = environmentVariableKey; + ColumnName = columnName; + this.cellValue = Environment.GetEnvironmentVariable(environmentVariableKey, EnvironmentVariableTarget.Process); + + // Fallbacks for windows + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + this.cellValue ??= Environment.GetEnvironmentVariable(environmentVariableKey, EnvironmentVariableTarget.User); + this.cellValue ??= Environment.GetEnvironmentVariable(environmentVariableKey, EnvironmentVariableTarget.Machine); + } + + this.cellValue ??= defaultValue; + + Id = nameof(EnvironmentVariableColumn) + "." + ColumnName; + } + + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => cellValue; + + public bool IsAvailable(Summary summary) => true; + + public bool AlwaysShow { get; set; } = false; + public ColumnCategory Category => ColumnCategory.Custom; + public int PriorityInCategory { get; set; } = 0; + public bool IsNumeric => false; + public UnitType UnitType => UnitType.Dimensionless; + public string Legend => $"Environment variable {environmentVariableKey} has value '{cellValue}'"; + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); + public override string ToString() => ColumnName; + } +} diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/FixedColumn.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/FixedColumn.cs index 9d07db2..00f3e55 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/FixedColumn.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/FixedColumn.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/GarbageBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/GarbageBenchmarkConfig.cs deleted file mode 100644 index 5a1e06a..0000000 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/GarbageBenchmarkConfig.cs +++ /dev/null @@ -1,53 +0,0 @@ -// -------------------------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2020 Johannes Deml. All rights reserved. -// -// -// Johannes Deml -// public@deml.io -// -// -------------------------------------------------------------------------------------------------------------------- - -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Exporters.Csv; -using BenchmarkDotNet.Jobs; - -namespace NetworkBenchmark -{ - public class GarbageBenchmarkConfig : ManualConfig - { - public GarbageBenchmarkConfig() - { - Add(DefaultConfig.Instance); - - Job baseJob = Job.Default - .WithLaunchCount(1) - .WithWarmupCount(1) - .WithIterationCount(10) - .WithGcServer(true) - .WithGcConcurrent(true) - .WithGcForce(true); - - AddJob(baseJob - .WithRuntime(CoreRuntime.Core50) - .WithPlatform(Platform.X64)); - - AddJob(baseJob - .WithRuntime(CoreRuntime.Core31) - .WithPlatform(Platform.X64)); - - AddColumn(FixedColumn.VersionColumn); - AddColumn(FixedColumn.OperatingSystemColumn); - AddColumn(FixedColumn.DateTimeColumn); - - AddExporter(MarkdownExporter.GitHub); - AddExporter(new CsvExporter(CsvSeparator.Comma, ConfigConstants.CsvStyle)); - - AddDiagnoser(MemoryDiagnoser.Default); - AddDiagnoser(new EventPipeProfiler(EventPipeProfile.GcVerbose)); - } - } -} diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/MessagesPerSecondColumn.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/MessagesPerSecondColumn.cs index 5c9481f..bf14643 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/MessagesPerSecondColumn.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/MessagesPerSecondColumn.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/NumClientsColumn.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/NumClientsColumn.cs index 897e919..973c269 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/NumClientsColumn.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/NumClientsColumn.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs index 614a00f..bbb2dc4 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/PerformanceBenchmarkConfig.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -10,8 +10,6 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Jobs; namespace NetworkBenchmark @@ -22,24 +20,20 @@ public PerformanceBenchmarkConfig() { Add(DefaultConfig.Instance); - AddJob(Job.Default + Job baseJob = Job.Default .WithLaunchCount(1) .WithWarmupCount(1) .WithIterationCount(10) .WithGcServer(true) .WithGcConcurrent(true) .WithGcForce(true) - .WithRuntime(CoreRuntime.Core50) - .WithPlatform(Platform.X64)); + .WithPlatform(Platform.X64); + AddJob(baseJob.WithRuntime(CoreRuntime.Core50)); + + ConfigHelper.AddDefaultColumns(this); AddColumn(new NumClientsColumn()); AddColumn(new MessagesPerSecondColumn()); - AddColumn(FixedColumn.VersionColumn); - AddColumn(FixedColumn.OperatingSystemColumn); - AddColumn(FixedColumn.DateTimeColumn); - - AddExporter(MarkdownExporter.GitHub); - AddExporter(new CsvExporter(CsvSeparator.Comma, ConfigConstants.CsvStyle)); } } } diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs index fccba4d..4e1fd37 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/QuickBenchmarkConfig.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml @@ -9,11 +9,8 @@ // -------------------------------------------------------------------------------------------------------------------- using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Jobs; namespace NetworkBenchmark @@ -31,20 +28,15 @@ public QuickBenchmarkConfig() .WithIterationCount(5) .WithGcServer(true) .WithGcConcurrent(true) - .WithGcForce(true); + .WithGcForce(true) + .WithPlatform(Platform.X64); // Here you can test different runtimes - AddJob(baseJob - .WithRuntime(CoreRuntime.Core50) - .WithPlatform(Platform.X64)); + AddJob(baseJob.WithRuntime(CoreRuntime.Core50)); + ConfigHelper.AddDefaultColumns(this); + AddColumn(new NumClientsColumn()); AddColumn(new MessagesPerSecondColumn()); - AddColumn(FixedColumn.VersionColumn); - AddColumn(FixedColumn.OperatingSystemColumn); - AddColumn(FixedColumn.DateTimeColumn); - - AddExporter(MarkdownExporter.GitHub); - AddExporter(new CsvExporter(CsvSeparator.Comma, ConfigConstants.CsvStyle)); // You can also use additional diagnosers. // Those might result in large trace files and can take some time to process after the benchmark finished diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs new file mode 100644 index 0000000..473bf8f --- /dev/null +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/Config/SamplingBenchmarkConfig.cs @@ -0,0 +1,55 @@ +// -------------------------------------------------------------------------------------------------------------------- +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. +// +// +// Johannes Deml +// public@deml.io +// +// -------------------------------------------------------------------------------------------------------------------- + +using System.Diagnostics.Tracing; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using Microsoft.Diagnostics.NETCore.Client; +using Microsoft.Diagnostics.Tracing.Parsers; + +namespace NetworkBenchmark +{ + public class SamplingBenchmarkConfig : ManualConfig + { + public SamplingBenchmarkConfig() + { + Add(DefaultConfig.Instance); + + Job baseJob = Job.Default + .WithLaunchCount(1) + .WithWarmupCount(1) + .WithIterationCount(1) + .WithGcServer(true) + .WithGcConcurrent(true) + .WithGcForce(true) + .WithPlatform(Platform.X64); + + AddJob(baseJob.WithRuntime(CoreRuntime.Core50)); + + ConfigHelper.AddDefaultColumns(this); + + var providers = new[] + { + new EventPipeProvider( + name: ClrTraceEventParser.ProviderName, + eventLevel: EventLevel.Verbose, + keywords: (long) ClrTraceEventParser.Keywords.Default | + (long) ClrTraceEventParser.Keywords.GC | + (long) ClrTraceEventParser.Keywords.GCHandle | + (long) ClrTraceEventParser.Keywords.Exception + ), + }; + + AddDiagnoser(new EventPipeProfiler(providers: providers, performExtraBenchmarksRun: false)); + } + } +} diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/ReliablePerformanceBenchmark.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/ReliablePerformanceBenchmark.cs index 135c4d6..5041db6 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/ReliablePerformanceBenchmark.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/ReliablePerformanceBenchmark.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/SamplingBenchmark.cs similarity index 68% rename from NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs rename to NetworkBenchmarkDotNet/PredefinedBenchmarks/SamplingBenchmark.cs index 80c5165..6062acd 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/SamplingBenchmark.cs @@ -1,5 +1,5 @@ // -------------------------------------------------------------------------------------------------------------------- -// +// // Copyright (c) 2020 Johannes Deml. All rights reserved. // // @@ -12,19 +12,19 @@ namespace NetworkBenchmark { - [Config(typeof(GarbageBenchmarkConfig))] - public class GarbageBenchmark : APredefinedBenchmark + [Config(typeof(SamplingBenchmarkConfig))] + public class SamplingBenchmark : APredefinedBenchmark { [Params(NetworkLibrary.ENet, NetworkLibrary.LiteNetLib, NetworkLibrary.NetCoreServer)] public NetworkLibrary Library { get; set; } - public override int ClientCount { get; set; } = 10; - public override int MessageTarget { get; set; } = 10_000; - protected override BenchmarkMode Mode => BenchmarkMode.Garbage; + public override int ClientCount { get; set; } = 1; + public override int MessageTarget { get; set; } = 100_000; + protected override BenchmarkMode Mode => BenchmarkMode.Sampling; protected override NetworkLibrary LibraryTarget => Library; - [GlobalSetup(Target = nameof(Garbage))] - public void PrepareGarbageBenchmark() + [GlobalSetup(Target = nameof(SampleSimpleEcho))] + public void PrepareSamplingBenchmark() { BenchmarkCoordinator.ApplyPredefinedConfiguration(); var config = BenchmarkCoordinator.Config; @@ -35,7 +35,7 @@ public void PrepareGarbageBenchmark() } [Benchmark] - public long Garbage() + public long SampleSimpleEcho() { return RunBenchmark(); } diff --git a/NetworkBenchmarkDotNet/PredefinedBenchmarks/UnreliablePerformanceBenchmark.cs b/NetworkBenchmarkDotNet/PredefinedBenchmarks/UnreliablePerformanceBenchmark.cs index f3b184c..023f1d1 100644 --- a/NetworkBenchmarkDotNet/PredefinedBenchmarks/UnreliablePerformanceBenchmark.cs +++ b/NetworkBenchmarkDotNet/PredefinedBenchmarks/UnreliablePerformanceBenchmark.cs @@ -1,5 +1,5 @@ // -------------------------------------------------------------------------------------------------------------------- -// +// // Copyright (c) 2020 Johannes Deml. All rights reserved. // // diff --git a/NetworkBenchmarkDotNet/Program.cs b/NetworkBenchmarkDotNet/Program.cs index 3079d34..c60995f 100644 --- a/NetworkBenchmarkDotNet/Program.cs +++ b/NetworkBenchmarkDotNet/Program.cs @@ -58,10 +58,10 @@ private static void RunPredefinedBenchmarks(BenchmarkMode mode) Console.WriteLine($"Finished {BenchmarkMode.Quick} Benchmark"); } - if ((mode & BenchmarkMode.Garbage) != 0) + if ((mode & BenchmarkMode.Sampling) != 0) { - RunBenchmark(); - Console.WriteLine($"Finished {BenchmarkMode.Garbage} Benchmark"); + RunBenchmark(); + Console.WriteLine($"Finished {BenchmarkMode.Sampling} Benchmark"); } } @@ -72,14 +72,7 @@ private static void RunCustomBenchmark() try { BenchmarkCoordinator.PrepareBenchmark(networkBenchmark); - if (BenchmarkCoordinator.Config.Duration < 0) - { - BenchmarkCoordinator.RunIndefinitely(networkBenchmark); - } - else - { - BenchmarkCoordinator.RunTimedBenchmark(networkBenchmark); - } + BenchmarkCoordinator.RunBenchmark(networkBenchmark); } catch (Exception e) { diff --git a/NetworkBenchmarkDotNet/Utils/TimeUtilities.cs b/NetworkBenchmarkDotNet/Utils/TimeUtilities.cs index 2a797fd..f2bebb9 100644 --- a/NetworkBenchmarkDotNet/Utils/TimeUtilities.cs +++ b/NetworkBenchmarkDotNet/Utils/TimeUtilities.cs @@ -19,7 +19,7 @@ public static class TimeUtilities #if WINDOWS // See https://github.com/Leandros/WindowsHModular/blob/7f5df60fe3711b9a878cd3cba755f9f71b5d01ca/include/win32/windows.h#L2655 private const uint TIMERR_NOERROR = 0u; - + /// Get higher precision for Thread.Sleep on Windows /// See https://web.archive.org/web/20051125042113/http://www.dotnet247.com/247reference/msgs/57/289291.aspx /// See https://docs.microsoft.com/en-us/windows/win32/api/timeapi/nf-timeapi-timebeginperiod diff --git a/NetworkBenchmarkDotNet/Utils/Utilities.cs b/NetworkBenchmarkDotNet/Utils/Utilities.cs index 1ba7ca4..7c6ee94 100644 --- a/NetworkBenchmarkDotNet/Utils/Utilities.cs +++ b/NetworkBenchmarkDotNet/Utils/Utilities.cs @@ -1,6 +1,6 @@ // -------------------------------------------------------------------------------------------------------------------- // -// Copyright (c) 2020 Johannes Deml. All rights reserved. +// Copyright (c) 2021 Johannes Deml. All rights reserved. // // // Johannes Deml diff --git a/README.md b/README.md index 17fa415..35e1a25 100644 --- a/README.md +++ b/README.md @@ -60,10 +60,10 @@ This test is for multiplexing / message merging performance. Runs the benchmark with **500** clients, which pingpong **1 message** each with the server with **reliable** transmission. The benchmark runs until a total of **500,000** messages are sent to the server and back to the clients. Message size is **32 bytes**. This test is for getting an idea of an average roundtrip time. -### Benchmark [Garbage](./NetworkBenchmarkDotNet/PredefinedBenchmarks/GarbageBenchmark.cs) +### Benchmark [SampleEchoSimple](./NetworkBenchmarkDotNet/PredefinedBenchmarks/SamplingBenchmark.cs) -Runs the benchmark with **10** clients, which pingpong **10 messages** each with the server. The benchmark runs until a total of **10,000** messages are sent to the server and back to the clients. Message size is **128 bytes**. -This test collects information about generated garbage while running the benchmark. +Runs the benchmark with **1** client, which pingpong **10 messages** each with the server. The benchmark runs until a total of **100,000** messages are sent to the server and back to the clients. Message size is **128 bytes**. +This test collects information about generated garbage and CPU times while running the benchmark. Those results can be analyzed with [PerfView](https://github.com/microsoft/perfview) on Windows. ## Benchmark Results @@ -131,9 +131,9 @@ DateTime=02/18/2021 16:18:02 ### Notes * The tests perform very different on Linux compared to Windows 10, since there are a lot of client threads involved and Linux seems to handle them a lot better. -* Creation, Connection and Disconnection and Disposal of the Server and Clients is not included in the performance benchmarks, but is included in the Garbage benchmark. +* Creation, Connection and Disconnection and Disposal of the Server and Clients is not included in the performance benchmarks, but is included in the .nettrace files from the Sampling benchmark. * Since the clients and the server run on the same machine, there is a lot less network latency as in a real world application. On the other hand, the CPU pressure is a lot higher than for a normal server, since all the clients get there own threads and run on the same machine. Take the results with a grain of salt. -* To access the Garbage results, you can use [PerfView](https://github.com/microsoft/perfview) to open the `.nettrace` files. +* To access the Sampling results, you can use [PerfView](https://github.com/microsoft/perfview) to open the `.nettrace` files. * Kcp2k has been recently added and might have some room for improvements. Especially using `Thread.Sleep` on Windows creates [noticeable delays](https://social.msdn.microsoft.com/Forums/vstudio/en-US/facc2b57-9a27-4049-bb32-ef093fbf4c29/threadsleep1-sleeps-for-156-ms?forum=clr). For now it is excluded of the predefined benchmarks, until its execution and cleanup are improved. @@ -157,23 +157,24 @@ Usage: NetworkBenchmarkDotNet [options] Options: - -b, --benchmark Run predefined benchmarks [default: Custom] - -m, --execution-mode Control what parts to run [default: Complete] - -t, --test Test type [default: PingPong] - --transmission Transmission type [default: Unreliable] - -l, --library Library target [default: ENet] - -d, --duration Test duration in seconds (-1 for manual stopping) [default: 10] - --address
IP Address, can be ipv4 (e.g. 127.0.0.1) or ipv6 (e.g. ::1) [default: ::1] - --port Socket Port [default: 3330] - --clients # Simultaneous clients [default: 500] - --parallel-messages # Parallel messages per client [default: 1] - --message-byte-size Message byte size sent by clients [default: 32] - --message-payload Message load sent by clients [default: Random] - --verbose Verbose output of test steps and errors [default: True] - --client-tick-rate Client ticks per second if supported [default: 60] - --server-tick-rate Server ticks per second if supported [default: 60] - --version Show version information - -?, -h, --help Show help and usage information + -b, --benchmark Run predefined benchmarks [default: Custom] + -m, --execution-mode Control what parts to run [default: Complete] + -t, --test Test type [default: PingPong] + --transmission Transmission type [default: Unreliable] + -l, --library Library target [default: ENet] + -d, --duration Test duration in seconds (-1 for manual stopping) [default: 10] + --address
IP Address, can be ipv4 (e.g. 127.0.0.1) or ipv6 (e.g. ::1) [default: ::1] + --port Socket Port [default: 3330] + --clients # Simultaneous clients [default: 500] + --parallel-messages # Parallel messages per client [default: 1] + --message-byte-size Message byte size sent by clients [default: 32] + --message-payload Message load sent by clients [default: Random] + --verbose Verbose output of test steps and errors [default: True] + --client-tick-rate Client ticks per second if supported [default: 60] + --server-tick-rate Server ticks per second if supported [default: 60] + --version Show version information + -?, -h, --help Show help and usage information + ``` ### Predefined Benchmarks @@ -184,7 +185,7 @@ Predefined benchmarks take some time to run, but generate reproducible numbers. * **Quick** (<1min): Runs a quick benchmark with whatever is set in [QuickBenchmark.cs](../../blob/master/NetworkBenchmarkDotNet/PredefinedBenchmarks/QuickBenchmark.cs) * **Performance** (>15min): High Performance statistical test with all included libraries -* **Garbage** (<1min): Test with all included libraries using cpu sampling and memory allocation statistics +* **Sampling** (<1min): Test with all included libraries using cpu sampling and memory allocation statistics * **Essential** (>15min): Running Performance + Garbage Benchmark ![Run Predefined Benchmark windows command-line screenshot](./Docs/run-predefined-benchmark.png) diff --git a/linux-benchmark.sh b/linux-benchmark.sh index f5a523a..812bbe0 100644 --- a/linux-benchmark.sh +++ b/linux-benchmark.sh @@ -1,5 +1,7 @@ #!/bin/bash +# build and run benchmark for linux + # Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build # Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog dotnet build --configuration Release --framework net5.0 --output ./bin/NetworkBenchmarkDotNet-Linux/ @@ -12,3 +14,11 @@ else fi ./bin/NetworkBenchmarkDotNet-Linux/NetworkBenchmarkDotNet -b "$mode" + +echo "--- Benchmark finished ---" +echo "Save current process list" +# Folder should exist, just to be sure create it if it does not +mkdir -p BenchmarkDotNet.Artifacts + +ps -aux > ./BenchmarkDotNet.Artifacts/running-processes.txt +ps -e -o %p, -o lstart -o ,%C, -o %mem -o ,%c > ./BenchmarkDotNet.Artifacts/running-processes.csv \ No newline at end of file diff --git a/win-benchmark.bat b/win-benchmark.bat index 8b648b4..1792040 100644 --- a/win-benchmark.bat +++ b/win-benchmark.bat @@ -1,21 +1,33 @@ :: build and run benchmark for windows -:: Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build -:: Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog echo off Echo --- NBN Predefined Benchmark runner --- Echo. -Echo Benchmark types: [Quick/Performance/Garbage/Essential] +Echo Benchmark types: [Quick/Performance/Sampling/Essential] Echo * Quick (^<1min): Runs a quick benchmark with whatever is set in QuickBenchmark.cs Echo * Performance (^>15min): High Performance statistical test with all included libraries -Echo * Garbage (^<1min): Test with all included libraries using cpu sampling and memory allocation statistics -Echo * Essential (^>15min): Running Performance + Garbage Benchmark +Echo * Sampling (^<1min): Test with all included libraries using cpu sampling and memory allocation statistics +Echo * Essential (^>15min): Running Performance + Sampling Benchmark Echo * Custom: Use the commandline with .\NetworkBenchmarkDotNet --help to see how to use it Echo. set benchmark=Essential -set /p benchmark=Which benchmark do you want to run [Quick/Performance/Garbage/Essential] (default - %benchmark%)?: +set /p benchmark=Which benchmark do you want to run [Quick/Performance/Sampling/Essential] (default - %benchmark%)?: echo on +:: Options: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build +:: Build targets: https://docs.microsoft.com/en-us/dotnet/core/rid-catalog dotnet build --configuration Release --framework net5.0 --output .\bin\NetworkBenchmarkDotNet-Windows\ .\bin\NetworkBenchmarkDotNet-Windows\NetworkBenchmarkDotNet -b %benchmark% + +echo off +Echo --- Benchmarks finished --- +Echo Save current process list +:: Folder should exist, just to be sure create it if it does not +if not exist "BenchmarkDotNet.Artifacts" mkdir BenchmarkDotNet.Artifacts + +echo on +:: Store currently running processes +tasklist /V /FO CSV > "BenchmarkDotNet.Artifacts\running-processes.csv" +tasklist /V > "BenchmarkDotNet.Artifacts\running-processes.txt" + PAUSE