KaneAI - World’s First E2E Software Testing Agent.Signup for Private Beta Join TestMu Conference

How to use PerfSummary method of Microsoft.Coyote.Benchmarking.Storage class

Best Coyote code snippet using Microsoft.Coyote.Benchmarking.Storage.PerfSummary

Program.cs

Source:Program.cs

...170                using (var file = new StreamWriter(summaryFile, true, Encoding.UTF8))171                {172                    if (writeHeaders)173                    {174                        PerfSummary.WriteHeaders(file);175                    }176                    foreach (var summary in await storage.DownloadAsync(this.DownloadPartition, rowKeys))177                    {178                        if (summary is null)179                        {180                            Console.WriteLine("Summary missing for {0}", b.Name);181                        }182                        else183                        {184                            summary.CommitId = this.CommitId;185                            summary.SetPartitionKey(this.DownloadPartition);186                            summary.WriteCsv(file);187                        }188                    }189                }190            }191        }192        private async Task<int> Run()193        {194            if (!string.IsNullOrEmpty(this.DownloadPartition))195            {196                await this.DowwnloadResults();197                return 0;198            }199            if (this.UploadCommits)200            {201                return await UploadCommitHistory();202            }203            if (string.IsNullOrEmpty(this.CommitId))204            {205                this.CommitId = Guid.NewGuid().ToString().Replace("-", string.Empty);206            }207            Storage storage = new Storage();208            List<PerfSummary> results = new List<PerfSummary>();209            int matching = 0;210            foreach (var b in Benchmarks)211            {212                if (FilterMatches(b.Name, this.Filters))213                {214                    matching++;215                    var config = DefaultConfig.Instance.WithArtifactsPath(this.OutputDir)216                        .WithOption(ConfigOptions.DisableOptimizationsValidator, true);217                    config.AddDiagnoser(new CpuDiagnoser());218                    config.AddDiagnoser(new TotalMemoryDiagnoser());219                    var summary = BenchmarkRunner.Run(b.Test, config);220                    foreach (var report in summary.Reports)221                    {222                        var data = this.GetEntities(report);223                        if (data.Count > 0)224                        {225                            results.Add(new PerfSummary(data));226                        }227                    }228                }229            }230            if (matching is 0)231            {232                Console.ForegroundColor = ConsoleColor.Red;233                Console.WriteLine("No benchmarks matching filter: {0}", string.Join(",", this.Filters));234                Console.ResetColor();235                PrintUsage();236                return 1;237            }238            else if (this.Cosmos)239            {240                await storage.UploadAsync(results);241                await UploadCommitHistory(1); // upload this commit id and it's commit date.242            }243            return 0;244        }245        private List<PerfEntity> GetEntities(BenchmarkReport report)246        {247            List<PerfEntity> results = new List<PerfEntity>();248            string testName = report.BenchmarkCase.Descriptor.DisplayInfo;249            foreach (var p in report.BenchmarkCase.Parameters.Items)250            {251                testName += string.Format(" {0}={1}", p.Name, p.Value);252            }253            // Right now we are choosing NOT to return each test result as254            // a separate entity, as this is too much information, so for now255            // we return the "Min" time from this run, based on the idea that the256            // minimum time has the least OS noise in it so it should be more stable.257            List<double> times = new List<double>();258            foreach (var row in report.GetResultRuns())259            {260                double msPerTest = row.Nanoseconds / 1000000.0 / row.Operations;261                times.Add(msPerTest);262            }263            var e = new PerfEntity(this.MachineName, this.RuntimeVersion, this.CommitId, testName, 0)264            {265                Time = times.Min(),266                TimeStdDev = MathHelpers.StandardDeviation(times),267            };268            if (report.Metrics.ContainsKey("CPU"))269            {270                e.Cpu = report.Metrics["CPU"].Value;271            }272            if (report.Metrics.ContainsKey("TotalMemory"))273            {274                e.Memory = report.Metrics["TotalMemory"].Value;275            }276            results.Add(e);277            return results;278        }279        private void ExportToCsv(List<PerfSummary> results)280        {281            this.SaveSummary(results);282            foreach (var item in results)283            {284                this.SaveReport(item.Data);285            }286        }287        private void SaveSummary(List<PerfSummary> report)288        {289            string filename = Path.Combine(this.OutputDir, "summary.csv");290            bool writeHeaders = !File.Exists(filename);291            using (StreamWriter writer = new StreamWriter(filename, true, Encoding.UTF8))292            {293                if (writeHeaders)294                {295                    PerfSummary.WriteHeaders(writer);296                }297                foreach (var item in report)298                {299                    item.WriteCsv(writer);300                }301            }302        }303        private void SaveReport(List<PerfEntity> data)304        {305            var testName = data[0].TestName.Split(' ')[0];306            string filename = Path.Combine(this.OutputDir, testName + ".csv");307            bool writeHeaders = !File.Exists(filename);308            using (StreamWriter writer = new StreamWriter(filename, true, Encoding.UTF8))309            {...

Storage.cs

Source:Storage.cs

...43            this.CosmosClient = new CosmosClient(endpointUri, primaryKey);44            var response = await this.CosmosClient.CreateDatabaseIfNotExistsAsync(CosmosDatabaseId);45            this.CosmosDatabase = response.Database;46        }47        internal async Task<int> UploadAsync(List<PerfSummary> summaries)48        {49            if (this.CosmosDatabase is null)50            {51                await this.Connect();52            }53            if (this.CosmosDatabase is null)54            {55                return 0;56            }57            if (this.SummaryContainer is null)58            {59                var response = await this.CosmosDatabase.CreateContainerIfNotExistsAsync(SummaryTableName, "/PartitionKey");60                this.SummaryContainer = response.Container;61            }62            int count = 0;63            foreach (var s in summaries)64            {65                bool better = true;66                try67                {68                    var response = await this.SummaryContainer.ReadItemAsync<PerfSummary>(s.Id, new PartitionKey(s.PartitionKey));69                    PerfSummary old = response.Resource;70                    if (old.TimeMean < s.TimeMean)71                    {72                        better = false;73                    }74                }75                catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)76                {77                }78                if (better)79                {80                    Console.WriteLine("===> Uploading summary for {0}...", s.TestName);81                    await this.SummaryContainer.UpsertItemAsync(s, new PartitionKey(s.PartitionKey));82                    count++;83                }84                else85                {86                    Console.WriteLine("===> Existing record is better for {0}...", s.TestName);87                }88            }89            return count;90        }91        internal async Task<List<PerfSummary>> DownloadAsync(string partitionKey, List<string> rowKeys)92        {93            List<PerfSummary> results = new List<PerfSummary>();94            if (this.CosmosDatabase is null)95            {96                await this.Connect();97            }98            if (this.CosmosDatabase is null)99            {100                return results;101            }102            if (this.SummaryContainer is null)103            {104                var response = await this.CosmosDatabase.CreateContainerIfNotExistsAsync(SummaryTableName, "/PartitionKey");105                this.SummaryContainer = response.Container;106            }107            foreach (var rowKey in rowKeys)108            {109                try110                {111                    var response = await this.SummaryContainer.ReadItemAsync<PerfSummary>(rowKey, new PartitionKey(partitionKey));112                    results.Add(response.Resource);113                }114                catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)115                {116                }117            }118            return results;119        }120        internal async Task UploadLogAsync(List<CommitHistoryEntity> log)121        {122            if (this.CosmosDatabase is null)123            {124                await this.Connect();125            }126            if (this.CosmosDatabase is null)127            {128                return;129            }130            var response = await this.CosmosDatabase.CreateContainerIfNotExistsAsync(CommitLogTableName, "/PartitionKey");131            var container = response.Container;132            foreach (var item in log)133            {134                Console.WriteLine("===> Uploading commit info {0}...", item.Id);135                await container.UpsertItemAsync(item, new PartitionKey(item.PartitionKey));136            }137        }138    }139    /// <summary>140    /// Entity representing a test result.141    /// </summary>142    public class PerfEntity143    {144        /// <summary>145        /// The row id.146        /// </summary>147        [JsonProperty(PropertyName = "id")]148        public string Id { get; set; }149        /// <summary>150        /// The parition key for the data.151        /// </summary>152        public string PartitionKey { get; set; }153        /// <summary>154        /// The computer name where the test was run.155        /// </summary>156        public string MachineName { get; set; }157        /// <summary>158        /// The .net runtime version used.159        /// </summary>160        public string RuntimeVersion { get; set; }161        /// <summary>162        /// The git commit id of code being tested.163        /// </summary>164        public string CommitId { get; set; }165        /// <summary>166        /// UTC date and time the test was run.167        /// </summary>168        public DateTime Date { get; set; }169        /// <summary>170        /// The unique name of the test.171        /// </summary>172        public string TestName { get; set; }173        /// <summary>174        /// The test iteration.175        /// </summary>176        public int Iteration { get; set; }177        /// <summary>178        /// Time to complete the test in milliseconds.179        /// </summary>180        public double Time { get; set; }181        /// <summary>182        /// Standard deviation in the times.183        /// </summary>184        public double TimeStdDev { get; set; }185        /// <summary>186        /// Process working set during the test.187        /// </summary>188        public double Memory { get; set; }189        /// <summary>190        /// Standard deviation in the memory numbers.191        /// </summary>192        public double MemoryStdDev { get; set; }193        /// <summary>194        /// Process total CPU usage during the test as a % of total number of cores.195        /// </summary>196        public double Cpu { get; set; }197        /// <summary>198        /// Standard deviation in the CPU numbers.199        /// </summary>200        public double CpuStdDev { get; set; }201        /// <summary>202        /// Additional notes about the test.203        /// </summary>204        public string Comments { get; set; }205        /// <summary>206        /// Initializes a new instance of the <see cref="PerfEntity"/> class.207        /// </summary>208        public PerfEntity(string machine, string runtime, string commit, string testName, int iteration)209        {210            this.MachineName = machine;211            this.RuntimeVersion = runtime;212            this.CommitId = commit;213            this.TestName = testName;214            this.Iteration = iteration;215            this.Date = DateTime.Now.ToUniversalTime();216            this.PartitionKey = string.Format("{0}.{1}", machine, runtime);217            this.Id = string.Format("{0}.{1}.{2}", commit, testName, iteration);218        }219        /// <summary>220        /// Initializes a new instance of the <see cref="PerfEntity"/> class.221        /// Needed for retreival.222        /// </summary>223        public PerfEntity()224        {225        }226        internal static void WriteHeaders(TextWriter outFile)227        {228            outFile.WriteLine("Name,Iteration,MinTime,StdDevTime,Memory,StdDevMemory,Cpu,StdDevCpu");229        }230        internal void WriteCsv(TextWriter outFile)231        {232            outFile.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", this.TestName, this.Iteration, this.Time, this.TimeStdDev, this.Memory, this.MemoryStdDev, this.Cpu, this.CpuStdDev);233        }234    }235    /// <summary>236    /// An entity representing the summary of all test iterations on a given test.237    /// </summary>238    public class PerfSummary239    {240        /// <summary>241        /// The row id.242        /// </summary>243        [JsonProperty(PropertyName = "id")]244        public string Id { get; set; }245        /// <summary>246        /// The parition key for the data.247        /// </summary>248        public string PartitionKey { get; set; }249        /// <summary>250        /// The computer name where the test was run.251        /// </summary>252        public string MachineName { get; set; }253        /// <summary>254        /// The .net runtime version used.255        /// </summary>256        public string RuntimeVersion { get; set; }257        /// <summary>258        /// The git commit id of code being tested.259        /// </summary>260        public string CommitId { get; set; }261        /// <summary>262        /// UTC date and time the test was run.263        /// </summary>264        public DateTime Date { get; set; }265        /// <summary>266        /// The unique name of the test.267        /// </summary>268        public string TestName { get; set; }269        /// <summary>270        /// The mean time in milliseconds.271        /// </summary>272        public double TimeMean { get; set; }273        /// <summary>274        /// The standard deviation of the test times.275        /// </summary>276        public double TimeStdDev { get; set; }277        /// <summary>278        /// The slope of the linear regression of the test times.279        /// </summary>280        public double TimeSlope { get; set; }281        /// <summary>282        /// The mean time in milliseconds.283        /// </summary>284        public double MemoryMean { get; set; }285        /// <summary>286        /// The standard deviation of the memory usage.287        /// </summary>288        public double MemoryStdDev { get; set; }289        /// <summary>290        /// The slope of the linear regression of the memory usage.291        /// </summary>292        public double MemorySlope { get; set; }293        /// <summary>294        /// The process cpu utilization as a percentage of total cores.295        /// </summary>296        public double CpuMean { get; set; }297        /// <summary>298        /// The standard deviation of the cpu times.299        /// </summary>300        public double CpuStdDev { get; set; }301        /// <summary>302        /// The slope of the linear regression of the cpu utilization.303        /// </summary>304        public double CpuSlope { get; set; }305        /// <summary>306        /// Additional notes about the test.307        /// </summary>308        public string Comments { get; set; }309        /// <summary>310        /// The raw test iterations.311        /// </summary>312        [IgnoreDataMember]313        internal readonly List<PerfEntity> Data;314        /// <summary>315        /// Initializes a new instance of the <see cref="PerfSummary"/> class.316        /// </summary>317        public PerfSummary(List<PerfEntity> data)318        {319            PerfEntity e = data[0];320            this.MachineName = e.MachineName;321            this.RuntimeVersion = e.RuntimeVersion;322            this.CommitId = e.CommitId;323            this.Data = data;324            this.TestName = e.TestName;325            this.Date = DateTime.Now.ToUniversalTime();326            this.PartitionKey = string.Format("{0}.{1}", e.MachineName, e.RuntimeVersion);327            this.Id = string.Format("{0}.{1}", e.CommitId, e.TestName);328            // summaryize the data.329            double meanTime = MathHelpers.Mean(from i in data select i.Time);330            double meanMemory = MathHelpers.Mean(from i in data select i.Memory);331            double meanCpu = MathHelpers.Mean(from i in data select i.Cpu);332            double meanStdDevTime = MathHelpers.Mean(from i in data select i.TimeStdDev);333            double meanStdDevMemory = MathHelpers.Mean(from i in data select i.MemoryStdDev);334            double meanStdDevCpu = MathHelpers.Mean(from i in data select i.CpuStdDev);335            if (meanStdDevTime is 0)336            {337                meanStdDevTime = MathHelpers.StandardDeviation(from i in data select i.Time);338                meanStdDevMemory = MathHelpers.StandardDeviation(from i in data select i.Memory);339                meanStdDevCpu = MathHelpers.StandardDeviation(from i in data select i.Cpu);340            }341            double timeSlope = MathHelpers.LinearRegression(MathHelpers.ToDataPoints(from i in data select i.Time)).Slope / meanTime;342            double memSlope = MathHelpers.LinearRegression(MathHelpers.ToDataPoints(from i in data select i.Memory)).Slope / meanMemory;343            double cpuSlope = MathHelpers.LinearRegression(MathHelpers.ToDataPoints(from i in data select i.Cpu)).Slope / meanCpu;344            // more than 10% slope we have a problem!345            if (timeSlope > 0.1)346            {347                this.Comments = "Slow down?";348            }349            else if (memSlope > 0.1)350            {351                this.Comments = "Memory leak?";352            }353            else if (cpuSlope > 0.1)354            {355                this.Comments = "Thread leak?";356            }357            this.TimeMean = meanTime;358            this.TimeStdDev = meanStdDevTime;359            this.TimeSlope = timeSlope;360            this.MemoryMean = meanMemory;361            this.MemoryStdDev = meanStdDevMemory;362            this.MemorySlope = memSlope;363            this.CpuMean = meanCpu;364            this.CpuStdDev = meanStdDevCpu;365            this.CpuSlope = cpuSlope;366        }367        /// <summary>368        /// Initializes a new instance of the <see cref="PerfSummary"/> class.369        /// Needed for retreival.370        /// </summary>371        public PerfSummary()372        {373        }374        internal static void WriteHeaders(TextWriter outFile)375        {376            outFile.WriteLine("Machine,Runtime,Commit,Date,Test,TimeMean,TimeStdDev,TimeSlope,MemoryMean,MemoryStdDev,MemorySlope,CpuMean,CpuStdDev,CpuSlope");377        }378        internal void WriteCsv(TextWriter outFile)379        {380            outFile.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13}", this.MachineName,381                this.RuntimeVersion, this.CommitId,  this.Date.ToLocalTime(), this.TestName, this.TimeMean,382                this.TimeStdDev, this.TimeSlope, this.MemoryMean, this.MemoryStdDev, this.MemorySlope,383                this.CpuMean, this.CpuStdDev, this.CpuSlope);384        }385        internal void SetPartitionKey(string partitionKey)...

PerfSummary

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Microsoft.Coyote.Benchmarking;7{8    {9        static void Main(string[] args)10        {

PerfSummary

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3using System.Collections.Generic;4using System.Linq;5using System.Text;6using System.Threading.Tasks;7{8    {9        static void Main(string[] args)10        {11            Storage.PerfSummary();12        }13    }14}

PerfSummary

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Microsoft.Coyote.Benchmarking;7using System.IO;8{9    {10        static void Main(string[] args)11        {12            Storage storage = new Storage();13            storage.PerfSummary();14        }15    }16}17using System;18using System.Collections.Generic;19using System.Linq;20using System.Text;21using System.Threading.Tasks;22using Microsoft.Coyote.Benchmarking;23{24    {25        static void Main(string[] args)26        {27            Storage storage = new Storage();28        }29    }30}31using System;32using System.Collections.Generic;33using System.Linq;34using System.Text;35using System.Threading.Tasks;36using Microsoft.Coyote.Benchmarking;37{38    {39        static void Main(string[] args)40        {41            Storage storage = new Storage();42            storage.PerfSummary();43            storage.PerfSummary();44        }45    }46}47using System;48using System.Collections.Generic;49using System.Linq;50using System.Text;51using System.Threading.Tasks;52using Microsoft.Coyote.Benchmarking;53{54    {55        static void Main(string[] args)56        {57            Storage storage = new Storage();58            storage.PerfSummary();59            storage.PerfSummary();60            storage.PerfSummary();61        }62    }63}64using System;65using System.Collections.Generic;66using System.Linq;67using System.Text;68using System.Threading.Tasks;69using Microsoft.Coyote.Benchmarking;70{71    {72        static void Main(string[] args)73        {74            Storage storage = new Storage();75            storage.PerfSummary();76            storage.PerfSummary();77            storage.PerfSummary();78            storage.PerfSummary();79        }80    }81}

PerfSummary

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3{4    {5        static void Main(string[] args)6        {7            Storage.PerfSummary();8        }9    }10}11using Microsoft.Coyote.Benchmarking;12using System;13{14    {15        static void Main(string[] args)16        {17            Storage.PerfSummary();18        }19    }20}21dotnet test --logger "console;verbosity=detailed" --logger "trx;LogFileName=2.trx"22using Microsoft.Coyote.Benchmarking;23using System;24{25    {26        static void Main(string[] args)27        {28            Storage.PerfSummary("merged.csv");29        }30    }31}

PerfSummary

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3{4    {5        static void Main(string[] args)6        {7            Storage storage = new Storage();8            storage.PerfSummary();9        }10    }11}12using Microsoft.Coyote.Benchmarking;13using System;14{15    {16        static void Main(string[] args)17        {18            Storage storage = new Storage();19            storage.PerfSummary();20        }21    }22}23using Microsoft.Coyote.Benchmarking;24using System;25{26    {27        static void Main(string[] args)28        {29            Storage storage = new Storage();30            storage.PerfSummary();31        }32    }33}34using Microsoft.Coyote.Benchmarking;35using System;36{37    {38        static void Main(string[] args)39        {40            Storage storage = new Storage();41            storage.PerfSummary();42        }43    }44}45using Microsoft.Coyote.Benchmarking;46using System;47{48    {49        static void Main(string[] args)50        {51            Storage storage = new Storage();52            storage.PerfSummary();53        }54    }55}56using Microsoft.Coyote.Benchmarking;57using System;58{59    {60        static void Main(string[] args)61        {62            Storage storage = new Storage();63            storage.PerfSummary();64        }65    }66}67using Microsoft.Coyote.Benchmarking;68using System;69{70    {71        static void Main(string[] args)72        {73            Storage storage = new Storage();

PerfSummary

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3{4    {5        static void Main(string[] args)6        {7            Storage.PerfSummary();8        }9    }10}11  #   Name                             Min (ms)    Max (ms)    Average (ms)    StdDev (ms)    Total (ms)    # of Samples

PerfSummary

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3using System.IO;4{5    {6        static void Main(string[] args)7        {8            string perfFile = "CoyotePerf.csv";9            string perfSummary = Storage.PerfSummary(perfFile);10            Console.WriteLine(perfSummary);

PerfSummary

Using AI Code Generation

1using System;2using Microsoft.Coyote.Benchmarking;3{4    {5        static void Main(string[] args)6        {7            string[] files = new string[] { "1.txt", "2.txt", "3.txt" };8            Storage.PerfSummary(files, "summary.txt");9        }10    }11}12using System;13using Microsoft.Coyote.Benchmarking;14{15    {16        static void Main(string[] args)17        {18            string[] files = new string[] { "1.txt", "2.txt", "3.txt" };19            Storage.PerfSummary(files, "summary.txt");20        }21    }22}

PerfSummary

Using AI Code Generation

1Total Time (ms): 02Total Time (s): 03Average Time (ms): 04Average Time (s): 05Min Time (ms): 06Min Time (s): 07Max Time (ms): 08Max Time (s): 09Number of Operations (K): 010Number of Operations (M): 011Throughput (K): 012Throughput (M): 013Throughput (ops/s): 014Throughput (Kops/s): 015Throughput (Mops/s): 016Total Time (ms): 017Total Time (s): 018Average Time (ms): 019Average Time (s): 020Min Time (ms): 021Min Time (s): 022Max Time (ms): 023Max Time (s): 0

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.