Skip to content

Commit 3ba552b

Browse files
brdeyobrdeyo
committed
Few more fixes to handle eccentricities with SockPerf and Latte workloads.
1 parent 1373da0 commit 3ba552b

23 files changed

Lines changed: 628 additions & 208 deletions

‎src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithInstallationInLin
5959
$"sudo gcc -dumpversion",
6060
$"sudo chmod -R ugo=rwx {this.mockPackage.Path}",
6161
$"sudo umount {this.mockFixture.GetPackagePath()}/speccpu_mount",
62-
$"sudo bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies 4 --threads 8 --tune all --reportable intrate\""
62+
$"bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies 4 --threads 8 --tune all --reportable intrate\""
6363
};
6464

6565
int processCount = 0;
@@ -180,7 +180,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithSpecificBenchmark
180180
$"sudo gcc -dumpversion",
181181
$"sudo chmod -R ugo=rwx {this.mockPackage.Path}",
182182
$"sudo umount {this.mockFixture.GetPackagePath()}/speccpu_mount",
183-
$"sudo bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies 4 --threads 8 --tune all --noreportable 549.fotonik3d_r\""
183+
$"bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies 4 --threads 8 --tune all --noreportable 549.fotonik3d_r\""
184184
};
185185

186186
int processCount = 0;
@@ -245,7 +245,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
245245
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
246246
{
247247
int coreCount = Environment.ProcessorCount;
248-
if (arguments == $"bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies {coreCount} --threads {coreCount} --tune base --reportable fprate\"")
248+
if (arguments == $"runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies {coreCount} --threads {coreCount} --tune base --reportable fprate\"")
249249
{
250250
commandCalled = true;
251251
}
@@ -283,7 +283,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
283283

284284
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
285285
{
286-
if (arguments == $"bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies {coreCount} --threads {coreCount} --tune all --reportable intspeed\"")
286+
if (arguments == $"runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 2 --copies {coreCount} --threads {coreCount} --tune all --reportable intspeed\"")
287287
{
288288
commandCalled = true;
289289
}
@@ -321,7 +321,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
321321

322322
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
323323
{
324-
if (arguments == $"bash runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 1 --copies {coreCount} --threads {coreCount} --tune all --noreportable intspeed\"")
324+
if (arguments == $"runspeccpu.sh \"--config vc-linux-x64.cfg --iterations 1 --copies {coreCount} --threads {coreCount} --tune all --noreportable intspeed\"")
325325
{
326326
commandCalled = true;
327327
}

‎src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuMetricsParserTests.cs‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33

44
namespace VirtualClient.Actions
55
{
6+
using System;
67
using System.Collections.Generic;
78
using System.IO;
9+
using System.Linq;
810
using System.Reflection;
911
using NUnit.Framework;
1012
using VirtualClient;
@@ -151,6 +153,52 @@ public void SpecCpuMetricsParserParsesExpectedMetricsFromFpRateBaseOnlyCsvResult
151153
MetricAssert.Exists(metrics, "SPECrate(R)2017_fp_base", 26.914269, "score");
152154
}
153155

156+
[Test]
157+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromFpRateResults()
158+
{
159+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
160+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuFpRateExample.txt");
161+
this.rawText = File.ReadAllText(outputPath);
162+
this.testParser = new SpecCpuMetricsParser(this.rawText);
163+
164+
IList<Metric> metrics = this.testParser.Parse();
165+
166+
foreach (Metric metric in metrics)
167+
{
168+
if (metric.Name.Contains("base"))
169+
{
170+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_base_rate");
171+
}
172+
else if (metric.Name.Contains("peak"))
173+
{
174+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_peak_rate");
175+
}
176+
}
177+
}
178+
179+
[Test]
180+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromFpRateCsvResults()
181+
{
182+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
183+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuFpRateExample.csv");
184+
this.rawText = File.ReadAllText(outputPath);
185+
this.testParser = new SpecCpuMetricsParser(this.rawText, csv: true);
186+
187+
IList<Metric> metrics = this.testParser.Parse();
188+
189+
foreach (Metric metric in metrics)
190+
{
191+
if (metric.Name.Contains("base"))
192+
{
193+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_base_rate");
194+
}
195+
else if (metric.Name.Contains("peak"))
196+
{
197+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_peak_rate");
198+
}
199+
}
200+
}
201+
154202
[Test]
155203
public void SpecCpuMetricsParserParsesExpectedMetricsFromIntRateBaseOnlyResults()
156204
{
@@ -242,6 +290,52 @@ public void SpecCpuMetricsParserParsesExpectedMetricsFromFpSpeedCsvResults()
242290
MetricAssert.Exists(metrics, "SPECspeed(R)2017_fp_peak", 252.731489, "score");
243291
}
244292

293+
[Test]
294+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromFpSpeedResults()
295+
{
296+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
297+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuFpSpeedExample.txt");
298+
this.rawText = File.ReadAllText(outputPath);
299+
this.testParser = new SpecCpuMetricsParser(this.rawText);
300+
301+
IList<Metric> metrics = this.testParser.Parse();
302+
303+
foreach (Metric metric in metrics)
304+
{
305+
if (metric.Name.Contains("base"))
306+
{
307+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_base_speed");
308+
}
309+
else if (metric.Name.Contains("peak"))
310+
{
311+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_peak_speed");
312+
}
313+
}
314+
}
315+
316+
[Test]
317+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromFpSpeedCsvResults()
318+
{
319+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
320+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuFpSpeedExample.csv");
321+
this.rawText = File.ReadAllText(outputPath);
322+
this.testParser = new SpecCpuMetricsParser(this.rawText, csv: true);
323+
324+
IList<Metric> metrics = this.testParser.Parse();
325+
326+
foreach (Metric metric in metrics)
327+
{
328+
if (metric.Name.Contains("base"))
329+
{
330+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_base_speed");
331+
}
332+
else if (metric.Name.Contains("peak"))
333+
{
334+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "floating_point_peak_speed");
335+
}
336+
}
337+
}
338+
245339
[Test]
246340
public void SpecCpuMetricsParserParsesExpectedMetricsFromIntRateResults()
247341
{
@@ -310,6 +404,52 @@ public void SpecCpuMetricsParserParsesExpectedMetricsFromIntRateCsvResults()
310404
MetricAssert.Exists(metrics, "SPECrate(R)2017_int_peak", 812.169896, "score");
311405
}
312406

407+
[Test]
408+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromIntRateResults()
409+
{
410+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
411+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuIntRateExample.txt");
412+
this.rawText = File.ReadAllText(outputPath);
413+
this.testParser = new SpecCpuMetricsParser(this.rawText);
414+
415+
IList<Metric> metrics = this.testParser.Parse();
416+
417+
foreach (Metric metric in metrics)
418+
{
419+
if (metric.Name.Contains("base"))
420+
{
421+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_base_rate");
422+
}
423+
else if (metric.Name.Contains("peak"))
424+
{
425+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_peak_rate");
426+
}
427+
}
428+
}
429+
430+
[Test]
431+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromIntRateCsvResults()
432+
{
433+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
434+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuIntRateExample.csv");
435+
this.rawText = File.ReadAllText(outputPath);
436+
this.testParser = new SpecCpuMetricsParser(this.rawText, csv: true);
437+
438+
IList<Metric> metrics = this.testParser.Parse();
439+
440+
foreach (Metric metric in metrics)
441+
{
442+
if (metric.Name.Contains("base"))
443+
{
444+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_base_rate");
445+
}
446+
else if (metric.Name.Contains("peak"))
447+
{
448+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_peak_rate");
449+
}
450+
}
451+
}
452+
313453
[Test]
314454
public void SpecCpuMetricsParserParsesExpectedMetricsFromIntSpeedResults()
315455
{
@@ -378,6 +518,52 @@ public void SpecCpuMetricsParserParsesExpectedMetricsFromIntSpeedCsvResults()
378518
MetricAssert.Exists(metrics, "SPECspeed(R)2017_int_peak", 12.293316, "score");
379519
}
380520

521+
[Test]
522+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromIntSpeedResults()
523+
{
524+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
525+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuIntSpeedExample.txt");
526+
this.rawText = File.ReadAllText(outputPath);
527+
this.testParser = new SpecCpuMetricsParser(this.rawText);
528+
529+
IList<Metric> metrics = this.testParser.Parse();
530+
531+
foreach (Metric metric in metrics)
532+
{
533+
if (metric.Name.Contains("base"))
534+
{
535+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_base_speed");
536+
}
537+
else if (metric.Name.Contains("peak"))
538+
{
539+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_peak_speed");
540+
}
541+
}
542+
}
543+
544+
[Test]
545+
public void SpecCpuMetricsParserParsesExpectedBenchmarkMetadataFromIntSpeedCsvResults()
546+
{
547+
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
548+
string outputPath = Path.Combine(workingDirectory, "Examples", "SpecCpu", "SpecCpuIntSpeedExample.csv");
549+
this.rawText = File.ReadAllText(outputPath);
550+
this.testParser = new SpecCpuMetricsParser(this.rawText, csv: true);
551+
552+
IList<Metric> metrics = this.testParser.Parse();
553+
554+
foreach (Metric metric in metrics)
555+
{
556+
if (metric.Name.Contains("base"))
557+
{
558+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_base_speed");
559+
}
560+
else if (metric.Name.Contains("peak"))
561+
{
562+
Assert.IsTrue(metric.Metadata.TryGetValue("workload", out IConvertible workload) && workload.ToString() == "integer_peak_speed");
563+
}
564+
}
565+
}
566+
381567
[Test]
382568
public void SpecCpuParserVerifyMetricsIntRateBaseWinArm64Incomplete()
383569
{

‎src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteClientExecutor.cs‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,8 @@ public LatteClientExecutor(IServiceCollection dependencies, IDictionary<string,
4141
}
4242

4343
/// <inheritdoc/>
44-
protected override Task<IProcessProxy> ExecuteWorkloadAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken, TimeSpan? timeout = null)
44+
protected override Task ExecuteWorkloadAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken, TimeSpan? timeout = null)
4545
{
46-
IProcessProxy process = null;
47-
4846
EventContext relatedContext = telemetryContext.Clone()
4947
.AddContext("command", this.ExecutablePath)
5048
.AddContext("commandArguments", commandArguments);
@@ -55,7 +53,7 @@ protected override Task<IProcessProxy> ExecuteWorkloadAsync(string commandArgume
5553
{
5654
await this.ProcessStartRetryPolicy.ExecuteAsync(async () =>
5755
{
58-
using (process = this.SystemManagement.ProcessManager.CreateProcess(this.ExecutablePath, commandArguments))
56+
using (IProcessProxy process = this.SystemManagement.ProcessManager.CreateProcess(this.ExecutablePath, commandArguments))
5957
{
6058
int processId = -1;
6159

@@ -98,8 +96,6 @@ await this.ProcessStartRetryPolicy.ExecuteAsync(async () =>
9896
}
9997
});
10098
}
101-
102-
return process;
10399
});
104100
}
105101

@@ -111,8 +107,7 @@ protected override string GetCommandLineArguments()
111107
string clientIPAddress = this.GetLayoutClientInstances(ClientRole.Client).First().IPAddress;
112108
string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress;
113109

114-
return $"-so -c -a {serverIPAddress}:{this.Port} -rio -i {this.Iterations} -riopoll {this.RioPoll} -{this.Protocol.ToString().ToLowerInvariant()} " +
115-
$"-hist -hl 1 -hc 9998 -bl {clientIPAddress}";
110+
return $"-so -c -a {serverIPAddress}:{this.Port} -t {this.TestDuration.TotalSeconds} -rio -riopoll {this.RioPoll} -{this.Protocol.ToString().ToLowerInvariant()} -hist -hl 1 -hc 9998 -bl {clientIPAddress}";
116111
}
117112

118113
/// <summary>

‎src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteExecutor.cs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,6 @@ public LatteExecutor(IServiceCollection dependencies, IDictionary<string, IConve
4545
.WaitAndRetryAsync(3, retries => TimeSpan.FromSeconds(retries * 3));
4646
}
4747

48-
/// <summary>
49-
/// The number of iterations for the network send/receive operations.
50-
/// </summary>
51-
public int Iterations
52-
{
53-
get
54-
{
55-
return this.Parameters.GetValue<int>(nameof(this.Iterations), 100100);
56-
}
57-
}
58-
5948
/// <summary>
6049
/// The starting port for the range of ports that will be used for client/server
6150
/// network connections.
@@ -90,6 +79,17 @@ public int RioPoll
9079
}
9180
}
9281

82+
/// <summary>
83+
/// Parameter defines the duration for running the Latte workload.
84+
/// </summary>
85+
public TimeSpan TestDuration
86+
{
87+
get
88+
{
89+
return this.Parameters.GetTimeSpanValue(nameof(this.TestDuration), TimeSpan.FromSeconds(60));
90+
}
91+
}
92+
9393
/// <summary>
9494
/// The retry policy to apply to the startup of the Latte workload to handle
9595
/// transient issues.

‎src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteServerExecutor.cs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,8 @@ public LatteServerExecutor(IServiceCollection dependencies, IDictionary<string,
4040
}
4141

4242
/// <inheritdoc/>
43-
protected override Task<IProcessProxy> ExecuteWorkloadAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken, TimeSpan? timeout = null)
43+
protected override Task ExecuteWorkloadAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken, TimeSpan? timeout = null)
4444
{
45-
IProcessProxy process = null;
46-
4745
EventContext relatedContext = telemetryContext.Clone()
4846
.AddContext("command", this.ExecutablePath)
4947
.AddContext("commandArguments", commandArguments);
@@ -56,7 +54,7 @@ await this.ProcessStartRetryPolicy.ExecuteAsync(async () =>
5654
{
5755
try
5856
{
59-
using (process = this.SystemManagement.ProcessManager.CreateProcess(this.ExecutablePath, commandArguments))
57+
using (IProcessProxy process = this.SystemManagement.ProcessManager.CreateProcess(this.ExecutablePath, commandArguments))
6058
{
6159
if (!process.Start())
6260
{
@@ -101,8 +99,6 @@ await this.ProcessStartRetryPolicy.ExecuteAsync(async () =>
10199
}
102100
});
103101
}
104-
105-
return process;
106102
});
107103
}
108104

@@ -113,7 +109,7 @@ await this.ProcessStartRetryPolicy.ExecuteAsync(async () =>
113109
protected override string GetCommandLineArguments()
114110
{
115111
string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress;
116-
return $"-a {serverIPAddress}:{this.Port} -rio -i {this.Iterations} -riopoll {this.RioPoll} -{this.Protocol.ToLowerInvariant()}";
112+
return $"-a {serverIPAddress}:{this.Port} -t {this.TestDuration.TotalSeconds} -rio -riopoll {this.RioPoll} -{this.Protocol.ToLowerInvariant()}";
117113
}
118114
}
119115
}

0 commit comments

Comments
 (0)