From 2a91b1baf26d42d5022e1ca5b5b67402507f6c60 Mon Sep 17 00:00:00 2001 From: Bjørn Ivar Teigen Date: Mon, 10 Jul 2023 19:58:56 +0200 Subject: Added QoO and Cablelabs latency histogram --- networkQuality.go | 32 ++++++++ qualityattenuation/qualityattenuation.go | 136 +++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) diff --git a/networkQuality.go b/networkQuality.go index aa8d854..5360666 100644 --- a/networkQuality.go +++ b/networkQuality.go @@ -846,6 +846,38 @@ func main() { if *debugCliFlag { fmt.Printf("(%s RPM Calculation stats): %v\n", direction.DirectionLabel, directionResult.ToString()) } + if *printQualityAttenuation { + fmt.Println("Quality Attenuation Statistics:") + fmt.Printf( + `Number of losses: %d +Number of samples: %d +Loss: %f +Min: %.6f +Max: %.6f +Mean: %.6f +Variance: %.6f +Standard Deviation: %.6f +PDV(90): %.6f +PDV(99): %.6f +P(90): %.6f +P(99): %.6f +RPM: %.0f +Gaming QoO: %.0f +`, selfRttsQualityAttenuation.GetNumberOfLosses(), + selfRttsQualityAttenuation.GetNumberOfSamples(), + selfRttsQualityAttenuation.GetLossPercentage(), + selfRttsQualityAttenuation.GetMinimum(), + selfRttsQualityAttenuation.GetMaximum(), + selfRttsQualityAttenuation.GetAverage(), + selfRttsQualityAttenuation.GetVariance(), + selfRttsQualityAttenuation.GetStandardDeviation(), + selfRttsQualityAttenuation.GetPDV(90), + selfRttsQualityAttenuation.GetPDV(99), + selfRttsQualityAttenuation.GetPercentile(90), + selfRttsQualityAttenuation.GetPercentile(99), + selfRttsQualityAttenuation.GetRPM(), + selfRttsQualityAttenuation.GetGamingQoO()) + } if !testRanToStability { fmt.Printf("Test did not run to stability, these results are estimates:\n") diff --git a/qualityattenuation/qualityattenuation.go b/qualityattenuation/qualityattenuation.go index 279959e..d9b8e0a 100644 --- a/qualityattenuation/qualityattenuation.go +++ b/qualityattenuation/qualityattenuation.go @@ -9,6 +9,39 @@ import ( "github.com/influxdata/tdigest" ) +type cablelabsHist struct { + hist [256]float64 +} + +func (h *cablelabsHist) GetHist() [256]float64 { + return h.hist +} + +func (h *cablelabsHist) AddSample(sample float64) error { + bin := 0 + if sample < 0.050 { + // Round down + bin = int(sample / 0.0005) + h.hist[bin]++ + } else if sample < 0.150 { + bin = int((sample - 0.050) / 0.001) + h.hist[100+bin]++ + } else if sample < 1.150 { + bin = int((sample - 0.150) / 0.020) + h.hist[200+bin]++ + } else if sample < 1.400 { + bin = 250 + h.hist[bin]++ + } else if sample < 3.000 { + bin = int((sample - 1.400) / 0.400) + h.hist[251+bin]++ + } else { + bin = 255 + h.hist[bin]++ + } + return nil +} + type SimpleQualityAttenuation struct { empiricalDistribution *tdigest.TDigest offset float64 @@ -19,6 +52,17 @@ type SimpleQualityAttenuation struct { latencyEqLossThreshold float64 minimumLatency float64 maximumLatency float64 + hist cablelabsHist +} + +type PercentileLatencyPair struct { + percentile float64 + perfectLatency float64 + uselessLatency float64 +} + +type QualityRequirement struct { + latencyRequirements []PercentileLatencyPair } func NewSimpleQualityAttenuation() *SimpleQualityAttenuation { @@ -41,6 +85,7 @@ func (qa *SimpleQualityAttenuation) AddSample(sample float64) error { // TODO: This should raise a warning and/or trigger error handling. return fmt.Errorf("sample is zero or negative") } + qa.hist.AddSample(sample) qa.numberOfSamples++ if sample > qa.latencyEqLossThreshold { qa.numberOfLosses++ @@ -108,6 +153,32 @@ func (qa *SimpleQualityAttenuation) GetPDV(percentile float64) float64 { return qa.GetPercentile(percentile) - qa.GetMinimum() } +func (qa *SimpleQualityAttenuation) PrintCablelabsStatisticsSummary() string { + // Prints a digest based on Cablelabs Latency Measurements Metrics and Architeture, CL-TR-LM-Arch-V01-221123, https://www.cablelabs.com/specifications/CL-TR-LM-Arch + // The recommendation is to report the following percentiles: 0, 10, 25, 50, 75, 90, 95, 99, 99.9 and 100 + return fmt.Sprintf("Cablelabs Statistics Summary:\n"+ + "0th Percentile: %f\n"+ + "10th Percentile: %f\n"+ + "25th Percentile: %f\n"+ + "50th Percentile: %f\n"+ + "75th Percentile: %f\n"+ + "90th Percentile: %f\n"+ + "95th Percentile: %f\n"+ + "99th Percentile: %f\n"+ + "99.9th Percentile: %f\n"+ + "100th Percentile: %f\n", + qa.GetPercentile(0.0), + qa.GetPercentile(10.0), + qa.GetPercentile(25.0), + qa.GetPercentile(50.0), + qa.GetPercentile(75.0), + qa.GetPercentile(90.0), + qa.GetPercentile(95.0), + qa.GetPercentile(99.0), + qa.GetPercentile(99.9), + qa.GetPercentile(100.0)) +} + // Merge two quality attenuation values. This operation assumes the two samples have the same offset and latency_eq_loss_threshold, and // will return an error if they do not. // It also assumes that the two quality attenuation values are measurements of the same thing (path, outcome, etc.). @@ -134,3 +205,68 @@ func (qa *SimpleQualityAttenuation) Merge(other *SimpleQualityAttenuation) error } return nil } + +func (qa *SimpleQualityAttenuation) GetHist() [256]float64 { + return qa.hist.GetHist() +} + +func (qa *SimpleQualityAttenuation) EmpiricalDistributionHistogram() []float64 { + // Convert the tdigest to a histogram on the format defined by CableLabs, with the following bucket edges: + // 100 bins from 0 to 50 ms, each 0.5 ms wide + // 100 bins from 50 to 100 ms, each 1 ms wide + // 50 bins from 150 to 1150 ms, each 20 ms wide + // 1 bin from 1150 to 1400 ms, 250 ms wide + // 4 bins from 1400 to 3000 ms, each 400 ms wide + hist := make([]float64, 256) + for i := 0; i < 100; i++ { + hist[i] = float64(qa.numberOfSamples) * (qa.empiricalDistribution.CDF(float64(i+1)*0.0005) - qa.empiricalDistribution.CDF(float64(i)*0.0005)) + } + for i := 100; i < 200; i++ { + hist[i] = float64(qa.numberOfSamples) * (qa.empiricalDistribution.CDF(0.050+float64(i-99)*0.001) - qa.empiricalDistribution.CDF(0.050+float64(i-100)*0.001)) + } + for i := 200; i < 250; i++ { + hist[i] = float64(qa.numberOfSamples) * (qa.empiricalDistribution.CDF(0.150+float64(i-199)*0.020) - qa.empiricalDistribution.CDF(0.150+float64(i-200)*0.020)) + } + for i := 250; i < 251; i++ { + hist[i] = float64(qa.numberOfSamples) * (qa.empiricalDistribution.CDF(1.150+0.250) - qa.empiricalDistribution.CDF(1.150)) + } + for i := 251; i < 255; i++ { + hist[i] = float64(qa.numberOfSamples) * (qa.empiricalDistribution.CDF(1.400+float64(i-250)*0.400) - qa.empiricalDistribution.CDF(1.400+float64(i-251)*0.400)) + } + hist[255] = float64(qa.numberOfSamples) * (1 - qa.empiricalDistribution.CDF(3.000)) + return hist +} + +// Compute the Quality of Outcome (QoO) for a given quality requirement. +// The details and motivation for the QoO metric are described in the following internet draft: +// https://datatracker.ietf.org/doc/draft-olden-ippm-qoo/ +func (qa *SimpleQualityAttenuation) QoO(requirement QualityRequirement) float64 { + QoO := 100.0 + for _, percentileLatencyPair := range requirement.latencyRequirements { + score := 0.0 + percentile := percentileLatencyPair.percentile + perfectLatency := percentileLatencyPair.perfectLatency + uselessLatency := percentileLatencyPair.uselessLatency + latency := qa.GetPercentile(percentile) + if latency >= uselessLatency { + score = 0.0 + } else if latency <= perfectLatency { + score = 100.0 + } else { + score = 100 * ((uselessLatency - latency) / (uselessLatency - perfectLatency)) + } + if score < QoO { + QoO = score + } + } + return QoO +} + +func (qa *SimpleQualityAttenuation) GetGamingQoO() float64 { + qualReq := QualityRequirement{} + qualReq.latencyRequirements = []PercentileLatencyPair{} + qualReq.latencyRequirements = append(qualReq.latencyRequirements, PercentileLatencyPair{percentile: 50.0, perfectLatency: 0.030, uselessLatency: 0.150}) + qualReq.latencyRequirements = append(qualReq.latencyRequirements, PercentileLatencyPair{percentile: 95.0, perfectLatency: 0.065, uselessLatency: 0.200}) + qualReq.latencyRequirements = append(qualReq.latencyRequirements, PercentileLatencyPair{percentile: 99.0, perfectLatency: 0.100, uselessLatency: 0.250}) + return qa.QoO(qualReq) +} -- cgit v1.2.3 From db0aea8354eb6f4d9732af5fd1cf0718e86f8f04 Mon Sep 17 00:00:00 2001 From: Bjørn Ivar Teigen Date: Wed, 12 Jul 2023 10:09:12 +0200 Subject: Addressed pull request comments * Added units to printout * Made QualityRequirement and PercentileLatencyPair private *Removed the hist struct from the SimpleQualityAttenuation struct (it was only used for testing) --- networkQuality.go | 20 ++++++++++---------- qualityattenuation/qualityattenuation.go | 26 ++++++++++---------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/networkQuality.go b/networkQuality.go index 5360666..0fff8a6 100644 --- a/networkQuality.go +++ b/networkQuality.go @@ -851,16 +851,16 @@ func main() { fmt.Printf( `Number of losses: %d Number of samples: %d -Loss: %f -Min: %.6f -Max: %.6f -Mean: %.6f -Variance: %.6f -Standard Deviation: %.6f -PDV(90): %.6f -PDV(99): %.6f -P(90): %.6f -P(99): %.6f +Loss: %f %% +Min: %.6f s +Max: %.6f s +Mean: %.6f s +Variance: %.6f s +Standard Deviation: %.6f s +PDV(90): %.6f s +PDV(99): %.6f s +P(90): %.6f s +P(99): %.6f s RPM: %.0f Gaming QoO: %.0f `, selfRttsQualityAttenuation.GetNumberOfLosses(), diff --git a/qualityattenuation/qualityattenuation.go b/qualityattenuation/qualityattenuation.go index d9b8e0a..c6a2f85 100644 --- a/qualityattenuation/qualityattenuation.go +++ b/qualityattenuation/qualityattenuation.go @@ -13,7 +13,7 @@ type cablelabsHist struct { hist [256]float64 } -func (h *cablelabsHist) GetHist() [256]float64 { +func (h *cablelabsHist) GetHistogram() [256]float64 { return h.hist } @@ -52,17 +52,16 @@ type SimpleQualityAttenuation struct { latencyEqLossThreshold float64 minimumLatency float64 maximumLatency float64 - hist cablelabsHist } -type PercentileLatencyPair struct { +type percentileLatencyPair struct { percentile float64 perfectLatency float64 uselessLatency float64 } -type QualityRequirement struct { - latencyRequirements []PercentileLatencyPair +type qualityRequirement struct { + latencyRequirements []percentileLatencyPair } func NewSimpleQualityAttenuation() *SimpleQualityAttenuation { @@ -85,7 +84,6 @@ func (qa *SimpleQualityAttenuation) AddSample(sample float64) error { // TODO: This should raise a warning and/or trigger error handling. return fmt.Errorf("sample is zero or negative") } - qa.hist.AddSample(sample) qa.numberOfSamples++ if sample > qa.latencyEqLossThreshold { qa.numberOfLosses++ @@ -206,10 +204,6 @@ func (qa *SimpleQualityAttenuation) Merge(other *SimpleQualityAttenuation) error return nil } -func (qa *SimpleQualityAttenuation) GetHist() [256]float64 { - return qa.hist.GetHist() -} - func (qa *SimpleQualityAttenuation) EmpiricalDistributionHistogram() []float64 { // Convert the tdigest to a histogram on the format defined by CableLabs, with the following bucket edges: // 100 bins from 0 to 50 ms, each 0.5 ms wide @@ -240,7 +234,7 @@ func (qa *SimpleQualityAttenuation) EmpiricalDistributionHistogram() []float64 { // Compute the Quality of Outcome (QoO) for a given quality requirement. // The details and motivation for the QoO metric are described in the following internet draft: // https://datatracker.ietf.org/doc/draft-olden-ippm-qoo/ -func (qa *SimpleQualityAttenuation) QoO(requirement QualityRequirement) float64 { +func (qa *SimpleQualityAttenuation) QoO(requirement qualityRequirement) float64 { QoO := 100.0 for _, percentileLatencyPair := range requirement.latencyRequirements { score := 0.0 @@ -263,10 +257,10 @@ func (qa *SimpleQualityAttenuation) QoO(requirement QualityRequirement) float64 } func (qa *SimpleQualityAttenuation) GetGamingQoO() float64 { - qualReq := QualityRequirement{} - qualReq.latencyRequirements = []PercentileLatencyPair{} - qualReq.latencyRequirements = append(qualReq.latencyRequirements, PercentileLatencyPair{percentile: 50.0, perfectLatency: 0.030, uselessLatency: 0.150}) - qualReq.latencyRequirements = append(qualReq.latencyRequirements, PercentileLatencyPair{percentile: 95.0, perfectLatency: 0.065, uselessLatency: 0.200}) - qualReq.latencyRequirements = append(qualReq.latencyRequirements, PercentileLatencyPair{percentile: 99.0, perfectLatency: 0.100, uselessLatency: 0.250}) + qualReq := qualityRequirement{} + qualReq.latencyRequirements = []percentileLatencyPair{} + qualReq.latencyRequirements = append(qualReq.latencyRequirements, percentileLatencyPair{percentile: 50.0, perfectLatency: 0.030, uselessLatency: 0.150}) + qualReq.latencyRequirements = append(qualReq.latencyRequirements, percentileLatencyPair{percentile: 95.0, perfectLatency: 0.065, uselessLatency: 0.200}) + qualReq.latencyRequirements = append(qualReq.latencyRequirements, percentileLatencyPair{percentile: 99.0, perfectLatency: 0.100, uselessLatency: 0.250}) return qa.QoO(qualReq) } -- cgit v1.2.3