Skip to content

Commit cb72b94

Browse files
committed
Merge pull request #797 from mattrjacobs/take-histogram-off-multithreaded-read-path
Move all reads of Histograms to a single-threaded path
2 parents 3246e46 + d66dae4 commit cb72b94

File tree

4 files changed

+164
-23
lines changed

4 files changed

+164
-23
lines changed

hystrix-core/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ jar {
3737
jmh {
3838
fork = 10
3939
iterations = 3
40-
jmhVersion = '1.9'
40+
jmhVersion = '1.9.3'
4141
profilers = ['gc']
4242
threads = 8
4343
warmup = '1s'

hystrix-core/src/jmh/java/com/netflix/hystrix/perf/MultiThreadedMetricsTest.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.netflix.hystrix.HystrixCommand;
44
import com.netflix.hystrix.HystrixCommandGroupKey;
5+
import com.netflix.hystrix.HystrixCommandMetrics;
56
import com.netflix.hystrix.HystrixCommandProperties;
67
import com.netflix.hystrix.HystrixThreadPoolProperties;
78
import org.openjdk.jmh.annotations.Benchmark;
@@ -72,7 +73,8 @@ public Integer writeHeavyCommandExecution(CommandState state) {
7273
@BenchmarkMode({Mode.Throughput})
7374
@OutputTimeUnit(TimeUnit.MILLISECONDS)
7475
public Integer writeHeavyReadMetrics(CommandState state) {
75-
return state.command.getMetrics().getCurrentConcurrentExecutionCount();
76+
HystrixCommandMetrics metrics = state.command.getMetrics();
77+
return metrics.getExecutionTimeMean() + metrics.getExecutionTimePercentile(50) + metrics.getExecutionTimePercentile(75) + metrics.getExecutionTimePercentile(99);
7678
}
7779

7880
@Benchmark
@@ -90,7 +92,8 @@ public Integer evenSplitOfWritesAndReadsCommandExecution(CommandState state) {
9092
@BenchmarkMode({Mode.Throughput})
9193
@OutputTimeUnit(TimeUnit.MILLISECONDS)
9294
public Integer evenSplitOfWritesAndReadsReadMetrics(CommandState state) {
93-
return state.command.getMetrics().getCurrentConcurrentExecutionCount();
95+
HystrixCommandMetrics metrics = state.command.getMetrics();
96+
return metrics.getExecutionTimeMean() + metrics.getExecutionTimePercentile(50) + metrics.getExecutionTimePercentile(75) + metrics.getExecutionTimePercentile(99);
9497
}
9598

9699
@Benchmark
@@ -108,6 +111,7 @@ public Integer readHeavyCommandExecution(CommandState state) {
108111
@BenchmarkMode({Mode.Throughput})
109112
@OutputTimeUnit(TimeUnit.MILLISECONDS)
110113
public Integer readHeavyReadMetrics(CommandState state) {
111-
return state.command.getMetrics().getCurrentConcurrentExecutionCount();
114+
HystrixCommandMetrics metrics = state.command.getMetrics();
115+
return metrics.getExecutionTimeMean() + metrics.getExecutionTimePercentile(50) + metrics.getExecutionTimePercentile(75) + metrics.getExecutionTimePercentile(99);
112116
}
113117
}

hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java

Lines changed: 65 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ public void addValue(int... value) {
110110

111111
for (int v : value) {
112112
try {
113-
getCurrentBucket().data.addValue(v);
113+
getCurrentBucket().bucketData.addValue(v);
114114
} catch (Exception e) {
115115
logger.error("Failed to add value: " + v, e);
116116
}
@@ -285,7 +285,7 @@ private static class PercentileBucketData {
285285
private final IntCountsHistogram histogram;
286286

287287
public PercentileBucketData() {
288-
this.histogram = new IntCountsHistogram(3);
288+
this.histogram = new IntCountsHistogram(4);
289289
}
290290

291291
public void addValue(int... latency) {
@@ -304,36 +304,83 @@ public int length() {
304304
*/
305305
/* package for testing */ static class PercentileSnapshot {
306306
private final IntCountsHistogram aggregateHistogram;
307+
private final long count;
308+
private final int mean;
309+
private final int p0;
310+
private final int p5;
311+
private final int p10;
312+
private final int p25;
313+
private final int p50;
314+
private final int p75;
315+
private final int p90;
316+
private final int p95;
317+
private final int p99;
318+
private final int p995;
319+
private final int p999;
320+
private final int p100;
321+
307322

308323
/* package for testing */ PercentileSnapshot() {
309324
this(new Bucket[0]);
310325
}
311326

312-
/* package for testing */ PercentileSnapshot(int... data) {
313-
aggregateHistogram = new IntCountsHistogram(4);
314-
for (int latency: data) {
315-
aggregateHistogram.recordValue(latency);
316-
}
327+
/* package for testing */ PercentileSnapshot(long startTime, int... data) {
328+
this(new Bucket[]{new Bucket(startTime, data)});
317329
}
318330

319331
/* package for testing */ PercentileSnapshot(Bucket[] buckets) {
320332
aggregateHistogram = new IntCountsHistogram(4);
321333
for (Bucket bucket: buckets) {
322-
aggregateHistogram.add(bucket.data.histogram);
334+
aggregateHistogram.add(bucket.bucketData.histogram);
323335
}
336+
337+
count = aggregateHistogram.getTotalCount();
338+
mean = (int) aggregateHistogram.getMean();
339+
p0 = (int) aggregateHistogram.getValueAtPercentile(0);
340+
p5 = (int) aggregateHistogram.getValueAtPercentile(5);
341+
p10 = (int) aggregateHistogram.getValueAtPercentile(10);
342+
p25 = (int) aggregateHistogram.getValueAtPercentile(25);
343+
p50 = (int) aggregateHistogram.getValueAtPercentile(50);
344+
p75 = (int) aggregateHistogram.getValueAtPercentile(75);
345+
p90 = (int) aggregateHistogram.getValueAtPercentile(90);
346+
p95 = (int) aggregateHistogram.getValueAtPercentile(95);
347+
p99 = (int) aggregateHistogram.getValueAtPercentile(99);
348+
p995 = (int) aggregateHistogram.getValueAtPercentile(99.5);
349+
p999 = (int) aggregateHistogram.getValueAtPercentile(99.9);
350+
p100 = (int) aggregateHistogram.getValueAtPercentile(100);
324351
}
325352

326353
/* package for testing */ int getMean() {
327-
return (int) aggregateHistogram.getMean();
354+
return mean;
328355
}
329356

330357
/**
331358
* Provides percentile computation.
332359
*/
333360
public int getPercentile(double percentile) {
334-
if (aggregateHistogram.getTotalCount() == 0) {
361+
if (count == 0) {
335362
return 0;
336363
}
364+
365+
int permyriad = (int) (percentile * 100);
366+
switch(permyriad) {
367+
case 0 : return p0;
368+
case 500 : return p5;
369+
case 1000: return p10;
370+
case 2500: return p25;
371+
case 5000: return p50;
372+
case 7500: return p75;
373+
case 9000: return p90;
374+
case 9500: return p95;
375+
case 9900: return p99;
376+
case 9950: return p995;
377+
case 9990: return p999;
378+
case 10000: return p100;
379+
default: return getArbitraryPercentile(percentile);
380+
}
381+
}
382+
383+
private synchronized int getArbitraryPercentile(double percentile) {
337384
return (int) aggregateHistogram.getValueAtPercentile(percentile);
338385
}
339386
}
@@ -519,13 +566,19 @@ private Bucket[] getArray() {
519566
*/
520567
/* package for testing */ static class Bucket {
521568
final long windowStart;
522-
final PercentileBucketData data;
569+
final PercentileBucketData bucketData;
523570

524571
Bucket(long startTime) {
525572
this.windowStart = startTime;
526-
this.data = new PercentileBucketData();
573+
this.bucketData = new PercentileBucketData();
527574
}
528575

576+
public Bucket(long startTime, int[] data) {
577+
this.windowStart = startTime;
578+
579+
this.bucketData = new PercentileBucketData();
580+
bucketData.addValue(data);
581+
}
529582
}
530583

531584
/* package for testing */ static interface Time {

hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java

Lines changed: 91 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,17 @@
1818
import static org.junit.Assert.assertEquals;
1919
import static org.junit.Assert.fail;
2020

21+
import java.util.Random;
22+
import java.util.concurrent.CountDownLatch;
23+
import java.util.concurrent.ExecutorService;
24+
import java.util.concurrent.Executors;
25+
import java.util.concurrent.Future;
26+
import java.util.concurrent.TimeUnit;
2127
import java.util.concurrent.atomic.AtomicInteger;
2228

29+
import org.junit.AfterClass;
2330
import org.junit.Assert;
31+
import org.junit.BeforeClass;
2432
import org.junit.Test;
2533

2634
import com.netflix.hystrix.strategy.properties.HystrixProperty;
@@ -33,6 +41,23 @@ public class HystrixRollingPercentileTest {
3341
private static final HystrixProperty<Integer> numberOfBuckets = HystrixProperty.Factory.asProperty(12); // 12 buckets at 5000ms each
3442
private static final HystrixProperty<Boolean> enabled = HystrixProperty.Factory.asProperty(true);
3543

44+
private static ExecutorService threadPool;
45+
46+
@BeforeClass
47+
public static void setUp() {
48+
threadPool = Executors.newFixedThreadPool(10);
49+
}
50+
51+
@AfterClass
52+
public static void tearDown() {
53+
threadPool.shutdown();
54+
try {
55+
threadPool.awaitTermination(10, TimeUnit.SECONDS);
56+
} catch (InterruptedException ie) {
57+
System.out.println("Thread pool never terminated in HystrixRollingPercentileTest");
58+
}
59+
}
60+
3661
@Test
3762
public void testRolling() {
3863
MockedTime time = new MockedTime();
@@ -78,7 +103,7 @@ public void testRolling() {
78103
time.increment(6000);
79104

80105
// the rolling version should have the same data as creating a snapshot like this
81-
PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600);
106+
PercentileSnapshot ps = new PercentileSnapshot(System.currentTimeMillis(), 1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600);
82107

83108
assertEquals(ps.getPercentile(0.15), p.getPercentile(0.15));
84109
assertEquals(ps.getPercentile(0.50), p.getPercentile(0.50));
@@ -201,36 +226,36 @@ public void testSampleDataOverTime2() {
201226
}
202227

203228
public PercentileSnapshot getPercentileForValues(int... values) {
204-
return new PercentileSnapshot(values);
229+
return new PercentileSnapshot(System.currentTimeMillis(), values);
205230
}
206231

207232
@Test
208233
public void testPercentileAlgorithm_Median1() {
209-
PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 300);
234+
PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 300);
210235
Assert.assertEquals(200, list.getPercentile(50));
211236
}
212237

213238
@Test
214239
public void testPercentileAlgorithm_Median2() {
215-
PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 500);
240+
PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 500);
216241
Assert.assertEquals(100, list.getPercentile(50));
217242
}
218243

219244
@Test
220245
public void testPercentileAlgorithm_Median3() {
221-
PercentileSnapshot list = new PercentileSnapshot(50, 75, 100, 125, 160, 170, 180, 200, 210, 300, 500);
246+
PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 50, 75, 100, 125, 160, 170, 180, 200, 210, 300, 500);
222247
Assert.assertEquals(170, list.getPercentile(50));
223248
}
224249

225250
@Test
226251
public void testPercentileAlgorithm_Median4() {
227-
PercentileSnapshot list = new PercentileSnapshot(300, 75, 125, 500, 100, 160, 180, 200, 210, 50, 170);
252+
PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 300, 75, 125, 500, 100, 160, 180, 200, 210, 50, 170);
228253
Assert.assertEquals(170, list.getPercentile(50));
229254
}
230255

231256
@Test
232257
public void testPercentileAlgorithm_Extremes() {
233-
PercentileSnapshot p = new PercentileSnapshot(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 800, 768, 657, 700, 867);
258+
PercentileSnapshot p = new PercentileSnapshot(System.currentTimeMillis(), 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 800, 768, 657, 700, 867);
234259

235260
System.out.println("0.01: " + p.getPercentile(0.01));
236261
System.out.println("10th: " + p.getPercentile(10));
@@ -317,6 +342,65 @@ public void increment(int millis) {
317342

318343
}
319344

345+
@Test
346+
public void testThreadSafety() {
347+
final MockedTime time = new MockedTime();
348+
final HystrixRollingPercentile p = new HystrixRollingPercentile(time, HystrixProperty.Factory.asProperty(100), HystrixProperty.Factory.asProperty(25), HystrixProperty.Factory.asProperty(true));
349+
350+
final int NUM_THREADS = 1000;
351+
final int NUM_ITERATIONS = 1000000;
352+
353+
final CountDownLatch latch = new CountDownLatch(NUM_THREADS);
354+
355+
final AtomicInteger aggregateMetrics = new AtomicInteger(); //same as a blackhole
356+
357+
final Random r = new Random();
358+
359+
Future<?> metricsPoller = threadPool.submit(new Runnable() {
360+
@Override
361+
public void run() {
362+
while(!Thread.currentThread().isInterrupted()) {
363+
aggregateMetrics.addAndGet(p.getMean() + p.getPercentile(10) + p.getPercentile(50) + p.getPercentile(90));
364+
//System.out.println("AGGREGATE : " + p.getPercentile(10) + " : " + p.getPercentile(50) + " : " + p.getPercentile(90));
365+
}
366+
}
367+
});
368+
369+
for (int i = 0; i < NUM_THREADS; i++) {
370+
final int threadId = i;
371+
threadPool.submit(new Runnable() {
372+
@Override
373+
public void run() {
374+
for (int j = 1; j < NUM_ITERATIONS / NUM_THREADS + 1; j++) {
375+
int nextInt = r.nextInt(100);
376+
p.addValue(nextInt);
377+
if (threadId == 0) {
378+
time.increment(1);
379+
}
380+
}
381+
latch.countDown();
382+
}
383+
});
384+
}
385+
386+
try {
387+
latch.await(100, TimeUnit.SECONDS);
388+
metricsPoller.cancel(true);
389+
} catch (InterruptedException ex) {
390+
fail("Timeout on all threads writing percentiles");
391+
}
392+
393+
aggregateMetrics.addAndGet(p.getMean() + p.getPercentile(10) + p.getPercentile(50) + p.getPercentile(90));
394+
System.out.println(p.getMean() + " : " + p.getPercentile(50) + " : " + p.getPercentile(75) + " : " + p.getPercentile(90) + " : " + p.getPercentile(95) + " : " + p.getPercentile(99));
395+
}
396+
397+
@Test
398+
public void testThreadSafetyMulti() {
399+
for (int i = 0; i < 100; i++) {
400+
testThreadSafety();
401+
}
402+
}
403+
320404
/* sub-class to avoid 65k limit of a single class */
321405
private static class SampleDataHolder1 {
322406
/*

0 commit comments

Comments
 (0)