ExecutorServiceの最大プールサイズを考える(Java) †
概要 †「N個のデータの塊を 何個のスレッドで処理するのが一番効率が良いのか」を コア数との関係を考慮しながら 確認する。 今回、検証に使用したプログラムの概要は以下の通り。
結果 †先に結果を書いてしまう。 今回のような単純なMapインスタンスの生成を繰り返すようなプログラムの場合は、 良くないのはスレッド数 ≧ コア数の時 で、スループットもヒープ効率もかなり悪い結果となった。 恐らく、コア数以上のプール(スレッド)数で処理した場合、コンテキストスイッチが多数発生する事になる為、 ただし、ヒープの割り当て方によっては今回の結果も大きく変わる可能性がある事には注意したい。 結果(補足) †尚、Executors.newCachedThreadPool 及び Executors.newWorkStealingPool で同様の動作確認も行ってみたが、 またメインスレッドの起動時にスレッドプールを生成し、以降はそれを使い回す方式も確認してみたが、 プロジェクト作成 †確認用のプロジェクトは以下の通り作成した。 build.gradle †plugins { id 'java' id 'application' } // ヒープサイズは512M、New、OLD比率は1:1(256M:256M)、Edenとsurvivor比率は4:1(170M:42M) applicationDefaultJvmArgs = ["-Xms512m", "-Xmx512m", "-XX:NewRatio=1", "-XX:SurvivorRatio=4"] mainClassName = 'App' dependencies { compile 'com.google.guava:guava:23.0' testCompile 'junit:junit:4.12' } repositories { jcenter() } run { // argsが指定された時はメインクラスに渡す if (project.hasProperty('args')) { args project.args.split('\\s+') } } src/main/java/App.java †import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.ArrayList; import java.util.List; import java.time.LocalDateTime; public class App { ExecutorService executor = null; private static int THREAD_GROUP_NUM = 100; // 繰り返し回数 private static int THREAD_NUM = 10; // 1グループ辺りで何個のスレッドを生成するか private static boolean POOL_SETUP_EACH_TIME = true; // スレッドプールの生成を繰り返し毎に行う場合はtrue private static int THREAD_POOL_MAX_NUM = 5; // 確保するスレッドプールの数 public static void main(String[] args){ for (String arg : args) { String[] argArray = arg.split("="); if (argArray.length < 2) { continue; } if ("POOL_SETUP_EACH_TIME".equals(argArray[0])) { POOL_SETUP_EACH_TIME = "true".equals(argArray[1].toLowerCase()); } if ("THREAD_POOL_MAX_NUM".equals(argArray[0])) { THREAD_POOL_MAX_NUM = Integer.valueOf(argArray[1]); } if ("THREAD_GROUP_NUM".equals(argArray[0])) { THREAD_GROUP_NUM = Integer.valueOf(argArray[1]); } if ("THREAD_NUM".equals(argArray[0])) { THREAD_NUM = Integer.valueOf(argArray[1]); } } new App().main(); } public void main() { info("App.main", "START"); long start = System.currentTimeMillis(); info("App.main", "ENV POOL_SETUP_EACH_TIME : " + POOL_SETUP_EACH_TIME); info("App.main", "ENV THREAD_POOL_MAX_NUM : " + THREAD_POOL_MAX_NUM); info("App.main", "ENV THREAD_GROUP_NUM : " + THREAD_GROUP_NUM); info("App.main", "ENV THREAD_NUM : " + THREAD_NUM); // スレッドプールのセットアップ if (!POOL_SETUP_EACH_TIME) { setupExecutor(); } // スレッドグループの実行 executeThreadsGroup(); // スレッドプールの後片付け if (!POOL_SETUP_EACH_TIME) { shutdownExecutor(); } long end = System.currentTimeMillis(); info("App.main", "END", "("+(end-start)+"ms)"); } private void setupExecutor(){ info("App.setupExecutor", "START"); long start = System.currentTimeMillis(); executor = Executors.newFixedThreadPool(THREAD_POOL_MAX_NUM); //executor = Executors.newCachedThreadPool(); //executor = Executors.newWorkStealingPool(); long end = System.currentTimeMillis(); info("App.setupExecutor", "END", "("+(end-start)+"ms)"); } private void executeThreadsGroup(){ info("App.executeThreadsGroup", "START"); long start = System.currentTimeMillis(); for (int i = 0; i < THREAD_GROUP_NUM; i++) { executeThreads(); } long end = System.currentTimeMillis(); info("App.executeThreadsGroup", "END", "("+(end-start)+"ms)"); } private void shutdownExecutor(){ info("App.shutdownExecutor", "START"); long start = System.currentTimeMillis(); executor.shutdown(); long end = System.currentTimeMillis(); info("App.shutdownExecutor", "END", "("+(end-start)+"ms)"); } public void executeThreads(){ info("App.executeThreads", "START"); long start = System.currentTimeMillis(); if (POOL_SETUP_EACH_TIME) { setupExecutor(); } // スレッドの起動 List<Future<Boolean>> futureList = new ArrayList<Future<Boolean>>(); for (int i = 0; i < THREAD_NUM; i++) { MyThread thread = new MyThread(); thread.setParam("thread-"+i); futureList.add(executor.submit(thread)); } // 各スレッドの結果確認 for (Future<Boolean> future : futureList) { try { if (!future.get()){ System.err.println("thread error!"); } } catch (Exception e) { e.printStackTrace(); } } if (POOL_SETUP_EACH_TIME) { shutdownExecutor(); } long end = System.currentTimeMillis(); info("App.executeThreads", "END", "("+(end-start)+"ms)"); } public static void info(String...messages){ String delimiter = ""; StringBuilder sb = new StringBuilder(); for (String message : messages) { sb.append(delimiter); sb.append(message); delimiter = " "; } System.out.println(LocalDateTime.now() + " " + sb.toString()); } } src/main/java/MyThread.java †import java.util.Map; import java.util.HashMap; import java.util.concurrent.Callable; public class MyThread implements Callable<Boolean> { Object param = null; public Boolean call(){ String param = (String)getParam(); App.info("MyThread.call", "START", param); // 適当にオブジェクト生成を繰り返す Map map = new HashMap(); for (int i = 0; i < 1000; i++) { map.put("key-" + Math.random(), "val-" + Math.random()); map.put("subMap", createSubMap()); } App.info("MyThread.call", "END", param); return true; } public void setParam(Object param){ this.param = param; } public Object getParam(){ return this.param; } private Map createSubMap(){ Map map = new HashMap(); for (int i = 0; i < 100; i++) { map.put("key-" + Math.random(), "val-" + Math.random()); } return map; } } start.sh †#!/bin/bash args="" log_sufix=`date +%Y%m%d_%H%M%S` if [ "$#" != "0" ]; then args="$*" fi echo gradle run -Pargs="$args" gradle run -Pargs="$args">app_${log_sufix}.log 2>&1 & ./watch.sh | tee heap_${log_sufix}.log cat app_${log_sufix}.log | grep "App.main ENV" | awk '{print $4" "$5" "$6}' cat app_${log_sufix}.log | grep "App.executeThreadsGroup END" | awk '{print $4}' | sed -E "s/(\(|\))//g" | awk '{print "App.executeThreadsGroup "$1}' cat app_${log_sufix}.log | grep "App.main END" | awk '{print $4}' | sed -E "s/(\(|\))//g" | awk '{print "App.main "$1}' cat app_${log_sufix}.log | grep "App.setupExecutor END" | wc -l | awk '{print "executor setup count: "$1}' cat app_${log_sufix}.log | grep "App.executeThreads END" | wc -l | awk '{print "threads count: "$1}' cat app_${log_sufix}.log | grep "App.executeThreads END" | awk '{print $4}' | sed -E 's/[^0-9]//g' | awk 'BEGIN{sum=0}{sum=sum+$1}END{print "threads summary: "sum}' watch.sh †#!/bin/bash while [ true ]; do pid=`jps | grep App | awk '{print $1}'` if [ "$pid" != "" ]; then break fi done jstat -gccapacity $pid jstat -gcutil $pid 1000 確認 †事前にコア数の確認 †Macの場合 system_profiler SPHardwareDataType | grep Cores Total Number of Cores: 6 ※linuxの場合は、cat /proc/cpuinfo 等で確認する。 MAXスレッド数>コア数 †$ ./start.sh THREAD_POOL_MAX_NUM=10 gradle run -Pargs=THREAD_POOL_MAX_NUM=10 NGCMN NGCMX NGC S0C S1C EC OGCMN OGCMX OGC OC MCMN MCMX MC CCSMN CCSMX CCSC YGC FGC 262144.0 262144.0 262144.0 43520.0 43520.0 175104.0 262144.0 262144.0 262144.0 262144.0 0.0 1056768.0 4864.0 0.0 1048576.0 512.0 1 0 S0 S1 E O M CCS YGC YGCT FGC FGCT GCT 5.96 0.00 67.65 0.01 74.07 75.14 3 0.003 0 0.000 0.003 37.50 0.00 91.41 0.34 74.33 75.35 8 0.010 0 0.000 0.010 0.00 44.64 74.91 0.54 74.56 75.35 13 0.014 0 0.000 0.014 58.33 0.00 58.63 1.78 74.96 75.35 18 0.019 0 0.000 0.019 0.00 51.04 54.05 4.19 75.54 75.35 23 0.026 0 0.000 0.026 72.50 0.00 31.38 6.50 75.67 75.35 28 0.033 0 0.000 0.033 0.00 60.00 7.41 8.92 75.85 75.35 33 0.039 0 0.000 0.039 0.00 62.50 80.24 10.83 75.97 75.35 37 0.045 0 0.000 0.045 68.75 0.00 41.94 13.18 76.35 75.44 42 0.051 0 0.000 0.051 0.00 85.42 9.99 15.52 76.49 75.44 47 0.058 0 0.000 0.058 68.75 0.00 3.43 17.58 76.78 75.44 52 0.064 0 0.000 0.064 62.50 0.00 62.61 18.03 76.94 75.44 56 0.068 0 0.000 0.068 0.00 73.75 23.27 20.36 76.96 75.44 61 0.075 0 0.000 0.075 0.00 71.25 89.16 22.25 76.96 75.44 65 0.081 0 0.000 0.081 61.25 0.00 50.58 24.68 76.97 75.44 70 0.088 0 0.000 0.088 0.00 70.00 14.43 27.00 77.21 75.71 75 0.095 0 0.000 0.095 81.25 0.00 5.43 29.42 77.58 75.71 80 0.102 0 0.000 0.102 71.88 0.00 60.04 31.29 77.60 75.71 84 0.108 0 0.000 0.108 0.00 67.50 46.93 33.08 77.60 75.71 89 0.113 0 0.000 0.113 61.25 0.00 12.88 33.94 77.60 75.71 94 0.117 0 0.000 0.117 71.25 0.00 75.99 35.74 77.60 75.71 98 0.122 0 0.000 0.122 0.00 61.25 30.30 38.18 77.71 75.71 103 0.128 0 0.000 0.128 70.00 0.00 9.30 40.50 77.79 75.71 108 0.134 0 0.000 0.134 70.00 0.00 10.47 42.43 77.79 75.71 112 0.142 0 0.000 0.142 0.00 61.25 57.95 43.89 77.79 75.71 115 0.147 0 0.000 0.147 89.06 0.00 81.14 45.29 77.80 75.71 118 0.151 0 0.000 0.151 87.50 0.00 35.72 47.20 77.80 75.71 122 0.157 0 0.000 0.157 67.50 0.00 8.54 48.67 77.80 75.71 126 0.165 0 0.000 0.165 62.50 0.00 17.73 49.20 77.80 75.71 130 0.169 0 0.000 0.169 0.00 71.25 87.37 50.48 77.80 75.71 133 0.173 0 0.000 0.173 0.00 71.25 19.96 52.38 77.80 75.71 137 0.179 0 0.000 0.179 61.25 0.00 51.56 53.86 77.80 75.71 140 0.183 0 0.000 0.183 62.50 0.00 9.79 55.75 77.80 75.71 144 0.189 0 0.000 0.189 61.25 0.00 20.54 57.64 77.80 75.71 148 0.198 0 0.000 0.198 65.00 0.00 0.00 59.54 77.80 75.71 152 0.204 0 0.000 0.204 0.00 87.50 58.35 60.93 77.80 75.71 155 0.209 0 0.000 0.209 78.12 0.00 95.57 62.38 77.80 75.71 158 0.216 0 0.000 0.216 0.00 67.50 50.26 64.31 77.80 75.71 163 0.222 0 0.000 0.222 0.00 62.50 0.00 65.27 77.82 75.71 167 0.226 0 0.000 0.226 71.25 0.00 58.12 65.75 77.88 75.71 170 0.229 0 0.000 0.229 71.25 0.00 48.27 67.66 77.88 75.71 174 0.235 0 0.000 0.235 72.50 0.00 17.31 69.57 77.88 75.71 178 0.240 0 0.000 0.240 0.00 62.50 49.19 71.06 77.88 75.71 181 0.244 0 0.000 0.244 71.25 0.00 31.21 73.39 77.88 75.71 186 0.251 0 0.000 0.251 0.00 63.75 73.59 74.86 77.88 75.71 189 0.255 0 0.000 0.255 0.00 61.25 60.69 76.75 77.88 75.71 193 0.261 0 0.000 0.261 POOL_SETUP_EACH_TIME : true THREAD_POOL_MAX_NUM : 10 THREAD_GROUP_NUM : 100 THREAD_NUM : 10 App.executeThreadsGroup 46504ms App.main 46504ms executor setup count: 100 threads count: 100 threads summary: 46496 MAXスレッド数<コア数 †$ ./start.sh THREAD_POOL_MAX_NUM=3 gradle run -Pargs=THREAD_POOL_MAX_NUM=3 NGCMN NGCMX NGC S0C S1C EC OGCMN OGCMX OGC OC MCMN MCMX MC CCSMN CCSMX CCSC YGC FGC 262144.0 262144.0 262144.0 43520.0 43520.0 175104.0 262144.0 262144.0 262144.0 262144.0 0.0 1056768.0 4480.0 0.0 1048576.0 384.0 0 0 S0 S1 E O M CCS YGC YGCT FGC FGCT GCT 0.00 2.43 78.00 0.00 73.96 75.14 1 0.001 0 0.000 0.001 0.00 62.50 47.82 0.33 74.27 75.35 9 0.007 0 0.000 0.007 47.92 0.00 32.67 0.41 74.65 75.35 16 0.012 0 0.000 0.012 0.00 50.00 36.21 0.49 75.21 75.35 23 0.016 0 0.000 0.016 37.50 0.00 98.14 0.57 75.38 75.35 30 0.020 0 0.000 0.020 58.33 0.00 0.00 0.65 75.84 75.44 38 0.025 0 0.000 0.025 0.00 78.12 0.00 0.74 76.11 75.44 45 0.029 0 0.000 0.029 52.08 0.00 0.00 0.82 76.20 75.44 52 0.033 0 0.000 0.033 34.38 0.00 30.79 0.87 76.21 75.44 58 0.037 0 0.000 0.037 0.00 22.92 17.57 0.94 76.26 75.44 65 0.041 0 0.000 0.041 81.25 0.00 0.00 1.00 76.35 75.44 72 0.045 0 0.000 0.045 0.00 10.42 5.55 1.07 76.69 75.44 79 0.049 0 0.000 0.049 71.88 0.00 0.00 1.14 76.73 75.44 86 0.053 0 0.000 0.053 0.00 56.25 0.00 1.19 76.74 75.44 93 0.056 0 0.000 0.056 71.88 0.00 17.13 1.27 76.75 75.44 100 0.060 0 0.000 0.060 18.75 0.00 94.65 1.32 76.75 75.44 106 0.064 0 0.000 0.064 0.00 78.12 52.86 1.40 76.76 75.44 113 0.068 0 0.000 0.068 0.00 43.75 52.48 1.46 76.82 75.44 119 0.071 0 0.000 0.071 40.62 0.00 27.65 1.54 76.83 75.44 126 0.075 0 0.000 0.075 43.75 0.00 93.45 1.60 76.90 75.44 132 0.079 0 0.000 0.079 0.00 31.25 30.08 1.67 76.98 75.44 139 0.083 0 0.000 0.083 53.12 0.00 0.00 1.77 76.98 75.44 146 0.087 0 0.000 0.087 81.25 0.00 13.57 1.84 77.00 75.44 152 0.090 0 0.000 0.090 31.25 0.00 39.10 1.90 77.04 75.44 158 0.093 0 0.000 0.093 50.00 0.00 52.23 1.96 77.32 75.44 164 0.096 0 0.000 0.096 0.00 50.00 90.85 2.03 77.53 75.44 169 0.099 0 0.000 0.099 0.00 47.92 57.15 2.11 77.53 75.44 175 0.102 0 0.000 0.102 37.50 0.00 84.69 2.16 77.53 75.44 180 0.105 0 0.000 0.105 78.12 0.00 21.86 2.23 77.54 75.44 186 0.108 0 0.000 0.108 0.00 35.42 77.59 2.27 77.56 75.44 191 0.110 0 0.000 0.110 POOL_SETUP_EACH_TIME : true THREAD_POOL_MAX_NUM : 3 THREAD_GROUP_NUM : 100 THREAD_NUM : 10 App.executeThreadsGroup 29948ms App.main 29949ms executor setup count: 100 threads count: 100 threads summary: 29940 MAXスレッド数=コア数 †$ ./start.sh THREAD_POOL_MAX_NUM=6 gradle run -Pargs=THREAD_POOL_MAX_NUM=6 NGCMN NGCMX NGC S0C S1C EC OGCMN OGCMX OGC OC MCMN MCMX MC CCSMN CCSMX CCSC YGC FGC 262144.0 262144.0 262144.0 43520.0 43520.0 175104.0 262144.0 262144.0 262144.0 262144.0 0.0 1056768.0 4864.0 0.0 1048576.0 512.0 1 0 S0 S1 E O M CCS YGC YGCT FGC FGCT GCT 0.00 4.23 15.67 0.00 74.01 75.14 3 0.003 0 0.000 0.003 0.00 57.50 28.24 0.39 74.39 75.35 11 0.010 0 0.000 0.010 23.75 0.00 61.83 1.36 74.77 75.35 18 0.016 0 0.000 0.016 45.31 0.00 2.99 1.49 75.53 75.35 26 0.021 0 0.000 0.021 0.00 56.25 71.33 1.60 75.72 75.35 33 0.027 0 0.000 0.027 68.75 0.00 85.08 1.69 75.85 75.35 40 0.033 0 0.000 0.033 53.12 0.00 0.00 2.92 76.10 75.35 48 0.040 0 0.000 0.040 0.00 37.50 13.11 3.34 76.43 75.44 55 0.045 0 0.000 0.045 50.00 0.00 35.20 3.46 76.48 75.44 62 0.049 0 0.000 0.049 0.00 83.33 32.56 3.54 76.51 75.44 69 0.055 0 0.000 0.055 32.81 0.00 45.74 4.08 76.69 75.44 76 0.061 0 0.000 0.061 0.00 20.31 67.75 5.10 77.10 75.44 83 0.066 0 0.000 0.066 0.00 45.83 0.00 5.23 77.30 75.44 91 0.071 0 0.000 0.071 81.25 0.00 0.00 5.32 77.34 75.44 98 0.076 0 0.000 0.076 0.00 18.75 4.97 5.42 77.44 75.44 105 0.080 0 0.000 0.080 26.56 0.00 0.00 6.98 77.44 75.44 112 0.087 0 0.000 0.087 0.00 37.50 43.03 7.13 77.44 75.44 117 0.091 0 0.000 0.091 41.67 0.00 64.72 7.22 77.48 75.44 122 0.094 0 0.000 0.094 0.00 77.08 56.45 7.27 77.74 75.71 127 0.098 0 0.000 0.098 0.00 91.67 16.10 7.36 77.79 75.71 133 0.105 0 0.000 0.105 0.00 70.31 17.80 8.03 77.79 75.71 139 0.109 0 0.000 0.109 34.38 0.00 43.40 8.97 77.79 75.71 144 0.113 0 0.000 0.113 0.00 26.56 72.22 9.06 77.81 75.71 149 0.117 0 0.000 0.117 0.00 47.92 54.32 9.15 77.81 75.71 155 0.121 0 0.000 0.121 0.00 62.50 61.12 9.25 77.81 75.71 161 0.125 0 0.000 0.125 68.75 0.00 61.09 9.31 77.83 75.71 166 0.131 0 0.000 0.131 45.31 0.00 39.81 10.51 77.86 75.71 172 0.135 0 0.000 0.135 26.56 0.00 0.00 10.84 77.86 75.71 178 0.139 0 0.000 0.139 0.00 64.58 0.00 10.90 77.86 75.71 183 0.143 0 0.000 0.143 62.50 0.00 0.00 11.02 77.86 75.71 190 0.147 0 0.000 0.147 0.00 67.19 17.57 11.07 77.86 75.71 195 0.151 0 0.000 0.151 POOL_SETUP_EACH_TIME : true THREAD_POOL_MAX_NUM : 6 THREAD_GROUP_NUM : 100 THREAD_NUM : 10 App.executeThreadsGroup 30566ms App.main 30567ms executor setup count: 100 threads count: 100 threads summary: 30555 MAXスレッド数=(コア数-1) †$ ./start.sh THREAD_POOL_MAX_NUM=5 gradle run -Pargs=THREAD_POOL_MAX_NUM=5 NGCMN NGCMX NGC S0C S1C EC OGCMN OGCMX OGC OC MCMN MCMX MC CCSMN CCSMX CCSC YGC FGC 262144.0 262144.0 262144.0 43520.0 43520.0 175104.0 262144.0 262144.0 262144.0 262144.0 0.0 1056768.0 4480.0 0.0 1048576.0 384.0 0 0 S0 S1 E O M CCS YGC YGCT FGC FGCT GCT 3.38 4.85 0.00 0.00 74.01 75.14 3 0.003 0 0.000 0.003 22.50 0.00 19.31 0.37 74.45 75.35 12 0.012 0 0.000 0.012 0.00 39.06 0.00 0.52 75.35 75.35 21 0.018 0 0.000 0.018 0.00 75.00 41.65 0.63 75.56 75.35 29 0.024 0 0.000 0.024 23.44 0.00 37.88 0.75 75.91 75.35 38 0.030 0 0.000 0.030 0.00 47.92 0.00 0.89 76.19 75.44 47 0.036 0 0.000 0.036 0.00 64.58 31.21 1.00 76.34 75.44 55 0.042 0 0.000 0.042 64.06 0.00 0.00 1.14 76.47 75.44 64 0.048 0 0.000 0.048 21.88 0.00 65.90 1.22 76.54 75.44 72 0.052 0 0.000 0.052 0.00 52.08 0.00 1.37 76.86 75.44 81 0.056 0 0.000 0.056 0.00 70.83 9.66 1.49 76.88 75.44 89 0.061 0 0.000 0.061 0.00 53.12 37.36 1.57 76.92 75.44 95 0.066 0 0.000 0.066 0.00 23.44 8.96 1.64 77.21 75.44 101 0.069 0 0.000 0.069 41.67 0.00 67.82 1.75 77.29 75.44 108 0.073 0 0.000 0.073 60.42 0.00 52.98 1.84 77.34 75.44 114 0.078 0 0.000 0.078 79.17 0.00 51.47 1.92 77.39 75.44 120 0.084 0 0.000 0.084 20.31 0.00 26.53 1.99 77.43 75.44 126 0.089 0 0.000 0.089 35.42 0.00 78.47 2.07 77.45 75.44 133 0.094 0 0.000 0.094 0.00 54.17 4.56 2.19 77.45 75.44 139 0.098 0 0.000 0.098 0.00 72.92 0.00 2.27 77.45 75.44 145 0.102 0 0.000 0.102 0.00 54.69 0.00 2.35 77.67 75.71 151 0.108 0 0.000 0.108 0.00 23.44 21.65 2.42 77.85 75.71 157 0.112 0 0.000 0.112 0.00 43.75 21.83 2.50 77.85 75.71 163 0.116 0 0.000 0.116 0.00 60.42 37.79 2.58 77.92 75.71 169 0.120 0 0.000 0.120 79.17 0.00 14.04 2.70 77.92 75.71 176 0.125 0 0.000 0.125 0.00 53.12 78.88 2.76 77.92 75.71 181 0.128 0 0.000 0.128 23.44 0.00 78.34 2.84 77.92 75.71 188 0.132 0 0.000 0.132 0.00 54.17 0.00 2.96 77.92 75.71 195 0.137 0 0.000 0.137 POOL_SETUP_EACH_TIME : true THREAD_POOL_MAX_NUM : 5 THREAD_GROUP_NUM : 100 THREAD_NUM : 10 App.executeThreadsGroup 27507ms App.main 27507ms executor setup count: 100 threads count: 100 threads summary: 27500 |