Addition Muilt gpus & some logic

ryujaehun · Jan 9, 2019 · 674806e · 674806e
1 parent 3d02706
commit 674806e
Show file tree

Hide file tree

Showing 93 changed files with 731 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ ResNet152, DenseNet121, DenseNet169, DenseNet201, DenseNet161**
 * python3-tk
 * matplotlib
 * pandas
-* PyTorch
+* PyTorch >=0.4
 * torchvision
 
 ### Environment
@@ -58,6 +58,20 @@ ResNet152, DenseNet121, DenseNet169, DenseNet201, DenseNet161**
 * CUDNN version= `7301`
 
 
+
+
+
+### Change Log
+
+* 2019/01/09
+  * PR Update typo (thank for johmathe)
+  * Add requirements.txt
+  * Add result figures
+  * Add ('TkAgg') for cli
+  * Addition Muilt GPUS (DGX-station)
+
+
+
 ### Comparison between networks (single GPU)
 
 Each network is fed with 12 images with 224x224x3 dimensions.
@@ -225,3 +239,80 @@ _I conducted the experiment using two rtx 2080ti._
 ![](fig/device/squeezenet_inference_half.png)
 ##### Double
 ![](fig/device/squeezenet_inference_double.png)
+
+
+### DGX STATION SPEC
+
+| Spec                        | NVIDIA DGX Station                                          |
+|-----------------------------|-------------------------------------------------------------|
+| GPUs                        | 4 x Tesla V100                                              |
+| TFLOPS (GPU FP16)           |  480                                                        |
+| GPU Memory                  | 64 GB total system                                          |
+| CPU                         | 20-Core Intel Xeon E5-2698 v4 2.2 GHz                       |
+| NVIDIA CUDA Cores           | 20,480                                                      |
+| NVIDIA Tensor Cores         | 2,560                                                       |
+| Maximum Power Requirements  | 1,500 W                                                     |
+| System Memory               | 256 GB DDR4 LRDIMM                                          |
+| Storage                     |  4 (data: 3 and OS: 1) x 1.92 TB SSD RAID 0                 |
+| Network                     |  Dual 10 GbE, 4 IB EDR                                      |
+| Display                     | 3X DisplayPort, 4K resolution                               |
+| Acoustics                   | < 35 dB                                                     |
+| Software                    |  Ubuntu Linux Host OSDGX Recommended GPU DriverCUDA Toolkit |
+| System Weight               |  88 lbs / 40 kg                                             |
+| System Dimensions           |  518 D x 256 W x 639 H (mm)                                 |
+| Operating Temperature Range | 10 – 30 °C                                                  |
+
+
+### result
+![](/results/dgx.png)
+
+|        | batchs | gpus | times            |
+|--------|--------|------|------------------|
+| half   | 16     | 1    | 15.6316900253296 |
+| half   | 16     | 2    | 25.2950036525726 |
+| half   | 16     | 3    | 32.5298488140106 |
+| half   | 16     | 4    | 39.5952260494232 |
+| half   | 32     | 1    | 28.9202857017517 |
+| half   | 32     | 2    | 26.9314527511597 |
+| half   | 32     | 3    | 32.6970362663269 |
+| half   | 32     | 4    | 40.0277709960938 |
+| half   | 64     | 1    | 54.6519541740418 |
+| half   | 64     | 2    | 36.9417870044708 |
+| half   | 64     | 3    | 35.1460886001587 |
+| half   | 64     | 4    | 39.9034130573273 |
+| half   | 128    | 1    | 105.689181089401 |
+| half   | 128    | 2    | 62.5697267055512 |
+| half   | 128    | 3    | 50.5970776081085 |
+| half   | 128    | 4    | 45.686126947403  |
+| single | 16     | 1    | 15.7001733779907 |
+| single | 16     | 2    | 25.2602100372314 |
+| single | 16     | 3    | 32.5334632396698 |
+| single | 16     | 4    | 39.9562275409698 |
+| single | 32     | 1    | 29.0114963054657 |
+| single | 32     | 2    | 26.9594860076904 |
+| single | 32     | 3    | 32.7185535430908 |
+| single | 32     | 4    | 39.8312091827393 |
+| single | 64     | 1    | 54.7226464748383 |
+| single | 64     | 2    | 38.2881510257721 |
+| single | 64     | 3    | 35.2633249759674 |
+| single | 64     | 4    | 40.4890751838684 |
+| single | 128    | 1    | 105.767976045609 |
+| single | 128    | 2    | 62.6480567455292 |
+| single | 128    | 3    | 50.3757321834564 |
+| single | 128    | 4    | 45.5866599082947 |
+| double | 16     | 1    | 15.703741312027  |
+| double | 16     | 2    | 25.3219473361969 |
+| double | 16     | 3    | 33.0831336975098 |
+| double | 16     | 4    | 40.441951751709  |
+| double | 32     | 1    | 29.0125107765198 |
+| double | 32     | 2    | 27.3240101337433 |
+| double | 32     | 3    | 33.0090951919556 |
+| double | 32     | 4    | 40.2768909931183 |
+| double | 64     | 1    | 54.7836709022522 |
+| double | 64     | 2    | 36.7958390712738 |
+| double | 64     | 3    | 35.0011682510376 |
+| double | 64     | 4    | 39.9146497249603 |
+| double | 128    | 1    | 105.872387886047 |
+| double | 128    | 2    | 62.9272031784058 |
+| double | 128    | 3    | 48.4100317955017 |
+| double | 128    | 4    | 45.5989670753479 |
diff --git a/__pycache__/info_utils.cpython-36.pyc b/__pycache__/info_utils.cpython-36.pyc
diff --git a/__pycache__/plot.cpython-36.pyc b/__pycache__/plot.cpython-36.pyc
diff --git a/benchmark_models.py b/benchmark_models.py
@@ -9,6 +9,7 @@
 import argparse
 import os
 from plot import *
+
 print_info()
 
 MODEL_LIST = {

diff --git a/fig/1080ti/densenet.png b/fig/1080ti/densenet.png
diff --git a/fig/1080ti/densenet_training.png b/fig/1080ti/densenet_training.png
diff --git a/fig/1080ti/resnet.png b/fig/1080ti/resnet.png
diff --git a/fig/1080ti/resnet_training.png b/fig/1080ti/resnet_training.png
diff --git a/fig/1080ti/squeezenet.png b/fig/1080ti/squeezenet.png
diff --git a/fig/1080ti/squeezenet_training.png b/fig/1080ti/squeezenet_training.png
diff --git a/fig/1080ti/total.png b/fig/1080ti/total.png
diff --git a/fig/1080ti/vgg.png b/fig/1080ti/vgg.png
diff --git a/fig/1080ti/vgg_training.png b/fig/1080ti/vgg_training.png
diff --git a/fig/TITANV/densenet.png b/fig/TITANV/densenet.png
diff --git a/fig/TITANV/densenet_training.png b/fig/TITANV/densenet_training.png
diff --git a/fig/TITANV/resnet.png b/fig/TITANV/resnet.png
diff --git a/fig/TITANV/resnet_training.png b/fig/TITANV/resnet_training.png
diff --git a/fig/TITANV/squeezenet.png b/fig/TITANV/squeezenet.png
diff --git a/fig/TITANV/squeezenet_training.png b/fig/TITANV/squeezenet_training.png
diff --git a/fig/TITANV/total.png b/fig/TITANV/total.png
diff --git a/fig/TITANV/vgg.png b/fig/TITANV/vgg.png
diff --git a/fig/TITANV/vgg_training.png b/fig/TITANV/vgg_training.png
diff --git a/fig/device/densenet_inference_double.png b/fig/device/densenet_inference_double.png
diff --git a/fig/device/densenet_inference_half.png b/fig/device/densenet_inference_half.png
diff --git a/fig/device/densenet_inference_single.png b/fig/device/densenet_inference_single.png
diff --git a/fig/device/densenet_training_double.png b/fig/device/densenet_training_double.png
diff --git a/fig/device/densenet_training_half.png b/fig/device/densenet_training_half.png
diff --git a/fig/device/densenet_training_single.png b/fig/device/densenet_training_single.png
diff --git a/fig/device/resnet_inference_double.png b/fig/device/resnet_inference_double.png
diff --git a/fig/device/resnet_inference_half.png b/fig/device/resnet_inference_half.png
diff --git a/fig/device/resnet_inference_single.png b/fig/device/resnet_inference_single.png
diff --git a/fig/device/resnet_training_double.png b/fig/device/resnet_training_double.png
diff --git a/fig/device/resnet_training_half.png b/fig/device/resnet_training_half.png
diff --git a/fig/device/resnet_training_single.png b/fig/device/resnet_training_single.png
diff --git a/fig/device/squeezenet_inference_double.png b/fig/device/squeezenet_inference_double.png
diff --git a/fig/device/squeezenet_inference_half.png b/fig/device/squeezenet_inference_half.png
diff --git a/fig/device/squeezenet_inference_single.png b/fig/device/squeezenet_inference_single.png
diff --git a/fig/device/squeezenet_training_double.png b/fig/device/squeezenet_training_double.png
diff --git a/fig/device/squeezenet_training_half.png b/fig/device/squeezenet_training_half.png
diff --git a/fig/device/squeezenet_training_single.png b/fig/device/squeezenet_training_single.png
diff --git a/fig/device/vgg_inference_double.png b/fig/device/vgg_inference_double.png
diff --git a/fig/device/vgg_inference_half.png b/fig/device/vgg_inference_half.png
diff --git a/fig/device/vgg_inference_single.png b/fig/device/vgg_inference_single.png
diff --git a/fig/device/vgg_training_double.png b/fig/device/vgg_training_double.png
diff --git a/fig/device/vgg_training_half.png b/fig/device/vgg_training_half.png
diff --git a/fig/device/vgg_training_single.png b/fig/device/vgg_training_single.png
diff --git a/fig/dgx.png b/fig/dgx.png
diff --git a/fig/rtx2080ti(1)/densenet.png b/fig/rtx2080ti(1)/densenet.png
diff --git a/fig/rtx2080ti(1)/densenet_training.png b/fig/rtx2080ti(1)/densenet_training.png
diff --git a/fig/rtx2080ti(1)/resnet.png b/fig/rtx2080ti(1)/resnet.png
diff --git a/fig/rtx2080ti(1)/resnet_training.png b/fig/rtx2080ti(1)/resnet_training.png
diff --git a/fig/rtx2080ti(1)/squeezenet.png b/fig/rtx2080ti(1)/squeezenet.png
diff --git a/fig/rtx2080ti(1)/squeezenet_training.png b/fig/rtx2080ti(1)/squeezenet_training.png
diff --git a/fig/rtx2080ti(1)/total.png b/fig/rtx2080ti(1)/total.png
diff --git a/fig/rtx2080ti(1)/vgg.png b/fig/rtx2080ti(1)/vgg.png
diff --git a/fig/rtx2080ti(1)/vgg_training.png b/fig/rtx2080ti(1)/vgg_training.png
diff --git a/fig/rtx2080ti(2)/densenet.png b/fig/rtx2080ti(2)/densenet.png
diff --git a/fig/rtx2080ti(2)/densenet_training.png b/fig/rtx2080ti(2)/densenet_training.png
diff --git a/fig/rtx2080ti(2)/resnet.png b/fig/rtx2080ti(2)/resnet.png
diff --git a/fig/rtx2080ti(2)/resnet_training.png b/fig/rtx2080ti(2)/resnet_training.png
diff --git a/fig/rtx2080ti(2)/squeezenet.png b/fig/rtx2080ti(2)/squeezenet.png
diff --git a/fig/rtx2080ti(2)/squeezenet_training.png b/fig/rtx2080ti(2)/squeezenet_training.png
diff --git a/fig/rtx2080ti(2)/total.png b/fig/rtx2080ti(2)/total.png
diff --git a/fig/rtx2080ti(2)/vgg.png b/fig/rtx2080ti(2)/vgg.png
diff --git a/fig/rtx2080ti(2)/vgg_training.png b/fig/rtx2080ti(2)/vgg_training.png
diff --git a/muilt_gpus.py b/muilt_gpus.py
@@ -0,0 +1,79 @@
+#! /usr/bin/python3
+import torch.nn as nn
+from torch.autograd import Variable
+from torchvision.models.resnet import resnet50
+import torch
+import time
+import numpy as np
+import argparse
+import pandas as pd
+import matplotlib
+matplotlib.use('TkAgg')
+
+import matplotlib.pyplot as plt
+
+# Inference settings
+parser = argparse.ArgumentParser(description='PyTorch Benchmarking')
+parser.add_argument('--WARM_UP','-w', type=int,default=10, required=False, help="Num of warm up")
+parser.add_argument('--NUM_TEST','-n', type=int,default=200,required=False, help="Num of Test")
+args = parser.parse_args()
+
+torch.backends.cudnn.benchmark = True
+NUM_GPUS = torch.cuda.device_count()
+BATCH_LIST = [2 ** x for x in range(4, 8)]  # 16 to 2048
+precision=[torch.float16,torch.float32,torch.float64] # precision
+
+def main(i):
+    benchmark = {}
+
+    for batch_size in BATCH_LIST:
+        benchmark[batch_size] = []
+        for gpu in range(1, NUM_GPUS + 1):
+            print('Benchmarking  type %s ResNet50 on batch size %i with %i GPUs' % (str(i).split('.')[-1],batch_size, gpu))
+            model = resnet50()
+            if gpu > 1:
+                model = nn.DataParallel(model,device_ids=range(0,gpu))
+            model.cuda()
+            model.eval()
+
+            img = torch.randn(batch_size, 3, 224, 224, device='cuda', requires_grad=False,dtype=torch.float32)
+            durations = []
+            for step in range(args.NUM_TEST + args.WARM_UP):
+                # test
+                torch.cuda.synchronize()
+                start = time.time()
+                model(img)
+                torch.cuda.synchronize()
+                end = time.time()
+                if step >= args.WARM_UP:
+                    duration = (end - start) * 1000
+                    durations.append(duration)
+            benchmark[batch_size].append(durations)
+            del model
+    return benchmark
+
+
+if __name__ == '__main__':
+    result=[]
+for i in precision:
+    result.append(main(i))
+temp=[]
+for bench,dtype in zip(result,['half','single','double']):
+    for key in bench.keys():
+        for gpu, duration in enumerate(bench[key]):
+            print('Data Type %s, Batch size %i, # of GPUs %i, time cost %.4fms' % (dtype,key, gpu + 1, np.mean(duration)))
+            temp.append([dtype,key, gpu + 1, np.mean(duration)])
+
+# save csv
+temp=np.array(temp)
+df=pd.DataFrame(temp[:,1:],index=temp[:,0],columns=['batchs','gpus','times'])
+df.to_csv('results/dgx.csv')
+df=df.astype(np.float16)
+
+
+# save fig
+fig, axes = plt.subplots(nrows=4, ncols=3)
+for i in range(0,len(df),4):
+    df[i:i+4].plot(ax=axes[i//12,i%12//4],figsize=(12,10),x='gpus',y='times',grid=True,kind='bar',title=str(df['batchs'][i])+"batchs "+df.index[i]+" type")
+fig.tight_layout()
+fig.savefig('fig/dgx.png',dpi=600)
diff --git a/plot.py b/plot.py
@@ -2,13 +2,14 @@
 import pandas as pd
 import os
 import glob
+import matplotlib
+matplotlib.use('TkAgg')
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib.ticker import MaxNLocator
 from collections import namedtuple
 
 
-
 def arr_train():
     result_path=os.path.join(os.getcwd(),'results')
     arr_train=[i for i in glob.glob(result_path+'/*training*.csv')]

diff --git a/requirement.txt b/requirement.txt
@@ -0,0 +1,4 @@
+matplotlib
+torchvision
+torch==0.4.1
+pandas
diff --git a/results/GeForce GTX 1080 Ti_double_model_inference_benchmark.csv b/results/GeForce GTX 1080 Ti_double_model_inference_benchmark.csv
@@ -0,0 +1,21 @@
+densenet121,densenet161,densenet169,densenet201,resnet101,resnet152,resnet18,resnet34,resnet50,squeezenet1_0,squeezenet1_1,vgg16,vgg16_bn,vgg19,vgg19_bn
+275.27689933776855,1157.184362411499,328.7699222564697,414.7911071777344,1081.2571048736572,1589.6754264831543,181.33091926574707,390.880823135376,529.9007892608643,111.09209060668945,49.97110366821289,2092.3688411712646,2104.647636413574,2775.4247188568115,2782.4301719665527
+275.191068649292,1157.1948528289795,328.5984992980957,414.75462913513184,1079.8389911651611,1588.7415409088135,181.21886253356934,391.0074234008789,529.2463302612305,111.16266250610352,50.16517639160156,2093.881368637085,2101.701021194458,2775.087833404541,2785.6807708740234
+275.2981185913086,1156.9633483886719,328.68194580078125,414.84785079956055,1080.0518989562988,1589.115858078003,181.09440803527832,390.69151878356934,529.8037528991699,110.84651947021484,50.15707015991211,2096.381425857544,2103.3260822296143,2777.519464492798,2783.987522125244
+275.21777153015137,1157.6392650604248,328.7515640258789,414.9746894836426,1080.1196098327637,1588.4418487548828,180.99617958068848,390.67745208740234,529.4334888458252,110.81051826477051,50.1408576965332,2095.0114727020264,2103.778123855591,2778.05233001709,2783.5707664489746
+275.1765251159668,1156.233549118042,328.58943939208984,415.0364398956299,1080.2056789398193,1589.3497467041016,181.17666244506836,390.6211853027344,529.5617580413818,110.53037643432617,49.70955848693848,2096.644163131714,2102.504014968872,2772.886276245117,2784.9161624908447
+275.20108222961426,1157.2072505950928,328.643798828125,414.8697853088379,1079.789400100708,1589.296817779541,181.1234951019287,390.9306526184082,529.210090637207,111.18936538696289,50.02307891845703,2094.830274581909,2104.4046878814697,2775.226831436157,2785.510540008545
+275.2645015716553,1157.7191352844238,328.5636901855469,414.86430168151855,1079.9520015716553,1589.4649028778076,181.16021156311035,390.8696174621582,529.3786525726318,110.82911491394043,50.01211166381836,2093.9769744873047,2101.332664489746,2778.5086631774902,2785.057544708252
+275.2187252044678,1156.9101810455322,328.66954803466797,414.9026870727539,1080.540657043457,1589.9295806884766,181.13255500793457,390.6412124633789,529.6840667724609,110.68010330200195,50.02331733703613,2095.735788345337,2102.3287773132324,2777.4722576141357,2780.8704376220703
+275.15435218811035,1156.980037689209,328.55725288391113,414.9818420410156,1079.7145366668701,1589.2047882080078,181.10966682434082,390.77019691467285,530.0579071044922,110.40067672729492,50.19211769104004,2094.186782836914,2103.314161300659,2772.8872299194336,2782.503843307495
+275.1944065093994,1157.5899124145508,328.5846710205078,414.9816036224365,1080.2981853485107,1588.7458324432373,181.13279342651367,390.6581401824951,529.961347579956,111.27567291259766,50.02140998840332,2096.4760780334473,2100.987672805786,2772.8123664855957,2784.7609519958496
+275.2494812011719,1157.51051902771,328.61804962158203,414.823055267334,1080.4884433746338,1589.3886089324951,181.16426467895508,390.73848724365234,529.7136306762695,110.98742485046387,50.07624626159668,2095.710515975952,2101.6435623168945,2774.803638458252,2786.098003387451
+275.221586227417,1157.2282314300537,328.6285400390625,414.84880447387695,1080.2576541900635,1588.8161659240723,181.16283416748047,390.6717300415039,529.6895503997803,111.09519004821777,49.91626739501953,2094.292402267456,2103.2872200012207,2778.792142868042,2781.2345027923584
+275.2540111541748,1156.6588878631592,328.66644859313965,414.9057865142822,1080.5060863494873,1589.1194343566895,181.15925788879395,390.90967178344727,529.7524929046631,110.36109924316406,49.78775978088379,2096.041679382324,2103.2004356384277,2775.458812713623,2783.2419872283936
+275.38466453552246,1157.53173828125,328.54413986206055,415.13609886169434,1080.5070400238037,1588.8521671295166,181.1356544494629,390.69223403930664,529.4234752655029,111.07540130615234,50.04715919494629,2094.2089557647705,2105.8895587921143,2774.0325927734375,2788.1064414978027
+275.1154899597168,1156.1098098754883,328.6154270172119,415.01617431640625,1079.9245834350586,1589.123010635376,181.2601089477539,390.81406593322754,529.9923419952393,110.69869995117188,50.09150505065918,2094.2142009735107,2103.8155555725098,2773.9779949188232,2783.3642959594727
+275.27689933776855,1155.8349132537842,328.57418060302734,415.03310203552246,1079.683780670166,1589.2608165740967,181.15472793579102,390.66100120544434,529.7553539276123,111.04297637939453,49.91936683654785,2095.4086780548096,2099.839210510254,2778.4059047698975,2785.635471343994
+275.03204345703125,1157.4089527130127,328.6454677581787,415.01832008361816,1080.277919769287,1589.2224311828613,181.31446838378906,390.93542098999023,529.8240184783936,110.81457138061523,49.54838752746582,2095.277786254883,2104.121685028076,2776.404619216919,2783.728837966919
+275.12645721435547,1156.2864780426025,328.7689685821533,415.04883766174316,1080.4767608642578,1588.862419128418,181.17356300354004,390.8078670501709,529.4899940490723,111.16456985473633,49.57866668701172,2094.4089889526367,2104.85577583313,2776.249885559082,2783.2164764404297
+275.3021717071533,1158.3847999572754,328.54247093200684,414.9212837219238,1080.6753635406494,1588.937520980835,181.29992485046387,391.04270935058594,529.2887687683105,111.2966537475586,49.64327812194824,2092.952251434326,2102.637529373169,2775.989294052124,2784.698724746704
+275.18367767333984,1157.240629196167,328.61971855163574,414.95585441589355,1079.9553394317627,1589.0295505523682,181.15234375,391.0396099090576,529.6313762664795,110.69178581237793,50.202369689941406,2093.963384628296,2102.959156036377,2777.5652408599854,2782.874822616577
diff --git a/results/GeForce GTX 1080 Ti_double_model_training_benchmark.csv b/results/GeForce GTX 1080 Ti_double_model_training_benchmark.csv
@@ -0,0 +1,21 @@
+densenet121,densenet161,densenet169,densenet201,resnet101,resnet152,resnet18,resnet34,resnet50,squeezenet1_0,squeezenet1_1,vgg16,vgg16_bn,vgg19,vgg19_bn
+779.5779705047607,2521.775007247925,940.2995109558105,1196.3605880737305,2412.79935836792,3548.431873321533,463.1688594818115,969.4628715515137,1218.0991172790527,262.71653175354004,131.730318069458,4227.349042892456,4274.938344955444,5474.944829940796,5520.171880722046
+779.5536518096924,2522.479295730591,940.0417804718018,1197.000503540039,2409.986972808838,3547.506093978882,463.2539749145508,969.973087310791,1215.238094329834,260.2832317352295,131.7312717437744,4224.5774269104,4275.1829624176025,5476.2890338897705,5523.429393768311
+779.5064449310303,2521.4219093322754,940.3128623962402,1196.840524673462,2410.5119705200195,3545.8383560180664,462.89992332458496,969.8486328125,1217.7934646606445,260.8153820037842,131.52432441711426,4225.840091705322,4270.016670227051,5476.691484451294,5518.359422683716
+779.8304557800293,2522.575616836548,940.2890205383301,1196.6004371643066,2412.243604660034,3546.654224395752,463.0846977233887,969.6764945983887,1217.2226905822754,260.481595993042,131.5619945526123,4228.270053863525,4272.846221923828,5475.972414016724,5522.813558578491
+779.3407440185547,2522.094964981079,940.248966217041,1197.0233917236328,2410.402297973633,3546.1065769195557,463.05012702941895,969.5048332214355,1217.2229290008545,258.76760482788086,131.52408599853516,4228.338956832886,4272.484302520752,5473.5307693481445,5525.737524032593
+779.1059017181396,2523.0553150177,940.964937210083,1196.3491439819336,2408.9221954345703,3548.2263565063477,463.28020095825195,969.8011875152588,1217.5500392913818,260.12253761291504,131.40416145324707,4228.645801544189,4272.087335586548,5478.617191314697,5521.843910217285
+779.0071964263916,2521.796226501465,940.6938552856445,1196.9048976898193,2410.423994064331,3545.429468154907,464.0352725982666,969.7577953338623,1217.2696590423584,260.235071182251,131.75582885742188,4225.056171417236,4274.237155914307,5479.026079177856,5521.048307418823
+779.8733711242676,2522.1657752990723,940.3927326202393,1196.8860626220703,2407.7351093292236,3547.0783710479736,463.1063938140869,969.6173667907715,1215.6291007995605,258.8462829589844,131.64806365966797,4229.239463806152,4270.251274108887,5472.555637359619,5525.039196014404
+779.7191143035889,2520.9436416625977,939.8624897003174,1196.8843936920166,2412.1506214141846,3548.981189727783,463.3023738861084,969.318151473999,1217.2956466674805,259.6909999847412,131.5314769744873,4228.9934158325195,4271.765947341919,5474.7607707977295,5521.821737289429
+779.5631885528564,2522.4084854125977,940.4158592224121,1196.986436843872,2408.856153488159,3547.0223426818848,463.1333351135254,971.0109233856201,1214.547872543335,259.22465324401855,131.3626766204834,4227.1058559417725,4270.457744598389,5476.619482040405,5521.64101600647
+779.9177169799805,2521.0673809051514,940.6344890594482,1196.805477142334,2410.522222518921,3548.4395027160645,464.3669128417969,969.9065685272217,1217.968463897705,258.71777534484863,131.62636756896973,4225.007772445679,4272.768020629883,5476.911306381226,5520.698547363281
+779.5429229736328,2521.958827972412,940.3390884399414,1197.04008102417,2411.906957626343,3547.405958175659,463.4673595428467,969.7439670562744,1217.7469730377197,257.39097595214844,131.58750534057617,4227.298974990845,4272.3047733306885,5476.249694824219,5520.978212356567
+779.3474197387695,2521.4664936065674,940.253496170044,1197.387933731079,2410.207748413086,3547.3873615264893,462.9251956939697,970.0770378112793,1216.9926166534424,260.8499526977539,131.5755844116211,4229.658603668213,4273.0677127838135,5473.834991455078,5521.616697311401
+780.2989482879639,2523.251533508301,940.1402473449707,1196.7973709106445,2410.334348678589,3547.499179840088,463.1781578063965,969.7866439819336,1217.7138328552246,258.58211517333984,132.02142715454102,4226.225852966309,4270.529747009277,5473.350524902344,5523.295640945435
+779.1705131530762,2521.547555923462,940.9699440002441,1196.9633102416992,2410.5658531188965,3545.0515747070312,463.12808990478516,970.8940982818604,1216.5451049804688,260.47253608703613,131.53910636901855,4225.259065628052,4269.794225692749,5473.800897598267,5521.015167236328
+779.3745994567871,2523.23317527771,940.4432773590088,1196.6500282287598,2410.2425575256348,3544.889211654663,463.26422691345215,970.1576232910156,1215.6078815460205,260.8513832092285,131.27684593200684,4227.103471755981,4272.888660430908,5473.813772201538,5522.21155166626
+779.3848514556885,2522.357225418091,940.7253265380859,1196.789264678955,2410.839796066284,3546.975612640381,462.9967212677002,969.7983264923096,1216.3732051849365,259.32836532592773,131.74724578857422,4225.943565368652,4268.872499465942,5476.525068283081,5523.60725402832
+779.8871994018555,2522.1152305603027,940.579891204834,1196.6793537139893,2410.1028442382812,3547.189950942993,462.9819393157959,969.9103832244873,1217.6969051361084,259.89747047424316,131.56986236572266,4230.077266693115,4271.387338638306,5477.8852462768555,5521.852493286133
+779.7486782073975,2522.857427597046,940.6981468200684,1196.3670253753662,2411.1685752868652,3546.626091003418,464.30349349975586,971.1790084838867,1217.5266742706299,260.64395904541016,131.03652000427246,4227.705001831055,4270.67494392395,5474.617481231689,5522.245168685913
+778.8183689117432,2522.3746299743652,940.6757354736328,1196.5951919555664,2409.12127494812,3546.8196868896484,463.18697929382324,969.7632789611816,1217.0171737670898,260.4029178619385,131.3035488128662,4226.501226425171,4272.454738616943,5473.072528839111,5522.516965866089