Skip to content

Commit adae90b

Browse files
Coderx7Coderx7
committed
update results and add benchmark artifacts
-- updated the results-imagenet.csv to contain latest results of simplenet variants so far -- added the benchmark results for inference with fp32, NHWC for both pytorch 1.10 and 1.11 for better comparison, accuracies have been added to the benchmark results. The hardware and software stack used to run benchmark is as follows: OS: Ubuntu 20.04.4 kernel version: 5.13.0-51-generic Driver version: 515.86.01 Python version: 3.9.7 (anaconda installation) GPU: GTX1080 CPU: 4790K RAM: 32Gig
1 parent 67fc2e7 commit adae90b

19 files changed

+957
-533
lines changed

‎ImageNet/training_scripts/imagenet_training/model_list_normal.txt

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
mobilenetv3_rw
2+
tf_mobilenetv3_large_100
3+
mobilenetv2_100
4+
tf_mobilenetv3_large_minimal_100
5+
mobilenetv2_110d
6+
mobilenetv3_large_100
7+
tf_mobilenetv3_large_075
8+
efficientnet_lite0
9+
tf_efficientnet_lite0
10+
densenet121
11+
tv_densenet121
12+
mnasnet_100
13+
dla34
14+
tinynet_b
15+
tf_mixnet_s
16+
ghostnet_100
17+
crossvit_9_240
18+
regnetx_006
19+
vit_base_patch32_224_sam
20+
resnest14d
21+
tv_resnet34
22+
swsl_resnet18
23+
resnet26
24+
resnet34
25+
legacy_seresnet18
26+
resnet18
27+
gluon_resnet18_v1b
28+
resnet18d
29+
deit_tiny_patch16_224
30+
mixer_l16_224
31+
vit_tiny_r_s16_p8_224
32+
repvgg_b0
33+
vgg13_bn
34+
vgg16
35+
vgg11_bn
36+
vgg13
37+
vgg11
38+
vgg19
39+
vgg16_bn
40+
vgg19_bn
41+
simplenetv1_5m_m1
42+
simplenetv1_5m_m2
43+
simplenetv1_9m_m1
44+
simplenetv1_9m_m2
45+
simplenetv1_small_m1_075
46+
simplenetv1_small_m2_075

‎ImageNet/training_scripts/imagenet_training/model_list_normal_known.txt

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
simplenetv1_5m_m1
2+
simplenetv1_5m_m2
3+
simplenetv1_9m_m1
4+
simplenetv1_9m_m2
5+
simplenetv1_small_m1_075
6+
simplenetv1_small_m2_075
7+
mobilenetv3_large_100
8+
mobilenetv2_100
9+
densenet121
10+
tf_mobilenetv3_large_100
11+
efficientnet_lite0
12+
resnet26
13+
resnet34
14+
mobilenetv2_110d
15+
tinynet_b
16+
tf_efficientnet_lite0
17+
mnasnet_100
18+
dla34
19+
ghostnet_100
20+
crossvit_9_240
21+
regnetx_006
22+
vit_base_patch32_224_sam
23+
tf_mobilenetv3_large_075
24+
tf_mobilenetv3_large_minimal_100
25+
deit_tiny_patch16_224
26+
vit_tiny_r_s16_p8_224
27+
repvgg_b0
28+
vgg19_bn
29+
vgg19
30+
vgg13_bn
31+
vgg16_bn
32+
vgg16
33+
vgg11_bn
34+
vgg13
35+
vgg11
36+
resnet18
37+

‎ImageNet/training_scripts/imagenet_training/model_list_small.txt

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
tf_mobilenetv3_small_100
2+
dla60x_c
3+
mobilenetv3_small_100
4+
mnasnet_small
5+
dla46x_c
6+
mobilenetv2_050
7+
tf_mobilenetv3_small_075
8+
mobilenetv3_small_075
9+
dla46_c
10+
lcnet_050
11+
tf_mobilenetv3_small_minimal_100
12+
mobilenetv3_small_050
13+
simplenetv1_small_m1_05
14+
simplenetv1_small_m2_05
15+
simplenetv1_small_m1_075
16+
simplenetv1_small_m2_075

‎ImageNet/training_scripts/imagenet_training/results/add_acc_to_benchmarks.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import pandas as pd
2+
import argparse
3+
4+
parser = argparse.ArgumentParser(description="A small utility to merge model accuracy with timm benchmarks")
5+
parser.add_argument(
6+
"--imagenet-results",
7+
default="./results-imagenet.csv",
8+
type=str,
9+
metavar="FILENAME",
10+
help="the imagenet results csv file to get the accuracies from",
11+
)
12+
parser.add_argument(
13+
"--bench-csv",
14+
default="./benchmark_inference_GTX1080_fp32_small_torch1.10.csv",
15+
type=str,
16+
metavar="FILENAME",
17+
help="the csv file for which you want to add accuracy",
18+
)
19+
20+
21+
def add_acc_to_csv(imagenet_results, csv_filename):
22+
df_imagenet_results = pd.read_csv(imagenet_results)
23+
df_imagenet_accs = df_imagenet_results[["model", "top1", "top5"]]
24+
df_csv = pd.read_csv(csv_filename)
25+
df_csv_acc = pd.merge(df_csv, df_imagenet_accs, on=["model"])
26+
df_csv_acc.to_csv(csv_filename.replace(".csv", "_with_accuracy.csv"), index=False)
27+
print(f"{csv_filename} is done")
28+
29+
30+
if __name__ == "__main__":
31+
args = parser.parse_args()
32+
add_acc_to_csv(args.imagenet_results, args.bench_csv)

‎ImageNet/training_scripts/imagenet_training/results/benchmark_inference_GTX1080.csv

-119
This file was deleted.

‎ImageNet/training_scripts/imagenet_training/results/benchmark_inference_GTX1080_fp32_normal_known_torch1.10.csv

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
model,infer_samples_per_sec,infer_step_time,infer_batch_size,infer_img_size,infer_gmacs,infer_macts,param_count
2+
vit_tiny_r_s16_p8_224,1809.8,141.431,256,224,0.44,2.06,6.34
3+
simplenetv1_small_m1_075,1498.12,170.855,256,224,0.83,1.56,3.29
4+
simplenetv1_small_m2_075,1222.18,209.433,256,224,1.02,1.79,3.29
5+
simplenetv1_5m_m1,1034.1,247.529,256,224,1.46,2.09,5.75
6+
deit_tiny_patch16_224,910.68,281.08,256,224,1.26,5.97,5.72
7+
resnet18,832.07,307.634,256,224,1.82,2.48,11.69
8+
simplenetv1_5m_m2,818.45,312.755,256,224,1.81,2.39,5.75
9+
vit_base_patch32_224_sam,550.96,464.615,256,224,4.41,5.01,88.22
10+
crossvit_9_240,540.26,473.812,256,240,1.85,9.52,8.55
11+
tinynet_b,530.52,482.515,256,188,0.21,4.44,3.73
12+
resnet26,519.5,492.742,256,224,2.36,7.35,16.0
13+
tf_mobilenetv3_large_075,505.34,506.555,256,224,0.16,4.0,3.99
14+
regnetx_006,475.48,538.373,256,224,0.61,3.98,6.2
15+
resnet34,456.22,561.098,256,224,3.67,3.74,21.8
16+
simplenetv1_9m_m1,455.52,561.959,256,224,2.96,3.41,9.51
17+
dla34,441.47,579.845,256,224,3.07,5.02,15.74
18+
repvgg_b0,434.24,589.509,256,224,3.41,6.15,15.82
19+
ghostnet_100,406.6,629.583,256,224,0.15,3.55,5.18
20+
tf_mobilenetv3_large_minimal_100,406.07,630.397,256,224,0.22,4.4,3.92
21+
mobilenetv3_large_100,399.88,640.158,256,224,0.23,4.41,5.48
22+
tf_mobilenetv3_large_100,387.42,660.742,256,224,0.23,4.41,5.48
23+
simplenetv1_9m_m2,387.08,661.332,256,224,3.74,3.86,9.51
24+
mobilenetv2_100,294.32,869.767,256,224,0.31,6.68,3.5
25+
densenet121,271.19,943.952,256,224,2.87,6.9,7.98
26+
vgg11,265.22,965.196,256,224,7.61,7.44,132.86
27+
mnasnet_100,261.47,979.059,256,224,0.33,5.46,4.38
28+
vgg11_bn,252.21,507.471,128,224,7.62,7.44,132.87
29+
mobilenetv2_110d,230.59,1110.181,256,224,0.45,8.71,4.52
30+
efficientnet_lite0,223.51,1145.336,256,224,0.4,6.74,4.65
31+
tf_efficientnet_lite0,219.46,1166.486,256,224,0.4,6.74,4.65
32+
vgg13,140.34,912.059,128,224,11.31,12.25,133.05
33+
vgg13_bn,132.22,968.059,128,224,11.33,12.25,133.05
34+
vgg16,115.5,1108.21,128,224,15.47,13.56,138.36
35+
vgg16_bn,109.38,1170.163,128,224,15.5,13.56,138.37
36+
vgg19,98.14,1304.284,128,224,19.63,14.86,143.67
37+
vgg19_bn,93.53,1368.463,128,224,19.66,14.86,143.68

‎ImageNet/training_scripts/imagenet_training/results/benchmark_inference_GTX1080_fp32_normal_known_torch1.10_with_accuracy.csv

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
model,infer_samples_per_sec,infer_step_time,infer_batch_size,infer_img_size,infer_gmacs,infer_macts,param_count,top1,top5
2+
vit_tiny_r_s16_p8_224,1809.8,141.431,256,224,0.44,2.06,6.34,71.792,90.822
3+
simplenetv1_small_m1_075,1498.12,170.855,256,224,0.83,1.56,3.29,67.764,87.66
4+
simplenetv1_small_m2_075,1222.18,209.433,256,224,1.02,1.79,3.29,68.15,87.762
5+
simplenetv1_5m_m1,1034.1,247.529,256,224,1.46,2.09,5.75,71.37,90.1
6+
deit_tiny_patch16_224,910.68,281.08,256,224,1.26,5.97,5.72,72.172,91.114
7+
resnet18,832.07,307.634,256,224,1.82,2.48,11.69,69.744,89.082
8+
simplenetv1_5m_m2,818.45,312.755,256,224,1.81,2.39,5.75,71.936,90.3
9+
vit_base_patch32_224_sam,550.96,464.615,256,224,4.41,5.01,88.22,73.694,91.01
10+
crossvit_9_240,540.26,473.812,256,240,1.85,9.52,8.55,73.96,91.968
11+
tinynet_b,530.52,482.515,256,188,0.21,4.44,3.73,74.976,92.184
12+
resnet26,519.5,492.742,256,224,2.36,7.35,16.0,75.3,92.578
13+
tf_mobilenetv3_large_075,505.34,506.555,256,224,0.16,4.0,3.99,73.436,91.344
14+
regnetx_006,475.48,538.373,256,224,0.61,3.98,6.2,73.86,91.672
15+
resnet34,456.22,561.098,256,224,3.67,3.74,21.8,75.114,92.284
16+
simplenetv1_9m_m1,455.52,561.959,256,224,2.96,3.41,9.51,73.376,91.048
17+
dla34,441.47,579.845,256,224,3.07,5.02,15.74,74.62,92.072
18+
repvgg_b0,434.24,589.509,256,224,3.41,6.15,15.82,75.16,92.418
19+
ghostnet_100,406.6,629.583,256,224,0.15,3.55,5.18,73.974,91.46
20+
tf_mobilenetv3_large_minimal_100,406.07,630.397,256,224,0.22,4.4,3.92,72.25,90.63
21+
mobilenetv3_large_100,399.88,640.158,256,224,0.23,4.41,5.48,75.766,92.544
22+
tf_mobilenetv3_large_100,387.42,660.742,256,224,0.23,4.41,5.48,75.518,92.604
23+
simplenetv1_9m_m2,387.08,661.332,256,224,3.74,3.86,9.51,74.17,91.614
24+
mobilenetv2_100,294.32,869.767,256,224,0.31,6.68,3.5,72.97,91.02
25+
densenet121,271.19,943.952,256,224,2.87,6.9,7.98,75.584,92.652
26+
vgg11,265.22,965.196,256,224,7.61,7.44,132.86,69.028,88.626
27+
mnasnet_100,261.47,979.059,256,224,0.33,5.46,4.38,74.658,92.112
28+
vgg11_bn,252.21,507.471,128,224,7.62,7.44,132.87,70.36,89.802
29+
mobilenetv2_110d,230.59,1110.181,256,224,0.45,8.71,4.52,75.038,92.184
30+
efficientnet_lite0,223.51,1145.336,256,224,0.4,6.74,4.65,75.476,92.512
31+
tf_efficientnet_lite0,219.46,1166.486,256,224,0.4,6.74,4.65,74.832,92.174
32+
vgg13,140.34,912.059,128,224,11.31,12.25,133.05,69.926,89.246
33+
vgg13_bn,132.22,968.059,128,224,11.33,12.25,133.05,71.594,90.376
34+
vgg16,115.5,1108.21,128,224,15.47,13.56,138.36,71.59,90.382
35+
vgg16_bn,109.38,1170.163,128,224,15.5,13.56,138.37,73.35,91.504
36+
vgg19,98.14,1304.284,128,224,19.63,14.86,143.67,72.366,90.87
37+
vgg19_bn,93.53,1368.463,128,224,19.66,14.86,143.68,74.214,91.848

‎ImageNet/training_scripts/imagenet_training/results/benchmark_inference_GTX1080_fp32_normal_known_torch1.11.csv

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
model,infer_samples_per_sec,infer_step_time,infer_batch_size,infer_img_size,infer_gmacs,infer_macts,param_count
2+
vit_tiny_r_s16_p8_224,1882.23,135.988,256,224,0.44,2.06,6.34
3+
simplenetv1_small_m1_075,1516.74,168.762,256,224,0.83,1.56,3.29
4+
simplenetv1_small_m2_075,1260.89,203.01,256,224,1.02,1.79,3.29
5+
simplenetv1_5m_m1,1107.7,231.088,256,224,1.46,2.09,5.75
6+
deit_tiny_patch16_224,991.41,258.198,256,224,1.26,5.97,5.72
7+
resnet18,876.92,291.907,256,224,1.82,2.48,11.69
8+
simplenetv1_5m_m2,835.17,306.502,256,224,1.81,2.39,5.75
9+
crossvit_9_240,602.13,425.137,256,240,1.85,9.52,8.55
10+
vit_base_patch32_224_sam,571.37,448.024,256,224,4.41,5.01,88.22
11+
tinynet_b,530.15,482.86,256,188,0.21,4.44,3.73
12+
resnet26,524.36,488.193,256,224,2.36,7.35,16.0
13+
tf_mobilenetv3_large_075,505.13,506.778,256,224,0.16,4.0,3.99
14+
resnet34,491.96,520.334,256,224,3.67,3.74,21.8
15+
regnetx_006,478.41,535.075,256,224,0.61,3.98,6.2
16+
dla34,472.49,541.773,256,224,3.07,5.02,15.74
17+
simplenetv1_9m_m1,459.21,557.458,256,224,2.96,3.41,9.51
18+
repvgg_b0,455.36,562.169,256,224,3.41,6.15,15.82
19+
ghostnet_100,407.03,628.922,256,224,0.15,3.55,5.18
20+
tf_mobilenetv3_large_minimal_100,406.84,629.211,256,224,0.22,4.4,3.92
21+
mobilenetv3_large_100,402.08,636.663,256,224,0.23,4.41,5.48
22+
simplenetv1_9m_m2,389.94,656.492,256,224,3.74,3.86,9.51
23+
tf_mobilenetv3_large_100,388.3,659.264,256,224,0.23,4.41,5.48
24+
mobilenetv2_100,295.68,865.772,256,224,0.31,6.68,3.5
25+
densenet121,293.94,870.881,256,224,2.87,6.9,7.98
26+
mnasnet_100,262.25,976.131,256,224,0.33,5.46,4.38
27+
vgg11,260.38,983.145,256,224,7.61,7.44,132.86
28+
vgg11_bn,248.92,514.193,128,224,7.62,7.44,132.87
29+
mobilenetv2_110d,230.8,1109.144,256,224,0.45,8.71,4.52
30+
efficientnet_lite0,224.81,1138.729,256,224,0.4,6.74,4.65
31+
tf_efficientnet_lite0,219.93,1163.953,256,224,0.4,6.74,4.65
32+
vgg13,154.03,830.996,128,224,11.31,12.25,133.05
33+
vgg13_bn,144.39,886.483,128,224,11.33,12.25,133.05
34+
vgg16,123.7,1034.687,128,224,15.47,13.56,138.36
35+
vgg16_bn,117.06,1093.467,128,224,15.5,13.56,138.37
36+
vgg19,103.71,1234.193,128,224,19.63,14.86,143.67
37+
vgg19_bn,98.59,1298.317,128,224,19.66,14.86,143.68

0 commit comments

Comments
 (0)