forked from scikit-learn/scikit-learn
-
Notifications
You must be signed in to change notification settings - Fork 6
/
bench_mnist.py
230 lines (201 loc) · 6.83 KB
/
bench_mnist.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
"""
=======================
MNIST dataset benchmark
=======================
Benchmark on the MNIST dataset. The dataset comprises 70,000 samples
and 784 features. Here, we consider the task of predicting
10 classes - digits from 0 to 9 from their raw images. By contrast to the
covertype dataset, the feature space is homogeneous.
Example of output :
[..]
Classification performance:
===========================
Classifier train-time test-time error-rate
------------------------------------------------------------
MLP_adam 53.46s 0.11s 0.0224
Nystroem-SVM 112.97s 0.92s 0.0228
MultilayerPerceptron 24.33s 0.14s 0.0287
ExtraTrees 42.99s 0.57s 0.0294
RandomForest 42.70s 0.49s 0.0318
SampledRBF-SVM 135.81s 0.56s 0.0486
LinearRegression-SAG 16.67s 0.06s 0.0824
CART 20.69s 0.02s 0.1219
dummy 0.00s 0.01s 0.8973
"""
# Author: Issam H. Laradji
# Arnaud Joly <[email protected]>
# License: BSD 3 clause
import argparse
import os
from time import time
import numpy as np
from joblib import Memory
from sklearn.datasets import fetch_openml, get_data_home
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier
from sklearn.kernel_approximation import Nystroem, RBFSampler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import zero_one_loss
from sklearn.neural_network import MLPClassifier
from sklearn.pipeline import make_pipeline
from sklearn.svm import LinearSVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.utils import check_array
# Memoize the data extraction and memory map the resulting
# train / test splits in readonly mode
memory = Memory(os.path.join(get_data_home(), "mnist_benchmark_data"), mmap_mode="r")
@memory.cache
def load_data(dtype=np.float32, order="F"):
"""Load the data, then cache and memmap the train/test split"""
######################################################################
# Load dataset
print("Loading dataset...")
data = fetch_openml("mnist_784", as_frame=True, parser="pandas")
X = check_array(data["data"], dtype=dtype, order=order)
y = data["target"]
# Normalize features
X = X / 255
# Create train-test split (as [Joachims, 2006])
print("Creating train-test split...")
n_train = 60000
X_train = X[:n_train]
y_train = y[:n_train]
X_test = X[n_train:]
y_test = y[n_train:]
return X_train, X_test, y_train, y_test
ESTIMATORS = {
"dummy": DummyClassifier(),
"CART": DecisionTreeClassifier(),
"ExtraTrees": ExtraTreesClassifier(),
"RandomForest": RandomForestClassifier(),
"Nystroem-SVM": make_pipeline(
Nystroem(gamma=0.015, n_components=1000), LinearSVC(C=100, dual="auto")
),
"SampledRBF-SVM": make_pipeline(
RBFSampler(gamma=0.015, n_components=1000), LinearSVC(C=100, dual="auto")
),
"LogisticRegression-SAG": LogisticRegression(solver="sag", tol=1e-1, C=1e4),
"LogisticRegression-SAGA": LogisticRegression(solver="saga", tol=1e-1, C=1e4),
"MultilayerPerceptron": MLPClassifier(
hidden_layer_sizes=(100, 100),
max_iter=400,
alpha=1e-4,
solver="sgd",
learning_rate_init=0.2,
momentum=0.9,
verbose=1,
tol=1e-4,
random_state=1,
),
"MLP-adam": MLPClassifier(
hidden_layer_sizes=(100, 100),
max_iter=400,
alpha=1e-4,
solver="adam",
learning_rate_init=0.001,
verbose=1,
tol=1e-4,
random_state=1,
),
}
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--classifiers",
nargs="+",
choices=ESTIMATORS,
type=str,
default=["ExtraTrees", "Nystroem-SVM"],
help="list of classifiers to benchmark.",
)
parser.add_argument(
"--n-jobs",
nargs="?",
default=1,
type=int,
help=(
"Number of concurrently running workers for "
"models that support parallelism."
),
)
parser.add_argument(
"--order",
nargs="?",
default="C",
type=str,
choices=["F", "C"],
help="Allow to choose between fortran and C ordered data",
)
parser.add_argument(
"--random-seed",
nargs="?",
default=0,
type=int,
help="Common seed used by random number generator.",
)
args = vars(parser.parse_args())
print(__doc__)
X_train, X_test, y_train, y_test = load_data(order=args["order"])
print("")
print("Dataset statistics:")
print("===================")
print("%s %d" % ("number of features:".ljust(25), X_train.shape[1]))
print("%s %d" % ("number of classes:".ljust(25), np.unique(y_train).size))
print("%s %s" % ("data type:".ljust(25), X_train.dtype))
print(
"%s %d (size=%dMB)"
% (
"number of train samples:".ljust(25),
X_train.shape[0],
int(X_train.nbytes / 1e6),
)
)
print(
"%s %d (size=%dMB)"
% (
"number of test samples:".ljust(25),
X_test.shape[0],
int(X_test.nbytes / 1e6),
)
)
print()
print("Training Classifiers")
print("====================")
error, train_time, test_time = {}, {}, {}
for name in sorted(args["classifiers"]):
print("Training %s ... " % name, end="")
estimator = ESTIMATORS[name]
estimator_params = estimator.get_params()
estimator.set_params(
**{
p: args["random_seed"]
for p in estimator_params
if p.endswith("random_state")
}
)
if "n_jobs" in estimator_params:
estimator.set_params(n_jobs=args["n_jobs"])
time_start = time()
estimator.fit(X_train, y_train)
train_time[name] = time() - time_start
time_start = time()
y_pred = estimator.predict(X_test)
test_time[name] = time() - time_start
error[name] = zero_one_loss(y_test, y_pred)
print("done")
print()
print("Classification performance:")
print("===========================")
print(
"{0: <24} {1: >10} {2: >11} {3: >12}".format(
"Classifier ", "train-time", "test-time", "error-rate"
)
)
print("-" * 60)
for name in sorted(args["classifiers"], key=error.get):
print(
"{0: <23} {1: >10.2f}s {2: >10.2f}s {3: >12.4f}".format(
name, train_time[name], test_time[name], error[name]
)
)
print()