Experimenting with optimizers – Orbifold Consulting

    import numpy as np
    import pandas as pd

    from sklearn.model_selection import train_test_split
    from keras.models import Sequential
    from keras.layers import Dense, Dropout
    from keras.callbacks import EarlyStopping, ModelCheckpoint
    from keras.optimizers import SGD, Adadelta, Adam, RMSprop, Adagrad, Nadam, Adamax

    SEED = 2017

Using TensorFlow backend.



    # Data can be downloaded at https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv


    data = pd.read_csv('winequality-red.csv', sep=';')
    y = data['quality']
    X = data.drop(['quality'], axis=1)


    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=SEED)
    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=SEED)


    def create_model(opt):
        model = Sequential()
        model.add(Dense(100, input_dim=X_train.shape[1],
        activation='relu'))
        model.add(Dense(50, activation='relu'))
        model.add(Dense(25, activation='relu'))
        model.add(Dense(10, activation='relu'))
        model.add(Dense(1, activation='linear'))
        return model


    def create_callbacks(opt):
        callbacks = [
        EarlyStopping(monitor='val_acc', patience=200, verbose=2),
        ModelCheckpoint('/Users/swa/Desktop/temp/checkpoints/optimizers_best_' + opt + '.h5', monitor='val_acc', save_best_only=True, verbose=0)
        ]
        return callbacks


    opts = dict({
        'sgd': SGD(),
         'sgd-0001': SGD(lr=0.0001, decay=0.00001),
         'adam': Adam(),
         'adadelta': Adadelta(),
         'rmsprop': RMSprop(),
         'rmsprop-0001': RMSprop(lr=0.0001),
         'nadam': Nadam(),
         'adamax': Adamax()
        })


    batch_size = 128
    n_epochs = 1000

    results = []
    # Loop through the optimizers
    for opt in opts:
        model = create_model(opt)
        callbacks = create_callbacks(opt)
        model.compile(loss='mse', optimizer=opts[opt], metrics=['accuracy'])
        hist = model.fit(X_train.values, y_train, batch_size=batch_size, epochs=n_epochs, validation_data=(X_val.values, y_val), verbose=0,
        callbacks=callbacks)
        best_epoch = np.argmax(hist.history['val_acc'])
        best_acc = hist.history['val_acc'][best_epoch]
        best_model = create_model(opt)
        # Load the model weights with the highest validation accuracy
        best_model.load_weights('/Users/swa/Desktop/temp/checkpoints/optimizers_best_' + opt + '.h5')
        best_model.compile(loss='mse', optimizer=opts[opt], metrics=['accuracy'])
        score = best_model.evaluate(X_test.values, y_test, verbose=0)
        results.append([opt, best_epoch, best_acc, score[1]])

Epoch 00201: early stopping
WARNING:tensorflow:Variable *= will be deprecated. Use `var.assign(var * other)` if you want assignment to the variable value or `x = x * y` if you want a new python Tensor object.
Epoch 00359: early stopping
Epoch 00459: early stopping
Epoch 00525: early stopping
Epoch 00392: early stopping
Epoch 00251: early stopping
Epoch 00379: early stopping
Epoch 00382: early stopping



    res = pd.DataFrame(results)
    res.columns = ['optimizer', 'epochs', 'val_accuracy', 'test_accuracy']
    res

import numpy as np

import pandas as pd

from sklearn.model_selection import train_test_split

from keras.models import Sequential

from keras.layers import Dense, Dropout

from keras.callbacks import EarlyStopping, ModelCheckpoint

from keras.optimizers import SGD, Adadelta, Adam, RMSprop, Adagrad, Nadam, Adamax

SEED = 2017

Using TensorFlow backend.

# Data can be downloaded at https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv

data = pd.read_csv('winequality-red.csv', sep=';')

y = data['quality']

X = data.drop(['quality'], axis=1)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=SEED)

X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=SEED)

def create_model(opt):

model = Sequential()

model.add(Dense(100, input_dim=X_train.shape[1],

activation='relu'))

model.add(Dense(50, activation='relu'))

model.add(Dense(25, activation='relu'))

model.add(Dense(10, activation='relu'))

model.add(Dense(1, activation='linear'))

return model

def create_callbacks(opt):

callbacks = [

EarlyStopping(monitor='val_acc', patience=200, verbose=2),

ModelCheckpoint('/Users/swa/Desktop/temp/checkpoints/optimizers_best_' + opt + '.h5', monitor='val_acc', save_best_only=True, verbose=0)

]

return callbacks

opts = dict({

'sgd': SGD(),

'sgd-0001': SGD(lr=0.0001, decay=0.00001),

'adam': Adam(),

'adadelta': Adadelta(),

'rmsprop': RMSprop(),

'rmsprop-0001': RMSprop(lr=0.0001),

'nadam': Nadam(),

'adamax': Adamax()

})

batch_size = 128

n_epochs = 1000

results = []

# Loop through the optimizers

for opt in opts:

model = create_model(opt)

callbacks = create_callbacks(opt)

model.compile(loss='mse', optimizer=opts[opt], metrics=['accuracy'])

hist = model.fit(X_train.values, y_train, batch_size=batch_size, epochs=n_epochs, validation_data=(X_val.values, y_val), verbose=0,

callbacks=callbacks)

best_epoch = np.argmax(hist.history['val_acc'])

best_acc = hist.history['val_acc'][best_epoch]

best_model = create_model(opt)

# Load the model weights with the highest validation accuracy

best_model.load_weights('/Users/swa/Desktop/temp/checkpoints/optimizers_best_' + opt + '.h5')

best_model.compile(loss='mse', optimizer=opts[opt], metrics=['accuracy'])

score = best_model.evaluate(X_test.values, y_test, verbose=0)

results.append([opt, best_epoch, best_acc, score[1]])

Epoch 00201: early stopping

WARNING:tensorflow:Variable *= will be deprecated. Use `var.assign(var * other)` if you want assignment to the variable value or `x = x * y` if you want a new python Tensor object.

Epoch 00359: early stopping

Epoch 00459: early stopping

Epoch 00525: early stopping

Epoch 00392: early stopping

Epoch 00251: early stopping

Epoch 00379: early stopping

Epoch 00382: early stopping

res = pd.DataFrame(results)

res.columns = ['optimizer', 'epochs', 'val_accuracy', 'test_accuracy']

res

	optimizer	epochs	val_accuracy	test_accuracy
0	sgd	0	0.000000	0.000000
1	sgd-0001	158	0.539062	0.543750
2	adam	258	0.593750	0.609375
3	adadelta	324	0.566406	0.584375
4	rmsprop	191	0.566406	0.550000
5	rmsprop-0001	50	0.546875	0.578125
6	nadam	178	0.570312	0.612500
7	adamax	181	0.570312	0.593750

You might also like