Pre-Training ANN mit RBM in pylearn2

Question

Ich versuche, ein mehrschichtiges ANN mit pylearn2, mit Pre-Training mit RBM zu trainieren. Ich habe leicht modifiziert, um das Skript run_deep_trainer genannt, die in pylearn2 \ pylearn2 \ scripts \ tutorials \ deep_trainer enthalten ist. Ich möchte ein 4-lagiges Netz, wobei die ersten 3 mit 500 GaussianBinaryRBM hergestellt werden und die letzte ist eine mlp.Softmax Schicht.

Hier ist das Skript, das ich erstellt habe:

from pylearn2.models.rbm import GaussianBinaryRBM 
from pylearn2.models.softmax_regression import SoftmaxRegression 
from pylearn2.models.mlp import Softmax 
from pylearn2.training_algorithms.sgd import SGD 
from pylearn2.costs.autoencoder import MeanSquaredReconstructionError 
from pylearn2.termination_criteria import EpochCounter 
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix 
from pylearn2.energy_functions.rbm_energy import GRBM_Type_1 
from pylearn2.blocks import StackedBlocks 
from pylearn2.datasets.transformer_dataset import TransformerDataset 
from pylearn2.costs.ebm_estimation import SMD 
from pylearn2.training_algorithms.sgd import MonitorBasedLRAdjuster 
from pylearn2.train import Train 
from optparse import OptionParser 

import numpy 

def get_dataset_timitConsSmall(): 
    print('loading timitConsSmall dataset...') 

    template = \ 
     """!obj:pylearn2.datasets.timitConsSmall.timit.TIMIT { 
classes_number: 32, 
which_set: %s, 
}""" 
    trainset = yaml_parse.load(template % "train") 
    # testset = yaml_parse.load(template % "test") 

    print('...done loading timitConsSmall.') 

    return trainset 

def get_grbm(structure): 
    n_input, n_output = structure 
    config = { 
     'nvis': n_input, 
     'nhid': n_output, 
     "irange": 0.05, 
     "energy_function_class": GRBM_Type_1, 
     "learn_sigma": True, 
     "init_sigma": .4, 
     "init_bias_hid": -2., 
     "mean_vis": False, 
     "sigma_lr_scale": 1e-3 
    } 

    return GaussianBinaryRBM(**config) 


def get_logistic_regressor(structure): 
    n_input, n_output = structure 

    layer = SoftmaxRegression(n_classes=n_output, irange=0.02, nvis=n_input) 

    return layer 

def get_mlp_softmax(structure): 
    n_input, n_output = structure 

    layer = Softmax(n_classes=n_output, irange=0.02, layer_name='y') 

    return layer 

def get_layer_trainer_softmax(layer, trainset): 
    # configs on sgd 

    config = {'learning_rate': 000.1, 
       'cost': Default(), 
       'batch_size': 100, 
       'monitoring_batches': 10, 
       'monitoring_dataset': trainset, 
       'termination_criterion': EpochCounter(max_epochs=MAX_EPOCHS_SUPERVISED), 
       'update_callbacks': None 
       } 

    train_algo = SGD(**config) 
    model = layer 
    return Train(model=model, 
       dataset=trainset, 
       algorithm=train_algo, 
       extensions=None) 

def get_layer_trainer_logistic(layer, trainset): 
    # configs on sgd 

    config = {'learning_rate': 0.1, 
       'cost': Default(), 
       'batch_size': 10, 
       'monitoring_batches': 10, 
       'monitoring_dataset': trainset, 
       'termination_criterion': EpochCounter(max_epochs=MAX_EPOCHS_SUPERVISED), 
       'update_callbacks': None 
       } 

    train_algo = SGD(**config) 
    model = layer 
    return Train(model=model, 
       dataset=trainset, 
       algorithm=train_algo, 
       extensions=None) 

def get_layer_trainer_sgd_rbm(layer, trainset): 
    train_algo = SGD(
     learning_rate=1e-2, 
     batch_size=100, 
     # "batches_per_iter" : 2000, 
     monitoring_batches=20, 
     monitoring_dataset=trainset, 
     cost=SMD(corruptor=GaussianCorruptor(stdev=0.4)), 
     termination_criterion=EpochCounter(max_epochs=MAX_EPOCHS_UNSUPERVISED), 
    ) 
    model = layer 
    extensions = [MonitorBasedLRAdjuster()] 
    return Train(model=model, algorithm=train_algo, 
       save_path='grbm.pkl', save_freq=1, 
       extensions=extensions, dataset=trainset) 

def main(args=None): 
    trainset = get_dataset_timitConsSmall() 
    n_output = 32 

    design_matrix = trainset.get_design_matrix() 
    n_input = design_matrix.shape[1] 

    # build layers 
    layers = [] 
    structure = [[n_input, 500], [500, 500], [500, 500], [500, n_output]] 
    # layer 0: gaussianRBM 
    layers.append(get_grbm(structure[0])) 
    # # layer 1: denoising AE 
    # layers.append(get_denoising_autoencoder(structure[1])) 
    # # layer 2: AE 
    # layers.append(get_autoencoder(structure[2])) 
    # # layer 3: logistic regression used in supervised training 
    # layers.append(get_logistic_regressor(structure[3])) 

    # layer 1: gaussianRBM 
    layers.append(get_grbm(structure[1])) 
    # layer 2: gaussianRBM 
    layers.append(get_grbm(structure[2])) 
    # layer 3: logistic regression used in supervised training 
    # layers.append(get_logistic_regressor(structure[3])) 
    layers.append(get_mlp_softmax(structure[3])) 



    # construct training sets for different layers 
    trainset = [trainset, 
       TransformerDataset(raw=trainset, transformer=layers[0]), 
       TransformerDataset(raw=trainset, transformer=StackedBlocks(layers[0:2])), 
       TransformerDataset(raw=trainset, transformer=StackedBlocks(layers[0:3]))] 

    # construct layer trainers 
    layer_trainers = [] 
    layer_trainers.append(get_layer_trainer_sgd_rbm(layers[0], trainset[0])) 
    # layer_trainers.append(get_layer_trainer_sgd_autoencoder(layers[1], trainset[1])) 
    # layer_trainers.append(get_layer_trainer_sgd_autoencoder(layers[2], trainset[2])) 
    layer_trainers.append(get_layer_trainer_sgd_rbm(layers[1], trainset[1])) 
    layer_trainers.append(get_layer_trainer_sgd_rbm(layers[2], trainset[2])) 
    # layer_trainers.append(get_layer_trainer_logistic(layers[3], trainset[3])) 
    layer_trainers.append(get_layer_trainer_softmax(layers[3], trainset[3])) 

    # unsupervised pretraining 
    for i, layer_trainer in enumerate(layer_trainers[0:3]): 
     print('-----------------------------------') 
     print(' Unsupervised training layer %d, %s' % (i, layers[i].__class__)) 
     print('-----------------------------------') 
     layer_trainer.main_loop() 

    print('\n') 
    print('------------------------------------------------------') 
    print(' Unsupervised training done! Start supervised training...') 
    print('------------------------------------------------------') 
    print('\n') 

    # supervised training 
    layer_trainers[-1].main_loop() 


if __name__ == '__main__': 
    main() 

Es tut richtig unbeaufsichtigt vor dem Training Teil, aber es ist ein Fehler in dem überwachten Training Teil:

Traceback (most recent call last): 
    File "run_deep_trainer.py", line 404, in <module> 
    main() 
    File "run_deep_trainer.py", line 400, in main 
    layer_trainers[-1].main_loop() 
    File "/home/gortolan/pylearn2/pylearn2/train.py", line 141, in main_loop 
    self.setup() 
    File "/home/gortolan/pylearn2/pylearn2/train.py", line 121, in setup 
    self.algorithm.setup(model=self.model, dataset=self.dataset) 
    File "/home/gortolan/pylearn2/pylearn2/training_algorithms/sgd.py", line 243, in setup 
    inf_params = [param for param in model.get_params() 
    File "/home/gortolan/pylearn2/pylearn2/models/model.py", line 503, in get_params 
    return list(self._params) 
AttributeError: 'Softmax' object has no attribute '_params' 

Wenn ich das verwenden SoftmaxRegression (als Modell) in der letzten Schicht, die die Funktionen und get_mlp_softmax()get_layer_trainer_softmax() mit get_logistic_regressor() und get_layer_trainer_logistic() Mitteln Substitution, funktioniert alles in Ordnung.

Es scheint, dass das Modell mlp.Softmax die Parameter (_params) nicht über die Funktion get_params() zurückgibt.

Kann jemand das beheben?

Answer 1

Das Problem liegt daran, SoftmaxRegressor ist ein Modell, aber Softmax ist eine Schicht für eine MLP. Ein Weg, um es zu beheben wäre etwas entlang der Linien von

def get_mlp_softmax(structure): 
    n_input, n_output = structure 

    layer = MLP(nvis=500, layers=[Softmax(n_classes=n_output, irange=0.02, layer_name='y')]) 

    return layer 

seine Wo MLPmlp.MLP