Keine Dimension erhöhen ValueError in Batch_Norm mit Tensorflow

Question

Ich habe eine bestimmte Art von neuronalen Netzen (GAN: Generative Adversarial Networks) in tensorflow implementiert.

Es funktionierte wie erwartet, bis ich (siehe vollständigen Code unten) die folgende Batch Normalisierung Schicht in der generator(z) Methode hinzufügen entschieden:

out = tf.contrib.layers.batch_norm(out, is_training=False) 

wie bekomme ich folgende Fehlermeldung:

G_sample = generator(Z) 
    File "/Users/Florian/Documents/DeepLearning/tensorflow_stuff/tensorflow_stuff/DCGAN.py", line 84, in generator 
    out = tf.contrib.layers.batch_norm(out, is_training=False)          
    File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/arg_scope.py", line 181, in func_with_args 
    return func(*args, **current_args) 
    File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/layers/python/layers/layers.py", line 551, in batch_norm 
    outputs = layer.apply(inputs, training=is_training) 
    File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 381, in apply 
    return self.__call__(inputs, **kwargs) 
    File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 328, in __call__ 
    self.build(input_shapes[0]) 
    File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/normalization.py", line 143, in build 
    input_shape) 
ValueError: ('Input has undefined `axis` dimension. Input shape: ', TensorShape([Dimension(None), Dimension(None), Dimension(None), Dimension(None)])) 

Das Problem scheint an der [None, None, None, None] Form des Eingangs out zu liegen, aber ich weiß nicht, wie ich das beheben kann. Hier

ist der vollständige Code:

from __future__ import division 
from __future__ import division 
from __future__ import print_function 
import tensorflow as tf 
from tensorflow.examples.tutorials.mnist import input_data 
from tensorflow.contrib.layers import batch_norm 
import numpy as np 
import matplotlib.pyplot as plt 
import matplotlib.gridspec as gridspec 
import os 




def leaky_relu(x, alpha): 
    return tf.maximum(alpha * x, x) 




def discriminator(x): 

    with tf.variable_scope('discriminator', reuse=True): 

     # conv_2D accepts shape (batch, height, width, channel) as input so 
     # reshape it 
     x = tf.reshape(x, shape=[-1, 28, 28, 1]) 
     out = tf.nn.conv2d(x, tf.get_variable('D_w_1'), strides=[1, 2, 2, 1], padding='SAME') 
     out = leaky_relu(out, alpha=0.2) 
     #out = tf.nn.dropout(out, keep_prob=0.2) 
     out = tf.nn.conv2d(out, tf.get_variable('D_w_2'), strides=[1, 2, 2, 1], padding='SAME') 
     out = leaky_relu(out, alpha=0.2) 
     #out = tf.nn.dropout(out, keep_prob=0.2) 

     # fully connected layer 
     out = tf.reshape(out, shape=[-1, 7*7*128]) 
     D_logits = tf.matmul(out, tf.get_variable('D_w_fc_1')) 
     #D_logits = tf.nn.sigmoid(D_logits) 
     D_logits = leaky_relu(D_logits, alpha=0.2) 

    return D_logits 




def generator(z): 

    with tf.variable_scope('generator', reuse=True): 
     out = tf.matmul(z, tf.get_variable('G_w_fc_1')) 
     out = tf.nn.relu(out) 

     out = tf.reshape(out, shape=[-1, 7, 7, 128]) 

     out = tf.nn.conv2d_transpose(out, 
            tf.get_variable('G_w_deconv_1'), 
            output_shape=tf.stack([tf.shape(out)[0], 14, 14, 64]), 
            strides=[1, 2, 2, 1], 
            padding='SAME') 
     print(out.get_shape().as_list()) 
     out = tf.contrib.layers.batch_norm(out, is_training=False)          
     out = tf.nn.relu(out) 

     out = tf.nn.conv2d_transpose(out, 
            tf.get_variable('G_w_deconv_2'), 
            output_shape=tf.stack([tf.shape(out)[0], 28, 28, 1]), 
            strides=[1, 2, 2, 1], 
            padding='SAME') 
     out = tf.nn.tanh(out) 


    return out 







def sample_Z(m, n): 
    return np.random.uniform(-1., 1., size=[m, n]) 


def plot(samples): 
    fig = plt.figure(figsize=(4, 4)) 
    gs = gridspec.GridSpec(4, 4) 
    gs.update(wspace=0.05, hspace=0.05) 

    for i, sample in enumerate(samples): 
     ax = plt.subplot(gs[i]) 
     plt.axis('off') 
     ax.set_xticklabels([]) 
     ax.set_yticklabels([]) 
     ax.set_aspect('equal') 
     plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 

    return fig 


if __name__ == '__main__': 


    mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 

    batch_size = 128 
    # size of generator input 
    Z_dim = 10 
    # batch within an epoch 
    batches_per_epoch = int(np.floor(mnist.train.num_examples/batch_size)) 
    nb_epochs = 20 

    # learning rate 
    learning_rate = 0.00005 # 0.0002 

    Z = tf.placeholder(tf.float32, [batch_size, Z_dim]) 
    X = tf.placeholder(tf.float32, [batch_size, 784]) 

    with tf.variable_scope('discriminator'): 
     D_w_1 = tf.get_variable('D_w_1', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02)) 
     D_w_2 = tf.get_variable('D_w_2', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02)) 
     D_w_fc_1 = tf.get_variable('D_w_fc_1', initializer=tf.random_normal([7*7*128, 1], stddev=0.02)) 

    D_var_list = [D_w_1, D_w_2, D_w_fc_1] 


    with tf.variable_scope('generator'): 
     G_w_fc_1 = tf.get_variable('G_w_fc_1', initializer=tf.random_normal([Z_dim, 128*7*7], stddev=0.02)) 
     G_w_deconv_1 = tf.get_variable('G_w_deconv_1', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02)) 
     G_w_deconv_2 = tf.get_variable('G_w_deconv_2', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02)) 

    G_var_list = [G_w_fc_1, G_w_deconv_1, G_w_deconv_2] 


    G_sample = generator(Z) 
    D_logit_real = discriminator(X) 
    D_logit_fake = discriminator(G_sample) 


    # objective functions 
    # discriminator aims at maximizing the probability of TRUE data (i.e. from the dataset) and minimizing the probability 
    # of GENERATED/FAKE data: 
    D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real))) 
    D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake))) 
    D_loss = D_loss_real + D_loss_fake 

    # generator aims at maximizing the probability of GENERATED/FAKE data (i.e. fool the discriminator) 
    G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake))) 

    D_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(D_loss, var_list=D_var_list) 
    # when optimizing generator, discriminator is kept fixed 
    G_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(G_loss, var_list=G_var_list) 


    with tf.Session() as sess:  

     sess.run(tf.global_variables_initializer()) 

     if not os.path.exists('out/'): 
      os.makedirs('out/') 

     for i_epoch in range(nb_epochs): 

      G_loss_val = 0 
      D_loss_val = 0 

      for i_batch in range(batches_per_epoch): 
       print('batch %i/%i' % (i_batch+1, batches_per_epoch)) 

       X_mb, _ = mnist.train.next_batch(batch_size) 

       # train discriminator 
       _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(batch_size, Z_dim)}) 
       D_loss_val += D_loss_curr 

       # train generator 
       _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(batch_size, Z_dim)}) 
       G_loss_val += G_loss_curr 

       if i_batch % 50 == 0: 
        samples = sess.run(G_sample, feed_dict={Z: sample_Z(16, Z_dim)}) 

        fig = plot(samples) 
        plt.savefig('out/%i_%i.png' % (i_epoch, i_batch), bbox_inches='tight') 
        plt.close(fig) 





      print('Iter: {}'.format(i_epoch)) 
      print('D loss: {:.4}'.format(D_loss)) 
      print('G_loss: {:.4}'.format(G_loss)) 

Answer 1

Wenn Sie eine konstante Form wie [100, 14, 14, 64] wie output_shape passieren, conv2d_transpose einen Tensor mit der richtigen Form Satz zurück. Wenn Sie jedoch einen nichtkonstanten Tensor übergeben (was Sie tun müssen, wenn Sie die Losgröße nicht im Voraus kennen), geht davon aus, dass er die Form erst kennen kann, wenn das Diagramm ausgeführt wird. Keine Form während der Konstruktion.

Theoretisch könnte es sich herausgestellt haben, dass einige der Dimensionen konstant sind, aber das ist im Moment nicht getan. Sie können dies umgehen, indem Sie out.set_shape([None, 14, 14, 64]) oder out = tf.reshape(out, [-1, 14, 14, 64]) verwenden. Die Größe der Chargendimension muss nicht festgelegt werden, da batch_norm dies nicht erfordert.

Dies wird auf Tensorflow-Probleme 833 und 8972 diskutiert.

Answer 2

Arbeitscode ist unten. Es gab ein paar kleinere Fehler im Code - vielleicht von Ihren Tests vor dem Posten der Frage - oder vielleicht hatten Sie es noch nicht vollständig ausgeführt, meine Änderungen werden mit #EDIT: notiert. Sie müssen die Form definieren, um die Batch-Normalisierung zu verwenden, und Sie können dies im Voraus tun, wenn Sie möchten, aber Ihr Vorschlag ist in Ordnung. Ich bevorzuge die Umformung mit einer variablen Dimension, indem ich -1 out = tf.reshape(out, [-1, 14, 14, 64]) verwende. Der folgende Code funktioniert bei TF> 1 und Python> 3.5.

from __future__ import division 
from __future__ import division 
from __future__ import print_function 
import tensorflow as tf 
from tensorflow.examples.tutorials.mnist import input_data 
from tensorflow.contrib.layers import batch_norm 
import numpy as np 
import matplotlib.pyplot as plt 
import matplotlib.gridspec as gridspec 
import os 




def leaky_relu(x, alpha): 
    return tf.maximum(alpha * x, x) 




def discriminator(x): 

    with tf.variable_scope('discriminator', reuse=True): 

     # conv_2D accepts shape (batch, height, width, channel) as input so 
     # reshape it 
     x = tf.reshape(x, shape=[-1, 28, 28, 1]) 
     out = tf.nn.conv2d(x, tf.get_variable('D_w_1'), strides=[1, 2, 2, 1], padding='SAME') 
     out = leaky_relu(out, alpha=0.2) 
     #out = tf.nn.dropout(out, keep_prob=0.2) 
     out = tf.nn.conv2d(out, tf.get_variable('D_w_2'), strides=[1, 2, 2, 1], padding='SAME') 
     out = leaky_relu(out, alpha=0.2) 
     #out = tf.nn.dropout(out, keep_prob=0.2) 

     # fully connected layer 
     out = tf.reshape(out, shape=[-1, 7*7*128]) 
     D_logits = tf.matmul(out, tf.get_variable('D_w_fc_1')) 
     #D_logits = tf.nn.sigmoid(D_logits) 
     D_logits = leaky_relu(D_logits, alpha=0.2) 

    return D_logits 




def generator(z): 

    with tf.variable_scope('generator', reuse=True): 
     out = tf.matmul(z, tf.get_variable('G_w_fc_1')) 
     out = tf.nn.relu(out) 

     out = tf.reshape(out, shape=[-1, 7, 7, 128]) 

     out = tf.nn.conv2d_transpose(out, 
            tf.get_variable('G_w_deconv_1'), 
            output_shape=tf.stack([tf.shape(out)[0], 14, 14, 64]), 
            strides=[1, 2, 2, 1], 
            padding='SAME') 
     print(out.get_shape().as_list()) 
     out = tf.reshape(out, [-1, 14, 14, 64]) #EDIT: You need to define the shape for batch_norm 

     #out.set_shape([out.get_shape().as_list()[0], 14, 14, 64]) 
     out = tf.contrib.layers.batch_norm(out, is_training=False) 
     out = tf.nn.relu(out) 

     out = tf.nn.conv2d_transpose(out, 
            tf.get_variable('G_w_deconv_2'), 
            output_shape=tf.stack([tf.shape(out)[0], 28, 28, 1]), 
            strides=[1, 2, 2, 1], 
            padding='SAME') 
     out = tf.nn.tanh(out) 


    return out 

def sample_Z(m, n): 
    return np.random.uniform(-1., 1., size=[m, n]) 


def plot(samples): 
    fig = plt.figure(figsize=(4, 4)) 
    gs = gridspec.GridSpec(12, 12) #EDIT: This wasn't large enough for the dataset. 
    gs.update(wspace=0.05, hspace=0.05) 

    for i, sample in enumerate(samples): 
     ax = plt.subplot(gs[i]) 
     plt.axis('off') 
     ax.set_xticklabels([]) 
     ax.set_yticklabels([]) 
     ax.set_aspect('equal') 
     plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 

    return fig 


if __name__ == '__main__': 


    mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 

    batch_size = 128 
    # size of generator input 
    Z_dim = 10 
    # batch within an epoch 
    batches_per_epoch = int(np.floor(mnist.train.num_examples/batch_size)) 
    nb_epochs = 20 

    # learning rate 
    learning_rate = 0.00005 # 0.0002 

    Z = tf.placeholder(tf.float32, [batch_size, Z_dim]) 
    X = tf.placeholder(tf.float32, [batch_size, 784]) 

    with tf.variable_scope('discriminator'): 
     D_w_1 = tf.get_variable('D_w_1', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02)) 
     D_w_2 = tf.get_variable('D_w_2', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02)) 
     D_w_fc_1 = tf.get_variable('D_w_fc_1', initializer=tf.random_normal([7*7*128, 1], stddev=0.02)) 

    D_var_list = [D_w_1, D_w_2, D_w_fc_1] 


    with tf.variable_scope('generator'): 
     G_w_fc_1 = tf.get_variable('G_w_fc_1', initializer=tf.random_normal([Z_dim, 128*7*7], stddev=0.02)) 
     G_w_deconv_1 = tf.get_variable('G_w_deconv_1', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02)) 
     G_w_deconv_2 = tf.get_variable('G_w_deconv_2', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02)) 

    G_var_list = [G_w_fc_1, G_w_deconv_1, G_w_deconv_2] 


    G_sample = generator(Z) 
    D_logit_real = discriminator(X) 
    D_logit_fake = discriminator(G_sample) 


    # objective functions 
    # discriminator aims at maximizing the probability of TRUE data (i.e. from the dataset) and minimizing the probability 
    # of GENERATED/FAKE data: 
    D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real))) 
    D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake))) 
    D_loss = D_loss_real + D_loss_fake 

    # generator aims at maximizing the probability of GENERATED/FAKE data (i.e. fool the discriminator) 
    G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake))) 

    D_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(D_loss, var_list=D_var_list) 
    # when optimizing generator, discriminator is kept fixed 
    G_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(G_loss, var_list=G_var_list) 


    with tf.Session() as sess:  

     sess.run(tf.global_variables_initializer()) 

     if not os.path.exists('out/'): 
      os.makedirs('out/') 

     for i_epoch in range(nb_epochs): 

      G_loss_val = 0 
      D_loss_val = 0 

      for i_batch in range(batches_per_epoch): 
       print('batch %i/%i' % (i_batch+1, batches_per_epoch)) 

       X_mb, _ = mnist.train.next_batch(batch_size) 

       # train discriminator 
       _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(batch_size, Z_dim)}) 
       D_loss_val += D_loss_curr 

       # train generator 
       _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(batch_size, Z_dim)}) 
       G_loss_val += G_loss_curr 

       if i_batch % 50 == 0: 
        samples = sess.run(G_sample, feed_dict={Z: sample_Z(batch_size, Z_dim)}) #EDIT: changed to batch_size to match the tensor 

        fig = plot(samples) 
        plt.savefig('out/%i_%i.png' % (i_epoch, i_batch), bbox_inches='tight') 
        plt.close(fig) 

      print('Iter: {}'.format(i_epoch)) 
      print('D loss: {:.4}'.format(D_loss_curr)) #EDIT: You were trying to print the tensor. 
      print('G_loss: {:.4}'.format(G_loss_curr))#EDIT: You were trying to print the tensor.