Batch normalization added and training works

Author: forcecore

README.md

@@ -1,8 +1,5 @@
 # Keras-GAN-Animeface-Character
 
-WORK IN PROGRESS.
-DOESN'T WORK YET!!
-
 GAN example for Keras. Cuz MNIST is too small and there
 should an example on something more realistic.
 
@@ -21,6 +18,7 @@ should an example on something more realistic.
     * https://github.com/tdrussell/IllustrationGAN
 * I used slow implementation for the sake of simplicity. However, the correct way is:
     * https://ctmakro.github.io/site/on_learning/fast_gan_in_keras.html
+* https://github.com/shekkizh/neuralnetworks.thought-experiments/blob/master/Generative%20Models/GAN/Readme.md
 
 
 ## How to run this example
@@ -101,3 +99,6 @@ What I experienced during my training of GAN.
   If it stays there for too long, it isn't good, I think.
 * In case you're seeing high G loss, it could mean it can't keep up with discriminator.
   You might need to increase LR. (Must be slower than discriminator though)
+* One final piece of the training I was missing was the parameter in BatchNormalization.
+  I found about it in this link:
+  https://github.com/shekkizh/neuralnetworks.thought-experiments/blob/master/Generative%20Models/GAN/Readme.md

args.py

@@ -9,7 +9,6 @@ class Args :
 
     # images size we will work on. (sz, sz, 3)
     sz = 64
-    ch = 1
 
     # alpha, used by leaky relu of D and G networks.
     alpha_D = 0.2
@@ -44,3 +43,10 @@ class Args :
     # Same as default in Keras, but good for GAN, says
     # https://github.com/gheinrich/DIGITS-GAN/blob/master/examples/weight-init/README.md#experiments-with-lenet-on-mnist
     kernel_initializer = 'glorot_uniform'
+
+    # Since DCGAN paper, everybody uses 0.5 and for me, it works the best too.
+    # I tried 0.9, 0.1.
+    adam_beta = 0.5
+
+    # BatchNormalization matters too.
+    bn_momentum = 0.3

data.py

@@ -15,9 +15,11 @@ def normalize4gan(im):
     Convert colorspace and
     cale the input in [-1, 1] range, as described in ganhacks
     '''
-    im = cv2.cvtColor(im, cv2.COLOR_RGB2YCR_CB).astype(np.float32)
+    #im = cv2.cvtColor(im, cv2.COLOR_RGB2YCR_CB).astype(np.float32)
+    # HSV... not helpful.
+    im = im.astype(np.float32)
     im /= 128.0
-    im -= 1 # now in [-1, 1]
+    im -= 1.0 # now in [-1, 1]
     return im
 
 
@@ -30,8 +32,7 @@ def denormalize4gan(im):
     '''
     im += 1.0 # in [0, 2]
     im *= 127.0 # in [0, 255]
-    #im = cv2.cvtColor(im.astype(np.uint8), cv2.COLOR_YCR_CB2RGB)
-    return im[:,:,0]
+    return im.astype(np.uint8)
 
 
 

gan.py

@@ -85,7 +85,7 @@ def dump_batch(imgs, cnt, ofname):
 
 
 def build_networks():
-    shape = (Args.sz, Args.sz, Args.ch)
+    shape = (Args.sz, Args.sz, 3)
 
     # Learning rate is important.
     # Optimizers are important too, try experimenting them yourself to fit your dataset.
@@ -115,8 +115,8 @@ def build_networks():
     # now same lr, as we are using history to train D multiple times.
     # I don't exactly understand how decay parameter in Adam works. Certainly not exponential.
     # Actually faster than exponential, when I look at the code and plot it in Excel.
-    dopt = Adam(lr=0.00005, beta_1=0.5)
-    opt  = Adam(lr=0.00005, beta_1=0.5)
+    dopt = Adam(lr=0.0002, beta_1=Args.adam_beta)
+    opt  = Adam(lr=0.0001, beta_1=Args.adam_beta)
 
     # too slow
     # Another thing about LR.
@@ -160,7 +160,7 @@ def train_autoenc( dataf ):
 
     opt = Adam(lr=0.001)
 
-    shape = (Args.sz, Args.sz, Args.ch)
+    shape = (Args.sz, Args.sz, 3)
     enc = build_enc( shape )
     enc.compile(optimizer=opt, loss='mse')
     enc.summary()
@@ -201,8 +201,8 @@ def load_weights(model, wf):
     try:
         model.load_weights(wf)
     except:
-        print("failed to load weight", wf)
-        raise
+        print("failed to load weight, network changed or corrupt hdf5", wf)
+        sys.exit(1)
 
 
 
@@ -218,12 +218,6 @@ def train_gan( dataf ) :
     logger.on_train_begin() # initialize csv file
     with h5py.File( dataf, 'r' ) as f :
         faces = f.get( 'faces' )
-
-        if Args.ch == 1:
-            faces = np.array(faces[:,:,:,0])
-            faces = np.expand_dims(faces, 3)
-            print("xxxxxxxxxxxxxx", faces.shape)
-
         run_batches(gen, disc, gan, faces, logger, range(50000))
     logger.on_train_end()
 
@@ -290,7 +284,7 @@ def end_of_batch_task(batch, gen, disc, reals, fakes):
         dump_batch(reals, 4, "reals.png")
         dump_batch(fakes, 4, "fakes.png") # to check how noisy the image is
         frame = gen.predict(_bits)
-        animf = os.path.join(Args.anim_dir, "frame_{:08d}.png".format(batch))
+        animf = os.path.join(Args.anim_dir, "frame_{:08d}.png".format(int(batch/10)))
         dump_batch(frame, 4, animf)
         dump_batch(frame, 4, "frame.png")
 
@@ -309,7 +303,7 @@ def end_of_batch_task(batch, gen, disc, reals, fakes):
 
 
 def generate( genw, cnt ):
-    shape = (Args.sz, Args.sz, Args.ch)
+    shape = (Args.sz, Args.sz, 3)
     gen = build_gen( shape )
     gen.compile(optimizer='sgd', loss='mse')
     load_weights(gen, Args.genw)

nets.py

@@ -47,7 +47,7 @@ def conv2d( x, filters, shape=(4, 4), **kwargs ) :
             kernel_initializer=Args.kernel_initializer,
             **kwargs )( x )
         #x = MaxPooling2D()( x )
-        x = BatchNormalization()( x )
+        x = BatchNormalization(momentum=Args.bn_momentum)( x )
         x = LeakyReLU(alpha=Args.alpha_D)( x )
         return x
 
@@ -113,7 +113,7 @@ def deconv2d( x, filters, shape=(4, 4) ) :
         #x = bilinear2x( x, filters )
         #x = Conv2D( filters, shape, padding='same' )( x )
 
-        x = BatchNormalization()( x )
+        x = BatchNormalization(momentum=Args.bn_momentum)( x )
         x = LeakyReLU(alpha=Args.alpha_G)( x )
         return x
 
@@ -127,7 +127,7 @@ def deconv2d( x, filters, shape=(4, 4) ) :
 
     x= Conv2DTranspose( 512, (4, 4),
         kernel_initializer=Args.kernel_initializer )(x)
-    x = BatchNormalization()( x )
+    x = BatchNormalization(momentum=Args.bn_momentum)( x )
     x = LeakyReLU(alpha=Args.alpha_G)( x )
     # 4x4
     x = deconv2d( x, 256 )
@@ -140,11 +140,11 @@ def deconv2d( x, filters, shape=(4, 4) ) :
     # Extra layer
     x = Conv2D( 64, (3, 3), padding='same',
         kernel_initializer=Args.kernel_initializer )( x )
-    x = BatchNormalization()( x )
+    x = BatchNormalization(momentum=Args.bn_momentum)( x )
     x = LeakyReLU(alpha=Args.alpha_G)( x )
     # 32x32
 
-    x= Conv2DTranspose( Args.ch, (4, 4), padding='same', activation='tanh',
+    x= Conv2DTranspose( 3, (4, 4), padding='same', activation='tanh',
         strides=(2, 2), kernel_initializer=Args.kernel_initializer )(x)
     # 64x64