tpu: tpu-v3-128-euw4a-52; run: shawn-bigrun65-uncond-evonorm; description: Unconditional BigGAN 128x128 + evonorm on many datasets; logdir: gs://darnbooru-euw4a/runs/bigrun66/

[shawwn]

Branch: https://github.com/shawwn/compare_gan/blob/2020-05-09/dynamicvars/run_bigrun61.sh

dataset.name = "images_128"
options.datasets = "gs://darnbooru-euw4a/datasets/danbooru2019-s/danbooru2019-s-0*,gs://darnbooru-euw4a/datasets/danbooru2019-s/danbooru2019-s-0*,gs://darnbooru-euw4a/datasets/imagenet/train-0*,gs://darnbooru-euw4a/datasets/flickr3m/flickr3m-0*,gs://darnbooru-euw4a/datasets/ffhq1024/ffhq1024-0*,gs://darnbooru-euw4a/datasets/portraits/portraits-0*,gs://darnbooru-euw4a/datasets/ffhq1024/ffhq1024-0*,gs://darnbooru-euw4a/datasets/portraits/portraits-0*"
options.random_labels = False
options.num_classes = 1000
train_imagenet_transform.crop_method = "random"
options.z_dim = 120
resnet_biggan.Generator.ch = 128
resnet_biggan.Discriminator.ch = 128
resnet_biggan.Generator.blocks_with_attention = "64"
resnet_biggan.Discriminator.blocks_with_attention = "64"

options.architecture = "resnet_biggan_arch"
ModularGAN.conditional = False
options.batch_size = 2048
options.gan_class = @ModularGAN
options.lamba = 1
options.training_steps = 250000
weights.initializer = "orthogonal"
spectral_norm.singular_value = "auto"

# Generator
G.batch_norm_fn = @batch_norm
G.spectral_norm = True
ModularGAN.g_use_ema = True
resnet_biggan.Generator.hierarchical_z = True
resnet_biggan.Generator.embed_z = True
resnet_biggan.Generator.embed_y = False
standardize_batch.decay = 0.9
standardize_batch.epsilon = 1e-5
standardize_batch.use_moving_averages = False
standardize_batch.use_cross_replica_mean = None
standardize_batch.use_evonorm = True

# Discriminator
options.disc_iters = 1
ModularGAN.experimental_joint_gen_for_disc = False
ModularGAN.experimental_force_graph_unroll = False
D.spectral_norm = True
resnet_biggan.Discriminator.project_y = False

# Loss and optimizer
loss.fn = @hinge
penalty.fn = @no_penalty
ModularGAN.g_lr = 0.0000666
ModularGAN.d_lr = 0.0005
ModularGAN.g_lr_mul = 1.0
ModularGAN.d_lr_mul = 1.0
ModularGAN.g_optimizer_fn = @tf.train.AdamOptimizer
ModularGAN.d_optimizer_fn = @tf.train.AdamOptimizer
tf.train.AdamOptimizer.beta1 = 0.0
tf.train.AdamOptimizer.beta2 = 0.999

z.distribution_fn = @tf.random.normal
eval_z.distribution_fn = @tf.random.normal

run_config.experimental_host_call_every_n_steps = 50
TpuSummaries.save_image_steps = 50
run_config.iterations_per_loop = 500
run_config.save_checkpoints_steps = 2000

options.d_flood = -128.0
options.g_flood = -128.0
options.d_stop_g_above = 128.0
options.g_stop_d_above = 128.0
options.d_stop_d_below = -128.0
options.g_stop_g_below = -128.0

options.d_stop_d_below = 0.20
#options.g_stop_g_below = 0.05
#options.d_stop_g_above = 1.00
options.g_stop_d_above = 1.50

knobs.stop = False
#knobs.rollback = 46000
knobs.rollback = False
ModularGAN.g_lr_mul = 1.0
ModularGAN.d_lr_mul = 0.1
options.g_stop_d_above = 1.50