From 0ccc9676a42efc89963d96f682d7aea93f3666eb Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 23 Sep 2020 22:10:09 +0300 Subject: [PATCH 01/24] refactoring main script --- cuda_gpu_config.py | 15 ++ models/baseline_fc_v4_8x16.py | 6 + models/scalers.py | 17 ++ test_script_data_v4.py | 377 +++++++++++++++++++--------------- 4 files changed, 255 insertions(+), 160 deletions(-) create mode 100644 cuda_gpu_config.py create mode 100644 models/scalers.py diff --git a/cuda_gpu_config.py b/cuda_gpu_config.py new file mode 100644 index 0000000..73e860d --- /dev/null +++ b/cuda_gpu_config.py @@ -0,0 +1,15 @@ +import os + +import tensorflow as tf + +def setup_gpu(gpu_num=None): + os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' + if gpu_num is not None: + os.environ['CUDA_VISIBLE_DEVICES'] = gpu_num + + gpus = tf.config.experimental.list_physical_devices('GPU') + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + + logical_devices = tf.config.experimental.list_logical_devices('GPU') + assert len(logical_devices) > 0, "Not enough GPU hardware devices available" diff --git a/models/baseline_fc_v4_8x16.py b/models/baseline_fc_v4_8x16.py index 9ebe96b..c0339d5 100644 --- a/models/baseline_fc_v4_8x16.py +++ b/models/baseline_fc_v4_8x16.py @@ -1,5 +1,7 @@ import tensorflow as tf +from . import scalers + @tf.function(experimental_relax_shapes=True) def preprocess_features(features): # features: @@ -145,6 +147,10 @@ def __init__(self, activation=tf.keras.activations.relu, kernel_init='glorot_uni # loss='mean_squared_error') # self.discriminator.compile(optimizer=self.disc_opt, # loss='mean_squared_error') + self.scaler = scalers.Logarithmic() + self.pad_range = (-3, 5) + self.time_range = (-7, 9) + self.data_version = 'data_v4' @tf.function diff --git a/models/scalers.py b/models/scalers.py new file mode 100644 index 0000000..91ebf46 --- /dev/null +++ b/models/scalers.py @@ -0,0 +1,17 @@ +import numpy as np + + +class Identity: + def scale(self, x): + return x + + def unscale(self, x): + return x + + +class Logarithmic: + def scale(self, x): + return np.log10(1 + x) + + def unscale(self, x): + return 10 ** x - 1 \ No newline at end of file diff --git a/test_script_data_v4.py b/test_script_data_v4.py index 536099c..22b38e7 100644 --- a/test_script_data_v4.py +++ b/test_script_data_v4.py @@ -12,9 +12,9 @@ from models.training import train from models.baseline_fc_v4_8x16 import BaselineModel_8x16 from metrics import make_metric_plots, make_histograms +import cuda_gpu_config - -def main(): +def make_parser(): parser = argparse.ArgumentParser(fromfile_prefix_chars='@') parser.add_argument('--checkpoint_name', type=str, required=True) parser.add_argument('--batch_size', type=int, default=32, required=False) @@ -36,16 +36,10 @@ def main(): parser.add_argument('--stochastic_stepping', action='store_true', default=True) parser.add_argument('--feature_noise_power', type=float, default=None) parser.add_argument('--feature_noise_decay', type=float, default=None) + return parser - args = parser.parse_args() - - - assert ( - (args.feature_noise_power is None) == - (args.feature_noise_decay is None) - ), 'Noise power and decay must be both provided' - +def print_args(args): print("") print("----" * 10) print("Arguments:") @@ -54,197 +48,249 @@ def main(): print("----" * 10) print("") - os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' - if args.gpu_num is not None: - os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_num - gpus = tf.config.experimental.list_physical_devices('GPU') - for gpu in gpus: - tf.config.experimental.set_memory_growth(gpu, True) - logical_devices = tf.config.experimental.list_logical_devices('GPU') - assert len(logical_devices) > 0, "Not enough GPU hardware devices available" +def parse_args(): + args = make_parser().parse_args() - model_path = Path('saved_models') / args.checkpoint_name - if args.prediction_only: - assert model_path.exists(), "Couldn't find model directory" - else: - assert not model_path.exists(), "Model directory already exists" - model_path.mkdir(parents=True) + assert ( + (args.feature_noise_power is None) == + (args.feature_noise_decay is None) + ), 'Noise power and decay must be both provided' - with open(model_path / 'arguments.txt', 'w') as f: - raw_args = [a for a in sys.argv[1:] if a[0] != '@'] - fnames = [a[1:] for a in sys.argv[1:] if a[0] == '@'] + print_args(args) - f.write('\n'.join(raw_args)) - for fname in fnames: - with open(fname, 'r') as f_in: - if len(raw_args) > 0: f.write('\n') - f.write(f_in.read()) + return args - model = BaselineModel_8x16(kernel_init=args.kernel_init, lr=args.lr, - num_disc_updates=args.num_disc_updates, latent_dim=args.latent_dim, - gp_lambda=args.gp_lambda, gpdata_lambda=args.gpdata_lambda, - num_additional_layers=args.num_additional_disc_layers, - cramer=args.cramer_gan, features_to_tail=args.features_to_tail, - dropout_rate=args.dropout_rate, - stochastic_stepping=args.stochastic_stepping) - if args.prediction_only: - def epoch_from_name(name): - epoch, = re.findall('\d+', name) - return int(epoch) - - gen_checkpoints = model_path.glob("generator_*.h5") - disc_checkpoints = model_path.glob("discriminator_*.h5") - latest_gen_checkpoint = max( - gen_checkpoints, - key=lambda path: epoch_from_name(path.stem) - ) - latest_disc_checkpoint = max( - disc_checkpoints, - key=lambda path: epoch_from_name(path.stem) - ) +def write_args(model_path, fname='arguments.txt'): + with open(model_path / fname, 'w') as f: + raw_args = [a for a in sys.argv[1:] if a[0] != '@'] + fnames = [a[1:] for a in sys.argv[1:] if a[0] == '@'] - assert ( - epoch_from_name(latest_gen_checkpoint.stem) == epoch_from_name(latest_disc_checkpoint.stem) - ), "Latest disc and gen epochs differ" + f.write('\n'.join(raw_args)) + f.write('\n') + for fname in fnames: + with open(fname, 'r') as f_in: + f.write(f_in.read()) - print(f'Loading generator weights from {str(latest_gen_checkpoint)}') - model.generator.load_weights(str(latest_gen_checkpoint)) - print(f'Loading discriminator weights from {str(latest_disc_checkpoint)}') - model.discriminator.load_weights(str(latest_disc_checkpoint)) +def epoch_from_name(name): + epoch, = re.findall('\d+', name) + return int(epoch) - def save_model(step): - if step % args.save_every == 0: - print(f'Saving model on step {step} to {model_path}') - model.generator.save(str(model_path.joinpath("generator_{:05d}.h5".format(step)))) - model.discriminator.save(str(model_path.joinpath("discriminator_{:05d}.h5".format(step)))) +def load_weights(model, model_path): + gen_checkpoints = model_path.glob("generator_*.h5") + disc_checkpoints = model_path.glob("discriminator_*.h5") + latest_gen_checkpoint = max( + gen_checkpoints, + key=lambda path: epoch_from_name(path.stem) + ) + latest_disc_checkpoint = max( + disc_checkpoints, + key=lambda path: epoch_from_name(path.stem) + ) - preprocessing._VERSION = 'data_v4' - pad_range = (-3, 5) - time_range = (-7, 9) - data, features = preprocessing.read_csv_2d(pad_range=pad_range, time_range=time_range) - features = features.astype('float32') + assert ( + epoch_from_name(latest_gen_checkpoint.stem) == epoch_from_name(latest_disc_checkpoint.stem) + ), "Latest disc and gen epochs differ" + + print(f'Loading generator weights from {str(latest_gen_checkpoint)}') + model.generator.load_weights(str(latest_gen_checkpoint)) + print(f'Loading discriminator weights from {str(latest_disc_checkpoint)}') + model.discriminator.load_weights(str(latest_disc_checkpoint)) + + return latest_gen_checkpoint, latest_disc_checkpoint + + +def get_images(model, + sample, + return_raw_data=False, + calc_chi2=False, + gen_more=None, + batch_size=128): + X, Y = sample + assert X.ndim == 2 + assert X.shape[1] == 4 + + if gen_more is None: + gen_features = X + else: + gen_features = np.tile( + X, + [gen_more] + [1] * (X.ndim - 1) + ) + gen_scaled = np.concatenate([ + model.make_fake(gen_features[i:i+batch_size]).numpy() + for i in range(0, len(gen_features), batch_size) + ], axis=0) + real = model.scaler.unscale(Y) + gen = model.scaler.unscale(gen_scaled) + gen[gen < 0] = 0 + gen1 = np.where(gen < 1., 0, gen) + + features = { + 'crossing_angle' : (X[:, 0], gen_features[:,0]), + 'dip_angle' : (X[:, 1], gen_features[:,1]), + 'drift_length' : (X[:, 2], gen_features[:,2]), + 'time_bin_fraction' : (X[:, 2] % 1, gen_features[:,2] % 1), + 'pad_coord_fraction' : (X[:, 3] % 1, gen_features[:,3] % 1) + } + + images = make_metric_plots(real, gen, features=features, calc_chi2=calc_chi2) + if calc_chi2: + images, chi2 = images + + images1 = make_metric_plots(real, gen1, features=features) + + img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) + + result = [images, images1, img_amplitude] + + if return_raw_data: + result += [(gen_features, gen)] + + if calc_chi2: + result += [chi2] + + return result + + +class SaveModelCallback: + def __init__(self, model, path, save_period): + self.model = model + self.path = path + self.save_period = save_period + + def __call__(self, step): + if step % self.save_period == 0: + print(f'Saving model on step {step} to {self.path}') + self.model.generator.save( + str(self.path.joinpath("generator_{:05d}.h5".format(step)))) + self.model.discriminator.save( + str(self.path.joinpath("discriminator_{:05d}.h5".format(step)))) + + +class WriteHistSummaryCallback: + def __init__(self, model, sample, save_period, writer): + self.model = model + self.sample = sample + self.save_period = save_period + self.writer = writer + + def __call__(self, step): + if step % self.save_period == 0: + images, images1, img_amplitude, chi2 = get_images(self.model, + sample=self.sample, + calc_chi2=True) + with self.writer.as_default(): + tf.summary.scalar("chi2", chi2, step) - data_scaled = np.log10(1 + data).astype('float32') - Y_train, Y_test, X_train, X_test = train_test_split(data_scaled, features, test_size=0.25, random_state=42) + for k, img in images.items(): + tf.summary.image(k, img, step) + for k, img in images1.items(): + tf.summary.image("{} (amp > 1)".format(k), img, step) + tf.summary.image("log10(amplitude + 1)", img_amplitude, step) - if not args.prediction_only: - writer_train = tf.summary.create_file_writer(f'logs/{args.checkpoint_name}/train') - writer_val = tf.summary.create_file_writer(f'logs/{args.checkpoint_name}/validation') - unscale = lambda x: 10 ** x - 1 +class ScheduleLRCallback: + def __init__(self, model, decay_rate, writer): + self.model = model + self.decay_rate = decay_rate + self.writer = writer - def get_images(return_raw_data=False, calc_chi2=False, gen_more=None, sample=(X_test, Y_test), batch_size=128): - X, Y = sample - assert X.ndim == 2 - assert X.shape[1] == 4 + def __call__(self, step): + self.model.disc_opt.lr.assign(self.model.disc_opt.lr * self.decay_rate) + self.model.gen_opt.lr.assign(self.model.gen_opt.lr * self.decay_rate) + with self.writer.as_default(): + tf.summary.scalar("discriminator learning rate", self.model.disc_opt.lr, step) + tf.summary.scalar("generator learning rate", self.model.gen_opt.lr, step) - if gen_more is None: - gen_features = X - else: - gen_features = np.tile( - X, - [gen_more] + [1] * (X.ndim - 1) - ) - gen_scaled = np.concatenate([ - model.make_fake(gen_features[i:i+batch_size]).numpy() - for i in range(0, len(gen_features), batch_size) - ], axis=0) - real = unscale(Y) - gen = unscale(gen_scaled) - gen[gen < 0] = 0 - gen1 = np.where(gen < 1., 0, gen) - features = { - 'crossing_angle' : (X[:, 0], gen_features[:,0]), - 'dip_angle' : (X[:, 1], gen_features[:,1]), - 'drift_length' : (X[:, 2], gen_features[:,2]), - 'time_bin_fraction' : (X[:, 2] % 1, gen_features[:,2] % 1), - 'pad_coord_fraction' : (X[:, 3] % 1, gen_features[:,3] % 1) - } +def evaluate_model(model, path, sample, gen_sample_name=None): + path.mkdir() + ( + images, images1, img_amplitude, + gen_dataset, chi2 + ) = get_images(model, sample=sample, + calc_chi2=True, return_raw_data=True, gen_more=10) - images = make_metric_plots(real, gen, features=features, calc_chi2=calc_chi2) - if calc_chi2: - images, chi2 = images + array_to_img = lambda arr: PIL.Image.fromarray(arr.reshape(arr.shape[1:])) - images1 = make_metric_plots(real, gen1, features=features) + for k, img in images.items(): + array_to_img(img).save(str(path / f"{k}.png")) + for k, img in images1.items(): + array_to_img(img).save(str(path / f"{k}_amp_gt_1.png")) + array_to_img(img_amplitude).save(str(path / "log10_amp_p_1.png")) - img_amplitude = make_histograms(Y_test.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) + if gen_sample_name is not None: + with open(str(path / gen_sample_name), 'w') as f: + for event_X, event_Y in zip(*gen_dataset): + f.write('params: {:.3f} {:.3f} {:.3f} {:.3f}\n'.format(*event_X)) + for ipad, time_distr in enumerate(event_Y, model.pad_range[0] + event_X[3].astype(int)): + for itime, amp in enumerate(time_distr, model.time_range[0] + event_X[2].astype(int)): + if amp < 1: + continue + f.write(" {:2d} {:3d} {:8.3e} ".format(ipad, itime, amp)) + f.write('\n') - result = [images, images1, img_amplitude] + with open(str(path / 'stats'), 'w') as f: + f.write(f"{chi2:.2f}\n") - if return_raw_data: - result += [(gen_features, gen)] - if calc_chi2: - result += [chi2] +def main(): + args = parse_args() - return result + cuda_gpu_config.setup_gpu(args.gpu_num) + model_path = Path('saved_models') / args.checkpoint_name - def write_hist_summary(step): - if step % args.save_every == 0: - images, images1, img_amplitude, chi2 = get_images(calc_chi2=True) + if args.prediction_only: + assert model_path.exists(), "Couldn't find model directory" + else: + assert not model_path.exists(), "Model directory already exists" + model_path.mkdir(parents=True) - with writer_val.as_default(): - tf.summary.scalar("chi2", chi2, step) + write_args(model_path) - for k, img in images.items(): - tf.summary.image(k, img, step) - for k, img in images1.items(): - tf.summary.image("{} (amp > 1)".format(k), img, step) - tf.summary.image("log10(amplitude + 1)", img_amplitude, step) + model = BaselineModel_8x16(kernel_init=args.kernel_init, lr=args.lr, + num_disc_updates=args.num_disc_updates, latent_dim=args.latent_dim, + gp_lambda=args.gp_lambda, gpdata_lambda=args.gpdata_lambda, + num_additional_layers=args.num_additional_disc_layers, + cramer=args.cramer_gan, features_to_tail=args.features_to_tail, + dropout_rate=args.dropout_rate, + stochastic_stepping=args.stochastic_stepping) + + if args.prediction_only: + latest_gen_checkpoint, latest_disc_checkpoint = load_weights(model, model_path) + preprocessing._VERSION = model.data_version + data, features = preprocessing.read_csv_2d(pad_range=model.pad_range, time_range=model.time_range) + features = features.astype('float32') + + data_scaled = model.scaler.scale(data).astype('float32') + + Y_train, Y_test, X_train, X_test = train_test_split(data_scaled, features, test_size=0.25, random_state=42) + + if not args.prediction_only: + writer_train = tf.summary.create_file_writer(f'logs/{args.checkpoint_name}/train') + writer_val = tf.summary.create_file_writer(f'logs/{args.checkpoint_name}/validation') - def schedule_lr(step): - model.disc_opt.lr.assign(model.disc_opt.lr * args.lr_schedule_rate) - model.gen_opt.lr.assign(model.gen_opt.lr * args.lr_schedule_rate) - with writer_val.as_default(): - tf.summary.scalar("discriminator learning rate", model.disc_opt.lr, step) - tf.summary.scalar("generator learning rate", model.gen_opt.lr, step) if args.prediction_only: prediction_path = model_path / f"prediction_{epoch_from_name(latest_gen_checkpoint.stem):05d}" assert not prediction_path.exists(), "Prediction path already exists" prediction_path.mkdir() - array_to_img = lambda arr: PIL.Image.fromarray(arr.reshape(arr.shape[1:])) - for part in ['train', 'test']: - path = prediction_path / part - path.mkdir() - ( - images, images1, img_amplitude, - gen_dataset, chi2 - ) = get_images( - calc_chi2=True, return_raw_data=True, gen_more=10, + evaluate_model( + model, path=prediction_path / part, sample=( - (X_train, Y_train) if part=='train' + (X_train, Y_train) if part == 'train' else (X_test, Y_test) - ) + ), + gen_sample_name=(None if part == 'train' else 'generated.dat') ) - for k, img in images.items(): - array_to_img(img).save(str(path / f"{k}.png")) - for k, img in images1.items(): - array_to_img(img).save(str(path / f"{k}_amp_gt_1.png")) - array_to_img(img_amplitude).save(str(path / "log10_amp_p_1.png")) - - if part == 'test': - with open(str(path / 'generated.dat'), 'w') as f: - for event_X, event_Y in zip(*gen_dataset): - f.write('params: {:.3f} {:.3f} {:.3f} {:.3f}\n'.format(*event_X)) - for ipad, time_distr in enumerate(event_Y, pad_range[0] + event_X[3].astype(int)): - for itime, amp in enumerate(time_distr, time_range[0] + event_X[2].astype(int)): - if amp < 1: - continue - f.write(" {:2d} {:3d} {:8.3e} ".format(ipad, itime, amp)) - f.write('\n') - - with open(str(path / 'stats'), 'w') as f: - f.write(f"{chi2:.2f}\n") else: features_noise = None @@ -256,6 +302,17 @@ def features_noise(epoch): return current_power + + save_model = SaveModelCallback( + model=model, path=model_path, save_period=args.save_every + ) + write_hist_summary = WriteHistSummaryCallback( + model, sample=(X_test, Y_test), + save_period=args.save_every, writer=writer_val + ) + schedule_lr = ScheduleLRCallback( + model, decay_rate=args.lr_schedule_rate, writer=writer_val + ) train(Y_train, Y_test, model.training_step, model.calculate_losses, args.num_epochs, args.batch_size, train_writer=writer_train, val_writer=writer_val, callbacks=[write_hist_summary, save_model, schedule_lr], From 0febbe6d25dba25ba9049c530090054b5627af77 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Thu, 24 Sep 2020 16:26:56 +0300 Subject: [PATCH 02/24] architecture from yaml [wip] --- models/architectures/baseline_fc_8x16.yaml | 39 ++++++ models/nn.py | 136 +++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 models/architectures/baseline_fc_8x16.yaml create mode 100644 models/nn.py diff --git a/models/architectures/baseline_fc_8x16.yaml b/models/architectures/baseline_fc_8x16.yaml new file mode 100644 index 0000000..43424da --- /dev/null +++ b/models/architectures/baseline_fc_8x16.yaml @@ -0,0 +1,39 @@ +generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128] + activations: ['relu', 'relu', 'relu', 'relu', 'relu'] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: [8, 16] + name: 'generator' + +discriminator: + - block_type: 'connect' + arguments: + vector_shape: [5,] + img_shape: [8, 16] + vector_bypass: False + concat_outputs: True + name: 'discriminator_tail' + block: + block_type: 'conv' + arguments: + filters: [16, 16, 32, 32, 64, 64] + kernel_sizes: [3, 3, 3, 3, 3, 2] + paddings: ['same', 'same', 'same', 'same', 'valid', 'valid'] + activations: ['relu', 'relu', 'relu', 'relu', 'relu', 'relu'] + poolings: [NULL, [1, 2], NULL, 2, NULL, NULL] + kernel_init: glorot_uniform + input_shape: NULL + output_shape: [64,] + dropouts: [0.02, 0.02, 0.02, 0.02, 0.02, 0.02] + name: discriminator_conv_block + - block_type: 'fully_connected' + arguments: + units: [128, 1] + activations: ['relu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [69,] + output_shape: NULL + name: 'discriminator_head' \ No newline at end of file diff --git a/models/nn.py b/models/nn.py new file mode 100644 index 0000000..4bc7d1e --- /dev/null +++ b/models/nn.py @@ -0,0 +1,136 @@ +import tensorflow as tf + + +def fully_connected_block(units, activations, + kernel_init='glorot_uniform', input_shape=None, + output_shape=None, dropouts=None, name=None): + assert len(units) == len(activations) + if dropouts: + assert len(dropouts) == len(units) + + layers = [] + for i, (size, act) in enumerate(zip(units, activations)): + args = dict(units=size, activation=act, kernel_initializer=kernel_init) + if i == 0 and input_shape: + args['input_shape'] = input_shape + + layers.append(tf.keras.layers.Dense(**args)) + + if dropouts and dropouts[i]: + layers.append(tf.keras.layers.Dropout(dropouts[i])) + + if output_shape: + layers.append(tf.keras.layers.Reshape(output_shape)) + + args = {} + if name: + args['name'] = name + + return tf.keras.Sequential(layers, **args) + + + +def conv_block(filters, kernel_sizes, paddings, activations, poolings, + kernel_init='glorot_uniform', input_shape=None, output_shape=None, + dropouts=None, name=None): + assert len(filters) == len(kernel_sizes) == len(paddings) == len(activations) == len(poolings) + if dropouts: + assert len(dropouts) == len(filters) + + layers = [] + for i, (nfilt, ksize, padding, act, pool) in enumerate(zip(filters, kernel_sizes, paddings, + activations, poolings)): + args = dict(filters=nfilt, kernel_size=ksize, + padding=padding, activation=act, kernel_initializer=kernel_init) + if i == 0 and input_shape: + args['input_shape'] = input_shape + + layers.append(tf.keras.layers.Conv2D(**args)) + + if dropouts and dropouts[i]: + layers.append(tf.keras.layers.Dropout(dropouts[i])) + + if pool: + layers.append(tf.keras.layers.MaxPool2D(pool)) + + if output_shape: + layers.append(tf.keras.layers.Reshape(output_shape)) + + args = {} + if name: + args['name'] = name + + return tf.keras.Sequential(layers, **args) + + +def vector_img_connect_block(vector_shape, img_shape, block, + vector_bypass=False, concat_outputs=True, name=None): + vector_shape = tuple(vector_shape) + img_shape = tuple(img_shape) + + assert len(vector_shape) == 1 + assert 2 <= len(img_shape) <= 3 + + input_vec = tf.keras.Input(shape=vector_shape) + input_img = tf.keras.Input(shape=img_shape) + + block_input = input_img + if len(img_shape) == 2: + block_input = tf.keras.layers.Reshape(img_shape + (1,))(block_input) + if not vector_bypass: + reshaped_vec = tf.tile( + tf.keras.layers.Reshape((1, 1) + vector_shape)(input_vec), + (1, *img_shape[:2], 1) + ) + block_input = tf.keras.layers.Concatenate(axis=-1)([reshaped_vec, block_input]) + + block_output = block(block_input) + + outputs = [input_vec, block_output] + if concat_outputs: + outputs = tf.keras.layers.Concatenate(axis=-1)(outputs) + + args = dict( + inputs=[input_vec, input_img], + outputs=outputs, + ) + + if name: + args['name'] = name + + return tf.keras.Model(**args) + + +def build_block(block_type, arguments): + if block_type == 'fully_connected': + block = fully_connected_block(**arguments) + elif block_type == 'conv': + block = conv_block(**arguments) + elif block_type == 'connect': + inner_block = build_block(**arguments['block']) + arguments['block'] = inner_block + block = vector_img_connect_block(**arguments) + else: + raise(NotImplementedError(block_type)) + + return block + +def build_architecture(block_descriptions, name=None): + blocks = [build_block(**descr) + for descr in block_descriptions] + + inputs = [ + tf.keras.Input(shape=i.shape[1:]) + for i in blocks[0].inputs + ] + outputs = inputs + for block in blocks: + outputs = block(outputs) + + args = dict( + inputs=inputs, + outputs=outputs + ) + if name: + args['name'] = name + return tf.keras.Model(**args) \ No newline at end of file From 956c603fabfe88fe6888d1bb87567add6192f5d0 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Thu, 24 Sep 2020 19:27:42 +0300 Subject: [PATCH 03/24] making use of the yaml constuctor; moving old code to legacy --- .../models}/baseline_10x10.py | 0 .../models}/baseline_10x15.py | 0 .../models}/baseline_fc_v4_8x16.py | 0 .../models}/baseline_v2_10x10.args.txt | 0 .../models}/baseline_v2_10x10.py | 0 .../models}/baseline_v3_6x15.py | 0 .../models}/baseline_v4_8x16.py | 0 .../test_script_data_v0.py | 0 .../test_script_data_v1.py | 0 .../test_script_data_v1_normed.py | 0 .../test_script_data_v2.py | 0 .../test_script_data_v3.py | 0 .../test_script_data_v4.py | 0 models/model_v4.py | 166 +++++++++ models/nn.py | 2 +- run_model_v4.py | 315 ++++++++++++++++++ 16 files changed, 482 insertions(+), 1 deletion(-) rename {models => legacy_code/models}/baseline_10x10.py (100%) rename {models => legacy_code/models}/baseline_10x15.py (100%) rename {models => legacy_code/models}/baseline_fc_v4_8x16.py (100%) rename {models => legacy_code/models}/baseline_v2_10x10.args.txt (100%) rename {models => legacy_code/models}/baseline_v2_10x10.py (100%) rename {models => legacy_code/models}/baseline_v3_6x15.py (100%) rename {models => legacy_code/models}/baseline_v4_8x16.py (100%) rename test_script_data_v0.py => legacy_code/test_script_data_v0.py (100%) rename test_script_data_v1.py => legacy_code/test_script_data_v1.py (100%) rename test_script_data_v1_normed.py => legacy_code/test_script_data_v1_normed.py (100%) rename test_script_data_v2.py => legacy_code/test_script_data_v2.py (100%) rename test_script_data_v3.py => legacy_code/test_script_data_v3.py (100%) rename test_script_data_v4.py => legacy_code/test_script_data_v4.py (100%) create mode 100644 models/model_v4.py create mode 100644 run_model_v4.py diff --git a/models/baseline_10x10.py b/legacy_code/models/baseline_10x10.py similarity index 100% rename from models/baseline_10x10.py rename to legacy_code/models/baseline_10x10.py diff --git a/models/baseline_10x15.py b/legacy_code/models/baseline_10x15.py similarity index 100% rename from models/baseline_10x15.py rename to legacy_code/models/baseline_10x15.py diff --git a/models/baseline_fc_v4_8x16.py b/legacy_code/models/baseline_fc_v4_8x16.py similarity index 100% rename from models/baseline_fc_v4_8x16.py rename to legacy_code/models/baseline_fc_v4_8x16.py diff --git a/models/baseline_v2_10x10.args.txt b/legacy_code/models/baseline_v2_10x10.args.txt similarity index 100% rename from models/baseline_v2_10x10.args.txt rename to legacy_code/models/baseline_v2_10x10.args.txt diff --git a/models/baseline_v2_10x10.py b/legacy_code/models/baseline_v2_10x10.py similarity index 100% rename from models/baseline_v2_10x10.py rename to legacy_code/models/baseline_v2_10x10.py diff --git a/models/baseline_v3_6x15.py b/legacy_code/models/baseline_v3_6x15.py similarity index 100% rename from models/baseline_v3_6x15.py rename to legacy_code/models/baseline_v3_6x15.py diff --git a/models/baseline_v4_8x16.py b/legacy_code/models/baseline_v4_8x16.py similarity index 100% rename from models/baseline_v4_8x16.py rename to legacy_code/models/baseline_v4_8x16.py diff --git a/test_script_data_v0.py b/legacy_code/test_script_data_v0.py similarity index 100% rename from test_script_data_v0.py rename to legacy_code/test_script_data_v0.py diff --git a/test_script_data_v1.py b/legacy_code/test_script_data_v1.py similarity index 100% rename from test_script_data_v1.py rename to legacy_code/test_script_data_v1.py diff --git a/test_script_data_v1_normed.py b/legacy_code/test_script_data_v1_normed.py similarity index 100% rename from test_script_data_v1_normed.py rename to legacy_code/test_script_data_v1_normed.py diff --git a/test_script_data_v2.py b/legacy_code/test_script_data_v2.py similarity index 100% rename from test_script_data_v2.py rename to legacy_code/test_script_data_v2.py diff --git a/test_script_data_v3.py b/legacy_code/test_script_data_v3.py similarity index 100% rename from test_script_data_v3.py rename to legacy_code/test_script_data_v3.py diff --git a/test_script_data_v4.py b/legacy_code/test_script_data_v4.py similarity index 100% rename from test_script_data_v4.py rename to legacy_code/test_script_data_v4.py diff --git a/models/model_v4.py b/models/model_v4.py new file mode 100644 index 0000000..c943574 --- /dev/null +++ b/models/model_v4.py @@ -0,0 +1,166 @@ +import tensorflow as tf +import yaml + +from . import scalers, nn + +@tf.function(experimental_relax_shapes=True) +def preprocess_features(features): + # features: + # crossing_angle [-20, 20] + # dip_angle [-60, 60] + # drift_length [35, 290] + # pad_coordinate [40-something, 40-something] + bin_fractions = features[:,-2:] % 1 + features = ( + features[:,:3] - tf.constant([[0., 0., 162.5]]) + ) / tf.constant([[20., 60., 127.5]]) + return tf.concat([features, bin_fractions], axis=-1) + +_f = preprocess_features + +def disc_loss(d_real, d_fake): + return tf.reduce_mean(d_fake - d_real) + + +def gen_loss(d_real, d_fake): + return tf.reduce_mean(d_real - d_fake) + + +def disc_loss_cramer(d_real, d_fake, d_fake_2): + return -tf.reduce_mean( + tf.norm(d_real - d_fake, axis=-1) + + tf.norm(d_fake_2, axis=-1) - + tf.norm(d_fake - d_fake_2, axis=-1) - + tf.norm(d_real, axis=-1) + ) + +def gen_loss_cramer(d_real, d_fake, d_fake_2): + return -disc_loss_cramer(d_real, d_fake, d_fake_2) + +class Model_v4: + def __init__(self, description_file='models/architectures/baseline_fc_8x16.yaml', + lr=1e-4, latent_dim=32, gp_lambda=10., num_disc_updates=8, + gpdata_lambda=0., cramer=False, stochastic_stepping=True): + self.disc_opt = tf.keras.optimizers.RMSprop(lr) + self.gen_opt = tf.keras.optimizers.RMSprop(lr) + self.latent_dim = latent_dim + self.gp_lambda = gp_lambda + self.gpdata_lambda = gpdata_lambda + self.num_disc_updates = num_disc_updates + self.cramer = cramer + self.stochastic_stepping = stochastic_stepping + + with open(description_file, 'r') as f: + architecture_descr = yaml.load(f, Loader=yaml.FullLoader) + self.generator = nn.build_architecture(architecture_descr['generator']) + self.discriminator = nn.build_architecture(architecture_descr['discriminator']) + + self.step_counter = tf.Variable(0, dtype='int32', trainable=False) + + self.scaler = scalers.Logarithmic() + self.pad_range = (-3, 5) + self.time_range = (-7, 9) + self.data_version = 'data_v4' + + + @tf.function + def make_fake(self, features): + size = tf.shape(features)[0] + latent_input = tf.random.normal(shape=(size, self.latent_dim), dtype='float32') + return self.generator( + tf.concat([_f(features), latent_input], axis=-1) + ) + + def gradient_penalty(self, features, real, fake): + alpha = tf.random.uniform(shape=[len(real), 1, 1]) + interpolates = alpha * real + (1 - alpha) * fake + with tf.GradientTape() as t: + t.watch(interpolates) + d_int = self.discriminator([_f(features), interpolates]) + + grads = tf.reshape(t.gradient(d_int, interpolates), [len(real), -1]) + return tf.reduce_mean(tf.maximum(tf.norm(grads, axis=-1) - 1, 0)**2) + + def gradient_penalty_on_data(self, features, real): + with tf.GradientTape() as t: + t.watch(real) + d_real = self.discriminator([_f(features), real]) + + grads = tf.reshape(t.gradient(d_real, real), [len(real), -1]) + return tf.reduce_mean(tf.reduce_sum(grads**2, axis=-1)) + + @tf.function + def calculate_losses(self, feature_batch, target_batch): + fake = self.make_fake(feature_batch) + d_real = self.discriminator([_f(feature_batch), target_batch]) + d_fake = self.discriminator([_f(feature_batch), fake]) + if self.cramer: + fake_2 = self.make_fake(feature_batch) + d_fake_2 = self.discriminator([_f(feature_batch), fake_2]) + + if not self.cramer: + d_loss = disc_loss(d_real, d_fake) + else: + d_loss = disc_loss_cramer(d_real, d_fake, d_fake_2) + + if self.gp_lambda > 0: + d_loss = ( + d_loss + + self.gradient_penalty( + feature_batch, target_batch, fake + ) * self.gp_lambda + ) + if self.gpdata_lambda > 0: + d_loss = ( + d_loss + + self.gradient_penalty_on_data( + feature_batch, target_batch + ) * self.gpdata_lambda + ) + if not self.cramer: + g_loss = gen_loss(d_real, d_fake) + else: + g_loss = gen_loss_cramer(d_real, d_fake, d_fake_2) + + return {'disc_loss': d_loss, 'gen_loss': g_loss} + + def disc_step(self, feature_batch, target_batch): + feature_batch = tf.convert_to_tensor(feature_batch) + target_batch = tf.convert_to_tensor(target_batch) + + with tf.GradientTape() as t: + losses = self.calculate_losses(feature_batch, target_batch) + + grads = t.gradient(losses['disc_loss'], self.discriminator.trainable_variables) + self.disc_opt.apply_gradients(zip(grads, self.discriminator.trainable_variables)) + return losses + + def gen_step(self, feature_batch, target_batch): + feature_batch = tf.convert_to_tensor(feature_batch) + target_batch = tf.convert_to_tensor(target_batch) + + with tf.GradientTape() as t: + losses = self.calculate_losses(feature_batch, target_batch) + + grads = t.gradient(losses['gen_loss'], self.generator.trainable_variables) + self.gen_opt.apply_gradients(zip(grads, self.generator.trainable_variables)) + return losses + + @tf.function + def training_step(self, feature_batch, target_batch): + if self.stochastic_stepping: + if tf.random.uniform( + shape=[], dtype='int32', + maxval=self.num_disc_updates + 1 + ) == self.num_disc_updates: + result = self.gen_step(feature_batch, target_batch) + else: + result = self.disc_step(feature_batch, target_batch) + else: + if self.step_counter == self.num_disc_updates: + result = self.gen_step(feature_batch, target_batch) + self.step_counter.assign(0) + else: + result = self.disc_step(feature_batch, target_batch) + self.step_counter.assign_add(1) + return result diff --git a/models/nn.py b/models/nn.py index 4bc7d1e..2807f24 100644 --- a/models/nn.py +++ b/models/nn.py @@ -82,7 +82,7 @@ def vector_img_connect_block(vector_shape, img_shape, block, tf.keras.layers.Reshape((1, 1) + vector_shape)(input_vec), (1, *img_shape[:2], 1) ) - block_input = tf.keras.layers.Concatenate(axis=-1)([reshaped_vec, block_input]) + block_input = tf.keras.layers.Concatenate(axis=-1)([block_input, reshaped_vec]) block_output = block(block_input) diff --git a/run_model_v4.py b/run_model_v4.py new file mode 100644 index 0000000..fa37d7c --- /dev/null +++ b/run_model_v4.py @@ -0,0 +1,315 @@ +import os, sys +import re +from pathlib import Path +import argparse + +import numpy as np +from sklearn.model_selection import train_test_split +import tensorflow as tf +import PIL + +from data import preprocessing +from models.training import train +from models.model_v4 import Model_v4 +from metrics import make_metric_plots, make_histograms +import cuda_gpu_config + +def make_parser(): + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + parser.add_argument('--checkpoint_name', type=str, required=True) + parser.add_argument('--batch_size', type=int, default=32, required=False) + parser.add_argument('--lr', type=float, default=1e-4, required=False) + parser.add_argument('--num_disc_updates', type=int, default=8, required=False) + parser.add_argument('--lr_schedule_rate', type=float, default=0.999, required=False) + parser.add_argument('--save_every', type=int, default=50, required=False) + parser.add_argument('--num_epochs', type=int, default=10000, required=False) + parser.add_argument('--latent_dim', type=int, default=32, required=False) + parser.add_argument('--gpu_num', type=str, required=False) + parser.add_argument('--gp_lambda', type=float, default=10., required=False) + parser.add_argument('--gpdata_lambda', type=float, default=0., required=False) + parser.add_argument('--cramer_gan', action='store_true', default=False) + parser.add_argument('--prediction_only', action='store_true', default=False) + parser.add_argument('--stochastic_stepping', action='store_true', default=True) + parser.add_argument('--feature_noise_power', type=float, default=None) + parser.add_argument('--feature_noise_decay', type=float, default=None) + return parser + + +def print_args(args): + print("") + print("----" * 10) + print("Arguments:") + for k, v in vars(args).items(): + print(f" {k} : {v}") + print("----" * 10) + print("") + + +def parse_args(): + args = make_parser().parse_args() + + assert ( + (args.feature_noise_power is None) == + (args.feature_noise_decay is None) + ), 'Noise power and decay must be both provided' + + print_args(args) + + return args + + +def write_args(model_path, fname='arguments.txt'): + with open(model_path / fname, 'w') as f: + raw_args = [a for a in sys.argv[1:] if a[0] != '@'] + fnames = [a[1:] for a in sys.argv[1:] if a[0] == '@'] + + f.write('\n'.join(raw_args)) + f.write('\n') + for fname in fnames: + with open(fname, 'r') as f_in: + f.write(f_in.read()) + + +def epoch_from_name(name): + epoch, = re.findall('\d+', name) + return int(epoch) + + +def load_weights(model, model_path): + gen_checkpoints = model_path.glob("generator_*.h5") + disc_checkpoints = model_path.glob("discriminator_*.h5") + latest_gen_checkpoint = max( + gen_checkpoints, + key=lambda path: epoch_from_name(path.stem) + ) + latest_disc_checkpoint = max( + disc_checkpoints, + key=lambda path: epoch_from_name(path.stem) + ) + + assert ( + epoch_from_name(latest_gen_checkpoint.stem) == epoch_from_name(latest_disc_checkpoint.stem) + ), "Latest disc and gen epochs differ" + + print(f'Loading generator weights from {str(latest_gen_checkpoint)}') + model.generator.load_weights(str(latest_gen_checkpoint)) + print(f'Loading discriminator weights from {str(latest_disc_checkpoint)}') + model.discriminator.load_weights(str(latest_disc_checkpoint)) + + return latest_gen_checkpoint, latest_disc_checkpoint + + +def get_images(model, + sample, + return_raw_data=False, + calc_chi2=False, + gen_more=None, + batch_size=128): + X, Y = sample + assert X.ndim == 2 + assert X.shape[1] == 4 + + if gen_more is None: + gen_features = X + else: + gen_features = np.tile( + X, + [gen_more] + [1] * (X.ndim - 1) + ) + gen_scaled = np.concatenate([ + model.make_fake(gen_features[i:i+batch_size]).numpy() + for i in range(0, len(gen_features), batch_size) + ], axis=0) + real = model.scaler.unscale(Y) + gen = model.scaler.unscale(gen_scaled) + gen[gen < 0] = 0 + gen1 = np.where(gen < 1., 0, gen) + + features = { + 'crossing_angle' : (X[:, 0], gen_features[:,0]), + 'dip_angle' : (X[:, 1], gen_features[:,1]), + 'drift_length' : (X[:, 2], gen_features[:,2]), + 'time_bin_fraction' : (X[:, 2] % 1, gen_features[:,2] % 1), + 'pad_coord_fraction' : (X[:, 3] % 1, gen_features[:,3] % 1) + } + + images = make_metric_plots(real, gen, features=features, calc_chi2=calc_chi2) + if calc_chi2: + images, chi2 = images + + images1 = make_metric_plots(real, gen1, features=features) + + img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) + + result = [images, images1, img_amplitude] + + if return_raw_data: + result += [(gen_features, gen)] + + if calc_chi2: + result += [chi2] + + return result + + +class SaveModelCallback: + def __init__(self, model, path, save_period): + self.model = model + self.path = path + self.save_period = save_period + + def __call__(self, step): + if step % self.save_period == 0: + print(f'Saving model on step {step} to {self.path}') + self.model.generator.save( + str(self.path.joinpath("generator_{:05d}.h5".format(step)))) + self.model.discriminator.save( + str(self.path.joinpath("discriminator_{:05d}.h5".format(step)))) + + +class WriteHistSummaryCallback: + def __init__(self, model, sample, save_period, writer): + self.model = model + self.sample = sample + self.save_period = save_period + self.writer = writer + + def __call__(self, step): + if step % self.save_period == 0: + images, images1, img_amplitude, chi2 = get_images(self.model, + sample=self.sample, + calc_chi2=True) + with self.writer.as_default(): + tf.summary.scalar("chi2", chi2, step) + + for k, img in images.items(): + tf.summary.image(k, img, step) + for k, img in images1.items(): + tf.summary.image("{} (amp > 1)".format(k), img, step) + tf.summary.image("log10(amplitude + 1)", img_amplitude, step) + + +class ScheduleLRCallback: + def __init__(self, model, decay_rate, writer): + self.model = model + self.decay_rate = decay_rate + self.writer = writer + + def __call__(self, step): + self.model.disc_opt.lr.assign(self.model.disc_opt.lr * self.decay_rate) + self.model.gen_opt.lr.assign(self.model.gen_opt.lr * self.decay_rate) + with self.writer.as_default(): + tf.summary.scalar("discriminator learning rate", self.model.disc_opt.lr, step) + tf.summary.scalar("generator learning rate", self.model.gen_opt.lr, step) + + +def evaluate_model(model, path, sample, gen_sample_name=None): + path.mkdir() + ( + images, images1, img_amplitude, + gen_dataset, chi2 + ) = get_images(model, sample=sample, + calc_chi2=True, return_raw_data=True, gen_more=10) + + array_to_img = lambda arr: PIL.Image.fromarray(arr.reshape(arr.shape[1:])) + + for k, img in images.items(): + array_to_img(img).save(str(path / f"{k}.png")) + for k, img in images1.items(): + array_to_img(img).save(str(path / f"{k}_amp_gt_1.png")) + array_to_img(img_amplitude).save(str(path / "log10_amp_p_1.png")) + + if gen_sample_name is not None: + with open(str(path / gen_sample_name), 'w') as f: + for event_X, event_Y in zip(*gen_dataset): + f.write('params: {:.3f} {:.3f} {:.3f} {:.3f}\n'.format(*event_X)) + for ipad, time_distr in enumerate(event_Y, model.pad_range[0] + event_X[3].astype(int)): + for itime, amp in enumerate(time_distr, model.time_range[0] + event_X[2].astype(int)): + if amp < 1: + continue + f.write(" {:2d} {:3d} {:8.3e} ".format(ipad, itime, amp)) + f.write('\n') + + with open(str(path / 'stats'), 'w') as f: + f.write(f"{chi2:.2f}\n") + + +def main(): + args = parse_args() + + cuda_gpu_config.setup_gpu(args.gpu_num) + + model_path = Path('saved_models') / args.checkpoint_name + + if args.prediction_only: + assert model_path.exists(), "Couldn't find model directory" + else: + assert not model_path.exists(), "Model directory already exists" + model_path.mkdir(parents=True) + + write_args(model_path) + + model = Model_v4(lr=args.lr, latent_dim=args.latent_dim, gp_lambda=args.gp_lambda, + num_disc_updates=args.num_disc_updates, gpdata_lambda=args.gpdata_lambda, + cramer=args.cramer_gan, stochastic_stepping=args.stochastic_stepping) + + if args.prediction_only: + latest_gen_checkpoint, latest_disc_checkpoint = load_weights(model, model_path) + + preprocessing._VERSION = model.data_version + data, features = preprocessing.read_csv_2d(pad_range=model.pad_range, time_range=model.time_range) + features = features.astype('float32') + + data_scaled = model.scaler.scale(data).astype('float32') + + Y_train, Y_test, X_train, X_test = train_test_split(data_scaled, features, test_size=0.25, random_state=42) + + if not args.prediction_only: + writer_train = tf.summary.create_file_writer(f'logs/{args.checkpoint_name}/train') + writer_val = tf.summary.create_file_writer(f'logs/{args.checkpoint_name}/validation') + + + if args.prediction_only: + prediction_path = model_path / f"prediction_{epoch_from_name(latest_gen_checkpoint.stem):05d}" + assert not prediction_path.exists(), "Prediction path already exists" + prediction_path.mkdir() + + for part in ['train', 'test']: + evaluate_model( + model, path=prediction_path / part, + sample=( + (X_train, Y_train) if part == 'train' + else (X_test, Y_test) + ), + gen_sample_name=(None if part == 'train' else 'generated.dat') + ) + + else: + features_noise = None + if args.feature_noise_power is not None: + def features_noise(epoch): + current_power = args.feature_noise_power / (10**(epoch / args.feature_noise_decay)) + with writer_train.as_default(): + tf.summary.scalar("features noise power", current_power, epoch) + + return current_power + + + save_model = SaveModelCallback( + model=model, path=model_path, save_period=args.save_every + ) + write_hist_summary = WriteHistSummaryCallback( + model, sample=(X_test, Y_test), + save_period=args.save_every, writer=writer_val + ) + schedule_lr = ScheduleLRCallback( + model, decay_rate=args.lr_schedule_rate, writer=writer_val + ) + train(Y_train, Y_test, model.training_step, model.calculate_losses, args.num_epochs, args.batch_size, + train_writer=writer_train, val_writer=writer_val, + callbacks=[write_hist_summary, save_model, schedule_lr], + features_train=X_train, features_val=X_test, features_noise=features_noise) + + +if __name__ == '__main__': + main() From eb005199f633e4f59e02afef71e4d49587c7ded8 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Sun, 27 Sep 2020 20:13:03 +0300 Subject: [PATCH 04/24] Moving all model setup to yaml config --- models/architectures/baseline_fc_8x16.yaml | 39 ------------- models/configs/baseline_fc_8x16.yaml | 62 ++++++++++++++++++++ models/model_v4.py | 34 +++++------ models/scalers.py | 10 +++- run_model_v4.py | 68 +++++++++------------- 5 files changed, 115 insertions(+), 98 deletions(-) delete mode 100644 models/architectures/baseline_fc_8x16.yaml create mode 100644 models/configs/baseline_fc_8x16.yaml diff --git a/models/architectures/baseline_fc_8x16.yaml b/models/architectures/baseline_fc_8x16.yaml deleted file mode 100644 index 43424da..0000000 --- a/models/architectures/baseline_fc_8x16.yaml +++ /dev/null @@ -1,39 +0,0 @@ -generator: - - block_type: 'fully_connected' - arguments: - units: [32, 64, 64, 64, 128] - activations: ['relu', 'relu', 'relu', 'relu', 'relu'] - kernel_init: 'glorot_uniform' - input_shape: [37,] - output_shape: [8, 16] - name: 'generator' - -discriminator: - - block_type: 'connect' - arguments: - vector_shape: [5,] - img_shape: [8, 16] - vector_bypass: False - concat_outputs: True - name: 'discriminator_tail' - block: - block_type: 'conv' - arguments: - filters: [16, 16, 32, 32, 64, 64] - kernel_sizes: [3, 3, 3, 3, 3, 2] - paddings: ['same', 'same', 'same', 'same', 'valid', 'valid'] - activations: ['relu', 'relu', 'relu', 'relu', 'relu', 'relu'] - poolings: [NULL, [1, 2], NULL, 2, NULL, NULL] - kernel_init: glorot_uniform - input_shape: NULL - output_shape: [64,] - dropouts: [0.02, 0.02, 0.02, 0.02, 0.02, 0.02] - name: discriminator_conv_block - - block_type: 'fully_connected' - arguments: - units: [128, 1] - activations: ['relu', NULL] - kernel_init: 'glorot_uniform' - input_shape: [69,] - output_shape: NULL - name: 'discriminator_head' \ No newline at end of file diff --git a/models/configs/baseline_fc_8x16.yaml b/models/configs/baseline_fc_8x16.yaml new file mode 100644 index 0000000..42b5787 --- /dev/null +++ b/models/configs/baseline_fc_8x16.yaml @@ -0,0 +1,62 @@ +latent_dim: 32 +batch_size: 32 +lr: 1.e-4 +lr_schedule_rate: 0.999 + +num_disc_updates: 8 +gp_lambda: 10. +gpdata_lambda: 0. +cramer: False +stochastic_stepping: True + +save_every: 50 +num_epochs: 10000 + +feature_noise_power: NULL +feature_noise_decay: NULL + +data_version: 'data_v4' +pad_range: [-3, 5] +time_range: [-7, 9] +scaler: 'logarithmic' + +architecture: + generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128] + activations: ['relu', 'relu', 'relu', 'relu', 'relu'] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: [8, 16] + name: 'generator' + + discriminator: + - block_type: 'connect' + arguments: + vector_shape: [5,] + img_shape: [8, 16] + vector_bypass: False + concat_outputs: True + name: 'discriminator_tail' + block: + block_type: 'conv' + arguments: + filters: [16, 16, 32, 32, 64, 64] + kernel_sizes: [3, 3, 3, 3, 3, 2] + paddings: ['same', 'same', 'same', 'same', 'valid', 'valid'] + activations: ['relu', 'relu', 'relu', 'relu', 'relu', 'relu'] + poolings: [NULL, [1, 2], NULL, 2, NULL, NULL] + kernel_init: glorot_uniform + input_shape: NULL + output_shape: [64,] + dropouts: [0.02, 0.02, 0.02, 0.02, 0.02, 0.02] + name: discriminator_conv_block + - block_type: 'fully_connected' + arguments: + units: [128, 1] + activations: ['relu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [69,] + output_shape: NULL + name: 'discriminator_head' \ No newline at end of file diff --git a/models/model_v4.py b/models/model_v4.py index c943574..0b92176 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -1,5 +1,4 @@ import tensorflow as tf -import yaml from . import scalers, nn @@ -38,29 +37,26 @@ def gen_loss_cramer(d_real, d_fake, d_fake_2): return -disc_loss_cramer(d_real, d_fake, d_fake_2) class Model_v4: - def __init__(self, description_file='models/architectures/baseline_fc_8x16.yaml', - lr=1e-4, latent_dim=32, gp_lambda=10., num_disc_updates=8, - gpdata_lambda=0., cramer=False, stochastic_stepping=True): - self.disc_opt = tf.keras.optimizers.RMSprop(lr) - self.gen_opt = tf.keras.optimizers.RMSprop(lr) - self.latent_dim = latent_dim - self.gp_lambda = gp_lambda - self.gpdata_lambda = gpdata_lambda - self.num_disc_updates = num_disc_updates - self.cramer = cramer - self.stochastic_stepping = stochastic_stepping - - with open(description_file, 'r') as f: - architecture_descr = yaml.load(f, Loader=yaml.FullLoader) + def __init__(self, config): + self.disc_opt = tf.keras.optimizers.RMSprop(config['lr']) + self.gen_opt = tf.keras.optimizers.RMSprop(config['lr']) + self.gp_lambda = config['gp_lambda'] + self.gpdata_lambda = config['gpdata_lambda'] + self.num_disc_updates = config['num_disc_updates'] + self.cramer = config['cramer'] + self.stochastic_stepping = config['stochastic_stepping'] + self.latent_dim = config['latent_dim'] + + architecture_descr = config['architecture'] self.generator = nn.build_architecture(architecture_descr['generator']) self.discriminator = nn.build_architecture(architecture_descr['discriminator']) self.step_counter = tf.Variable(0, dtype='int32', trainable=False) - self.scaler = scalers.Logarithmic() - self.pad_range = (-3, 5) - self.time_range = (-7, 9) - self.data_version = 'data_v4' + self.scaler = scalers.get_scaler(config['scaler']) + self.pad_range = tuple(config['pad_range']) + self.time_range = tuple(config['time_range']) + self.data_version = config['data_version'] @tf.function diff --git a/models/scalers.py b/models/scalers.py index 91ebf46..f1e8303 100644 --- a/models/scalers.py +++ b/models/scalers.py @@ -14,4 +14,12 @@ def scale(self, x): return np.log10(1 + x) def unscale(self, x): - return 10 ** x - 1 \ No newline at end of file + return 10 ** x - 1 + +def get_scaler(scaler_type): + if scaler_type == 'identity': + return Identity() + elif scaler_type == 'logarithmic': + return Logarithmic() + else: + raise NotImplementedError(scaler_type) diff --git a/run_model_v4.py b/run_model_v4.py index fa37d7c..eab7429 100644 --- a/run_model_v4.py +++ b/run_model_v4.py @@ -1,12 +1,14 @@ import os, sys import re from pathlib import Path +import shutil import argparse import numpy as np from sklearn.model_selection import train_test_split import tensorflow as tf import PIL +import yaml from data import preprocessing from models.training import train @@ -16,22 +18,11 @@ def make_parser(): parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + parser.add_argument('--config', type=str, required=False) parser.add_argument('--checkpoint_name', type=str, required=True) - parser.add_argument('--batch_size', type=int, default=32, required=False) - parser.add_argument('--lr', type=float, default=1e-4, required=False) - parser.add_argument('--num_disc_updates', type=int, default=8, required=False) - parser.add_argument('--lr_schedule_rate', type=float, default=0.999, required=False) - parser.add_argument('--save_every', type=int, default=50, required=False) - parser.add_argument('--num_epochs', type=int, default=10000, required=False) - parser.add_argument('--latent_dim', type=int, default=32, required=False) parser.add_argument('--gpu_num', type=str, required=False) - parser.add_argument('--gp_lambda', type=float, default=10., required=False) - parser.add_argument('--gpdata_lambda', type=float, default=0., required=False) - parser.add_argument('--cramer_gan', action='store_true', default=False) parser.add_argument('--prediction_only', action='store_true', default=False) - parser.add_argument('--stochastic_stepping', action='store_true', default=True) - parser.add_argument('--feature_noise_power', type=float, default=None) - parser.add_argument('--feature_noise_decay', type=float, default=None) + return parser @@ -48,27 +39,20 @@ def print_args(args): def parse_args(): args = make_parser().parse_args() - assert ( - (args.feature_noise_power is None) == - (args.feature_noise_decay is None) - ), 'Noise power and decay must be both provided' - print_args(args) return args +def load_config(file): + with open(file, 'r') as f: + config = yaml.load(f, Loader=yaml.FullLoader) -def write_args(model_path, fname='arguments.txt'): - with open(model_path / fname, 'w') as f: - raw_args = [a for a in sys.argv[1:] if a[0] != '@'] - fnames = [a[1:] for a in sys.argv[1:] if a[0] == '@'] - - f.write('\n'.join(raw_args)) - f.write('\n') - for fname in fnames: - with open(fname, 'r') as f_in: - f.write(f_in.read()) + assert ( + (config['feature_noise_power'] is None) == + (config['feature_noise_decay'] is None) + ), 'Noise power and decay must be both provided' + return config def epoch_from_name(name): epoch, = re.findall('\d+', name) @@ -243,15 +227,21 @@ def main(): if args.prediction_only: assert model_path.exists(), "Couldn't find model directory" + assert not args.config, "Config should be read from model path when doing prediction" + args.config = str(model_path / 'config.yaml') else: assert not model_path.exists(), "Model directory already exists" + assert args.config, "No config provided" + model_path.mkdir(parents=True) + config_destination = str(model_path / 'config.yaml') + shutil.copy(args.config, config_destination) + + args.config = config_destination - write_args(model_path) + config = load_config(args.config) - model = Model_v4(lr=args.lr, latent_dim=args.latent_dim, gp_lambda=args.gp_lambda, - num_disc_updates=args.num_disc_updates, gpdata_lambda=args.gpdata_lambda, - cramer=args.cramer_gan, stochastic_stepping=args.stochastic_stepping) + model = Model_v4(config) if args.prediction_only: latest_gen_checkpoint, latest_disc_checkpoint = load_weights(model, model_path) @@ -286,9 +276,9 @@ def main(): else: features_noise = None - if args.feature_noise_power is not None: + if config['feature_noise_power'] is not None: def features_noise(epoch): - current_power = args.feature_noise_power / (10**(epoch / args.feature_noise_decay)) + current_power = config['feature_noise_power'] / (10**(epoch / config['feature_noise_decay'])) with writer_train.as_default(): tf.summary.scalar("features noise power", current_power, epoch) @@ -296,17 +286,17 @@ def features_noise(epoch): save_model = SaveModelCallback( - model=model, path=model_path, save_period=args.save_every + model=model, path=model_path, save_period=config['save_every'] ) write_hist_summary = WriteHistSummaryCallback( model, sample=(X_test, Y_test), - save_period=args.save_every, writer=writer_val + save_period=config['save_every'], writer=writer_val ) schedule_lr = ScheduleLRCallback( - model, decay_rate=args.lr_schedule_rate, writer=writer_val + model, decay_rate=config['lr_schedule_rate'], writer=writer_val ) - train(Y_train, Y_test, model.training_step, model.calculate_losses, args.num_epochs, args.batch_size, - train_writer=writer_train, val_writer=writer_val, + train(Y_train, Y_test, model.training_step, model.calculate_losses, config['num_epochs'], + config['batch_size'], train_writer=writer_train, val_writer=writer_val, callbacks=[write_hist_summary, save_model, schedule_lr], features_train=X_train, features_val=X_test, features_noise=features_noise) From c92f90003bb29a35758659f5c3129e476cc2c6ae Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Sun, 27 Sep 2020 21:08:21 +0300 Subject: [PATCH 05/24] refactoring metric plots and callbacks --- metrics/__init__.py | 243 +++++++------------- metrics/gaussian_metrics.py | 62 +++++ plotting/__init__.py => metrics/plotting.py | 0 metrics/trends.py | 110 +++++++++ models/callbacks.py | 53 +++++ run_model_v4.py | 142 +----------- 6 files changed, 312 insertions(+), 298 deletions(-) create mode 100644 metrics/gaussian_metrics.py rename plotting/__init__.py => metrics/plotting.py (100%) create mode 100644 metrics/trends.py create mode 100644 models/callbacks.py diff --git a/metrics/__init__.py b/metrics/__init__.py index 4143dbc..71d0ebd 100644 --- a/metrics/__init__.py +++ b/metrics/__init__.py @@ -7,68 +7,9 @@ import PIL -from plotting import _bootstrap_error - - -def _gaussian_fit(img): - assert img.ndim == 2, '_gaussian_fit: Wrong image dimentions' - assert (img >= 0).all(), '_gaussian_fit: negative image content' - assert (img > 0).any(), '_gaussian_fit: blank image' - img_n = img / img.sum() - - mu = np.fromfunction( - lambda i, j: (img_n[np.newaxis,...] * np.stack([i, j])).sum(axis=(1, 2)), - shape=img.shape - ) - cov = np.fromfunction( - lambda i, j: ( - (img_n[np.newaxis,...] * np.stack([i * i, j * i, i * j, j * j])).sum(axis=(1, 2)) - ) - np.stack([mu[0]**2, mu[0]*mu[1], mu[0]*mu[1], mu[1]**2]), - shape=img.shape - ).reshape(2, 2) - return mu, cov - - -def _get_val_metric_single(img): - """Returns a vector of gaussian fit results to the image. - The components are: [mu0, mu1, sigma0^2, sigma1^2, covariance, integral] - """ - assert img.ndim == 2, '_get_val_metric_single: Wrong image dimentions' - - img = np.where(img < 0, 0, img) - - mu, cov = _gaussian_fit(img) - - return np.array((*mu, *cov.diagonal(), cov[0, 1], img.sum())) - - -_METRIC_NAMES = ['Mean0', 'Mean1', 'Sigma0^2', 'Sigma1^2', 'Cov01', 'Sum'] - - -get_val_metric = np.vectorize(_get_val_metric_single, signature='(m,n)->(k)') - - -def get_val_metric_v(imgs): - """Returns a vector of gaussian fit results to the image. - The components are: [mu0, mu1, sigma0^2, sigma1^2, covariance, integral] - """ - assert imgs.ndim == 3, 'get_val_metric_v: Wrong images dimentions' - assert (imgs >= 0).all(), 'get_val_metric_v: Negative image content' - assert (imgs > 0).any(axis=(1, 2)).all(), 'get_val_metric_v: some images are empty' - imgs_n = imgs / imgs.sum(axis=(1, 2), keepdims=True) - mu = np.fromfunction( - lambda i, j: (imgs_n[:,np.newaxis,...] * np.stack([i, j])[np.newaxis,...]).sum(axis=(2, 3)), - shape=imgs.shape[1:] - ) - - cov = np.fromfunction( - lambda i, j: ( - (imgs_n[:,np.newaxis,...] * np.stack([i * i, j * j, i * j])[np.newaxis,...]).sum(axis=(2, 3)) - ) - np.stack([mu[:,0]**2, mu[:,1]**2, mu[:,0] * mu[:,1]]).T, - shape=imgs.shape[1:] - ) - - return np.concatenate([mu, cov, imgs.sum(axis=(1, 2))[:,np.newaxis]], axis=1) +from .plotting import _bootstrap_error +from .gaussian_metrics import get_val_metric_v, _METRIC_NAMES +from .trends import make_trend_plot def make_histograms(data_real, data_gen, title, figsize=(8, 8), n_bins=100, logy=False): @@ -110,13 +51,13 @@ def make_metric_plots(images_real, images_gen, features=None, calc_chi2=False): for metric_name, real, gen in zip(_METRIC_NAMES, metric_real.T, metric_gen.T): name = f'{metric_name} vs {feature_name}' if calc_chi2 and (metric_name != "Sum"): - plots[name], chi2_i = make_trend(feature_real, real, - feature_gen, gen, - name, calc_chi2=True) + plots[name], chi2_i = make_trend_plot(feature_real, real, + feature_gen, gen, + name, calc_chi2=True) chi2 += chi2_i else: - plots[name] = make_trend(feature_real, real, - feature_gen, gen, name) + plots[name] = make_trend_plot(feature_real, real, + feature_gen, gen, name) except AssertionError as e: print(f"WARNING! Assertion error ({e})") @@ -126,104 +67,86 @@ def make_metric_plots(images_real, images_gen, features=None, calc_chi2=False): return plots -def calc_trend(x, y, do_plot=True, bins=100, window_size=20, **kwargs): - assert x.ndim == 1, 'calc_trend: wrong x dim' - assert y.ndim == 1, 'calc_trend: wrong y dim' - - if 'alpha' not in kwargs: - kwargs['alpha'] = 0.7 - if isinstance(bins, int): - bins = np.linspace(np.min(x), np.max(x), bins + 1) - sel = (x >= bins[0]) - x, y = x[sel], y[sel] - cats = (x[:,np.newaxis] < bins[np.newaxis,1:]).argmax(axis=1) - - def stats(arr): - return ( - arr.mean(), - arr.std() / (len(arr) - 1)**0.5, - arr.std(), - _bootstrap_error(arr, np.std) +def make_images_for_model(model, + sample, + return_raw_data=False, + calc_chi2=False, + gen_more=None, + batch_size=128): + X, Y = sample + assert X.ndim == 2 + assert X.shape[1] == 4 + + if gen_more is None: + gen_features = X + else: + gen_features = np.tile( + X, + [gen_more] + [1] * (X.ndim - 1) ) - - mean, mean_err, std, std_err, bin_centers = np.array([ - stats( - y[(cats >= left) & (cats < right)] - ) + ((bins[left] + bins[right]) / 2,) for left, right in zip( - range(len(bins) - window_size), - range(window_size, len(bins)) - ) - ]).T - - - if do_plot: - mean_p_std_err = (mean_err**2 + std_err**2)**0.5 - plt.fill_between(bin_centers, mean - mean_err, mean + mean_err, **kwargs) - kwargs['alpha'] *= 0.5 - kwargs = {k : v for k, v in kwargs.items() if k != 'label'} - plt.fill_between(bin_centers, mean - std - mean_p_std_err, mean - std + mean_p_std_err, **kwargs) - plt.fill_between(bin_centers, mean + std - mean_p_std_err, mean + std + mean_p_std_err, **kwargs) - kwargs['alpha'] *= 0.25 - plt.fill_between(bin_centers, mean - std + mean_p_std_err, mean + std - mean_p_std_err, **kwargs) - - return (mean, std), (mean_err, std_err) - + gen_scaled = np.concatenate([ + model.make_fake(gen_features[i:i+batch_size]).numpy() + for i in range(0, len(gen_features), batch_size) + ], axis=0) + real = model.scaler.unscale(Y) + gen = model.scaler.unscale(gen_scaled) + gen[gen < 0] = 0 + gen1 = np.where(gen < 1., 0, gen) + + features = { + 'crossing_angle' : (X[:, 0], gen_features[:,0]), + 'dip_angle' : (X[:, 1], gen_features[:,1]), + 'drift_length' : (X[:, 2], gen_features[:,2]), + 'time_bin_fraction' : (X[:, 2] % 1, gen_features[:,2] % 1), + 'pad_coord_fraction' : (X[:, 3] % 1, gen_features[:,3] % 1) + } + + images = make_metric_plots(real, gen, features=features, calc_chi2=calc_chi2) + if calc_chi2: + images, chi2 = images -def make_trend(feature_real, real, feature_gen, gen, name, calc_chi2=False, figsize=(8, 8)): - feature_real = feature_real.squeeze() - feature_gen = feature_gen.squeeze() - real = real.squeeze() - gen = gen.squeeze() + images1 = make_metric_plots(real, gen1, features=features) - bins = np.linspace( - min(feature_real.min(), feature_gen.min()), - max(feature_real.max(), feature_gen.max()), - 100 - ) + img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) - fig = plt.figure(figsize=figsize) - calc_trend(feature_real, real, bins=bins, label='real', color='blue') - calc_trend(feature_gen, gen, bins=bins, label='generated', color='red') - plt.legend() - plt.title(name) + result = [images, images1, img_amplitude] - buf = io.BytesIO() - fig.savefig(buf, format='png') - plt.close(fig) - buf.seek(0) - - img = PIL.Image.open(buf) - img_data = np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[0], img.size[1], -1) + if return_raw_data: + result += [(gen_features, gen)] if calc_chi2: - bins = np.linspace( - min(feature_real.min(), feature_gen.min()), - max(feature_real.max(), feature_gen.max()), - 20 - ) - ( - (real_mean, real_std), - (real_mean_err, real_std_err) - ) = calc_trend(feature_real, real, do_plot=False, bins=bins, window_size=1) - ( - (gen_mean, gen_std), - (gen_mean_err, gen_std_err) - ) = calc_trend(feature_gen, gen, do_plot=False, bins=bins, window_size=1) - - gen_upper = gen_mean + gen_std - gen_lower = gen_mean - gen_std - gen_err2 = gen_mean_err**2 + gen_std_err**2 - - real_upper = real_mean + real_std - real_lower = real_mean - real_std - real_err2 = real_mean_err**2 + real_std_err**2 - - chi2 = ( - ((gen_upper - real_upper)**2 / (gen_err2 + real_err2)).sum() + - ((gen_lower - real_lower)**2 / (gen_err2 + real_err2)).sum() - ) - - return img_data, chi2 - - return img_data + result += [chi2] + + return result + + +def evaluate_model(model, path, sample, gen_sample_name=None): + path.mkdir() + ( + images, images1, img_amplitude, + gen_dataset, chi2 + ) = make_images_for_model(model, sample=sample, + calc_chi2=True, return_raw_data=True, gen_more=10) + + array_to_img = lambda arr: PIL.Image.fromarray(arr.reshape(arr.shape[1:])) + + for k, img in images.items(): + array_to_img(img).save(str(path / f"{k}.png")) + for k, img in images1.items(): + array_to_img(img).save(str(path / f"{k}_amp_gt_1.png")) + array_to_img(img_amplitude).save(str(path / "log10_amp_p_1.png")) + + if gen_sample_name is not None: + with open(str(path / gen_sample_name), 'w') as f: + for event_X, event_Y in zip(*gen_dataset): + f.write('params: {:.3f} {:.3f} {:.3f} {:.3f}\n'.format(*event_X)) + for ipad, time_distr in enumerate(event_Y, model.pad_range[0] + event_X[3].astype(int)): + for itime, amp in enumerate(time_distr, model.time_range[0] + event_X[2].astype(int)): + if amp < 1: + continue + f.write(" {:2d} {:3d} {:8.3e} ".format(ipad, itime, amp)) + f.write('\n') + + with open(str(path / 'stats'), 'w') as f: + f.write(f"{chi2:.2f}\n") \ No newline at end of file diff --git a/metrics/gaussian_metrics.py b/metrics/gaussian_metrics.py new file mode 100644 index 0000000..af490d0 --- /dev/null +++ b/metrics/gaussian_metrics.py @@ -0,0 +1,62 @@ +import numpy as np + + +def _gaussian_fit(img): + assert img.ndim == 2, '_gaussian_fit: Wrong image dimentions' + assert (img >= 0).all(), '_gaussian_fit: negative image content' + assert (img > 0).any(), '_gaussian_fit: blank image' + img_n = img / img.sum() + + mu = np.fromfunction( + lambda i, j: (img_n[np.newaxis,...] * np.stack([i, j])).sum(axis=(1, 2)), + shape=img.shape + ) + cov = np.fromfunction( + lambda i, j: ( + (img_n[np.newaxis,...] * np.stack([i * i, j * i, i * j, j * j])).sum(axis=(1, 2)) + ) - np.stack([mu[0]**2, mu[0]*mu[1], mu[0]*mu[1], mu[1]**2]), + shape=img.shape + ).reshape(2, 2) + return mu, cov + + +def _get_val_metric_single(img): + """Returns a vector of gaussian fit results to the image. + The components are: [mu0, mu1, sigma0^2, sigma1^2, covariance, integral] + """ + assert img.ndim == 2, '_get_val_metric_single: Wrong image dimentions' + + img = np.where(img < 0, 0, img) + + mu, cov = _gaussian_fit(img) + + return np.array((*mu, *cov.diagonal(), cov[0, 1], img.sum())) + + +_METRIC_NAMES = ['Mean0', 'Mean1', 'Sigma0^2', 'Sigma1^2', 'Cov01', 'Sum'] + + +get_val_metric = np.vectorize(_get_val_metric_single, signature='(m,n)->(k)') + + +def get_val_metric_v(imgs): + """Returns a vector of gaussian fit results to the image. + The components are: [mu0, mu1, sigma0^2, sigma1^2, covariance, integral] + """ + assert imgs.ndim == 3, 'get_val_metric_v: Wrong images dimentions' + assert (imgs >= 0).all(), 'get_val_metric_v: Negative image content' + assert (imgs > 0).any(axis=(1, 2)).all(), 'get_val_metric_v: some images are empty' + imgs_n = imgs / imgs.sum(axis=(1, 2), keepdims=True) + mu = np.fromfunction( + lambda i, j: (imgs_n[:,np.newaxis,...] * np.stack([i, j])[np.newaxis,...]).sum(axis=(2, 3)), + shape=imgs.shape[1:] + ) + + cov = np.fromfunction( + lambda i, j: ( + (imgs_n[:,np.newaxis,...] * np.stack([i * i, j * j, i * j])[np.newaxis,...]).sum(axis=(2, 3)) + ) - np.stack([mu[:,0]**2, mu[:,1]**2, mu[:,0] * mu[:,1]]).T, + shape=imgs.shape[1:] + ) + + return np.concatenate([mu, cov, imgs.sum(axis=(1, 2))[:,np.newaxis]], axis=1) \ No newline at end of file diff --git a/plotting/__init__.py b/metrics/plotting.py similarity index 100% rename from plotting/__init__.py rename to metrics/plotting.py diff --git a/metrics/trends.py b/metrics/trends.py new file mode 100644 index 0000000..89025dc --- /dev/null +++ b/metrics/trends.py @@ -0,0 +1,110 @@ +import io + +import numpy as np +import matplotlib.pyplot as plt +import PIL + +from .plotting import _bootstrap_error + + +def calc_trend(x, y, do_plot=True, bins=100, window_size=20, **kwargs): + assert x.ndim == 1, 'calc_trend: wrong x dim' + assert y.ndim == 1, 'calc_trend: wrong y dim' + + if 'alpha' not in kwargs: + kwargs['alpha'] = 0.7 + + if isinstance(bins, int): + bins = np.linspace(np.min(x), np.max(x), bins + 1) + sel = (x >= bins[0]) + x, y = x[sel], y[sel] + cats = (x[:,np.newaxis] < bins[np.newaxis,1:]).argmax(axis=1) + + def stats(arr): + return ( + arr.mean(), + arr.std() / (len(arr) - 1)**0.5, + arr.std(), + _bootstrap_error(arr, np.std) + ) + + mean, mean_err, std, std_err, bin_centers = np.array([ + stats( + y[(cats >= left) & (cats < right)] + ) + ((bins[left] + bins[right]) / 2,) for left, right in zip( + range(len(bins) - window_size), + range(window_size, len(bins)) + ) + ]).T + + + if do_plot: + mean_p_std_err = (mean_err**2 + std_err**2)**0.5 + plt.fill_between(bin_centers, mean - mean_err, mean + mean_err, **kwargs) + kwargs['alpha'] *= 0.5 + kwargs = {k : v for k, v in kwargs.items() if k != 'label'} + plt.fill_between(bin_centers, mean - std - mean_p_std_err, mean - std + mean_p_std_err, **kwargs) + plt.fill_between(bin_centers, mean + std - mean_p_std_err, mean + std + mean_p_std_err, **kwargs) + kwargs['alpha'] *= 0.25 + plt.fill_between(bin_centers, mean - std + mean_p_std_err, mean + std - mean_p_std_err, **kwargs) + + return (mean, std), (mean_err, std_err) + + +def make_trend_plot(feature_real, real, feature_gen, gen, name, calc_chi2=False, figsize=(8, 8)): + feature_real = feature_real.squeeze() + feature_gen = feature_gen.squeeze() + real = real.squeeze() + gen = gen.squeeze() + + bins = np.linspace( + min(feature_real.min(), feature_gen.min()), + max(feature_real.max(), feature_gen.max()), + 100 + ) + + fig = plt.figure(figsize=figsize) + calc_trend(feature_real, real, bins=bins, label='real', color='blue') + calc_trend(feature_gen, gen, bins=bins, label='generated', color='red') + plt.legend() + plt.title(name) + + buf = io.BytesIO() + fig.savefig(buf, format='png') + plt.close(fig) + buf.seek(0) + + img = PIL.Image.open(buf) + img_data = np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[0], img.size[1], -1) + + if calc_chi2: + bins = np.linspace( + min(feature_real.min(), feature_gen.min()), + max(feature_real.max(), feature_gen.max()), + 20 + ) + ( + (real_mean, real_std), + (real_mean_err, real_std_err) + ) = calc_trend(feature_real, real, do_plot=False, bins=bins, window_size=1) + ( + (gen_mean, gen_std), + (gen_mean_err, gen_std_err) + ) = calc_trend(feature_gen, gen, do_plot=False, bins=bins, window_size=1) + + gen_upper = gen_mean + gen_std + gen_lower = gen_mean - gen_std + gen_err2 = gen_mean_err**2 + gen_std_err**2 + + real_upper = real_mean + real_std + real_lower = real_mean - real_std + real_err2 = real_mean_err**2 + real_std_err**2 + + chi2 = ( + ((gen_upper - real_upper)**2 / (gen_err2 + real_err2)).sum() + + ((gen_lower - real_lower)**2 / (gen_err2 + real_err2)).sum() + ) + + return img_data, chi2 + + return img_data \ No newline at end of file diff --git a/models/callbacks.py b/models/callbacks.py new file mode 100644 index 0000000..08ce60e --- /dev/null +++ b/models/callbacks.py @@ -0,0 +1,53 @@ +import tensorflow as tf + +from metrics import make_images_for_model + +class SaveModelCallback: + def __init__(self, model, path, save_period): + self.model = model + self.path = path + self.save_period = save_period + + def __call__(self, step): + if step % self.save_period == 0: + print(f'Saving model on step {step} to {self.path}') + self.model.generator.save( + str(self.path.joinpath("generator_{:05d}.h5".format(step)))) + self.model.discriminator.save( + str(self.path.joinpath("discriminator_{:05d}.h5".format(step)))) + + +class WriteHistSummaryCallback: + def __init__(self, model, sample, save_period, writer): + self.model = model + self.sample = sample + self.save_period = save_period + self.writer = writer + + def __call__(self, step): + if step % self.save_period == 0: + images, images1, img_amplitude, chi2 = make_images_for_model(self.model, + sample=self.sample, + calc_chi2=True) + with self.writer.as_default(): + tf.summary.scalar("chi2", chi2, step) + + for k, img in images.items(): + tf.summary.image(k, img, step) + for k, img in images1.items(): + tf.summary.image("{} (amp > 1)".format(k), img, step) + tf.summary.image("log10(amplitude + 1)", img_amplitude, step) + + +class ScheduleLRCallback: + def __init__(self, model, decay_rate, writer): + self.model = model + self.decay_rate = decay_rate + self.writer = writer + + def __call__(self, step): + self.model.disc_opt.lr.assign(self.model.disc_opt.lr * self.decay_rate) + self.model.gen_opt.lr.assign(self.model.gen_opt.lr * self.decay_rate) + with self.writer.as_default(): + tf.summary.scalar("discriminator learning rate", self.model.disc_opt.lr, step) + tf.summary.scalar("generator learning rate", self.model.gen_opt.lr, step) \ No newline at end of file diff --git a/run_model_v4.py b/run_model_v4.py index eab7429..78afa63 100644 --- a/run_model_v4.py +++ b/run_model_v4.py @@ -12,8 +12,9 @@ from data import preprocessing from models.training import train +from models.callbacks import SaveModelCallback, WriteHistSummaryCallback, ScheduleLRCallback from models.model_v4 import Model_v4 -from metrics import make_metric_plots, make_histograms +from metrics import evaluate_model import cuda_gpu_config def make_parser(): @@ -38,11 +39,10 @@ def print_args(args): def parse_args(): args = make_parser().parse_args() - print_args(args) - return args + def load_config(file): with open(file, 'r') as f: config = yaml.load(f, Loader=yaml.FullLoader) @@ -54,6 +54,7 @@ def load_config(file): return config + def epoch_from_name(name): epoch, = re.findall('\d+', name) return int(epoch) @@ -83,141 +84,6 @@ def load_weights(model, model_path): return latest_gen_checkpoint, latest_disc_checkpoint -def get_images(model, - sample, - return_raw_data=False, - calc_chi2=False, - gen_more=None, - batch_size=128): - X, Y = sample - assert X.ndim == 2 - assert X.shape[1] == 4 - - if gen_more is None: - gen_features = X - else: - gen_features = np.tile( - X, - [gen_more] + [1] * (X.ndim - 1) - ) - gen_scaled = np.concatenate([ - model.make_fake(gen_features[i:i+batch_size]).numpy() - for i in range(0, len(gen_features), batch_size) - ], axis=0) - real = model.scaler.unscale(Y) - gen = model.scaler.unscale(gen_scaled) - gen[gen < 0] = 0 - gen1 = np.where(gen < 1., 0, gen) - - features = { - 'crossing_angle' : (X[:, 0], gen_features[:,0]), - 'dip_angle' : (X[:, 1], gen_features[:,1]), - 'drift_length' : (X[:, 2], gen_features[:,2]), - 'time_bin_fraction' : (X[:, 2] % 1, gen_features[:,2] % 1), - 'pad_coord_fraction' : (X[:, 3] % 1, gen_features[:,3] % 1) - } - - images = make_metric_plots(real, gen, features=features, calc_chi2=calc_chi2) - if calc_chi2: - images, chi2 = images - - images1 = make_metric_plots(real, gen1, features=features) - - img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) - - result = [images, images1, img_amplitude] - - if return_raw_data: - result += [(gen_features, gen)] - - if calc_chi2: - result += [chi2] - - return result - - -class SaveModelCallback: - def __init__(self, model, path, save_period): - self.model = model - self.path = path - self.save_period = save_period - - def __call__(self, step): - if step % self.save_period == 0: - print(f'Saving model on step {step} to {self.path}') - self.model.generator.save( - str(self.path.joinpath("generator_{:05d}.h5".format(step)))) - self.model.discriminator.save( - str(self.path.joinpath("discriminator_{:05d}.h5".format(step)))) - - -class WriteHistSummaryCallback: - def __init__(self, model, sample, save_period, writer): - self.model = model - self.sample = sample - self.save_period = save_period - self.writer = writer - - def __call__(self, step): - if step % self.save_period == 0: - images, images1, img_amplitude, chi2 = get_images(self.model, - sample=self.sample, - calc_chi2=True) - with self.writer.as_default(): - tf.summary.scalar("chi2", chi2, step) - - for k, img in images.items(): - tf.summary.image(k, img, step) - for k, img in images1.items(): - tf.summary.image("{} (amp > 1)".format(k), img, step) - tf.summary.image("log10(amplitude + 1)", img_amplitude, step) - - -class ScheduleLRCallback: - def __init__(self, model, decay_rate, writer): - self.model = model - self.decay_rate = decay_rate - self.writer = writer - - def __call__(self, step): - self.model.disc_opt.lr.assign(self.model.disc_opt.lr * self.decay_rate) - self.model.gen_opt.lr.assign(self.model.gen_opt.lr * self.decay_rate) - with self.writer.as_default(): - tf.summary.scalar("discriminator learning rate", self.model.disc_opt.lr, step) - tf.summary.scalar("generator learning rate", self.model.gen_opt.lr, step) - - -def evaluate_model(model, path, sample, gen_sample_name=None): - path.mkdir() - ( - images, images1, img_amplitude, - gen_dataset, chi2 - ) = get_images(model, sample=sample, - calc_chi2=True, return_raw_data=True, gen_more=10) - - array_to_img = lambda arr: PIL.Image.fromarray(arr.reshape(arr.shape[1:])) - - for k, img in images.items(): - array_to_img(img).save(str(path / f"{k}.png")) - for k, img in images1.items(): - array_to_img(img).save(str(path / f"{k}_amp_gt_1.png")) - array_to_img(img_amplitude).save(str(path / "log10_amp_p_1.png")) - - if gen_sample_name is not None: - with open(str(path / gen_sample_name), 'w') as f: - for event_X, event_Y in zip(*gen_dataset): - f.write('params: {:.3f} {:.3f} {:.3f} {:.3f}\n'.format(*event_X)) - for ipad, time_distr in enumerate(event_Y, model.pad_range[0] + event_X[3].astype(int)): - for itime, amp in enumerate(time_distr, model.time_range[0] + event_X[2].astype(int)): - if amp < 1: - continue - f.write(" {:2d} {:3d} {:8.3e} ".format(ipad, itime, amp)) - f.write('\n') - - with open(str(path / 'stats'), 'w') as f: - f.write(f"{chi2:.2f}\n") - - def main(): args = parse_args() From 2db1d20922014bfb781f873d5f149f07d12532db Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Mon, 28 Sep 2020 16:21:13 +0300 Subject: [PATCH 06/24] concat block --- models/nn.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/models/nn.py b/models/nn.py index 2807f24..2809f3a 100644 --- a/models/nn.py +++ b/models/nn.py @@ -29,6 +29,21 @@ def fully_connected_block(units, activations, return tf.keras.Sequential(layers, **args) +def concat_block(input1_shape, input2_shape, reshape_input1=None, + reshape_input2=None, axis=-1, name=None): + in1 = tf.keras.Input(shape=input1_shape) + in2 = tf.keras.Input(shape=input2_shape) + concat1, concat2 = in1, in2 + if reshape_input1: + concat1 = tf.keras.layers.Reshape(reshape_input1)(concat1) + if reshape_input2: + concat2 = tf.keras.layers.Reshape(reshape_input2)(concat2) + out = tf.keras.layers.Concatenate(axis=axis)([concat1, concat2]) + args = dict(inputs=[in1, in2], outputs=out) + if name: + args['name'] = name + return tf.keras.Model(**args) + def conv_block(filters, kernel_sizes, paddings, activations, poolings, kernel_init='glorot_uniform', input_shape=None, output_shape=None, @@ -110,11 +125,14 @@ def build_block(block_type, arguments): inner_block = build_block(**arguments['block']) arguments['block'] = inner_block block = vector_img_connect_block(**arguments) + elif block_type == 'concat': + block = concat_block(**arguments) else: raise(NotImplementedError(block_type)) return block + def build_architecture(block_descriptions, name=None): blocks = [build_block(**descr) for descr in block_descriptions] From a845c54bbcee2e463a6cda734c1592213fc38f5a Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 30 Sep 2020 14:02:48 +0300 Subject: [PATCH 07/24] model generating moments --- models/configs/moments.yaml | 50 +++++++++++++++++++++++++++++++++++++ models/model_v4.py | 2 +- models/scalers.py | 47 ++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 models/configs/moments.yaml diff --git a/models/configs/moments.yaml b/models/configs/moments.yaml new file mode 100644 index 0000000..b162135 --- /dev/null +++ b/models/configs/moments.yaml @@ -0,0 +1,50 @@ +latent_dim: 32 +batch_size: 32 +lr: 1.e-4 +lr_schedule_rate: 0.999 + +num_disc_updates: 8 +gp_lambda: 10. +gpdata_lambda: 0. +cramer: False +stochastic_stepping: True + +save_every: 50 +num_epochs: 10000 + +feature_noise_power: NULL +feature_noise_decay: NULL + +data_version: 'data_v4' +pad_range: [-3, 5] +time_range: [-7, 9] +scaler: 'gaussian' + +architecture: + generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 6] + activations: ['relu', 'relu', 'relu', 'relu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: NULL + name: 'generator' + + discriminator: + - block_type: 'concat' + arguments: + input1_shape: [5,] + input2_shape: [6,] + reshape_input1: NULL + reshape_input2: NULL + axis: -1 + name: 'discriminator_concat' + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128, 1] + activations: ['relu', 'relu', 'relu', 'relu', 'relu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [11,] + output_shape: NULL + name: 'discriminator_head' diff --git a/models/model_v4.py b/models/model_v4.py index 0b92176..4ce2dd0 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -68,7 +68,7 @@ def make_fake(self, features): ) def gradient_penalty(self, features, real, fake): - alpha = tf.random.uniform(shape=[len(real), 1, 1]) + alpha = tf.random.uniform(shape=[len(real),] + [1] * (len(real.shape) - 1)) interpolates = alpha * real + (1 - alpha) * fake with tf.GradientTape() as t: t.watch(interpolates) diff --git a/models/scalers.py b/models/scalers.py index f1e8303..99812ba 100644 --- a/models/scalers.py +++ b/models/scalers.py @@ -1,5 +1,7 @@ import numpy as np +from metrics.gaussian_metrics import get_val_metric_v as gaussian_fit + class Identity: def scale(self, x): @@ -16,10 +18,55 @@ def scale(self, x): def unscale(self, x): return 10 ** x - 1 + +class Gaussian: + def __init__(self, shape=(8, 16)): + self.shape = shape + + def scale(self, x): + result = gaussian_fit(x) + result[:,-1] = np.log1p(result[:,-1]) + result[:,4] /= (result[:,2] * result[:,3]) + return result + + def unscale(self, x): + m0, m1, D00, D11, D01, logA = x.T + D00 = np.clip(D00, 0.1, None) + D11 = np.clip(D11, 0.1, None) + D01 = np.clip(D01, -1., 1.) + D01 *= D00 * D11 + + A = np.expm1(logA) + + cov = np.stack([ + np.stack([D00, D01], axis=1), + np.stack([D01, D11], axis=1) + ], axis=2) # N x 2 x 2 + invcov = np.linalg.inv(cov) + mu = np.stack([m0, m1], axis=1) + + xx0 = np.arange(self.shape[0]) + xx1 = np.arange(self.shape[1]) + xx0, xx1 = np.meshgrid(xx0, xx1, indexing='ij') + xx = np.stack([xx0, xx1], axis=2) + residuals = xx[None,...] - mu[:,None,None,:] # N x H x W x 2 + + result = np.exp(-0.5 * + np.einsum('ijkl,ilm,ijkm->ijk', residuals, invcov, residuals) + ) + + result /= result.sum(axis=(1, 2), keepdims=True) + result *= A[:,None,None] + + return result + + def get_scaler(scaler_type): if scaler_type == 'identity': return Identity() elif scaler_type == 'logarithmic': return Logarithmic() + elif scaler_type == 'gaussian': + return Gaussian() else: raise NotImplementedError(scaler_type) From 455039e667f3a71b2cb03d82b203a9f023362d64 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Tue, 6 Oct 2020 18:23:17 +0300 Subject: [PATCH 08/24] fixing moments clip rule for gaussian scaler --- models/scalers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/scalers.py b/models/scalers.py index 99812ba..83969e5 100644 --- a/models/scalers.py +++ b/models/scalers.py @@ -31,8 +31,8 @@ def scale(self, x): def unscale(self, x): m0, m1, D00, D11, D01, logA = x.T - D00 = np.clip(D00, 0.1, None) - D11 = np.clip(D11, 0.1, None) + D00 = np.clip(D00, 0.05, None) + D11 = np.clip(D11, 0.05, None) D01 = np.clip(D01, -1., 1.) D01 *= D00 * D11 From b0cb980af17af64a2304e52bb1f8d5da249ece0b Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Tue, 6 Oct 2020 19:13:20 +0300 Subject: [PATCH 09/24] moving load_model to utils --- models/utils.py | 36 ++++++++++++++++++++++++++++++++++++ run_model_v4.py | 36 ++++-------------------------------- 2 files changed, 40 insertions(+), 32 deletions(-) create mode 100644 models/utils.py diff --git a/models/utils.py b/models/utils.py new file mode 100644 index 0000000..6825d82 --- /dev/null +++ b/models/utils.py @@ -0,0 +1,36 @@ +import re + + +def epoch_from_name(name): + epoch, = re.findall('\d+', name) + return int(epoch) + + +def latest_epoch(model_path): + gen_checkpoints = model_path.glob("generator_*.h5") + disc_checkpoints = model_path.glob("discriminator_*.h5") + + gen_epochs = [epoch_from_name(path.stem) for path in gen_checkpoints] + disc_epochs = [epoch_from_name(path.stem) for path in disc_checkpoints] + + latest_gen_epoch = max(gen_epochs) + latest_disc_epoch = max(disc_epochs) + + assert ( + latest_gen_epoch == latest_disc_epoch + ), "Latest disc and gen epochs differ" + + return latest_gen_epoch + + +def load_weights(model, model_path, epoch=None): + if epoch is None: + epoch = latest_epoch(model_path) + + latest_gen_checkpoint = model_path / f"generator_{epoch:05d}.h5" + latest_disc_checkpoint = model_path / f"discriminator_{epoch:05d}.h5" + + print(f'Loading generator weights from {str(latest_gen_checkpoint)}') + model.generator.load_weights(str(latest_gen_checkpoint)) + print(f'Loading discriminator weights from {str(latest_disc_checkpoint)}') + model.discriminator.load_weights(str(latest_disc_checkpoint)) \ No newline at end of file diff --git a/run_model_v4.py b/run_model_v4.py index 78afa63..95c7854 100644 --- a/run_model_v4.py +++ b/run_model_v4.py @@ -1,5 +1,4 @@ import os, sys -import re from pathlib import Path import shutil import argparse @@ -11,6 +10,7 @@ import yaml from data import preprocessing +from models.utils import latest_epoch, load_weights from models.training import train from models.callbacks import SaveModelCallback, WriteHistSummaryCallback, ScheduleLRCallback from models.model_v4 import Model_v4 @@ -55,35 +55,6 @@ def load_config(file): return config -def epoch_from_name(name): - epoch, = re.findall('\d+', name) - return int(epoch) - - -def load_weights(model, model_path): - gen_checkpoints = model_path.glob("generator_*.h5") - disc_checkpoints = model_path.glob("discriminator_*.h5") - latest_gen_checkpoint = max( - gen_checkpoints, - key=lambda path: epoch_from_name(path.stem) - ) - latest_disc_checkpoint = max( - disc_checkpoints, - key=lambda path: epoch_from_name(path.stem) - ) - - assert ( - epoch_from_name(latest_gen_checkpoint.stem) == epoch_from_name(latest_disc_checkpoint.stem) - ), "Latest disc and gen epochs differ" - - print(f'Loading generator weights from {str(latest_gen_checkpoint)}') - model.generator.load_weights(str(latest_gen_checkpoint)) - print(f'Loading discriminator weights from {str(latest_disc_checkpoint)}') - model.discriminator.load_weights(str(latest_disc_checkpoint)) - - return latest_gen_checkpoint, latest_disc_checkpoint - - def main(): args = parse_args() @@ -110,7 +81,7 @@ def main(): model = Model_v4(config) if args.prediction_only: - latest_gen_checkpoint, latest_disc_checkpoint = load_weights(model, model_path) + load_weights(model, model_path) preprocessing._VERSION = model.data_version data, features = preprocessing.read_csv_2d(pad_range=model.pad_range, time_range=model.time_range) @@ -126,7 +97,8 @@ def main(): if args.prediction_only: - prediction_path = model_path / f"prediction_{epoch_from_name(latest_gen_checkpoint.stem):05d}" + epoch = latest_epoch(model_path) + prediction_path = model_path / f"prediction_{epoch:05d}" assert not prediction_path.exists(), "Prediction path already exists" prediction_path.mkdir() From d74ad57a1c650e6e947569fcb4646b0979ce6130 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 7 Oct 2020 16:06:34 +0300 Subject: [PATCH 10/24] batching validation losses to save memory --- models/training.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/models/training.py b/models/training.py index 7525b8c..2aea9c3 100644 --- a/models/training.py +++ b/models/training.py @@ -41,16 +41,25 @@ def train(data_train, data_val, train_step_fn, loss_eval_fn, num_epochs, batch_s for k, l in losses_train_batch.items(): losses_train[k] = losses_train.get(k, 0) + l.numpy() * len(batch) losses_train = {k : l / len(data_train) for k, l in losses_train.items()} - + tf.keras.backend.set_learning_phase(0) # testing - - if features_train is None: - losses_val = {k : l.numpy() for k, l in loss_eval_fn(data_val).items()} - else: - losses_val = {k : l.numpy() for k, l in loss_eval_fn(features_val, data_val).items()} + + losses_val = {} + for i_sample in trange(0, len(data_val), batch_size): + batch = data_val[i_sample:i_sample + batch_size] + + if features_train is None: + losses_val_batch = {k : l.numpy() for k, l in loss_eval_fn(batch).items()} + else: + feature_batch = features_val[i_sample:i_sample + batch_size] + losses_val_batch = {k : l.numpy() for k, l in loss_eval_fn(feature_batch, batch).items()} + for k, l in losses_val_batch.items(): + losses_val[k] = losses_val.get(k, 0) + l * len(batch) + losses_val = {k : l / len(data_val) for k, l in losses_val.items()} + for f in callbacks: f(i_epoch) - + if train_writer is not None: with train_writer.as_default(): for k, l in losses_train.items(): From c311c7f7841e3939deccf611e87f88958dd6d36f Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 7 Oct 2020 16:07:56 +0300 Subject: [PATCH 11/24] fixing save img to tensorboard; adding examples plots --- metrics/__init__.py | 63 +++++++++++++++++++++++++++++++++++++++++++-- metrics/trends.py | 2 +- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/metrics/__init__.py b/metrics/__init__.py index 71d0ebd..b64643d 100644 --- a/metrics/__init__.py +++ b/metrics/__init__.py @@ -31,7 +31,7 @@ def make_histograms(data_real, data_gen, title, figsize=(8, 8), n_bins=100, logy buf.seek(0) img = PIL.Image.open(buf) - return np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[0], img.size[1], -1) + return np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[1], img.size[0], -1) def make_metric_plots(images_real, images_gen, features=None, calc_chi2=False): @@ -110,6 +110,9 @@ def make_images_for_model(model, img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) + images['examples'] = plot_individual_images(Y, gen_scaled) + images['examples_mask'] = plot_images_mask(Y, gen_scaled) + result = [images, images1, img_amplitude] if return_raw_data: @@ -149,4 +152,60 @@ def evaluate_model(model, path, sample, gen_sample_name=None): f.write('\n') with open(str(path / 'stats'), 'w') as f: - f.write(f"{chi2:.2f}\n") \ No newline at end of file + f.write(f"{chi2:.2f}\n") + + +def plot_individual_images(real, gen, n=10): + assert real.ndim == 3 == gen.ndim + assert real.shape[1:] == gen.shape[1:] + N_max = min(len(real), len(gen)) + assert n * 2 <= N_max + + idx = np.sort(np.random.choice(N_max, n * 2, replace=False)) + real = real[idx] + gen = gen[idx] + + size_x = 12 + size_y = size_x / real.shape[2] * real.shape[1] * n * 1.2 / 4 + + fig, axx = plt.subplots(n, 4, figsize=(size_x, size_y)) + axx = [(ax[0], ax[1]) for ax in axx] + \ + [(ax[2], ax[3]) for ax in axx] + + for ax, img_real, img_fake in zip(axx, real, gen): + ax[0].imshow(img_real, aspect='auto') + ax[0].set_title("real") + ax[0].axis('off') + ax[1].imshow(img_fake, aspect='auto') + ax[1].set_title('generated') + ax[1].axis('off') + + buf = io.BytesIO() + fig.savefig(buf, format='png') + plt.close(fig) + buf.seek(0) + + img = PIL.Image.open(buf) + return np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[1], img.size[0], -1) + + +def plot_images_mask(real, gen): + assert real.ndim == 3 == gen.ndim + assert real.shape[1:] == gen.shape[1:] + + size_x = 6 + size_y = size_x / real.shape[2] * real.shape[1] * 2.4 + + fig, [ax0, ax1] = plt.subplots(2, 1, figsize=(size_x, size_y)) + ax0.imshow(real.any(axis=0), aspect='auto') + ax0.set_title("real") + ax1.imshow(gen.any(axis=0), aspect='auto') + ax1.set_title("generated") + + buf = io.BytesIO() + fig.savefig(buf, format='png') + plt.close(fig) + buf.seek(0) + + img = PIL.Image.open(buf) + return np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[1], img.size[0], -1) \ No newline at end of file diff --git a/metrics/trends.py b/metrics/trends.py index 89025dc..de6ef42 100644 --- a/metrics/trends.py +++ b/metrics/trends.py @@ -75,7 +75,7 @@ def make_trend_plot(feature_real, real, feature_gen, gen, name, calc_chi2=False, buf.seek(0) img = PIL.Image.open(buf) - img_data = np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[0], img.size[1], -1) + img_data = np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[1], img.size[0], -1) if calc_chi2: bins = np.linspace( From 1da7ddee8f3a39662ca83f3bbd54a87c2250ba0a Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 7 Oct 2020 21:42:36 +0300 Subject: [PATCH 12/24] elu activation --- models/configs/baseline_fc_8x16_elu.yaml | 62 ++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 models/configs/baseline_fc_8x16_elu.yaml diff --git a/models/configs/baseline_fc_8x16_elu.yaml b/models/configs/baseline_fc_8x16_elu.yaml new file mode 100644 index 0000000..1c72edc --- /dev/null +++ b/models/configs/baseline_fc_8x16_elu.yaml @@ -0,0 +1,62 @@ +latent_dim: 32 +batch_size: 32 +lr: 1.e-4 +lr_schedule_rate: 0.999 + +num_disc_updates: 8 +gp_lambda: 10. +gpdata_lambda: 0. +cramer: False +stochastic_stepping: True + +save_every: 50 +num_epochs: 10000 + +feature_noise_power: NULL +feature_noise_decay: NULL + +data_version: 'data_v4' +pad_range: [-3, 5] +time_range: [-7, 9] +scaler: 'logarithmic' + +architecture: + generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128] + activations: ['elu', 'elu', 'elu', 'elu', 'elu'] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: [8, 16] + name: 'generator' + + discriminator: + - block_type: 'connect' + arguments: + vector_shape: [5,] + img_shape: [8, 16] + vector_bypass: False + concat_outputs: True + name: 'discriminator_tail' + block: + block_type: 'conv' + arguments: + filters: [16, 16, 32, 32, 64, 64] + kernel_sizes: [3, 3, 3, 3, 3, 2] + paddings: ['same', 'same', 'same', 'same', 'valid', 'valid'] + activations: ['elu', 'elu', 'elu', 'elu', 'elu', 'elu'] + poolings: [NULL, [1, 2], NULL, 2, NULL, NULL] + kernel_init: glorot_uniform + input_shape: NULL + output_shape: [64,] + dropouts: [0.02, 0.02, 0.02, 0.02, 0.02, 0.02] + name: discriminator_conv_block + - block_type: 'fully_connected' + arguments: + units: [128, 1] + activations: ['elu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [69,] + output_shape: NULL + name: 'discriminator_head' From 034ccbf3f7fc1f27c3838d035cd172f2315c2cdf Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 7 Oct 2020 22:09:45 +0300 Subject: [PATCH 13/24] prototyping evaluation with a classifier [wip] --- notebooks/ClassifierEvaluation.ipynb | 522 +++++++++++++++++++++++++++ 1 file changed, 522 insertions(+) create mode 100644 notebooks/ClassifierEvaluation.ipynb diff --git a/notebooks/ClassifierEvaluation.ipynb b/notebooks/ClassifierEvaluation.ipynb new file mode 100644 index 0000000..31fa64e --- /dev/null +++ b/notebooks/ClassifierEvaluation.ipynb @@ -0,0 +1,522 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "current_path = %pwd\n", + "current_path = Path(current_path)\n", + "sys.path.insert(0, str(current_path.parent))" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import roc_auc_score\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import cuda_gpu_config\n", + "from run_model_v4 import load_config\n", + "from models.model_v4 import Model_v4\n", + "from models.utils import load_weights, latest_epoch\n", + "from data import preprocessing\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXQV9f3/8ecHSICEEJawhxD2LQkKgYigolgFxAXRWrVuqKjV1vbbCsgiKC6IWrVFpbgh1WorCYuouKKgggoq2SAQwha2AIEkZCHL/fz+SH49VIEEuDeTO/f1OIcDyXy88xoDrzOZzLyvsdYiIiL+r57TAURExDtU6CIiLqFCFxFxCRW6iIhLqNBFRFyigVM7joiIsNHR0U7tXkTEL61bt+6AtbbV8bY5VujR0dGsXbvWqd2LiPglY8z2E23TJRcREZdQoYuIuIQKXUTEJVToIiIuoUIXEXGJagvdGNPRGLPCGJNujEkzxtx/nDXGGPM3Y0ymMSbZGNPfN3FFROREanLbYjnwZ2vtD8aYMGCdMeYTa236MWtGAt2rfiUAL1X9LiIitaTaM3Rr7R5r7Q9Vfy4ANgAdfrbsSmCBrbQGaGaMaef1tCIifqyswsOLX2Syfudhn7z+KV1DN8ZEA2cD3/5sUwdg5zEfZ/PL0scYM94Ys9YYs3b//v2nllRExI+l7srjqhe+ZvbyDD5M3euTfdT4SVFjTBMgEfijtTb/dHZmrZ0HzAOIj4/XO2uIiOuVlFXw9883M/fLLJqHBPPSjf0ZGeubCxg1KnRjTBCVZf6WtTbpOEt2AR2P+Tiy6nMiIgFr7bZcJiQmk7W/kGsHRDL1sj6EhwT5bH/VFroxxgCvAhustX89wbKlwH3GmHeo/GFonrV2j/diioj4jyNHy3lq+UYWrNlO+/DGLBg3iPN7HHeellfV5Ax9CHATkGKM+anqc5OBKABr7VzgA2AUkAkUAbd5P6qISN335ab9TE5KYXdeMbcMjuaBS3sS2rB25iBWuxdr7VeAqWaNBe71VigREX9zuKiUmcs2kPhDNl1bhfLuXYOJj25RqxkcG58rIuIWH6bsYdqSNA4VlXLfhd2476JuNAqqX+s5VOgiIqcpJ7+Eh5aksTxtLzEdmvLGuIH0bR/uWB4VuojIKbLW8u66bB5dlk5JuYeJI3px53mdaVDf2fFYKnQRkVOwM7eIyYtSWLX5AIOiWzBrbCxdWjVxOhagQhcRqZEKj2XB6m089VEGBph5ZV9uTOhEvXonvWekVqnQRUSqkZlTwMTEFNZtP8QFPVrx+NWxdGjW2OlYv6BCFxE5gbIKD//4cgt/+yyTkIb1efa6flx1Vgcqn7ese1ToIiLHkZKdx4TEZDbsyeeyuHY8fEVfIpo0dDrWSanQRUSOUVJWwXOfbublVVm0DA3mHzcN4NK+bZ2OVSMqdBGRKt9mHWRSUgpbDxRyXXxHJl/Wm/DGvhum5W0qdBEJeAUlZcxensE/12ynY4vGvHVHAkO6RTgd65Sp0EUkoK3IyGFKUgp78ksYN6Qzf7m0ByHB/lmN/plaROQMHSosZeaydJJ+3EX31k1IvOdc+kc1dzrWGVGhi0hAsdbyfsoepi9JI6+4jD8M7869F3alYYPaH6blbSp0EQkY+/JLmLo4lU/S9xEXGc6bdyTQu11Tp2N5jQpdRFzPWst/1u7k0fc3UFruYfKoXowb4vwwLW9ToYuIq+04WMSkpGS+2XKQhM4teHJsHNERoU7H8gkVuoi4UoXHMv+bbTz9UQb16xkeGxPD9QOj6tQwLW9ToYuI62zaV8CEhcn8tPMwF/VqzWNjYmgXXveGaXmbCl1EXKO03MNLX2xhzorNhDUK4vnfnMUV/drX2WFa3qZCFxFXWL/zMBMTk9m4t4Ar+rVn+uV9aFnHh2l5mwpdRPxacWkFz366iVdWZdE6rBGv3BzPxX3aOB3LESp0EfFbq7cc5MGkZLYdLOL6QVE8OKoXTRv5zzAtb1Ohi4jfyS8pY9aHG/nXtzvo1DKEf92ZwLld/W+Ylrep0EXEr3y2YR9TFqWSU1DC+PO78KeLe9A42P8f2/cGFbqI+IWDR47y8HvpLF2/m55twph70wDO6tjM6Vh1igpdROo0ay1L1+/m4ffSKSgp408X9+CeYV0JbuCux/a9QYUuInXWnrxipi5K5bONOfTr2IzZY+Po2TbM6Vh1lgpdROocj8fyzvc7eeKDDZR5PEy9rDe3DelMfRc/tu8NKnQRqVO2HShkUlIya7JyObdrS564OpZOLd05TMvbVOgiUieUV3h47eutPPPxJoLr12PW1bFcN7BjwDy27w3VFrox5jVgNJBjrY05zvZw4E0gqur1nrbWvu7toCLiXhv35jNxYTLrs/O4uHcbHr0qhrbhjZyO5XdqcoY+H5gDLDjB9nuBdGvt5caYVkCGMeYta22plzKKiEsdLa/ghRVbeHFFJuGNg/j79WczOq6dzspPU7WFbq1daYyJPtkSIMxUfgWaALlAuVfSiYhr/bjjEBMTk9m07whjzu7AtNF9aBEa7HQsv+aNa+hzgKXAbiAMuM5a6zneQmPMeGA8QFRUlBd2LSL+pqi0nGc+3sRrX2+lbdNGvHZrPBf1CsxhWt7mjUK/FPgJuAjoCnxijFllrc3/+UJr7TxgHkB8fLz1wr5FxI98k3mASUkp7Mgt4rfnRDFxRC/CAniYlrd5o9BvA2ZZay2QaYzZCvQCvvPCa4uIC+QVl/HEBxt45/uddI4I5d/jzyGhS0unY7mONwp9BzAcWGWMaQP0BLK88Loi4gIfp+1l6uJUDhw5yl0XVA7TahSkYVq+UJPbFt8GhgERxphsYDoQBGCtnQvMBOYbY1IAA0y01h7wWWIR8QsHjhxlxtI0liXvoVfbMF65JZ64SA3T8qWa3OVyfTXbdwOXeC2RiPg1ay2Lf9rFw++lU3S0gj//qgd3D+tKUH0N0/I1PSkqIl6z+3AxUxalsCJjP2dHVQ7T6t5Gw7RqiwpdRM6Yx2N567sdzPpgAx4L0y/vw82DozVMq5ap0EXkjGTtP8KkxBS+25bL0G4RPHF1LB1bhDgdKyCp0EXktJRXeHjlq608+8kmGjaox+xr4rh2QKQe23eQCl1ETln67nwmJK4ndVc+l/Ztw8wrY2jdVMO0nKZCF5EaO1pewZzPM3npiy00CwnixRv7MzKmrc7K6wgVuojUyLrtuUxMTCEz5whj+0cy9bLeNNcwrTpFhS4iJ1V4tJynPsrgjdXbaB/emDfGDeKCHq2cjiXHoUIXkRNatXk/DyalkH2omFsGd+KBEb1o0lC1UVfpKyMiv5BXVMaj76fz7rpsurQK5d27BzMwuoXTsaQaKnQR+R/LU/cybUkquYWl/G5YV/4wvLuGafkJFbqIAJBTUMKMpWl8kLKXPu2a8vqtA4npEO50LDkFKnSRAGetJfGHXcxclk5xWQUPXNqT8ed30TAtP6RCFwlg2YeKmLwolZWb9hPfqTmzxsbRrXUTp2PJaVKhiwQgj8fyzzXbeXL5RgAevqIvN53TiXoapuXXVOgiAWbL/iNMXJjM2u2HOL9HKx4fE0Nkcw3TcgMVukiAKKvwMG9lFs9/tpnGQfV5+tp+jO3fQY/tu4gKXSQApO7KY8LCZNL35DMqti0zruhL6zAN03IbFbqIi5WUVfD8Z5uZtzKLFqHBzP1tf0bEtHM6lviICl3Epb7flsvEhclkHSjk2gGRTL2sD+EhQU7HEh9SoYu4zJGj5cxevpEFq7cT2bwx/7x9EOd11zCtQKBCF3GRLzftZ3JSCrvzirn13GgeuLQnoRqmFTD0lRZxgcNFpTyyLJ2kH3bRtVUoC+8ezIBOGqYVaFToIn7MWsuHqXt5aEkqh4vKuO/Cbtx3UTcN0wpQKnQRP5WTX8K0Jal8lLaPmA5NeWPcIPq21zCtQKZCF/Ez1lreXZfNo8vSOVruYdLIXtwxtDMNNEwr4KnQRfzIztwiHkxK4avMAwyKbsGssbF0aaVhWlJJhS7iByo8lgWrtzF7eQb1DMy8KoYbB0VpmJb8DxW6SB23eV8BExOT+WHHYYb1bMVjY2Lp0Kyx07GkDlKhi9RRZRUe5n6xhb9/nklow/o8e10/rjpLw7TkxKotdGPMa8BoIMdaG3OCNcOA54Ag4IC19gJvhhQJNCnZeTywcD0b9xYwOq4dM67oS0SThk7HkjquJmfo84E5wILjbTTGNANeBEZYa3cYY1p7L55IYCkpq+DZTzfx8sosIpo0ZN5NA7ikb1unY4mfqLbQrbUrjTHRJ1lyA5Bkrd1RtT7HO9FEAsu3WQeZlJTC1gOF/GZgRx4c1ZvwxhqmJTXnjWvoPYAgY8wXQBjwvLX2RGfz44HxAFFRUV7YtYj/Kygp48nlG3lzzQ46tmjMW3ckMKRbhNOxxA95o9AbAAOA4UBjYLUxZo21dtPPF1pr5wHzAOLj460X9i3i11ZszGHyohT25pdw+9DO/PmSHoQE614FOT3e+JuTDRy01hYChcaYlUA/4BeFLiKVcgtLeeS9NBb/tJvurZuQeM+59I9q7nQs8XPeKPQlwBxjTAMgGEgAnvXC64q4jrWWZcl7mLE0jbziMu4f3p3fXdiVhg00TEvOXE1uW3wbGAZEGGOygelU3p6ItXautXaDMWY5kAx4gFestam+iyzin/bllzBlUSqfbthHXGQ4b92ZQK+2TZ2OJS5Sk7tcrq/BmqeAp7ySSMRlrLX8+/udPPbBBkrLPUwZ1ZvbhkRrmJZ4nX76IuJD2w8W8mBSCt9sOUhC5xY8OTaO6IhQp2OJS6nQRXygwmN5/eutPP1xBg3q1ePxMbH8ZmBHDdMSn1Khi3hZxt4CJiQms37nYYb3as2jY2JoF65hWuJ7KnQRLykt9/DiF5m8sCKTsEZBPP+bs7iiX3sN05Jao0IX8YL1Ow8zYWEyGfsKuPKs9jw0ug8tNUxLapkKXeQMFJdW8NdPMnj1q620DmvEKzfHc3GfNk7HkgClQhc5Td9sOcCDSSlsP1jEDQlRTBrZi6aNNExLnKNCFzlF+SVlPPHBRt7+bgedWobwrzsTOLerhmmJ81ToIqfg0/R9TFmcwv6Co4w/vwt/urgHjYP12L7UDSp0kRo4eOQoD7+XztL1u+nVNox5N8XTr2Mzp2OJ/A8VushJWGtZun43M5amceRoOX+6uAf3DOtKcAM9ti91jwpd5AT25BUzdVEqn23M4ayOzZh9TRw92oQ5HUvkhFToIj/j8Vje/n4HT3ywkXKPh6mX9ea2IZ2pr8f2pY5ToYscY+uBQiYlJvPt1lzO7dqSWVfHEdUyxOlYIjWiQhcByis8vPb1Vp75eBPBDerx5NhYfh3fUY/ti19RoUvA27Ann4mJySRn5/GrPm149KoY2jRt5HQskVOmQpeAdbS8ghdWbOHFFZmENw5izg1nc1lsO52Vi99SoUtA+mHHISYuTGZzzhHGnN2Bh0b3oXlosNOxRM6ICl0CSlFpOU9/tInXv9lK26aNeP3WgVzYq7XTsUS8QoUuAePrzANMSkpmZ24xN53TiQkjehKmYVriIip0cb284jIef38D/167k84Rofx7/DkkdGnpdCwRr1Ohi6t9nLaXqYtTOVhYyt0XdOWPF3enUZCGaYk7qdDFlfYXHGXGe2m8n7yH3u2a8uotA4mNDHc6lohPqdDFVay1LPpxF48sS6foaAV/uaQHd13QlaD6GqYl7qdCF9fYdbiYKYtS+CJjP/2jKodpdWutYVoSOFTo4vc8Hstb325n1ocb8ViYfnkfbh4crWFaEnBU6OLXsvYfYVJiCt9ty+W87hE8PiaWji00TEsCkwpd/FJ5hYeXV23l2U830ahBPZ66Jo5rBkTqsX0JaCp08Ttpu/OYmJhM6q58Lu3bhplXxtBaw7REVOjiP0rKKvj755uZ+2UWzUOCeenG/oyMbed0LJE6Q4UufmHd9lwmLExmy/5CxvaPZNro3jQL0TAtkWNVe3OuMeY1Y0yOMSa1mnUDjTHlxphrvBdPAl3h0XJmLE3jmrmrKSnz8Ma4QTzz634qc5HjqMkZ+nxgDrDgRAuMMfWBJ4GPvRNLBFZu2s+DSSnszivm5nM68cCIXjRpqG8qRU6k2n8d1tqVxpjoapb9HkgEBnohkwS4vKIyZr6fzsJ12XRpFcp/7hrMwOgWTscSqfPO+HTHGNMBGANcSDWFbowZD4wHiIqKOtNdiwstT93DtCVp5BaW8rthXfnDcA3TEqkpb3z/+hww0Vrrqe4eYGvtPGAeQHx8vPXCvsUlcgpKmL4kjQ9T99KnXVNev3UgMR00TEvkVHij0OOBd6rKPAIYZYwpt9Yu9sJri8tZa1m4LptH399AcVkFE0b05M7zumiYlshpOONCt9Z2/v9/NsbMB5apzKUmduYWMXlRCqs2H2BgdHNmjY2ja6smTscS8VvVFrox5m1gGBBhjMkGpgNBANbauT5NJ67k8VgWrN7G7I8yMMAjV/bltwmdqKdhWiJnpCZ3uVxf0xez1t56RmnE9TJzjjApMZm12w9xfo9WPD4mhsjmGqYl4g26qVdqRVmFh3krs3j+0800Dq7PM9f24+r+HTRMS8SLVOjic6m78piwMJn0PfmMim3Lw1fE0CqsodOxRFxHhS4+U1JWwfOfbWbeyixahAYz97cDGBHT1ulYIq6lQhef+H5bLhMXJpN1oJBfx0cyZVQfwkOCnI4l4moqdPGqI0fLmb18IwtWbyeyeWPevD2Bod0jnI4lEhBU6OI1KzJymJKUwp78Em4bEs1fLulJqIZpidQa/WuTM3aosJSZy9JJ+nEX3Vo3YeHd5zKgU3OnY4kEHBW6nDZrLR+k7GX60lQOF5Xx+4u6cd9F3WjYQMO0RJygQpfTkpNfwtTFqXycvo/YDuEsGJdAn/ZNnY4lEtBU6HJKrLW8uzabme+nU1ru4cGRvbh9aGcaaJiWiONU6FJjO3OLeDApha8yDzCocwtmXR1LFw3TEqkzVOhSrQqP5Y1vtvHURxnUr2d49KoYbhgUpWFaInWMCl1OavO+AiYkJvPjjsMM69mKx8fE0r5ZY6djichxqNDluErLPcz9cgtzPs8ktGF9nrvuLK48q72GaYnUYSp0+YXk7MNMWJjMxr0FXN6vPdMv70NEEw3TEqnrVOjyXyVlFTz7ySZeXpVFq7CGvHxzPL/q08bpWCJSQyp0AWBN1kEmJSaz7WAR1w/qyKSRvQlvrGFaIv5EhR7gCkrKmPXhRt76dgdRLUL41x0JnNtNw7RE/JEKPYB9vnEfUxalsi+/hDuGdub/LulBSLD+Soj4K/3rDUC5haU88l4ai3/aTY82TXjxxnM5O0rDtET8nQo9gFhreS95DzOWplFQUsb9w7tz74XdCG6gx/ZF3ECFHiD25lUO0/p0wz76RYbz5DUJ9GqrYVoibqJCdzlrLe98v5PH399AmcfDlFG9GTe0M/X12L6I66jQXWz7wUImJaawOusg53Rpwayr44iOCHU6loj4iArdhSo8lte/3srTH2cQVK8ej4+J5TcDO2qYlojLqdBdJmNv5TCt9TsPM7xXax4dE0O7cA3TEgkEKnSXKC338OIXmbywIpOwRkH87fqzuTyunYZpiQQQFboL/LTzMBMXJpOxr4Arz2rP9Mv70iI02OlYIlLLVOh+rLi0gmc+zuC1r7fSOqwRr94Sz/DeGqYlEqhU6H7qmy0HmJSYwo7cIm5IiGLSyF40baRhWiKBrNpCN8a8BowGcqy1McfZfiMwETBAAXCPtXa9t4NKpfySMp74YANvf7eTTi1DePvOcxjctaXTsUSkDqjJGfp8YA6w4ATbtwIXWGsPGWNGAvOABO/Ek2N9mr6PKYtT2F9wlLvO78IfL+5B4+D6TscSkTqi2kK31q40xkSfZPs3x3y4Bog881hyrINHjjLjvXTeW7+bXm3DePnmeOIimzkdS0TqGG9fQ78d+PBEG40x44HxAFFRUV7etftYa1ny024efi+NI0fL+b9f9eDuC7pqmJaIHJfXCt0YcyGVhT70RGustfOovCRDfHy89da+3Wj34WKmLk7l8405nNWxGbOviaNHmzCnY4lIHeaVQjfGxAGvACOttQe98ZqByuOx/Ou7Hcz6cCMVHsu00X249dxoDdMSkWqdcaEbY6KAJOAma+2mM48UuLYeKGRSYjLfbs1lSLeWPDEmjqiWIU7HEhE/UZPbFt8GhgERxphsYDoQBGCtnQs8BLQEXqx6zLzcWhvvq8BuVF7h4dWvtvLXTzYR3KAes8fGcW18pB7bF5FTUpO7XK6vZvsdwB1eSxRg0nfnMzExmZRdefyqTxsevSqGNk0bOR1LRPyQnhR1yNHyCuZ8nslLX2yhWUgQL9zQn1GxbXVWLiKnTYXugHXbDzExMZnMnCNcfXYHpo3uQ3MN0xKRM6RCr0VFpeU89VEG87/ZRrumjXj9toFc2LO107FExCVU6LXkq80HmJSUTPahYm46pxMTRvQkTMO0RMSLVOg+lldcxmPvp/Oftdl0jgjlP3cNZlDnFk7HEhEXUqH70Edpe5m2OJWDhaXcM6wr9w/vTqMgDdMSEd9QofvA/oKjzFiaxvspe+jdrimv3jKQ2Mhwp2OJiMup0L3IWkvSD7t4ZFk6xaUVPHBpT8af34Wg+hqmJSK+p0L3kl2Hi5mclMKXm/bTP6pymFa31hqmJSK1R4V+hjwey5vfbufJDzdigRmX9+GmwRqmJSK1T4V+BrbsP8KkxGS+33aI87pH8PiYWDq20DAtEXGGCv00lFV4eHlVFs99uplGDerx1DVxXDNAw7RExFkq9FOUuiuPiYnJpO3OZ0TftjxyVV9ah2mYlog4T4VeQyVlFfz9883M/TKL5iHBvHRjf0bGtnM6lojIf6nQa2DttlwmJCaTtb+Qsf0jmTa6N81CNExLROoWFfpJFB6tHKb1xupttA9vzBvjBnFBj1ZOxxIROS4V+gl8uWk/k5NS2J1XzC2Do3ng0p6ENtT/LhGpu9RQP3O4qJSZyzaQ+EM2XVqF8u5dg4mP1jAtEan7VOjH+DBlD9OWpHGoqJR7L+zK7y/SMC0R8R8qdCAnv4SHlqSxPG0vfds35Y1xA+nbXsO0RMS/BHShW2tZuC6bmcvSKSn3MHFEL+44r7OGaYmIXwrYQt+ZW8TkRSms2nyAgdHNmTU2jq6tmjgdS0TktAVcoVd4LP9cvY3ZH2VggJlX9uXGhE7U0zAtEfFzAVXomTkFTExMYd32Q1zQoxWPjYkhsrmGaYmIOwREoZdVePjHl1v422eZhDSsz19/3Y8xZ3fQMC0RcRXXF3rqrjweWJjMhj35XBbbjhlX9KVVWEOnY4mIeJ1rC72krILnPt3My6uyaBEazNzfDmBETFunY4mI+IwrC/27rblMSkwm60Ah18V3ZPKo3oSHBDkdS0TEp1xV6AUlZcxensE/12wnsnlj3rw9gaHdI5yOJSJSK1xT6CsycpiSlMKe/BLGDenMXy7tQUiwaw5PRKRaft94hwpLmbksnaQfd9GtdRMW3n0uAzo1dzqWiEitq7bQjTGvAaOBHGttzHG2G+B5YBRQBNxqrf3B20F/zlrL+yl7mL4kjbziMv5wUTfuvagbDRtomJaIBKaanKHPB+YAC06wfSTQvepXAvBS1e8+sy+/hGmLU/k4fR+xHcJ5844Eerdr6stdiojUedUWurV2pTEm+iRLrgQWWGstsMYY08wY085au8dLGf/Hio05/OGdHykt9/DgyF7cPrQzDTRMS0TEK9fQOwA7j/k4u+pzvyh0Y8x4YDxAVFTUae2sc0Qo/aOaM+OKvnSOCD2t1xARcaNaPbW11s6z1sZba+NbtTq99+aMjgjljXGDVOYiIj/jjULfBXQ85uPIqs+JiEgt8kahLwVuNpXOAfJ8df1cREROrCa3Lb4NDAMijDHZwHQgCMBaOxf4gMpbFjOpvG3xNl+FFRGRE6vJXS7XV7PdAvd6LZGIiJwW3e8nIuISKnQREZdQoYuIuIQKXUTEJUzlzzQd2LEx+4Htp/mfRwAHvBjHH+iYA4OOOTCcyTF3stYe98lMxwr9TBhj1lpr453OUZt0zIFBxxwYfHXMuuQiIuISKnQREZfw10Kf53QAB+iYA4OOOTD45Jj98hq6iIj8kr+eoYuIyM+o0EVEXKJOF7ox5jVjTI4xJvUE240x5m/GmExjTLIxpn9tZ/SmGhzvjVXHmWKM+cYY06+2M3pbdcd8zLqBxphyY8w1tZXNV2pyzMaYYcaYn4wxacaYL2szny/U4O92uDHmPWPM+qpj9vuprcaYjsaYFcaY9Kpjuv84a7zaYXW60Kl8g+oRJ9l+7BtUj6fyDar92XxOfrxbgQustbHATNzxw6T5nPyYMcbUB54EPq6NQLVgPic5ZmNMM+BF4AprbV/g2lrK5UvzOfnX+V4g3Vrbj8px3c8YY4JrIZcvlQN/ttb2Ac4B7jXG9PnZGq92WJ0udGvtSiD3JEv++wbV1to1QDNjTLvaSed91R2vtfYba+2hqg/XUPnuUH6tBl9jgN8DiUCO7xP5Xg2O+QYgyVq7o2q93x93DY7ZAmHGGAM0qVpbXhvZfMVau8da+0PVnwuADVS+3/KxvNphdbrQa+BEb1AdCG4HPnQ6hK8ZYzoAY/D/775ORQ+guTHmC2PMOmPMzU4HqgVzgN7AbiAFuN9a63E2kvcYY6KBs4Fvf7bJqx1W7RtcSN1jjLmQykIf6nSWWvAcMNFa66k8eQsIDYABwHCgMbDaGLPGWrvJ2Vg+dSnwE3AR0BX4xBizylqb72ysM2eMaULld5h/9PXx+HuhB9wbVBtj4oBXgJHW2oNO56kF8cA7VWUeAYwyxpRbaxc7G8unsoGD1tpCoNAYsxLoB7i50G8DZlW9A1qmMWYr0Av4ztlYZ8YYE0Rlmb9lrU06zhKvdpi/X3IJqDeoNsZEAUnATS4/W/sva21na220tTYaWAj8zuVlDrAEGGqMaWCMCQESqLz+6mY7qPyOBGNMG6AnkOVoojNU9fOAV4EN1tq/nmCZVzusTp+hB9obVNfgeB8CWgIvVp2xlvv7lLoaHNsP+d4AAABtSURBVLPrVHfM1toNxpjlQDLgAV6x1p70ts66rgZf55nAfGNMCmCovMzm7yN1hwA3ASnGmJ+qPjcZiALfdJge/RcRcQl/v+QiIiJVVOgiIi6hQhcRcQkVuoiIS6jQRURcQoUuIuISKnQREZf4fx6nnL5fWyXoAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot([1,2], [1,2])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "cuda_gpu_config.setup_gpu('3')\n", + "\n", + "checkpoint_name = 'baseline_fc_8x16_elu_t1'\n", + "\n", + "model_path = Path('../saved_models/cluster') / checkpoint_name\n", + "config = load_config(str(model_path / 'config.yaml'))\n", + "# config = load_config(str(model_path / '../baseline_fc_8x16_test/config.yaml'))\n", + "epoch = latest_epoch(model_path)\n", + "\n", + "prediction_npz_file = model_path / f'prediction_{epoch:05d}.npz'\n", + "\n", + "if prediction_npz_file.exists():\n", + " f = np.load(prediction_npz_file)\n", + " Y_train_fake = f['Y_train_fake']\n", + " Y_train = f['Y_train' ]\n", + " X_train = f['X_train' ]\n", + " Y_test_fake = f['Y_test_fake' ]\n", + " Y_test = f['Y_test' ]\n", + " X_test = f['X_test' ]\n", + "else:\n", + " model = Model_v4(config)\n", + " load_weights(model, model_path, epoch=epoch)\n", + "\n", + " preprocessing._VERSION = model.data_version\n", + " data, features = preprocessing.read_csv_2d(pad_range=model.pad_range, time_range=model.time_range)\n", + " features = features.astype('float32')\n", + "\n", + " data_scaled = model.scaler.scale(data).astype('float32')\n", + "\n", + " Y_train, Y_test, X_train, X_test = train_test_split(data_scaled, features, test_size=0.25, random_state=42)\n", + " Y_train_fake = model.make_fake(X_train).numpy()\n", + " Y_test_fake = model.make_fake(X_test).numpy()\n", + " \n", + " Y_train_fake[Y_train_fake < np.log10(2)] = 0\n", + " Y_test_fake[Y_test_fake < np.log10(2)] = 0\n", + "\n", + " np.savez(\n", + " prediction_npz_file,\n", + " Y_train_fake = Y_train_fake,\n", + " Y_train = Y_train ,\n", + " X_train = X_train ,\n", + " Y_test_fake = Y_test_fake ,\n", + " Y_test = Y_test ,\n", + " X_test = X_test ,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(15000, 8, 16) [0. 0.7473341 1.8435442 2.626956 2.9893162]\n", + "(5000, 8, 16) [0. 0. 0.57518786 1.8517474 2.771808 ]\n", + "(15000, 4) [105.694 41.372 0.597 -16.378 111.295]\n", + "(5000, 4) [199.847 44.66 -0.518 -24.15 146.426]\n", + "(15000, 8, 16) [0. 0.97502446 1.7760193 2.4324412 2.863718 ]\n", + "(5000, 8, 16) [0. 0. 0.3912279 1.540107 2.4478 ]\n" + ] + } + ], + "source": [ + "for i in [Y_train, Y_test, X_train, X_test, Y_train_fake, Y_test_fake]:\n", + " print(i.shape, i.ravel()[50:55])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def prepare_classification(X, Y, Y_fake):\n", + " features = (np.concatenate([X, X], axis=0),\n", + " np.concatenate([Y, Y_fake], axis=0))\n", + " targets = np.concatenate([\n", + " np.ones(len(X), dtype=int),\n", + " np.zeros(len(X), dtype=int)\n", + " ])\n", + "\n", + " shuffle_ids = np.random.choice(len(targets), len(targets), replace=False)\n", + " return (tuple(f[shuffle_ids] for f in features), targets[shuffle_ids])\n", + "\n", + "ds_train = tf.data.Dataset.from_tensor_slices(\n", + " prepare_classification(X_train, Y_train, Y_train_fake)\n", + ")\n", + "ds_test = tf.data.Dataset.from_tensor_slices(\n", + " prepare_classification(X_test, Y_test, Y_test_fake)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from models.model_v4 import preprocess_features\n", + "\n", + "dropout = 0.02\n", + "\n", + "input_params = tf.keras.Input(shape=(4,))\n", + "input_img = tf.keras.Input(shape=(8, 16))\n", + "\n", + "params = preprocess_features.python_function(input_params)\n", + "\n", + "xx = tf.keras.layers.Reshape((8, 16, 1))(input_img)\n", + "\n", + "def conv_block(xx, filters, kernel_size, padding, dropout):\n", + " xx = tf.keras.layers.Conv2D(filters=filters,\n", + " kernel_size=kernel_size,\n", + " padding=padding)(xx)\n", + " xx = tf.keras.layers.BatchNormalization()(xx)\n", + " xx = tf.keras.layers.ELU()(xx)\n", + " xx = tf.keras.layers.Dropout(rate=dropout)(xx)\n", + " return xx\n", + "\n", + "def fc_block(xx, units, dropout):\n", + " xx = tf.keras.layers.Dense(units=units)(xx)\n", + " xx = tf.keras.layers.BatchNormalization()(xx)\n", + " xx = tf.keras.layers.ELU()(xx)\n", + " xx = tf.keras.layers.Dropout(rate=dropout)(xx)\n", + " return xx\n", + "\n", + "xx = conv_block(xx, filters=8, kernel_size=(2, 3), padding='valid', dropout=dropout) # 7x14\n", + "xx = conv_block(xx, filters=16, kernel_size=(2, 3), padding='valid', dropout=dropout) # 6x12\n", + "xx = conv_block(xx, filters=32, kernel_size=1, padding='valid', dropout=dropout)\n", + "xx = tf.keras.layers.MaxPool2D()(xx) # 3x6\n", + "\n", + "xx = conv_block(xx, filters=64, kernel_size=(2, 3), padding='valid', dropout=dropout) # 2x4\n", + "xx = conv_block(xx, filters=128, kernel_size=(2, 3), padding='valid', dropout=dropout) # 1x2\n", + "xx = tf.keras.layers.Reshape((256,))(xx)\n", + "xx = tf.keras.layers.Concatenate()([xx, input_params])\n", + "\n", + "xx = fc_block(xx, units=128, dropout=dropout)\n", + "xx = xx + fc_block(xx, units=128, dropout=dropout)\n", + "xx = xx + fc_block(xx, units=128, dropout=dropout)\n", + "xx = xx + fc_block(xx, units=128, dropout=dropout)\n", + "xx = xx + fc_block(xx, units=128, dropout=dropout)\n", + "xx = xx + fc_block(xx, units=128, dropout=dropout)\n", + "logits = tf.keras.layers.Dense(units=1)(xx)\n", + "\n", + "model = tf.keras.Model(inputs=[input_params, input_img],\n", + " outputs=logits, name='classifier')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer=tf.optimizers.Adam(0.001),\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " )#metrics=[tf.keras.metrics.AUC()])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train for 938 steps, validate for 20 steps\n", + "Epoch 1/10\n", + "938/938 [==============================] - 16s 17ms/step - loss: 0.2620 - val_loss: 0.0805\n", + "Epoch 2/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0610 - val_loss: 0.0360\n", + "Epoch 3/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0418 - val_loss: 0.0872\n", + "Epoch 4/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0314 - val_loss: 0.0421\n", + "Epoch 5/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0268 - val_loss: 0.0123\n", + "Epoch 6/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0221 - val_loss: 0.0530\n", + "Epoch 7/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0212 - val_loss: 0.0363\n", + "Epoch 8/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0192 - val_loss: 0.0292\n", + "Epoch 9/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0177 - val_loss: 0.0386\n", + "Epoch 10/10\n", + "938/938 [==============================] - 12s 13ms/step - loss: 0.0162 - val_loss: 0.0117\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.fit(ds_train.batch(32), epochs=10, validation_data=ds_test.batch(512))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "features_test, targets_test = prepare_classification(X_test, Y_test, Y_test_fake)\n", + "\n", + "preds_test = model.predict(features_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZxU1Z3//9e7q7sLurvYuqtRAQUEFzSISnBF3MV9j5rkF5PMxG8mMYlJZCTLmBkzxpgYzRi3r5lxovNNdBKiiRpUjBsuuOAGKIKIG4vSC0gvdFcvn98fdaspuxuo7q61+Twfj35Qde6tW+cW1f2555x7PkdmhnPOOZesKNcVcM45l388ODjnnOvBg4NzzrkePDg455zrwYODc865Hjw4OOec68GDg3MpkvQFSQtyXY/+kmSSJuW6Hq4wyOc5OLdzkGTAZDNbleu6uPznLQc3KEkqznUdnCtkHhxcwZE0TtK9kmok1Um6SdKXJT0r6QZJdcC/SiqS9GNJ70vaIOkuScODYwyR9P+C12+S9JKk0cG2L0taLalB0ruSvpBU/kxSPUzS1yW9HRzjZkkKtoUk/UpSbXCMS4P9txu0JH1F0vLgvVdL+j9J246WtEbS94PzWS/pK0nbKyU9IGlzcD7/nlzfbu8TlnSdpA8kfSzpNklDB/Df4gYZDw6uoEgKAQ8C7wPjgTHAPcHmQ4DVwGjgauDLwc8xwESgArgp2PdiYDgwDqgEvg5skVQO3AicbGYR4HDgte1U6TTgs8BU4HPASUH514CTgWnAQcBZKZ7ihuCYw4CvADdIOihp+y5BvccA/wDcLGlksO1moCnY5+LgZ1t+DuwV1G9ScLwrU6yj2xmYmf/4T8H8AIcBNUBxt/IvAx90K3sM+EbS872BNqAY+CrwHDC122vKgU3AucDQXt7jmaTnBhyZ9PyPwNzg8ePA/0nadnywf3Efz/cvwHeCx0cDW5KPQTyYHAqEgnPbO2nbv/dS30mAiAeRPbt9ru/m+v/Xf/Lnx1sOrtCMA943s/Zetn3Y7fluxFsYCe8TDwyjgf8BHgHukbRO0i8klZhZE3AB8ZbEekl/k7TPdurzUdLjZuKtk8R7J9ene916JelkSc9Lqpe0CTgFqErapa7buSfeMxqcWyrvGQXKgJeD7rBNwMNBuXOAdyu5wvMhsPs2+u6733q3Dtgj6fnuQDvwsZm1mdm/mdkU4l1HpwFfAjCzR8zsBGBX4C3gt/2o53pgbNLzcTt6gaQw8GfgOmC0mY0A5hO/0t+RGuLnlsp71hJvgexnZiOCn+FmVrGN/d1OyIODKzQvEv/D+3NJ5cHA8hHb2Pdu4LuSJkiqAH4G/K+ZtUs6RtJngjGMzcS7ZDoljZZ0ZjD20Ao0Ap39qOcfge9IGiNpBHBFCq8pBcIEf+glnQycmMqbmVkHcC/xgfiyoLXzpW3s20k84N0gqRogqOdJve3vdk4eHFxBCf4Ink687/wDYA3xbqDe3EG8+2gh8C7QAnwr2LYLMI94YFgOPBXsWwR8j3irox6YBfxTP6r6W2ABsAR4lXgLoB3o2M65NQDfJh5YNgKfB+7vw3teSnyw+iPi53I38QDXmyuAVcDzkjYDfyc+JuMc4JPgnMuKoBVwm5ntscOd0/ee1wK7mNn27lpyrlfecnAuAyQNlXSKpGJJY4CfAPdl+D33kTRVcTOI3+qa0fd0g5cHB+cyQ8C/Ee8eepV419WVAJIat/Ezc4DvGSE+7tAE/C/wK+CvAzym20l5t5JzzrkevOXgnHOuh0GRnKyqqsrGjx+f62o451xBefnll2vNrNfJj4MiOIwfP57FixfnuhrOOVdQJL2/rW3ereScc64HDw7OOed68ODgnHOuBw8OzjnnevDg4JxzroeUgoOkO4JlCZdtY7sk3ShplaQlyStXSbo4WEbxbUkXJ5UfLGlp8Jobk5ZXHCXp0WD/R5NWuXLOOZclqbYcfgfM3s72k4HJwc8lwK0Q/0NPPKfMIcAM4CdJf+xvJb6UYuJ1iePPBR4zs8nEV/Kam2IdnXPOpUlK8xzMbKGk8dvZ5UzgLovn4nhe0ghJuxJf1vBRM6sHkPQoMFvSk8AwM3s+KL+L+Bq7DwXHOjo47p3Ak6SWC7/P3qttYt7LazhiUhUH7TGCcHEoE2+TMS1tHdzx7Lu0xLaZBXrQC5eEuPjw8VSEB8WUnYLR1tHJ6x9u4qX3NrIl1tuifKkpKhKnH7Abe0azv85Qa3sH972ylqP3rmaX4UOy/v7p8Ou/r+TgPUYyc3L6F/FL12/UGD69JOGaoGx75Wt6KYf4Cljrg8cfEV/SsQdJlxBvpbD77rv3q9JL1n7CLU+u4qYnVjG0JMSMCaM4clIVR0yqYp9dIhQVpbIAV+48u6qWXzy8AgDld1UzxgxKQuKSo/bMdVUGtc5O4831m1n0Th3PvlPLi+/W0xxclAzku2cGtzz5DpcdP5lLZk6kOJSdYdBXP9jIP89bwtsbGjl67yi/+8qMrLxvOnV2Gjc+9jbfPGZSXgeHjDAzk9RrZkAzux24HWD69On9yh54xgG7cfTeUZ5/p45nV9XyzKparp6/HIDK8lIOn1TFkZMqOWJSFWNHlvX3NDJmQ0N8HZfn5h7LbiOG5rg2uXHOLc/yp8Vr+NrMiWhnjZAZYGasrm3iuVW1PPdOHYtW17GpuQ2ASdUVnHfwWA7fs5JDJ1Yyoqy03+9T09DKT+5fxi8eXsH8peu59typ7Lfb8HSdRg9bYh38asEK7nj2XUYPG8JZ03bjL6+t4/nVdRw6sTJj75sJm7a00WkwcgCf//akKzis5dPr1Y4NytaytYsoUf5kUD62l/0BPpa0q5mtD7qmNqSpjr0aNqSEE/fbhRP32wWA9Z9s4dlVW4PFA6+vA2B8ZRlHTKriyElVHL5nFcPLSjJZrZTUBMGhsiIzX45CcP70cfzg3qW8vuYTpo0bkevqFLS1m7Z0BYPn3qnl483x79eYEUM5Yd/RHDGpisP2rGT0sPR1wUQjYW75wsE8vGw9P/7LG5x507N8fdaefOu4SWnv5l30Th1z713C+3XNfOGQ3Zl78j6UhIp4fnU9P3/oLe77xuEFdYFR3xQDMvf7n67gcD9wqaR7iA8+fxL8cX8E+FnSIPSJwA/MrF7SZkmHAi8QX+v2N0nHuhj4efBvVvPR7zp8KOcdPJbzDh6LmfH2hkaeebuWZ1fV8pdX1/L7Fz5AgqljhnPEpCqO2aeaz44flc0qdqltbGVEWUnBjZWk02lTd+XfHniDPy7+sOCCw2sfbuLnDy3njAPGcNGMcVn/w9TZaTy6/GOeXFHDondqea+uGYi3mg/bM95iPnzPSnYfVZbxus3ef1cOnVjJVQ++yU1PrOKRNz7i2vOmctDuA79ZsaGljWseeos/vPABe1SWcffXDuWwPbe2Ei47fjJz713Kgjc/5qTgIrEQJILDqPIcBgdJdxNvAVRJWkP8DqQSADO7jfj6uKcQX5O2GfhKsK1e0k+Bl4JDXZUYnAa+QfwuqKHEB6IfCsp/DvxR0j8A7wOf6//pDYwk9hodYa/REb565ISuQbhnVsWDxe0LV3PLk+/w9+8dxaTqSNbrV9PQSlVFOOvvm08iQ0o4ef9deeD1dVx52hSGlOR/oDQz7nzuPa6ev5wiiedX1/PA6+u49typ7F6Zne7LlR838MN7l7L4/Y1EwsUcMnEUXzpsPIdPqmTv0ZGcXEGPKCvl+s9N4/QDduNH9y7l3Fuf46tHTOD7J+5FWWn/rmOfeGsDP7xvKR9vbuFrMyfwvRP2Zmjpp78j5x08lt8+vZpfPrKC4/apztq4x0DVN8VbdjkNDmZ20Q62G/DNbWy7g/hC793LFwP791JeBxyXSr2yrSRUxPTxo5g+fhSXHb8XL6yu44Lbn+fDjVtyFhyiO3lwADh/+ljue3Utj7zxEWdOG7PjF+RQQ0sbc/+8lL8tXc9x+1Rz3fkH8NCyj/jZ/OWc9OuFzDlpby4+fDyhDN0M0dLWwY2Pvc3tC1cTGVLML86byjkHjsmrP4jH7F3Ngu/N4tqH3uK/nnmXR9/8mJ+f+xkO37Mq5WNsbIpx1YNvct+ra9lrdAW3fvGIbbYsi0NFzDlpb77+/17h3lfW8rnPjut1v3xTl+GWQ/58IwrQrsPjg8CJvv9sq21sJRrx4HDohErGjhzKnxav2fHOObR8/WbOuOlZHn7jI+aevA+//dJ0RpaX8vlDdmfBd4/i0ImjuOrBNzn/tudYtaEx7e//1MoaTrxhIbc8+Q5nHTiGx75/NJ+bPi6vAkNCRbiYn561P/dccihFgs//9gV+cO9SNre07fC185eu54QbnuKB19fx7eMm88C3jtxhl+NJ++3CAeNGcMPfV9LSVhi3hm/04JC/En+YcxUcvFsprqhInHfwWJ59p5Y1G5tzXZ1e/XHxh5x187M0tbbzh388hK/P2vNTt0rvNmIod3z5s1z/uQNYXdvEKTc+zc1PrKK9o3PA772hoYVv3/0qF9/xIsVF4g9fO4Trzj8gY39U0unQiZU89J2juOSoifzvSx9w4vULefytj3vdd8PmFr7+Py/zjd+/wq7Dh/LAt47keyfsldKYnCSumL036z9p4X8WbXOJg7xS1xSjIlycsTFHDw4DMLQ0RCRcnJPg0BxrpynW4S2HwLkHjcUM/vzy2h3vnEVbYh1c/qfX+ed5S5g+fiR/+/ZMDtnGLZOSOOegsTz63Vkcv281v3xkBWfd8ixvrtvcr/fu7DT+8MIHHP+rp3h42UdcdvxkHrpsZp+6Z/LB0NIQPzxlX+79xhEMH1rCV3+3mMvuebVrQNbM+NPiDzn++qd4fMUG5p68D/d943D23XVYn97n8D2rOGqvKDc/uSqlFkqu1TfFMhrgPTgMUDQSzklwqG2Idb2/g3GjyjhiUiXzXvmQzs5+TXtJu9U1jZx9y7P8+ZU1fPvYSdz11UNS+v9K3N556xcO4qNPWjjjpme4fsEKWttT7+5Y8VED5//fRfzwvqVM2W0YD102k8uOT+0qOl9NGzeCB751JN85bjIPLlnPCdc/xT0vfsDF//0Sc+YtYe9dIjz8nZl8fdae/e4q++eT9mZTcxu3P7U6zbVPv/qmGCM9OOSvXAWHmsYWAKp24jkO3Z1/8Dg+rN/CC+/W73jnDHtwyTpO/80zbGho5XdfmcH3Tty7z4PMJ39mVx797izOOGA3bnx8Faf/5hle+3DTdl+zJdbBtQ+/xak3Ps3qmkZ+df4B3P21Q3OSniITSouL+O4Je/Hgt49kzMihzL13KYvfq+eqM/fjfy85jIkDPM/9xwzn9AN247+eeZcNm1vSVOvMqG+KUenBIX9FI2FqGnMQHIKA5C2HrU7abxci4WL+tPjDHe+cIa3tHfzkr8u49A+vss+uw/jbt49k1l79T20wsryU6y+Yxn9/+bM0tLRzzi3P8rP5y3sdNH1yxQZO/PVT3PrkO5wdDDife/DYgprYlap9dhnGvf90ODdedCALvnsUXzpsfNrS3Xz/hL1o6+jkxsffTsvxMsW7lfJc7loO3q3U3dDSEKdP2435y9bTkIM+4w/rm/ncbYu4c9H7/OORE7jnkkO77mgbqGP2qWbBd4/iwhm7c/vC1Zz8H0/zYtBC2rC5hUv/8Apf/u+XKA0Vcc8lh/LLAhlwHojiUBFnHLBb2lPbjK8q58IZ47jnxQ95r7YprcdOFzOjzlsO+S0aCdPY2k7zADJT9kdNQysSjMpQXpVCdf7BY2lp6+RvS9bveOc0emz5x5z2m2dYXdvEbV88mB+fNoWSNN8iGhlSws/O/gx/+NohdHQan/u/i/jm71/huOufYsGbH/O9E/Zi/ndmFlyOoHz07eMmUxIq4lePrsx1VXrVFOsg1t7pYw75rDoSzzOT7dZDTUMrleWleXmPei5NGzeCSdUV/Onl7Mx5aO/o5OcPvcU/3LmYsSOH8uC3jmT2/plNwXD4nlU8fNlMvnrEBOYvW89nxgzn4e/M5NvHTS7oAed8Uh0Zwj8cOYEHXl/HsrWf5Lo6PWR6jgN4cBiwXM11qG30OQ69kcT5B4/l5fc3ZmQiWbLmWDtf+M8XuO2pd/j8Ibvz5386nD0qyzP6ngllpcVcefoUXvnxCfz+Hw8Z8ECs6+mSWRMZUVbCtQ+/leuq9JCYHe3dSnkskb4iFy0HH2/o3dkHjSFUJOZluPXwm8dX8cK79fzq/AP42dmfyUlep5HlpYNywDkfDBtSwjePnsTTb9fy3KraXFfnUzKdVwk8OAxY4g/0hlwEB2859Ko6MoRj9o5y7ytr0jLDuDdvf9zAbxeu5vyDx3LuwWN3/AJXkP6/w/Zgt+FDuPaRFcRTyOWHukbvVsp7o8pLCRUpqy0HM/O8Sjtw3sHj2NDQytNvp/+Kz8z4l78uozxczNyT90n78V3+GFIS4rIT9uL1Dzfx8LKPcl2dLhubPTjkvVCRqCwvzWpwaGhtp7W908cctuPYfaoZVV7KHzMw5+Gvr63j+dX1XDF7Hyr9/2DQO+fAMUyqruCXC1ZkrCXaV3VNMUpDRRldO92DQxpkeyKcT4DbsdLiIs4+cAx/X/5xVw6edPhkSxv//rflHDBuBBcWSGpnNzCJlN6ra5oyPo6VqvrG+AS4TI43eXBIg+pImA0N2ZtqX+vBISXnTx9LW4fx19fSl4zv+gUrqG9q5eqz9k/bjFyX/06cMpoDdx/Br//+dl6k9M50XiXw4JAW2Z4lnWileLfS9u2zyzA+M2Z42tZ5WLrmE/7n+ff50mHj2X/M8LQc0xWGeErvffhocwu/e+69XFeH+ubMzo6GFIODpNmSVkhaJWluL9v3kPSYpCWSnpQ0NmnbtZKWBT8XJJUfK+mVoPxOScVB+UhJ9wXHelFSj9Xi8k00Eqa2MZa1bKDerZS6z00fy5vrNw94IlNHp/HjvyxlVHmY7524V5pq5wrJoRMrOXrvKLc8sYpPmnOb0jvTeZUgheAgKQTcDJwMTAEukjSl227XAXeZ2VTgKuCa4LWnAgcB04BDgMslDZNUBNwJXGhm+xNfK/ri4Fg/BF4LjvUl4D8GdoqZF60I09FpXXcQZFptYyvFRWLE0JKsvF8hO+OAMZQWFw24r/ielz7g9TWf8C+n7cuwIf6576z++aR92NzSzm0L38lpPRJjDpmUSsthBrDKzFabWQy4Bziz2z5TgMeDx08kbZ8CLDSzdjNrApYAs4FKIGZmicQljwLndj+Wmb0FjJc0us9nlkXVw+IpNLI116GmoZXKilLv807B8LISTpwymr+8trZP6yEkq21s5RcPr+CwiZWcccBuaa6hKyRTdhvGmdN247+ffZePc5TSu7W9g4bW9rwIDmOA5PsB1wRlyV4Hzgkenw1EJFUG5bMllUmqAo4BxgG1QLGk6cFrzgvKP3UsSTOAPYAes4wkXSJpsaTFNTU1KZxG5mQ7hYbPju6b86ePY1NzG48t39Cv1//8obdojrXz07P289nIju+fsDcdncZ/PJablN6bgi6tfAgOqbgcmCXpVWAWsBboMLMFwHzgOeBuYFFQbsCFwA2SXgQagMRl3c+BEZJeA74FvJq0rYuZ3W5m081sejTa/3z56ZDtFBq1jTGfHd0HR06qYtfhQ/o15+HFd+uZ9/IavjZzIpOqIxmonSs0u1eW8fkZu/O/L33I6prM5u/qTWJ2dD4MSK9l61U9xK/iP3VvoJmtM7NzzOxA4EdB2abg36vNbJqZnQAIWBmULzKzmWY2A1iYVL7ZzL5iZtOIjzlEgbxesy/bKTRqGjzpXl+EisS5B41l4coaPvok9a6Ato5O/uUvyxgzYijfOnZyBmvoCs2lx04mXFzErxZkP6V3Yt5OPtzK+hIwWdIESaXEr/jvT95BUlUwyAzwA+COoDwUdC8haSowFVgQPK8O/g0DVwC3Bc9HBO8D8I/Exyz6t8J6lpSHiykvDWWl5dDZ6akz+uO8g8fSaXDvq6kPTP/u2fdY8XEDPzl9CkNLPRW22yoaCfOPR07gb0vXs2TN9pduTbe6IOlezlsOZtYOXAo8AiwH/mhmb0i6StIZwW5HAyskrQRGA1cH5SXA05LeBG4HvhgcD2COpOXEB6kfMLPEgPa+wDJJK4jfIfWdgZ5kNmRrlvSmLW20d5oHhz4aX1XOjAmj+NPiNSklUFv/yRZu+PtKjtunmhOm5PX9EC5HvnbURIqLxCNvZDfnUjbWcgBIKTGHmc0nPnaQXHZl0uN5wLxeXtdC/O6j3o45B5jTS/kioOBuJI9PhMv83Qu1PgGu384/eCxz5i3h5fc3Mn38qO3u+9MH36Sj0/jXM3wQ2vUuMqSEqorsLxNc3xRDghEZXgXSZ0inSTQSzsqYg0+A679TPrMrZaWhHc6YfnLFBuYv/YhvHTuJcaPSuz6xG1yy9XufrK4pxoihJYQyfCu7B4c0qY4MycoVhAeH/isPF3PqZ3blwSXrtrnmd0tbBz+5/w0mVpXztaMmZrmGrtBUZzl1DsTTdWe6Swk8OKRNNBKmoaU940m5vFtpYD732XE0xTqYv7T3fuLbnnqH9+uauerM/X09ZrdDOWk5NMaoLM/8778HhzTJ1lyHmoZWSouLGDYkc3ncB7Ppe4xkQlU5f+plzsN7tU3c8uQ7nH7Abhw5uSoHtXOFJhoJU9fYSkeW8qpBdvIqgQeHtIkOy85ch8TyoD5I2j+SOO/gsbzwbj0f1DV3lZsZV97/BqWhIn586r45rKErJNWRMJ229fbSbMhGum7w4JA2WWs5NLZS5eMNA3LOQWMoEsx7eWvr4eFlH7FwZQ3fO2EvRge5spzbkWynzukMEnxmeo4DeHBIm+rElyTDcx0SLQfXf7sOH8rMyVHmvbyGjk6jsbWdf3vgTfbddRhfOmyPXFfPFZBoJH4hka3g8MmWNjot83McwIND2sSX7Mv8l8RnR6fH+dPHsu6TFp57p5YbH3ubjza38O9n7U9xyH8lXOqqs5w6py6YAFdZkfng4KOaaVIcKqKyPLMT4To6jfqmGNEsfDEGu+P3Hc3woSVc/+hKlqz5hAs/O46D9xiZ62q5ApPtbqWuvEoZngAH3nJIq0wvF1rX1Eqn+RyHdBhSEuKsabvx6gebGDakmCtm75PrKrkCNKQkRGRIcdaDg3crFZhMBwefAJdeF3x2d0JF4oen7JuVuz/c4JTNiXD13q1UmKIVYVZ93JCx49cGedx9Alx6TNltGK/8+ASGl/myn67/4hPhsrMqXH1wy6x3KxWY6mHxzKypZP3sD285pJ8HBjdQ2UqdA/EB6fLSEENKMj9734NDGkUrwrR1WNcyfumW+AJ6y8G5/JHNFBobm2KMytINKR4c0iia4bkOtY2tlJWGKA97b6Bz+SIaCdMc66CptfdkjulU1xRjVBbyKoEHh7TK9G1tNQ0+x8G5fJPNuQ71TdmZHQ0eHNJq65ckM4NTPjvaufyTzbkO9U2xrAxGQ4rBQdJsSSskrZI0t5fte0h6TNISSU9KGpu07VpJy4KfC5LKj5X0SlB+p6TioHy4pAckvS7pDUlfSceJZkOmvyS1ja0+3uBcnqkOUmhk+o4ls/gk2GzcxgopBAdJIeBm4us5TwEuktR96c/rgLvMbCpwFXBN8NpTgYOAacAhwOWShkkqAu4ELjSz/YH3gYuDY30TeNPMDiC+NvWvJBXETegV4WKGlBRlrlvJU2c4l3ey1XJojnXQ2t6ZlQlwkFrLYQawysxWm1kMuAc4s9s+U4DHg8dPJG2fAiw0s3YzawKWALOBSiBmZiuD/R4Fzg0eGxBRPCd1BVAPZH6kJw0kUR0ZkpG+x1h7J5ua2zw4OJdnRgwtoSSkjAeHbM6OhtSCwxggeWWUNUFZsteBc4LHZxP/414ZlM+WVCapCjgGGAfUAsWSpgevOS8oB7gJ2BdYBywFvmNmnd0rJekSSYslLa6pqUnhNLIjU7OkE/nivVvJufxSVCSqKjJ/O2si6d6ofBpzSMHlwCxJrwKzgLVAh5ktAOYDzwF3A4uCcgMuBG6Q9CLQACTW1zwJeA3YjXh31E2ShnV/QzO73cymm9n0aDSaptMYuGhFZoKDT4BzLn9lI4XGxkRwyJcxB+J/6MclPR8blHUxs3Vmdo6ZHQj8KCjbFPx7tZlNM7MTAAErg/JFZjbTzGYACxPlwFeAey1uFfAuUDBZ0aKRcEbmOXhwcC5/ZWMiXFe67jzqVnoJmCxpQjAwfCFwf/IOkqqCQWaAHwB3BOWhoHsJSVOBqcCC4Hl18G8YuAK4LXj9B8BxwbbRwN7A6v6eYLZVR8Jsam6jtb1jxzv3QW1jolupIMbmndupRLOQQiORVylvxhzMrB24FHgEWA780czekHSVpDOC3Y4GVkhaCYwGrg7KS4CnJb0J3A58MTgewBxJy4kPUj9gZokB7Z8Ch0taCjwGXGFmtQM90WxJXNknkuSli6fOcC5/RSNh6ppaae/oMTyaNnVNMUpCoiJLGRJSehczm0987CC57Mqkx/OAeb28roX4HUu9HXMOMKeX8nXAianUKx8l39Y2ZsTQtB23pqGVYUOKs5JwyznXN9FIGLP4HUXVGVqDfGNTLFhxUhk5fnc+QzrNMnXPc21jjCofb3AuL2UjhUZ9FvMqgQeHtMvUbElPneFc/srGRLi6LOZVAg8OaZeY2p7uL4nPjnYuf2U6rxoEeZU8OBSuklARo8pL09+t1OB5lZzLV4nfzUy2HLKZkRU8OGREuifCbYl10NDa7i0H5/LUkJIQw4eWZGzMIdbeSUNLe9ZuYwUPDhlRPSy9E2IScxw8ODiXvzKVOgdgY3N28yqBB4eMSHfLITHj2geknctf1RmcJZ3tpHvgwSEjEik04imkBs5TZziX/zLZcvDgMEhEI2Fi7Z1sbklPpnEPDs7lv0TyvXRdFCbLdl4l8OCQEVvveU7PbW2JMYdsXjU45/omGgmzpa2Dxtb0Lz9Tn4O/AR4cMiCa5tmSNQ2tjCovpSTk/13O5atMToSrb25DghFZWssBPDhkRHWavyQ+O9q5/Lc1O0IGgkNTKyOGlhAqyk5eJfDgkBHR4EuSruBQ29hKVcS7lJzLZxltOSBD27wAABzDSURBVARJ97LJg0MGDBtSTGlxUfpaDo3ecnAu32Uy+V5doweHQUFS2uY6mFm8W8nvVHIurw0fWkJpKH0Xhck2NntwGDTStVxoU6yDlrZOz6vkXJ6TFCwXmv7ke9lO1w0eHDKmOhJmw+aBBwef4+Bc4ajKwES4zk5jY3NbVuc4QIrBQdJsSSskrZI0t5fte0h6TNISSU9KGpu07VpJy4KfC5LKj5X0SlB+p6TioHyOpNeCn2WSOiSNSsfJZlO6Wg4eHJwrHNUZCA6fbGmjo9Oymq4bUggOkkLAzcDJxJf8vEhS96U/rwPuMrOpwFXANcFrTwUOAqYBhwCXSxomqQi4E7jQzPYH3gcuBjCzX5rZNDObBvwAeMrM6gd+qtkVjYSpb4rRNsA1ZRMT4Lxbybn8l4kUGvXN2Z8dDam1HGYAq8xstZnFgHuAM7vtMwV4PHj8RNL2KcBCM2s3syZgCTAbqARiZrYy2O9R4Nxe3vsi4O5UTyafJK706xpjAzqOtxycKxzVkTB1abgoTJaLvEqQWnAYA3yY9HxNUJbsdeCc4PHZQERSZVA+W1KZpCrgGGAcUAsUS5oevOa8oLyLpDLigeTPvVVK0iWSFktaXFNTk8JpZFe6lgutaWglVCRGZnFmpHOuf9J1UZgscax8DA6puByYJelVYBawFugwswXAfOA54i2ARUG5ARcCN0h6EWgAOrod83Tg2W11KZnZ7WY23cymR6PRNJ1G+qRrQkxtYzx1RjZnRjrn+ieagRXhctVyKE5hn7V8+qp+bFDWxczWEbQcJFUA55rZpmDb1cDVwbY/ACuD8kXAzKD8RGCvbu97IQXapQTpCw6eOsO5wlE9LLnHYHhajpmLhX4gtZbDS8BkSRMklRL/o31/8g6SqoJBZogPIt8RlIeC7iUkTQWmAguC59XBv2HgCuC2pOMNJ94C+Wv/Ty23qiri/5EDDg6NPgHOuUKRiRQadY0xyktDDCkJpe2YqdhhcDCzduBS4BFgOfBHM3tD0lWSzgh2OxpYIWklMJqgpQCUAE9LehO4HfhicDyAOZKWEx+kfsDMEgPaEB+3WBAMYhekcHGIEWUDX1O2tqHV71RyrkAkLgrTmUKjvqmVURXZH3NMpVsJM5tPfOwguezKpMfzgHm9vK6F+B1LvR1zDjBnG9t+B/wulbrls4Gm0DAzbzk4V0ASF4VpbTk0xRiVgxtSfIZ0Bg10ItwnW9po6zAPDs4VkOo0p9DIRV4l8OCQUQOdLZl4bVUOmpTOuf5J90S4+sbs51UCDw4ZlUjC1d81ZROtDm85OFc4qiND0jbmYGbUNcWozMEFogeHDIpGwrS0dfZ7TdnE1Ue1BwfnCkai5dDfi8JkzbEOWts7czIJ1oNDBg30trat3UoeHJwrFNWRMK3tnWxu6d9FYbLEBLhs51UCDw4ZVT3A5UJrG2OUhMTwoSXprJZzLoPSOdchV7OjwYNDRiW+JP3tf0zMjpY8dYZzhWLr7/3A71jqCg4+5jC4DDTPSk1jK1U+3uBcQalOY8uhLhEcfMxhcBlRVkJJSP2e61DreZWcKzjRioF1Jyfb6C2HwUnSgGZJ++xo5wrPsKHFlBYXpa3lUBISkXBKySzSyoNDhsXnOvT9S9LRadR5cHCu4Az0ojBZfVM8ZX8uxh09OGRYf2dLbmyO0Wl+G6tzhah6WP8uCrurb2rL2UJfHhwyLBoZ0q/g4MuDOle40tlyyMXsaPDgkHHRSJi6plba+7imrAcH5wpXvOWQnltZc5FXCTw4ZFw0EsZs6/3Kqapt9NnRzhWqaMUQNja3EWvv20Vhd3VNsZzMjgYPDhmXuBW1r/2P3nJwrnBVD4v/3tYOIGV/W0cnDS3tPuYwWCW+JH2d61DT0MrQkhDlpdldGtA5N3ADnQALuZ3jACkGB0mzJa2QtErS3F627yHpMUlLJD0paWzStmslLQt+LkgqP1bSK0H5nZKKk7YdLek1SW9IemqgJ5lLXV+SzX37ktQ2tlIVyc0tbM65gUlcFA7kjqW6HCbdgxSCg6QQcDNwMvElPy+S1H3pz+uAu8xsKnAVcE3w2lOBg4BpwCHA5ZKGSSoC7gQuNLP9gfeBi4PXjABuAc4ws/2A8wd8ljnUlYSrry2HRp8d7VyhSkfyvVwm3YPUWg4zgFVmttrMYsA9wJnd9pkCPB48fiJp+xRgoZm1m1kTsASYDVQCMTNbGez3KHBu8PjzwL1m9gGAmW3o+2nljyElISJDivv8Jalp8AlwzhWqyvKBJ98rhOAwBvgw6fmaoCzZ68A5weOzgYikyqB8tqQySVXAMcA4oBYoljQ9eM15QTnAXsDIoHvqZUlf6q1Ski6RtFjS4pqamhROI3f6s1xobWPM71RyrkCVFhcxqrx00LccUnE5MEvSq8AsYC3QYWYLgPnAc8DdwKKg3IALgRskvQg0AB3BsYqBg4FTgZOAf5G0V/c3NLPbzWy6mU2PRqNpOo3MiPZxwfG2jk7qm2LecnCugEUrBjZLuq4phkTO7lZKJZvTWrZe1QOMDcq6mNk6gpaDpArgXDPbFGy7Grg62PYHYGVQvgiYGZSfSLzFAPGWSV3QDdUkaSFwQOJ1hSgaGcLSNZtS3r+uMRa8zoODc4WqetjAZknXN7UyYmgJoaLc3JSSSsvhJWCypAmSSolf8d+fvIOkqmCQGeAHwB1BeSjoXkLSVGAqsCB4Xh38GwauAG4LXv9X4EhJxZLKiA9kL+//KeZeX7uVfAKcc4VvoCk0Nja1MTJHXUqQQsvBzNolXQo8AoSAO8zsDUlXAYvN7H7gaOAaSQYsBL4ZvLwEeDq4HXMz8EUzSyysOkfSacQD1K1m9njwfsslPUx88LoT+E8zW5ae082NaCRMU6yDptZ2ylNIvesT4JwrfNGg5WBm/bolva6pNWe3sUJq3UqY2XziYwfJZVcmPZ4HzOvldS3E71jq7ZhzgDnb2PZL4Jep1K0QJE+I6VNw8JaDcwUrWhEm1tHJ5i3tDC/r+zrw9U0xJlSVZ6BmqfEZ0lnQ17kOif285eBc4aoeFl8Rrr+3s+Yy6R54cMiKrhQaKfY/1jS0EgkXM6TEU2c4V6gGkkKjs9PY2NzGqPK+tzjSxYNDFvT1S+LLgzpX+AaSQmNzSxsdneYth8FuZFkpoSKl3LysbWj1O5WcK3ADSaGR67xK4MEhK4qKRFVF6rMlveXgXOGLhIsJFxf1a8whMTs6l7eyenDIkuo+LBfqeZWcK3yS+j0Rrt5bDjuPaCSc0t1KLW0dNLS0U5WjHO7OufTpbwqNXOdVAg8OWROtCLMhhTUdav02VucGjb70GCTz4LATiUbC1DXF6Oi07e7ns6OdGzziSTf7MSDdGKOsNJTT29k9OGRJ9bAwHZ3GxubYdverDZLu+d1KzhW+6kiYT7a00dreseOdk2xsjuW01QAeHLImMddhR11L3nJwbvDo7+2sdU2xnA5GgweHrEk1hUbiS1SZw8kvzrn06Gt2hIT6plZvOewsqiPxPCs7+pLUNrYyoqyE0mL/r3Gu0EUrUvu9766+MZbTOQ7gwSFrqiLx/+gdfUlqGlo9G6tzg0R/U2jUN3u30k6jrLSYinDxDmdL+uxo5waPUeWlSH1rOTTH2mlp68xpXiXw4JBV0RRWhKtt9LxKzg0WJaEiRpWV9qnlkFgm2FsOO5FUgoOnznBucEnl9z5Z4nb3ghhzkDRb0gpJqyTN7WX7HpIek7RE0pOSxiZtu1bSsuDngqTyYyW9EpTfKak4KD9a0ieSXgt+ruz+foVqRyk0mlrbaY51eHBwbhCJB4fUk+/V5cHsaEghOEgKATcDJxNf8vMiSd2X/rwOuMvMpgJXAdcErz0VOAiYBhwCXC5pmKQi4E7gQjPbH3gfuDjpeE+b2bTg56oBnWEeiVaEqdnOPIdE6gzvVnJu8OhrCo36AupWmgGsMrPVZhYD7gHO7LbPFODx4PETSdunAAvNrN3MmoAlwGygEoiZ2cpgv0eBc/t/GoUhGgnT0NrOlljvsyV9Apxzg0+ix8Bs+6lzErryKuU4+WYqwWEM8GHS8zVBWbLXgXOCx2cDEUmVQflsSWWSqoBjgHFALVAsaXrwmvOC8oTDJL0u6SFJ+/VWKUmXSFosaXFNTU0Kp5F71cEf/dptdC11BQdvOTg3aFRHwrR1GJua21Lav745RklIRMLFGa7Z9qVrQPpyYJakV4FZwFqgw8wWAPOB54C7gUVBuQEXAjdIehFoABKX068Ae5jZAcBvgL/09oZmdruZTTez6dFoNE2nkVmJFsG27lzo6laKeLpu5waLHf3ed1ffGGNkWSmSMlmtHUolOKzl01f1Y4OyLma2zszOMbMDgR8FZZuCf68Oxg5OAASsDMoXmdlMM5sBLEwq32xmjcHj+UBJ0OooeFvzrPQ+OFXT0EqRPHWGc4NJdR/zK9U15T7pHqQWHF4CJkuaIKmU+BX//ck7SKoKBpkBfgDcEZSHgu4lJE0FpgILgufVwb9h4ArgtuD5LgpCpqQZQR3rBnKS+WJHSbhqGlsZVR4mVJTbKwbnXPpsbTmkdsdSfVMrlXmw2NcOO7XMrF3SpcAjQAi4w8zekHQVsNjM7geOBq6RZMRbAd8MXl4CPB38rd8MfNHM2oNtcySdRvyP/61mlhjQPg/4J0ntwBbidzSlNpKT5yrLwxRtZ7ZkTUPMV4BzbpCpHta3/Eobm9vYbcTQTFYpJSmNeATdO/O7lV2Z9HgeMK+X17UQv2Opt2POAeb0Un4TcFMq9So0oSJRWbHtuQ6eOsO5wae8NMTQklDq3UqNrTm/jRV8hnTWbW+50FqfHe3coCMp5RXh2jo62dzSnvO8SuDBIeuqh/XecjCzeMvBb2N1btCpTjGFxsY8meMAHhyyLlrR+5dkc0s7sfZObzk4NwjFWw47HpCuD/IqjSrz4LDTSSTh6uz89Bi7z452bvBKteWQSJ1RKLeyujSKRsK0dxqbtnx6tqTnVXJu8IpGwmxuaaelrffUOQmJpHv5cCurB4cs29Zyod5ycG7wSnWZ4Po8ycgKHhyyblsT4TyvknODV6opNBLBYcTQkozXaUc8OGTZtmZL1jS2UlwkhufBl8I5l147yo6QUN8UY0RZCcWh3P9pzn0NdjLb+pLUNsSXBy3y1BnODTrVO8irllCfJ3mVwIND1lWEiykr7Tlb0mdHOzd4VVZsP3VOQl1TfsyOBg8OOdHbcqE1Da2eV8m5QSpUJEaV73iW9MamNkbmwRwH8OCQE72l0Kj1loNzg1oqcx3qmmJ5cRsreHDIie4th85Oo7Yx5sHBuUGstx6DZJ2dxsZmH3PYqXW/gtjYHKOj03wCnHODWDSy7aSbAJtb2ujotLxIugceHHIiGgnzyZY2WtvjsyVrgynz3nJwbvCqjoSpbeyZOidh6wS4/Lid3YNDDnS/ndUnwDk3+CVS52wMkut1tzU45MffAQ8OOdB9Kn1NY/ze5ypvOTg3aHX93m9j3KErr1IhjTlImi1phaRVkub2sn0PSY9JWiLpSUljk7ZdK2lZ8HNBUvmxkl4Jyu+UVNztmJ+V1C7pvIGcYD7q3nKobfBuJecGu67sCNsYd8invEqQQnCQFAJuBk4mvuTnRZK6L/15HXCXmU0FrgKuCV57KnAQMA04BLhc0jBJRcCdxNeH3h94H7i423teCywY2Onlp67g0Nja9W+4uIhIOKVVW51zBah6Byk0Ci44ADOAVWa22sxiwD3Amd32mQI8Hjx+Imn7FGChmbWbWROwBJgNVAIxM1sZ7PcocG7S8b4F/BnY0MfzKQiV5aVIW68gaoLUGZKnznBusNpR8r36phhlpSGGlISyWa1tSiU4jAE+THq+JihL9jpwTvD4bCAiqTIony2pTFIVcAwwDqgFiiVND15zXlCOpDHBMW7dXqUkXSJpsaTFNTU1KZxG/igOFVFZXtrVcvAJcM4NfuXhYsp7SZ2TkE95lSB9A9KXA7MkvQrMAtYCHWa2AJgPPAfcDSwKyg24ELhB0otAA5BYBePXwBVm1rm9NzSz281suplNj0ajaTqN7KlKWi60psGDg3M7g+0tF5pvwSGVTu61BFf1gbFBWRczW0fQcpBUAZxrZpuCbVcDVwfb/gCsDMoXATOD8hOBvYLDTQfuCbpYqoBTJLWb2V/6cX55Kxr5dHA4cPeROa6Rcy7TqiNDtttyyJfUGZBay+ElYLKkCZJKiV/x35+8g6SqYJAZ4AfAHUF5KOheQtJUYCrBILOk6uDfMHAFcBuAmU0ws/FmNh6YB3xjsAUG2Boc2js6qW/21BnO7Qyi28mvlG8thx0GBzNrBy4FHgGWA380szckXSXpjGC3o4EVklYCowlaCkAJ8LSkN4HbgS8GxwOYI2k58UHqB8wsMaC9U0hcQdQ1xTDz21id2xlsLzjkU7puSK1bCTObT3zsILnsyqTH84hf5Xd/XQvxO5Z6O+YcYM4O3vfLqdSvEEUjYWIdnbyzoTH+PI+ak865zIhGwjS0trMl1sHQ0q13JW2JddDS1snIPAoOPkM6RxIthTfXb/7Uc+fc4LWtlSDrmuLP86nl4MEhRxITYt5cFwSHiiG5rI5zLgu6JsI1fvqOpXzLqwQeHHKme8uhKpI/VwzOuczYVgqNujybHQ0eHHIm8SVZtaGR8tIQZaWeOsO5wW5byfc2enBwCZFwMeHiIto7zccbnNtJjCovpUg9Ww75llcJPDjkjCSqh8WDgq8A59zOIVSkT2VHSKhrilESEsOG5E8PggeHHEos7uMtB+d2Hr2l0KhvjDGyrDSvkm96cMihRFDw4ODczqM6Eu4x5lDfnF+zo8GDQ04lgoJ3Kzm384hGwr2OOXhwcF0Sdy54y8G5nUc0EqauKUZHp3WVeXBwn9LVreQtB+d2GtWRIXR0WtcdSgB1jfmVVwk8OOTUHpVlAOwe/OucG/y6p9Bo6+hkc0t7XuVVAg8OOXXYxEoe+/4s9hodyXVVnHNZUt21XGj8jqWNzfEWhLccXBdJ7BmtyHU1nHNZ1L3lkI95lcCDg3POZVVXcAhuZ61vzL/Z0eDBwTnnsqqstJiKcHHX7az1zR4cnHPO8emJcPmYVwlSDA6SZktaIWmVpLm9bN9D0mOSlkh6UtLYpG3XSloW/FyQVH6spFeC8jslFQflZwbHeU3SYklHpuNEnXMuX1RFwtQELYe6oFtpZFlJLqvUww6Dg6QQcDNwMvElPy+S1H3pz+uAu8xsKnAVcE3w2lOBg4BpwCHA5ZKGSSoC7gQuNLP9gfeBi4NjPQYcYGbTgK8C/zmwU3TOufzSveUwoqyE4lB+deSkUpsZwCozW21mMeAe4Mxu+0wBHg8eP5G0fQqw0MzazawJWALMBiqBmJmtDPZ7FDgXwMwazSwxdbAc2DqN0DnnBoF4Co34raz1zTFGleVXlxKkFhzGAB8mPV8TlCV7HTgneHw2EJFUGZTPllQmqQo4BhgH1ALFkqYHrzkvKAdA0tmS3gL+Rrz10IOkS4Jup8U1NTUpnIZzzuWHaCRMU6yDptZ26hvzL3UGpG9A+nJglqRXgVnAWqDDzBYA84HngLuBRUG5ARcCN0h6EWgAOhIHM7P7zGwf4Czgp729oZndbmbTzWx6NBpN02k451zmda0I19Cal3mVILXgsJakq3pgbFDWxczWmdk5ZnYg8KOgbFPw79VmNs3MTgAErAzKF5nZTDObASxMlHc77kJgYtDqcM65QSF5rkNdU4zKisIMDi8BkyVNkFRK/Ir//uQdJFUFg8wAPwDuCMpDQfcSkqYCU4EFwfPq4N8wcAVwW/B8koIVLyQdBISBuoGcpHPO5ZNECo2PN7ewsTm+0E++2eGadGbWLulS4BEgBNxhZm9IugpYbGb3A0cD10gy4q2AbwYvLwGeDv7Wbwa+aGbtwbY5kk4jHqBuNbPEgPa5wJcktQFbgAuSBqidc67gJVoO72xooqPT8rJbKaUFS81sPvGxg+SyK5MezwPm9fK6FuJ3LPV2zDnAnF7KrwWuTaVezjlXiEaVlRIqEis+3gxQsN1Kzjnn0qioSFRVlPLWRw1A/iXdAw8OzjmXE9WRIbxX2wRQsPMcnHPOpVk0EiaxUugo71ZyzjkHW+9Ygvxb6Ac8ODjnXE4k7lgqKw0xpCSU49r05MHBOedyINFyyMc5DuDBwTnnciLRcsjH21jBg4NzzuVEIjjk4wQ48ODgnHM5kUi+l4+3sYIHB+ecy4l8bzmklD7DOedceg0pCfHDU/Zh5uT8XHLAg4NzzuXIJUftmesqbJN3KznnnOvBg4NzzrkePDg455zrwYODc865Hjw4OOec68GDg3POuR48ODjnnOvBg4NzzrkeZGa5rsOASaoB3t/G5iqgNovV6Suv38B4/QbG6zcwhV6/Pcys1ynagyI4bI+kxWY2Pdf12Bav38B4/QbG6zcwg7l+3q3knHOuBw8OzjnnetgZgsPtua7ADnj9BsbrNzBev4EZtPUb9GMOzjnn+m5naDk455zrIw8Ozjnnehi0wUHSLyW9JWmJpPskjUja9gNJqyStkHRSjup3vqQ3JHVKmp5UPl7SFkmvBT+35VP9gm05//y61edfJa1N+sxOyXWdACTNDj6jVZLm5ro+3Ul6T9LS4DNbnAf1uUPSBknLkspGSXpU0tvBvyPzrH558d2TNE7SE5LeDH5vvxOU9//zM7NB+QOcCBQHj68Frg0eTwFeB8LABOAdIJSD+u0L7A08CUxPKh8PLMuDz29b9cuLz69bXf8VuDzXn1m3OoWCz2YiUBp8ZlNyXa9udXwPqMp1PZLqcxRwUPL3H/gFMDd4PDfxe5xH9cuL7x6wK3BQ8DgCrAx+V/v9+Q3aloOZLTCz9uDp88DY4PGZwD1m1mpm7wKrgBk5qN9yM1uR7fdN1XbqlxefXwGYAawys9VmFgPuIf7ZuW0ws4VAfbfiM4E7g8d3AmdltVJJtlG/vGBm683sleBxA7AcGMMAPr9BGxy6+SrwUPB4DPBh0rY1QVk+mSDpVUlPSZqZ68p0k6+f36VBF+Iduex6SJKvn1MyAxZIelnSJbmuzDaMNrP1weOPgNG5rMw25NV3T9J44EDgBQbw+RWnvWZZJOnvwC69bPqRmf012OdHQDvw+2zWLXjvHdavF+uB3c2sTtLBwF8k7Wdmm/OkfjmxvboCtwI/Jf7H7qfAr4hfELjtO9LM1kqqBh6V9FZwdZyXzMwk5du993n13ZNUAfwZuMzMNkvq2tbXz6+gg4OZHb+97ZK+DJwGHGdBpxuwFhiXtNvYoCzr9dvGa1qB1uDxy5LeAfYC0j5g2J/6kcXPL1mqdZX0W+DBDFcnFTn5nPrCzNYG/26QdB/xrrB8Cw4fS9rVzNZL2hXYkOsKJTOzjxOPc/3dk1RCPDD83szuDYr7/fkN2m4lSbOBfwbOMLPmpE33AxdKCkuaAEwGXsxFHXsjKSopFDyeSLx+q3Nbq0/Ju88v+NInnA0s29a+WfQSMFnSBEmlwIXEP7u8IKlcUiTxmPgNHPnwuXV3P3Bx8PhiIN9atHnx3VO8ifBfwHIzuz5pU/8/v1yPsmdw9H4V8T7f14Kf25K2/Yj4nSQrgJNzVL+zifdDtwIfA48E5ecCbwR1fgU4PZ/qly+fX7e6/g+wFFgS/DLsmus6BfU6hfhdI+8Q76rLeZ2S6jaR+B1Urwfft5zXD7ibeLdqW/Dd+wegEngMeBv4OzAqz+qXF9894EjiXVtLkv7mnTKQz8/TZzjnnOth0HYrOeec6z8PDs4553rw4OCcc64HDw7OOed68ODgnHOuBw8OzjnnevDg4Jxzrof/H5JYvITuxISUAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3ycd3nn/c93RhrJI9mWRj4QYslOqdOumxglNQG6SxMOBUMoKYEtSUsTDt28tgW2+zyNn5DylN0NzQM85Vm6XVJ4sjSQ9EBKAy2BDU3SEBq2CRCTxCaHxnVDHR9Co2OskewZjXTtH3Pf8sSSpZE0cx+k6/166aXR777nnt8vE881v9N1y8xwzjnnamXiroBzzrnk8eDgnHNuFg8OzjnnZvHg4JxzbhYPDs4552bx4OCcc24WDw5u1ZP0RUm/J+k1kp6Ouz5nIuk/S/rTuOvhVoeWuCvgXFKY2XeAn4q7Hs4lgfccnHPOzeLBwa06ki6Q9IikMUl/AbQH5ZdIOlJz3j9Lul7Sk5JGJH1BUvsC1+6W9A1JA8FzviFpS83xb0v6mKS/D17/Hkkbao5fJemQpCFJvxvU4Q1neK1XSXpQ0qikfZIuWe5/G+dCHhzcqiIpB/w18CdAAfhL4B3zPOVXgTcBLwPOBf7vBV4iA3wB2Ar0ASeAz5x2zq8A7wU2ATng2qBuO4A/Cl7zLGA9cPYZ2nE28D+B3wvacS3wFUkbF6ifc3Xx4OBWm1cBrcAfmNmkmd0BPDzP+Z8xs8NmNgzcCFw538XNbMjMvmJmE2Y2Fjzn4tNO+4KZHTCzE8CXgf6g/J3A183sf5lZGfgocKbkZ+8G7jKzu8xs2szuBfYCb5mvfs7Vyyek3WrzUuCovTjj5KF5zj982nkvne/ikvLAp4HdQHdQvFZS1symgr9/XPOUCaCzpm4zr2dmE5KGzvBSW4F/K+kXa8pagfvnq59z9fKeg1ttngPOlqSasr55zu897bxjC1z/t6mueHqlma0Dfj4o15mf8qK61c5PrAF6znDuYeBPzKyr5qfDzD5Rx+s4tyAPDm61eQioAP9BUquky4GL5jn/A5K2SCoAHwH+YoHrr6U6zzAaPOc/LaJudwC/KOnngrmR/8yZg8qfBue+SVJWUnswob7lDOc7tygeHNyqEozlXw68BxgG3gV8dZ6n/DlwD/AM8E9UJ4Dn8wfAGmAQ+C7wN4uo2xPAh4DbqfYiisDzQGmOcw8DlwG/AwxQ7Unswf9NuwaR3+zHublJ+mfg183sb2N6/U5gFNhuZj+Kow5u9fJvGc4liKRflJSX1AF8Cvgh8M/x1sqtRh4cnFskSb8jqTjHzzcbcPnLqE56HwO2A1eYd+9dDHxYyTnn3Czec3DOOTfLitgEt2HDBtu2bVvc1XDOuVT5wQ9+MGhmc6ZcWRHBYdu2bezduzfuajjnXKpIOmN2AB9Wcs45N4sHB+ecc7N4cHDOOTeLBwfnnHOzeHBwzjk3S13BQdItkp6X9PgZjkvSH0o6KGm/pAtrjl0t6R+Dn6tryn9W0g+D5/xhmEJZUkHSvcH590rqnus1nXPONU+9PYcvUr15yZm8mepW/+3ANcBnofpBTzVl8SuppkX+TzUf9p8F/l3N88Lrfxi4z8y2A/cFfzvnnItQXfsczOwBSdvmOeUy4LYgB8x3JXVJOgu4BLg3uMUiku4Fdkv6NrDOzL4blN8G/BLwzeBalwTXvRX4NnDdYhpVr6d/PMb/3L/QvVucO7NcS4arfm4b69pb467KopkZf/q9Zxk4fnLJ12jJZviVV/axobOtgTWLhpnxl3uPcGRkIu6qLMuubQV+/tzG3zq8UZvgzubFt1M8EpTNV35kjnKAzWb2XPD4x8DmuV5Q0jVUeyn09c13I68zO/h8kf9+/8ElPde5MC3Z+jWt/Nqrt8Val6U4MnKC3/3r6kix6rlP3WnC9ne2tfC+f3NOA2sWjU/d8zQ33f9PwNLanxT//uKXJTo4NIWZmaQ5MwOa2c3AzQC7du1aUvbAS3eexaU7L11GDd1qZma84sa/5dHDo/zaq+OuzeINFqv3EPrjq3fx+n8153eweZkZP/mRbzI0PuteRIn3xb//ETfd/09ceVEf/8/bz0Npjg5N0qjVSkd58b12twRl85VvmaMc4F+CISmC3883qI7ONZQk+nu7eOzwaNxVWZLh8TIAhY7ckp4vie58buY6afGN/cf4L994kl/YsZmPXfYzHhjOoFHB4U7gqmDV0quAF4KhobuBN0rqDiai3wjcHRw7LulVwSqlq4Cv1VwrXNV0dU25c4nT39vFMwPjvDAxGXdVFm0o+FDv6Vj6fEFPR46hYnqCw4P/NMj/+Rf72LW1m/9+5QW0ZH01/5nUNawk6UtUJ4k3SDpCdQVSK4CZfQ64C3gLcBCYAN4bHBuW9DHg4eBSN4ST08BvUl0FtYbqRHR4o5RPAF+W9H7gEPDLS2+ec83V31tdfLfvyGhTxn2baabn0Lm0ngNUex0jE+kIDk8ce4FrbvsB2zbk+fxVr6C9NRt3lRKt3tVKVy5w3IAPnOHYLcAtc5TvBc6bo3wIeH099XIubjt71yPBY4fTGRxyLRk6ckv/kCx05njqueMNrFVzHB6e4D1feJh17S3c+r6LWJ9P3+qyqHmfyrllWNfeyss2dqZy3mGoWKanI7esMfdCCuYchoolrrrl+0xOTXPb+y/irPVr4q5SKnhwcG6ZwknptN1yd3i8tOTJ6FChI8foxCSVqekG1aqxxksV3vfFh3nuhRP88dWv4Cc3rY27SqnhwcG5Zerv7WJ4vMzh4RNxV2VRhsbL9Cxz81pPMF8xeiJ5E/LlyjS/8WeP8Pix49z0Kxfys1s9E89ieHBwbpn6e7sAePTwSMw1WZxwWGk5wp5H0oaWpqeN676ynwcODPDxy89f0j6O1c6Dg3PL9NMvWUt7ayZ18w7D4+XlDyvlq89P2nLWT/zNP/BXjx5lz5t+il/e1bvwE9wsHhycW6aWbIbzz16fquBwojzFicmp5QeHzuT1HP7HA89w8wPPcPWrt/Kbl7ws7uqklgcH5xqgv7eLJ44dp1xJ5sTs6cKUFw0bVkrIXoe/evQIN971FJeefxYf/UXf/bwcHhyca4D+3m7KlelUrPmH5afOCHUHw0rDCRhW+rsDA+z5y/28+id6+K/vejnZjAeG5fDg4FwD9PdVJ6XTMrQ0kzpjGbujAVqzGda1tzAcc/K9fYdH+Y0//QHbN6/l/7/qZ2lr8d3Py+XBwbkGeOn6djaubUtNcAi/6ReWkVcp1NPZNhNs4vCjwXHe+8WHKXTkuPW9r0jlvTWSyIODcw2Qtgytww3qOUC8+ZWeHzvJVbd8D4Db3ncRm9a1x1KPlciDg3MN0t/bxY8GxxlNyOTsfIbGy7Rmxdq25d/SpRBTZtbpaeP9X9zLULHMF97zCn5iY2fkdVjJPDg41yAX9KZn3mGoWE2d0YjVPHHlVxoslvjh0Rf4j2/YzsuD//aucTw4ONcg5285laE16aob4Bpz3+dCZ3VYKercUgPBnex6u/ORvu5q4cHBuQZZ297K9k3pyNA6NL781Bmhno4ck1PGWKnSkOvVazAYytqwtjFBzr2YBwfnGqi/t4t9KcjQ2ojUGaGZjXARzzsMBT2HDctMHujm5sHBuQbq7+1mZGKSQ0MTcVdlXo0MDt3BdaJezjo4Exwa0w73Yh4cnGug/hRMSpcqUxRLlYYOK0H0+ZUGi9U72XU2YMWVm82Dg3MNdO7mTta0ZhMdHBpx7+haYQ9kJOrgMFZiY2eb509qEg8OzjVQSzbD+VvW82iCg0O4J6GnUauV4hpWGi/7kFITeXBwrsEu6O3iqWPHKVWm4q7KnBq5Oxogn2uhvTUTeX6lwbGST0Y3kQcH5xqsv7eL8tQ0Tx5LZobWMF13oyakodoLiWNC2oND83hwcK7Bkp6h9dSwUuOCQ6EjF+mcw/S0BffA9mGlZvHg4FyDnbV+DZvXJTdD6/B4mWxGDc1e2t0RbQqN0ROTTE2b9xyayIODc02Q5Aytw+NluvM5Mg28GU5PRy7SYaWZDXC+O7ppPDg41wT9vd0cGpqIfHlnPRqZOiNUiLjnMOAb4JrOg4NzTTCzGe5I8noPjdwdHSp05JgoT3FyMpoVWmFepY0+rNQ0dQUHSbslPS3poKQPz3F8q6T7JO2X9G1JW2qOfVLS48HPu2rKXyfpkaD8VkktQfl6SV+XtE/SE5Le24iGOhelnVvWkxE89mwyg0OjJ3ILEe+SHhyr9hx6PDg0zYLBQVIWuAl4M7ADuFLSjtNO+xRwm5ntBG4APh4891LgQqAfeCVwraR1kjLArcAVZnYecAi4OrjWB4AnzezlwCXA/yfJ+44uVTraWjh389pEzjsMFUtNGVaCCINDsUQ2I7rW+C1Bm6WensNFwEEze8bMysDtwGWnnbMD+Fbw+P6a4zuAB8ysYmbjwH5gN9ADlM3sQHDevcA7gscGrFV1T3wnMAxEmwvYuQbo7+1i35FkZWgtV6Y5frLSsHs5hHoi3iU9VKzOmzRyUt29WD3B4WzgcM3fR4KyWvuAy4PHb6f64d4TlO+WlJe0AXgt0AsMAi2SdgXPeWdQDvAZ4F8Bx4AfAr9lZtOnV0rSNZL2Sto7MDBQRzOci1Z/bxejE5P8c4IytIb3em5UXqVQ1PmVfANc8zVqQvpa4GJJjwIXA0eBKTO7B7gLeBD4EvBQUG7AFcCnJX0fGAPCmaw3AY8BL6U6HPUZSetOf0Ezu9nMdpnZro0bNzaoGc41zqnNcCMx1+SUZmyAg+jzKw0WS74BrsnqCQ5HOfWtHmBLUDbDzI6Z2eVmdgHwkaBsNPh9o5n1m9kvAAIOBOUPmdlrzOwi4IGwHHgv8FWrOgj8CPjpJbfQuZhs37SWjlw2UZPSMxlZGxwc1rW3ks0osvxKg8Wyr1RqsnqCw8PAdknnBBPDVwB31p4gaUMwyQxwPXBLUJ4NhpeQtBPYCdwT/L0p+N0GXAd8Lnj+s8Drg2ObgZ8CnllqA52LSzYjzt+yPlGT0mFepUb3HDIZ0Z2PZq+DmTFQLPkGuCZbMDiYWQX4IHA38BTwZTN7QtINkt4WnHYJ8LSkA8Bm4MagvBX4jqQngZuBdwfXA9gj6Smqk9RfN7NwQvtjwM9J+iFwH3CdmQ0ut6HOxaG/t5snnzse2fr/hTSr5wDVgBNFcCiWKpQr074BrsnquoWSmd1Fde6gtuyjNY/vAO6Y43knqa5Ymuuae4A9c5QfA95YT72cS7r+3i4mp4wnnzvOhX3dcVeH4fEyEnTlG//B2t3RGklwCDfA+YR0c/kOaeea6IJwUjoh8w5D42UK+RzZJiwBjSptd3jvaN8A11weHJxros3r2jlrfXti5h2Gi41PnRGKKr9SuDvah5Way4ODc02WpAytzcirFCp05HjhxCSVqVnbkhpqcNzzKkXBg4NzTdbf28WzwxMzaabjNDjevP0BhY4cZtV7LTTT4FgJqTmT6u4UDw7ONVmYoXVfAjK0NrvnEL5GMw0WS3Tnc7Rk/eOrmfy/rnNNdv6W9WQzin1SujI1zejEZMPzKoVm8isVmx8cGr1Pw83mwcG5JsvnqhlaH4153mFkojrc06wP1jBfU5i/qVkGi2VfxhoBDw7ORaC/t4t9h0eZno4vQ2szN8ABFPLR5Fca8t3RkfDg4FwELujt4vjJCj8aGo+tDjOpM5o0Id0dzjk0fVip7MtYI+DBwbkI9CdgM1zYc+hp0pxDazbDuvaWpibfOzk5RbFU8WGlCHhwcC4CL9vYSWdbS6z7HZo9rATVXcvDE81byjrgG+Ai48HBuQhkM2JnzBlaw1VE3fnm3VqzO9/a1J5DmDrDew7N58HBuYj093bxVIwZWofGS3TlW5u6P6DQ0dbUpaxDnnQvMh4cnItIf28XlWnjiWMvxPL6zdwAF2p22u6ZnoOvVmo6Dw7ORSSclH40pknpoWK56ZvHCp05RibKVO8E3HgzGVl9E1zTeXBwLiKb1rZzdtea2OYdoug5FPI5JqeMsVJl4ZOXYLBYZm1bC+2t2aZc353iwcG5CMWZobUaHJo7HFNo8l4Hvz1odDw4OBeh/t4ujoycmBkeicr0tDEy0fzNY2EKjWbtkh4qlnwZa0Q8ODgXobg2w42emGTamp/mOkyhMdKk4OB5laLjwcG5CJ330iBDa8RDS+Heg6YHhyan7R4sNu9+FO7FPDg4F6E1uSw//ZK1kQeHcH9As1JnhHqaOKw0GaQc955DNDw4OBexODK0DkWQOgOq6cnbWzNN2SUd9kY8OETDg4NzEevv7WKsVOGZwWJkrxkGhyiGZAr5HMPjjc+vdCqvkgeHKHhwcC5iF8SwGW54Jq9SBMGhM9eUnkO4wmvjWp9ziIIHB+ci9hMbOlnbHm2G1uHxEmvbW8i1NP+ffKGjrSkT0oMRzZu4Kg8OzkUskxEv3xLtZrih8eiWgPZ05JoyIe15laJVV3CQtFvS05IOSvrwHMe3SrpP0n5J35a0pebYJyU9Hvy8q6b8dZIeCcpvldRSc+wSSY9JekLS3y23kc4lTX9vF//w4zFOlKPJ0BpF6oxQdz7XlH0OQ8US7a0ZOnKeOiMKCwYHSVngJuDNwA7gSkk7TjvtU8BtZrYTuAH4ePDcS4ELgX7glcC1ktZJygC3AleY2XnAIeDq4DldwB8BbzOznwH+7bJb6VzC9Pd2MTVtPB5RhtYog0NPZ47x8lTDU5OHG+AkNfS6bm719BwuAg6a2TNmVgZuBy477ZwdwLeCx/fXHN8BPGBmFTMbB/YDu4EeoGxmB4Lz7gXeETz+FeCrZvYsgJk9v/hmOZdsUe+UHhpvfkbWULM2wg0WS75SKUL1BIezgcM1fx8JymrtAy4PHr8dWCupJyjfLSkvaQPwWqAXGARaJO0KnvPOoBzgXKA7GJ76gaSr5qqUpGsk7ZW0d2BgoI5mOJccGzrb2NIdTYbW6WljJMKeQ7OCw8CY51WKUqMmpK8FLpb0KHAxcBSYMrN7gLuAB4EvAQ8F5QZcAXxa0veBMSDsg7YAPwtcCrwJ+F1J557+gmZ2s5ntMrNdGzdubFAznItOVBlaj5+cpDJtqQ8OnlcpWvUEh6Oc+lYPsCUom2Fmx8zscjO7APhIUDYa/L7RzPrN7BcAAQeC8ofM7DVmdhHwQFhOtWdyt5mNm9lgcOzlS26hcwnV39vF0dETPD92sqmvE+UGOGhOcJieNobHfVgpSvUEh4eB7ZLOkZSj+o3/ztoTJG0IJpkBrgduCcqzwfASknYCO4F7gr83Bb/bgOuAzwXP/xrwbyS1SMpTnch+aulNdC6Zws1w+w43d1J6eCZ1RnRLWaGx+ZVGJspMGz6sFKGWhU4ws4qkDwJ3A1ngFjN7QtINwF4zuxO4BPi4JKP6Tf8DwdNbge8EqwuOA+82s/AWUXskvZVqgPqsmX0reL2nJP0N1cnraeDzZvZ4Y5rrXHJs37wWgGcGisDmpr3OqaR70XywrmtvJZtRQ3dJhxvgfI9DdBYMDgBmdhfVuYPaso/WPL4DuGOO552kumJprmvuAfac4djvA79fT92cS6t17a1051t5dniiqa8zHPGwUiYjuvOtDc2vdOre0R4couI7pJ2LUV8hH0FwiOZeDrUKHY3Nr+R5laLnwcG5GPVGEByGxst0trXQ1hLdzuJqcGjcnMPMsJJPSEfGg4NzMdrak+foyAkqU9NNe40od0eHejraGjohPVgs0ZoV69e0Nuyabn4eHJyLUV8hT2XaeO6F5i1njSM4dHe0NjS/0uBYiZ4OT50RJQ8OzsWot5AH4HATh5YGi9GlzggVOtoYPTHJVIPuduf3jo6eBwfnYrS1pwOAQ00MDsPjpRiGlXKYVfcnNEKUKcddlQcH52L0knXttGbVtElpM6sOK0X8rbvRu6QHx3x3dNQ8ODgXo2xGbOlu3oqlsVKFySljQ8T7AxoZHMysmlfJl7FGyoODczHrLeR5dqg5wSG8d3TUw0qNDA7HT1YoT01HHuBWOw8OzsVsaxP3OoTLSaMeVmpkfqVTtwf1nkOUPDg4F7O+Qp4XTkzywkTj0k2EZlJnRL6UNeg5FJcfHIZ8A1wsPDg4F7OZ5awjje89xJE6A6A1m2Fte0tDVivN9Bw8OETKg4NzMesLgsOhJsw7zNzLIYbx+p6OXGOHlTw4RMqDg3Mx6+upBodmzDsMFcusac2yJhddXqVQo5LvDY6VkKA776kzouTBwbmYdba10NORa0pwiCN1RqjQ0TYzX7AcA8UyhXyOlqx/XEXJ/2s7lwDV7KzjDb/u0Hg5trQThY7Whsw5DBV9A1wcPDg4lwBbe5qznHV4vBT5SqVQoaON4fEyZsvLrzRYLPky1hh4cHAuAfoKeY6NnmSywam7h4vlyO4dfbqejhyTU8ZYqbLwyfMYLHpepTh4cHAuAXoLeaamjedGG5e628xiHVZq1F6HwWLJbw8aAw8OziXAzHLWBs47TJSnKFWmY5uQDoezhpcx7zBRrjBRnvJhpRh4cHAuAbY2YTlruDs6vtVKy+85+O7o+HhwcC4BNq9tJ5fNNDQ4DMWUOiPUiOR7A8EGuI0eHCLnwcG5BMhkxJbCmobeEW6oGE/qjFChAcn3Bsd8d3RcPDg4lxB9hXxDU2jEmToDIJ/L0taSWdZeh8FgWMlvERo9Dw7OJcTW4L4Oy90XEJrJyBrTB6ukan6lZc05VHsOHhyi58HBuYToLeQZK1V44URjUncPj5dpa8mQjyGvUqjQubz8SoPFEuvaW2hria8Nq5UHB+cSotHZWYeKZXo6ckhqyPWWojufW9aEdPX2oD7fEIe6goOk3ZKelnRQ0ofnOL5V0n2S9kv6tqQtNcc+Kenx4OddNeWvk/RIUH6rpJbTrvkKSRVJ71xOA51Li609HUDjlrMOj5civwPc6Xo6csva5zBQLPntQWOyYHCQlAVuAt4M7ACulLTjtNM+BdxmZjuBG4CPB8+9FLgQ6AdeCVwraZ2kDHArcIWZnQccAq4+7TU/CdyzvOY5lx69hTVAI4NDfKkzQoWOtmXtc/C8SvGpp+dwEXDQzJ4xszJwO3DZaefsAL4VPL6/5vgO4AEzq5jZOLAf2A30AGUzOxCcdy/wjprrfQj4CvD8ItvjXGrlcy1s6Gxr2HLWofFybHscQj2dOcbLU5ycnFrS84c8r1Js6gkOZwOHa/4+EpTV2gdcHjx+O7BWUk9QvltSXtIG4LVALzAItEjaFTznnUE5ks4OrvHZ+Sol6RpJeyXtHRgYqKMZziVfX2FNw+Yc4ryXQ6g7v/SNcOXKNC+cmPTgEJNGTUhfC1ws6VHgYuAoMGVm9wB3AQ8CXwIeCsoNuAL4tKTvA2NA+NXiD4DrzGze9JRmdrOZ7TKzXRs3bmxQM5yL19aejoYMK50oTzFRnoo9OCxnl/TQuG+Ai1PLwqdwlOBbfWBLUDbDzI4R9BwkdQLvMLPR4NiNwI3BsT8HDgTlDwGvCcrfCJwbXG4XcHuwwmID8BZJFTP76yW0z7lU6S3k+dpjRylXpsm1LP27W/jBmoRhJVhacBgc8w1wcarn/76Hge2SzpGUo/qN/87aEyRtCCaZAa4HbgnKs8HwEpJ2AjsJJpklbQp+twHXAZ8DMLNzzGybmW0D7gB+0wODWy36CnmmDY6OnljWdU5tgIt7QnoZwaHoPYc4LRgczKwCfBC4G3gK+LKZPSHpBklvC067BHha0gFgM0FPAWgFviPpSeBm4N3B9QD2SHqK6iT1180snNB2btUK9zosd2hpKOaMrKFCfun5lQY96V6s6hlWwszuojp3UFv20ZrHd1D9ln/6805SXbE01zX3AHsWeN331FM/51aKRqXuDpePxj2stH5NK9mMGFlScAjSdftS1lj4DmnnEmRjZxttLZllL2eduZdDzOP1mYzozrcuueeQz2XJ5+r6DusazIODcwmSyYjeQp5DQ8u7I9zQeJnWrFjbFv8Ha6FjafmVBosln4yOkQcH5xJmayHPs8PLnZAuUYg5r1KoO59jZHzxyQR9A1y8PDg4lzC9hTyHh5eXujsJqTNCPZ25maW1izFYLHlwiJEHB+cSpq+Qp1iqMDKx9NTdSUidEaoOKy1tzsGDQ3w8ODiXMKdSdy993mGoWE7MeH2ho43RE5NMTdffE5qaNobHy2xMSBtWIw8OziVMI5azJiGvUqiQb8UMRheRunt4vMy0xb+JbzXz4OBcwmzprgaHpS5nLVWmKJYqyRlWCj7gFzO05HmV4ufBwbmEWZPLsmlt25Kzs87scUjKhHTH4ndJh3mVNviwUmw8ODiXQH2F/JKHlYaKyUidEVpKfqWZvEp+i9DYeHBwLoH6evJLHlY6lXRvBQQHH1aKjQcH5xKor5DnueMnKVUWfwe14YQk3Qst5YY/A8USuWyGde3x7/BerTw4OJdAfYU8ZnBkZPE7pcOx/aRMSOdaMqxtb1nchHSwFDcJO7xXKw8OziXQcpazDo+XyGbEuvbWRldryXo6coubkPYNcLHz4OBcAvUWlr6cdahY3eOQySTnW3d3R25RaburwSEZPZ/VyoODcwm0sbON9tbMkpazJil1RmjRPYexsm+Ai5kHB+cSSNKSl7MmaXd0aDFpu82MoXEfVoqbBwfnEqqv0LGkYaUkBofuIPlePZlmj5+oMDllPqwUMw8OziVU2HNYbOruoWIpkcNKk1NGsVRZ8NyB8N7RvgEuVh4cnEuovsIaJspTM/dSrsfk1DTHT1YSkzojFNannuWsvgEuGTw4OJdQfUtYzjqSkHtHn24x+ZXC4JCUHd6rlQcH5xKqr9ABLG45a9I2wIW6wxQadfSCwtxQ3nOIlwcH5xJqS/cagEUtZx1OaHAI6zNcxz0dBoslMjqVdsPFw4ODcwnV3prlJevaFzWslNQhmcUk3xsslih0tJFN0Ca+1ciDg3MJttjsrEm7l0Mon8vS1pKpKzgMjJV9GWsCeHBwLsEWuxFueLxMRtC1Jjl5laC6qa/QkZuZT5iP51VKhrqCg6Tdkp6WdFDSh+c4vlXSfZL2S/q2pC01xz4p6fHg51015a+T9EhQfi0ZV3QAABAbSURBVKuklqD8V4Pr/FDSg5Je3oiGOpdGfYU8Pz5+kpOT9aXuHhov051PVl6lUKEjx0gdcw7V3dHec4jbgsFBUha4CXgzsAO4UtKO0077FHCbme0EbgA+Hjz3UuBCoB94JXCtpHWSMsCtwBVmdh5wCLg6uNaPgIvN7HzgY8DNy2uic+nVFyTgOzJSX+9huJi83dGhQp35lQbHyt5zSIB6eg4XAQfN7BkzKwO3A5edds4O4FvB4/trju8AHjCzipmNA/uB3UAPUDazA8F59wLvADCzB81sJCj/LjDTC3FutVnsXockps4I9dSRX2m8VOHE5JTfHjQB6gkOZwOHa/4+EpTV2gdcHjx+O7BWUk9QvltSXtIG4LVALzAItEjaFTznnUH56d4PfHOuSkm6RtJeSXsHBgbqaIZz6RP2HJ6tcznr0HgpcSuVQt0duQX3Ofju6ORo1IT0tcDFkh4FLgaOAlNmdg9wF/Ag8CXgoaDcgCuAT0v6PjAGvGhQVdJrqQaH6+Z6QTO72cx2mdmujRs3NqgZziVLT0eOfC7LoRXScxgvT807f5LUpbirUT3B4Sgv/la/JSibYWbHzOxyM7sA+EhQNhr8vtHM+s3sFwABB4Lyh8zsNWZ2EfBAWA4gaSfweeAyMxtacuucS7kwdXc9y1krU9OMnpikJ2HLWEPh8tr5JqXDPFIbvecQu3qCw8PAdknnSMpR/cZ/Z+0JkjYEk8wA1wO3BOXZYHgp/MDfCdwT/L0p+N1GtXfwueDvPuCrwK/VzEk4t2rVu5x1ZGISs+R+6w57NPMtZ/VhpeRYMDiYWQX4IHA38BTwZTN7QtINkt4WnHYJ8LSkA8Bm4MagvBX4jqQnqa46endwPYA9kp6iOkn9dTMLJ7Q/SnXC+o8kPSZp77Jb6VyK1Zu6+9QGuGQHh/k2wg2OBek/EhrgVpOWek4ys7uozh3Uln205vEdwB1zPO8k1RVLc11zD7BnjvJfB369nno5txr09eQ5OTnNwFiJTevaz3jeULASKOnBYf5hpRJd+VZas74/N27+DjiXcDMrlhYYWjqVdC+ZQzI9dQwrDY0n70ZFq5UHB+cSbrHBIak9h/VrWslo4WEln29IBg8OziXc2d1rkBZO3R1+I+/OJyuvUiiTEd35+XdJDxZLvgEuITw4OJdwbS1ZzlrXvuBy1uHxMl35VloSPF5f6MjN3K1uLgPFki9jTYjk/l/knJvR17Pwctbh8XLix+sLHbkzDiudnJxi7GQl8W1YLTw4OJcCfYX8grukq5O5yf7W3dOZm1lVdbowaPiwUjJ4cHAuBfoKeQbGSpwonzn1xFCCM7KGuvNn7jn4Brhk8eDgXAr0BiuWDs+Tunt4vEwh4ZvHejpyjJ6YZGp69oa+U8Eh2W1YLTw4OJcCW3s6gDNnZ52eNkYm0jHnYAajc2yEC3dHe88hGTw4OJcC4V6HM807jJ6YZNqSu8chVAg++OcaWhrwYaVE8eDgXAp051vpbGs543LW4YSnzggV8sEu6TmCw1CxTEcuy5pcNupquTl4cHAuBcLU3WdazhpugEv6aqWZ/EpzBAffAJcsHhycS4m+Qp5DQ+NzHkt66oxQmG11rp7DYLHkQ0oJ4sHBuZTo68lzeOQE03Os9Ak/bJO+0qc7f+a03dXgkOz6ryYeHJxLid5CnnJlmufHZm8iCz9suxPec8i1ZFjb1jJncBgqlunxnkNieHBwLiW2zpOddahYYl17Syrug1DonL0RrjI1zfCEZ2RNkuT/n+ScA2qWs84x7zA0np5v3XPlVxqeKGMGG31YKTE8ODiXEi/tWkNGzLmcdXg8+akzQoU50nb7Brjk8eDgXErkWjKctX7NnMNKqQoOHbmZfRmhmdQZvpQ1MTw4OJciW8+QunsoBem6Q4XOHCPjk5idWnUVZmpNSxtWAw8OzqXIXBvhzIyRFPUcejpylKemKZYqM2Uzw0rec0gMDw7OpUhvIc9gscx4zQfr8RMVKtOWmgnpufY6DBZLM8tcXTJ4cHAuRbb2zE7dnbYhmbl2SYe3B5UUV7XcaTw4OJcip5azngoOaUmdESoE+Z9GXtRzKM8EDZcMHhycS5EwONQuZx0spis4hD2c2p7DkOdVShwPDs6lyPo1raxtb3nRpHTYc0jLN+8wxcfpcw6eVylZPDg4lyKSZi1nTcu9HEIduSy5lsxMcJieNoaKnjojaeoKDpJ2S3pa0kFJH57j+FZJ90naL+nbkrbUHPukpMeDn3fVlL9O0iNB+a2SWoJySfrD4LX2S7qwEQ11bqXoK+RfdLvQofEynW0ttLWk4yY5kuipSaHxwolJKtPmwSFhFgwOkrLATcCbgR3AlZJ2nHbap4DbzGwncAPw8eC5lwIXAv3AK4FrJa2TlAFuBa4ws/OAQ8DVwbXeDGwPfq4BPrusFjq3wvQW8hwZOcFUkLo7TbujQ7X5lcLd0WkZFlst6uk5XAQcNLNnzKwM3A5cdto5O4BvBY/vrzm+A3jAzCpmNg7sB3YDPUDZzA4E590LvCN4fBnVQGNm9l2gS9JZS2ibcytSXyFPeWqafzl+EkhvcBiaCQ7V3xu955Ao9QSHs4HDNX8fCcpq7QMuDx6/HVgrqSco3y0pL2kD8FqgFxgEWiTtCp7zzqC83tdD0jWS9kraOzAwUEcznFsZthY6gFPLWavj9ekLDuFciedVSqZGTUhfC1ws6VHgYuAoMGVm9wB3AQ8CXwIeCsoNuAL4tKTvA2PA1GJe0MxuNrNdZrZr48aNDWqGc8l3+nLWtPYcRsYngZrg4D2HRKlnr/pRTn2rB9gSlM0ws2MEPQdJncA7zGw0OHYjcGNw7M+BA0H5Q8BrgvI3AufW+3rOrWZndbWTzYhnhycwsyA4pOuDtacjR7FUoVSZYrBYIpsRXWta466Wq1FPz+FhYLukcyTlqH7jv7P2BEkbgklmgOuBW4LybDC8hKSdwE7gnuDvTcHvNuA64HPB8+8ErgpWLb0KeMHMnltGG51bUVqzGc7uqqbuHitVKE9NpyZ1Rqh2r8NQsdrzyWQ8dUaSLNhzMLOKpA8CdwNZ4BYze0LSDcBeM7sTuAT4uCQDHgA+EDy9FfhOkC/lOPBuMwszhu2R9FaqAeqzZhZOaN8FvAU4CEwA711+M51bWfoKeQ4NTzCcst3RoZld0sVysAEuXT2f1aCuFIhmdhfVD+3aso/WPL4DuGOO552kumJprmvuAfbMUW6cCi7OuTn0FvLc/cSPZ1b8FFI3IR3kV5ooM5DCCfXVwHdIO5dCfYU8w+PlmUnptA0rFWqGlQbHSr6MNYE8ODiXQmHq7scOjwLpG1YqnD6s5MtYE8eDg3MpFC5nfTQIDj0pW63UtaaVjODZ4QlKlfRNqK8GHhycS6HeIDg8eewF8rksa3LpyKsUymREdz7HPz4/BvgehyTy4OBcCq1f08r6Na1MTlnqhpRChY4cB/6lCPju6CTy4OBcSoXzDmkdkunuyDEwFu6OTmcbVjIPDs6lVDi0lNaeQ21Q89VKyePBwbmU6psJDun8YK0Nat0pDXArmQcH51JqaxAc0nofhLDn0J1vpTXrH0VJ4++IcynVl/JhpbC34CuVksmDg3Mp9ZObOmnNim09HXFXZUkKHhwSra7cSs655Nm0rp2/v+51bEzpMtBw415ah8VWOg8OzqXYpnXtcVdhybznkGw+rOSci0UYHNLa81npPDg452KxeV0b/8cbzuWtO8+KuypuDj6s5JyLhSR+6w3b466GOwPvOTjnnJvFg4NzzrlZPDg455ybxYODc865WTw4OOecm8WDg3POuVk8ODjnnJvFg4NzzrlZZGZx12HZJA0Ah85weAMwGGF1orIS27US2wQrs13epvSYr11bzWzjXAdWRHCYj6S9ZrYr7no02kps10psE6zMdnmb0mOp7fJhJeecc7N4cHDOOTfLaggON8ddgSZZie1aiW2Cldkub1N6LKldK37OwTnn3OKthp6Dc865RfLg4JxzbpYVHRwkfUjSP0h6QtL/W1N+vaSDkp6W9KY467gUkn5bkknaEPwtSX8YtGm/pAvjruNiSPr94H3aL+mvJHXVHEvteyVpd1Dvg5I+HHd9lkJSr6T7JT0Z/Dv6raC8IOleSf8Y/O6Ou65LISkr6VFJ3wj+PkfS94L37C8k5eKu42JI6pJ0R/Dv6SlJr17qe7Vig4Ok1wKXAS83s58BPhWU7wCuAH4G2A38kaRsbBVdJEm9wBuBZ2uK3wxsD36uAT4bQ9WW417gPDPbCRwArod0v1dBPW+i+t7sAK4M2pM2FeC3zWwH8CrgA0E7PgzcZ2bbgfuCv9Pot4Cnav7+JPBpM/tJYAR4fyy1Wrr/BvyNmf008HKqbVvSe7VigwPwG8AnzKwEYGbPB+WXAbebWcnMfgQcBC6KqY5L8Wng/wJqVxJcBtxmVd8FuiSl5sa8ZnaPmVWCP78LbAkep/m9ugg4aGbPmFkZuJ1qe1LFzJ4zs0eCx2NUP2zOptqWW4PTbgV+KZ4aLp2kLcClwOeDvwW8DrgjOCVV7ZK0Hvh54I8BzKxsZqMs8b1aycHhXOA1QRfx7yS9Iig/Gzhcc96RoCzxJF0GHDWzfacdSm2b5vA+4JvB4zS3K811n5OkbcAFwPeAzWb2XHDox8DmmKq1HH9A9YvWdPB3DzBa80Ulbe/ZOcAA8IVgqOzzkjpY4nvV0qRKRkLS3wIvmePQR6i2rUC1K/wK4MuSfiLC6i3JAm36HapDSqkzX7vM7GvBOR+hOozxZ1HWzS1MUifwFeA/mtnx6pfsKjMzSalaEy/prcDzZvYDSZfEXZ8GaQEuBD5kZt+T9N84bQhpMe9VqoODmb3hTMck/QbwVatu5Pi+pGmqCaiOAr01p24JyhLhTG2SdD7Vbwb7gn+YW4BHJF1EwtsE879XAJLeA7wVeL2d2nyT+HbNI811fxFJrVQDw5+Z2VeD4n+RdJaZPRcMYT5/5isk0r8G3ibpLUA7sI7qeH2XpJag95C29+wIcMTMvhf8fQfV4LCk92olDyv9NfBaAEnnAjmqmQnvBK6Q1CbpHKqTuN+PrZZ1MrMfmtkmM9tmZtuo/o9woZn9mGqbrgpWLb0KeKGmG5l4knZT7d6/zcwmag6l8r0KPAxsD1a/5KhOrN8Zc50WLRiH/2PgKTP7rzWH7gSuDh5fDXwt6roth5ldb2Zbgn9LVwDfMrNfBe4H3hmclqp2BZ8FhyX9VFD0euBJlvhepbrnsIBbgFskPQ6UgauDb6RPSPoy1f9oFeADZjYVYz0b4S7gLVQnbCeA98ZbnUX7DNAG3Bv0ir5rZv/ezFL7XplZRdIHgbuBLHCLmT0Rc7WW4l8Dvwb8UNJjQdnvAJ+gOlT7fqrp8n85pvo12nXA7ZJ+D3iUYHI3RT4E/FnwheQZqp8FGZbwXnn6DOecc7Os5GEl55xzS+TBwTnn3CweHJxzzs3iwcE559wsHhycc87N4sHBOefcLB4cnHPOzfK/AalIQGNTRJkdAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3ycZZnw8d81k0zaTHrIJOn5lEJBslCgREAFCirYgi5yWEFFcFcX3RV1VRB4XdEXX2RxWXVZEeyurOAqyOKB4hYpZ3BBoRxaWgptaWlJekpzaHNqppO53j/mnnSaTJJnZp6ZNJPr+/nkk8n9HOa5Cc019+m6RVUxxhhjUgVG+gGMMcYcfiw4GGOMGcCCgzHGmAEsOBhjjBnAgoMxxpgBLDgYY4wZwIKDGVNE5Gci8v8GOXa6iLyZ8vPRIvKqiLSLyJcyfJ+3ReSDuT5vpkRknoioiJQU+r1NcbHgYIyjqs+q6tEpRV8HnlTVCap621CBZaSMVBAyxc+CgzHAIJ+05wLrCv0sxhwOLDiYoiYiJ4rIy65r6FfAOFd+pog0iMi1IrIT+M9kmTv+BHAW8CMR6RCRK4FPAl93Pz+UwTMEROQ6EXlLRJpF5H4RibhjyW6gK0Rkm4jsEZFvpFw7XkTuFpFWEVkvIl9PecafA3OAh9wzfT3lbT+Z7n7GeGXBwRQtEQkBvwN+DkSA/wYuSjllmiufC1yZeq2qvh94FrhKVStUdRnwC+B77uePZPAoXwQ+CiwGZgCtwO39zjkNOBr4AHCDiBzjyr8FzAPmA2cDl6U846eAbcBH3DN9z8P9jPHEgoMpZqcCpcAPVfWAqj4AvJhyPA58S1V7VLU7j8/xeeAbqtqgqj3At4GL+3Vl/V9V7VbV1cBq4HhX/jHgu6raqqoNwG0e33Ow+xnjic1oMMVsBtCoh2aX3JryuklV9xfgOeYCvxWReEpZLzA15eedKa+7gAr3egbwTsqx1NdDGex+xnhiLQdTzHYAM0VEUsrmpLzONCVxtimM3wGWqurklK9xqtro4dodwKyUn2f79EzGDMmCgylmzwMx4EsiUioiFwIn53C/XST6/jN1J3CTiMwFEJEaETnf47X3A9eLSKWIzASu8umZjBmSBQdTtFQ1ClwIfBpoAS4BfpPDLX8K1IlIm4j8LoPr/hVYDqwUkXbgT8ApHq+9EWgAtgCPAQ8APSnHbwb+0T3T1Rk8kzFDEtvsx5jRQ0T+DrhUVReP9LOY4mYtB2MOYyIyXUTe59ZKHA18DfjtSD+XKX42W8mYLIjIHOD1QQ7Xqeo2n94qBPwEqAXagPuAH/t0b2MGZd1KxhhjBrBuJWOMMQMURbdSdXW1zps3b6QfwxhjRpWXXnppj6rWpDtWFMFh3rx5rFq1aqQfwxhjRhUR2TrYMetWMsYYM4AFB2OMMQNYcDDGGDOABQdjjDEDWHAwxhgzgKfgICJ3ichuEVk7yHERkdtEZJOIrBGRRSnHrhCRje7ripTyk0TkNXfNbcm0yiISEZFH3fmPikhlrpU0xhiTGa8th58BS4Y4vhRY4L6uBO6AxB96EtscnkIiVfK3Uv7Y3wH8bcp1yftfBzyuqguAx93PxhhjCsjTOgdVfUZE5g1xyvnAPW7HrT+JyGQRmQ6cCTyqqi0AIvIosEREngImquqfXPk9JPbYfdjd60x337uBp4BrM6mUV2/ubOd/1mzPx609mzi+lMtOncu40mDB37uzJ8ZvX2nkY/WzCZUUvofxQG+c//zfLXTsj2V9j9JggKXHTefIKYXf6Gzn3v38Ye0OLjhxFpPKSwv+/rlSVf7rz9to2leIzfDyRITZleM5ZvpEFkytoKyk8P+OstXZE+M3LzfQ1N4z/MlDqJ8X4Yyj0q5jy4lfi+Bmcuj2hQ2ubKjyhjTlAFNVdYd7vZNDt1LsIyJX4jaFnzNnTrpThrVpdwf/9uSmrK71iyqs39HOrX+1kEM3LMv3+ypX//dqHl67k9rqMO87srpg75306jttfHfFGwBkW3VV+P5jGzj7mKl8/swjWDQn/72Qm3a3c+fTm3nw1UYO9Covvt3K7Z9cNPyFh5ltLV1883eJnuIC/q/nq9TUcCUB4cgpFRwzfSLHTJ/gvk+kuqJs5B4wje5oL//1p63c+fRbNHdGc/5v//nFRxzWwSEvVFVFJG1mQFVdBiwDqK+vzyp74HkLp3PewvNyeMLc/eDRDfzr4xs5buZEPv2+2oK974+feouH1ya2Gd7Tkdsnl2ztcZ+YVnzpdOpmTMzqHs0dPdz93Nvc/fxWVr6+i5NrI3x+8XzOOnqK78H2pa0t3PHUZh5bv4txpQE+cfIcSoMB/uOPWzh3zQ7OWzjd1/fLt3daugG4929P5T1HVI3w02SnN6683dzJ+h373Fc7z7/VzG9fObgDa82EMo6ZPpE6FzTqpk+ktjpMSbCwreX9B3q594Vt/Pipt2hq7+H0BdV85eyjCvKBJht+BYdGDt3bdpYra+RgF1Gy/ClXPivN+QC7RGS6qu5wXVO7fXrGw9KXP7CAddv38Z3/Wc/R0yYW5B/pE2/s4taVb3LW0TU8+WYTLZ3RvL9nOs3ufSPhUNb3qKoo46vnHM3nFh/BfS++w0+f3czf/GwVR0+dwOcWz+cjx8+gNIc/AvG48sQbu7nz6bdYtbWVyeWlfOkDC7jiPXOpqigj1hvnhbdbuOHBtZw6P0LVYfYpdSiNbV0AzKocP8JPkr1gQDiipoIjair48MIZfeWtnVHW79jH6+5r/Y52fvrWZg70Jj5HlpUEOGpqIlCctqCas941hYqy/HxW7on1cv+qBm5/YhM79+3n1PkRbv/EIk6ujeTl/fzi13+N5cBVInIficHnve6P+yPAd1MGoc8BrlfVFhHZJyKnAn8GLgf+LeVeVwD/5L4/6NMzHpYCAeEHlxzPR2//X77wy5d56IunMXNy/v6xbtnTyZfve5Vjpk3kR59YxLHffmTEgkPyfSvDuffXh8tK+MxptVz+nrk8tHo7P3l6M1+9fzW3PvImnzl9Ppe+ezbhDP7xR2Nxlq/ezk+efouNuzuYOXk83/pIHZe8ezbloYP3KQkG+OeLj+cj//ZHbnhw3ajqXmps7SYgMG3SuJF+FN9VhkO898hq3pvSXRqNxXmrqSMRNLbvY/3OfTzy+k5+teodQiUBzlhQw9Jjp/HBY6b6MoZ0oDfOr19q4N+e2ERjWzf1cyv5/iXH894jCt+Fmw1P/1pE5F4SLYBqEWkgMQOpFEBV7wRWAOcCm4Au4K/dsRYR+Q7worvVjcnBaeDvScyCGk9iIPphV/5PwP0i8hlgK/Cx7Ks3OkwYV8qyy+v56I/+l8/9fBUPfP69eRmg7uiJceU9qygNBlh2+UmEy0qoLA+NaHCYUFbi6yBiaTDAhYtmccGJM3nyzd3c+fRmvvP717nt8Y1c8Z65XP7eeUP2QXf0xLjvhW389I9b2LF3P++aNoEfXnIC5y2cPmgL5OhpE/jyBxfwz4+8Oaq6lxraupk6cVxOLavRJFQS6BuHuNDF8N648tLWVh5eu4M/rN3JY+t3URIQ3ndkNUuPncbZdVMzbg3GeuP87tXt3Pb4Rra1dHH87MncfOFxnL6guqDjirkqis1+6uvrtRiysj72+i4+e88qLjhxJt//2PG+/o8Ujyuf/6+XePyN3fz8Myf3fXr54PefZsGUCu647CTf3surL937Cqsb2nj6mrPy+j4vb2vlzqfe4tH1uwgFA3ysfjZ/e/p85lSV953T1N7Dz57bws+f38q+/TFOnR/hc4uP4Myjajz9HmK9cS684zkaWrt59CtnjIrupY/95HlUlf/+/HtH+lEOC/G4srqhjT+s3cnDa3eyraWLgMAptVWce9w0PvQX05gycfBWVm9ceWj1dv718Y1s2dPJsTMn8tWzj8rL+JdfROQlVa1Pd+ywHpAeaz5YN5Wvnn0U3390A8fOnMRnTvNvgPpHT25i5eu7+OaH6w5p1kbCob6+/0Jr6YxSWZ79eINXi+ZUsuzyejbt7uDfn9nMfS9u4xd/3sq5x03nr+pn88i6nTzwUgMHeuN8qG4an1s8nxMzHCQcjd1Lja3dvHve4TkYOhICAeHEOZWcOKeS65a+i3Xb97lAsYNvPriOG5av46Q5lSw9bjpLjp3W1/0bjysPr93JDx/bwMbdHbxr2gR+8qmTOKdu6mEbFLyw4HCYueqsI1nbuJfvrljPMdMn+NI/+fj6XfzgsQ1ccOJM/uZ98w45FikPsampI+f3yEZzZ5QZBezvPnJKBbdcvJCvnnMUd/1xC7/48zZ+v2YHoWCAi06ayWdPn88RNdmvlxhN3Uux3jg79+1n5igejM4nEeHYmZM4duYkrv7Q0Wzc1c6K1xKB4ju/f53v/P51jp81icVH1bDy9V28sbOdI6dUcPsnFrH02GkEAqM3KCRZcDjMBALC9y85ITFA/YuXWX7VacyOlA9/4SDeaurgH+57lb+YMZGbLzxuwCeZSEWI1rdHpuXQ2hnl2CynsOZi6sRxXH/uMfz9WUfy7MYmTp4XGbK7IBOfO2M+j6zbyTcP89lLu9p76I0rMydn///WWLJg6gS+PDUR/Lfs6ewbo7jtiU3UVof510tP4MMLZxAsgqCQNDZGokaZirISln3qJGJx5XM/f4nuaG9W92nff4Ar71lFqCTATz5Vn3aQuyocorUrSjxe2LEnVaWlM0qkIv/dSoOZNL6UDy+c4VtggIPdSx37Y9zw4Drf7uu3xtbEGgdrOWSutjrM3595JMuvOo1Xvnk2j37lDM4/YWZRBQaw4HDYml9TwW2Xnsj6nfu47jdryHTiQDyufOVXq3m7uYsffWLRoNNjI+EQcYW27gN+PLZnHT0xor1xIgUYcyi0ZPfS/7y2g/9Zs2P4C0ZAco1DPqdNjwWV4VDBF9MVSnHWqkic9a4pfO3so3jw1e389I9bMrr2tic28tj6XfzjeccMubAuuQCtpbOwq6RbfFgAdzj73BnzWThrEt98cO2IrUAfSl/LwYKDGYQFh8PcF846kiV/MY3vrljPHzfu8XTNo6/v4oePbeSiRbP49HvnDXnuweBQ2JZDMjhUjWC3Uj6ldi996zDsXmpo7aa6IsT40OhJVGcKy4LDYU5EuPVjx3PklAquuvdl3mnpGvL8Tbs7+MqvXmXhrEncdMGxw06lG/mWw+E5YOuHw7l7qbGt21oNZkgWHEaBxAB1PfG4cuUQA9T73AD0uNIAd152kqdV1lXuj3Oh1zr05VUqwjGHVIdr91Jja7cNRpshWXAYJeZVh7nt4yfyxs59fP3XAweo43HlK/e9yraWLm7/xCJmePxUmMxr1NJR2ODQ13Io0m6lpENnL6XdSLHgVNVaDmZYFhxGkTOPnsI1Hzqah1ZvZ9kzmw859sPHNvD4G7u54SN1nDLfe2bXspIgFWUltHQVNji0dkYJlQQIj4E+72T30orXdh4W3Ut7OqL0xOIWHMyQLDiMMn+3+AjOO246t/zhDZ7d2ATQtxjnr06axadOnZvxPSPhwiffa+6MUhUOjer0Apk4nLqXGtuSaxxsAZwZnAWHUUZE+N7FCzlq6gSu+uUrPPHGLr52/6scP3sy3/no8APQ6YxEcChUXqXDxeHUvWTTWI0XFhxGoXBZCT/5VCKL6t/8bBXjQ0HuvGxR1mm+I+EQzQUec2jujBbtNNbBpHYv/X4E9y5vaHUL4GxA2gzBgsMoNbcqzI8+cSLzq8PccdlJTJ+U/T/0iEuhUUitndGiXQA3lGT30g0Prhux7qXGtm4mjCth0vjcN7QxxcuCwyh2+oIanrj6TN49L7ftBqtc2u5C7u3RMkaDw+HQvdTYajOVzPAsOBgi4RDRWJzOLBP8Zaon1ktHT6zo1zgMZqS7lxrbukf1vtGmMCw4GCqTq6QLNO4wVtY4DCW1e6m5wN1L1nIwXlhwMFQlg0OBxh368iqNwW6lpJJggO9ecBwtnVH+sG5nwd53b/cB2ntiNhhthmXBwRQ8v9JYyKvkRd30iZSVBNjS1Fmw90zOVJplaxzMMCw4mIP5lQrdrRQe27NlAgGhtjrMlj2FCw62xsF4ZcHBHMyvVKCFcNZyOKjgwaHNdoAz3lhwMFSUlRAKBgo65hAQmGzz7KmtDrOtpYsDvfGCvF9jazfjSgNjerzHeGPBwSAiiRQaBepWanapMwJFtuduNmqrw8TiSoPr7sm3xrZuZkweP2ZyWpnsWXAwQGHzK7V0RPumz45182vCAGzZ01GQ97NU3cYrT8FBRJaIyJsisklErktzfK6IPC4ia0TkKRGZlXLsFhFZ674uSSl/v4i87MrvFpESVz5JRB4SkdUisk5E/tqPipqhRdwq6UJo6Rqbq6PTmV9dAcDmAs1Yamy1BXDGm2GDg4gEgduBpUAd8HERqet32q3APaq6ELgRuNldex6wCDgBOAW4WkQmikgAuBu4VFWPBbYCV7h7fQF4XVWPB84E/kVE7C9JnhUyv1KLS9dtEgsQJ5eXsrkAg9Jd0RjNnVGbxmo88dJyOBnYpKqbVTUK3Aec3++cOuAJ9/rJlON1wDOqGlPVTmANsASoAqKqusGd9yhwkXutwARJdIpWAC1ALOOamYwUcsxhrOZVGkxtdbggax22t9k0VuOdl+AwE3gn5ecGV5ZqNXChe30BiT/uVa58iYiUi0g1cBYwG9gDlIhIvbvmYlcO8CPgGGA78BrwZVUdMJVDRK4UkVUisqqpqclDNcxQqsIh2nti9MTym1+pN660WrfSIQo1nTU56G3TWI0Xfg1IXw0sFpFXgMVAI9CrqiuBFcBzwL3A865cgUuBH4jIC0A7kPyr9CHgVWAGie6oH4nIxP5vqKrLVLVeVetramp8qsbYlRwgbu08kNf32dt9AFUsOKSYXx1m5779dPbkt4HcaC0HkwEvwaGRg5/qAWa5sj6qul1VL1TVE4FvuLI29/0mVT1BVc8GBNjgyp9X1dNV9WTgmWQ58NfAbzRhE7AFeFfWNTSe9OVXyvOgdDJFhwWHg2rdoPTbzfltPTS2dlMSEKZOHJfX9zHFwUtweBFYICK1bmD4UmB56gkiUu0GmQGuB+5y5UHXvYSILAQWAivdz1Pc9zLgWuBOd/024APu2FTgaGBzthU03kQKFBySKTqqbHV0n9rq5HTWPAeHtm6mTRpH0NaXGA9KhjtBVWMichXwCBAE7lLVdSJyI7BKVZeTmFV0s4goiVbAF9zlpcCzbsHNPuAyVU22na8RkQ+TCFB3qGpyQPs7wM9E5DUSLY1rVXWPD3U1Q0hu2dmc5+R7yeBTOcbzKqWaV52YPZTvQekGm8ZqMjBscABQ1RUkxg5Sy25Ief0A8ECa6/aTmLGU7p7XANekKd8OnOPluYx/KssL1K3UZS2H/spDJcyYNC7/LYfWbt53ZHVe38MUD1shbQCYXB5CJLG3cz4lp8tay+FQtTXhvK51iMbi7GrfbzOVjGcWHAwAwYBQWZ7/VdLNnVEmlJVQVhLM6/uMNrXVYTY3deRtH++de/ejCrNsppLxyIKD6VOI/EotnZZXKZ3a6gr27Y/l7b9/Q1tikx9rORivLDiYPpECtBxsAVx68/M8Y8k2+TGZsuBg+kTCobyPOTR3WF6ldJLTWfM17pBcADd9sq1xMN5YcDB9IhWF6VaylsNAsyrHUxKQvLUcGlq7mTqxzMZ6jGcWHEyfKpeZNR7Pz6CoqlpwGERJMMCcqvK8rXVobLV9HExmLDiYPpXlIeIKbd35ya/UGe0l2hu34DCI+XlMwNfY1s1MS9VtMmDBwfRJrpLOV9dSco2DBYf05tdUsKW50/eWWzyu7NhrLQeTGQsOpk++8yslU3Mkg5A5VG11mGgszva9/u4nvbu9hwO9atNYTUYsOJg+B4NDfvIr9eVVKrfgkE7fjCWfxx0a3RoHWwBnMmHBwfRJBod8rXVIBgfLq5RevtY62CY/JhsWHEyfSN+GP/kNDhHrVkqrZkIZ4VAwf8HBWg4mAxYcTJ+ykiAVZSV5bTmESgKEQzbXPh0RyUsCvsa2birLSwmXeUrCbAxgwcH0k8/8Ss2dUSLlIdz+HiaN2uoKtuzp8PWeja3d1qVkMmbBwRyiMo/BodUWwA2rtjpMQ2s3PbHe4U/2qLHNprGazFlwMIeoynPLwaaxDm1+dRhV2Nbc5cv9VNWtjrYFcCYzFhzMIfLZrWSpM4Y3v8bfBHytXQfoPtBr3UomYxYczCEi4UTa7nxsOtPSGbU1DsOY5/N01oZWt8bBgoPJkAUHc4hIOEQ0Fqcz6l+fN0BPrJeOnpil6x7GxHGlVFeUsbnJn0Fp28fBZMuCgzlEvtY6tHYmkvnZGofh+ZmAL7mPg7UcTKYsOJhDVOVplXRfXiVrOQyr1sfg0NDaTTgUZNL4Ul/uZ8YOCw7mEJV5yq+UbDnYmMPwamvC7OmIsteH1OmJVN3jbW2JyZgFB3OIvpZDR55aDtatNKxkAr63fWg92CY/JlsWHMwh+sYcuvwNDn15lSzp3rD8TMCXbDkYkylPwUFElojImyKySUSuS3N8rog8LiJrROQpEZmVcuwWEVnrvi5JKX+/iLzsyu8WkZKUY2eKyKsisk5Ens61ksa7irISQsGA72MOLZ1RAoL1fXswp6qcgOS+1qF9/wH2dh9glu0AZ7IwbHAQkSBwO7AUqAM+LiJ1/U67FbhHVRcCNwI3u2vPAxYBJwCnAFeLyEQRCQB3A5eq6rHAVuAKd81k4MfAX6rqXwB/lXMtjWciQmW4tG/XNr+0dEaZXB4iGLC+7+GUlQSZVVmec8shOVPJupVMNry0HE4GNqnqZlWNAvcB5/c7pw54wr1+MuV4HfCMqsZUtRNYAywBqoCoqm5w5z0KXORefwL4japuA1DV3ZlXy+QiEi7zfZW0rY7OTGLGUm5rHRptHweTAy/BYSbwTsrPDa4s1WrgQvf6AmCCiFS58iUiUi4i1cBZwGxgD1AiIvXumotdOcBRQKXrnnpJRC5P91AicqWIrBKRVU1NTR6qYbyqCodo8XnModmCQ0Zqq8NsaerMaaV63xoHazmYLPg1IH01sFhEXgEWA41Ar6quBFYAzwH3As+7cgUuBX4gIi8A7UBySW4JcBJwHvAh4JsiclT/N1TVZapar6r1NTU1PlXDQH7yK7V0Rm2NQwbm14TpjPayuz37KcWNrd2EggGqK2wSgMmcl90/Gjn4qR5glivro6rbcS0HEakALlLVNnfsJuAmd+yXwAZX/jxwuis/h0SLARItk2bXDdUpIs8AxyevM/kXCYd8H3No7Yz2raEww0vdT3rqxHFZ3aOhrZsZk8cRsHEekwUvLYcXgQUiUisiIRKf+JenniAi1W6QGeB64C5XHnTdS4jIQmAhsNL9PMV9LwOuBe501z8InCYiJSJSTmIge332VTSZioRDtPfEfNtTIB5XWrus5ZCJWh+msza0dttMJZO1YYODqsaAq4BHSPyRvl9V14nIjSLyl+60M4E3RWQDMBXXUgBKgWdF5HVgGXCZux/ANSKynsQg9UOq+oR7v/XAH1z5C8B/qOra3KtqvEqODbR15b5CF6Ct+wBxxcYcMjBj0nhCJYGcBqVtAZzJhadNZVV1BYmxg9SyG1JePwA8kOa6/SRmLKW75zXANYMc+2fgn708m/Ff6irpbLs0UiVTcVhw8C4QEGqrss+xtP9AL3s6emymksmarZA2AxzMr+TPuENLMiOrBYeM1FaHs14It93WOJgcWXAwAxzMzOpP8j1rOWRnfk2Ybc1dxHrjGV/btwDOWg4mSxYczAB+7+mQTMVRZXmVMlJbHSYWVxrcYrZM2CY/JlcWHMwAk8tDiPjYreSmxVaGLa9SJg7uJ535oHRjWzcBgWmTch8zMmOTBQczQDAgTB5f6lvyvZauKBVlJZSVBH2531hRW10BJNY6ZKqhtZvpk8ZTGrR/4iY79n+OScvPVdKWVyk7leWlTBpfmtWMJZvGanJlwcGkVeVj8j0LDtkRkay3DLV9HEyuLDiYtPxsOTR3WHDI1vwsgkOsN87Offut5WByYsHBpFXpY3Bo7bLgkK3a6jA79u6nKxob/mRn57799MbVWg4mJxYcTFpV4RCtXVHi8exTRgOoKs2WkTVr82sSg9Jv7+nyfI1NYzV+sOBg0oqEQ8QV9nbnll+pM9pLNBa3lkOWsknAZwvgjB8sOJi0qiqSq6Rz61o6uMbBgkM25lUnsqpmkoCvwVoOxgcWHExaleX+5FdK7ihn3UrZKQ+VMH3SuIzWOjS2dlNdUca4UltXYrJnwcGkFelLvpdbfiXLq5S7TBPw2TRW4wcLDiatZLdSMqNqtpo7LK9Srmqrw2xu6vC8n3RjW7ftG21yZsHBpHWwWynXloPlVcpVbXWYfftjtHrYfCkeV2s5GF9YcDBpjSsNEg4Fcx+Q7ooSCgaoKPO0r5RJI5mAz8ug9J7OHqKxuA1Gm5xZcDCDilTkvhCuxa2OFrFN7rOVSQK+5EylWdZyMDmy4GAGFfEhv5LlVcrd7MrxlATE01qHvgVwFhxMjiw4mEFV+ZBCo9mCQ85KggHmVJV7Cw62PajxiQUHM6jK8tyDg+VV8ofXBHyNrd1MHFfChHE2AcDkxoKDGVRVRYjmzqjnKZTptFhGVl8kU3cPl+sqMVOpvEBPZYqZBQczqEg4RDQWpyvam9X1PbFe2ntitjraB7XVFfTE4mzfO/R+0rbJj/GLBQczqIOrpLPrWmp1C+gsr1LuvCTgU02scbCZSsYPFhzMoCLluSXfSwYVaznk7uBah8GDw97uA3T0xCw4GF94Cg4iskRE3hSRTSJyXZrjc0XkcRFZIyJPicislGO3iMha93VJSvn7ReRlV363iJT0u+e7RSQmIhfnUkGTvUhFbqukk8HBxhxyN2VCGeWh4JBrHSwbq/HTsMFBRILA7cBSoA74uIjU9TvtVuAeVV0I3Ajc7K49D1gEnACcAlwtIhNFJADcDVyqqscCW4Er+r3nLcDK3KpnclEVzi2/UrMLKsk8TSZ7XvaTtn0cjJ+8tBxOBjap6mZVjQL3Aef3O6cOeMK9fjLleB3wjKrGVLUTWAMsAaqAqCjwkS0AABPGSURBVKpucOc9ClyUcr8vAr8GdmdYH+OjXDOztibzKpVbcPDD/JqKoYODtRyMj7wEh5nAOyk/N7iyVKuBC93rC4AJIlLlypeISLmIVANnAbOBPUCJiNS7ay525YjITHePO4Z6KBG5UkRWiciqpqYmD9UwmaooK6E0KDmNOYjAZAsOvqitDtPQ2kVPLP3ssca2bsaVBqwbz/jCrwHpq4HFIvIKsBhoBHpVdSWwAngOuBd43pUrcCnwAxF5AWgHkv/H/xC4VlXjQ72hqi5T1XpVra+pqfGpGiaViBAJh/p2c8tUc2eUyvIQwYDlVfLD/OowcYV3WtLvJ52cxmp5rIwfvKTKbMR9qndmubI+qrod13IQkQrgIlVtc8duAm5yx34JbHDlzwOnu/JzgKPc7eqB+9z/4NXAuSISU9XfZVE/k6NIuIzWruxbDvYp1j/J6axvNXVy5JQJA443tHUxyxbAGZ94aTm8CCwQkVoRCZH4xL889QQRqXaDzADXA3e58qDrXkJEFgILcYPMIjLFfS8DrgXuBFDVWlWdp6rzgAeAv7fAMHKqwqGcupUi1qXkm3nDrHVobLV9HIx/hg0OqhoDrgIeAdYD96vqOhG5UUT+0p12JvCmiGwApuJaCkAp8KyIvA4sAy5z9wO4RkTWkxikfkhVkwPa5jBSmUPyPWs5+GvS+FKqK0JsSTOdtSua2AzIBqONXzztwKKqK0iMHaSW3ZDy+gESn/L7X7efxIyldPe8BrhmmPf9tJfnM/lTlcOYQ0tnlHfXWnDw02DTWRttHwfjM1shbYYUCYdo74kRjQ05P2CAeFxp7Yra6mif1VaH2ZwmODRYqm7jMwsOZkjJbqFMB6X3dh8grrbGwW+11RXs6ehh3/5DFybaJj/GbxYczJCSwaE5w66l5CC2rY72VzLH0tv9Wg+Nbd2UBIQpE8aNxGOZImTBwQwp25aD5VXKj/mDzFhqaO1mxuTxtqbE+MaCgxlScswg0+msyZQbFhz8NaeqHBEGJOBrbO2y8QbjKwsOZkh9+ZU6MsuvlEzWZ8HBX2UlQWZVjh8wKJ3YAc6Cg/GPBQczpMnlIUQy3/DHWg75U1tdwZY9HX0/R2Nxdrf3WMvB+MqCgxlSMCBMHl9KS4ZjDs2dUSrKSigrCebpycau+dVhtjR19u3tvWNvN6o2U8n4y4KDGVYki1XStjo6f2qrw3RGe2lqT7TO+hbAWcvB+MiCgxlWVbgs46msLZ1R2zs6T5IJ+JLjDg22yY/JAwsOZliV4dKsWg62Ojo/+u8n3dDajQhMn2TBwfjHgoMZVjZpu61bKX9mTBpPqCTQFxwaW7uZOmEcoRL752z8Y/83mWFVhUO0dh0gHldP56sqzdZyyJtAQKitCvetdWhs67IuJeM7Cw5mWJFwiN64srf7wPAnA13RXqKxuI055FEiO2tiOmtjW7dNYzW+s+BghhXJcJW0pc7Iv9qaMNtaEvtJ72jbby0H4zsLDmZYmeZX6ku6Z8Ehb2qrwxzoVV7e2kYsrtZyML6z4GCGlWlmVlsdnX/JBHzPbmwCbJMf4z8LDmZYffmVPHcrWV6lfEuudfjjpj2ABQfjPwsOZlgHg4O35HvWcsi/SDjExHElvNa4F4AZ1q1kfGbBwQxrXGmQcCjY1yIYTnNnlFAwQEWZpy3KTRZEhPk1FagmAkV5yP5bG39ZcDCeRCpC3lsOHYkFcCK28Uw+JccdbDDa5IMFB+NJpDzkeSpra5flVSqEWgsOJo8sOBhPMsnMaqujC6PW5ViyNQ4mHyw4GE8i4TJaM1gEZ4PR+WctB5NPFhyMJ1UViW6l5AYzQ0mOOZj8OmbaRL529lF85PgZI/0opgh5Cg4iskRE3hSRTSJyXZrjc0XkcRFZIyJPicislGO3iMha93VJSvn7ReRlV363iJS48k+6+7wmIs+JyPF+VNTkprI8RE8sTle0d8jzorE47T0xCw4FEAgIX/zAAmomlI30o5giNGxwEJEgcDuwFKgDPi4idf1OuxW4R1UXAjcCN7trzwMWAScApwBXi8hEEQkAdwOXquqxwFbgCnevLcBiVT0O+A6wLLcqGj9UeVwIl0yxYcHBmNHNS8vhZGCTqm5W1ShwH3B+v3PqgCfc6ydTjtcBz6hqTFU7gTXAEqAKiKrqBnfeo8BFAKr6nKq2uvI/AX2tEDNyvK6STqbYsAFpY0Y3L8FhJvBOys8NrizVauBC9/oCYIKIVLnyJSJSLiLVwFnAbGAPUCIi9e6ai115f58BHk73UCJypYisEpFVTU1NHqphchGpsJaDMWOJXwPSVwOLReQVYDHQCPSq6kpgBfAccC/wvCtX4FLgByLyAtAOHNKZLSJnkQgO16Z7Q1Vdpqr1qlpfU1PjUzXMYCLl3tJ2N1u6bmOKgpc1940c+ql+livro6rbcS0HEakALlLVNnfsJuAmd+yXwAZX/jxwuis/BzgqeT8RWQj8B7BUVZuzqZjx18GWw9CrpFs6LK+SMcXAS8vhRWCBiNSKSIjEJ/7lqSeISLUbZAa4HrjLlQdd91LyD/5CYKX7eYr7XkaidXCn+3kO8BvgUyljEmaETSgroTQow+ZXaumMIgKTyy04GDOaDdtyUNWYiFwFPAIEgbtUdZ2I3AisUtXlwJnAzSKiwDPAF9zlpcCzLsfOPuAyVY25Y9eIyIdJBKg7VDU5oH0DiQHrH7vrYqqaHJswI0RE3CrpYVoOXVEqy0MEA5ZXyZjRzFMqR1VdQWLsILXshpTXDwAPpLluP4kZS+nueQ1wTZryzwKf9fJcprAqy4dPodHSGaWyvLRAT2SMyRdbIW08S66SHkpzR5SqsC3KMma0s+BgPPOSX8nyKhlTHCw4GM+qwsO3HFq7on0zm4wxo5cFB+NZZXmI9v0xorF42uPxuNLadaBvTYQxZvSy4GA8S7YIkqug+9vbfYDeuFq3kjFFwIKD8Wy45HvJLqcq61YyZtSz4GA8Gy75nuVVMqZ4WHAwniX/6A82KJ3MyFppYw7GjHoWHIxnfS2HjvSrpFusW8mYomHBwXhWWR5CBFq60udXSqbWsG4lY0Y/Cw7Gs2BAmDy+dND8Si2dB6goK6GsJFjgJzPG+M2Cg8lIZXjw/EotnT1Uhi2vkjHFwIKDyUhVONQ38Nxfc2eUiOVVMqYoWHAwGYmEQ4MugmvpjNre0cYUCQsOJiORcNng6xws6Z4xRcOCg8lIJFxKa9cB4nE9pFxVXbeSBQdjioEFB5ORSLiM3riyt/vQ6axd0V56YnELDsYUCQsOJiN9+ZX6jTsku5osOBhTHCw4mIwMll+pb3W0BQdjioIFB5ORvvxKHemDQ6UFB2OKggUHk5HBWg7N1nIwpqhYcDAZSQaH/msdLK+SMcXFgoPJyLjSIOFQME230gFCwQAVZSUj9GTGGD9ZcDAZS+RXOjT5XjKvkoiM0FMZY/xkwcFkrCocGrDhT4vlVTKmqHgKDiKyRETeFJFNInJdmuNzReRxEVkjIk+JyKyUY7eIyFr3dUlK+ftF5GVXfreIlLhyEZHb3HutEZFFflTU+CddfqVmy6tkTFEZNjiISBC4HVgK1AEfF5G6fqfdCtyjqguBG4Gb3bXnAYuAE4BTgKtFZKKIBIC7gUtV9VhgK3CFu9dSYIH7uhK4I6caGt9FwmW09BtzsLxKxhQXLy2Hk4FNqrpZVaPAfcD5/c6pA55wr59MOV4HPKOqMVXtBNYAS4AqIKqqG9x5jwIXudfnkwg0qqp/AiaLyPQs6mbyJBIupbkziurB/EqWV8mY4uIlOMwE3kn5ucGVpVoNXOheXwBMEJEqV75ERMpFpBo4C5gN7AFKRKTeXXOxK/f6fojIlSKySkRWNTU1eaiG8UskXEZPLE5XtBeAaCxO+/6YBQdjiohfA9JXA4tF5BVgMdAI9KrqSmAF8BxwL/C8K1fgUuAHIvIC0A70ZvKGqrpMVetVtb6mpsanahgvqvothGvrsrxKxhQbL8GhkYOf6gFmubI+qrpdVS9U1ROBb7iyNvf9JlU9QVXPBgTY4MqfV9XTVfVk4JlkuZf3MyOr/yppWx1tTPHxEhxeBBaISK2IhEh84l+eeoKIVLtBZoDrgbtcedB1LyEiC4GFwEr38xT3vQy4FrjTXb8cuNzNWjoV2KuqO3Koo/FZZb/gYHmVjCk+wy5nVdWYiFwFPAIEgbtUdZ2I3AisUtXlwJnAzSKiJFoBX3CXlwLPuoVR+4DLVDXmjl0jIh8mEaDuUNXkgPYK4FxgE9AF/HXu1TR+SrYQmq3lYEzR8pTrQFVXkPijnVp2Q8rrB4AH0ly3n8SMpXT3vAa4Jk25cjC4mMNQpMLlV3JBodX2cjCm6NgKaZOxCWUllAblkJaDCEwut+BgTLGw4GAyJiJUlh/Mr9TS2cPk8aUEA5ZXyZhiYcHBZCUSDh0yIG1dSsYUFwsOJitVFYcGhypLumdMUbHgYLISCZdZy8GYImbBwWQlUl7aNyDd0hm1NQ7GFBkLDiYrkXAZ7ftj9MR6ae06YGscjCkyFhxMVpJrHbY2d9EbV+tWMqbIWHAwWUm2FDbt7kj8XGHBwZhiYsHBZKXSLXjbuKvjkJ+NMcXBgoPJSrKlsHF3O2CpM4wpNhYcTFYi1q1kTFGz4GCyMnl8KSKweU8nYN1KxhQbCw4mKyXBAJPGlxKNxQmHgowrDY70IxljfGTBwWQt2bUUsS4lY4qOBQeTteR01ojlVTKm6FhwMFnrazmUl47wkxhj/GbBwWQtYi0HY4qWBQeTtWRwsGmsxhQfCw4ma8kWgy2AM6b4WHAwWYuEE2MNEVvjYEzRseBgsmYtB2OKlwUHk7WT50W48oz5vOeIqpF+FGOMz0pG+gHM6DU+FOT/nHvMSD+GMSYPrOVgjDFmAE/BQUSWiMibIrJJRK5Lc3yuiDwuImtE5CkRmZVy7BYRWeu+Lkkp/4CIvCwir4rIH0XkSFc+R0SeFJFX3P3O9aOixhhjvBs2OIhIELgdWArUAR8Xkbp+p90K3KOqC4EbgZvdtecBi4ATgFOAq0VkorvmDuCTqnoC8EvgH135PwL3q+qJwKXAj7OvnjHGmGx4aTmcDGxS1c2qGgXuA87vd04d8IR7/WTK8TrgGVWNqWonsAZY4o4pkAwUk4Dtw5QbY4wpEC/BYSbwTsrPDa4s1WrgQvf6AmCCiFS58iUiUi4i1cBZwGx33meBFSLSAHwK+CdX/m3gMle+AvhiuocSkStFZJWIrGpqavJQDWOMMV75NSB9NbBYRF4BFgONQK+qriTxB/454F7geaDXXfMV4FxVnQX8J/B9V/5x4Geu/Fzg5yIy4DlVdZmq1qtqfU1NjU/VMMYYA96CQyMHP+0DzHJlfVR1u6pe6MYJvuHK2tz3m1T1BFU9GxBgg4jUAMer6p/dLX4FvNe9/gxwv7v2eWAcUJ1N5YwxxmTHS3B4EVggIrUiEiIxSLw89QQRqU75dH89cJcrD7ruJURkIbAQWAm0ApNE5Ch3zdnAevd6G/ABd80xJIKD9RsZY0wBDbsITlVjInIV8AgQBO5S1XUiciOwSlWXA2cCN4uIAs8AX3CXlwLPigjAPuAyVY0BiMjfAr8WkTiJYPE37pqvAf8uIl8hMTj9aVXVoZ7xpZde2iMiWzOodzaqgT15fo/DldV9bLK6F7+5gx2QYf7uGkdEVqlq/Ug/x0iwulvdx5qxXPckWyFtjDFmAAsOxhhjBrDg4N2ykX6AEWR1H5us7mOYjTkYY4wZwFoOxhhjBrDgYIwxZgALDoMQkbdF5DWXUnyVK4uIyKMistF9rxzp5/SDiNwlIrtFZG1KWdq6SsJtLn37GhFZNHJPnrtB6v5tEWl0v/tXU9PGi8j1ru5visiHRuapcycis11q/NdFZJ2IfNmVF/3vfYi6F/3vPSOqal9pvoC3gep+Zd8DrnOvrwNuGenn9KmuZ5BIrb52uLqSyHf1MIlUKKcCfx7p589D3b8NXJ3m3DoSySTLgFrgLSA40nXIst7TgUXu9QRgg6tf0f/eh6h70f/eM/mylkNmzgfudq/vBj46gs/iG1V9BmjpVzxYXc8nsXeHquqfgMkiMr0wT+q/Qeo+mPOB+1S1R1W3AJtIpLQfdVR1h6q+7F63k0hfM5Mx8Hsfou6DKZrfeyYsOAxOgZUi8pKIXOnKpqrqDvd6JzB1ZB6tIAarq5cU7sXgKtd9cldK92FR1l1E5gEnAn9mjP3e+9UdxtDvfTgWHAZ3mqouIrED3hdE5IzUg5pob46JecBjqa7OHcARJHYw3AH8y8g+Tv6ISAXwa+AfVHVf6rFi/72nqfuY+b17YcFhEKra6L7vBn5Lohm5K9mUdt93j9wT5t1gdR02hftop6q7VLVXVePAv3OwC6Go6i4ipST+OP5CVX/jisfE7z1d3cfK790rCw5piEhYRCYkXwPnAGtJpCq/wp12BfDgyDxhQQxW1+XA5W72yqnA3pRuiKLQry/9AhK/e0jU/VIRKRORWmAB8EKhn88PkkiV/FNgvap+P+VQ0f/eB6v7WPi9Z2SkR8QPxy9gPonZCauBdcA3XHkV8DiwEXgMiIz0s/pU33tJNKMPkOhP/cxgdSUxW+V2EjM2XgPqR/r581D3n7u6rSHxh2F6yvnfcHV/E1g60s+fQ71PI9FltAZ41X2dOxZ+70PUveh/75l8WfoMY4wxA1i3kjHGmAEsOBhjjBnAgoMxxpgBLDgYY4wZwIKDMcaYASw4GGOMGcCCgzHGmAH+P15kKnAutZTeAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZxcZZno8d9T1V3d6erupKsqCSQhCyQBGgyLMbhhWAyERRnRERgX9Oowjuj16sAgVy8qDoN+xqteHcTLeFFwBtBhxhnUIIlsYQyKYQtZJAQIkD29ZOnqTld39XP/OG9VV3pJV3dVnapTeb6fT39S/Z5zqt5TqernvNtzRFUxxhhjcoXKXQFjjDGVx4KDMcaYYSw4GGOMGcaCgzHGmGEsOBhjjBnGgoMxxphhLDiYo46IfFVE/rnc9RiNiGwVkXe7x/9TRH5U7jqZo09NuStgjBmdqv59sZ5LRBRYoKpbivWcpnpZy8GYMhERuzgzFcuCgwkM191yo4hsFJFOEfmxiNSLSIuI/EpE9rryX4nIrJzj5onI4yJyUERWAYk8X++dIrJGRPaJyBsi8jFXPllE7nav95qIfFlEQm5byP3+mojscftNdtvmioiKyCdE5HXgEVf+Ebd/u4h8aUgdsl1gOcdfLSKvi0hb7v4iskREnnT13Ski/ygiEbdttdvteRHpEpErXPmlIvKcO2aNiCyayP+NqT4WHEzQfAi4EDgBWAh8Ge9z/GNgDjAb6AH+MeeYe4Cn8YLC14Grx3oREZkDPAh8H5gKnA485zZ/H5gMHA8sBT4KfNxt+5j7OddtbxxSF9wxJwMXikgrcDvwEWAGEAdmcWTvBE4EzgduEpGTXXka+Lw7z7e57Z8GUNV3uX1OU9VGVf2ZiJwB3An8lXvd/ws8ICJ1Y7y+ORqoqv3YTyB+gK3Ap3J+vxh4eYT9Tgc63ePZQD8Qzdl+D/DPY7zWjcAvRigPAymgNafsr4DH3OOHgU/nbDsR6MMb35sLKHB8zvabgPtyfo+653+3+/2rmbrmHD8rZ/+ngCtHOYf/kXsO7tj5Ob/fDnx9yDEvAkvL/X9tP+X/sT5PEzRv5Dx+DZghIg3Ad4DlQIvb1iQiYbyr8U5VTQ457rgxXuc44OURyhNArXuO3Oeb6R7PGGFbDTB9lHOYkfu7qiZFpH2Muu3KedyN1zpBRBYC3wYWAw3udZ8+wvPMAa4Wkc/mlEVcncxRzrqVTNDk/lGfDewA/gbvCv0sVW0GMl0oAuwEWkQkOuS4sbyB13U1VBteS2DOkOfb7h7vGGFbP7A7pyw3FfJOcs7JBbp4HvUbye3An/BmJDUD/xPvPRjNG8Atqjol56dBVe+d4OubKmLBwQTNtSIyS0RiwJeAnwFNeOMM+1z5VzI7q+prwFrgayISEZF3Au/J43X+BXi3iHxQRGpEJC4ip6tqGvg5cIuINLmxiS8AmXUT9wKfd4PgjcDfAz9T1f5RXud+4FI3+B0Bbmbi38sm4ADQJSInAX89ZPtuvHGQjH8CPiUiZ4knKiKXiEjTBF/fVBELDiZo7gFWAq/gdfv8HfBdYBLeVf3vgd8MOeYvgLOADrzAcfdYL6Kqr+ONafyNO+454DS3+bNA0tXhv1yd7nTb7gR+CqwGXgUOuf1He50NwLXuOXYCncC2seo3iuvwzvUg3h/+nw3Z/lXgLjcz6YOquhb4S7wB805gC95gujGIqt3sxwSDiGwFPqmqvy13XYypdtZyMMYYM4wFB3PUEpEPuQVhQ382lLtuxpSbdSsZY4wZxloOxhhjhqmKRXCJRELnzp1b7moYY0ygPP30022qOnWkbVURHObOncvatWvLXQ1jjAkUEXlttG3WrWSMMWYYCw7GGGOGseBgjDFmGAsOxhhjhrHgYIwxZpi8goOI3Oluebh+lO0iIt8TkS0isk5EzszZdrWIvOR+rs4pf7OIvOCO+Z6IiCuPicgqt/8qEWkZ6TWNMcaUTr4th5/g3UhlNBcBC9zPNXh55clJn3wWsAT4Ss4f+9vxMkJmjss8/xeBh1V1Ad5dtb6YZx2NMcYUSV7rHFR1tYjMPcIulwF3q5eL4/ciMkVEjgXOAVapageAeDd3Xy4ijwHNqvp7V3438Gd49+y9zB0HcBfwGHDDeE7KL9s6u/nFM9vpSw9M+DlqwiHmxBs4YWojx0+N0hAJxtKTgQHl2Tc6Wb/9AFctmU2kxv8eyv70AD/+3VYOHurz/bUzIjUhPvr2uTTX15atDkerrW1JVm3cXdD/fygkXPGW4zh28qQi1ix///Hsdl7Z21XQcyyeG+NdC0dcx1aQYv0lmsnhtz7c5sqOVL5thHKA6aq60z3exeG3V8wSkWvwWinMnp3Pjb2K51Bfmv/7+Cvc/vgWDvUNIEe619YYhqa2mjllEvOnNXLC1EbmT2t0j6PEG8t/z/e+9AC/f6Wd36zfxaqNu9lzsBeAOfEGzjlxmu/1Wbd9P7es2ARQ0P9BIVShNhzir5aOdNM4U2zbOrv59bqd/HLdDtZvPwAU9n+vCoLwuXcvKFIN89eXHuALP3+OAS3sHD619ISKDg4loaoqIiNmBlTVO4A7ABYvXuxL9kBV5Tfrd/F3v97E9n09XLroWG68+GRmTpn4VUdvf5rX2rvZsqeLl/d0sWVvF1v2dPGHV9s51DfYImlpqM0JFo2cMK2R+VMbmTllEqFQ6f4y9qTSPL55Lys37OK3m3Zz4FA/k2rDnHPiVM6c3cItKzax1wUJv2Ve91effSenzpxcljpc/H+eYNXG3RYcSmj3gUP8et1OfrVuB8+8vg+A046bwpcvOZmL33QsMwr4/p32tZW0J8vz+e3sTjGg8PU/O5WPvHXO2Af4rFjBYTuH39t3livbzmAXUab8MVc+a4T9AXaLyLGqutN1Te0pUh0L8uKug3ztlxtY83I7Jx3TxH3XvJW3Hj/RW/0OqqsJs3B6EwunH35nxoEBZcf+Hrbs8YLFy3uTvLyni4c27KYjOdgYq68NcXyi8bDAMX9aI3MTDdTVhCdUp/09fTzyp938Zv0uHt+8l0N9A0xpqGVZ6zEsP/UYzl6QoL42TLK3n1tWbKI9mSroPZio9i7vdRNlbFVdcMp0/s/DL9HW1et7PVSV/3xuB21dvaQHlLQqAwNKegAGVBlQPax8QCE9MFg+oIqIcPkZM1k8N+Zr3cfS3tXLivW7+NXzO3hqaweqcPKxzfzt8hO59E0zmB1vKMrrxKKRsn1+O9zrxhoiZXn9sRQrODwAfEZE7sMbfN7v/rg/BPx9ziD0BcCNqtohIgdE5K3AH4CPAt/Pea6rgW+4f/+zSHWckP3dfXznt5v56e9fo7Guhq9fdgpXLZlNTbi0feyhkDCrpYFZLcO7bDqSKV52LYzMzzOvd/LA8zuy+4RDwuxYg2tlRJnvgsYJ0xpH7B/fc/AQKzfs5qENu3jy5Xb6B5TpzXV8cPFxXHjKMSyZF6N2yDk3RMLU14Zo7yrPlVfmdWPR8n25lrVO57u/fYlHNu3hg285buwDiuiZ1/fxP3723KjbQwIhEUIhISxCOCSIeJ+NsCvvSaW55w+vc/kZM/niRScxrbnexzM43P7uPh7asItfrtvBmpfbSQ8oJ0yN8rnzF3DpohnMn9ZY9NeMRSN0dJUpOLjXLefn90jyCg4ici9eCyAhItvwZiDVAqjqD4EVePfb3QJ0Ax932zpE5OvAH91T3ZwZnAY+jTcLahLeQPSDrvwbwM9F5BPAa8AHJ356E5ceUH72xzf41soX2ded4kNnzeELyxbSUgH/kbFohFg0xluGXO31pNK8vLfL+8nponp88x760oM9b9Oa6rItjXi0jide2svTr3eiCvMSUT5x9jyWn3IMp82acsQuKxEhHq3LXsH7rT2Zorm+piyD4RmtxzYzc8okVm7c7XtwePCFnUTCIZ644Vya6mu8QOCCQEi8/5+xdKf6ue3RLfzT6ldZuXE3nzt/AR97x9xhFwKlcvBQH7/dtJtfPb+T1S/tpS+tzI418Kmlx3PpohmcdExTXucxUbFohDc6ukv2/EfS0e19b+KN5f+bMpJ8ZytdNcZ2xbtJ+kjb7mTw5uu55WuBU0cobwfOz6depbJ2awdfeWADG3YcYMm8GF99zym0zmguZ5XyMikS5tSZk4f1v/enB3ijs+ewlsbLe7v4xTPbOdjbzykzmvn8uxey/NRjWDCtcVxfxkRTHW1lapaXoytnKBFhWet07n3qdbpT/b7NNlNVHly/i3ctTDC9gKv9hkgN1194En/+5uP42i83cMuKTfxs7Rt89T2n8M4FiSLW+HAv7T7Ij9ds5RfPbKenL82MyfV87O1zec9pM3jTzMklDQi54tEIz72xz5fXGirbrVQBF5wjqegBab/t2n+Ibzy4if94bgfHTq7n+1edwaWLjvXtg1oqNeEQ8xJR5iWiLGsdnPylqiRTaRrrJv4xSEQj7DpwqBjVHLf2rlRFXHUta53OT9Zs5YmX2rjwlGN8ec112/azfV8PX1i2sCjPNzcR5ccfX8LDm3bztV9u5MP/7w9c/KZj+NIlrQVNuMg1MKA8+uIefvy7rfzXljYiNSEuO20GV7zlOM6c3VLSiRWjaYlG6EymUDf+4qdMi3vKpMqcBm3BAW/G0I+eeJXbHt1C/4Dy2fPm89fnnBCYNQcTJSIFBQbwmsQbdhwoUo3Gpz3Zy/GJ4vdDj9eSeTGa62tYtXG3b8FhxQs7qQ0L7z55xJneE3b+ydN5x/wE/7T6FW57bAuP/GkP154zn7981/HU105sgsPBQ33869pt3PXkVl5r72Z6cx3XX3giV77luLJP0Y5HI/QPKAd6+pnc4O8f6Y5kiikNtSUfv5yo6v7rNwZV5eFNe/j6rzfyWns3F54ynS9f0spxseLMhDgaxBvraE/2luXKq60rxVvmlr/lUBsOcd5J03h402760wMl/7KrKivW7+Qd8xMl+YNWXxvms+cv4PI3z+KWX2/kf6/azL8+vY2vvKeV88cRjF5tS3LXmq3869o3SKbSnDl7CtddcCLLTz3GtzGNsWS6dDq6U/4Hh+5UxXYpwVEeHP517Tb+9t/WMX9aIz/9xBLOXlD8hSTVLh6N0Jf2/8qrPz1AZ3eq7GMOGctaj+E/ntvB0691clYRpjgfyYYdB3ijo4fPnlfahVszp0ziBx96M7/b0sZXHtjAJ+5ay7knTuWm95zCvER0xGNUlSdeauMna7by6It7qAkJly6awcfePpfTjptS0vpORDY4JHtHPadS6ehKEbfgUJkuWXQsvf1prlwyu2KuZIIm88e5Ldnra3Do7O5DFRIVMOYAsPTEqUTCIVZt3F3y4LDihZ3UhIQLWovbpTSad8xP8ODnzuauNVv57m9f4sLvrOYv3zWPa8+dn+167U7182/PbOeuNVvZsqeLRGOE/37eAj701tlMayrf9NixZIJDOWbcdSRTzCnSeo1SOKqDQ7Suho+8bW65qxFomQHh9q4UJ/jY8Mqsai13n3VGY10NbzshzqpNu/nSJSeXrItNVVnxwk7edkKcKT4unqoNh/jk2cfz3tNm8I3f/InbHn2Zf39mO19YtpCX9nRx31Ovc+BQP2+aOZlvf/A0Lll07IQXYfppsOXgf3BoT6Y4c07ltaYyjurgYAoXj3p/nP1eCJe50qukZvkFp0znS79Yz0t7uoateC+WTTsPsrW9u2zpOqY11/PtD57OXyyZzU3/uYHr719HOCQsP/UY/ts75nLm7JZAze7LfH4zaw78oqp02piDqWaJJu/D7fdah7auymo5ALz7ZC84rNq4u2TB4cH1Own72KU0msVzY/zys+/kd1vamD+tsaD8RuU0KRJmUm3Y91XSB3r6SQ8osWjlfH6Hso52U5BMXphytRwqZcwBYHpzPacdN4WVG3aV5PlVlV+/sJO3Hh+riKAYDgnvWjg1sIEhIxaN+N6tlOkWjUUrc40DWHAwBaoJh2hpqPV9QK892UtNSCruPgoXtE7n+W372V2ChYEv7enilb1JLjr12KI/99GsHMn3BldHlz/Ij8aCgylYZq2Dn9oOev215VhVeySZFeirNu4u+nOveGEnIvi20O5oEYtG6PR5zCETHCppzGwoCw6mYPFohLaD/rccKmWNQ64F0xqZG28oWXBYMjfG1KbKO+8gi0cjvrd8Kz2vElhwMEWQaKyjze+WQ4XkVRoqk4jvyZfbi3r70i17DrJ5dxcXv8m6lIqtpSxjDhYczFEg3uj/lVelthzAWy2dSg/w+Oa9RXvOB1/YhQgsP9W6lIotFo3Q05emJ5X27TU7kil3P5TKXQtiwcEULB6tY39PH6n+gbF3LpL2Ck498OY5LcSikaJ2La1Yv4vFc1oKSs9tRhbPya/kl85kZa9xAAsOpggyax38GtTrTvXTnUpXxHTOkYRDwnknTePRP+2hL114wHy1LcmmnQdsllKJZFdJ+9j6bU9W7sVNhgUHU7DMKtM2n9Y6ZFdHV+CYQ8YFrdM5cKifp17tGHvnMax4YSdgXUqlks2v5OO4WUcyVRF3lTwSCw6mYIlGf5OXZQbzKmkB3FBnL5hKfW2oKF1LD67fyRmzpwR+sVmlKkd+pQ7rVjJHg0z3jl9XXm0HXeqMCl5ANCkS5p3zp7Jywy68u+hOzOvt3azffoBLbJZSyWTzK/kcHKxbyVS9TPeOX2sdBjOyVvaX64LW6ezYf6igO+U9uN66lEqteVIN4ZD4Fhx6Uml6+tLWrWSqX1NdDZFwyLe1Dm3ZvEqV23IAOO/kaYgUtlp6xQs7OW3WZGa1VG7e/6ATEVoa/FvrkL24seBgqp2I+LrWob0rRWNdTUXPEQcveC2e0zLh4LCts5vnt+3nIutSKrm4jwvhgpBXCSw4mCLxgoNPs5WSvRXfpZSxrHU6G3ceYFtn97iP/c16L7vrxTaFteT8zMwahNQZYMHBFEk8WudbZstKXgA31LJWb6xgIq2HFS/s5NSZzcyu4FtJVotYowWHoSw4mKJINNb51q3U1tVbsQvghpqXiDJ/WuO4g8PO/T088/o+W/jmk1iDf2m7qyo4iMhyEXlRRLaIyBdH2D5HRB4WkXUi8piIzMrZ9k0RWe9+rsgpP09EnnHld4lIjSufLCK/FJHnRWSDiHy8GCdqSivRGKGtq7egaZv5ak+mKnqNw1AXtE7nD692sL87/0R8D77gdSldZLOUfBGLRtjf00d/EVa0j6U9maI2LDTXV/aNOMcMDiISBm4DLgJagatEpHXIbt8C7lbVRcDNwK3u2EuAM4HTgbOA60SkWURCwF3Alap6KvAacLV7rmuBjap6GnAO8L9FJDh/CY5S8cYIvf0DJEucvGxgQN0c8WC0HMAbd0gPKI++uCfvYx5cv5OTjmni+KmNJayZyciMYXWOI4BPVGcyRUtDpOLvtZ1Py2EJsEVVX1HVFHAfcNmQfVqBR9zjR3O2twKrVbVfVZPAOmA5EAdSqrrZ7bcKeL97rECTeO9cI9AB9I/7zIyvMn+sSz0ova+nj/SABmZAGuC0WVOY1lTHyo353T5094FDrH2t0xa++cjPVdLtAVgdDfkFh5nAGzm/b3NluZ4HLneP34f3xz3uypeLSIOIJIBzgeOANqBGRBa7Yz7gygH+ETgZ2AG8AHxOVYe19UTkGhFZKyJr9+4tXmpkMzHZhXAlDg6Z4BOUMQeAUEg4/+TpPP7iXnr7x25ZPbRhF6rYFFYfZe+F7sNanSCkzoDiDUhfBywVkWeBpcB2IK2qK4EVwBrgXuBJV67AlcB3ROQp4CCQ+dZcCDwHzMDrjvpHEWke+oKqeoeqLlbVxVOnTi3SaZiJyixIayvxoPTgArjK/3LluqB1OslUmjUvt4+574oXdrJweiPzp1mXkl9imW6lZOm7laopOGxn8KoeYJYry1LVHap6uaqeAXzJle1z/96iqqer6jJAgM2u/ElVPVtVlwCrM+XAx4F/V88W4FXgpAmfofFF3Kfke5kru0pfHT3U206IE42Ex5y1tPdgL0+92mGzlHw22K3kT8shCFOx8wkOfwQWiMg8NzB8JfBA7g4iknCDzAA3Ane68rDrXkJEFgGLgJXu92nu3zrgBuCH7vjXgfPdtunAicArEz1B449s2uOSdytV/o3ZR1JfG2bpiVP57cbdDAyMPqProQ27GFDsdqA+a8l2K5X24qYvPcD+nr6Kz6sEeQQHVe0HPgM8BGwCfq6qG0TkZhF5r9vtHOBFEdkMTAduceW1wBMishG4A/iwez6A60VkE94g9S9VNTOg/XXg7SLyAvAwcIOqthV6oqa06mrCNNfXlPzL1d7VS0hgSkPlf7mGWtY6nT0He3l+275R93lw/U5OmBpl4XTrUvJTbThEc31NyQekMzfECsLFTV4TbVV1Bd7YQW7ZTTmP7wfuH+G4Q3gzlkZ6zuuB60co3wFckE+9TGVJNNaVfEC6zfXXhkOVPQ1wJOeeOI1wSFi1cTdnzG4Ztr29q5ffv9LBp885oeKnOVajeGNdyYNDUPIqga2QNkXkR/K9toO9gVrjkGtKQ4Qlc2Ojjjus2rib9IDaeEOZ+JFfKSiro8GCgykiL79SiccckqlArXEY6oJTpvPSni62tiWHbfv1CzuZG2/g5GObylAzY8HhcBYcTNHEGyMln8raHqC8SiNZ1jodGJ6IrzOZYs3L7Vz0pmOtS6lM/MivZMHBHJXijXV0dqdKmp+mvStYeZWGmtXSwMnHNg9bLb1qk9elZKuiyyfWGKEzmSppfrBMt2tLQ23JXqNYLDiYokk0RlAtXX6aQ31pDvb2B26Nw1DLWqfz9Gudh037ffCFnRwXm8QpM4at9zQ+iUcj9A8oBw6VLltPZ3eKKQ211IQr/09v5dfQBEY2v1KJxh0yTfIgTAM8kgtapzOg8PCfvER8+3v6+K8tbVx8qnUplZMf+ZXak6lsqo5KZ8HBFE2pV0lnF8AFvOVwyoxmZkyuz447/HbjbvrSarmUyqzFh1XSHV3BSJ0BFhxMEQ3mVyrNl6stc2P2AI85gHfP7WWt03nipb30pNI8uH4nM6dM4rRZk8tdtaNaPBscSpdfKSh5lcCCgymihE8th0RA1znkWtZ6DIf6Bnhw/U5Wb27jolOPsS6lMvMjv1JHd3CmYltwMEXTXF9LTUhKNubQ1lUdLQeAs46P0VRfw9+v+BOp9IB1KVWAwTGz0lzcqGr2Rj9BYMHBFE0oJMSipVsl3d7VS31tiIZIuCTP76facIhzT5xGW1cvxzTXc8ZxU8pdpaPepEiY+toQHSX6/B7o6ad/QK1byRyd4iXMr9Te5d0etFq6Xy44xVsQt/zUYwgFMFdUNYpH6+joLtHFTcDGzCw4mKJKlHCVdFsyRaIp+OMNGeedNI33njaDj75tTrmrYpxSptDIZGQNQtI9yDMrqzH5ikcjbG0fnjeoGNpdF0y1aIjU8L2rzih3NUyOUgaHTHerrXMwR6V4Y11JZysFpUlugqmUY2bZvEoB+QxbcDBFlWisozuVpjtV3BQEqkp7MthJ90zli0Uj2e6fYmsP2Ap/Cw6mqEq1SvrAoX760hqYL5YJplg0QncqzaG+dNGfuzOZoiESpr42GLPtLDiYosouhCtyv21mBlTQk+6Zypa5+CjFWoeOAK1xAAsOpsiyC4mKPJ11MK9ScL5cJniy+ZVKMO4QtBtVWXAwRVWqbqVMsAnqLUJNMGTzK5Vg3CFIeZXAgoMpsswf771Fbjm0uWZ+kG/0YypfKfMrdQQoXTdYcDBFNikSJhoJl6zlEKQrLxM8g92i1nKw4GCKLt5YV/Tke+1dKVoCcgctE1xN9TWEQ1L0hXA9qTQ9fenArHEACw6mBOKNxV9IZGscjB9CIaGlofhrHbJ5lazlYI5miRIk32vrSgXqi2WCK16CVdKd7gZCVTeVVUSWi8iLIrJFRL44wvY5IvKwiKwTkcdEZFbOtm+KyHr3c0VO+Xki8owrv0tEanK2nSMiz4nIBhF5vNCTNP5KNEZKss7B1jgYP5Qiv1LQMrJCHsFBRMLAbcBFQCtwlYi0DtntW8DdqroIuBm41R17CXAmcDpwFnCdiDSLSAi4C7hSVU8FXgOudsdMAX4AvFdVTwH+vOCzNL6KR+voSKYYGNCiPaflVTJ+iUUjRZ/Kms2rFKCp2Pm0HJYAW1T1FVVNAfcBlw3ZpxV4xD1+NGd7K7BaVftVNQmsA5YDcSClqpvdfquA97vHfwH8u6q+DqCqe8Z/Wqac4o0R0gPK/p7i3Is31T/A/p4+W+NgfFGKlsNgcAjOBU4+wWEm8EbO79tcWa7ngcvd4/cBTSISd+XLRaRBRBLAucBxQBtQIyKL3TEfcOUAC4EW1z31tIh8dKRKicg1IrJWRNbu3bs3j9MwfskMHBdr3CEzOGgtB+OHWDTCvu4++tMDRXvOjmSKmpDQXB+cuyQUa0D6OmCpiDwLLAW2A2lVXQmsANYA9wJPunIFrgS+IyJPAQeBTKarGuDNwCXAhcD/EpGFQ19QVe9Q1cWqunjq1KlFOg1TDAl3dVSsm/5YXiXjp8xFSGd3cVq+4PIqRSOBuothPmFsO4NX9QCzXFmWqu7AtRxEpBF4v6ruc9tuAW5x2+4BNrvyJ4GzXfkFeC0G8Fom7a4bKikiq4HTMseZypdpORRrrUNm5oitjjZ+yMwo6uxOMbVIdx5sTwZvtl0+LYc/AgtEZJ6IRPCu+B/I3UFEEm6QGeBG4E5XHnbdS4jIImARsNL9Ps39WwfcAPzQHf+fwDtFpEZEGvAGsjdN/BSN34qdX2lwpoe1HEzpZTOzFnE6a9BWR0MeLQdV7ReRzwAPAWHgTlXdICI3A2tV9QHgHOBWEVFgNXCtO7wWeMI1pQ4AH1bVzF1grheRS/EC1O2q+oh7vU0i8hu8wesB4Eequr44p2v80NIQISTFy8xqGVmNnzKrmIs5KN2ZTHHyjOaiPZ8f8hodUdUVeGMHuWU35Ty+H7h/hOMO4c1YGuk5rweuH2XbPwD/kE/dTOUJh4RYNJJNlleovV29RMIhmuqCM5hngqsUyfeqtVvJmHGLR+uK2nKINwZrMM8EV2bMoSNZnAHpvrQ3FTto3VC3NAkAABrxSURBVEoWHExJFDO/UntXr3UpGd/UhkM019cUreWQnYptwcGYTGbWYg1Ip2wBnPFVMT+/2bxKFhyM8a6S2g4Wt1vJGL+0NNQWbUA6M9vOupWMwVuTcLC3n0N96bF3PgJVpa2rl6k2jdX4KObygxVD5nmC1vq14GBKIrMmodAvWDKVprd/wFoOxlfxIuZXCmJeJbDgYEqkWAuJMjOegnbVZYIt1ujd8MfL9FOYTHCY0lBb8HP5yYKDKYmESzvQVuCMjzZbAGfKINYQoS+tHDjUP/bOY+hIppg8qZbagN3iNli1NYGRKNKN2i3pnimHTBdQZxG6loK4AA4sOJgSGcyvVFjLwVJnmHLIpNAoxnTWjq7g5VUCCw6mRBoiYeprQwV/uTLBJYhfLhNc8Wjx8it1dqcCt8YBLDiYEhER4tG6gtc6tCdTNNXXUFcTLlLNjBnbYAqNwtfqWLeSMUMkGgtPvtfW1WvjDcZ38cbi5FdSVToDmK4bLDiYEoo3Fp58r70rZTf5Mb5riNRQXxsquOVwoKef/gG14GBMrni08OR77cleW+NgyiIeLTy/Ukd3MBfAgQUHU0KJpjrak70FLSSyvEqmXFqihedX6ghoXiWw4GBKKB4tbCFRf3qAju6U3R7UlEUsWlfwOofsVOwAtn4tOJiSyQwkT3TcobO7D1VszMGURTwaKbxbKZNXKYCfYQsOpmTiBS4kyqQ6DuJVlwm+WBGS72XHHBosOBiTFY8W1nKw1dGmnGLRCN2pdEFp5zu6UkyqDTMpErx1OhYcTMlkuoP2TnDG0mBeJQsOxn+xIqyS7gjoGgew4GBKqCVaWH6lIA/mmeArRnBot+BgzHC14RBTGmonvNahPdlLTUiYPClYefBNdcjek6SA4NDZbcHBmBF5Mz4m3nKIRSOEQlLkWhkztpZo4fmV2ruCmVcJLDiYEks01mVv2DNebV29tsbBlM1gZtaJ51eq+jEHEVkuIi+KyBYR+eII2+eIyMMisk5EHhORWTnbviki693PFTnl54nIM678LhGpGfKcbxGRfhH5QCEnaMorUUB+pTbLq2TKqLm+lnBIJtxy6Eml6elLBzJdN+QRHEQkDNwGXAS0AleJSOuQ3b4F3K2qi4CbgVvdsZcAZwKnA2cB14lIs4iEgLuAK1X1VOA14Oohr/lNYGVhp2fKLd448YVEXl6lYH6xTPCFQkJLw8TXOmTWOAT1M5xPy2EJsEVVX1HVFHAfcNmQfVqBR9zjR3O2twKrVbVfVZPAOmA5EAdSqrrZ7bcKeH/O830W+DdgzzjPx1SYeLSOfd199KUHxn2sl1fJupVM+cQKyK/U0RXcpHuQX3CYCbyR8/s2V5breeBy9/h9QJOIxF35chFpEJEEcC5wHNAG1IjIYnfMB1w5IjLTPcftR6qUiFwjImtFZO3evXvzOA1TDpkFbOPNUdOd6qc7lbYFcKasClklnV3hH9DPcLEGpK8DlorIs8BSYDuQVtWVwApgDXAv8KQrV+BK4Dsi8hRwEMgsQ/wucIOqHvFSU1XvUNXFqrp46tSpRToNU2yDC+HG12+bmf6asDUOpowKSdudCSotAUydAVAz9i5sx13VO7NcWZaq7sC1HESkEXi/qu5z224BbnHb7gE2u/IngbNd+QXAQvd0i4H7RAQgAVwsIv2q+h8TOD9TZvFs8r3xfcEyX8hEUzC/WKY6FNJyyBwX1EWc+bQc/ggsEJF5IhLBu+J/IHcHEUm4QWaAG4E7XXnYdS8hIouARbhBZhGZ5v6tA24AfgigqvNUda6qzgXuBz5tgSG4BhcSjbflYEn3TPm1RCPs7+mjfwJjZh3JFDUhoXlSPtfglWfM4KCq/cBngIeATcDPVXWDiNwsIu91u50DvCgim4HpuJYCUAs8ISIbgTuAD7vnA7heRDbhDVL/UlUzA9qmiiSaJtZyyORVCmp/rakO8WgEVdjXM/61Dh3JFC3RCK4XJHDyCmmqugJv7CC37Kacx/fjXeUPPe4Q3oylkZ7zeuD6MV73Y/nUz1SuproaIuHQuBfCtVleJVMBcvMrJcY5c649mQpkqu4MWyFtSkpEvLUOExiQjkaCmerYVI9st+gEVvl3Bnh1NFhwMD6YyEK49qSlzjDll1nd3Nk9/uDQkUwF8g5wGRYcTMnFo+NPoeEtgAvuF8tUh0Iys7Yng5t0Dyw4GB/EGyMTGHPotfEGU3bZzKzj/Pz2pwfY39MX2DUOYMHB+MDLzNqLt/YxP+1JS7pnyq82HKK5vmbcyfc6u73ZTUFu/VpwMCUXj0bo7R8gmcrvXrwDAzqh2SHGlEIsGqGje3xTWTML4GxA2pgjGFwlnd/V176ePtIDGuirLlM9vFXS4xwzc/tbcDDmCDLdQ/mOO2RXR1vLwVSAWLRu3FNZO90Ngiw4GHMEiXG2HNqySfeC+8Uy1SM+gfxKHdZyMGZsme6hfKcDDqY6tpaDKb+WaITO7tS4J1RAcDOyggUH44NYdpVpfi2HTBPexhxMJYhHI/SllYO9/WPv7HQkU0yeVEttOLh/YoNbcxMYdTVhmuprxjXmIBLsqy5TPWITWOvQEfDUGWDBwfgks9YhH20uYVk4FMxslqa6xMbZLQoWHIzJWzwayXvGR3tXr3UpmYqRyaw6nlvdWnAwJk9e8r38xxxsAZypFLlpu/MV9LxKYMHB+CTRmP9c8bYuy8hqKsd4Z9upKp3uRj9BZsHB+CLeWEdHd4r0wNjTAdu7gn/VZapHQ6SG+tpQ3qukDxzqp39AA/8ZtuBgfJFo9G63OFZe/EN9aQ729lvSPVNRYg0ROpL55VeqhrxKYMHB+CSTfnusrqXMF8u6lUwliTXmn1+pGlZHgwUH45Nsv+0Y01mzC+AC/sUy1SUWrct7QLqjCvIqgQUH45Ns8r0xvmBtljrDVKB4NP9b3VrLwZhxyHQrtR3Mr+VgYw6mkrQ0RPJe55AJIkG/k6EFB+OLyZNqCYdkzLUOmW4nW+dgKkm8MUIyleZQ39g3rOroSlFfG2JSJOxDzUrHgoPxRSgkxPJYJd3W1Ut9bYiGgH+xTHUZz0K4ju5U4FsNYMHB+MjLr3TkL5e3xqEOEcurZCrHuIJDFaTOgDyDg4gsF5EXRWSLiHxxhO1zRORhEVknIo+JyKycbd8UkfXu54qc8vNE5BlXfpeI1LjyD7nneUFE1ojIacU4UVN+iTxSaLQlUzbeYCqOBYcRiEgYuA24CGgFrhKR1iG7fQu4W1UXATcDt7pjLwHOBE4HzgKuE5FmEQkBdwFXquqpwGvA1e65XgWWquqbgK8DdxR2iqZS5JN8r91SZ5gKNJ7g0N51lAQHYAmwRVVfUdUUcB9w2ZB9WoFH3ONHc7a3AqtVtV9Vk8A6YDkQB1Kqutnttwp4P4CqrlHVTlf+eyDbCjHBFm+sy2udg61xMJUm85nMZzprZ/fRExxmAm/k/L7NleV6HrjcPX4f0CQicVe+XEQaRCQBnAscB7QBNSKy2B3zAVc+1CeAB0eqlIhcIyJrRWTt3r178zgNU26ZGR89qZFnfKgq7UlrOZjK01zvzbYbazrrob403an0URMc8nEdsFREngWWAtuBtKquBFYAa4B7gSdduQJXAt8RkaeAg8BhfzFE5Fy84HDDSC+oqneo6mJVXTx16tQinYYppURmrcMorYcDh/rpS6uNOZiKEwoJLQ21Y7YcBtc4BP8znE9w2M7hV/WzXFmWqu5Q1ctV9QzgS65sn/v3FlU9XVWXAQJsduVPqurZqroEWJ0pBxCRRcCPgMtUtX3CZ2cqylipjzNdTnajH1OJYtGx8ytlbiUa9HTdkF9w+COwQETmiUgE74r/gdwdRCThBpkBbgTudOVh172U+YO/CFjpfp/m/q3Dax380P0+G/h34CM5YxKmCmS6i0Ybd2jLro62biVTebzgMEbiyO7qaTnUjLWDqvaLyGeAh4AwcKeqbhCRm4G1qvoAcA5wq4goXivgWnd4LfCEm7N+APiwqva7bdeLyKV4Aep2Vc0MaN+EN2D9A3dcv6pmxiZMgCWyyffGaDlUwQIiU33i0Tr+tOvAEfeplrxKkEdwAFDVFXhjB7llN+U8vh+4f4TjDuHNWBrpOa8Hrh+h/JPAJ/OplwmWbH6lUZrmmaR8NuZgKlFLtHbMlsNgVuHgX+DYCmnjm0mRMNFIeMyWQzX015rqE4vWsa+n74h3M+xIpgiHhKb6vK67K5oFB+OrI611aO9KMaWhltqwfSxN5YlHx76bYWd3ipaGCKFQ8NO/2LfQ+CreOHpe/PZkb1UM5JnqlBlHONJah2paxGnBwfgqHh09+V5bV8oWwJmKFctjlXS15FUCCw7GZ4nGyKiL4Nq7em0w2lSsfPIrWXAwZoLijd5c8YERBvXauqojD76pTvnkV+qokrxKYMHB+CwerSM9oOzv6TusPNU/wP6ePlsAZypWyxhjDv3pAfZ191lwMGYiEk1ulfSQtQ6ZGSCWOsNUqtpwiKb6mlG7lTq7vQueavkMW3Awvkq4q6qhg9Jt2XtHV8cXy1SneHT02XaZoNHSUB2fYQsOxleD+ZUO/4JlV5Zat5KpYEdKvtdRRRlZwYKD8dlgZtbDv2CZ36vli2WqUyxaR0eyb8RtmeAQq5LWrwUH46uWhggiw7uVrOVggiAWrT1Cy6F6ku6BBQfjs3BIiDUMX+vQ1pWiNiw0V0FOGlO9vJZDCu9+ZYdrtzEHYwoTb4wMy6/U1tVLPFqHS9NuTEWKRyP0pZWDvf3DtnUmUzTX11RNbrDqOAsTKPFo3QgD0r0kmqrjistUryPlV2pPVlf6FwsOxneJprph0wHbk7Y62lS+I+VXqqbUGWDBwZRBPDp8zKG9K1U1i4dM9crmVxoheWRHMlU14w1gwcGUQaIxwsFD/fT2pwFQVdq6ei11hql4R0q+15GsnnTdYMHBlEGmXzbzBUum0vT2D1TVF8tUp0zrtmPIDX9Ulc7uVNWscQALDqYMstktXdM8M3OpmgbzTHWaVBumriY0rOVw4FA/fWmtqgscCw7Gd5kgkBl3aOuypHsmGETEy680ZMyh2vIqgQUHUwaZ5HptQ1oOCZutZAIg1jg8v1K1pc4ACw6mDAaT71nLwQRPLFpHR/fh+ZWqLekeWHAwZRCNhKmvDWXnig+OOVTPF8tUr1jD8PxKmd+tW8mYAnj9tnXZMYf2ZIqm+hrqasJlrpkxY4tF64atc8hc6FTTBU5ewUFElovIiyKyRUS+OML2OSLysIisE5HHRGRWzrZvish693NFTvl5IvKMK79LRGpcuYjI99xrrRORM4txoqayJBoHB/VsjYMJknhjhGQqzaG+dLasM5mivjZEQ6R6EkeOGRxEJAzcBlwEtAJXiUjrkN2+BdytqouAm4Fb3bGXAGcCpwNnAdeJSLOIhIC7gCtV9VTgNeBq91wXAQvczzXA7QWdoalI8ca67D0c2ruqa/GQqW7Z/Eo5ax2qMf1LPi2HJcAWVX1FVVPAfcBlQ/ZpBR5xjx/N2d4KrFbVflVNAuuA5UAcSKnqZrffKuD97vFleIFGVfX3wBQROXYC52YqWO50wPZkb1U1x011y4wr5E5n7UimaInWlqtKJZFPcJgJvJHz+zZXlut54HL3+H1Ak4jEXflyEWkQkQRwLnAc0AbUiMhid8wHXHm+r4eIXCMia0Vk7d69e/M4DVNJ4o1eZlZVdXmVquuqy1Sv7Crp5OHBIXYUthzycR2wVESeBZYC24G0qq4EVgBrgHuBJ125AlcC3xGRp4CDQHrEZx6Fqt6hqotVdfHUqVOLdBrGL4nGCKn0APt7+ujoTpGwbiUTECN1K1VbXiWAfEZPtjN4VQ8wy5VlqeoOXMtBRBqB96vqPrftFuAWt+0eYLMrfxI425VfACzM9/VM8GWuvl7a04Wqpc4wwTE0/QtUX7puyK/l8EdggYjME5EI3hX/A7k7iEjCDTID3Ajc6crDrnsJEVkELAJWut+nuX/rgBuAH7rjHwA+6mYtvRXYr6o7CzhHU4Eyg3d/2nXQ+93GHExANNfXEg5JtlvpUF+a7lS66oLDmC0HVe0Xkc8ADwFh4E5V3SAiNwNrVfUB4BzgVhFRYDVwrTu8FnjC3frxAPBhVc3cX+96EbkUL0DdrqqZAe0VwMXAFqAb+Hjhp2kqTWbq6mYXHGwqqwmKUEhoaagdXMSZSZ1xtAUHAFVdgfdHO7fsppzH9wP3j3DcIbwZSyM95/XA9SOUK4PBxVSpTH6lF3cfPOx3Y4IgFo1kbxXaWaXBwVZIm7JocV+kzS44VNsccVPdYtFItlupvQrzKoEFB1MmteEQUxpq2dfdRzgkTJ5UXXPETXWLRSPZRZzZvEoWHIwpjsyVViwaIRSSMtfGmPwd1nLospaDMUWVmb5abV8qU/1i0Tr29fSRHvBuDxoOCc311dX6teBgyiYzCG0zlUzQxKMRVGFfd8pLndFQfa1fCw6mbDKD0LbGwQRNZnyhI5mivStFrMryKoEFB1NGmRaDtRxM0GRXSSdTVbk6Giw4mDLKtBis5WCCJptfKZmio7v60nWDBQdTRtkxhyr8YpnqZi0HY0romMmT3L/1Za6JMeMzxd3TYe/BXvZ191XdGgfIM32GMaVw2qzJ/PQTS3jHCYlyV8WYcYnUhGiqr+HlvV1AdU7HtuBgykZEOHuB3YvDBFM8GmHLHi84WLeSMcYYwAsIr7QlgepsOVhwMMaYCYhFI6T6B4Dqy6sEFhyMMWZCcruSrOVgjDEG8PIrZVjLwRhjDEA2ZUZzfQ214er7U1p9Z2SMMT7ItByqcaYSWHAwxpgJyb0fSTWy4GCMMRMQywaH6kz/YsHBGGMmIBMcqnGmElhwMMaYCckEh2qcqQSWPsMYYyYkWlfDDctP4vyTp5W7KiVhwcEYYybor885odxVKBnrVjLGGDNMXsFBRJaLyIsiskVEvjjC9jki8rCIrBORx0RkVs62b4rIevdzRU75+SLyjIg8JyL/JSLzXflsEXlURJ51z3dxMU7UGGNM/sYMDiISBm4DLgJagatEpHXIbt8C7lbVRcDNwK3u2EuAM4HTgbOA60Sk2R1zO/AhVT0duAf4siv/MvBzVT0DuBL4wcRPzxhjzETk03JYAmxR1VdUNQXcB1w2ZJ9W4BH3+NGc7a3AalXtV9UksA5Y7rYpkAkUk4EdY5QbY4zxST7BYSbwRs7v21xZrueBy93j9wFNIhJ35ctFpEFEEsC5wHFuv08CK0RkG/AR4Buu/KvAh135CuCzI1VKRK4RkbUisnbv3r15nIYxxph8FWtA+jpgqYg8CywFtgNpVV2J9wd+DXAv8CSQdsd8HrhYVWcBPwa+7cqvAn7iyi8Gfioiw+qpqneo6mJVXTx1qt1NzBhjiimf4LCdwat9gFmuLEtVd6jq5W6c4EuubJ/79xZVPV1VlwECbBaRqcBpqvoH9xQ/A97uHn8C+Lk79kmgHrCbDBtjjI/yCQ5/BBaIyDwRieANEj+Qu4OIJHKu7m8E7nTlYde9hIgsAhYBK4FOYLKILHTHLAM2ucevA+e7Y07GCw7Wb2SMMT4acxGcqvaLyGeAh4AwcKeqbhCRm4G1qvoAcA5wq4gosBq41h1eCzwhIgAHgA+raj+AiPwl8G8iMoAXLP6bO+ZvgH8Skc/jDU5/TFX1SHV8+umn20TktXGcd7ElgLYyvv5YrH6FsfoVxupXmFLWb85oG2SMv7smDyKyVlUXl7seo7H6FcbqVxirX2HKVT9bIW2MMWYYCw7GGGOGseBQHHeUuwJjsPoVxupXGKtfYcpSPxtzMMYYM4y1HIwxxgxjwcEYY8wwFhzG4BbyPSsiv3K/f8alLleXL2q049IuHflzIvLAaPuVqI7/4lKsrxeRO0WkdpTjrhaRl9zP1RVYP1/ewxHq9/9E5HmXMv5+EWkc5bgb3WfhRRG5sJLqJyJzRaQn5/37oV/1yyn/noh0HeG4srx/+dSvnO+fiPxERF7Nee3TRzmupN9fuxPc2D6Ht3o7kyn2d8CvgMfGOK7HpSP3w9A6/gvwYff4Hrwkh7fnHiAiMeArwGK8xYZPi8gDqtpZCfVz/HoPh9bv86p6AEBEvg18hsHEkLjyVrxsAacAM4DfishCVU1TfOOun/Nymd4/RGQx0DLaAWV+/8asn1O29w+4XlXvH+0AP76/1nI4AvFuWnQJ8KNMmao+q6pby1apIUap4wp1gKfw8mENdSGwSlU73AdqFYPp1Cuhfr4YpX6ZP7wCTML78g11GXCfqvaq6qvAFrz09pVSP1+MVD/x7gHzD8DfHuHQsr1/edbPFyPVL08l//5acDiy7+J9gAYmcGy9eCnFfy8if1bkeuUatY6uu+YjwG9GOC6fVOzlrB/48x6OWD8R+TGwCzgJ+P4Ix5X1/cujfgDzXHfF4yJydgnqNlr9PgM8oKo7j3BcOd+/fOoH5Xv/AG5x3YbfEZG6EY4r+ftnwWEUInIpsEdVn57gU8xxS97/AviuiBT9TuR51PEHeDdbeqLYr52PItSvpO/hkeqnqh/H6+7YBFwxdLsfCqzfTmC2y5T8BeAeGbwLY8nqJyIzgD9n9IDlmwLrV5b3z7kRL+i/BYgBNxTzdfNlwWF07wDeKyJb8e5+d56I/HO+B6vqdvfvK3jjE2f4WUcR+QowFe+DPZIxU7GXuX5+vIdH/D92/d/3Ae8f4diyvn9j1c9117S7x08DLwMLh+5X7PoBG4D5wBZX3iAiW0Y4tizvX771K9f7JyL/rKo7Xa9rL969bkbqbiv9+6eq9jPGD17W2V8NKdsKJEbZvwWoc48TwEtAq191xBvgXQNMOsL+MeBVV9cW9zhWQfXz9T3M1A/vniPzXZng3R/9WyPsfwrenQ7rgHnAK0C4guo3NVMf4Hi8Pxy+/P8OKe8aZf+yvH/jqF/Z3j/g2Jz/3+8C3xhh/5J/f63lME4i8t/Fu4XpLGCdiPzIlS/OPAZOBtaKyPN499T+hqpu9LGaPwSmA0+6qXA3Da2jqnYAX8e7X8cfgZtdWUXUj/K9hwLcJSIvAC8AxwI3u/q9V7xU9ajqBrybUm3EGzO5Vksz02ZC9QPehff5fA64H/iUj/+/I6qQ9y+v+lHe9+9fcv5/E8Dfufr5+v219BnGGGOGsZaDMcaYYSw4GGOMGcaCgzHGmGEsOBhjjBnGgoMxxphhLDgYY4wZxoKDMcaYYf4/u7STvC1Xi/IAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for f, label in zip(features_test[0].T,\n", + " ['crossing_angle', 'dip_angle', 'drift_length', 'pad_coordinate']):\n", + " plt.figure()\n", + " bins = np.linspace(f.min(), f.max() + 1e-5, 21)\n", + " scores = []\n", + " for left, right in zip(bins[:-1], bins[1:]):\n", + " selection = (f >= left) & (f < right)\n", + " scores.append(\n", + " roc_auc_score(targets_test[selection], preds_test[selection])\n", + " )\n", + " scores = np.array(scores)\n", + " plt.plot(0.5 * (bins[:-1] + bins[1:]), scores)\n", + " plt.title(label)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAATwklEQVR4nO3de4zV5ZnA8e+zCiXxtg2QtWFQ2KJUpG5kB5XaGNtdW6iijZddaOOlcSXWsu02TVw1293GZdPoHyaLN5ashq0xWKO2AaXVJd42Rg2joZVLkFm2rUOaOOIGRUVlffaPOcB0mGF+M+ecOcw7309Ccs77uz3z4/DMy/N7z/tGZiJJKssftToASVLjmdwlqUAmd0kqkMldkgpkcpekAh3d6gAAJk2alNOmTWt1GJI0qrzyyitvZebk/rYdEcl92rRpdHR0tDoMSRpVIuK3A22zLCNJBTK5S1KBTO6SVKAjouYuScP18ccf09XVxd69e1sdStNMmDCBtrY2xo0bV/kYk7ukUa2rq4vjjjuOadOmERGtDqfhMpNdu3bR1dXF9OnTKx/X0rJMRCyMiJW7d+9uZRiSRrG9e/cyceLEIhM7QEQwceLEIf/PpKXJPTPXZuaSE044oZVhSBrlSk3s+w3n5/OBqiQVaNTX3F9efiVnf/eBVoch6Qhx82OvNfR8P7708w09X3+uueYaLrroIi6//PKGndOeuyQ1UGbyySeftDoMk7sk1es3v/kNM2fO5KqrrmL27Nk88MADzJs3jzlz5nDFFVewZ88eAG699Vbmzp3L7NmzWbJkCc1cCc/kLkkNsH37dm644Qaee+457rvvPtavX8+rr75Ke3s7d9xxBwBLly5lw4YNbNq0iQ8++IDHH3+8afGM+pq7JB0JTj75ZM455xwef/xxtmzZwrnnngvARx99xLx58wB45plnuP3223n//fd5++23Of3001m4cGFT4ml4co+I84F/BjYDD2Xms42+hiQdaY455higp+Z+wQUXsHr16j/YvnfvXm644QY6OjqYOnUqP/rRj5r6rdpKZZmIuD8i3oyITX3a50fEtojojIibas0J7AEmAF2NDVeSjmznnHMOL7zwAp2dnQC89957vP766wcS+aRJk9izZw+PPPJIU+Oo2nNfBdwF/GR/Q0QcBdwNXEBPEt8QEWuA/8rM5yLiT4A7gG82NGJJOoyRGLp4OJMnT2bVqlUsXryYDz/8EIBly5Zx6qmnct111zF79mxOPPFE5s6d29Q4KiX3zHw+Iqb1aT4L6MzMHQAR8RBwSWZuqW3/X+BTA50zIpYASwBOOumkoUUtSUeQadOmsWnTwcLGl7/8ZTZs2HDIfsuWLWPZsmWHtK9atarhMdUzWmYK8Eav913AlIi4NCL+DXiAnt5+vzJzZWa2Z2b75Mn9rhIlSRqmhj9QzczHgMeq7BsRC4GFM2bMaHQYkjSm1dNz3wlM7fW+rdZWmROHSVJz1JPcNwCnRMT0iBgPLALWDOUETvkrSc1RdSjkauBFYGZEdEXEtZm5D1gKPAlsBR7OzM1Dubg9d0lqjqqjZRYP0L4OWDfci1tzl6TmaOn0A5m5Fljb3t5+XSvjkFSQtd9r7PkW/uuguyxfvpx7772XOXPm8OCDDx6yfdWqVXR0dHDXXQMOIGy4liZ3e+6SSnDPPfewfv162traWh3KAS6zJ0l1uP7669mxYwcLFizgtttuY968eZx55pl84QtfYNu2bYfs/8QTTzBv3jzeeusturu7ueyyy5g7dy5z587lhRdeaFhczgopSXVYsWIFv/zlL3nmmWcYP348P/jBDzj66KNZv349t9xyC48++uiBfX/2s59xxx13sG7dOj796U/zjW98g+9///t88Ytf5He/+x1f/epX2bp1a0PisiwjSQ2ye/durr76arZv305E8PHHHx/Y9vTTT9PR0cFTTz3F8ccfD8D69evZsmXLgX3eeecd9uzZw7HHHlt3LJZlJKlBfvjDH/KlL32JTZs2sXbt2j+Y0vezn/0s7777Lq+//vqBtk8++YSXXnqJjRs3snHjRnbu3NmQxA6uxCRJDbN7926mTJkCHDoZ2Mknn8yjjz7KVVddxebNPV8J+spXvsKdd955YJ+NGzc2LBZr7pLKUmHoYrPceOONXH311SxbtowLL7zwkO2f+9znePDBB7niiitYu3Yty5cv5zvf+Q5nnHEG+/bt47zzzmPFihUNiSWauUDroBc/WHO/bvv27cM6x8vLr+Ts7z7Q2MAkjRpbt27ltNNOa3UYTdffzxkRr2Rme3/7W3OXpAJZc5ekApncJY16rSwvj4Th/Hwmd0mj2oQJE9i1a1exCT4z2bVrFxMmTBjScX6JSdKo1tbWRldXF93d3a0OpWkmTJgw5HlrnBVS0qg2btw4pk+f3uowjjiWZSSpQCZ3SSqQyV2SCmRyl6QCmdwlqUAtTe4RsTAiVu7evbuVYUhScZxbRpIKZFlGkgpkcpekApncJalAJndJKpDJXZIKZHKXpAI1JblHxDER0RERFzXj/JKkw6uU3CPi/oh4MyI29WmfHxHbIqIzIm7qtenvgYcbGagkqbqqPfdVwPzeDRFxFHA3sACYBSyOiFkRcQGwBXizgXFKkoag0mIdmfl8REzr03wW0JmZOwAi4iHgEuBY4Bh6Ev4HEbEuMz/pe86IWAIsATjppJOGG78kqR/1rMQ0BXij1/su4OzMXAoQEdcAb/WX2AEycyWwEqC9vb3MxQ8lqUWatsxeZq4abB/XUJWk5qhntMxOYGqv9221tsqcOEySmqOe5L4BOCUipkfEeGARsGYoJ3DKX0lqjqpDIVcDLwIzI6IrIq7NzH3AUuBJYCvwcGZuHsrF7blLUnNUHS2zeID2dcC64V7cmrskNYeLdUhSgVxmT5IKZM9dkgrkrJCSVCDLMpJUIMsyklQgyzKSVCCTuyQVyJq7JBXImrskFciyjCQVyOQuSQUyuUtSgXygKkkF8oGqJBXIsowkFcjkLkkFMrlLUoFM7pJUIJO7JBXIoZCSVCCHQkpSgSzLSFKBTO6SVCCTuyQVyOQuSQUyuUtSgUzuklSghif3iDgtIlZExCMR8e1Gn1+SNLhKyT0i7o+INyNiU5/2+RGxLSI6I+ImgMzcmpnXA38FnNv4kCVJg6nac18FzO/dEBFHAXcDC4BZwOKImFXbdjHwBLCuYZFKkiqrlNwz83ng7T7NZwGdmbkjMz8CHgIuqe2/JjMXAN8c6JwRsSQiOiKio7u7e3jRS5L6dXQdx04B3uj1vgs4OyLOBy4FPsVheu6ZuRJYCdDe3p51xCFJ6qOe5N6vzHwWeLbKvhGxEFg4Y8aMRochSWNaPaNldgJTe71vq7VV5sRhktQc9ST3DcApETE9IsYDi4A1QzmBU/5KUnNUHQq5GngRmBkRXRFxbWbuA5YCTwJbgYczc/NQLm7PXZKao1LNPTMXD9C+jjqGO1pzl6TmcLEOSSqQy+xJUoHsuUtSgZwVUpIKZFlGkgpkWUaSCmRZRpIKZHKXpAJZc5ekAllzl6QCWZaRpAKZ3CWpQCZ3SSqQD1QlqUA+UJWkAlmWkaQCmdwlqUAmd0kqkMldkgpkcpekAjkUUpIK5FBISSqQZRlJKpDJXZIKZHKXpAKZ3CWpQCZ3SSqQyV2SCnR0M04aEV8HLgSOB+7LzKeacR1JUv8q99wj4v6IeDMiNvVpnx8R2yKiMyJuAsjMn2fmdcD1wF83NmRJ0mCGUpZZBczv3RARRwF3AwuAWcDiiJjVa5d/qG2XJI2gysk9M58H3u7TfBbQmZk7MvMj4CHgkuhxG/CLzHy1v/NFxJKI6IiIju7u7uHGL0nqR70PVKcAb/R631Vr+1vgL4HLI+L6/g7MzJWZ2Z6Z7ZMnT64zDElSb015oJqZy4Hlg+0XEQuBhTNmzGhGGJI0ZtXbc98JTO31vq3WVokTh0lSc9Sb3DcAp0TE9IgYDywC1lQ92Cl/Jak5hjIUcjXwIjAzIroi4trM3AcsBZ4EtgIPZ+bmque05y5JzVG55p6ZiwdoXwesG87FrblLUnO4WIckFchl9iSpQPbcJalAzgopSQWyLCNJBbIsI0kFsiwzAm5+7LVWhyBpjDG5jyCTvKSRYs1dkgpkzV2SCmRZRpIKZHKXpAKZ3I9QPnyVVA8fqLaYSVxSM/hAtQUOl9AH2uYvAUlDYVlmpK39HvCHyfrmx14bdvI26Uvqj8ldkgpkch8hLy+/8pC2vr1ue+GSGsXkPgK+3nX7gdcv/8/bLYxE0lhReQ3VZnAN1cPrr2f/40s/f9j9+tsuaexxtEzBLPNIY1dLe+5jTe/yzP73P2+78ZD9qiRlE7ekw7Hm3iJ9E31VJnVJVZjcC9N3zPxwvjDVqP0ltY7JvQWG22vfzyQraTAm9xarN9EP10j9gvAXkdQaJndJKpDJXf2qZ74bSa3X8OQeEX8aEfdFxCONPnfJmlme2Z+k+0vWVduGc01/OUitUym5R8T9EfFmRGzq0z4/IrZFRGdE3ASQmTsy89pmBKv6NXL0TLPOIal+VXvuq4D5vRsi4ijgbmABMAtYHBGzGhqdJGlYKiX3zHwe6Dvj1VlAZ62n/hHwEHBJ1QtHxJKI6IiIju7u7soBl2wkSjND3a+/kk4jFhSxhy81Vz019ynAG73edwFTImJiRKwAzoyImwc6ODNXZmZ7ZrZPnjy5jjAkSX01/IFqZu7KzOsz87OZ+ePD7TsW1lAdTg+1FWPfm12LH8p57NVL9asnue8EpvZ631Zrq8xZISWpOepJ7huAUyJiekSMBxYBa4ZygrHQc69XMyYYa2TPuNHnanWvvdXXlxql6lDI1cCLwMyI6IqIazNzH7AUeBLYCjycmZuHcnF77pLUHFVHyyzOzM9k5rjMbMvM+2rt6zLz1Fp9/V+GenF77j16985bNdfMUPXuZQ+lx11vz3goI3UO9+UtqXSuxCRJBWppch8LPfd6euIDHdvK3v1wesGDHXO4nv9g2+q9tlQqe+6SVCBnhZSkAlmWOUIN9pC1kaWZkShd9Dd9QaOGaw53ZsuqyxEONR7pSGBZRpIKZFlGkgpkcpekAllzH0UOV2fvu220DZds1PmrbKtaa2/E1MaNPFYaCmvuklQgyzKSVCCTuyQVyOQuSQXygeooMVpmi2ymemaerOdLUUN9CLp/PpzBzuPcOGomH6hKUoEsy0hSgUzuklQgk7skFcjkLkkFcrTMKNN3KuD9f3q/r+ecI63q1/8HGmky3NEkg42oGWh7vSNxBjq2ypQIVUbbODWC9nO0jCQVyLKMJBXI5C5JBTK5S1KBTO6SVCCTuyQVyOQuSQUyuUtSgY5u9Akj4hjgHuAj4NnMfLDR15AkHV6lnntE3B8Rb0bEpj7t8yNiW0R0RsRNteZLgUcy8zrg4gbHK0mqoGpZZhUwv3dDRBwF3A0sAGYBiyNiFtAGvFHb7f8aE6YkaSgqJffMfB54u0/zWUBnZu7IzI+Ah4BLgC56Evxhzx8RSyKiIyI6uru7hx65Kuk7b8xw558ZDYa7slHV+W36tg8010x/+x2urff8Mv297hvH4fYbisFiqHpsPcbSXDiNngtoMPU8UJ3CwR469CT1KcBjwGURcS+wdqCDM3NlZrZnZvvkyZPrCEOS1FfDH6hm5nvAt6rsGxELgYUzZsxodBiSNKbV03PfCUzt9b6t1laZs0JKUnPUk9w3AKdExPSIGA8sAtYM5QTO5y5JzVF1KORq4EVgZkR0RcS1mbkPWAo8CWwFHs7MzUO5uD13SWqOSjX3zFw8QPs6YN1wL27NXZKaw5WYJKlArqEqSQWy5y5JBYrMbHUMREQ38NthHj4JeKuB4Yxm3ouDvBcHeS8OKu1enJyZ/X4L9IhI7vWIiI7MbG91HEcC78VB3ouDvBcHjaV74XzuklQgk7skFaiE5L6y1QEcQbwXB3kvDvJeHDRm7sWor7lLkg5VQs9dktSHyV2SCjRqkvsA67X23v6piPhpbfvLETFt5KMcGRXuxTUR0R0RG2t//qYVcTbbQGv79toeEbG8dp9+HRFzRjrGkVLhXpwfEbt7fSb+caRjHCkRMTUinomILRGxOSK+188+5X82MvOI/wMcBfw38KfAeOBXwKw++9wArKi9XgT8tNVxt/BeXAPc1epYR+BenAfMATYNsP1rwC+AAM4BXm51zC28F+cDj7c6zhG6F58B5tReHwe83s+/keI/G6Ol5z7Qeq29XQL8R+31I8BfRESMYIwjpcq9GBOy/7V9e7sE+En2eAn444j4zMhEN7Iq3IsxIzN/n5mv1l6/S8+U5FP67Fb8Z2O0JPeB1mvtd5/smWt+NzBxRKIbWVXuBfSsY/vriHgkIqb2s30sqHqvxop5EfGriPhFRJze6mBGQq08eybwcp9NxX82Rkty19CsBaZl5hnAf3LwfzQau16lZx6SPwPuBH7e4niaLiKOBR4F/i4z32l1PCNttCT3Kuu1HtgnIo4GTgB2jUh0I2vQe5GZuzLzw9rbfwf+fIRiO9LUvc5vKTLznczcU3u9DhgXEZNaHFbTRMQ4ehL7g5n5WD+7FP/ZGC3Jvcp6rWuAq2uvLweeztqTk8IMei/61A4vpqfmOBatAa6qjYw4B9idmb9vdVCtEBEn7n8GFRFn0fNvv8TOD7Wf8z5ga2beMcBuxX82Ki2z12qZuS8i9q/XehRwf2ZujohbgY7MXEPPX+YDEdFJz4OlRa2LuHkq3ovvRsTFwD567sU1LQu4iWpr+54PTIqILuCfgHEAmbmCniUgvwZ0Au8D32pNpM1X4V5cDnw7IvYBHwCLCu38AJwLXAm8FhEba223ACfB2PlsOP2AJBVotJRlJElDYHKXpAKZ3CWpQCZ3SSqQyV2SCmRyl6QCmdwlqUD/D1yK5XnWAjVQAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "_, bins, _ = plt.hist(Y_test[:,-2:,:].ravel(), bins=300, alpha=0.6, label='real')\n", + "plt.hist(Y_test_fake[:,-2:,:].ravel(), bins=bins, alpha=0.6, label='fake')\n", + "plt.yscale('log')\n", + "plt.legend();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAAPwCAYAAAD0+SwIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzde5TV5Z3v+c+HKoqbKBJBuSsB8YLEaLxg92m7O/GYbu1Oa4smOelokpWMOZOLyTJj1pq1TDpL4/RMEjOOawVHZ3nSpztnCKOTExPjaRNdfbyAtATEC1G5igiIIlJcBKn6zh/7V8nWsfhutNj72cX7tdZesGp/fr/9VPHspz786gePI0IAAABAqYa0egAAAADAgVBYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwHmK2w/aMVo8DaDXeC2gW5hpQM5jeCxRWACnbV9l+uNXjAIDDBevuW1FYE7Y7Wz0G4FBijqM0zEkMdszxg0dhfQe219m+zvYKSbtsT7V9l+2tttfa/kpd9mzbi2xvt73J9q22u1o4fLQZ22fYXma72/ZC2wts31A9d7Ht5dX8etT2nLrj1tm+1vYK269Xxw2vez47tn6Od9r+pu3V1TiesX1JlT1Z0nxJc23vtL29+vgw29+z/YLtLbbn2x5R9xrfqN4TL9n+7CH/QqKtse6imVh321BE8HjbQ9I6ScslTZE0StJSSddL6pI0XdIaSRdW2TMlnSupU9LxklZKuqbuXCFpRqs/Jx5lPqo5tV7SVyUNlXSppH2SbpD0QUkvSzpHUoekK6u5Oaw6dp2kJZImShpbzb2rq+caObZvjo+oPjavOtcQSVdI2iVpQvXcVZIeftvYb5b08+q1R0u6R9JN1XMflbRF0uzqPfQT3gs8DvRg3eXRrAfrbns+uMLav1siYoNqf/DjIuI7EbEvItZIul3SxyUpIpZGxOKI2B8R6yTdJun8lo0a7abvm+4tEfFmRNyt2mIoSV+QdFtEPBYRPRHxY0l7q2P63BIRL0XENtUWrtMP8tgNEbFHkiJiYXWu3ohYIOl5SWe/06Btu3qNr0XEtojolvRdVe8LSZdLujMinoqIXZK+/a6/QjicsO6iGVh32xD3UPRvQ/XrNEkT+y7HVzokPSRJtk+U9ANJH5I0UrWv6dImjhPtbaKkjVH99bhSP/eutP3luue6qmP6bK77/e665xo5dkPd72X705K+rtoVK0k6QtIx/Yx7nGrzfWltDa2dQrX3Rt/nVf8+WN/PeYB6rLtoBtbdNkRh7V/fRN4gaW1EzOwn9yNJyyR9IiK6bV8j6bJmDBCDwiZJk2y7bvGcImm1anPvxoi48V2ct5Fjf79Y256m2hWsD0taFBE9tperthi+JVt5RdIeSadGxMZ3OPem6vPoM/Ugx4/DE+sumoF1tw1xS0BuiaTu6kbpEbY7bM+2fVb1/GhJOyTttH2SpC+2bKRoR4sk9Uj6UnUD/sf0hx8H3S7patvnuGaU7Ytsj27gvAd77CjVFsetkmT7M6r9WLbPFkmT+/5hS0T0Vq9xs+3x1TGTbF9Y5X8q6Srbp9geKelbjX05AEmsuzi0WHfbEIU1ERE9ki5W7R6Vtar9DecOSUdVkWslfVJSt2oTaUELhok2FRH7VLvh/3OStkv6lKRfSNobEY9L+rykWyW9JmmVajfhN3Legzo2Ip6R9H3VFvItkk6T9Ehd5AFJT0vabPuV6mPXVeddbHuHpF9LmlWd71eSflgdt6r6FWgI6y4OJdbd9uS33sIBoNVsPyZpfkTc2eqxAMDhgHW3fFxhBVrM9vm2j6t+NHWlpDmS7mv1uABgsGLdbT/8oyug9Wapdu/RKNX+r8nLImJTa4cEAIMa626b4ZYAAAAAFI1bAgAAAFC0A94ScMGQeVx+xXt2f+9C56mBxdzFQGDuol01e+4ybzEQDjRvucIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0TpbPYBDYfel56SZTX+7L81cfNKTaWbSsNfSzL2bZqeZbb+YlGaO+98fTTNob3v/8qw0s/6ySDOXfuC3aWaI8/Os3HFcmlnzmxPSzOTf7EozfvSJNINy7fjkuWlm8wVvpplpk15NM7v2daWZbavGppmp9/Wkma77/i3NoH31nv/BNPPCR4anmX2T807RiCNXDEszk/4lf4/0PP3sQAynKFxhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0QblTleb5uY9/EunP5hm/scxq9PMUHekmWM7X08z3zr50jQzcc5JaaZ3xe/SDJpvyOz8z06SXrgon7s3/dFP08wlR7ycZoZ5aJr5zVH5/L72Q/PSzKsvH51mjmEjtyLt/Yt89zVJev3SnWnmttMXpJl/PzLfDeuF/flr/WDCn6aZe/fkn9sJ96URFKjj5JkN5Z77ZF6D/uHPfpJmLj8i/z6/Yt8baebHc85LM//Ske8qN/HpNNJ2uMIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiDcqdro5alWcWbjhjQF5ruPNdWR7Ylu9yNGRP/neHnlHD0ozTBFrBvb0N5UZszHeW+seX5qaZV499Ms28tn9UmvnlxlPTzBvLxqaZ4x/ZlmYa+wqh2YZv3t1QrmflkWnm74/4qzSzYEy+S9vvto9PM68uOTbNvP+u7WmGedmeelY+31BuzJP5zlLffl8+b38x6YU08/z2cWlm61P53J553+G5nnKFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGiOiH6fvGDIvP6fPAzEH52eZnZPyLdL7enKN0wdvf6NNONHlqeZEt3fu7DpO8a269ztmDUjzbw5fnSaic78Sz5kX755X7vOuYHC3EW7avbcZd5iIBxo3nKFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARets9QBK1sguP6OaMA4cPnqeXZVmhjzbhIEAAFAQrrACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNEdEq8cAAAAA9IsrrAAAACgahfUQsx22Z7R6HECr8V5AszDXgJrB9F6gsAJI2b7K9sOtHgcAHC5Yd9+Kwpqw3dnqMQCHEnMcpWFOYrBjjh88Cus7sL3O9nW2V0jaZXuq7btsb7W91vZX6rJn215ke7vtTbZvtd3VwuGjzdg+w/Yy2922F9peYPuG6rmLbS+v5tejtufUHbfO9rW2V9h+vTpueN3z2bH1c7zT9jdtr67G8YztS6rsyZLmS5pre6ft7dXHh9n+nu0XbG+xPd/2iLrX+Eb1nnjJ9mcP+RcSbY11F83EutuGIoLH2x6S1klaLmmKpFGSlkq6XlKXpOmS1ki6sMqeKelcSZ2Sjpe0UtI1decKSTNa/TnxKPNRzan1kr4qaaikSyXtk3SDpA9KelnSOZI6JF1Zzc1h1bHrJC2RNFHS2GruXV0918ixfXN8RPWxedW5hki6QtIuSROq566S9PDbxn6zpJ9Xrz1a0j2Sbqqe+6ikLZJmV++hn/Be4HGgB+suj2Y9WHfb88EV1v7dEhEbVPuDHxcR34mIfRGxRtLtkj4uSRGxNCIWR8T+iFgn6TZJ57ds1Gg3fd90b4mINyPibtUWQ0n6gqTbIuKxiOiJiB9L2lsd0+eWiHgpIraptnCdfpDHboiIPZIUEQurc/VGxAJJz0s6+50GbdvVa3wtIrZFRLek76p6X0i6XNKdEfFUROyS9O13/RXC4YR1F83AutuGuIeifxuqX6dJmth3Ob7SIekhSbJ9oqQfSPqQpJGqfU2XNnGcaG8TJW2M6q/Hlfq5d6XtL9c911Ud02dz3e931z3XyLEb6n4v25+W9HXVrlhJ0hGSjuln3ONUm+9La2to7RSqvTf6Pq/698H6fs4D1GPdRTOw7rYhCmv/+ibyBklrI2JmP7kfSVom6RMR0W37GkmXNWOAGBQ2SZpk23WL5xRJq1WbezdGxI3v4ryNHPv7xdr2NNWuYH1Y0qKI6LG9XLXF8C3ZyiuS9kg6NSI2vsO5N1WfR5+pBzl+HJ5Yd9EMrLttiFsCckskdVc3So+w3WF7tu2zqudHS9ohaaftkyR9sWUjRTtaJKlH0peqG/A/pj/8OOh2SVfbPsc1o2xfZHt0A+c92GNHqbY4bpUk259R7ceyfbZImtz3D1siord6jZttj6+OmWT7wir/U0lX2T7F9khJ32rsywFIYt3FocW624YorImI6JF0sWr3qKxV7W84d0g6qopcK+mTkrpVm0gLWjBMtKmI2KfaDf+fk7Rd0qck/ULS3oh4XNLnJd0q6TVJq1S7Cb+R8x7UsRHxjKTvq7aQb5F0mqRH6iIPSHpa0mbbr1Qfu64672LbOyT9WtKs6ny/kvTD6rhV1a9AQ1h3cSix7rYnv/UWDgCtZvsxSfMj4s5WjwUADgesu+XjCivQYrbPt31c9aOpKyXNkXRfq8cFAIMV62774R9dAa03S7V7j0ap9n9NXhYRm1o7JAAY1Fh32wy3BAAAAKBo3BIAAACAoh3wloALhszj8ives/t7FzpPDSzmLgYCcxftqtlzl3mLgXCgecsVVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIrW2eoBHIz9f35mQ7n1Fw1NM13TdqaZffvyL0/H6hFpZsLi/Wlm2C//Lc1g8Nv6xblpZufU/Dy9Dbyzj16ZZ8Y9ti3N9Dz9bH4iQNKrn8/n966JTjOjNkaaed8dixoaEw5vHaecmGa2nTE2zbwxNr/+18i6PHJLb5o56p8X5ycahLjCCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAorXVTldbzhrWUG7OWc+nmeun/CLNnD4sf71Nf5zvmPVPl3wgzdz2yT9JMxMX5jt4jfjZkjSD5nv5P57XUO6Yv92QZn54wj1p5riOXWlmc8+oNPPL109PM3evzDPDfzsyzUz83qNpBs23+auNzd2j/nJTmvnBzB+lmXOGvZlmHtubr4XXX/43aea1eyemmeN+yLxsR7suO6eh3Mt/+0aa+Q+n/Pc0c/lRj6eZLue7WC14Pd/R866P553CP3tfmnnf/9Veu8FxhRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEVrq52ujlyX7xIhSc9sPi7NPHLMjDQztmNlmhlup5mzRqxNM/827fg08/Qps9LM5J+lEbSAIxrKvbhtTJp5/NjpaeYTR65IMycOz8dzpB9LM9ven++Y9Zutp+Uvhqbzmaemmd0TG5u7JwzfnWbejI40MyxfUvVHw/LvBX83ZXGaueHUv0ozx85tYFehRU+kGTRX5+7G+oKcz+8Thr2cZqYPzXdfG+Y8c96ofKfOu5TPyX1HNfBGajNcYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACia4wBbRl4wZF5je/K1oe4rzk0zr3wg39rszXH78xfryc8zam2+S+6kB7vz11ryZJ5psvt7FzZ9j7h2nbsdM/NtVzddkG89vPP4/NOPSW+kmZ6d+bwc/Vy+3eDke7fmr7Uy35Kw2Zi7UvzR6Q3l1v7ViDTjafn2rb2Rf8mjgTW1c32+9/C0e/Px+NH23Ha12XO3tHnbqO1/NzfPXLwrzZw6YVOa2b2/K808+/zENDPhgXyL49EL8q2JS3SgecsVVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFC3fxmaQamQXiNELmjAQoE7P82vSzPhGMgMxmAHU0+oB4F3zI8sbyk1/5BAPBDgExvznRQ1k8vPke2E15kRtHKAzDT5cYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBojohWjwEAAADoF1dYDzHbYXtGq8cBtBrvBTQLcw2oGUzvBQorgJTtq2w/3OpxAMDhgnX3rSisCdudrR4DcCgxx1Ea5iQGO+b4waOwvgPb62xfZ3uFpF22p9q+y/ZW22ttf6Uue7btRba3295k+1bbXS0cPtqM7TNsL7PdbXuh7QW2b6ieu9j28mp+PWp7Tt1x62xfa3uF7der44bXPZ8dWz/HO21/0/bqahzP2L6kyp4sab6kubZ32t5efXyY7e/ZfsH2FtvzbY+oe41vVO+Jl2x/9pB/IdHWWHfRTKy7bSgieLztIWmdpOWSpkgaJWmppOsldUmaLmmNpAur7JmSzpXUKel4SSslXVN3rpA0o9WfE48yH9WcWi/pq5KGSrpU0j5JN0j6oKSXJZ0jqUPSldXcHFYdu07SEkkTJY2t5t7V1XONHNs3x0dUH5tXnWuIpCsk7ZI0oXruKkkPv23sN0v6efXaoyXdI+mm6rmPStoiaXb1HvoJ7wUeB3qw7vJo1oN1tz0fXGHt3y0RsUG1P/hxEfGdiNgXEWsk3S7p45IUEUsjYnFE7I+IdZJuk3R+y0aNdtP3TfeWiHgzIu5WbTGUpC9Iui0iHouInoj4saS91TF9bomIlyJim2oL1+kHeeyGiNgjSRGxsDpXb0QskPS8pLPfadC2Xb3G1yJiW0R0S/quqveFpMsl3RkRT0XELknfftdfIRxOWHfRDKy7bYh7KPq3ofp1mqSJfZfjKx2SHpIk2ydK+oGkD0kaqdrXdGkTx4n2NlHSxqj+elypn3tX2v5y3XNd1TF9Ntf9fnfdc40cu6Hu97L9aUlfV+2KlSQdIemYfsY9TrX5vrS2htZOodp7o+/zqn8frO/nPEA91l00A+tuG6Kw9q9vIm+QtDYiZvaT+5GkZZI+ERHdtq+RdFkzBohBYZOkSbZdt3hOkbRatbl3Y0Tc+C7O28ixv1+sbU9T7QrWhyUtioge28tVWwzfkq28ImmPpFMjYuM7nHtT9Xn0mXqQ48fhiXUXzcC624a4JSC3RFJ3daP0CNsdtmfbPqt6frSkHZJ22j5J0hdbNlK0o0WSeiR9qboB/2P6w4+Dbpd0te1zXDPK9kW2Rzdw3oM9dpRqi+NWSbL9GdV+LNtni6TJff+wJSJ6q9e42fb46phJti+s8j+VdJXtU2yPlPStxr4cgCTWXRxarLttiMKaiIgeSRerdo/KWtX+hnOHpKOqyLWSPimpW7WJtKAFw0Sbioh9qt3w/zlJ2yV9StIvJO2NiMclfV7SrZJek7RKtZvwGznvQR0bEc9I+r5qC/kWSadJeqQu8oCkpyVttv1K9bHrqvMutr1D0q8lzarO9ytJP6yOW1X9CjSEdReHEutue2JrVqAwth+TND8i7mz1WADgcMC6Wz6usAItZvt828dVP5q6UtIcSfe1elwAMFix7rYf/tEV0HqzVLv3aJRq/9fkZRGxqbVDAoBBjXW3zXBLAAAAAIrGLQEAAAAo2gFvCbhgyDwuv+I9u793ofPUwGLuYiAwd9Gumj13mbcYCAeat1xhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGidrR7AobDrb89JMy+dn5/n/ae+lGaOP2JbmnnilYlpZuej4/LXWrglzfQ8tzrNoFz7//zMNLPur4emmeNOejnNjBz6ZppZs2JSmhm/JI3oyP+yOA+hre35m7PTzOZzOtJM16wdaWbMyD1pZuOLY9PM5Hvz8Yy8+7E0g/J0zJzeUG7Lnx+bZrpPyM+zf+LePBQNDOj1fH0fv8Rp5qh/GnxrLldYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQtLba6Wr3pfkOVpL0yhW708z1c/5bmrn8iBfTzMghXWnmkXG9aea2sX+aZh6aclKamb5wTJqRpM7fLG0oh+ZqZBerT/3ZQ2nmr49clmbOHJbP3f80aXya+YfJF6aZ3qFz08yYf1yUZtB8ja67my7Ld/r56gceTDOfOnJlmjm6Y2Sa+c3MfBerv5/y12lmwynnpZnj734lzUhSzzPPNZTDe7d/3OiGctvO3Zdm/n7uf00znz6ysTmQ+eXu4WnmphP/Ms1sGZvP22NvebShMZWCK6wAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAAChaW+10dcSa7oZyLz9/VJr5T2PyXSBePO53aWZqV767xYv73pdm1rx+TJoZvjn/4xq2+fU0I0k9DaXQbEeuyv8O+X9PODPNvPb+fCegxSM3p5lfvTw7zfQ+d0SaGbsin5f5fnBohRGb32go1/tqvkPP6jfGpZnNo/KdroY6H1OH8vfAEV357lyvNrBYsoNVeTpfb2zedm0cm2Z+t2dimnl51Po0M8z5+r51f7674PbdI9LM0S8Ovu/yXGEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAomiOi3ycvGDKv/ychSeo48f1pJkYOSzO9y58ZiOEU6f7ehW72ax7uc9cfyrdU9f58M9TBPC8bwdxt3JDTT0kz207Lt83edlr+Wp6yO830Rv5H17kq3+Ly+J/nW4LH40+lmWZr9txt13nbiFe+MDfNvD4zP0/viHzNHb6pI81MfmBXmvGiJ/IBFehA85YrrAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKFpnqwfQ7nqeW93qIQD/P43svDNot6VBSzSyK9qY5fl5xgzAWAYS7xMc838uyjNNGMfhjiusAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFM0R0eoxAAAAAP3iCisAAACKRmE9xGyH7RmtHgfQarwX0CzMNaBmML0XKKwAUravsv1wq8cBAIcL1t23orAmbHe2egzAocQcR2mYkxjsmOMHj8L6Dmyvs32d7RWSdtmeavsu21ttr7X9lbrs2bYX2d5ue5PtW213tXD4aDO2z7C9zHa37YW2F9i+oXruYtvLq/n1qO05dcets32t7RW2X6+OG173fHZs/RzvtP1N26urcTxj+5Iqe7Kk+ZLm2t5pe3v18WG2v2f7BdtbbM+3PaLuNb5RvSdesv3ZQ/6FRFtj3UUzse62oYjg8baHpHWSlkuaImmUpKWSrpfUJWm6pDWSLqyyZ0o6V1KnpOMlrZR0Td25QtKMVn9OPMp8VHNqvaSvShoq6VJJ+yTdIOmDkl6WdI6kDklXVnNzWHXsOklLJE2UNLaae1dXzzVybN8cH1F9bF51riGSrpC0S9KE6rmrJD38trHfLOnn1WuPlnSPpJuq5z4qaYuk2dV76Ce8F3gc6MG6y6NZD9bd9nxwhbV/t0TEBtX+4MdFxHciYl9ErJF0u6SPS1JELI2IxRGxPyLWSbpN0vktGzXaTd833Vsi4s2IuFu1xVCSviDptoh4LCJ6IuLHkvZWx/S5JSJeiohtqi1cpx/ksRsiYo8kRcTC6ly9EbFA0vOSzn6nQdt29Rpfi4htEdEt6buq3heSLpd0Z0Q8FRG7JH37XX+FcDhh3UUzsO62Ie6h6N+G6tdpkib2XY6vdEh6SJJsnyjpB5I+JGmkal/TpU0cJ9rbREkbo/rrcaV+7l1p+8t1z3VVx/TZXPf73XXPNXLshrrfy/anJX1dtStWknSEpGP6Gfc41eb70toaWjuFau+Nvs+r/n2wvp/zAPVYd9EMrLttiMLav76JvEHS2oiY2U/uR5KWSfpERHTbvkbSZc0YIAaFTZIm2Xbd4jlF0mrV5t6NEXHjuzhvI8f+frG2PU21K1gflrQoInpsL1dtMXxLtvKKpD2STo2Ije9w7k3V59Fn6kGOH4cn1l00A+tuG+KWgNwSSd3VjdIjbHfYnm37rOr50ZJ2SNpp+yRJX2zZSNGOFknqkfSl6gb8j+kPPw66XdLVts9xzSjbF9ke3cB5D/bYUaotjlslyfZnVPuxbJ8tkib3/avGZkQAACAASURBVMOWiOitXuNm2+OrYybZvrDK/1TSVbZPsT1S0rca+3IAklh3cWix7rYhCmsiInokXazaPSprVfsbzh2Sjqoi10r6pKRu1SbSghYME20qIvapdsP/5yRtl/QpSb+QtDciHpf0eUm3SnpN0irVbsJv5LwHdWxEPCPp+6ot5FsknSbpkbrIA5KelrTZ9ivVx66rzrvY9g5Jv5Y0qzrfryT9sDpuVfUr0BDWXRxKrLvtyW+9hQNAq9l+TNL8iLiz1WMBgMMB6275uMIKtJjt820fV/1o6kpJcyTd1+pxAcBgxbrbfvhHV0DrzVLt3qNRqv1fk5dFxKbWDgkABjXW3TbDLQEAAAAoGrcEAAAAoGgHvCXggiHzuPyK9+z+3oXOUwOLuYuBwNxFu2r23GXeYiAcaN5yhRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEU74E5XAAAAOLAhc05KM73Dh+YnWvLkAIxmcOIKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNHaamvWjpnTG8q9edxRaWbImz35iRavaOj1gEyjc3ff5DF55sj8bdu1Y38+pgd/29CYgEZsvXpumnnjI91p5t9NXZ1mhro3zSx9ZXKaefWJ8Wlm2q/eSDND/nVZmkGZhnzg5DTzuy+PSjMzTtiSZo4dkc//Jes/kGa8ZmT+WkvyjjPivy5JMyXhCisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpWzE5XG687L81M/8s1DZ3r21MXpJnZQyPNrNjXkWY27j86zbwZ+Xnu3npGPp4HTkwz0xdsSzOS1PP0sw3lkHv9U+emmS1/0sDOapI+f+5/TzMfGf1UQ+fKPL/vuDSzfNfUNHP3M6enmc71w9PMtHv3pBlJ8iPLG8qhud4Y5zTzkePzded/Gv9gmpnceUSaWX5M/l76P47+cJp5cMTsNDPjzXx3Iknyo080lMN713Hi+xvKrfwfRqeZf3fq79LMFePyXaMuGpnvmvaPY49JM7cd/Sdp5vWt+fo+Ik2UhSusAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoWjE7XXXsHbhz9UYjPTzfeWjW0HxQ5w7f0cBr5UZ3PJJmvjc3391ldcfEhl7vhP/n1DQTy55u6FyHu459ecZdvQ2da2fPsDQzZkj+gid05jtLnT3s1TRz/oj1aeb0s15IM3cc+8dpZt/D49OMJOVfIbTCkWvzOf7ghplpZnxXd5qZ3JXv6LdyT74WLtmU7+Q2el3+/YQdrMrT89zqhnIjNxybZhaNmZ5mtu8bmWZ+NfK1NPP8jnFp5qW1+W5YEzY39j2nnXCFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGiOiH6fvGDIvP6fLNj+Pz8zzfSMaKCrN7Cz2Y7j891t94x3muncnb/WERvzAY15+vX8RJJ6n1jZUG4g3N+7MP8CDLDS5m7Hie9vKNc7Ot9SdcfM0Wlm9/h8fu/PdxJsSOeuPHPkhv1pZsTPlgzAaAYWc7f5ev7sjDSzfXq+Qe/eo/M/ukbW1CP/y+I0U6Jmz93Dfd42omPWjDTT8+yqJoykXAeat1xhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0fJtmtpQ5wNL88wAvda4ATrPQGlgcy60QM9zqwfsXKPz6a18LyygTB0P/jbNvO/BJgwEGGCH+y5W7xVXWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACiaI6LVYwAAAAD6xRXWQ8x22J7R6nEArcZ7Ac3CXANqBtN7gcIKIGX7KtsPt3ocAHC4YN19KwprwnZnq8cAHErMcZSGOYnBjjl+8Cis78D2OtvX2V4haZftqbbvsr3V9lrbX6nLnm17ke3ttjfZvtV2VwuHjzZj+wzby2x3215oe4HtG6rnLra9vJpfj9qeU3fcOtvX2l5h+/XquOF1z2fH1s/xTtvftL26Gsczti+psidLmi9pru2dtrdXHx9m+3u2X7C9xfZ82yPqXuMb1XviJdufPeRfSLQ11l00E+tuG4oIHm97SFonabmkKZJGSVoq6XpJXZKmS1oj6cIqe6akcyV1Sjpe0kpJ19SdKyTNaPXnxKPMRzWn1kv6qqShki6VtE/SDZI+KOllSedI6pB0ZTU3h1XHrpO0RNJESWOruXd19Vwjx/bN8RHVx+ZV5xoi6QpJuyRNqJ67StLDbxv7zZJ+Xr32aEn3SLqpeu6jkrZIml29h37Ce4HHgR6suzya9WDdbc8HV1j7d0tEbFDtD35cRHwnIvZFxBpJt0v6uCRFxNKIWBwR+yNinaTbJJ3fslGj3fR9070lIt6MiLtVWwwl6QuSbouIxyKiJyJ+LGlvdUyfWyLipYjYptrCdfpBHrshIvZIUkQsrM7VGxELJD0v6ex3GrRtV6/xtYjYFhHdkr6r6n0h6XJJd0bEUxGxS9K33/VXCIcT1l00A+tuG+Ieiv5tqH6dJmli3+X4SoekhyTJ9omSfiDpQ5JGqvY1XdrEcaK9TZS0Maq/Hlfq596Vtr9c91xXdUyfzXW/3133XCPHbqj7vWx/WtLXVbtiJUlHSDqmn3GPU22+L62tobVTqPbe6Pu86t8H6/s5D1CPdRfNwLrbhiis/eubyBskrY2Imf3kfiRpmaRPRES37WskXdaMAWJQ2CRpkm3XLZ5TJK1Wbe7dGBE3vovzNnLs7xdr29NUu4L1YUmLIqLH9nLVFsO3ZCuvSNoj6dSI2PgO595UfR59ph7k+HF4Yt1FM7DutiFuCcgtkdRd3Sg9wnaH7dm2z6qeHy1ph6Sdtk+S9MWWjRTtaJGkHklfqm7A/5j+8OOg2yVdbfsc14yyfZHt0Q2c92CPHaXa4rhVkmx/RrUfy/bZImly3z9siYje6jVutj2+OmaS7Qur/E8lXWX7FNsjJX2rsS8HIIl1F4cW624borAmIqJH0sWq3aOyVrW/4dwh6agqcq2kT0rqVm0iLWjBMNGmImKfajf8f07SdkmfkvQLSXsj4nFJn5d0q6TXJK1S7Sb8Rs57UMdGxDOSvq/aQr5F0mmSHqmLPCDpaUmbbb9Sfey66ryLbe+Q9GtJs6rz/UrSD6vjVlW/Ag1h3cWhxLrbntiaFSiM7cckzY+IO1s9FgA4HLDulo8rrECL2T7f9nHVj6aulDRH0n2tHhcADFasu+2Hf3QFtN4s1e49GqXa/zV5WURsau2QAGBQY91tM9wSAAAAgKJxSwAAAACKdsBbAi4YMo/Lr3jP7u9d6Dw1sJi7GAjMXbSrZs9d5i0GwoHmLVdYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKFpnqwdwMDpmTm8ot+Fvjksze8/YlWaOG7sjzWzdcUSaiWdGp5lJ/7o3zXQ+sDTNoL3t+MS5aWbzn/akmdNO2pBmOp2fZ9mzx6eZ8Q/ly8iYf1yUZjD4bfr6eWmm+9R9aWb0+/L1u6cnvx6z7/kj08z0n+WvpcUr8gza1pr/ZW6aGTP71TQzqiuf2+tfOCbNTL63I82MvPuxNNNuuMIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICitdVOV1v/+NiGckd9eHOa+e6Jd6eZPxmev9aL+3emmfkz810y/vnYPDOj54NpZsi/LkszaL6OU2c1lNv8kf1p5rbzf5xm/v3IN9PMyz35Dj7/NPa0NDN/64VpZkyaQLtb/b/la9h//Iv70szVY36XZkYO6UozT+/bk2ZumvAXaWbFy6ekmQmL0wgKtfr7+e6C5533dJr5+4n3ppkThuY7Y/78hJFp5h8mfzTNvDo2fz++74722oGQK6wAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAAChaW+10dfTvdjeUW/P0+DTzvw7Pdzj5zdEvpJkX9xydZh589sQ0M+nXTjPsYtW+ep5+tqHckU+el2aun/CxNHPvsevSzBPbJqWZF56akGZm3rUjzUSaQLs7cnWe+dnGD6SZYUPyXdqGuifNLN85Nc08+vz0NDPluXz3ObSvYx/LMw8fMyPN/M+9f5VmLjpmRZp5cd/YNPPqjlFpZuTQNNJ2uMIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNEf0v2niBUPmsaMi3rP7exfm+84OMOYuBgJzt/k6ZuXbYO4fm29N2bEn3+K1d/kzDY2pHTV77h7u8zbm5tsOv3HssPw8Q/I/tpF3N7CfbJs60LzlCisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIrW2eoBAADQp+fZVWmmkS2cet/7UICGedETaWZEE8YxmHGFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKI5Ilo9BgAAAKBfXGEFAABA0Sish5jtsD2j1eMAWo33ApqFuQbUDKb3AoUVQMr2VbYfbvU4AOBwwbr7VhTWhO3OVo8BOJSY4ygNcxKDHXP84FFY34Htdbavs71C0i7bU23fZXur7bW2v1KXPdv2ItvbbW+yfavtrhYOH23G9hm2l9nutr3Q9gLbN1TPXWx7eTW/HrU9p+64dbavtb3C9uvVccPrns+OrZ/jnba/aXt1NY5nbF9SZU+WNF/SXNs7bW+vPj7M9vdsv2B7i+35tkfUvcY3qvfES7Y/e8i/kGhrrLtoJtbdNhQRPN72kLRO0nJJUySNkrRU0vWSuiRNl7RG0oVV9kxJ50rqlHS8pJWSrqk7V0ia0erPiUeZj2pOrZf0VUlDJV0qaZ+kGyR9UNLLks6R1CHpympuDquOXSdpiaSJksZWc+/q6rlGju2b4yOqj82rzjVE0hWSdkmaUD13laSH3zb2myX9vHrt0ZLukXRT9dxHJW2RNLt6D/2E9wKPAz1Yd3k068G6254PrrD275aI2KDaH/y4iPhOROyLiDWSbpf0cUmKiKURsTgi9kfEOkm3STq/ZaNGu+n7pntLRLwZEXerthhK0hck3RYRj0VET0T8WNLe6pg+t0TESxGxTbWF6/SDPHZDROyRpIhYWJ2rNyIWSHpe0tnvNGjbrl7jaxGxLSK6JX1X1ftC0uWS7oyIpyJil6Rvv+uvEA4nrLtoBtbdNsQ9FP3bUP06TdLEvsvxlQ5JD0mS7RMl/UDShySNVO1rurSJ40R7myhpY1R/Pa7Uz70rbX+57rmu6pg+m+t+v7vuuUaO3VD3e9n+tKSvq3bFSpKOkHRMP+Mep9p8X1pbQ2unUO290fd51b8P1vdzHqAe6y6agXW3DVFY+9c3kTdIWhsRM/vJ/UjSMkmfiIhu29dIuqwZA8SgsEnSJNuuWzynSFqt2ty7MSJufBfnbeTY3y/WtqepdgXrw5IWRUSP7eWqLYZvyVZekbRH0qkRsfEdzr2p+jz6TD3I8ePwxLqLZmDdbUPcEpBbIqm7ulF6hO0O27Ntn1U9P1rSDkk7bZ8k6YstGyna0SJJPZK+VN2A/zH94cdBt0u62vY5rhll+yLboxs478EeO0q1xXGrJNn+jGo/lu2zRdLkvn/YEhG91WvcbHt8dcwk2xdW+Z9Kusr2KbZHSvpWY18OQBLrLg4t1t02RGFNRESPpItVu0dlrWp/w7lD0lFV5FpJn5TUrdpEWtCCYaJNRcQ+1W74/5yk7ZI+JekXkvZGxOOSPi/pVkmvSVql2k34jZz3oI6NiGckfV+1hXyLpNMkPVIXeUDS05I2236l+th11XkX294h6deSZlXn+5WkH1bHrap+BRrCuotDiXW3Pfmtt3AAaDXbj0maHxF3tnosAHA4YN0tH1dYgRazfb7t46ofTV0paY6k+1o9LgAYrFh32w//6ApovVmq3Xs0SrX/a/KyiNjU2iEBwKDGuttmuCUAAAAAReOWAAAAABTtgLcEXDBkHpdf8Z7d37vQeWpgMXcxEJi7aFfNnrvMWwyEA81brrACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQtM5WD6BkPvPUATnPkB170kzP82sG5LUAAEBj/MH8+3zPEV1pJjqdZoa80ZNmOrfvzsez8vk0MxhxhRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEUblDtdbf7qeWmm8yOvpJm/mLIizUzo2p5mVu6amGbuXzMnzYx4+Ig0M+lnL6QZSdq/4cWGcmiu3Zeck2Y2fiQ/z7zzHkszozveSDPP7RqfZh56claaOfGO/LW05Mk8g6bb8YlzG8ptOS/SzJw569LM2UfnmdW7x6WZh9a+P80M++2oNDPlZ5vTDDsVlueFb+c9QJJO+8izaebMo/Lvqx8YkWeefGNymvnJ6rPSzO6n5qaZE/7f7jQTjz+VZkrCFVYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABStrXa66jgx37lEkrrf35Nm/m7q02nm80fnuwVN7sx3n1ox8vk0s6unK8089MLsNLN/wtFpRpLETldFem1WR5o5/bR8Pn366EVp5tSuEWnmn4dtTTOPj52aZt4cMzzNDE0TGGgdJ89MM6/OcUPnunju0jRz7fgH08zURtbUfU+kmVGde9PMPd2np5k3puVr6tD8LYkB5A/l3wvjlHynJ0maMuK1NPPpo5almfEdI9PMR0euTjNv9ua17I4X/zTN9AzPz9NuVyzbbbwAAAA4zFBYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQtLbamrXnuXxbM0ka92/j08x/HnNOmnny+Ilp5n3DdqeZZ7fn43npqWPTzNT730wzWvJknkGxJizak2aWTT0hzXxh939IM6O78q0rn10zIc1M/mW+nezQf8m3OUbz9azM9xSdfne+ra4k/ab7rDRzz8w5aaZzWL619v5d+Ua+o1blmZP+2/Y007v8mTSD5orHn0ozx/04n4+SdPffnJlmts7Otws+9YhNaWbV7rwL/ObZWWlm2j29aWbIw8vTTLvhCisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIrmiOj3yQuGzOv/SaBB9/cudLNfk7mLgcDcRbtq9txl3mIgHGjecoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAojkiWj0GAAAAoF9cYT3EbIftGa0eB9BqvBfQLMw1oGYwvRcorABStq+y/XCrxwEAhwvW3beisCZsd7Z6DMChxBxHaZiTGOyY4wePwvoObK+zfZ3tFZJ22Z5q+y7bW22vtf2VuuzZthfZ3m57k+1bbXe1cPhoM7bPsL3MdrfthbYX2L6heu5i28ur+fWo7Tl1x62zfa3tFbZfr44bXvd8dmz9HO+0/U3bq6txPGP7kip7sqT5kuba3ml7e/XxYba/Z/sF21tsz7c9ou41vlG9J16y/dlD/oVEW2PdRTOx7rahiODxtoekdZKWS5oiaZSkpZKul9QlabqkNZIurLJnSjpXUqek4yWtlHRN3blC0oxWf048ynxUc2q9pK9KGirpUkn7JN0g6YOSXpZ0jqQOSVdWc3NYdew6SUskTZQ0tpp7V1fPNXJs3xwfUX1sXnWuIZKukLRL0oTquaskPfy2sd8s6efVa4+WdI+km6rnPippi6TZ1XvoJ7wXeBzowbrL4/9r796DrKzvPI9/vt1Nc2mQi4AKAt4QVHTRKGgytSSbdc2uVoxZycXNCsZJxmzlYrKmTM3UTrKumpraRFOUqeiYHdfsJDuEMTObmNEZE2NKI0gkEOIlRi6NLTQIgYbmIg3d3/3jeTo5pmy+B9Oc8z3N+1V1iq4+n+d5fn34nV9/+ukHnlo9WHcb88EZ1oEtcfcOFX/xk9z9NnfvcfcNku6X9CFJcvdV7r7C3Q+7e7uk+yQtqNuo0Wj6v+kucfdD7v49FYuhJH1c0n3u/oy797r7g5IOltv0W+LuW9x9p4qFa+5Rbtvh7gckyd2Xlfvqc/elkl6WNO/NBm1mVh7js+6+0927Jd2p8n0h6QOSHnD359x9n6QvveVXCMcT1l3UAutuA+IaioF1lH/OkDSl/3R8qVnSk5JkZmdLukvSxZJGqXhNV9VwnGhsUyRt9vLH41Ll3FtkZp+qeK613Kbf1oqP91c8V822HRUfy8yul/Q5FWesJGm0pIkDjHuSivm+qlhDi12oeG/0f12V74NNA+wHqMS6i1pg3W1AFNaB9U/kDkkb3X3mALlvSFot6cPu3m1mN0u6thYDxJDQKWmqmVnF4jlN0noVc+8Od7/jLey3mm1/t1ib2QwVZ7DeLWm5u/ea2RoVi+EbsqUdkg5IOs/dN7/JvjvLr6Pf9KMcP45PrLuoBdbdBsQlAbGVkrrLC6VHmlmzmc0xs0vK58dI2iNpr5nNlvSJuo0UjWi5pF5JnywvwL9av/910P2SbjKz+VZoM7MrzWxMFfs92m3bVCyO2yXJzG5Q8WvZftskndr/D1vcva88xt1mNrncZqqZXVHmvytpsZmda2ajJH2xupcDkMS6i2OLdbcBUVgD7t4r6SoV16hsVPETzjcljS0jt0i6TlK3iom0tA7DRINy9x4VF/zfKKlL0kckPSzpoLs/K+ljku6RtEvSOhUX4Vez36Pa1t1fkPRVFQv5NknnS/pZReRxSc9L2mpmO8rP3Vrud4WZ7ZH0I0mzyv09Iulr5Xbryj+BqrDu4lhi3W1M3JoVSMbMnpF0r7s/UO+xAMDxgHU3P86wAnVmZgvM7OTyV1OLJF0g6dF6jwsAhirW3cbDP7oC6m+WimuP2lT8X5PXuntnfYcEAEMa626D4ZIAAAAApMYlAQAAAEjtiJcEXN60kNOv+KM91rfM4tTgYu5iMDB30ahqPXeZtxgMR5q3nGEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApNZS7wH0a5p7bph55T+Mq2pf+6cfjo/3etzVmw5amDl8Qm+YsbZ4PM2dw8PMlKfiY434wcowg9rrueLiqnL7Th4WZnafXcXxpvRUdbyIdcdLxJj1zWFmwkuHwkzroz+vakyosUsvqCq28eq2MDPugh1h5tQxXWHmpe2Tw0zT8rFhZvpDm8PM4Y2bwgwa196F88PM1vfG62lLa/z9uacr/j4/YVW85k786+VhZijiDCsAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUktzp6uO98R3sRr/r7dWta8/P/2nYebyUa/Ex2saEWa29R4IM5sOjwozHYdODDN/d+m8MPObuW8PM5I07X88XVUOsc23xq953yV7qtrXvz3tl2HmugkrwsyMlnheVmNbb3znrb/esSDMPPby7DAzYdJlVY1p3P85Pu/yUi/b5o2uKnfWpe1h5s7T/iHMzB0e3w1ozbSDYebPJ1wTZjb3nBZmTlrCna4a0Z7rLq0qN+KGzjDz8Mz/G2bOaY2/z796eG+Y+cL5V4WZVSfH33Om3zb0vsdzhhUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqaW501U1hjX3VpUb0xTf5WeUwK8WpAAAHqxJREFUNcfHqyLTZnHnP+Txy/zi61PCTNfBkWGmZ2xfmJEkf8fcMGM/W1PVvo53XsWPfaNHxnfmkaRZo+K7uU1qjuf3KS3x3YnWH4rvurK5d2yYqYa7hZlDbYNyKAyyk5ZUd8ecTW3x3XduftcHw8z8Se3xsfZPCDMv/mZqmJmx7lCYQWMat7arqtyvN0wOM18f964wc/GYjWFm26GZYealnfF42jo9zAxFnGEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRm7gPf4uvypoU1u/9X05zZYWbLu+Pb8UnS3tPi25P2jaziFqYtccb2x7dvHb4zzoxdFx9rwupdYab3+ZfCTK091rcsvi/nIKvl3K3GrsWXVZXrnlHFS1XFV9YzPp5Prbvjn1eH74yP1dYZ3zJ51NaeMNP05Or4YDXG3EWjqvXcPd7nbfPMM8JM78sbajCSxnakecsZVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACk1lLvAfTre+7XYebk52owkMTi+wkhq/H/e3l1uWM8DgDA4OMuVsceZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqZm713sMAAAAwIA4wwoAAIDUKKzHmJm5mZ1V73EA9cZ7AbXCXAMKQ+m9QGEFEDKzxWb2VL3HAQDHC9bdN6KwBsyspd5jAI4l5jiyYU5iqGOOHz0K65sws3Yzu9XM1kraZ2bTzewhM9tuZhvN7NMV2XlmttzMusys08zuMbPWOg4fDcbMLjKz1WbWbWbLzGypmd1ePneVma0p59fTZnZBxXbtZnaLma01s93ldiMqno+2rZzjLWb2BTNbX47jBTO7psyeI+leSZeZ2V4z6yo/P9zMvmJmr5jZNjO718xGVhzj8+V7YouZffSYv5BoaKy7qCXW3Qbk7jz+4CGpXdIaSdMktUlaJekvJbVKOkPSBklXlNm3SbpUUouk0yS9KOnmin25pLPq/TXxyPko59QmSZ+RNEzS+yX1SLpd0oWSXpM0X1KzpEXl3BxebtsuaaWkKZImlHPvpvK5arbtn+Mjy88tLPfVJOmDkvZJOqV8brGkp/5g7HdL+n557DGSfiDpy+Vz75G0TdKc8j30Hd4LPI70YN3lUasH625jPjjDOrAl7t6h4i9+krvf5u497r5B0v2SPiRJ7r7K3Ve4+2F3b5d0n6QFdRs1Gk3/N90l7n7I3b+nYjGUpI9Lus/dn3H3Xnd/UNLBcpt+S9x9i7vvVLFwzT3KbTvc/YAkufuycl997r5U0suS5r3ZoM3MymN81t13unu3pDtVvi8kfUDSA+7+nLvvk/Slt/wK4XjCuotaYN1tQFxDMbCO8s8Zkqb0n44vNUt6UpLM7GxJd0m6WNIoFa/pqhqOE41tiqTNXv54XKqce4vM7FMVz7WW2/TbWvHx/ornqtm2o+Jjmdn1kj6n4oyVJI2WNHGAcU9SMd9XFWtosQsV743+r6vyfbBpgP0AlVh3UQusuw2Iwjqw/oncIWmju88cIPcNSaslfdjdu83sZknX1mKAGBI6JU01M6tYPKdJWq9i7t3h7ne8hf1Ws+3vFmszm6HiDNa7JS13914zW6NiMXxDtrRD0gFJ57n75jfZd2f5dfSbfpTjx/GJdRe1wLrbgLgkILZSUnd5ofRIM2s2szlmdkn5/BhJeyTtNbPZkj5Rt5GiES2X1Cvpk+UF+Ffr978Oul/STWY23wptZnalmY2pYr9Hu22bisVxuySZ2Q0qfi3bb5ukU/v/YYu795XHuNvMJpfbTDWzK8r8dyUtNrNzzWyUpC9W93IAklh3cWyx7jYgCmvA3XslXaXiGpWNKn7C+aaksWXkFknXSepWMZGW1mGYaFDu3qPigv8bJXVJ+oikhyUddPdnJX1M0j2Sdklap+Ii/Gr2e1TbuvsLkr6qYiHfJul8ST+riDwu6XlJW81sR/m5W8v9rjCzPZJ+JGlWub9HJH2t3G5d+SdQFdZdHEusu43J3ngJB4B6M7NnJN3r7g/UeywAcDxg3c2PM6xAnZnZAjM7ufzV1CJJF0h6tN7jAoChinW38fCProD6m6Xi2qM2Ff/X5LXu3lnfIQHAkMa622C4JAAAAACpcUkAAAAAUjviJQGXNy3k9Cv+aI/1LbM4NbiYuxgMzF00qlrPXeYtBsOR5i1nWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACptdR7AJn1/cncMDNs+94w0/vSusEYDo4DLadNDzM9004MM69Pag0zzQf6wszIju4w0/fcr8MMctr//vlhpntqc1X76ounnA61xZmmw3FmVKeHmXHrXo+P9eTq+GBoSNXMbUnaOTue3wdO7g0zTQfj838jt1uYmfjLnjDT+s/PhpmhiDOsAAAASI3CCgAAgNQorAAAAEiNwgoAAIDUKKwAAABIjcIKAACA1CisAAAASI3CCgAAgNQorAAAAEhtSN7pqv32y8LMR6/+UZi5YdzX42Mdjm/vcucrV4WZ9Q+fGWam/M+nwwxy6n3nRVXl1r03nk/nva09zPzJuFfCzKjmg2Gm4/UJYeafXrowzIz/8YgwM+FvlocZVK/5rNPDzO7T47v89L59d1XHu+qM58PMfxz38zAzZ1h8F6u9fijMPHFgSpj5xqZ3hpnXHp8aZk5buiXMSNLhDe1V5XBkXdfH3+MPXbuzqn3dNvuRMDN/RPz3e2rL6DDz4wPx++2hnZeEmUeunhdmzv5WfKc3rVgbZxLhDCsAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUhuSd7rqi28WpFFNPWHmxKaRYWby8Ljz33zqY2HmS5fHx9r1Wnx3j/EPcregjHqrmCeS1DfucJh5x4T1YebGcWvCzMTmtjCz8dAvw8zwpnjMDx2O74Y17uU4I0lNT66uKne86123McxM/XG87mwaMa6q4/1T07lh5pxz4jsGzRn2apiZXMXcPa91a5iZN3FTmPnuzElhZu95k8OMJI3gTleDYty34u9zW05+e1X7+u99V4aZxWc9E2auGvOrMDPM4nk7Y8Rvw8yEaV1h5rW3TQwzp/z2jDAjSb0vb6gqd6xxhhUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkJq5+4BPXt60cOAnG9zOj8a3Od05J/7y+8bGt6W0A81hZuyLcWby158OMxk91rfMan3MRp27fQvi25O++q74dpqtc3eFmTEjDoaZtmHxLYw3/XZ8mJnwUHxLwjFLV4SZWmPuDq69C+eHmV2z47Xw4Pi+MGO98V/dyNfizKRfxu+BYf/ybJiptVrP3aE8bzXv/DCy++x4jdt9ZnyOsDlelnXCxnj+j+44EGbs6fjW27V2pHnLGVYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApNZS7wHUy4S/WR5najAOoFLTT1eHmek/rcFAjsIMvVrvIaBBjF72TJypwTiAo7LyV2Fk7Mp4N2MHYSjHM86wAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFIzd6/3GAAAAIABcYb1GDMzN7Oz6j0OoN54L6BWmGtAYSi9FyisAEJmttjMnqr3OADgeMG6+0YU1oCZtdR7DMCxxBxHNsxJDHXM8aNHYX0TZtZuZrea2VpJ+8xsupk9ZGbbzWyjmX26IjvPzJabWZeZdZrZPWbWWsfho8GY2UVmttrMus1smZktNbPby+euMrM15fx62swuqNiu3cxuMbO1Zra73G5ExfPRtpVzvMXMvmBm68txvGBm15TZcyTdK+kyM9trZl3l54eb2VfM7BUz22Zm95rZyIpjfL58T2wxs48e8xcSDY11F7XEutuA3J3HHzwktUtaI2mapDZJqyT9paRWSWdI2iDpijL7NkmXSmqRdJqkFyXdXLEvl3RWvb8mHjkf5ZzaJOkzkoZJer+kHkm3S7pQ0muS5ktqlrSonJvDy23bJa2UNEXShHLu3VQ+V822/XN8ZPm5heW+miR9UNI+SaeUzy2W9NQfjP1uSd8vjz1G0g8kfbl87j2StkmaU76HvsN7gceRHqy7PGr1YN1tzAdnWAe2xN07VPzFT3L329y9x903SLpf0ockyd1XufsKdz/s7u2S7pO0oG6jRqPp/6a7xN0Pufv3VCyGkvRxSfe5+zPu3uvuD0o6WG7Tb4m7b3H3nSoWrrlHuW2Hux+QJHdfVu6rz92XSnpZ0rw3G7SZWXmMz7r7TnfvlnSnyveFpA9IesDdn3P3fZK+9JZfIRxPWHdRC6y7DYhrKAbWUf45Q9KU/tPxpWZJT0qSmZ0t6S5JF0sapeI1XVXDcaKxTZG02csfj0uVc2+RmX2q4rnWcpt+Wys+3l/xXDXbdlR8LDO7XtLnVJyxkqTRkiYOMO5JKub7qmINLXah4r3R/3VVvg82DbAfoBLrLmqBdbcBUVgH1j+ROyRtdPeZA+S+IWm1pA+7e7eZ3Szp2loMEENCp6SpZmYVi+c0SetVzL073P2Ot7Dfarb93WJtZjNUnMF6t6Tl7t5rZmtULIZvyJZ2SDog6Tx33/wm++4sv45+049y/Dg+se6iFlh3GxCXBMRWSuouL5QeaWbNZjbHzC4pnx8jaY+kvWY2W9In6jZSNKLlknolfbK8AP9q/f7XQfdLusnM5luhzcyuNLMxVez3aLdtU7E4bpckM7tBxa9l+22TdGr/P2xx977yGHeb2eRym6lmdkWZ/66kxWZ2rpmNkvTF6l4OQBLrLo4t1t0GRGENuHuvpKtUXKOyUcVPON+UNLaM3CLpOkndKibS0joMEw3K3XtUXPB/o6QuSR+R9LCkg+7+rKSPSbpH0i5J61RchF/Nfo9qW3d/QdJXVSzk2ySdL+lnFZHHJT0vaauZ7Sg/d2u53xVmtkfSjyTNKvf3iKSvldutK/8EqsK6i2OJdbcxcWtWIBkze0bSve7+QL3HAgDHA9bd/DjDCtSZmS0ws5PLX00tknSBpEfrPS4AGKpYdxsP/+gKqL9ZKq49alPxf01e6+6d9R0SAAxprLsNhksCAAAAkBqXBAAAACC1I14ScHnTQk6/4o/2WN8yi1ODi7mLwcDcRaOq9dxl3mIwHGnecoYVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkFpLvQdwLHT958vCzPZ5fWFm2OQDYaanuzXMnPB8nJn66PYw0/viy2EGja2aubvryn1hZtyYeO62NveGmc2vnBhmZvy/MKLhP/x5HMKQt//988PMtovj8yiHxsdzd0Rn/O3t1Mf3h5mmp9aEGQxtv/3TeF3umhXvp3f8oTh0KJ7/Z/x9PP9bfrwqPlaD4QwrAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFJrqDtdVXMXIEna8969Yeauuf8QZt7XFu9nf19PmPnbd5wWZv7qoivCzPRvXxJmWh/ljkIZ7fiz6uauX7kzzNw686dh5saxW8PMIY/vlvLDM8eGmTun/vsw03Xm28PMSUueDjPIa9eieI43f+i1MPO3s/8uzMwbPizMPHEgPh/zZ+d9JMxMG3ZRmGn+yS/CDHLa+pl4bTr5va+EmaVnxfP2zGGjw8yj+4eHmU+MiOft7G2zw0zfc78OM5lwhhUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqTXUna7Gv7CnqtzO804IM986Kb4ry+snPRtmJjTHd8N65eCJYcbdwkxfS5xBThPvW15Vbv3Zl4aZh8bEd97Z3RvfwWTOiI4w094zMcxU41DboOwGiU16sjPMbJgxJcz8t9ZrwszcCa+GmWd/Oz3MDFsd33mo+SfcgW0oO/UHW8LMS6dOCzN/2vefwsy8iZvCTFXztrM1zPiIg2Gm0XCGFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQmrn7gE9e3rRw4CcTO/C+eWGme0p8V9oRXX1hpnVPnBnZuS/M+Krnw0yjeqxvWc3vKduoc7cauxbFtxXec0b8kltvfKzhXXFm4trXw0zzE7+Id5QQcxeNqtZzl3mLwXCkecsZVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkFt/uqQGN/MeVcaYG4+jH7T8wmMY/uDzO1GAcAADUCmdYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKmZu9d7DAAAAMCAOMMKAACA1Cisx5iZuZmdVe9xAPXGewG1wlwDCkPpvUBhBRAys8Vm9lS9xwEAxwvW3TeisAbMrKXeYwCOJeY4smFOYqhjjh89CuubMLN2M7vVzNZK2mdm083sITPbbmYbzezTFdl5ZrbczLrMrNPM7jGz1joOHw3GzC4ys9Vm1m1my8xsqZndXj53lZmtKefX02Z2QcV27WZ2i5mtNbPd5XYjKp6Ptq2c4y1m9gUzW1+O4wUzu6bMniPpXkmXmdleM+sqPz/czL5iZq+Y2TYzu9fMRlYc4/Ple2KLmX30mL+QaGisu6gl1t0G5O48/uAhqV3SGknTJLVJWiXpLyW1SjpD0gZJV5TZt0m6VFKLpNMkvSjp5op9uaSz6v018cj5KOfUJkmfkTRM0vsl9Ui6XdKFkl6TNF9Ss6RF5dwcXm7bLmmlpCmSJpRz76byuWq27Z/jI8vPLSz31STpg5L2STqlfG6xpKf+YOx3S/p+eewxkn4g6cvlc++RtE3SnPI99B3eCzyO9GDd5VGrB+tuYz44wzqwJe7eoeIvfpK73+buPe6+QdL9kj4kSe6+yt1XuPthd2+XdJ+kBXUbNRpN/zfdJe5+yN2/p2IxlKSPS7rP3Z9x9153f1DSwXKbfkvcfYu771SxcM09ym073P2AJLn7snJffe6+VNLLkua92aDNzMpjfNbdd7p7t6Q7Vb4vJH1A0gPu/py775P0pbf8CuF4wrqLWmDdbUBcQzGwjvLPGZKm9J+OLzVLelKSzOxsSXdJuljSKBWv6aoajhONbYqkzV7+eFyqnHuLzOxTFc+1ltv021rx8f6K56rZtqPiY5nZ9ZI+p+KMlSSNljRxgHFPUjHfVxVraLELFe+N/q+r8n2waYD9AJVYd1ELrLsNiMI6sP6J3CFpo7vPHCD3DUmrJX3Y3bvN7GZJ19ZigBgSOiVNNTOrWDynSVqvYu7d4e53vIX9VrPt7xZrM5uh4gzWuyUtd/deM1ujYjF8Q7a0Q9IBSee5++Y32Xdn+XX0m36U48fxiXUXtcC624C4JCC2UlJ3eaH0SDNrNrM5ZnZJ+fwYSXsk7TWz2ZI+UbeRohEtl9Qr6ZPlBfhX6/e/Drpf0k1mNt8KbWZ2pZmNqWK/R7ttm4rFcbskmdkNKn4t22+bpFP7/2GLu/eVx7jbzCaX20w1syvK/HclLTazc81slKQvVvdyAJJYd3Fsse42IAprwN17JV2l4hqVjSp+wvmmpLFl5BZJ10nqVjGRltZhmGhQ7t6j4oL/GyV1SfqIpIclHXT3ZyV9TNI9knZJWqfiIvxq9ntU27r7C5K+qmIh3ybpfEk/q4g8Lul5SVvNbEf5uVvL/a4wsz2SfiRpVrm/RyR9rdxuXfknUBXWXRxLrLuNyd54CQeAejOzZyTd6+4P1HssAHA8YN3NjzOsQJ2Z2QIzO7n81dQiSRdIerTe4wKAoYp1t/Hwj66A+pul4tqjNhX/1+S17t5Z3yEBwJDGuttguCQAAAAAqXFJAAAAAFI74iUBlzct5PQr/miP9S2zODW4mLsYDMxdNKpaz13mLQbDkeYtZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqbXUewDHwoH3zQszr76vN8zMPeOVMNPncedfu2lqmDnxJ8PDzIQHlocZNLZtn3p7mNl90cEw86/OeDXMTBm1O8zs7BkVZlY/MSvMnPYXzN2Met5zSZjZvKC6bxMtM7vDzNTx8Zw7pYp5edLw+FgvdZ8UZp7bGK/NY9bEa/PUH24NM5LU+/KGqnI4suZZZ4WZre+aVNW+ds2Nu8DwCQfCTGvr4TDT0tQXZnZvGB9mTqpiOR37YleY6Vv763hHiXCGFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpDck7XW1+Z9zD/+u8R8LMx8e2h5lh1hxm/mXqsDBz16n/Lsysn31ZmDnz7/eGGUnyn/+qqhxq68BJHmauv2hFmPn8iavCzOimEWHmiQPxe+kz58V3FNq1KJ674x/kbliDyS6eE2ZefWf8LeC8y6q7O9N/mfp4mJk/Yk+YGWWtYaaadXfjifFa+L/GxvPy24cuDTPjXzoxzEjScO50NShevHlCmPk3F1b3PW7R5KfCzLTmeC6dPmx0mPnNoX1h5smZZ4aZb856R5jZ/r3JYWZS3+wwI0l9z+W4IxZnWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQ2pC809WEtRZm7pn6zjDTfuZzYWbq8F1h5lfdp8bH2hHfuaO1K/66mvYdDDOS1FtVCrU29YmeMPPt0+eFmd2zR4aZycO6w8w/dlwQZg6vHB9mTn7w6TCDweXPxuvXpDPjuzj9qvX0qo73F93XhJlpJ8Tr5f7D8Z2u9vYMDzOvdcV3Hmp6Kc6c+cTrYab5J78IMxg8456Pq8vjLedUta9XzxgXZmaesD3MDG86FGb2HI7X5SfWzwwzviW+S+HUrfF3+Sx3sKoWZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKmZuw/45OVNCwd+8jjQc8XFYaZ72rAwM2xf/DKOe2F3mOn75YthJqPH+pbF95QdZMf73O1bcGEc6o1fomHb98a7eWldNUNqSMxdNKpaz13mLQbDkeYtZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkFpLvQeQWes/PxtmThykY/UN0n4ASWr66epB2U/voOwFAIA/DmdYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKmZu9d7DAAAAMCAOMMKAACA1CisAAAASI3CCgAAgNQorAAAAEiNwgoAAIDUKKwAAABI7f8DC+HjETqqSZEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAAPwCAYAAAD0+SwIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzde5TV5Z3v+c+HKoqbKBJBuSsB8YLEaLxg92m7O/GYbu1Oa4smOelokpWMOZOLyTJj1pq1TDpL4/RMEjOOawVHZ3nSpztnCKOTExPjaRNdfbyAtATEC1G5igiIIlJcBKn6zh/7V8nWsfhutNj72cX7tdZesGp/fr/9VPHspz786gePI0IAAABAqYa0egAAAADAgVBYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwHmK2w/aMVo8DaDXeC2gW5hpQM5jeCxRWACnbV9l+uNXjAIDDBevuW1FYE7Y7Wz0G4FBijqM0zEkMdszxg0dhfQe219m+zvYKSbtsT7V9l+2tttfa/kpd9mzbi2xvt73J9q22u1o4fLQZ22fYXma72/ZC2wts31A9d7Ht5dX8etT2nLrj1tm+1vYK269Xxw2vez47tn6Od9r+pu3V1TiesX1JlT1Z0nxJc23vtL29+vgw29+z/YLtLbbn2x5R9xrfqN4TL9n+7CH/QqKtse6imVh321BE8HjbQ9I6ScslTZE0StJSSddL6pI0XdIaSRdW2TMlnSupU9LxklZKuqbuXCFpRqs/Jx5lPqo5tV7SVyUNlXSppH2SbpD0QUkvSzpHUoekK6u5Oaw6dp2kJZImShpbzb2rq+caObZvjo+oPjavOtcQSVdI2iVpQvXcVZIeftvYb5b08+q1R0u6R9JN1XMflbRF0uzqPfQT3gs8DvRg3eXRrAfrbns+uMLav1siYoNqf/DjIuI7EbEvItZIul3SxyUpIpZGxOKI2B8R6yTdJun8lo0a7abvm+4tEfFmRNyt2mIoSV+QdFtEPBYRPRHxY0l7q2P63BIRL0XENtUWrtMP8tgNEbFHkiJiYXWu3ohYIOl5SWe/06Btu3qNr0XEtojolvRdVe8LSZdLujMinoqIXZK+/a6/QjicsO6iGVh32xD3UPRvQ/XrNEkT+y7HVzokPSRJtk+U9ANJH5I0UrWv6dImjhPtbaKkjVH99bhSP/eutP3luue6qmP6bK77/e665xo5dkPd72X705K+rtoVK0k6QtIx/Yx7nGrzfWltDa2dQrX3Rt/nVf8+WN/PeYB6rLtoBtbdNkRh7V/fRN4gaW1EzOwn9yNJyyR9IiK6bV8j6bJmDBCDwiZJk2y7bvGcImm1anPvxoi48V2ct5Fjf79Y256m2hWsD0taFBE9tperthi+JVt5RdIeSadGxMZ3OPem6vPoM/Ugx4/DE+sumoF1tw1xS0BuiaTu6kbpEbY7bM+2fVb1/GhJOyTttH2SpC+2bKRoR4sk9Uj6UnUD/sf0hx8H3S7patvnuGaU7Ytsj27gvAd77CjVFsetkmT7M6r9WLbPFkmT+/5hS0T0Vq9xs+3x1TGTbF9Y5X8q6Srbp9geKelbjX05AEmsuzi0WHfbEIU1ERE9ki5W7R6Vtar9DecOSUdVkWslfVJSt2oTaUELhok2FRH7VLvh/3OStkv6lKRfSNobEY9L+rykWyW9JmmVajfhN3Legzo2Ip6R9H3VFvItkk6T9Ehd5AFJT0vabPuV6mPXVeddbHuHpF9LmlWd71eSflgdt6r6FWgI6y4OJdbd9uS33sIBoNVsPyZpfkTc2eqxAMDhgHW3fFxhBVrM9vm2j6t+NHWlpDmS7mv1uABgsGLdbT/8oyug9Wapdu/RKNX+r8nLImJTa4cEAIMa626b4ZYAAAAAFI1bAgAAAFC0A94ScMGQeVx+xXt2f+9C56mBxdzFQGDuol01e+4ybzEQDjRvucIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0TpbPYBDYfel56SZTX+7L81cfNKTaWbSsNfSzL2bZqeZbb+YlGaO+98fTTNob3v/8qw0s/6ySDOXfuC3aWaI8/Os3HFcmlnzmxPSzOTf7EozfvSJNINy7fjkuWlm8wVvpplpk15NM7v2daWZbavGppmp9/Wkma77/i3NoH31nv/BNPPCR4anmX2T807RiCNXDEszk/4lf4/0PP3sQAynKFxhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0QblTleb5uY9/EunP5hm/scxq9PMUHekmWM7X08z3zr50jQzcc5JaaZ3xe/SDJpvyOz8z06SXrgon7s3/dFP08wlR7ycZoZ5aJr5zVH5/L72Q/PSzKsvH51mjmEjtyLt/Yt89zVJev3SnWnmttMXpJl/PzLfDeuF/flr/WDCn6aZe/fkn9sJ96URFKjj5JkN5Z77ZF6D/uHPfpJmLj8i/z6/Yt8baebHc85LM//Ske8qN/HpNNJ2uMIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiDcqdro5alWcWbjhjQF5ruPNdWR7Ylu9yNGRP/neHnlHD0ozTBFrBvb0N5UZszHeW+seX5qaZV499Ms28tn9UmvnlxlPTzBvLxqaZ4x/ZlmYa+wqh2YZv3t1QrmflkWnm74/4qzSzYEy+S9vvto9PM68uOTbNvP+u7WmGedmeelY+31BuzJP5zlLffl8+b38x6YU08/z2cWlm61P53J553+G5nnKFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGiOiH6fvGDIvP6fPAzEH52eZnZPyLdL7enKN0wdvf6NNONHlqeZEt3fu7DpO8a269ztmDUjzbw5fnSaic78Sz5kX755X7vOuYHC3EW7avbcZd5iIBxo3nKFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARets9QBK1sguP6OaMA4cPnqeXZVmhjzbhIEAAFAQrrACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNEdEq8cAAAAA9IsrrAAAACgahfUQsx22Z7R6HECr8V5AszDXgJrB9F6gsAJI2b7K9sOtHgcAHC5Yd9+Kwpqw3dnqMQCHEnMcpWFOYrBjjh88Cus7sL3O9nW2V0jaZXuq7btsb7W91vZX6rJn215ke7vtTbZvtd3VwuGjzdg+w/Yy2922F9peYPuG6rmLbS+v5tejtufUHbfO9rW2V9h+vTpueN3z2bH1c7zT9jdtr67G8YztS6rsyZLmS5pre6ft7dXHh9n+nu0XbG+xPd/2iLrX+Eb1nnjJ9mcP+RcSbY11F83EutuGIoLH2x6S1klaLmmKpFGSlkq6XlKXpOmS1ki6sMqeKelcSZ2Sjpe0UtI1decKSTNa/TnxKPNRzan1kr4qaaikSyXtk3SDpA9KelnSOZI6JF1Zzc1h1bHrJC2RNFHS2GruXV0918ixfXN8RPWxedW5hki6QtIuSROq566S9PDbxn6zpJ9Xrz1a0j2Sbqqe+6ikLZJmV++hn/Be4HGgB+suj2Y9WHfb88EV1v7dEhEbVPuDHxcR34mIfRGxRtLtkj4uSRGxNCIWR8T+iFgn6TZJ57ds1Gg3fd90b4mINyPibtUWQ0n6gqTbIuKxiOiJiB9L2lsd0+eWiHgpIraptnCdfpDHboiIPZIUEQurc/VGxAJJz0s6+50GbdvVa3wtIrZFRLek76p6X0i6XNKdEfFUROyS9O13/RXC4YR1F83AutuGuIeifxuqX6dJmth3Ob7SIekhSbJ9oqQfSPqQpJGqfU2XNnGcaG8TJW2M6q/Hlfq5d6XtL9c911Ud02dz3e931z3XyLEb6n4v25+W9HXVrlhJ0hGSjuln3ONUm+9La2to7RSqvTf6Pq/698H6fs4D1GPdRTOw7rYhCmv/+ibyBklrI2JmP7kfSVom6RMR0W37GkmXNWOAGBQ2SZpk23WL5xRJq1WbezdGxI3v4ryNHPv7xdr2NNWuYH1Y0qKI6LG9XLXF8C3ZyiuS9kg6NSI2vsO5N1WfR5+pBzl+HJ5Yd9EMrLttiFsCckskdVc3So+w3WF7tu2zqudHS9ohaaftkyR9sWUjRTtaJKlH0peqG/A/pj/8OOh2SVfbPsc1o2xfZHt0A+c92GNHqbY4bpUk259R7ceyfbZImtz3D1siord6jZttj6+OmWT7wir/U0lX2T7F9khJ32rsywFIYt3FocW624YorImI6JF0sWr3qKxV7W84d0g6qopcK+mTkrpVm0gLWjBMtKmI2KfaDf+fk7Rd0qck/ULS3oh4XNLnJd0q6TVJq1S7Cb+R8x7UsRHxjKTvq7aQb5F0mqRH6iIPSHpa0mbbr1Qfu64672LbOyT9WtKs6ny/kvTD6rhV1a9AQ1h3cSix7rYnv/UWDgCtZvsxSfMj4s5WjwUADgesu+XjCivQYrbPt31c9aOpKyXNkXRfq8cFAIMV62774R9dAa03S7V7j0ap9n9NXhYRm1o7JAAY1Fh32wy3BAAAAKBo3BIAAACAoh3wloALhszj8ives/t7FzpPDSzmLgYCcxftqtlzl3mLgXCgecsVVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIrW2eoBHIz9f35mQ7n1Fw1NM13TdqaZffvyL0/H6hFpZsLi/Wlm2C//Lc1g8Nv6xblpZufU/Dy9Dbyzj16ZZ8Y9ti3N9Dz9bH4iQNKrn8/n966JTjOjNkaaed8dixoaEw5vHaecmGa2nTE2zbwxNr/+18i6PHJLb5o56p8X5ycahLjCCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAorXVTldbzhrWUG7OWc+nmeun/CLNnD4sf71Nf5zvmPVPl3wgzdz2yT9JMxMX5jt4jfjZkjSD5nv5P57XUO6Yv92QZn54wj1p5riOXWlmc8+oNPPL109PM3evzDPDfzsyzUz83qNpBs23+auNzd2j/nJTmvnBzB+lmXOGvZlmHtubr4XXX/43aea1eyemmeN+yLxsR7suO6eh3Mt/+0aa+Q+n/Pc0c/lRj6eZLue7WC14Pd/R866P553CP3tfmnnf/9Veu8FxhRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEVrq52ujlyX7xIhSc9sPi7NPHLMjDQztmNlmhlup5mzRqxNM/827fg08/Qps9LM5J+lEbSAIxrKvbhtTJp5/NjpaeYTR65IMycOz8dzpB9LM9ven++Y9Zutp+Uvhqbzmaemmd0TG5u7JwzfnWbejI40MyxfUvVHw/LvBX83ZXGaueHUv0ozx85tYFehRU+kGTRX5+7G+oKcz+8Thr2cZqYPzXdfG+Y8c96ofKfOu5TPyX1HNfBGajNcYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACia4wBbRl4wZF5je/K1oe4rzk0zr3wg39rszXH78xfryc8zam2+S+6kB7vz11ryZJ5psvt7FzZ9j7h2nbsdM/NtVzddkG89vPP4/NOPSW+kmZ6d+bwc/Vy+3eDke7fmr7Uy35Kw2Zi7UvzR6Q3l1v7ViDTjafn2rb2Rf8mjgTW1c32+9/C0e/Px+NH23Ha12XO3tHnbqO1/NzfPXLwrzZw6YVOa2b2/K808+/zENDPhgXyL49EL8q2JS3SgecsVVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFC3fxmaQamQXiNELmjAQoE7P82vSzPhGMgMxmAHU0+oB4F3zI8sbyk1/5BAPBDgExvznRQ1k8vPke2E15kRtHKAzDT5cYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBojohWjwEAAADoF1dYDzHbYXtGq8cBtBrvBTQLcw2oGUzvBQorgJTtq2w/3OpxAMDhgnX3rSisCdudrR4DcCgxx1Ea5iQGO+b4waOwvgPb62xfZ3uFpF22p9q+y/ZW22ttf6Uue7btRba3295k+1bbXS0cPtqM7TNsL7PdbXuh7QW2b6ieu9j28mp+PWp7Tt1x62xfa3uF7der44bXPZ8dWz/HO21/0/bqahzP2L6kyp4sab6kubZ32t5efXyY7e/ZfsH2FtvzbY+oe41vVO+Jl2x/9pB/IdHWWHfRTKy7bSgieLztIWmdpOWSpkgaJWmppOsldUmaLmmNpAur7JmSzpXUKel4SSslXVN3rpA0o9WfE48yH9WcWi/pq5KGSrpU0j5JN0j6oKSXJZ0jqUPSldXcHFYdu07SEkkTJY2t5t7V1XONHNs3x0dUH5tXnWuIpCsk7ZI0oXruKkkPv23sN0v6efXaoyXdI+mm6rmPStoiaXb1HvoJ7wUeB3qw7vJo1oN1tz0fXGHt3y0RsUG1P/hxEfGdiNgXEWsk3S7p45IUEUsjYnFE7I+IdZJuk3R+y0aNdtP3TfeWiHgzIu5WbTGUpC9Iui0iHouInoj4saS91TF9bomIlyJim2oL1+kHeeyGiNgjSRGxsDpXb0QskPS8pLPfadC2Xb3G1yJiW0R0S/quqveFpMsl3RkRT0XELknfftdfIRxOWHfRDKy7bYh7KPq3ofp1mqSJfZfjKx2SHpIk2ydK+oGkD0kaqdrXdGkTx4n2NlHSxqj+elypn3tX2v5y3XNd1TF9Ntf9fnfdc40cu6Hu97L9aUlfV+2KlSQdIemYfsY9TrX5vrS2htZOodp7o+/zqn8frO/nPEA91l00A+tuG6Kw9q9vIm+QtDYiZvaT+5GkZZI+ERHdtq+RdFkzBohBYZOkSbZdt3hOkbRatbl3Y0Tc+C7O28ixv1+sbU9T7QrWhyUtioge28tVWwzfkq28ImmPpFMjYuM7nHtT9Xn0mXqQ48fhiXUXzcC624a4JSC3RFJ3daP0CNsdtmfbPqt6frSkHZJ22j5J0hdbNlK0o0WSeiR9qboB/2P6w4+Dbpd0te1zXDPK9kW2Rzdw3oM9dpRqi+NWSbL9GdV+LNtni6TJff+wJSJ6q9e42fb46phJti+s8j+VdJXtU2yPlPStxr4cgCTWXRxarLttiMKaiIgeSRerdo/KWtX+hnOHpKOqyLWSPimpW7WJtKAFw0Sbioh9qt3w/zlJ2yV9StIvJO2NiMclfV7SrZJek7RKtZvwGznvQR0bEc9I+r5qC/kWSadJeqQu8oCkpyVttv1K9bHrqvMutr1D0q8lzarO9ytJP6yOW1X9CjSEdReHEutue2JrVqAwth+TND8i7mz1WADgcMC6Wz6usAItZvt828dVP5q6UtIcSfe1elwAMFix7rYf/tEV0HqzVLv3aJRq/9fkZRGxqbVDAoBBjXW3zXBLAAAAAIrGLQEAAAAo2gFvCbhgyDwuv+I9u793ofPUwGLuYiAwd9Gumj13mbcYCAeat1xhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGidrR7AobDrb89JMy+dn5/n/ae+lGaOP2JbmnnilYlpZuej4/LXWrglzfQ8tzrNoFz7//zMNLPur4emmeNOejnNjBz6ZppZs2JSmhm/JI3oyP+yOA+hre35m7PTzOZzOtJM16wdaWbMyD1pZuOLY9PM5Hvz8Yy8+7E0g/J0zJzeUG7Lnx+bZrpPyM+zf+LePBQNDOj1fH0fv8Rp5qh/GnxrLldYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQtLba6Wr3pfkOVpL0yhW708z1c/5bmrn8iBfTzMghXWnmkXG9aea2sX+aZh6aclKamb5wTJqRpM7fLG0oh+ZqZBerT/3ZQ2nmr49clmbOHJbP3f80aXya+YfJF6aZ3qFz08yYf1yUZtB8ja67my7Ld/r56gceTDOfOnJlmjm6Y2Sa+c3MfBerv5/y12lmwynnpZnj734lzUhSzzPPNZTDe7d/3OiGctvO3Zdm/n7uf00znz6ysTmQ+eXu4WnmphP/Ms1sGZvP22NvebShMZWCK6wAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAAChaW+10dcSa7oZyLz9/VJr5T2PyXSBePO53aWZqV767xYv73pdm1rx+TJoZvjn/4xq2+fU0I0k9DaXQbEeuyv8O+X9PODPNvPb+fCegxSM3p5lfvTw7zfQ+d0SaGbsin5f5fnBohRGb32go1/tqvkPP6jfGpZnNo/KdroY6H1OH8vfAEV357lyvNrBYsoNVeTpfb2zedm0cm2Z+t2dimnl51Po0M8z5+r51f7674PbdI9LM0S8Ovu/yXGEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAomiOi3ycvGDKv/ychSeo48f1pJkYOSzO9y58ZiOEU6f7ehW72ax7uc9cfyrdU9f58M9TBPC8bwdxt3JDTT0kz207Lt83edlr+Wp6yO830Rv5H17kq3+Ly+J/nW4LH40+lmWZr9txt13nbiFe+MDfNvD4zP0/viHzNHb6pI81MfmBXmvGiJ/IBFehA85YrrAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKFpnqwfQ7nqeW93qIQD/P43svDNot6VBSzSyK9qY5fl5xgzAWAYS7xMc838uyjNNGMfhjiusAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFM0R0eoxAAAAAP3iCisAAACKRmE9xGyH7RmtHgfQarwX0CzMNaBmML0XKKwAUravsv1wq8cBAIcL1t23orAmbHe2egzAocQcR2mYkxjsmOMHj8L6Dmyvs32d7RWSdtmeavsu21ttr7X9lbrs2bYX2d5ue5PtW213tXD4aDO2z7C9zHa37YW2F9i+oXruYtvLq/n1qO05dcets32t7RW2X6+OG173fHZs/RzvtP1N26urcTxj+5Iqe7Kk+ZLm2t5pe3v18WG2v2f7BdtbbM+3PaLuNb5RvSdesv3ZQ/6FRFtj3UUzse62oYjg8baHpHWSlkuaImmUpKWSrpfUJWm6pDWSLqyyZ0o6V1KnpOMlrZR0Td25QtKMVn9OPMp8VHNqvaSvShoq6VJJ+yTdIOmDkl6WdI6kDklXVnNzWHXsOklLJE2UNLaae1dXzzVybN8cH1F9bF51riGSrpC0S9KE6rmrJD38trHfLOnn1WuPlnSPpJuq5z4qaYuk2dV76Ce8F3gc6MG6y6NZD9bd9nxwhbV/t0TEBtX+4MdFxHciYl9ErJF0u6SPS1JELI2IxRGxPyLWSbpN0vktGzXaTd833Vsi4s2IuFu1xVCSviDptoh4LCJ6IuLHkvZWx/S5JSJeiohtqi1cpx/ksRsiYo8kRcTC6ly9EbFA0vOSzn6nQdt29Rpfi4htEdEt6buq3heSLpd0Z0Q8FRG7JH37XX+FcDhh3UUzsO62Ie6h6N+G6tdpkib2XY6vdEh6SJJsnyjpB5I+JGmkal/TpU0cJ9rbREkbo/rrcaV+7l1p+8t1z3VVx/TZXPf73XXPNXLshrrfy/anJX1dtStWknSEpGP6Gfc41eb70toaWjuFau+Nvs+r/n2wvp/zAPVYd9EMrLttiMLav76JvEHS2oiY2U/uR5KWSfpERHTbvkbSZc0YIAaFTZIm2Xbd4jlF0mrV5t6NEXHjuzhvI8f+frG2PU21K1gflrQoInpsL1dtMXxLtvKKpD2STo2Ije9w7k3V59Fn6kGOH4cn1l00A+tuG+KWgNwSSd3VjdIjbHfYnm37rOr50ZJ2SNpp+yRJX2zZSNGOFknqkfSl6gb8j+kPPw66XdLVts9xzSjbF9ke3cB5D/bYUaotjlslyfZnVPuxbJ8tkib3/avGZkQAACAASURBVMOWiOitXuNm2+OrYybZvrDK/1TSVbZPsT1S0rca+3IAklh3cWix7rYhCmsiInokXazaPSprVfsbzh2Sjqoi10r6pKRu1SbSghYME20qIvapdsP/5yRtl/QpSb+QtDciHpf0eUm3SnpN0irVbsJv5LwHdWxEPCPp+6ot5FsknSbpkbrIA5KelrTZ9ivVx66rzrvY9g5Jv5Y0qzrfryT9sDpuVfUr0BDWXRxKrLvtyW+9hQNAq9l+TNL8iLiz1WMBgMMB6275uMIKtJjt820fV/1o6kpJcyTd1+pxAcBgxbrbfvhHV0DrzVLt3qNRqv1fk5dFxKbWDgkABjXW3TbDLQEAAAAoGrcEAAAAoGgHvCXggiHzuPyK9+z+3oXOUwOLuYuBwNxFu2r23GXeYiAcaN5yhRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEU74E5XAAAAOLAhc05KM73Dh+YnWvLkAIxmcOIKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNHaamvWjpnTG8q9edxRaWbImz35iRavaOj1gEyjc3ff5DF55sj8bdu1Y38+pgd/29CYgEZsvXpumnnjI91p5t9NXZ1mhro3zSx9ZXKaefWJ8Wlm2q/eSDND/nVZmkGZhnzg5DTzuy+PSjMzTtiSZo4dkc//Jes/kGa8ZmT+WkvyjjPivy5JMyXhCisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpWzE5XG687L81M/8s1DZ3r21MXpJnZQyPNrNjXkWY27j86zbwZ+Xnu3npGPp4HTkwz0xdsSzOS1PP0sw3lkHv9U+emmS1/0sDOapI+f+5/TzMfGf1UQ+fKPL/vuDSzfNfUNHP3M6enmc71w9PMtHv3pBlJ8iPLG8qhud4Y5zTzkePzded/Gv9gmpnceUSaWX5M/l76P47+cJp5cMTsNDPjzXx3Iknyo080lMN713Hi+xvKrfwfRqeZf3fq79LMFePyXaMuGpnvmvaPY49JM7cd/Sdp5vWt+fo+Ik2UhSusAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoWjE7XXXsHbhz9UYjPTzfeWjW0HxQ5w7f0cBr5UZ3PJJmvjc3391ldcfEhl7vhP/n1DQTy55u6FyHu459ecZdvQ2da2fPsDQzZkj+gid05jtLnT3s1TRz/oj1aeb0s15IM3cc+8dpZt/D49OMJOVfIbTCkWvzOf7ghplpZnxXd5qZ3JXv6LdyT74WLtmU7+Q2el3+/YQdrMrT89zqhnIjNxybZhaNmZ5mtu8bmWZ+NfK1NPP8jnFp5qW1+W5YEzY39j2nnXCFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGiOiH6fvGDIvP6fLNj+Pz8zzfSMaKCrN7Cz2Y7j891t94x3muncnb/WERvzAY15+vX8RJJ6n1jZUG4g3N+7MP8CDLDS5m7Hie9vKNc7Ot9SdcfM0Wlm9/h8fu/PdxJsSOeuPHPkhv1pZsTPlgzAaAYWc7f5ev7sjDSzfXq+Qe/eo/M/ukbW1CP/y+I0U6Jmz93Dfd42omPWjDTT8+yqJoykXAeat1xhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0fJtmtpQ5wNL88wAvda4ATrPQGlgcy60QM9zqwfsXKPz6a18LyygTB0P/jbNvO/BJgwEGGCH+y5W7xVXWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACiaI6LVYwAAAAD6xRXWQ8x22J7R6nEArcZ7Ac3CXANqBtN7gcIKIGX7KtsPt3ocAHC4YN19KwprwnZnq8cAHErMcZSGOYnBjjl+8Cis78D2OtvX2V4haZftqbbvsr3V9lrbX6nLnm17ke3ttjfZvtV2VwuHjzZj+wzby2x3215oe4HtG6rnLra9vJpfj9qeU3fcOtvX2l5h+/XquOF1z2fH1s/xTtvftL26Gsczti+psidLmi9pru2dtrdXHx9m+3u2X7C9xfZ82yPqXuMb1XviJdufPeRfSLQ11l00E+tuG4oIHm97SFonabmkKZJGSVoq6XpJXZKmS1oj6cIqe6akcyV1Sjpe0kpJ19SdKyTNaPXnxKPMRzWn1kv6qqShki6VtE/SDZI+KOllSedI6pB0ZTU3h1XHrpO0RNJESWOruXd19Vwjx/bN8RHVx+ZV5xoi6QpJuyRNqJ67StLDbxv7zZJ+Xr32aEn3SLqpeu6jkrZIml29h37Ce4HHgR6suzya9WDdbc8HV1j7d0tEbFDtD35cRHwnIvZFxBpJt0v6uCRFxNKIWBwR+yNinaTbJJ3fslGj3fR9070lIt6MiLtVWwwl6QuSbouIxyKiJyJ+LGlvdUyfWyLipYjYptrCdfpBHrshIvZIUkQsrM7VGxELJD0v6ex3GrRtV6/xtYjYFhHdkr6r6n0h6XJJd0bEUxGxS9K33/VXCIcT1l00A+tuG+Ieiv5tqH6dJmli3+X4SoekhyTJ9omSfiDpQ5JGqvY1XdrEcaK9TZS0Maq/Hlfq596Vtr9c91xXdUyfzXW/3133XCPHbqj7vWx/WtLXVbtiJUlHSDqmn3GPU22+L62tobVTqPbe6Pu86t8H6/s5D1CPdRfNwLrbhiis/eubyBskrY2Imf3kfiRpmaRPRES37WskXdaMAWJQ2CRpkm3XLZ5TJK1Wbe7dGBE3vovzNnLs7xdr29NUu4L1YUmLIqLH9nLVFsO3ZCuvSNoj6dSI2PgO595UfR59ph7k+HF4Yt1FM7DutiFuCcgtkdRd3Sg9wnaH7dm2z6qeHy1ph6Sdtk+S9MWWjRTtaJGkHklfqm7A/5j+8OOg2yVdbfsc14yyfZHt0Q2c92CPHaXa4rhVkmx/RrUfy/bZImly3z9siYje6jVutj2+OmaS7Qur/E8lXWX7FNsjJX2rsS8HIIl1F4cW624borAmIqJH0sWq3aOyVrW/4dwh6agqcq2kT0rqVm0iLWjBMNGmImKfajf8f07SdkmfkvQLSXsj4nFJn5d0q6TXJK1S7Sb8Rs57UMdGxDOSvq/aQr5F0mmSHqmLPCDpaUmbbb9Sfey66ryLbe+Q9GtJs6rz/UrSD6vjVlW/Ag1h3cWhxLrbntiaFSiM7cckzY+IO1s9FgA4HLDulo8rrECL2T7f9nHVj6aulDRH0n2tHhcADFasu+2Hf3QFtN4s1e49GqXa/zV5WURsau2QAGBQY91tM9wSAAAAgKJxSwAAAACKdsBbAi4YMo/Lr3jP7u9d6Dw1sJi7GAjMXbSrZs9d5i0GwoHmLVdYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKFpnqwdwMDpmTm8ot+Fvjksze8/YlWaOG7sjzWzdcUSaiWdGp5lJ/7o3zXQ+sDTNoL3t+MS5aWbzn/akmdNO2pBmOp2fZ9mzx6eZ8Q/ly8iYf1yUZjD4bfr6eWmm+9R9aWb0+/L1u6cnvx6z7/kj08z0n+WvpcUr8gza1pr/ZW6aGTP71TQzqiuf2+tfOCbNTL63I82MvPuxNNNuuMIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICitdVOV1v/+NiGckd9eHOa+e6Jd6eZPxmev9aL+3emmfkz810y/vnYPDOj54NpZsi/LkszaL6OU2c1lNv8kf1p5rbzf5xm/v3IN9PMyz35Dj7/NPa0NDN/64VpZkyaQLtb/b/la9h//Iv70szVY36XZkYO6UozT+/bk2ZumvAXaWbFy6ekmQmL0wgKtfr7+e6C5533dJr5+4n3ppkThuY7Y/78hJFp5h8mfzTNvDo2fz++74722oGQK6wAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAAChaW+10dfTvdjeUW/P0+DTzvw7Pdzj5zdEvpJkX9xydZh589sQ0M+nXTjPsYtW+ep5+tqHckU+el2aun/CxNHPvsevSzBPbJqWZF56akGZm3rUjzUSaQLs7cnWe+dnGD6SZYUPyXdqGuifNLN85Nc08+vz0NDPluXz3ObSvYx/LMw8fMyPN/M+9f5VmLjpmRZp5cd/YNPPqjlFpZuTQNNJ2uMIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNEf0v2niBUPmsaMi3rP7exfm+84OMOYuBgJzt/k6ZuXbYO4fm29N2bEn3+K1d/kzDY2pHTV77h7u8zbm5tsOv3HssPw8Q/I/tpF3N7CfbJs60LzlCisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIrW2eoBAADQp+fZVWmmkS2cet/7UICGedETaWZEE8YxmHGFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKI5Ilo9BgAAAKBfXGEFAABA0Sish5jtsD2j1eMAWo33ApqFuQbUDKb3AoUVQMr2VbYfbvU4AOBwwbr7VhTWhO3OVo8BOJSY4ygNcxKDHXP84FFY34Htdbavs71C0i7bU23fZXur7bW2v1KXPdv2ItvbbW+yfavtrhYOH23G9hm2l9nutr3Q9gLbN1TPXWx7eTW/HrU9p+64dbavtb3C9uvVccPrns+OrZ/jnba/aXt1NY5nbF9SZU+WNF/SXNs7bW+vPj7M9vdsv2B7i+35tkfUvcY3qvfES7Y/e8i/kGhrrLtoJtbdNhQRPN72kLRO0nJJUySNkrRU0vWSuiRNl7RG0oVV9kxJ50rqlHS8pJWSrqk7V0ia0erPiUeZj2pOrZf0VUlDJV0qaZ+kGyR9UNLLks6R1CHpympuDquOXSdpiaSJksZWc+/q6rlGju2b4yOqj82rzjVE0hWSdkmaUD13laSH3zb2myX9vHrt0ZLukXRT9dxHJW2RNLt6D/2E9wKPAz1Yd3k068G6254PrrD275aI2KDaH/y4iPhOROyLiDWSbpf0cUmKiKURsTgi9kfEOkm3STq/ZaNGu+n7pntLRLwZEXerthhK0hck3RYRj0VET0T8WNLe6pg+t0TESxGxTbWF6/SDPHZDROyRpIhYWJ2rNyIWSHpe0tnvNGjbrl7jaxGxLSK6JX1X1ftC0uWS7oyIpyJil6Rvv+uvEA4nrLtoBtbdNsQ9FP3bUP06TdLEvsvxlQ5JD0mS7RMl/UDShySNVO1rurSJ40R7myhpY1R/Pa7Uz70rbX+57rmu6pg+m+t+v7vuuUaO3VD3e9n+tKSvq3bFSpKOkHRMP+Mep9p8X1pbQ2unUO290fd51b8P1vdzHqAe6y6agXW3DVFY+9c3kTdIWhsRM/vJ/UjSMkmfiIhu29dIuqwZA8SgsEnSJNuuWzynSFqt2ty7MSJufBfnbeTY3y/WtqepdgXrw5IWRUSP7eWqLYZvyVZekbRH0qkRsfEdzr2p+jz6TD3I8ePwxLqLZmDdbUPcEpBbIqm7ulF6hO0O27Ntn1U9P1rSDkk7bZ8k6YstGyna0SJJPZK+VN2A/zH94cdBt0u62vY5rhll+yLboxs478EeO0q1xXGrJNn+jGo/lu2zRdLkvn/YEhG91WvcbHt8dcwk2xdW+Z9Kusr2KbZHSvpWY18OQBLrLg4t1t02RGFNRESPpItVu0dlrWp/w7lD0lFV5FpJn5TUrdpEWtCCYaJNRcQ+1W74/5yk7ZI+JekXkvZGxOOSPi/pVkmvSVql2k34jZz3oI6NiGckfV+1hXyLpNMkPVIXeUDS05I2236l+th11XkX294h6deSZlXn+5WkH1bHrap+BRrCuotDiXW3Pfmtt3AAaDXbj0maHxF3tnosAHA4YN0tH1dYgRazfb7t46ofTV0paY6k+1o9LgAYrFh32w//6ApovVmq3Xs0SrX/a/KyiNjU2iEBwKDGuttmuCUAAAAAReOWAAAAABTtgLcEXDBkHpdf8Z7d37vQeWpgMXcxEJi7aFfNnrvMWwyEA81brrACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQtM5WD6BkPvPUATnPkB170kzP82sG5LUAAEBj/MH8+3zPEV1pJjqdZoa80ZNmOrfvzsez8vk0MxhxhRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEUblDtdbf7qeWmm8yOvpJm/mLIizUzo2p5mVu6amGbuXzMnzYx4+Ig0M+lnL6QZSdq/4cWGcmiu3Zeck2Y2fiQ/z7zzHkszozveSDPP7RqfZh56claaOfGO/LW05Mk8g6bb8YlzG8ptOS/SzJw569LM2UfnmdW7x6WZh9a+P80M++2oNDPlZ5vTDDsVlueFb+c9QJJO+8izaebMo/Lvqx8YkWeefGNymvnJ6rPSzO6n5qaZE/7f7jQTjz+VZkrCFVYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABStrXa66jgx37lEkrrf35Nm/m7q02nm80fnuwVN7sx3n1ox8vk0s6unK8089MLsNLN/wtFpRpLETldFem1WR5o5/bR8Pn366EVp5tSuEWnmn4dtTTOPj52aZt4cMzzNDE0TGGgdJ89MM6/OcUPnunju0jRz7fgH08zURtbUfU+kmVGde9PMPd2np5k3puVr6tD8LYkB5A/l3wvjlHynJ0maMuK1NPPpo5almfEdI9PMR0euTjNv9ua17I4X/zTN9AzPz9NuVyzbbbwAAAA4zFBYAQAAUDQKKwAAAIpGYQUAAEDRKKwAAAAoGoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQtLbamrXnuXxbM0ka92/j08x/HnNOmnny+Ilp5n3DdqeZZ7fn43npqWPTzNT730wzWvJknkGxJizak2aWTT0hzXxh939IM6O78q0rn10zIc1M/mW+nezQf8m3OUbz9azM9xSdfne+ra4k/ab7rDRzz8w5aaZzWL619v5d+Ua+o1blmZP+2/Y007v8mTSD5orHn0ozx/04n4+SdPffnJlmts7Otws+9YhNaWbV7rwL/ObZWWlm2j29aWbIw8vTTLvhCisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAolFYAQAAUDQKKwAAAIrmiOj3yQuGzOv/SaBB9/cudLNfk7mLgcDcRbtq9txl3mIgHGjecoUVAAAARaOwAgAAoGgUVgAAABSNwgoAAICiUVgBAABQNAorAAAAikZhBQAAQNEorAAAACgahRUAAABFo7ACAACgaBRWAAAAFI3CCgAAgKJRWAEAAFA0CisAAACKRmEFAABA0SisAAAAKBqFFQAAAEWjsAIAAKBoFFYAAAAUjcIKAACAojkiWj0GAAAAoF9cYT3EbIftGa0eB9BqvBfQLMw1oGYwvRcorABStq+y/XCrxwEAhwvW3beisCZsd7Z6DMChxBxHaZiTGOyY4wePwvoObK+zfZ3tFZJ22Z5q+y7bW22vtf2VuuzZthfZ3m57k+1bbXe1cPhoM7bPsL3MdrfthbYX2L6heu5i28ur+fWo7Tl1x62zfa3tFbZfr44bXvd8dmz9HO+0/U3bq6txPGP7kip7sqT5kuba3ml7e/XxYba/Z/sF21tsz7c9ou41vlG9J16y/dlD/oVEW2PdRTOx7rahiODxtoekdZKWS5oiaZSkpZKul9QlabqkNZIurLJnSjpXUqek4yWtlHRN3blC0oxWf048ynxUc2q9pK9KGirpUkn7JN0g6YOSXpZ0jqQOSVdWc3NYdew6SUskTZQ0tpp7V1fPNXJs3xwfUX1sXnWuIZKukLRL0oTquaskPfy2sd8s6efVa4+WdI+km6rnPippi6TZ1XvoJ7wXeBzowbrL4/9r796DrKzvPI9/vt1Nc2mQi4AKAt4QVHTRKGgytSSbdc2uVoxZycXNCsZJxmzlYrKmTM3UTrKumpraRFOUqeiYHdfsJDuEMTObmNEZE2NKI0gkEOIlRi6NLTQIgYbmIg3d3/3jeTo5pmy+B9Oc8z3N+1V1iq4+n+d5fn34nV9/+ukHnlo9WHcb88EZ1oEtcfcOFX/xk9z9NnfvcfcNku6X9CFJcvdV7r7C3Q+7e7uk+yQtqNuo0Wj6v+kucfdD7v49FYuhJH1c0n3u/oy797r7g5IOltv0W+LuW9x9p4qFa+5Rbtvh7gckyd2Xlfvqc/elkl6WNO/NBm1mVh7js+6+0927Jd2p8n0h6QOSHnD359x9n6QvveVXCMcT1l3UAutuA+IaioF1lH/OkDSl/3R8qVnSk5JkZmdLukvSxZJGqXhNV9VwnGhsUyRt9vLH41Ll3FtkZp+qeK613Kbf1oqP91c8V822HRUfy8yul/Q5FWesJGm0pIkDjHuSivm+qlhDi12oeG/0f12V74NNA+wHqMS6i1pg3W1AFNaB9U/kDkkb3X3mALlvSFot6cPu3m1mN0u6thYDxJDQKWmqmVnF4jlN0noVc+8Od7/jLey3mm1/t1ib2QwVZ7DeLWm5u/ea2RoVi+EbsqUdkg5IOs/dN7/JvjvLr6Pf9KMcP45PrLuoBdbdBsQlAbGVkrrLC6VHmlmzmc0xs0vK58dI2iNpr5nNlvSJuo0UjWi5pF5JnywvwL9av/910P2SbjKz+VZoM7MrzWxMFfs92m3bVCyO2yXJzG5Q8WvZftskndr/D1vcva88xt1mNrncZqqZXVHmvytpsZmda2ajJH2xupcDkMS6i2OLdbcBUVgD7t4r6SoV16hsVPETzjcljS0jt0i6TlK3iom0tA7DRINy9x4VF/zfKKlL0kckPSzpoLs/K+ljku6RtEvSOhUX4Vez36Pa1t1fkPRVFQv5NknnS/pZReRxSc9L2mpmO8rP3Vrud4WZ7ZH0I0mzyv09Iulr5Xbryj+BqrDu4lhi3W1M3JoVSMbMnpF0r7s/UO+xAMDxgHU3P86wAnVmZgvM7OTyV1OLJF0g6dF6jwsAhirW3cbDP7oC6m+WimuP2lT8X5PXuntnfYcEAEMa626D4ZIAAAAApMYlAQAAAEjtiJcEXN60kNOv+KM91rfM4tTgYu5iMDB30ahqPXeZtxgMR5q3nGEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApNZS7wH0a5p7bph55T+Mq2pf+6cfjo/3etzVmw5amDl8Qm+YsbZ4PM2dw8PMlKfiY434wcowg9rrueLiqnL7Th4WZnafXcXxpvRUdbyIdcdLxJj1zWFmwkuHwkzroz+vakyosUsvqCq28eq2MDPugh1h5tQxXWHmpe2Tw0zT8rFhZvpDm8PM4Y2bwgwa196F88PM1vfG62lLa/z9uacr/j4/YVW85k786+VhZijiDCsAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUktzp6uO98R3sRr/r7dWta8/P/2nYebyUa/Ex2saEWa29R4IM5sOjwozHYdODDN/d+m8MPObuW8PM5I07X88XVUOsc23xq953yV7qtrXvz3tl2HmugkrwsyMlnheVmNbb3znrb/esSDMPPby7DAzYdJlVY1p3P85Pu/yUi/b5o2uKnfWpe1h5s7T/iHMzB0e3w1ozbSDYebPJ1wTZjb3nBZmTlrCna4a0Z7rLq0qN+KGzjDz8Mz/G2bOaY2/z796eG+Y+cL5V4WZVSfH33Om3zb0vsdzhhUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqaW501U1hjX3VpUb0xTf5WeUwK8WpAAAHqxJREFUNcfHqyLTZnHnP+Txy/zi61PCTNfBkWGmZ2xfmJEkf8fcMGM/W1PVvo53XsWPfaNHxnfmkaRZo+K7uU1qjuf3KS3x3YnWH4rvurK5d2yYqYa7hZlDbYNyKAyyk5ZUd8ecTW3x3XduftcHw8z8Se3xsfZPCDMv/mZqmJmx7lCYQWMat7arqtyvN0wOM18f964wc/GYjWFm26GZYealnfF42jo9zAxFnGEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRm7gPf4uvypoU1u/9X05zZYWbLu+Pb8UnS3tPi25P2jaziFqYtccb2x7dvHb4zzoxdFx9rwupdYab3+ZfCTK091rcsvi/nIKvl3K3GrsWXVZXrnlHFS1XFV9YzPp5Prbvjn1eH74yP1dYZ3zJ51NaeMNP05Or4YDXG3EWjqvXcPd7nbfPMM8JM78sbajCSxnakecsZVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACk1lLvAfTre+7XYebk52owkMTi+wkhq/H/e3l1uWM8DgDA4OMuVsceZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqZm713sMAAAAwIA4wwoAAIDUKKzHmJm5mZ1V73EA9cZ7AbXCXAMKQ+m9QGEFEDKzxWb2VL3HAQDHC9bdN6KwBsyspd5jAI4l5jiyYU5iqGOOHz0K65sws3Yzu9XM1kraZ2bTzewhM9tuZhvN7NMV2XlmttzMusys08zuMbPWOg4fDcbMLjKz1WbWbWbLzGypmd1ePneVma0p59fTZnZBxXbtZnaLma01s93ldiMqno+2rZzjLWb2BTNbX47jBTO7psyeI+leSZeZ2V4z6yo/P9zMvmJmr5jZNjO718xGVhzj8+V7YouZffSYv5BoaKy7qCXW3Qbk7jz+4CGpXdIaSdMktUlaJekvJbVKOkPSBklXlNm3SbpUUouk0yS9KOnmin25pLPq/TXxyPko59QmSZ+RNEzS+yX1SLpd0oWSXpM0X1KzpEXl3BxebtsuaaWkKZImlHPvpvK5arbtn+Mjy88tLPfVJOmDkvZJOqV8brGkp/5g7HdL+n557DGSfiDpy+Vz75G0TdKc8j30Hd4LPI70YN3lUasH625jPjjDOrAl7t6h4i9+krvf5u497r5B0v2SPiRJ7r7K3Ve4+2F3b5d0n6QFdRs1Gk3/N90l7n7I3b+nYjGUpI9Lus/dn3H3Xnd/UNLBcpt+S9x9i7vvVLFwzT3KbTvc/YAkufuycl997r5U0suS5r3ZoM3MymN81t13unu3pDtVvi8kfUDSA+7+nLvvk/Slt/wK4XjCuotaYN1tQFxDMbCO8s8Zkqb0n44vNUt6UpLM7GxJd0m6WNIoFa/pqhqOE41tiqTNXv54XKqce4vM7FMVz7WW2/TbWvHx/ornqtm2o+Jjmdn1kj6n4oyVJI2WNHGAcU9SMd9XFWtosQsV743+r6vyfbBpgP0AlVh3UQusuw2Iwjqw/oncIWmju88cIPcNSaslfdjdu83sZknX1mKAGBI6JU01M6tYPKdJWq9i7t3h7ne8hf1Ws+3vFmszm6HiDNa7JS13914zW6NiMXxDtrRD0gFJ57n75jfZd2f5dfSbfpTjx/GJdRe1wLrbgLgkILZSUnd5ofRIM2s2szlmdkn5/BhJeyTtNbPZkj5Rt5GiES2X1Cvpk+UF+Ffr978Oul/STWY23wptZnalmY2pYr9Hu22bisVxuySZ2Q0qfi3bb5ukU/v/YYu795XHuNvMJpfbTDWzK8r8dyUtNrNzzWyUpC9W93IAklh3cWyx7jYgCmvA3XslXaXiGpWNKn7C+aaksWXkFknXSepWMZGW1mGYaFDu3qPigv8bJXVJ+oikhyUddPdnJX1M0j2Sdklap+Ii/Gr2e1TbuvsLkr6qYiHfJul8ST+riDwu6XlJW81sR/m5W8v9rjCzPZJ+JGlWub9HJH2t3G5d+SdQFdZdHEusu43J3ngJB4B6M7NnJN3r7g/UeywAcDxg3c2PM6xAnZnZAjM7ufzV1CJJF0h6tN7jAoChinW38fCProD6m6Xi2qM2Ff/X5LXu3lnfIQHAkMa622C4JAAAAACpcUkAAAAAUjviJQGXNy3k9Cv+aI/1LbM4NbiYuxgMzF00qlrPXeYtBsOR5i1nWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACptdR7AJn1/cncMDNs+94w0/vSusEYDo4DLadNDzM9004MM69Pag0zzQf6wszIju4w0/fcr8MMctr//vlhpntqc1X76ounnA61xZmmw3FmVKeHmXHrXo+P9eTq+GBoSNXMbUnaOTue3wdO7g0zTQfj838jt1uYmfjLnjDT+s/PhpmhiDOsAAAASI3CCgAAgNQorAAAAEiNwgoAAIDUKKwAAABIjcIKAACA1CisAAAASI3CCgAAgNQorAAAAEhtSN7pqv32y8LMR6/+UZi5YdzX42Mdjm/vcucrV4WZ9Q+fGWam/M+nwwxy6n3nRVXl1r03nk/nva09zPzJuFfCzKjmg2Gm4/UJYeafXrowzIz/8YgwM+FvlocZVK/5rNPDzO7T47v89L59d1XHu+qM58PMfxz38zAzZ1h8F6u9fijMPHFgSpj5xqZ3hpnXHp8aZk5buiXMSNLhDe1V5XBkXdfH3+MPXbuzqn3dNvuRMDN/RPz3e2rL6DDz4wPx++2hnZeEmUeunhdmzv5WfKc3rVgbZxLhDCsAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUhuSd7rqi28WpFFNPWHmxKaRYWby8Ljz33zqY2HmS5fHx9r1Wnx3j/EPcregjHqrmCeS1DfucJh5x4T1YebGcWvCzMTmtjCz8dAvw8zwpnjMDx2O74Y17uU4I0lNT66uKne86123McxM/XG87mwaMa6q4/1T07lh5pxz4jsGzRn2apiZXMXcPa91a5iZN3FTmPnuzElhZu95k8OMJI3gTleDYty34u9zW05+e1X7+u99V4aZxWc9E2auGvOrMDPM4nk7Y8Rvw8yEaV1h5rW3TQwzp/z2jDAjSb0vb6gqd6xxhhUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkJq5+4BPXt60cOAnG9zOj8a3Od05J/7y+8bGt6W0A81hZuyLcWby158OMxk91rfMan3MRp27fQvi25O++q74dpqtc3eFmTEjDoaZtmHxLYw3/XZ8mJnwUHxLwjFLV4SZWmPuDq69C+eHmV2z47Xw4Pi+MGO98V/dyNfizKRfxu+BYf/ybJiptVrP3aE8bzXv/DCy++x4jdt9ZnyOsDlelnXCxnj+j+44EGbs6fjW27V2pHnLGVYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApNZS7wHUy4S/WR5najAOoFLTT1eHmek/rcFAjsIMvVrvIaBBjF72TJypwTiAo7LyV2Fk7Mp4N2MHYSjHM86wAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFIzd6/3GAAAAIABcYb1GDMzN7Oz6j0OoN54L6BWmGtAYSi9FyisAEJmttjMnqr3OADgeMG6+0YU1oCZtdR7DMCxxBxHNsxJDHXM8aNHYX0TZtZuZrea2VpJ+8xsupk9ZGbbzWyjmX26IjvPzJabWZeZdZrZPWbWWsfho8GY2UVmttrMus1smZktNbPby+euMrM15fx62swuqNiu3cxuMbO1Zra73G5ExfPRtpVzvMXMvmBm68txvGBm15TZcyTdK+kyM9trZl3l54eb2VfM7BUz22Zm95rZyIpjfL58T2wxs48e8xcSDY11F7XEutuA3J3HHzwktUtaI2mapDZJqyT9paRWSWdI2iDpijL7NkmXSmqRdJqkFyXdXLEvl3RWvb8mHjkf5ZzaJOkzkoZJer+kHkm3S7pQ0muS5ktqlrSonJvDy23bJa2UNEXShHLu3VQ+V822/XN8ZPm5heW+miR9UNI+SaeUzy2W9NQfjP1uSd8vjz1G0g8kfbl87j2StkmaU76HvsN7gceRHqy7PGr1YN1tzAdnWAe2xN07VPzFT3L329y9x903SLpf0ockyd1XufsKdz/s7u2S7pO0oG6jRqPp/6a7xN0Pufv3VCyGkvRxSfe5+zPu3uvuD0o6WG7Tb4m7b3H3nSoWrrlHuW2Hux+QJHdfVu6rz92XSnpZ0rw3G7SZWXmMz7r7TnfvlnSnyveFpA9IesDdn3P3fZK+9JZfIRxPWHdRC6y7DYhrKAbWUf45Q9KU/tPxpWZJT0qSmZ0t6S5JF0sapeI1XVXDcaKxTZG02csfj0uVc2+RmX2q4rnWcpt+Wys+3l/xXDXbdlR8LDO7XtLnVJyxkqTRkiYOMO5JKub7qmINLXah4r3R/3VVvg82DbAfoBLrLmqBdbcBUVgH1j+ROyRtdPeZA+S+IWm1pA+7e7eZ3Szp2loMEENCp6SpZmYVi+c0SetVzL073P2Ot7Dfarb93WJtZjNUnMF6t6Tl7t5rZmtULIZvyJZ2SDog6Tx33/wm++4sv45+049y/Dg+se6iFlh3GxCXBMRWSuouL5QeaWbNZjbHzC4pnx8jaY+kvWY2W9In6jZSNKLlknolfbK8AP9q/f7XQfdLusnM5luhzcyuNLMxVez3aLdtU7E4bpckM7tBxa9l+22TdGr/P2xx977yGHeb2eRym6lmdkWZ/66kxWZ2rpmNkvTF6l4OQBLrLo4t1t0GRGENuHuvpKtUXKOyUcVPON+UNLaM3CLpOkndKibS0joMEw3K3XtUXPB/o6QuSR+R9LCkg+7+rKSPSbpH0i5J61RchF/Nfo9qW3d/QdJXVSzk2ySdL+lnFZHHJT0vaauZ7Sg/d2u53xVmtkfSjyTNKvf3iKSvldutK/8EqsK6i2OJdbcxcWtWIBkze0bSve7+QL3HAgDHA9bd/DjDCtSZmS0ws5PLX00tknSBpEfrPS4AGKpYdxsP/+gKqL9ZKq49alPxf01e6+6d9R0SAAxprLsNhksCAAAAkBqXBAAAACC1I14ScHnTQk6/4o/2WN8yi1ODi7mLwcDcRaOq9dxl3mIwHGnecoYVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkFpLvQdwLHT958vCzPZ5fWFm2OQDYaanuzXMnPB8nJn66PYw0/viy2EGja2aubvryn1hZtyYeO62NveGmc2vnBhmZvy/MKLhP/x5HMKQt//988PMtovj8yiHxsdzd0Rn/O3t1Mf3h5mmp9aEGQxtv/3TeF3umhXvp3f8oTh0KJ7/Z/x9PP9bfrwqPlaD4QwrAAAAUqOwAgAAIDUKKwAAAFKjsAIAACA1CisAAABSo7ACAAAgNQorAAAAUqOwAgAAIDUKKwAAAFJrqDtdVXMXIEna8969Yeauuf8QZt7XFu9nf19PmPnbd5wWZv7qoivCzPRvXxJmWh/ljkIZ7fiz6uauX7kzzNw686dh5saxW8PMIY/vlvLDM8eGmTun/vsw03Xm28PMSUueDjPIa9eieI43f+i1MPO3s/8uzMwbPizMPHEgPh/zZ+d9JMxMG3ZRmGn+yS/CDHLa+pl4bTr5va+EmaVnxfP2zGGjw8yj+4eHmU+MiOft7G2zw0zfc78OM5lwhhUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqTXUna7Gv7CnqtzO804IM986Kb4ry+snPRtmJjTHd8N65eCJYcbdwkxfS5xBThPvW15Vbv3Zl4aZh8bEd97Z3RvfwWTOiI4w094zMcxU41DboOwGiU16sjPMbJgxJcz8t9ZrwszcCa+GmWd/Oz3MDFsd33mo+SfcgW0oO/UHW8LMS6dOCzN/2vefwsy8iZvCTFXztrM1zPiIg2Gm0XCGFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQmrn7gE9e3rRw4CcTO/C+eWGme0p8V9oRXX1hpnVPnBnZuS/M+Krnw0yjeqxvWc3vKduoc7cauxbFtxXec0b8kltvfKzhXXFm4trXw0zzE7+Id5QQcxeNqtZzl3mLwXCkecsZVgAAAKRGYQUAAEBqFFYAAACkRmEFAABAahRWAAAApEZhBQAAQGoUVgAAAKRGYQUAAEBqFFYAAACkFt/uqQGN/MeVcaYG4+jH7T8wmMY/uDzO1GAcAADUCmdYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKmZu9d7DAAAAMCAOMMKAACA1Cisx5iZuZmdVe9xAPXGewG1wlwDCkPpvUBhBRAys8Vm9lS9xwEAxwvW3TeisAbMrKXeYwCOJeY4smFOYqhjjh89CuubMLN2M7vVzNZK2mdm083sITPbbmYbzezTFdl5ZrbczLrMrNPM7jGz1joOHw3GzC4ys9Vm1m1my8xsqZndXj53lZmtKefX02Z2QcV27WZ2i5mtNbPd5XYjKp6Ptq2c4y1m9gUzW1+O4wUzu6bMniPpXkmXmdleM+sqPz/czL5iZq+Y2TYzu9fMRlYc4/Ple2KLmX30mL+QaGisu6gl1t0G5O48/uAhqV3SGknTJLVJWiXpLyW1SjpD0gZJV5TZt0m6VFKLpNMkvSjp5op9uaSz6v018cj5KOfUJkmfkTRM0vsl9Ui6XdKFkl6TNF9Ss6RF5dwcXm7bLmmlpCmSJpRz76byuWq27Z/jI8vPLSz31STpg5L2STqlfG6xpKf+YOx3S/p+eewxkn4g6cvlc++RtE3SnPI99B3eCzyO9GDd5VGrB+tuYz44wzqwJe7eoeIvfpK73+buPe6+QdL9kj4kSe6+yt1XuPthd2+XdJ+kBXUbNRpN/zfdJe5+yN2/p2IxlKSPS7rP3Z9x9153f1DSwXKbfkvcfYu771SxcM09ym073P2AJLn7snJffe6+VNLLkua92aDNzMpjfNbdd7p7t6Q7Vb4vJH1A0gPu/py775P0pbf8CuF4wrqLWmDdbUBcQzGwjvLPGZKm9J+OLzVLelKSzOxsSXdJuljSKBWv6aoajhONbYqkzV7+eFyqnHuLzOxTFc+1ltv021rx8f6K56rZtqPiY5nZ9ZI+p+KMlSSNljRxgHFPUjHfVxVraLELFe+N/q+r8n2waYD9AJVYd1ELrLsNiMI6sP6J3CFpo7vPHCD3DUmrJX3Y3bvN7GZJ19ZigBgSOiVNNTOrWDynSVqvYu7d4e53vIX9VrPt7xZrM5uh4gzWuyUtd/deM1ujYjF8Q7a0Q9IBSee5++Y32Xdn+XX0m36U48fxiXUXtcC624C4JCC2UlJ3eaH0SDNrNrM5ZnZJ+fwYSXsk7TWz2ZI+UbeRohEtl9Qr6ZPlBfhX6/e/Drpf0k1mNt8KbWZ2pZmNqWK/R7ttm4rFcbskmdkNKn4t22+bpFP7/2GLu/eVx7jbzCaX20w1syvK/HclLTazc81slKQvVvdyAJJYd3Fsse42IAprwN17JV2l4hqVjSp+wvmmpLFl5BZJ10nqVjGRltZhmGhQ7t6j4oL/GyV1SfqIpIclHXT3ZyV9TNI9knZJWqfiIvxq9ntU27r7C5K+qmIh3ybpfEk/q4g8Lul5SVvNbEf5uVvL/a4wsz2SfiRpVrm/RyR9rdxuXfknUBXWXRxLrLuNyd54CQeAejOzZyTd6+4P1HssAHA8YN3NjzOsQJ2Z2QIzO7n81dQiSRdIerTe4wKAoYp1t/Hwj66A+pul4tqjNhX/1+S17t5Z3yEBwJDGuttguCQAAAAAqXFJAAAAAFI74iUBlzct5PQr/miP9S2zODW4mLsYDMxdNKpaz13mLQbDkeYtZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqbXUewDHwoH3zQszr76vN8zMPeOVMNPncedfu2lqmDnxJ8PDzIQHlocZNLZtn3p7mNl90cEw86/OeDXMTBm1O8zs7BkVZlY/MSvMnPYXzN2Met5zSZjZvKC6bxMtM7vDzNTx8Zw7pYp5edLw+FgvdZ8UZp7bGK/NY9bEa/PUH24NM5LU+/KGqnI4suZZZ4WZre+aVNW+ds2Nu8DwCQfCTGvr4TDT0tQXZnZvGB9mTqpiOR37YleY6Vv763hHiXCGFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpDck7XW1+Z9zD/+u8R8LMx8e2h5lh1hxm/mXqsDBz16n/Lsysn31ZmDnz7/eGGUnyn/+qqhxq68BJHmauv2hFmPn8iavCzOimEWHmiQPxe+kz58V3FNq1KJ674x/kbliDyS6eE2ZefWf8LeC8y6q7O9N/mfp4mJk/Yk+YGWWtYaaadXfjifFa+L/GxvPy24cuDTPjXzoxzEjScO50NShevHlCmPk3F1b3PW7R5KfCzLTmeC6dPmx0mPnNoX1h5smZZ4aZb856R5jZ/r3JYWZS3+wwI0l9z+W4IxZnWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQ2pC809WEtRZm7pn6zjDTfuZzYWbq8F1h5lfdp8bH2hHfuaO1K/66mvYdDDOS1FtVCrU29YmeMPPt0+eFmd2zR4aZycO6w8w/dlwQZg6vHB9mTn7w6TCDweXPxuvXpDPjuzj9qvX0qo73F93XhJlpJ8Tr5f7D8Z2u9vYMDzOvdcV3Hmp6Kc6c+cTrYab5J78IMxg8456Pq8vjLedUta9XzxgXZmaesD3MDG86FGb2HI7X5SfWzwwzviW+S+HUrfF3+Sx3sKoWZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKmZuw/45OVNCwd+8jjQc8XFYaZ72rAwM2xf/DKOe2F3mOn75YthJqPH+pbF95QdZMf73O1bcGEc6o1fomHb98a7eWldNUNqSMxdNKpaz13mLQbDkeYtZ1gBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkFpLvQeQWes/PxtmThykY/UN0n4ASWr66epB2U/voOwFAIA/DmdYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKlRWAEAAJAahRUAAACpUVgBAACQGoUVAAAAqVFYAQAAkBqFFQAAAKmZu9d7DAAAAMCAOMMKAACA1CisAAAASI3CCgAAgNQorAAAAEiNwgoAAIDUKKwAAABI7f8DC+HjETqqSZEAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "def plot_images(real, gen, n=10):\n", + " assert real.shape == gen.shape\n", + " assert real.ndim == 3\n", + " assert n * 2 <= len(real)\n", + "\n", + " idx = np.sort(np.random.choice(len(real), n * 2, replace=False))\n", + " real = real[idx]\n", + " gen = gen[idx]\n", + "\n", + " size_x = 12\n", + " size_y = size_x / real.shape[2] * real.shape[1] * n * 1.2 / 4\n", + "\n", + " fig, axx = plt.subplots(n, 4, figsize=(size_x, size_y))\n", + " axx = [(ax[0], ax[1]) for ax in axx] + \\\n", + " [(ax[2], ax[3]) for ax in axx]\n", + "\n", + " for ax, img_real, img_fake in zip(axx, real, gen):\n", + " ax[0].imshow(img_real, aspect='auto')\n", + " ax[0].set_title(\"real\")\n", + " ax[0].axis('off')\n", + " ax[1].imshow(img_fake, aspect='auto')\n", + " ax[1].set_title('generated')\n", + " ax[1].axis('off')\n", + "\n", + " return fig\n", + "\n", + "plot_images(Y_train, Y_train_fake)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAG2CAYAAABWJMy3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAcNUlEQVR4nO3dfZBldX3n8fdnh4dxRiJR8YEZIlREsiyrAzVLNGRNBYKMSkF2K5WCja5Ea2ezGw24Zg0mVdnN1iZlVbI+lZaGIEKVBB9QKqyljKxoue4iOuCAwKAiEpnhYSAuASHhye/+cc9A03N7+nRyT5/f0O9XVVffe8/tM5/p6fvp3/zuOeeXqkKS1K5/MnYASdLeWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqKU5klSSl46dQ5rLopakxlnUekZJst/YGaRZs6i1z0tye5LfS3ID8FCSn0nymST3JvlBkt+Z89zjk1yd5P4kdyX5YJIDRowvLcqi1jPFmcDrgecClwHXA+uAk4BzkpzSPe8J4O3A84FXddv/47KnlZbAotYzxQeq6g7gGOCQqvpvVfVoVd0G/AVwBkBVXVtVX6+qx6vqduDPgV8aLbXUg/N5eqa4o/v8EuDQJPfP2bYK+N8ASV4GvAfYCKxh8hq4dhlzSkvmiFrPFLsvA3kH8IOqOnjOx0FV9bpu+4eBW4Ajq+qngN8HMkJeqTeLWs803wAe7N5cfFaSVUmOSfIvuu0HAQ8AP07yc8B/GC2p1JNFrWeUqnoCOBXYAPwAuA84H3hO95TfBf4N8CCTuetPjhBTWpK4cIAktc0RtSQ1zqKWpMZZ1JLUOItakho3yAkvB+TAWs3aIXYtrQgve/nDM9/nd29YM/N9anb+nod4tB6Zekz/IEW9mrX8fE4aYtfSirBly7aZ7/OUQzfMfJ+anWvqSwtuc+pDkhpnUUtS4yxqSWqcRS1JjbOoJalxFrUkNa5XUSfZlOQ7SW5Ncu7QoSRJT1m0qJOsAj4EvBY4GjgzydFDB5MkTfQZUR8P3FpVt1XVo8AngNOHjSVJ2q1PUa/jqfXoAHZ0j0mSlsHMTiFPshnYDLAarykgSbPSZ0S9Ezhszv313WNPU1XnVdXGqtq4PwfOKp8krXh9ivqbwJFJjkhyAHAGcPmwsSRJuy069VFVjyd5K7AFWAVcUFU3DZ5MkgT0nKOuqs8Dnx84iyRpCs9MlKTGWdSS1DiLWpIaZ1FLUuMsaklq3CCL2+4rttw5+wVEpVat5J/3fX1hX0fUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMb1WYX8giS7kty4HIEkSU/XZ0R9IbBp4BySpAUsWtRV9VXgR8uQRZI0hauQS1LjZvZmoquQS9IwPOpDkhpnUUtS4/ocnncJcDVwVJIdSd4yfCxJ0m6LvplYVWcuRxBJ0nROfUhS4yxqSWqcRS1JjbOoJalx+8wq5Ct5BWVJ/zj7Qn8cf8rDC25zRC1JjbOoJalxFrUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqXJ/LnB6W5MtJbk5yU5KzlyOYJGmiz5mJjwPvqKrrkhwEXJvkyqq6eeBskiT6rUJ+V1Vd191+ENgOrBs6mCRpYknX+khyOHAscM2Uba5CLkkD6P1mYpJnA58BzqmqB+ZvdxVySRpGr6JOsj+Tkr64qj47bCRJ0lx9jvoI8FFge1W9Z/hIkqS5+oyoTwDeCJyYZFv38bqBc0mSOn1WIf8akGXIIkmawjMTJalxFrUkNc6ilqTGWdSS1DiLWpIat6RTyPt62csfZsuW9pdnl6R/qFMO3TDT/X23/mbBbY6oJalxFrUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqXJ/rUa9O8o0k13erkP/RcgSTJE30OeHlEeDEqvpxt9LL15J8oaq+PnA2SRL9rkddwI+7u/t3HzVkKEnSU/qumbgqyTZgF3BlVU1dhTzJ1iRb7/2bJ2adU5JWrF5FXVVPVNUGYD1wfJJjpjznyVXID3neqlnnlKQVa0lHfVTV/cCXgU3DxJEkzdfnqI9Dkhzc3X4WcDJwy9DBJEkTfY76eDFwUZJVTIr9U1X1uWFjSZJ263PUxw3AscuQRZI0hWcmSlLjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMZZ1JLUOItakhpnUUtS4yxqSWqcRS1JjbOoJalxFrUkNa53UXfLcX0riZc4laRltJQR9dnA9qGCSJKm67u47Xrg9cD5w8aRJM3Xd0T9PuCdwE8WeoKrkEvSMPqsmXgqsKuqrt3b81yFXJKG0WdEfQJwWpLbgU8AJyb5+KCpJElPWrSoq+pdVbW+qg4HzgCuqqo3DJ5MkgR4HLUkNW/RVcjnqqqvAF8ZJIkkaSpH1JLUOItakhpnUUtS4yxqSWqcRS1JjbOoJalxFrUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcb0uc9qt7vIg8ATweFVtHDKUJOkpS7ke9S9X1X2DJZEkTeXUhyQ1rm9RF/DFJNcm2TztCUk2J9maZOu9f/PE7BJK0grXd+rjF6tqZ5IXAFcmuaWqvjr3CVV1HnAewMZXrK4Z55SkFavXiLqqdnafdwGXAccPGUqS9JRFizrJ2iQH7b4NvAa4cehgkqSJPlMfLwQuS7L7+X9ZVVcMmkqS9KRFi7qqbgNesQxZJElTeHieJDXOopakxlnUktQ4i1qSGreUa3309t0b1nDKoRuG2HXztty5bewIkubZ1/vIEbUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcb2KOsnBSS5NckuS7UleNXQwSdJE3zMT3w9cUVW/luQAYM2AmSRJcyxa1EmeA7waOAugqh4FHh02liRptz5TH0cA9wIfS/KtJOd3S3I9zdxVyB/jkZkHlaSVqk9R7wccB3y4qo4FHgLOnf+kqjqvqjZW1cb9OXDGMSVp5epT1DuAHVV1TXf/UibFLUlaBosWdVXdDdyR5KjuoZOAmwdNJUl6Ut+jPt4GXNwd8XEb8JvDRZIkzdWrqKtqG7Bx4CySpCk8M1GSGmdRS1LjLGpJapxFLUmNs6glqXF9D89TT0MsS7/lzm0z3+dKNcS/jzQ0R9SS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcRa1JDVu0aJOclSSbXM+HkhyznKEkyT1OOGlqr4DbABIsgrYCVw2cC5JUmepUx8nAd+vqr8eIowkaU9LPYX8DOCSaRuSbAY2A6xmzT8yliRpt94j6m4ZrtOAT0/b7irkkjSMpUx9vBa4rqruGSqMJGlPSynqM1lg2kOSNJxeRZ1kLXAy8Nlh40iS5uu7CvlDwPMGziJJmsIzEyWpcRa1JDXOopakxlnUktQ4i1qSGpeqmv1Ok3uBPtcDeT5w38wDzJ45Z2tfyLkvZARzztqYOV9SVYdM2zBIUfeVZGtVbRwtQE/mnK19Iee+kBHMOWut5nTqQ5IaZ1FLUuPGLurzRv7z+zLnbO0LOfeFjGDOWWsy56hz1JKkxY09opYkLcKilqTGjVbUSTYl+U6SW5OcO1aOhSQ5LMmXk9yc5KYkZ4+daW+SrEryrSSfGzvLQpIcnOTSJLck2Z7kVWNnmibJ27t/8xuTXJJk9diZAJJckGRXkhvnPPbcJFcm+V73+afHzNhlmpbzT7t/9xuSXJbk4DEzdpn2yDln2zuSVJLnj5FtvlGKulvN/ENMVo05GjgzydFjZNmLx4F3VNXRwCuB324w41xnA9vHDrGI9wNXVNXPAa+gwbxJ1gG/A2ysqmOAVUzWCm3BhcCmeY+dC3ypqo4EvtTdH9uF7JnzSuCYqno58F3gXcsdaooL2TMnSQ4DXgP8cLkDLWSsEfXxwK1VdVtVPQp8Ajh9pCxTVdVdVXVdd/tBJqWybtxU0yVZD7weOH/sLAtJ8hzg1cBHAarq0aq6f9xUC9oPeFaS/YA1wJ0j5wGgqr4K/Gjew6cDF3W3LwJ+dVlDTTEtZ1V9saoe7+5+HVi/7MHmWeD7CfBe4J1AM0dajFXU64A75tzfQaMlCJDkcOBY4JpxkyzofUx+sH4ydpC9OAK4F/hYN0VzfrdyUFOqaifwZ0xGU3cBf1tVXxw31V69sKru6m7fDbxwzDA9vRn4wtghpklyOrCzqq4fO8tcvpm4iCTPBj4DnFNVD4ydZ74kpwK7qurasbMsYj/gOODDVXUs8BBt/Df9abo53tOZ/GI5FFib5A3jpuqnJsfaNjMKnCbJHzCZVrx47CzzJVkD/D7wh2NnmW+sot4JHDbn/vrusaYk2Z9JSV9cVa2uF3kCcFqS25lMIZ2Y5OPjRppqB7Cjqnb/r+RSJsXdml8BflBV91bVY0zWCf2FkTPtzT1JXgzQfd41cp4FJTkLOBX4jWrzBI6fZfIL+vru9bQeuC7Ji0ZNxXhF/U3gyCRHJDmAyZs1l4+UZaokYTKfur2q3jN2noVU1buqan1VHc7k+3hVVTU3Aqyqu4E7khzVPXQScPOIkRbyQ+CVSdZ0PwMn0eCbnnNcDrypu/0m4K9GzLKgJJuYTM+dVlUPj51nmqr6dlW9oKoO715PO4Djup/dUY1S1N2bCm8FtjB5EXyqqm4aI8tenAC8kckIdVv38bqxQ+3j3gZcnOQGYAPwJyPn2UM34r8UuA74NpPXSBOnFSe5BLgaOCrJjiRvAd4NnJzke0z+N/DuMTPCgjk/CBwEXNm9lj4yakgWzNkkTyGXpMb5ZqIkNc6ilqTGWdSS1DiLWpIaZ1FLA+ou7PPSsXNo32ZRSwtIclaSr42dQ7KotSJ1F1yS9gkWtUaR5Lju4kwPJvl0kk8m+e/dtlO7kyLuT/J/k7x8ztfdnuR3u+sa/233davnbF/sa3+vO+HmoST7JTk3yfe7HDcn+Vfdc/8p8BHgVUl+nOT+7vEDk/xZkh8muSfJR5I8a86f8Z+T3JXkziRvHvwbqRXBotay6y4bcBmT6wE/F7gE2F2QxwIXAP8eeB7w58DlSQ6cs4tfZ3Id4SOAlwNnLeFrz2RySdiDuzNkvw/8S+A5wB8BH0/y4qraDvwWcHVVPbuqdl/o/t3Ay5icWflSJld9/MPuz98E/C5wMnAkkzMFpX80i1pjeCWTq+l9oKoe6y549Y1u22bgz6vqmqp6oqouAh7pvma3D1TVnVX1I+B/MinNpXztHVX1dwBV9eluXz+pqk8C32NyvfQ9dNf+2Ay8vap+1F2n/E94amGBXwc+VlU3VtVDwH/9B3+HpDmcp9MYDmVyzd+51y/YfX3ylwBvSvK2OdsO6L5mt7kXyXl4zrY+Xzv3Ougk+bfAfwIO7x56NrDQ8kuHMFlI4NpJZ092wWQVmN1/r7mXm/3rBfYjLYlFrTHcBaxLkjllfRiTaYg7gD+uqj/+B+y3z9c++cshyUuAv2Byhbyrq+qJJNuYlO/Tntu5D/g74J91CwzMdxdPv3zvzywxvzSVUx8aw9XAE8Bbuzf0Tuep6Ya/AH4ryc9nYm2S1yc5qMd+l/q1a5mU8b0ASX4TOGbO9nuA9d2cOlX1k+7PeG+SF3Rfsy7JKd3zPwWcleTo7iL0/6Xft0PaO4tay65bJ/NfA28B7gfeAHwOeKSqtgL/jsllMf8fcCvdm4U99rukr62qm4H/weQXxz3APwf+z5ynXAXcBNyd5L7usd/r9vv1JA8A/ws4qtvfF5gsi3ZV95yr+uSWFuNlTtWEJNcAH6mqj42dRWqNI2qNIskvJXlRN/XxJiaH2V0xdi6pRb6ZqLEcxWROdy1wG/Brc1bTljSHUx+S1DinPiSpcYNMfRyQA2s1a4fYtSQ9I/09D/FoPZJp2wYp6tWs5edz0hC7lqRnpGvqSwtuc+pDkhpnUUtS4yxqSWqcRS1JjbOoJalxFrUkNa5XUSfZlOQ7SW5Ncu7QoSRJT1m0qJOsAj4EvBY4GjgzydFDB5MkTfQZUR8P3FpVt3XXEf4EcPqwsSRJu/Up6nU8fZ25Hd1jT5Nkc5KtSbY+xiOzyidJK97M3kysqvOqamNVbdyfA2e1W0la8foU9U6evmDn+u4xSdIy6FPU3wSOTHJEt8jnGcDlw8aSJO226NXzqurxJG8FtgCrgAuq6qbBk0mSgJ6XOa2qzwOfHziLJGkKz0yUpMZZ1JLUOItakhpnUUtS4yxqSWrcIIvbqn1b7tw2dgTtxSmHbhg7ghriiFqSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMb1Wdz2giS7kty4HIEkSU/XZ0R9IbBp4BySpAUsWtRV9VXgR8uQRZI0xcxOIU+yGdgMsJo1s9qtJK14rkIuSY3zqA9JapxFLUmN63N43iXA1cBRSXYkecvwsSRJuy36ZmJVnbkcQSRJ0zn1IUmNs6glqXEWtSQ1zqKWpMZZ1JLUOFchnzFX99YsrOSfI1dg35MjaklqnEUtSY2zqCWpcRa1JDXOopakxlnUktQ4i1qSGtfnMqeHJflykpuT3JTk7OUIJkma6HPCy+PAO6rquiQHAdcmubKqbh44mySJfquQ31VV13W3HwS2A+uGDiZJmljSKeRJDgeOBa6Zss1VyCVpAL3fTEzybOAzwDlV9cD87a5CLknD6FXUSfZnUtIXV9Vnh40kSZqrz1EfAT4KbK+q9wwfSZI0V58R9QnAG4ETk2zrPl43cC5JUqfPKuRfA7IMWSRJU3hmoiQ1zqKWpMZZ1JLUOItakhpnUUtS4yxqSWrckq718Uyz5c5tY0eQNM8Qr8tTDt0w830uJ0fUktQ4i1qSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1rs/CAauTfCPJ9UluSvJHyxFMkjTR54SXR4ATq+rH3ZJcX0vyhar6+sDZJEn0WziggB93d/fvPmrIUJKkp/Rd3HZVkm3ALuDKqrpmynM2J9maZOtjPDLrnJK0YvUq6qp6oqo2AOuB45McM+U551XVxqrauD8HzjqnJK1YSzrqo6ruB74MbBomjiRpvj5HfRyS5ODu9rOAk4Fbhg4mSZroc9THi4GLkqxiUuyfqqrPDRtLkrRbn6M+bgCOXYYskqQpPDNRkhpnUUtS4yxqSWqcRS1JjbOoJalxFrUkNc6ilqTGWdSS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcb2LuluO61tJvMSpJC2jpYyozwa2DxVEkjRd38Vt1wOvB84fNo4kab6+I+r3Ae8EfrLQE1yFXJKG0WfNxFOBXVV17d6e5yrkkjSMPiPqE4DTktwOfAI4McnHB00lSXrSokVdVe+qqvVVdThwBnBVVb1h8GSSJMDjqCWpeYuuQj5XVX0F+MogSSRJUzmilqTGWdSS1DiLWpIaZ1FLUuMsaklq3JKO+nimOeXQDTPf55Y7t818n9JKMsTrcl/niFqSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1zqKWpMb1Oo66WzTgQeAJ4PGq2jhkKEnSU5ZywssvV9V9gyWRJE3l1IckNa5vURfwxSTXJtk87QmuQi5Jw+g79fGLVbUzyQuAK5PcUlVfnfuEqjoPOA/gp/LcmnFOSVqxeo2oq2pn93kXcBlw/JChJElPWbSok6xNctDu28BrgBuHDiZJmugz9fFC4LIku5//l1V1xaCpJElPWrSoq+o24BXLkEWSNIWH50lS4yxqSWqcRS1JjbOoJalxFrUkNW5Fr0I+hJW8grIrsM/OSv450p4cUUtS4yxqSWqcRS1JjbOoJalxFrUkNc6ilqTGWdSS1LheRZ3k4CSXJrklyfYkrxo6mCRpou8JL+8HrqiqX0tyALBmwEySpDkWLeokzwFeDZwFUFWPAo8OG0uStFufqY8jgHuBjyX5VpLzuyW5nsZVyCVpGH2Kej/gOODDVXUs8BBw7vwnVdV5VbWxqjbuz4EzjilJK1efot4B7Kiqa7r7lzIpbknSMli0qKvqbuCOJEd1D50E3DxoKknSk/oe9fE24OLuiI/bgN8cLpIkaa5eRV1V24CNA2eRJE3hmYmS1DiLWpIaZ1FLUuMsaklqnEUtSY2zqCWpcX2Po5YWdcqhG8aOIC2bLXdum+n+jj/l4QW3OaKWpMZZ1JLUOItakhpnUUtS4yxqSWqcRS1JjVu0qJMclWTbnI8HkpyzHOEkST2Oo66q7wAbAJKsAnYClw2cS5LUWerUx0nA96vqr4cII0na01KL+gzgkiGCSJKm613U3TJcpwGfXmD75iRbk2x9jEdmlU+SVryljKhfC1xXVfdM21hV51XVxqrauD8HziadJGlJRX0mTntI0rLrVdRJ1gInA58dNo4kab6+q5A/BDxv4CySpCk8M1GSGmdRS1LjLGpJapxFLUmNs6glqXEWtSQ1LlU1+50m9wJ9Ltz0fOC+mQeYPXPO1r6Qc1/ICOactTFzvqSqDpm2YZCi7ivJ1qraOFqAnsw5W/tCzn0hI5hz1lrN6dSHJDXOopakxo1d1OeN/Of3Zc7Z2hdy7gsZwZyz1mTOUeeoJUmLG3tELUlahEUtSY0braiTbErynSS3Jjl3rBwLSXJYki8nuTnJTUnOHjvT3iRZleRbST43dpaFJDk4yaVJbkmyPcmrxs40TZK3d//mNya5JMnqsTMBJLkgya4kN8557LlJrkzyve7zT4+Zscs0Leefdv/uNyS5LMnBY2bsMu2Rc862dySpJM8fI9t8oxR1klXAh5gs73U0cGaSo8fIshePA++oqqOBVwK/3WDGuc4Gto8dYhHvB66oqp8DXkGDeZOsA34H2FhVxwCrmCzq3IILgU3zHjsX+FJVHQl8qbs/tgvZM+eVwDFV9XLgu8C7ljvUFBeyZ06SHAa8BvjhcgdayFgj6uOBW6vqtqp6FPgEcPpIWaaqqruq6rru9oNMSmXduKmmS7IeeD1w/thZFpLkOcCrgY8CVNWjVXX/uKkWtB/wrCT7AWuAO0fOA0BVfRX40byHTwcu6m5fBPzqsoaaYlrOqvpiVT3e3f06sH7Zg82zwPcT4L3AO4FmjrQYq6jXAXfMub+DRksQIMnhwLHANeMmWdD7mPxg/WTsIHtxBHAv8LFuiub8bom3plTVTuDPmIym7gL+tqq+OG6qvXphVd3V3b4beOGYYXp6M/CFsUNMk+R0YGdVXT92lrl8M3ERSZ4NfAY4p6oeGDvPfElOBXZV1bVjZ1nEfsBxwIer6ljgIdr4b/rTdHO8pzP5xXIosDbJG8ZN1U9NjrVtZhQ4TZI/YDKtePHYWeZLsgb4feAPx84y31hFvRM4bM799d1jTUmyP5OSvriqWl3Y9wTgtCS3M5lCOjHJx8eNNNUOYEdV7f5fyaVMirs1vwL8oKrurarHmCzo/AsjZ9qbe5K8GKD7vGvkPAtKchZwKvAb1eYJHD/L5Bf09d3raT1wXZIXjZqK8Yr6m8CRSY5IcgCTN2suHynLVEnCZD51e1W9Z+w8C6mqd1XV+qo6nMn38aqqam4EWFV3A3ckOap76CTg5hEjLeSHwCuTrOl+Bk6iwTc957gceFN3+03AX42YZUFJNjGZnjutqh4eO880VfXtqnpBVR3evZ52AMd1P7ujGqWouzcV3gpsYfIi+FRV3TRGlr04AXgjkxHqtu7jdWOH2se9Dbg4yQ3ABuBPRs6zh27EfylwHfBtJq+RJk4rTnIJcDVwVJIdSd4CvBs4Ocn3mPxv4N1jZoQFc34QOAi4snstfWTUkCyYs0meQi5JjfPNRElqnEUtSY2zqCWpcRa1JDXOopakxlnUktQ4i1qSGvf/Ad/tqRJd61LPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "def plot_images_mask(real, gen):\n", + " assert real.shape == gen.shape\n", + " assert real.ndim == 3\n", + "\n", + " size_x = 6\n", + " size_y = size_x / real.shape[2] * real.shape[1] * 2.4\n", + "\n", + " fig, [ax0, ax1] = plt.subplots(2, 1, figsize=(size_x, size_y))\n", + " ax0.imshow(real.any(axis=0), aspect='auto')\n", + " ax0.set_title(\"real\")\n", + " ax1.imshow(gen.any(axis=0), aspect='auto')\n", + " ax1.set_title(\"generated\")\n", + "\n", + " return fig\n", + "\n", + "fig = plot_images_mask(Y_train, Y_train_fake)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From ba3c1539cdb737fb02fc794b710fc975449c0d15 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Thu, 8 Oct 2020 13:09:18 +0300 Subject: [PATCH 14/24] residual fc block --- .../baseline_fc_8x16_elu_residual.yaml | 73 +++++++++++++++++++ models/nn.py | 40 ++++++++++ 2 files changed, 113 insertions(+) create mode 100644 models/configs/baseline_fc_8x16_elu_residual.yaml diff --git a/models/configs/baseline_fc_8x16_elu_residual.yaml b/models/configs/baseline_fc_8x16_elu_residual.yaml new file mode 100644 index 0000000..3e24a3e --- /dev/null +++ b/models/configs/baseline_fc_8x16_elu_residual.yaml @@ -0,0 +1,73 @@ +latent_dim: 32 +batch_size: 32 +lr: 1.e-4 +lr_schedule_rate: 0.999 + +num_disc_updates: 8 +gp_lambda: 10. +gpdata_lambda: 0. +cramer: False +stochastic_stepping: True + +save_every: 50 +num_epochs: 10000 + +feature_noise_power: NULL +feature_noise_decay: NULL + +data_version: 'data_v4' +pad_range: [-3, 5] +time_range: [-7, 9] +scaler: 'logarithmic' + +architecture: + generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128] + activations: ['elu', 'elu', 'elu', 'elu', 'elu'] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: [8, 16] + name: 'generator' + + discriminator: + - block_type: 'connect' + arguments: + vector_shape: [5,] + img_shape: [8, 16] + vector_bypass: False + concat_outputs: True + name: 'discriminator_tail' + block: + block_type: 'conv' + arguments: + filters: [ 8 , 16 , 32, 64 , 128 ] + kernel_sizes: [[2, 3], [2, 3], 1, [2, 3], [2, 3]] + paddings: ['valid', 'valid', 'valid', 'valid', 'valid'] + activations: ['elu', 'elu', 'elu', 'elu', 'elu'] + poolings: [NULL, NULL, 2, NULL, NULL] + kernel_init: glorot_uniform + input_shape: NULL + output_shape: [256,] + dropouts: [0.02, 0.02, 0.02, 0.02, 0.02] + name: discriminator_conv_block + - block_type: 'fully_connected_residual' + arguments: + units: 128 + activations: ['elu', 'elu', 'elu', 'elu'] + input_shape: [261,] + kernel_init: 'glorot_uniform' + batchnorm: True + output_shape: NULL + dropouts: [0.02, 0.02, 0.02, 0.02] + name: 'discriminator_head' + - block_type: 'fully_connected' + arguments: + units: [1] + activations: [NULL] + kernel_init: 'glorot_uniform' + input_shape: NULL + output_shape: NULL + name: 'discriminator_head_output' + diff --git a/models/nn.py b/models/nn.py index 2809f3a..f96f373 100644 --- a/models/nn.py +++ b/models/nn.py @@ -29,6 +29,44 @@ def fully_connected_block(units, activations, return tf.keras.Sequential(layers, **args) +def fully_connected_residual_block(units, activations, input_shape, + kernel_init='glorot_uniform', batchnorm=True, + output_shape=None, dropouts=None, name=None): + assert isinstance(units, int) + if dropouts: + assert len(dropouts) == len(activations) + else: + dropouts = [None] * len(activations) + + def single_block(xx, units, activation, kernel_init, batchnorm, dropout): + xx = tf.keras.layers.Dense(units=units, kernel_initializer=kernel_init)(xx) + if batchnorm: + xx = tf.keras.layers.BatchNormalization()(xx) + xx = tf.keras.activations.get(activation)(xx) + if dropout: + xx = tf.keras.layers.Dropout(dropout)(xx) + return xx + + input_tensor = tf.keras.Input(shape=input_shape) + xx = input_tensor + for i, (act, dropout) in enumerate(zip(activations, dropouts)): + args = dict(units=units, activation=act, kernel_init=kernel_init, + batchnorm=batchnorm, dropout=dropout) + if len(xx.shape) == 2 and xx.shape[1] == units: + xx = xx + single_block(xx, **args) + else: + assert i == 0 + xx = single_block(xx, **args) + + if output_shape: + xx = tf.keras.layers.Reshape(output_shape)(xx) + + args = dict(inputs=input_tensor, outputs=xx) + if name: + args['name'] = name + return tf.keras.Model(**args) + + def concat_block(input1_shape, input2_shape, reshape_input1=None, reshape_input2=None, axis=-1, name=None): in1 = tf.keras.Input(shape=input1_shape) @@ -127,6 +165,8 @@ def build_block(block_type, arguments): block = vector_img_connect_block(**arguments) elif block_type == 'concat': block = concat_block(**arguments) + elif block_type == 'fully_connected_residual': + block = fully_connected_residual_block(**arguments) else: raise(NotImplementedError(block_type)) From 14b8ef4fb10ee9157ecf4a21050a0db56086911a Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Thu, 8 Oct 2020 15:31:22 +0300 Subject: [PATCH 15/24] batchnorm off --- models/configs/baseline_fc_8x16_elu_residual.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/configs/baseline_fc_8x16_elu_residual.yaml b/models/configs/baseline_fc_8x16_elu_residual.yaml index 3e24a3e..ee6371e 100644 --- a/models/configs/baseline_fc_8x16_elu_residual.yaml +++ b/models/configs/baseline_fc_8x16_elu_residual.yaml @@ -58,7 +58,7 @@ architecture: activations: ['elu', 'elu', 'elu', 'elu'] input_shape: [261,] kernel_init: 'glorot_uniform' - batchnorm: True + batchnorm: False output_shape: NULL dropouts: [0.02, 0.02, 0.02, 0.02] name: 'discriminator_head' From 830f31319ae893c1984c8bba3195c05695b47fc3 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Fri, 9 Oct 2020 20:31:53 +0300 Subject: [PATCH 16/24] js gan loss --- models/model_v4.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/models/model_v4.py b/models/model_v4.py index 4ce2dd0..b3681dd 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -36,6 +36,24 @@ def disc_loss_cramer(d_real, d_fake, d_fake_2): def gen_loss_cramer(d_real, d_fake, d_fake_2): return -disc_loss_cramer(d_real, d_fake, d_fake_2) + +def logloss(x): + return tf.where(x < -30., -x, tf.math.log(1. + tf.math.exp(-x))) + + +def disc_loss_js(d_real, d_fake): + return tf.reduce_sum( + logloss(d_real) + ) + tf.reduce_sum( + logloss(-d_fake) + ) / (len(d_real) + len(d_fake)) + +def gen_loss_js(d_real, d_fake): + return tf.reduce_mean( + logloss(d_fake) + ) + + class Model_v4: def __init__(self, config): self.disc_opt = tf.keras.optimizers.RMSprop(config['lr']) @@ -44,6 +62,9 @@ def __init__(self, config): self.gpdata_lambda = config['gpdata_lambda'] self.num_disc_updates = config['num_disc_updates'] self.cramer = config['cramer'] + self.js = config.get('js', False) + assert not (self.js and self.cramer) + self.stochastic_stepping = config['stochastic_stepping'] self.latent_dim = config['latent_dim'] @@ -95,7 +116,10 @@ def calculate_losses(self, feature_batch, target_batch): d_fake_2 = self.discriminator([_f(feature_batch), fake_2]) if not self.cramer: - d_loss = disc_loss(d_real, d_fake) + if self.js: + d_loss = disc_loss_js(d_real, d_fake) + else: + d_loss = disc_loss(d_real, d_fake) else: d_loss = disc_loss_cramer(d_real, d_fake, d_fake_2) @@ -114,7 +138,10 @@ def calculate_losses(self, feature_batch, target_batch): ) * self.gpdata_lambda ) if not self.cramer: - g_loss = gen_loss(d_real, d_fake) + if self.js: + g_loss = gen_loss_js(d_real, d_fake) + else: + g_loss = gen_loss(d_real, d_fake) else: g_loss = gen_loss_cramer(d_real, d_fake, d_fake_2) From db7193ec71cd4b83df96ba7ddb93d77b767b5320 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Mon, 12 Oct 2020 19:35:23 +0300 Subject: [PATCH 17/24] custom code activations --- models/nn.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/models/nn.py b/models/nn.py index f96f373..93a26a1 100644 --- a/models/nn.py +++ b/models/nn.py @@ -1,6 +1,14 @@ import tensorflow as tf +def get_activation(activation): + try: + activation = tf.keras.activations.get(activation) + except ValueError: + activation = eval(activation) + return activation + + def fully_connected_block(units, activations, kernel_init='glorot_uniform', input_shape=None, output_shape=None, dropouts=None, name=None): @@ -8,6 +16,8 @@ def fully_connected_block(units, activations, if dropouts: assert len(dropouts) == len(units) + activations = [get_activation(a) for a in activations] + layers = [] for i, (size, act) in enumerate(zip(units, activations)): args = dict(units=size, activation=act, kernel_initializer=kernel_init) @@ -38,11 +48,13 @@ def fully_connected_residual_block(units, activations, input_shape, else: dropouts = [None] * len(activations) + activations = [get_activation(a) for a in activations] + def single_block(xx, units, activation, kernel_init, batchnorm, dropout): xx = tf.keras.layers.Dense(units=units, kernel_initializer=kernel_init)(xx) if batchnorm: xx = tf.keras.layers.BatchNormalization()(xx) - xx = tf.keras.activations.get(activation)(xx) + xx = activation(xx) if dropout: xx = tf.keras.layers.Dropout(dropout)(xx) return xx @@ -90,6 +102,8 @@ def conv_block(filters, kernel_sizes, paddings, activations, poolings, if dropouts: assert len(dropouts) == len(filters) + activations = [get_activation(a) for a in activations] + layers = [] for i, (nfilt, ksize, padding, act, pool) in enumerate(zip(filters, kernel_sizes, paddings, activations, poolings)): From 31931f5a835c26dbe146266c1c4f6a9eee533084 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Tue, 13 Oct 2020 19:58:47 +0300 Subject: [PATCH 18/24] custom kinked activation --- models/configs/baseline_fc_8x16_kinked.yaml | 79 +++++++++++++++++++++ models/nn.py | 1 + run_docker_tensorboard.sh | 1 + 3 files changed, 81 insertions(+) create mode 100644 models/configs/baseline_fc_8x16_kinked.yaml diff --git a/models/configs/baseline_fc_8x16_kinked.yaml b/models/configs/baseline_fc_8x16_kinked.yaml new file mode 100644 index 0000000..d15d444 --- /dev/null +++ b/models/configs/baseline_fc_8x16_kinked.yaml @@ -0,0 +1,79 @@ +latent_dim: 32 +batch_size: 32 +lr: 1.e-4 +lr_schedule_rate: 0.999 + +num_disc_updates: 8 +gp_lambda: 10. +gpdata_lambda: 0. +cramer: False +stochastic_stepping: True + +save_every: 50 +num_epochs: 10000 + +feature_noise_power: NULL +feature_noise_decay: NULL + +data_version: 'data_v4' +pad_range: [-3, 5] +time_range: [-7, 9] +scaler: 'logarithmic' + +architecture: + generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128] + activations: [ + 'elu', 'elu', 'elu', 'elu', + " ( + lambda x, + shift=0.01, + val=np.log10(2), + v0=np.log10(2) / 10: ( + tf.where( + x > shift, + val + x - shift, + v0 + tf.keras.activations.elu( + x, + alpha=(v0 * shift / (val - v0)) + ) * (val - v0) / shift + ) + ) + )" + ] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: [8, 16] + name: 'generator' + + discriminator: + - block_type: 'connect' + arguments: + vector_shape: [5,] + img_shape: [8, 16] + vector_bypass: False + concat_outputs: True + name: 'discriminator_tail' + block: + block_type: 'conv' + arguments: + filters: [16, 16, 32, 32, 64, 64] + kernel_sizes: [3, 3, 3, 3, 3, 2] + paddings: ['same', 'same', 'same', 'same', 'valid', 'valid'] + activations: ['elu', 'elu', 'elu', 'elu', 'elu', 'elu'] + poolings: [NULL, [1, 2], NULL, 2, NULL, NULL] + kernel_init: glorot_uniform + input_shape: NULL + output_shape: [64,] + dropouts: [0.02, 0.02, 0.02, 0.02, 0.02, 0.02] + name: discriminator_conv_block + - block_type: 'fully_connected' + arguments: + units: [128, 1] + activations: ['elu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [69,] + output_shape: NULL + name: 'discriminator_head' diff --git a/models/nn.py b/models/nn.py index 93a26a1..9ef13d3 100644 --- a/models/nn.py +++ b/models/nn.py @@ -1,4 +1,5 @@ import tensorflow as tf +import numpy as np def get_activation(activation): diff --git a/run_docker_tensorboard.sh b/run_docker_tensorboard.sh index 846dda4..cf02ac1 100755 --- a/run_docker_tensorboard.sh +++ b/run_docker_tensorboard.sh @@ -1,5 +1,6 @@ docker run -it \ -u $(id -u):$(id -g) \ + --rm \ --env HOME=`pwd` \ -p 127.0.0.1:6126:6006/tcp \ --runtime nvidia \ From ea4c2d2828df22d4020ebca8430bb35dcca2165d Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Tue, 13 Oct 2020 23:54:55 +0300 Subject: [PATCH 19/24] save/load optimizer state [wip] --- models/model_v4.py | 39 +++++++++++++++++++++++++++++++++++++++ models/utils.py | 10 ++++------ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/models/model_v4.py b/models/model_v4.py index b3681dd..d4771d2 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -1,4 +1,7 @@ +import h5py import tensorflow as tf +from tensorflow.python.keras.saving import hdf5_format + from . import scalers, nn @@ -79,6 +82,42 @@ def __init__(self, config): self.time_range = tuple(config['time_range']) self.data_version = config['data_version'] + self.generator.compile(optimizer=self.gen_opt, loss='mean_squared_error') + self.discriminator.compile(optimizer=self.disc_opt, loss='mean_squared_error') + + def load_generator(self, checkpoint): + self._load_weights(checkpoint, 'gen') + + def load_discriminator(self, checkpoint): + self._load_weights(checkpoint, 'disc') + + def _load_weights(self, checkpoint, gen_or_disc): + if gen_or_disc == 'gen': + network = self.generator + step_fn = self.gen_step + elif gen_or_disc == 'disc': + network = self.discriminator + step_fn = self.disc_step + else: + raise ValueError(gen_or_disc) + + model_file = h5py.File(checkpoint, 'r') + if len(network.optimizer.weights) == 0 and 'optimizer_weights' in model_file: + # perform single optimization step to init optimizer weights + features_shape = self.discriminator.inputs[0].shape.as_list() + targets_shape = self.discriminator.inputs[1].shape.as_list() + features_shape[0], targets_shape[0] = 1, 1 + step_fn(tf.zeros(features_shape), tf.zeros(targets_shape)) + + print(f'Loading {gen_or_disc} weights from {str(checkpoint)}') + network.load_weights(str(checkpoint)) + + if 'optimizer_weights' in model_file: + opt_weight_values = hdf5_format.load_optimizer_weights_from_hdf5_group( + model_file + ) + network.optimizer.set_weights(opt_weight_values) + @tf.function def make_fake(self, features): diff --git a/models/utils.py b/models/utils.py index 6825d82..c2fa541 100644 --- a/models/utils.py +++ b/models/utils.py @@ -27,10 +27,8 @@ def load_weights(model, model_path, epoch=None): if epoch is None: epoch = latest_epoch(model_path) - latest_gen_checkpoint = model_path / f"generator_{epoch:05d}.h5" - latest_disc_checkpoint = model_path / f"discriminator_{epoch:05d}.h5" + gen_checkpoint = model_path / f"generator_{epoch:05d}.h5" + disc_checkpoint = model_path / f"discriminator_{epoch:05d}.h5" - print(f'Loading generator weights from {str(latest_gen_checkpoint)}') - model.generator.load_weights(str(latest_gen_checkpoint)) - print(f'Loading discriminator weights from {str(latest_disc_checkpoint)}') - model.discriminator.load_weights(str(latest_disc_checkpoint)) \ No newline at end of file + model.load_generator(gen_checkpoint) + model.load_discriminator(disc_checkpoint) \ No newline at end of file From 43701d09cfbf4ccebf73618d36677c862291a7cc Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 14 Oct 2020 16:27:56 +0300 Subject: [PATCH 20/24] resume training, functional lr scheduler --- models/callbacks.py | 20 +++++++++++++++----- models/model_v4.py | 5 +++-- models/training.py | 4 ++-- models/utils.py | 4 +++- run_model_v4.py | 42 ++++++++++++++++++++++++++++-------------- 5 files changed, 51 insertions(+), 24 deletions(-) diff --git a/models/callbacks.py b/models/callbacks.py index 08ce60e..e447d57 100644 --- a/models/callbacks.py +++ b/models/callbacks.py @@ -40,14 +40,24 @@ def __call__(self, step): class ScheduleLRCallback: - def __init__(self, model, decay_rate, writer): + def __init__(self, model, func_gen, func_disc, writer): self.model = model - self.decay_rate = decay_rate + self.func_gen = func_gen + self.func_disc = func_disc self.writer = writer def __call__(self, step): - self.model.disc_opt.lr.assign(self.model.disc_opt.lr * self.decay_rate) - self.model.gen_opt.lr.assign(self.model.gen_opt.lr * self.decay_rate) + self.model.disc_opt.lr.assign(self.func_disc(step)) + self.model.gen_opt.lr.assign(self.func_gen(step)) with self.writer.as_default(): tf.summary.scalar("discriminator learning rate", self.model.disc_opt.lr, step) - tf.summary.scalar("generator learning rate", self.model.gen_opt.lr, step) \ No newline at end of file + tf.summary.scalar("generator learning rate", self.model.gen_opt.lr, step) + + +def get_scheduler(lr, lr_decay): + if isinstance(lr_decay, str): + return eval(lr_decay) + + def schedule_lr(step): + return lr * lr_decay**step + return schedule_lr \ No newline at end of file diff --git a/models/model_v4.py b/models/model_v4.py index d4771d2..884a9b9 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -59,8 +59,8 @@ def gen_loss_js(d_real, d_fake): class Model_v4: def __init__(self, config): - self.disc_opt = tf.keras.optimizers.RMSprop(config['lr']) - self.gen_opt = tf.keras.optimizers.RMSprop(config['lr']) + self.disc_opt = tf.keras.optimizers.RMSprop(config['lr_disc']) + self.gen_opt = tf.keras.optimizers.RMSprop(config['lr_gen']) self.gp_lambda = config['gp_lambda'] self.gpdata_lambda = config['gpdata_lambda'] self.num_disc_updates = config['num_disc_updates'] @@ -113,6 +113,7 @@ def _load_weights(self, checkpoint, gen_or_disc): network.load_weights(str(checkpoint)) if 'optimizer_weights' in model_file: + print('Also recovering the optimizer state') opt_weight_values = hdf5_format.load_optimizer_weights_from_hdf5_group( model_file ) diff --git a/models/training.py b/models/training.py index 2aea9c3..afcf3e6 100644 --- a/models/training.py +++ b/models/training.py @@ -5,12 +5,12 @@ def train(data_train, data_val, train_step_fn, loss_eval_fn, num_epochs, batch_size, train_writer=None, val_writer=None, callbacks=[], features_train=None, features_val=None, - features_noise=None): + features_noise=None, first_epoch=0): if not ((features_train is None) or (features_val is None)): assert features_train is not None, 'train: features should be provided for both train and val' assert features_val is not None, 'train: features should be provided for both train and val' - for i_epoch in range(num_epochs): + for i_epoch in range(first_epoch, num_epochs): print("Working on epoch #{}".format(i_epoch), flush=True) tf.keras.backend.set_learning_phase(1) # training diff --git a/models/utils.py b/models/utils.py index c2fa541..dd5a41b 100644 --- a/models/utils.py +++ b/models/utils.py @@ -31,4 +31,6 @@ def load_weights(model, model_path, epoch=None): disc_checkpoint = model_path / f"discriminator_{epoch:05d}.h5" model.load_generator(gen_checkpoint) - model.load_discriminator(disc_checkpoint) \ No newline at end of file + model.load_discriminator(disc_checkpoint) + + return epoch \ No newline at end of file diff --git a/run_model_v4.py b/run_model_v4.py index 95c7854..c434129 100644 --- a/run_model_v4.py +++ b/run_model_v4.py @@ -12,7 +12,7 @@ from data import preprocessing from models.utils import latest_epoch, load_weights from models.training import train -from models.callbacks import SaveModelCallback, WriteHistSummaryCallback, ScheduleLRCallback +from models.callbacks import SaveModelCallback, WriteHistSummaryCallback, ScheduleLRCallback, get_scheduler from models.model_v4 import Model_v4 from metrics import evaluate_model import cuda_gpu_config @@ -52,6 +52,11 @@ def load_config(file): (config['feature_noise_decay'] is None) ), 'Noise power and decay must be both provided' + if 'lr_disc' not in config: config['lr_disc'] = config['lr'] + if 'lr_gen' not in config: config['lr_gen' ] = config['lr'] + if 'lr_schedule_rate_disc' not in config: config['lr_schedule_rate_disc'] = config['lr_schedule_rate'] + if 'lr_schedule_rate_gen' not in config: config['lr_schedule_rate_gen' ] = config['lr_schedule_rate'] + return config @@ -62,26 +67,29 @@ def main(): model_path = Path('saved_models') / args.checkpoint_name + config_path = str(model_path / 'config.yaml') + continue_training = False if args.prediction_only: assert model_path.exists(), "Couldn't find model directory" assert not args.config, "Config should be read from model path when doing prediction" - args.config = str(model_path / 'config.yaml') else: - assert not model_path.exists(), "Model directory already exists" - assert args.config, "No config provided" - - model_path.mkdir(parents=True) - config_destination = str(model_path / 'config.yaml') - shutil.copy(args.config, config_destination) + if not args.config: + assert model_path.exists(), "Couldn't find model directory" + continue_training = True + else: + assert not model_path.exists(), "Model directory already exists" - args.config = config_destination + model_path.mkdir(parents=True) + shutil.copy(args.config, config_path) + args.config = config_path config = load_config(args.config) model = Model_v4(config) - if args.prediction_only: - load_weights(model, model_path) + next_epoch = 0 + if args.prediction_only or continue_training: + next_epoch = load_weights(model, model_path) + 1 preprocessing._VERSION = model.data_version data, features = preprocessing.read_csv_2d(pad_range=model.pad_range, time_range=model.time_range) @@ -131,12 +139,18 @@ def features_noise(epoch): save_period=config['save_every'], writer=writer_val ) schedule_lr = ScheduleLRCallback( - model, decay_rate=config['lr_schedule_rate'], writer=writer_val + model, writer=writer_val, + func_gen=get_scheduler(config['lr_gen'], config['lr_schedule_rate_gen']), + func_disc=get_scheduler(config['lr_disc'], config['lr_schedule_rate_disc']) ) + if continue_training: + schedule_lr(next_epoch - 1) + train(Y_train, Y_test, model.training_step, model.calculate_losses, config['num_epochs'], config['batch_size'], train_writer=writer_train, val_writer=writer_val, - callbacks=[write_hist_summary, save_model, schedule_lr], - features_train=X_train, features_val=X_test, features_noise=features_noise) + callbacks=[schedule_lr, save_model, write_hist_summary], + features_train=X_train, features_val=X_test, features_noise=features_noise, + first_epoch=next_epoch) if __name__ == '__main__': From 9338b1167315f011ec11d48450a621c7c8f6dd49 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 14 Oct 2020 23:46:24 +0300 Subject: [PATCH 21/24] dynamic stepping; logloss fix --- models/model_v4.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/models/model_v4.py b/models/model_v4.py index 884a9b9..929094f 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -41,7 +41,7 @@ def gen_loss_cramer(d_real, d_fake, d_fake_2): def logloss(x): - return tf.where(x < -30., -x, tf.math.log(1. + tf.math.exp(-x))) + return tf.nn.softplus(-x) def disc_loss_js(d_real, d_fake): @@ -69,6 +69,11 @@ def __init__(self, config): assert not (self.js and self.cramer) self.stochastic_stepping = config['stochastic_stepping'] + self.dynamic_stepping = config.get('dynamic_stepping', False) + if self.dynamic_stepping: + assert not self.stochastic_stepping + self.dynamic_stepping_threshold = config['dynamic_stepping_threshold'] + self.latent_dim = config['latent_dim'] architecture_descr = config['architecture'] @@ -225,5 +230,9 @@ def training_step(self, feature_batch, target_batch): self.step_counter.assign(0) else: result = self.disc_step(feature_batch, target_batch) - self.step_counter.assign_add(1) + if self.dynamic_stepping: + if result['disc_loss'] < self.dynamic_stepping_threshold: + self.step_counter.assign(self.num_disc_updates) + else: + self.step_counter.assign_add(1) return result From b0a0cd447eb1f289397b4b8bb89fff551422132f Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Mon, 19 Oct 2020 19:40:07 +0000 Subject: [PATCH 22/24] updating dump graph --- combine_images.py | 49 +++++++++++++++++++++++++++++++ dump_graph_model_v4.py | 19 +++++++----- model_export/run_docker_centos.sh | 1 + run_docker.sh | 9 +++++- 4 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 combine_images.py diff --git a/combine_images.py b/combine_images.py new file mode 100644 index 0000000..622a207 --- /dev/null +++ b/combine_images.py @@ -0,0 +1,49 @@ +import argparse +from pathlib import Path + +from PIL import Image + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('path_to_images', type=str) + parser.add_argument('--output_name', type=str, default='plots.png') + + args = parser.parse_args() + + variables = [ + 'crossing_angle', + 'dip_angle', + 'drift_length', + 'pad_coord_fraction', + 'time_bin_fraction', + ] + + stats = [ + 'Mean0', + 'Mean1', + 'Sigma0^2', + 'Sigma1^2', + 'Cov01', + 'Sum', + ] + + img_path = Path(args.path_to_images) + images = [[Image.open(img_path / f'{s} vs {v}_amp_gt_1.png') for v in variables] for s in stats] + + width, height = images[0][0].size + + new_image = Image.new('RGB', (width * len(stats), height * len(variables))) + + x_offset = 0 + for img_line in images: + y_offset = 0 + for img in img_line: + new_image.paste(img, (x_offset, y_offset)) + y_offset += img.size[1] + x_offset += img.size[0] + + new_image.save(img_path / args.output_name) + +if __name__ == '__main__': + main() diff --git a/dump_graph_model_v4.py b/dump_graph_model_v4.py index 39f2cfa..0889bf0 100644 --- a/dump_graph_model_v4.py +++ b/dump_graph_model_v4.py @@ -3,20 +3,26 @@ import tensorflow as tf +from cuda_gpu_config import setup_gpu from model_export import dump_graph -from models.baseline_v4_8x16 import preprocess_features +from models.model_v4 import preprocess_features, Model_v4 +from models.utils import load_weights +from run_model_v4 import load_config def main(): parser = argparse.ArgumentParser(fromfile_prefix_chars='@') parser.add_argument('--checkpoint_name', type=str, required=True) parser.add_argument('--output_path', type=str, default='model_export/model_v4/graph.pbtxt') parser.add_argument('--latent_dim', type=int, default=32, required=False) - parser.add_argument('--dont_hack_upsampling_op', default=False, action='store_true') + parser.add_argument('--dont_hack_upsampling_op', default=True, action='store_true') parser.add_argument('--test_input', type=float, nargs=4, default=None) parser.add_argument('--constant_seed', type=float, default=None) + parser.add_argument('--gpu_num', type=str, default=None) args, _ = parser.parse_known_args() + setup_gpu(args.gpu_num) + print("") print("----" * 10) print("Arguments:") @@ -30,13 +36,10 @@ def epoch_from_name(name): return int(epoch) model_path = Path('saved_models') / args.checkpoint_name - gen_checkpoints = model_path.glob("generator_*.h5") - latest_gen_checkpoint = max( - gen_checkpoints, - key=lambda path: epoch_from_name(path.stem) - ) - model = tf.keras.models.load_model(str(latest_gen_checkpoint), compile=False) + full_model = Model_v4(load_config(model_path / 'config.yaml')) + load_weights(full_model, model_path) + model = full_model.generator if args.constant_seed is None: def preprocess(x): diff --git a/model_export/run_docker_centos.sh b/model_export/run_docker_centos.sh index 47e1afb..4df0f0e 100755 --- a/model_export/run_docker_centos.sh +++ b/model_export/run_docker_centos.sh @@ -1,4 +1,5 @@ docker run -it \ + --rm \ --env HOME=`pwd` \ -v `pwd`:`pwd` \ centos:centos7.8.2003 \ diff --git a/run_docker.sh b/run_docker.sh index dc1fb62..2fc7e28 100755 --- a/run_docker.sh +++ b/run_docker.sh @@ -1 +1,8 @@ -docker run -u $(id -u):$(id -g) --runtime nvidia -v `pwd`:`pwd` silikhon/tensorflow2:v1 /bin/bash -c 'cd '`pwd`'; python test_script.py' +docker run -it \ + --rm \ + -u $(id -u):$(id -g) \ + --env HOME=`pwd` \ + --runtime nvidia \ + -v `pwd`:`pwd` \ + silikhon/tensorflow2:v1 \ + /bin/bash From efb8fd1bdb54d5a16ffa7d5f708a287f92c86504 Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Wed, 4 Nov 2020 00:09:00 +0300 Subject: [PATCH 23/24] custom objects through config --- .../baseline_fc_8x16_kinked_trainable.yaml | 86 +++++++++++++++++++ models/model_v4.py | 6 +- models/nn.py | 10 ++- run_docker_jlab.sh | 1 + 4 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 models/configs/baseline_fc_8x16_kinked_trainable.yaml diff --git a/models/configs/baseline_fc_8x16_kinked_trainable.yaml b/models/configs/baseline_fc_8x16_kinked_trainable.yaml new file mode 100644 index 0000000..a1ed052 --- /dev/null +++ b/models/configs/baseline_fc_8x16_kinked_trainable.yaml @@ -0,0 +1,86 @@ +latent_dim: 32 +batch_size: 32 +lr: 1.e-4 +lr_schedule_rate: 0.999 + +num_disc_updates: 8 +gp_lambda: 10. +gpdata_lambda: 0. +cramer: False +stochastic_stepping: True + +save_every: 50 +num_epochs: 10000 + +feature_noise_power: NULL +feature_noise_decay: NULL + +data_version: 'data_v4' +pad_range: [-3, 5] +time_range: [-7, 9] +scaler: 'logarithmic' + +architecture: + generator: + - block_type: 'fully_connected' + arguments: + units: [32, 64, 64, 64, 128] + activations: [ + 'elu', 'elu', 'elu', 'elu', 'custom_objects["TrainableActivation"]()' + ] + kernel_init: 'glorot_uniform' + input_shape: [37,] + output_shape: [8, 16] + name: 'generator' + + discriminator: + - block_type: 'connect' + arguments: + vector_shape: [5,] + img_shape: [8, 16] + vector_bypass: False + concat_outputs: True + name: 'discriminator_tail' + block: + block_type: 'conv' + arguments: + filters: [16, 16, 32, 32, 64, 64] + kernel_sizes: [3, 3, 3, 3, 3, 2] + paddings: ['same', 'same', 'same', 'same', 'valid', 'valid'] + activations: ['elu', 'elu', 'elu', 'elu', 'elu', 'elu'] + poolings: [NULL, [1, 2], NULL, 2, NULL, NULL] + kernel_init: glorot_uniform + input_shape: NULL + output_shape: [64,] + dropouts: [0.02, 0.02, 0.02, 0.02, 0.02, 0.02] + name: discriminator_conv_block + - block_type: 'fully_connected' + arguments: + units: [128, 1] + activations: ['elu', NULL] + kernel_init: 'glorot_uniform' + input_shape: [69,] + output_shape: NULL + name: 'discriminator_head' + +custom_objects: | + class TrainableActivation(tf.keras.layers.Layer): + def __init__(self, val=np.log10(2)): + super().__init__(autocast=False) + self.v = tf.Variable(0., dtype='float32', trainable=True) + self.val = val + + def call(self, x): + val = self.val + slope = (tf.nn.elu(self.v * 50. + 50) + 1.) + return tf.where( + x >= 0, + val + x, + tf.exp(-tf.abs(x) * (slope + 1e-10)) * val + ) + + def get_config(self): + config = super().get_config().copy() + config.update(dict(val=self.val)) + return config + diff --git a/models/model_v4.py b/models/model_v4.py index 929094f..ded6544 100644 --- a/models/model_v4.py +++ b/models/model_v4.py @@ -77,8 +77,10 @@ def __init__(self, config): self.latent_dim = config['latent_dim'] architecture_descr = config['architecture'] - self.generator = nn.build_architecture(architecture_descr['generator']) - self.discriminator = nn.build_architecture(architecture_descr['discriminator']) + self.generator = nn.build_architecture(architecture_descr['generator'], + custom_objects_code=config.get('custom_objects', None)) + self.discriminator = nn.build_architecture(architecture_descr['discriminator'], + custom_objects_code=config.get('custom_objects', None)) self.step_counter = tf.Variable(0, dtype='int32', trainable=False) diff --git a/models/nn.py b/models/nn.py index 9ef13d3..f063306 100644 --- a/models/nn.py +++ b/models/nn.py @@ -2,6 +2,9 @@ import numpy as np +custom_objects = {} + + def get_activation(activation): try: activation = tf.keras.activations.get(activation) @@ -188,7 +191,12 @@ def build_block(block_type, arguments): return block -def build_architecture(block_descriptions, name=None): +def build_architecture(block_descriptions, name=None, custom_objects_code=None): + if custom_objects_code: + print("build_architecture(): got custom objects code, executing:") + print(custom_objects_code) + exec(custom_objects_code, globals(), custom_objects) + blocks = [build_block(**descr) for descr in block_descriptions] diff --git a/run_docker_jlab.sh b/run_docker_jlab.sh index 59c391f..a4e661b 100755 --- a/run_docker_jlab.sh +++ b/run_docker_jlab.sh @@ -6,6 +6,7 @@ else fi docker run -it \ + --rm \ -u $(id -u):$(id -g) \ --env HOME=`pwd` \ -p 127.0.0.1:$PORT:8888/tcp \ From 3e395e7d0a2015be77bc3552f43f44dc81d33eef Mon Sep 17 00:00:00 2001 From: Artem Maevskiy Date: Mon, 23 Nov 2020 22:05:56 +0000 Subject: [PATCH 24/24] saving images as PDFs; fixing mask plot --- metrics/__init__.py | 100 ++++++++++++++++++++++++++++++++++---------- metrics/trends.py | 3 +- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/metrics/__init__.py b/metrics/__init__.py index b64643d..6d9771d 100644 --- a/metrics/__init__.py +++ b/metrics/__init__.py @@ -12,7 +12,7 @@ from .trends import make_trend_plot -def make_histograms(data_real, data_gen, title, figsize=(8, 8), n_bins=100, logy=False): +def make_histograms(data_real, data_gen, title, figsize=(8, 8), n_bins=100, logy=False, pdffile=None): l = min(data_real.min(), data_gen.min()) r = max(data_real.max(), data_gen.max()) bins = np.linspace(l, r, n_bins + 1) @@ -27,6 +27,7 @@ def make_histograms(data_real, data_gen, title, figsize=(8, 8), n_bins=100, logy buf = io.BytesIO() fig.savefig(buf, format='png') + if pdffile is not None: fig.savefig(pdffile, format='pdf') plt.close(fig) buf.seek(0) @@ -34,8 +35,9 @@ def make_histograms(data_real, data_gen, title, figsize=(8, 8), n_bins=100, logy return np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[1], img.size[0], -1) -def make_metric_plots(images_real, images_gen, features=None, calc_chi2=False): +def make_metric_plots(images_real, images_gen, features=None, calc_chi2=False, make_pdfs=False): plots = {} + if make_pdfs: pdf_plots = {} if calc_chi2: chi2 = 0 @@ -43,29 +45,42 @@ def make_metric_plots(images_real, images_gen, features=None, calc_chi2=False): metric_real = get_val_metric_v(images_real) metric_gen = get_val_metric_v(images_gen ) - plots.update({name : make_histograms(real, gen, name) - for name, real, gen in zip(_METRIC_NAMES, metric_real.T, metric_gen.T)}) + for name, real, gen in zip(_METRIC_NAMES, metric_real.T, metric_gen.T): + pdffile = None + if make_pdfs: + pdffile = io.BytesIO() + pdf_plots[name] = pdffile + plots[name] = make_histograms(real, gen, name, pdffile=pdffile) + if features is not None: for feature_name, (feature_real, feature_gen) in features.items(): for metric_name, real, gen in zip(_METRIC_NAMES, metric_real.T, metric_gen.T): name = f'{metric_name} vs {feature_name}' + pdffile = None + if make_pdfs: + pdffile = io.BytesIO() + pdf_plots[name] = pdffile if calc_chi2 and (metric_name != "Sum"): plots[name], chi2_i = make_trend_plot(feature_real, real, feature_gen, gen, - name, calc_chi2=True) + name, calc_chi2=True, + pdffile=pdffile) chi2 += chi2_i else: plots[name] = make_trend_plot(feature_real, real, - feature_gen, gen, name) + feature_gen, gen, name, pdffile=pdffile) except AssertionError as e: print(f"WARNING! Assertion error ({e})") + result = {'plots' : plots} if calc_chi2: - return plots, chi2 + result['chi2'] = chi2 + if make_pdfs: + result['pdf_plots'] = pdf_plots - return plots + return result def make_images_for_model(model, @@ -73,10 +88,15 @@ def make_images_for_model(model, return_raw_data=False, calc_chi2=False, gen_more=None, - batch_size=128): + batch_size=128, + pdf_outputs=None): X, Y = sample assert X.ndim == 2 assert X.shape[1] == 4 + make_pdfs = (pdf_outputs is not None) + if make_pdfs: + assert isinstance(pdf_outputs, list) + assert len(pdf_outputs) == 0 if gen_more is None: gen_features = X @@ -102,16 +122,36 @@ def make_images_for_model(model, 'pad_coord_fraction' : (X[:, 3] % 1, gen_features[:,3] % 1) } - images = make_metric_plots(real, gen, features=features, calc_chi2=calc_chi2) + metric_plot_results = make_metric_plots(real, gen, features=features, + calc_chi2=calc_chi2, make_pdfs=make_pdfs) + images = metric_plot_results['plots'] if calc_chi2: - images, chi2 = images - - images1 = make_metric_plots(real, gen1, features=features) - - img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True) - - images['examples'] = plot_individual_images(Y, gen_scaled) - images['examples_mask'] = plot_images_mask(Y, gen_scaled) + chi2 = metric_plot_results['chi2'] + if make_pdfs: + images_pdf = metric_plot_results['pdf_plots'] + pdf_outputs.append(images_pdf) + + metric_plot_results1 = make_metric_plots(real, gen1, features=features, make_pdfs=make_pdfs) + images1 = metric_plot_results1['plots'] + if make_pdfs: + pdf_outputs.append(metric_plot_results1['pdf_plots']) + + pdffile = None + if make_pdfs: + pdffile = io.BytesIO() + pdf_outputs.append(pdffile) + img_amplitude = make_histograms(Y.flatten(), gen_scaled.flatten(), 'log10(amplitude + 1)', logy=True, + pdffile=pdffile) + + pdffile_examples = None + pdffile_examples_mask = None + if make_pdfs: + pdffile_examples = io.BytesIO() + pdffile_examples_mask = io.BytesIO() + images_pdf['examples'] = pdffile_examples + images_pdf['examples_mask'] = pdffile_examples_mask + images['examples'] = plot_individual_images(Y, gen_scaled, pdffile=pdffile_examples) + images['examples_mask'] = plot_images_mask(Y, gen_scaled, pdffile=pdffile_examples_mask) result = [images, images1, img_amplitude] @@ -126,11 +166,13 @@ def make_images_for_model(model, def evaluate_model(model, path, sample, gen_sample_name=None): path.mkdir() + pdf_outputs = [] ( images, images1, img_amplitude, gen_dataset, chi2 ) = make_images_for_model(model, sample=sample, - calc_chi2=True, return_raw_data=True, gen_more=10) + calc_chi2=True, return_raw_data=True, gen_more=10, pdf_outputs=pdf_outputs) + images_pdf, images1_pdf, img_amplitude_pdf = pdf_outputs array_to_img = lambda arr: PIL.Image.fromarray(arr.reshape(arr.shape[1:])) @@ -140,6 +182,16 @@ def evaluate_model(model, path, sample, gen_sample_name=None): array_to_img(img).save(str(path / f"{k}_amp_gt_1.png")) array_to_img(img_amplitude).save(str(path / "log10_amp_p_1.png")) + def buf_to_file(buf, filename): + with open(filename, 'wb') as f: + f.write(buf.getbuffer()) + + for k, img in images_pdf.items(): + buf_to_file(img, str(path / f"{k}.pdf")) + for k, img in images1_pdf.items(): + buf_to_file(img, str(path / f"{k}_amp_gt_1.pdf")) + buf_to_file(img_amplitude_pdf, str(path / "log10_amp_p_1.pdf")) + if gen_sample_name is not None: with open(str(path / gen_sample_name), 'w') as f: for event_X, event_Y in zip(*gen_dataset): @@ -155,7 +207,7 @@ def evaluate_model(model, path, sample, gen_sample_name=None): f.write(f"{chi2:.2f}\n") -def plot_individual_images(real, gen, n=10): +def plot_individual_images(real, gen, n=10, pdffile=None): assert real.ndim == 3 == gen.ndim assert real.shape[1:] == gen.shape[1:] N_max = min(len(real), len(gen)) @@ -182,6 +234,7 @@ def plot_individual_images(real, gen, n=10): buf = io.BytesIO() fig.savefig(buf, format='png') + if pdffile is not None: fig.savefig(pdffile, format='pdf') plt.close(fig) buf.seek(0) @@ -189,7 +242,7 @@ def plot_individual_images(real, gen, n=10): return np.array(img.getdata(), dtype=np.uint8).reshape(1, img.size[1], img.size[0], -1) -def plot_images_mask(real, gen): +def plot_images_mask(real, gen, pdffile=None): assert real.ndim == 3 == gen.ndim assert real.shape[1:] == gen.shape[1:] @@ -197,13 +250,14 @@ def plot_images_mask(real, gen): size_y = size_x / real.shape[2] * real.shape[1] * 2.4 fig, [ax0, ax1] = plt.subplots(2, 1, figsize=(size_x, size_y)) - ax0.imshow(real.any(axis=0), aspect='auto') + ax0.imshow((real >= 1.).any(axis=0), aspect='auto') ax0.set_title("real") - ax1.imshow(gen.any(axis=0), aspect='auto') + ax1.imshow((gen >= 1.).any(axis=0), aspect='auto') ax1.set_title("generated") buf = io.BytesIO() fig.savefig(buf, format='png') + if pdffile is not None: fig.savefig(pdffile, format='pdf') plt.close(fig) buf.seek(0) diff --git a/metrics/trends.py b/metrics/trends.py index de6ef42..2ea7d5c 100644 --- a/metrics/trends.py +++ b/metrics/trends.py @@ -51,7 +51,7 @@ def stats(arr): return (mean, std), (mean_err, std_err) -def make_trend_plot(feature_real, real, feature_gen, gen, name, calc_chi2=False, figsize=(8, 8)): +def make_trend_plot(feature_real, real, feature_gen, gen, name, calc_chi2=False, figsize=(8, 8), pdffile=None): feature_real = feature_real.squeeze() feature_gen = feature_gen.squeeze() real = real.squeeze() @@ -71,6 +71,7 @@ def make_trend_plot(feature_real, real, feature_gen, gen, name, calc_chi2=False, buf = io.BytesIO() fig.savefig(buf, format='png') + if pdffile is not None: fig.savefig(pdffile, format='pdf') plt.close(fig) buf.seek(0)