From 7204dc93dc557c5dfffbcbc5e838995fd9381095 Mon Sep 17 00:00:00 2001 From: enchanted123 <18701919513@163.com> Date: Tue, 23 May 2023 16:41:29 +0800 Subject: [PATCH] change some details --- .../quantum_rl/Quantum RL with Quafu.ipynb | 333 ------------------ examples/quantum_rl/misc/utils.py | 305 ++++++++++------ .../quantum_rl/models/quantum_genotypes.py | 51 +++ examples/quantum_rl/models/quantum_models.py | 19 +- .../search/quantum_evolution_search.py | 27 +- .../quantum_rl/search/quantum_train_search.py | 94 +---- .../quantum_rl/validation/quantum_test.py | 28 +- .../quantum_rl/validation/quantum_train.py | 87 ++--- examples/quantum_rl/visualization/plot_gif.py | 70 ++++ examples/quantum_rl/visualization/qrl.py | 140 -------- .../weights_id10_quafu_100.h5} | Bin 12568 -> 12568 bytes .../weights_id10_quafu_132.h5} | Bin 12600 -> 12568 bytes .../weights_id10_quafu_88.h5} | Bin 12600 -> 12568 bytes .../weights_id10_quafu_94.h5} | Bin 12600 -> 12568 bytes .../train_p136/weights_id10_quafu_91.h5 | Bin 0 -> 12568 bytes .../train_p136/weights_id10_quafu_94.h5 | Bin 0 -> 12568 bytes .../train_p18/weights_id10_quafu_80.h5 | Bin 0 -> 12568 bytes .../train_p18/weights_id10_quafu_86.h5 | Bin 0 -> 12568 bytes examples/quantum_rl/weights/weights_id10.h5 | Bin 12600 -> 0 bytes examples/quantum_rl/weights/weights_id97.h5 | Bin 12600 -> 0 bytes 20 files changed, 421 insertions(+), 733 deletions(-) delete mode 100644 examples/quantum_rl/Quantum RL with Quafu.ipynb create mode 100644 examples/quantum_rl/visualization/plot_gif.py delete mode 100644 examples/quantum_rl/visualization/qrl.py rename examples/quantum_rl/weights/{weights_id10_quafu.h5 => train_p10/weights_id10_quafu_100.h5} (87%) rename examples/quantum_rl/weights/{weights_id97_direct.h5 => train_p10/weights_id10_quafu_132.h5} (68%) rename examples/quantum_rl/weights/{weights_id97_quafu.h5 => train_p10/weights_id10_quafu_88.h5} (68%) rename examples/quantum_rl/weights/{weights_id10_direct.h5 => train_p10/weights_id10_quafu_94.h5} (68%) create mode 100644 examples/quantum_rl/weights/train_p136/weights_id10_quafu_91.h5 create mode 100644 examples/quantum_rl/weights/train_p136/weights_id10_quafu_94.h5 create mode 100644 examples/quantum_rl/weights/train_p18/weights_id10_quafu_80.h5 create mode 100644 examples/quantum_rl/weights/train_p18/weights_id10_quafu_86.h5 delete mode 100644 examples/quantum_rl/weights/weights_id10.h5 delete mode 100644 examples/quantum_rl/weights/weights_id97.h5 diff --git a/examples/quantum_rl/Quantum RL with Quafu.ipynb b/examples/quantum_rl/Quantum RL with Quafu.ipynb deleted file mode 100644 index d3fa12e..0000000 --- a/examples/quantum_rl/Quantum RL with Quafu.ipynb +++ /dev/null @@ -1,333 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5bbddd29", - "metadata": {}, - "source": [ - "# Reinforcement learining with quantum computing cloud Quafu" - ] - }, - { - "cell_type": "markdown", - "id": "d891b68d", - "metadata": {}, - "source": [ - "Before starting the journey of executing reinforcement learining(RL) task on real quantum devices with Quafu, you have to make sure the environment is consistent. The following code is based on python 3.8 to meet the need of the specific version of tensorflow. Then, you can install the follwing packages:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3fc952f8", - "metadata": {}, - "outputs": [], - "source": [ - "!pip install pyquafu \n", - "!pip install tensorflow==2.7.0\n", - "!pip install tensorflow-quantum==0.7.2\n", - "!pip install gym" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "40def399", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2023-03-15 14:42:36.950239: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-03-15 14:42:38.671011: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 20593 MB memory: -> device: 0, name: GeForce RTX 3090, pci bus id: 0000:5b:00.0, compute capability: 8.6\n", - "2023-03-15 14:42:38.672331: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 22306 MB memory: -> device: 1, name: GeForce RTX 3090, pci bus id: 0000:9b:00.0, compute capability: 8.6\n", - "2023-03-15 14:42:38.673411: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:2 with 22143 MB memory: -> device: 2, name: Tesla P40, pci bus id: 0000:25:00.0, compute capability: 6.1\n" - ] - } - ], - "source": [ - "# model imports\n", - "import tensorflow as tf\n", - "import argparse\n", - "import re\n", - "import cirq\n", - "import gym\n", - "import numpy as np\n", - "from functools import reduce\n", - "from PIL import Image\n", - "from quafu import QuantumCircuit as quafuQC\n", - "from quafu import Task, User\n", - "\n", - "import models.quantum_genotypes as genotypes\n", - "from models.quantum_models import generate_circuit\n", - "from models.quantum_models import generate_model_policy as Network\n", - "from models.quantum_models import get_model_circuit_params" - ] - }, - { - "cell_type": "markdown", - "id": "f3319328", - "metadata": {}, - "source": [ - "Set some parameters about the RL task:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "1f8d50c1", - "metadata": {}, - "outputs": [], - "source": [ - "parser = argparse.ArgumentParser('Reinforcement learining with quantum computing cloud Quafu')\n", - "parser.add_argument('--env_name', type=str, default='CartPole-v1', help='environment name')\n", - "parser.add_argument('--state_bounds', type=np.array, default=np.array([2.4, 2.5, 0.21, 2.5]), help='state bounds')\n", - "parser.add_argument('--n_qubits', type=int, default=4, help='the number of qubits')\n", - "parser.add_argument('--n_actions', type=int, default=2, help='the number of actions')\n", - "parser.add_argument('--arch', type=str, default='NSGANet_id10', help='which architecture to use')\n", - "parser.add_argument('--shots', type=int, default=1000, help='the number of sampling')\n", - "parser.add_argument('--backend', type=str, default='ScQ-P20', help='which backend to use')\n", - "parser.add_argument('--model_path', type=str, default='./weights/weights_id10_quafu.h5', help='path of pretrained model')\n", - "args = parser.parse_args(args=[])" - ] - }, - { - "cell_type": "markdown", - "id": "0b75c1ea", - "metadata": {}, - "source": [ - "According to the results retrieved by Quafu, you can compute expectations with observables($Z_{0} * Z_{1} * Z_{2} * Z_{3}$ for CartPole) as the follwing function:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "79aebf20", - "metadata": {}, - "outputs": [], - "source": [ - "def get_res_exp(res):\n", - " # access to probabilities of all possibilities \n", - " prob = res.probabilities\n", - " sumexp = 0\n", - " for k, v in prob.items():\n", - " count = 0\n", - " for i in range(len(k)):\n", - " if k[i] == '1':\n", - " count += 1\n", - " if count % 2 == 0:\n", - " sumexp += v\n", - " else:\n", - " sumexp -= v\n", - " return sumexp" - ] - }, - { - "cell_type": "markdown", - "id": "e78631a5", - "metadata": {}, - "source": [ - "It's important to construct a process to send circuits to Quafu and get results from it. The next part shows the whole pipeline of involing Cirq circuits with Quafu and acquire expectations with quantum devices." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "7b611594", - "metadata": {}, - "outputs": [], - "source": [ - "def get_quafu_exp(circuit):\n", - " # convert Cirq circuts to qasm\n", - " openqasm = circuit.to_qasm(header='')\n", - " openqasm = re.sub('//.*\\n', '', openqasm)\n", - " openqasm = \"\".join([s for s in openqasm.splitlines(True) if s.strip()])\n", - " \n", - " # fill in with your token, register on website http://quafu.baqis.ac.cn/\n", - " user = User()\n", - " user.save_apitoken(\" \")\n", - " \n", - " # initialize to Quafu circuits\n", - " q = quafuQC(args.n_qubits)\n", - " q.from_openqasm(openqasm)\n", - " \n", - " # create the task\n", - " task = Task()\n", - " task.load_account()\n", - " \n", - " # choose sampling number and specific quantum devices\n", - " shots = args.shots \n", - " task.config(backend=args.backend, shots=shots, compile=True)\n", - " task_id = task.send(q, wait=True).taskid\n", - " print('task_id:', task_id)\n", - " \n", - " # retrieve the result of completed tasks and compute expectations\n", - " task_status = task.retrieve(task_id).task_status\n", - " if task_status == 'Completed':\n", - " task = Task()\n", - " task.load_account()\n", - " res = task.retrieve(task_id)\n", - " OB = get_res_exp(res)\n", - " return task_id, tf.convert_to_tensor([[OB]])" - ] - }, - { - "cell_type": "markdown", - "id": "12875e09", - "metadata": {}, - "source": [ - "The next post-processing layer apply stored action-specific weights on expectation values." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "67e2e227", - "metadata": {}, - "outputs": [], - "source": [ - "class Alternating_(tf.keras.layers.Layer):\n", - " def __init__(self, obsw):\n", - " super(Alternating_, self).__init__()\n", - " self.w = tf.Variable(\n", - " initial_value=tf.constant(obsw), dtype=\"float32\", trainable=True, name=\"obsw\")\n", - "\n", - " def call(self, inputs):\n", - " return tf.matmul(inputs, self.w)" - ] - }, - { - "cell_type": "markdown", - "id": "7a0a0ef0", - "metadata": {}, - "source": [ - "Then the softmax layer outputs the policy of the agent to choose next actions." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "3c69ae1e", - "metadata": {}, - "outputs": [], - "source": [ - "def get_obs_policy(obsw):\n", - " process = tf.keras.Sequential([ Alternating_(obsw),\n", - " tf.keras.layers.Lambda(lambda x: x * 1.0),\n", - " tf.keras.layers.Softmax()\n", - " ], name=\"obs_policy\")\n", - " return process" - ] - }, - { - "cell_type": "markdown", - "id": "ed82bd69", - "metadata": {}, - "source": [ - "Prepare for loading model weights:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "3b1d0556", - "metadata": {}, - "outputs": [], - "source": [ - "qubits = cirq.GridQubit.rect(1, args.n_qubits)\n", - "genotype = eval(\"genotypes.%s\" % args.arch)\n", - "ops = [cirq.Z(q) for q in qubits]\n", - "observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3\n", - "model = Network(qubits, genotype, args.n_actions, observables)\n", - "model.load_weights(args.model_path)" - ] - }, - { - "cell_type": "markdown", - "id": "77547584", - "metadata": {}, - "source": [ - "The follwing part builds an interaction between the agent in CartPole environment and Quafu. Every action choice means a task completed by quantum devices and finally, you can get a gif picturing the whole process." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37cd2946", - "metadata": {}, - "outputs": [], - "source": [ - "# update gym to the version having render_mode, which is 0.26.1 in this file\n", - "# initialize the environment\n", - "env = gym.make(args.env_name, render_mode=\"rgb_array\")\n", - "state, _ = env.reset()\n", - "frames = []\n", - "\n", - "# set the number of episodes\n", - "for epi in range(20):\n", - " im = Image.fromarray(env.render())\n", - " frames.append(im) \n", - " \n", - " # get PQC model parameters and expectations\n", - " stateb = state/args.state_bounds\n", - " newtheta, newlamda = get_model_circuit_params(qubits, genotype, model)\n", - " circuit, _, _ = generate_circuit(qubits, genotype, newtheta, newlamda, stateb)\n", - " _, expectation = get_quafu_exp(circuit)\n", - " \n", - " # get policy model parameters\n", - " obsw = model.get_layer('observables-policy').get_weights()[0]\n", - " obspolicy = get_obs_policy(obsw)\n", - " policy = obspolicy(expectation)\n", - " print('policy:', policy)\n", - " \n", - " # choose actions and make a step\n", - " action = np.random.choice(args.n_actions, p=policy.numpy()[0])\n", - " state, reward, terminated, truncated, _ = env.step(action)\n", - " if terminated or truncated:\n", - " print(epi+1)\n", - " break\n", - "env.close()\n", - "\n", - "# save to your path\n", - "frames[1].save('./visualization/id10_quafu_gif/gym_CartPole_20.gif', save_all=True, append_images=frames[2:], optimize=False, duration=20, loop=0)\n", - "# for example, ./visualization/id10_quafu_gif/gym_CartPole_20.gif" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "96034943", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "tfjyx", - "language": "python", - "name": "tfjyx" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - }, - "vscode": { - "interpreter": { - "hash": "8453d71d5161753483e9e20455923c6b6a939f471903d1c75f4816d93f062f94" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/quantum_rl/misc/utils.py b/examples/quantum_rl/misc/utils.py index c38e294..2e88173 100644 --- a/examples/quantum_rl/misc/utils.py +++ b/examples/quantum_rl/misc/utils.py @@ -1,118 +1,14 @@ import os +import re import shutil +from collections import defaultdict +import gym import numpy as np -import torch -import torchvision.transforms as transforms -from torch.autograd import Variable - - -class AvgrageMeter(object): - def __init__(self): - self.reset() - - def reset(self): - self.avg = 0 - self.sum = 0 - self.cnt = 0 - - def update(self, val, n=1): - self.sum += val * n - self.cnt += n - self.avg = self.sum / self.cnt - - -def accuracy(output, target, topk=(1,)): - maxk = max(topk) - batch_size = target.size(0) - - _, pred = output.topk(maxk, 1, True, True) - pred = pred.t() - correct = pred.eq(target.view(1, -1).expand_as(pred)) - - res = [] - for k in topk: - correct_k = correct[:k].view(-1).float().sum(0) - res.append(correct_k.mul_(100.0 / batch_size)) - return res - - -class Cutout(object): - def __init__(self, length): - self.length = length - - def __call__(self, img): - h, w = img.size(1), img.size(2) - mask = np.ones((h, w), np.float32) - y = np.random.randint(h) - x = np.random.randint(w) - - y1 = np.clip(y - self.length // 2, 0, h) - y2 = np.clip(y + self.length // 2, 0, h) - x1 = np.clip(x - self.length // 2, 0, w) - x2 = np.clip(x + self.length // 2, 0, w) - - mask[y1: y2, x1: x2] = 0. - mask = torch.from_numpy(mask) - mask = mask.expand_as(img) - img *= mask - return img - - -def _data_transforms_cifar10(args): - CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124] - CIFAR_STD = [0.24703233, 0.24348505, 0.26158768] - - train_transform = transforms.Compose([ - transforms.RandomCrop(32, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor() - ]) - - if args.cutout: - train_transform.transforms.append(Cutout(args.cutout_length)) - - train_transform.transforms.append(transforms.Normalize(CIFAR_MEAN, CIFAR_STD)) - - valid_transform = transforms.Compose([ - transforms.ToTensor(), - transforms.Normalize(CIFAR_MEAN, CIFAR_STD), - ]) - return train_transform, valid_transform - - -def count_parameters_in_MB(model): - - n_params_from_auxiliary_head = np.sum(np.prod(v.size()) for name, v in model.named_parameters()) - \ - np.sum(np.prod(v.size()) for name, v in model.named_parameters() - if "auxiliary" not in name) - n_params_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) - return (n_params_trainable - n_params_from_auxiliary_head) / 1e6 - - -def save_checkpoint(state, is_best, save): - filename = os.path.join(save, 'checkpoint.pth.tar') - torch.save(state, filename) - if is_best: - best_filename = os.path.join(save, 'model_best.pth.tar') - shutil.copyfile(filename, best_filename) - - -def save(model, model_path): - torch.save(model.state_dict(), model_path) - - -def load(model, model_path): - model.load_state_dict(torch.load(model_path)) - - -def drop_path(x, drop_prob): - if drop_prob > 0.: - keep_prob = 1. - drop_prob - mask = Variable(torch.cuda.FloatTensor(x.size(0), 1, 1, 1).bernoulli_(keep_prob)) - x.div_(keep_prob) - x.mul_(mask) - return x +import tensorflow as tf +from models.quantum_models import generate_circuit, get_model_circuit_params +from quafu import QuantumCircuit as quafuQC +from quafu import Task, User def create_exp_dir(path, scripts_to_save=None): @@ -125,3 +21,190 @@ def create_exp_dir(path, scripts_to_save=None): for script in scripts_to_save: dst_file = os.path.join(path, 'scripts', os.path.basename(script)) shutil.copyfile(script, dst_file) + + +def get_res_exp(res): + """ + Access to probabilities of all possibilities, observable specifies to Z_0*Z_1*Z_2*Z_3. + """ + prob = res.probabilities + sumexp = 0 + for k, v in prob.items(): + count = 0 + for i in range(len(k)): + if k[i] == '1': + count += 1 + if count % 2 == 0: + sumexp += v + else: + sumexp -= v + return sumexp + + +def get_quafu_exp(circuit, n_qubits, backend_quafu, shots): + """ + Execute circuits on quafu cloud platform and return the expectation. + """ + # convert Cirq circuts to qasm + openqasm = circuit.to_qasm(header='') + openqasm = re.sub('//.*\n', '', openqasm) + openqasm = "".join([s for s in openqasm.splitlines(True) if s.strip()]) + + # fill in with your token, register on website http://quafu.baqis.ac.cn/ + user = User() + user.save_apitoken(" ") + + # initialize to Quafu circuits + q = quafuQC(n_qubits) + q.from_openqasm(openqasm) + + # create the task + task = Task() + + task.config(backend_quafu, shots, compile=True, priority=3) + task_id = task.send(q, wait=True).taskid + print('task_id:', task_id) + + # retrieve the result of completed tasks and compute expectations + task_status = task.retrieve(task_id).task_status + if task_status == 'Completed': + task = Task() + res = task.retrieve(task_id) + OB = get_res_exp(res) + return task_id, tf.convert_to_tensor([[OB]]) + + +def get_compiled_gates_depth(circuit, n_qubits, backend_quafu, shots): + """ + Get the gates and layered circuits of compiled circuits. + """ + openqasm = circuit.to_qasm(header='') + openqasm = re.sub('//.*\n', '', openqasm) + openqasm = "".join([s for s in openqasm.splitlines(True) if s.strip()]) + + user = User() + user.save_apitoken(" ") + + q = quafuQC(n_qubits) + q.from_openqasm(openqasm) + + task = Task() + + task.config(backend_quafu, shots, compile=True) + task_id = task.send(q, wait=True).taskid + print('task_id:', task_id) + + task_status = task.retrieve(task_id).task_status + if task_status == 'Completed': + task = Task() + res = task.retrieve(task_id) + gates = res.transpiled_circuit.gates + layered_circuit = res.transpiled_circuit.layered_circuit() + return task_id, gates, layered_circuit + + +class Alternating_(tf.keras.layers.Layer): + """ + Load observable weights of pre-trained models. + """ + def __init__(self, obsw): + super(Alternating_, self).__init__() + self.w = tf.Variable( + initial_value=tf.constant(obsw), dtype="float32", trainable=True, name="obsw") + + def call(self, inputs): + return tf.matmul(inputs, self.w) + + +def get_obs_policy(obsw, beta): + """ + Output the final policy. + """ + process = tf.keras.Sequential([ Alternating_(obsw), + tf.keras.layers.Lambda(lambda x: x * beta), + tf.keras.layers.Softmax() + ], name="obs_policy") + return process + + +def get_height(position): + """ + Get the height of position in MountainCar-v0. + """ + return np.sin(3 * position)*.45+.55 + + +def gather_episodes(state_bounds, n_actions, model, n_episodes, env_name, beta, backend, backend_quafu='ScQ-P10', shots=1000, + n_qubits=4, qubits=None, genotype=None): + """ + Interact with environment, you can choose the backend between `cirq` simulator and `quafu` cloud platform. + """ + trajectories = [defaultdict(list) for _ in range(n_episodes)] + envs = [gym.make(env_name) for _ in range(n_episodes)] + + done = [False for _ in range(n_episodes)] + states = [e.reset() for e in envs] + + tasklist = [] + + while not all(done): + unfinished_ids = [i for i in range(n_episodes) if not done[i]] + normalized_states = [s/state_bounds for i, s in enumerate(states) if not done[i]] + # height = [get_height(s[0]) for i, s in enumerate(states) if not done[i]] + + for i, state in zip(unfinished_ids, normalized_states): + trajectories[i]['states'].append(state) + + # Compute policy for all unfinished envs in parallel + states = tf.convert_to_tensor(normalized_states) + + if backend == 'cirq': + action_probs = model([states]) + elif backend == 'quafu': + newtheta, newlamda = get_model_circuit_params(qubits, genotype, model) + circuit, _, _ = generate_circuit(qubits, genotype, newtheta, newlamda, states.numpy()[0]) + taskid, expectation = get_quafu_exp(circuit, n_qubits, backend_quafu, shots) + tasklist.append(taskid) + # print('gather_episodes_exp:', expectation) + + obsw = model.get_layer('observables-policy').get_weights()[0] + obspolicy = get_obs_policy(obsw, beta) + action_probs = obspolicy(expectation) + else: + print('This backend is not supported now.') + + # Store action and transition all environments to the next state + states = [None for i in range(n_episodes)] + for i, policy in zip(unfinished_ids, action_probs.numpy()): + trajectories[i]['action_probs'].append(policy) + action = np.random.choice(n_actions, p=policy) + states[i], reward, done[i], _ = envs[i].step(action) + trajectories[i]['actions'].append(action) + if env_name == "CartPole-v1": + trajectories[i]['rewards'].append(reward) + elif env_name == "MountainCar-v0": + trajectories[i]['rewards'].append(reward + get_height(states[i][0])) + else: + print('This environment is not supported now.') + + return tasklist, trajectories + + +def compute_returns(rewards_history, gamma): + """ + Compute discounted returns with discount factor `gamma`. + """ + returns = [] + discounted_sum = 0 + for r in rewards_history[::-1]: + discounted_sum = r + gamma * discounted_sum + returns.insert(0, discounted_sum) + + # Normalize them for faster and more stable learning + returns = np.array(returns) + returns = (returns - np.mean(returns)) / (np.std(returns) + 1e-8) + returns = returns.tolist() + + return returns + + diff --git a/examples/quantum_rl/models/quantum_genotypes.py b/examples/quantum_rl/models/quantum_genotypes.py index 655e586..746424f 100644 --- a/examples/quantum_rl/models/quantum_genotypes.py +++ b/examples/quantum_rl/models/quantum_genotypes.py @@ -76,4 +76,55 @@ entangle=[ ('entanglement', 2) ] +) + +Layer5_CP = Genotype( + measure=[ + ('measurement', 15) + ], + vpqc=[ + ('variationalPQC', 0), + ('variationalPQC', 3), + ('variationalPQC', 6), + ('variationalPQC', 9), + ('variationalPQC', 12) + ], + dpqc=[ + ('dataencodingPQC', 2), + ('dataencodingPQC', 5), + ('dataencodingPQC', 8), + ('dataencodingPQC', 11), + ('dataencodingPQC', 14) + ], + entangle=[ + ('entanglement', 1), + ('entanglement', 4), + ('entanglement', 7), + ('entanglement', 10), + ('entanglement', 13) + ] +) + +Eqas_PQC = Genotype( + measure=[ + ('measurement', 12) + ], + vpqc=[ + ('variationalPQC', 5), + ('variationalPQC', 7) + ], + dpqc=[ + ('dataencodingPQC', 2), + ('dataencodingPQC', 6), + ('dataencodingPQC', 9), + ('dataencodingPQC', 11) + ], + entangle=[ + ('entanglement', 0), + ('entanglement', 1), + ('entanglement', 3), + ('entanglement', 4), + ('entanglement', 8), + ('entanglement', 10) + ] ) \ No newline at end of file diff --git a/examples/quantum_rl/models/quantum_models.py b/examples/quantum_rl/models/quantum_models.py index 68ecc81..5e90b52 100644 --- a/examples/quantum_rl/models/quantum_models.py +++ b/examples/quantum_rl/models/quantum_models.py @@ -122,23 +122,28 @@ def call(self, inputs): class Alternating(tf.keras.layers.Layer): """Apply action-specific weights.""" - def __init__(self, output_dim): + def __init__(self, output_dim, env): super(Alternating, self).__init__() - self.w = tf.Variable( - initial_value=tf.constant([[(-1.)**i for i in range(output_dim)]]), dtype="float32", - trainable=True, name="obs-weights") + if env == "CartPole-v1": + self.w = tf.Variable( + initial_value=tf.constant([[(-1.)**i for i in range(output_dim)]]), dtype="float32", + trainable=True, name="obs-weights") + elif env == "MountainCar-v0": + self.w = tf.Variable( + initial_value=tf.constant([[(-1.)**i for i in range(output_dim)], [(-1.)**i for i in range(output_dim)], + [(-1.)**i for i in range(output_dim)]]), dtype="float32", trainable=True, name="obs-weights") def call(self, inputs): return tf.matmul(inputs, self.w) -def generate_model_policy(qubits, genotype, n_actions, observables): +def generate_model_policy(qubits, genotype, n_actions, beta, observables, env): """Generate a Keras model for a NSGANet PQC policy.""" input_tensor = tf.keras.Input(shape=(len(qubits), ), dtype=tf.dtypes.float32, name='input') nsganet_pqc = NSGANetPQC(qubits, genotype, observables)([input_tensor]) process = tf.keras.Sequential([ - Alternating(n_actions), - tf.keras.layers.Lambda(lambda x: x * 1.0), + Alternating(n_actions, env), + tf.keras.layers.Lambda(lambda x: x * beta), tf.keras.layers.Softmax() ], name="observables-policy") policy = process(nsganet_pqc) diff --git a/examples/quantum_rl/search/quantum_evolution_search.py b/examples/quantum_rl/search/quantum_evolution_search.py index 3c602d5..df5d7ee 100644 --- a/examples/quantum_rl/search/quantum_evolution_search.py +++ b/examples/quantum_rl/search/quantum_evolution_search.py @@ -1,14 +1,16 @@ +# Note: If you want to search the architecture from scratch, please use the command 'pip install pymoo==0.3.0' first. import argparse import logging import os import sys + +sys.path.insert(0, ' ') import time from functools import reduce import cirq import numpy as np -import tensorflow as tf -from misc import utils +from misc.utils import create_exp_dir from pymoo.optimize import minimize from pymop.problem import Problem from search import nsganet as engine @@ -21,9 +23,10 @@ parser.add_argument('--n_gens', type=int, default=10, help='number of generation') parser.add_argument('--n_offspring', type=int, default=10, help='number of offspring created per generation') parser.add_argument('--n_episodes', type=int, default=300, help='number of episodes to train during architecture search') + args = parser.parse_args(args=[]) args.save = 'search-{}-{}'.format(args.save, time.strftime("%Y%m%d-%H%M%S")) -utils.create_exp_dir(args.save) +create_exp_dir(args.save) log_format = '%(asctime)s %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, @@ -104,19 +107,16 @@ def do_every_generations(algorithm): np.median(pop_obj[:, 1]), np.max(pop_obj[:, 1]), np.where(pop_obj[:, 1]==np.min(pop_obj[:, 1])))) -def main(): +def main(qubits, n_actions, observables): + """ + Main search process in multi-obj algorithms. + """ logging.info("args = %s", args) # setup NAS search problem lb = np.zeros(args.n_var) ub = np.ones(args.n_var) * 3 - n_qubits = 4 # Dimension of the state vectors in CartPole - n_actions = 2 # Number of actions in CartPole - qubits = cirq.GridQubit.rect(1, n_qubits) - ops = [cirq.Z(q) for q in qubits] - observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3 - problem = NAS(qubits, n_actions, observables, lb=lb, ub=ub, n_var=args.n_var, n_episodes=args.n_episodes, save_dir=args.save) @@ -134,4 +134,9 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + n_qubits = 4 # Dimension of the state vectors in CartPole + n_actions = 2 # Number of actions in CartPole + qubits = cirq.GridQubit.rect(1, n_qubits) + ops = [cirq.Z(q) for q in qubits] + observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3 + main(qubits, n_actions, observables) \ No newline at end of file diff --git a/examples/quantum_rl/search/quantum_train_search.py b/examples/quantum_rl/search/quantum_train_search.py index c6c9ae3..10a983c 100644 --- a/examples/quantum_rl/search/quantum_train_search.py +++ b/examples/quantum_rl/search/quantum_train_search.py @@ -2,87 +2,23 @@ import os import sys import time -from collections import defaultdict from functools import reduce -import cirq -import gym import numpy as np import tensorflow as tf -from misc import utils -from models.quantum_models import generate_circuit +from misc.utils import compute_returns, create_exp_dir, gather_episodes from models.quantum_models import generate_model_policy as Network -from models.quantum_models import get_model_circuit_params from search import quantum_encoding -from visualization.qrl import get_obs_policy, get_quafu_exp -def gather_episodes(state_bounds, n_actions, model, n_episodes, env_name, qubits=None, genotype=None): - """Interact with environment in batched fashion.""" - - trajectories = [defaultdict(list) for _ in range(n_episodes)] - envs = [gym.make(env_name) for _ in range(n_episodes)] - - done = [False for _ in range(n_episodes)] - states = [e.reset() for e in envs] - - tasklist = [] - - while not all(done): - unfinished_ids = [i for i in range(n_episodes) if not done[i]] - normalized_states = [s/state_bounds for i, s in enumerate(states) if not done[i]] - - for i, state in zip(unfinished_ids, normalized_states): - trajectories[i]['states'].append(state) - - # Compute policy for all unfinished envs in parallel - states = tf.convert_to_tensor(normalized_states) - - # You can choose action probabilities trained by Cirq simulator or Quafu cloud - # action_probs = model([states]) - newtheta, newlamda = get_model_circuit_params(qubits, genotype, model) - circuit, _, _ = generate_circuit(qubits, genotype, newtheta, newlamda, states.numpy()[0]) - taskid, expectation = get_quafu_exp(circuit) - tasklist.append(taskid) - # print('gather_episodes_exp:', expectation) - - obsw = model.get_layer('observables-policy').get_weights()[0] - obspolicy = get_obs_policy(obsw) - action_probs = obspolicy(expectation) - # print('gather_episodes_policy:', action_probs) - - # Store action and transition all environments to the next state - states = [None for i in range(n_episodes)] - for i, policy in zip(unfinished_ids, action_probs.numpy()): - action = np.random.choice(n_actions, p=policy) - states[i], reward, done[i], _ = envs[i].step(action) - trajectories[i]['actions'].append(action) - trajectories[i]['rewards'].append(reward) - - return tasklist, trajectories - - -def compute_returns(rewards_history, gamma): - """Compute discounted returns with discount factor `gamma`.""" - returns = [] - discounted_sum = 0 - for r in rewards_history[::-1]: - discounted_sum = r + gamma * discounted_sum - returns.insert(0, discounted_sum) - - # Normalize them for faster and more stable learning - returns = np.array(returns) - returns = (returns - np.mean(returns)) / (np.std(returns) + 1e-8) - returns = returns.tolist() - - return returns - - -def main(bit_string, qubits, n_actions, observables, n_episodes = 1000, batch_size = 10, gamma = 1, - state_bounds = np.array([2.4, 2.5, 0.21, 2.5]), env_name = "CartPole-v1", save='quantum', expr_root='search'): - """Main training process""" +def main(bit_string, qubits, n_actions, observables, n_episodes = 1000, batch_size = 10, gamma = 1, beta = 1.0, + state_bounds = np.array([2.4, 2.5, 0.21, 2.5]), env_name = "CartPole-v1", save='quantum', expr_root='search', + lr_in = 0.1, lr_var = 0.01, lr_out = 0.1, backend = 'cirq'): + """ + Main training process in multi-objective search. + """ save_pth = os.path.join(expr_root, '{}'.format(save)) - utils.create_exp_dir(save_pth) + create_exp_dir(save_pth) log_format = '%(asctime)s %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=log_format, datefmt='%m/%d %I:%M:%S %p') @@ -91,14 +27,14 @@ def main(bit_string, qubits, n_actions, observables, n_episodes = 1000, batch_si logging.getLogger().addHandler(fh) nb, genotype = quantum_encoding.convert2arch(bit_string) - model = Network(qubits, genotype, n_actions, observables) + model = Network(qubits, genotype, n_actions, beta, observables, env_name) logging.info("Genome = %s", nb) logging.info("Architecture = %s", genotype) - optimizer_in = tf.keras.optimizers.Adam(learning_rate=0.1, amsgrad=True) - optimizer_var = tf.keras.optimizers.Adam(learning_rate=0.01, amsgrad=True) - optimizer_out = tf.keras.optimizers.Adam(learning_rate=0.1, amsgrad=True) + optimizer_in = tf.keras.optimizers.Adam(learning_rate=lr_in, amsgrad=True) + optimizer_var = tf.keras.optimizers.Adam(learning_rate=lr_var, amsgrad=True) + optimizer_out = tf.keras.optimizers.Adam(learning_rate=lr_out, amsgrad=True) # Assign the model parameters to each optimizer w_in, w_var, w_out = 1, 0, 2 @@ -123,7 +59,7 @@ def reinforce_update(states, actions, returns, model): episode_reward_history = [] for batch in range(n_episodes // batch_size): # Gather episodes - episodes = gather_episodes(state_bounds, n_actions, model, batch_size, env_name) + _, episodes = gather_episodes(state_bounds, n_actions, model, batch_size, env_name, beta, backend) # Group states, actions and returns in numpy arrays states = np.concatenate([ep['states'] for ep in episodes]) @@ -146,6 +82,8 @@ def reinforce_update(states, actions, returns, model): logging.info('Finished episode: %f', (batch + 1) * batch_size) logging.info('Average rewards: %f', avg_rewards) - if avg_rewards >= 500.0: + if avg_rewards >= 500.0 and env_name == "CartPole-v1": + break + elif avg_rewards >= -110 and env_name == "MountainCar-v0": break return episode_reward_history diff --git a/examples/quantum_rl/validation/quantum_test.py b/examples/quantum_rl/validation/quantum_test.py index 233b774..df1c0ca 100644 --- a/examples/quantum_rl/validation/quantum_test.py +++ b/examples/quantum_rl/validation/quantum_test.py @@ -2,34 +2,38 @@ import logging import os import sys + +sys.path.insert(0, ' ') import time from functools import reduce import cirq -# model imports +import gym import models.quantum_genotypes as genotypes import numpy as np import tensorflow as tf -from misc import utils +from misc.utils import create_exp_dir, gather_episodes from models.quantum_models import generate_model_policy as Network -from search.quantum_train_search import gather_episodes -from sympy import im -parser = argparse.ArgumentParser('Quantum RL Testing') -parser.add_argument('--save', type=str, default='qEXP', help='experiment name') +parser = argparse.ArgumentParser('Quantum RL Inference') +parser.add_argument('--save', type=str, default='qEXP_quafu', help='experiment name') parser.add_argument('--batch_size', type=int, default=1, help='batch size') -parser.add_argument('--infer_episodes', type=int, default=10, help='the number of infer episodes') +parser.add_argument('--infer_episodes', type=int, default=100, help='the number of infer episodes') parser.add_argument('--gamma', type=float, default=1.0, help='discount parameter') parser.add_argument('--env_name', type=str, default="CartPole-v1", help='environment name') parser.add_argument('--state_bounds', type=np.array, default=np.array([2.4, 2.5, 0.21, 2.5]), help='state bounds') parser.add_argument('--n_qubits', type=int, default=4, help='the number of qubits') parser.add_argument('--n_actions', type=int, default=2, help='the number of actions') parser.add_argument('--arch', type=str, default='NSGANet_id10', help='which architecture to use') -parser.add_argument('--model_path', type=str, default='./weights/weights_id10_quafu.h5', help='path of pretrained model') +parser.add_argument('--model_path', type=str, default='./weights/train_p10/weights_id10_quafu_94.h5', help='path of pretrained model') +parser.add_argument('--beta', type=float, default=1.0, help='output parameter') +parser.add_argument('--backend', type=str, default='quafu', help='choose cirq simulator or quafu cloud platform') +parser.add_argument('--shots', type=int, default=1000, help='the number of sampling') +parser.add_argument('--backend_quafu', type=str, default='ScQ-P10', help='which quafu backend to use') args = parser.parse_args(args=[]) args.save = 'infer-{}-{}'.format(args.save, time.strftime("%Y%m%d-%H%M%S")) -utils.create_exp_dir(args.save) +create_exp_dir(args.save) log_format = '%(asctime)s %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, @@ -47,7 +51,7 @@ def main(): logging.info("args = %s", args) - model = Network(qubits, genotype, args.n_actions, observables) + model = Network(qubits, genotype, args.n_actions, args.beta, observables, args.env_name) model.load_weights(args.model_path) @@ -59,8 +63,10 @@ def infer(model): episode_reward_history = [] for batch in range(args.infer_episodes // args.batch_size): # Gather episodes - tasklist, episodes = gather_episodes(args.state_bounds, args.n_actions, model, args.batch_size, args.env_name, qubits, genotype) + tasklist, episodes = gather_episodes(args.state_bounds, args.n_actions, model, args.batch_size, + args.env_name, args.beta, args.backend, args.backend_quafu, args.shots, args.n_qubits, qubits, genotype) logging.info(tasklist) + logging.info(episodes) # Group states, actions and returns in numpy arrays states = np.concatenate([ep['states'] for ep in episodes]) diff --git a/examples/quantum_rl/validation/quantum_train.py b/examples/quantum_rl/validation/quantum_train.py index 087e4df..726f786 100644 --- a/examples/quantum_rl/validation/quantum_train.py +++ b/examples/quantum_rl/validation/quantum_train.py @@ -3,39 +3,43 @@ import logging import os import sys + +sys.path.insert(0, ' ') import time from functools import reduce import cirq import gym -# model imports import models.quantum_genotypes as genotypes import numpy as np import tensorflow as tf -from misc import utils -from models.quantum_models import generate_circuit +from misc.utils import compute_returns, create_exp_dir, gather_episodes from models.quantum_models import generate_model_policy as Network -from models.quantum_models import get_model_circuit_params -from search.quantum_train_search import compute_returns, gather_episodes -from sympy import im -from visualization.qrl import get_obs_policy, get_quafu_exp parser = argparse.ArgumentParser('Quantum RL Training') -parser.add_argument('--save', type=str, default='qEXP-quafu', help='experiment name') +parser.add_argument('--save', type=str, default='qEXP-quafu18_6', help='experiment name') parser.add_argument('--batch_size', type=int, default=1, help='batch size') -parser.add_argument('--n_episodes', type=int, default=10, help='the number of episodes') -parser.add_argument('--infer_episodes', type=int, default=5, help='the number of infer episodes') +parser.add_argument('--n_episodes', type=int, default=100, help='the number of episodes') +# parser.add_argument('--infer_episodes', type=int, default=5, help='the number of infer episodes') parser.add_argument('--gamma', type=float, default=1.0, help='discount parameter') parser.add_argument('--env_name', type=str, default="CartPole-v1", help='environment name') parser.add_argument('--state_bounds', type=np.array, default=np.array([2.4, 2.5, 0.21, 2.5]), help='state bounds') parser.add_argument('--n_qubits', type=int, default=4, help='the number of qubits') parser.add_argument('--n_actions', type=int, default=2, help='the number of actions') parser.add_argument('--arch', type=str, default='NSGANet_id10', help='which architecture to use') -parser.add_argument('--epochs', type=int, default=5, help='num of training epochs') +parser.add_argument('--epochs', type=int, default=1, help='num of training epochs') +parser.add_argument('--lr_in', type=float, default=0.1, help='learning rate of input parameter') +parser.add_argument('--lr_var', type=float, default=0.01, help='learning rate of variational parameter') +parser.add_argument('--lr_out', type=float, default=0.1, help='learning rate of output parameter') +parser.add_argument('--beta', type=float, default=1.0, help='output parameter') +parser.add_argument('--model_path', type=str, default='./weights/train_p18/weights_id10_quafu_86.h5', help='path of pretrained model') +parser.add_argument('--backend', type=str, default='quafu', help='choose cirq simulator or quafu cloud platform') +parser.add_argument('--shots', type=int, default=1000, help='the number of sampling') +parser.add_argument('--backend_quafu', type=str, default='ScQ-P10', help='which quafu backend to use') args = parser.parse_args(args=[]) args.save = 'train-{}-{}'.format(args.save, time.strftime("%Y%m%d-%H%M%S")) -utils.create_exp_dir(args.save) +create_exp_dir(args.save) log_format = '%(asctime)s %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, @@ -44,26 +48,23 @@ fh.setFormatter(logging.Formatter(log_format)) logging.getLogger().addHandler(fh) -qubits = cirq.GridQubit.rect(1, args.n_qubits) -genotype = eval("genotypes.%s" % args.arch) -ops = [cirq.Z(q) for q in qubits] -observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3 -def main(): +def main(qubits, genotype, observables): logging.info("args = %s", args) - model = Network(qubits, genotype, args.n_actions, observables) + model = Network(qubits, genotype, args.n_actions, args.beta, observables, args.env_name) + model.load_weights(args.model_path) n_epochs = args.epochs - optimizer_in = tf.keras.optimizers.Adam(learning_rate=0.1, amsgrad=True) - optimizer_var = tf.keras.optimizers.Adam(learning_rate=0.01, amsgrad=True) - optimizer_out = tf.keras.optimizers.Adam(learning_rate=0.1, amsgrad=True) + optimizer_in = tf.keras.optimizers.Adam(learning_rate=args.lr_in, amsgrad=True) + optimizer_var = tf.keras.optimizers.Adam(learning_rate=args.lr_var, amsgrad=True) + optimizer_out = tf.keras.optimizers.Adam(learning_rate=args.lr_out, amsgrad=True) # Assign the model parameters to each optimizer w_in, w_var, w_out = 1, 0, 2 - best_reward = 0 + # best_reward = 0 for epoch in range(n_epochs): logging.info('epoch %d', epoch) @@ -79,24 +80,18 @@ def main(): def train(model, optimizer_in, optimizer_var, optimizer_out, w_in, w_var, w_out): @tf.function - def reinforce_update(states, actions, returns, model): + def reinforce_update(states, actions, returns, logits2, model): states = tf.convert_to_tensor(states) actions = tf.convert_to_tensor(actions) returns = tf.convert_to_tensor(returns) + logits2 = tf.convert_to_tensor(logits2) with tf.GradientTape() as tape: tape.watch(model.trainable_variables) logits = model(states) - # newtheta, newlamda = get_model_circuit_params(qubits, genotype, model) - # circuit, _, _ = generate_circuit(qubits, genotype, newtheta, newlamda, states[0].numpy()[0]) - # expectation = get_quafu_exp(circuit) - # print('update_exp:', expectation) - - # obsw = model.get_layer('observables-policy').get_weights()[0] - # obspolicy = get_obs_policy(obsw) - # logits = obspolicy(expectation) - # print('update_policy:', logits) + delta = logits2 - logits + logits = logits + delta p_actions = tf.gather_nd(logits, actions) log_probs = tf.math.log(p_actions) loss = tf.math.reduce_sum(-log_probs * returns) / args.batch_size @@ -108,39 +103,42 @@ def reinforce_update(states, actions, returns, model): best_reward = 0 for batch in range(args.n_episodes // args.batch_size): # Gather episodes - tasklist, episodes = gather_episodes(args.state_bounds, args.n_actions, model, args.batch_size, args.env_name, qubits, genotype) + tasklist, episodes = gather_episodes(args.state_bounds, args.n_actions, model, args.batch_size, + args.env_name, args.beta, args.backend, args.backend_quafu, args.shots, args.n_qubits, qubits, genotype) logging.info(tasklist) + logging.info(episodes) # Group states, actions and returns in numpy arrays states = np.concatenate([ep['states'] for ep in episodes]) actions = np.concatenate([ep['actions'] for ep in episodes]) + logits = np.concatenate([ep['action_probs'] for ep in episodes]) rewards = [ep['rewards'] for ep in episodes] returns = np.concatenate([compute_returns(ep_rwds, args.gamma) for ep_rwds in rewards]) returns = np.array(returns, dtype=np.float32) id_action_pairs = np.array([[i, a] for i, a in enumerate(actions)]) - # Update model parameters. - reinforce_update(states, id_action_pairs, returns, model) - # Store collected rewards for ep_rwds in rewards: episode_reward_history.append(np.sum(ep_rwds)) if episode_reward_history[-1] >= best_reward: - model.save_weights(os.path.join(args.save, 'weights_id10_quafu.h5')) best_reward = episode_reward_history[-1] - # elif episode_reward_history[-1] >= 30: - # model.save_weights(os.path.join(args.save, 'weights_id10_quafu.h5')) + model.save_weights(os.path.join(args.save, 'weights_id10_quafu_{}.h5'.format(int(best_reward)))) + # Update model parameters. + reinforce_update(states, id_action_pairs, returns, logits, model) - # avg_rewards = np.mean(episode_reward_history[-10:]) + avg_rewards = np.mean(episode_reward_history[-5:]) logging.info('train finished episode: %f', (batch + 1) * args.batch_size) logging.info('train average rewards: %f', episode_reward_history[-1]) + logging.info('train moving average rewards: %f', avg_rewards) + + model.save_weights(os.path.join(args.save, 'weights_id10_quafu_latest.h5')) - if episode_reward_history[-1] >= 200.0: + if avg_rewards >= 100.0: break return episode_reward_history @@ -169,4 +167,9 @@ def reinforce_update(states, actions, returns, model): if __name__ == '__main__': - main() + qubits = cirq.GridQubit.rect(1, args.n_qubits) + genotype = eval("genotypes.%s" % args.arch) + ops = [cirq.Z(q) for q in qubits] + observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3 + + main(qubits, genotype, observables) diff --git a/examples/quantum_rl/visualization/plot_gif.py b/examples/quantum_rl/visualization/plot_gif.py new file mode 100644 index 0000000..d12bf47 --- /dev/null +++ b/examples/quantum_rl/visualization/plot_gif.py @@ -0,0 +1,70 @@ +# update gym to the version having render_mode, which is 0.26.1 in this file +import argparse +import sys + +sys.path.insert(0, ' ') +from functools import reduce + +import cirq +import gym +import models.quantum_genotypes as genotypes +import numpy as np +import tensorflow as tf +from misc.utils import get_obs_policy, get_quafu_exp +from models.quantum_models import generate_circuit +from models.quantum_models import generate_model_policy as Network +from models.quantum_models import get_model_circuit_params +from PIL import Image + +parser = argparse.ArgumentParser('Plot gif of pre-trained quantum models on quafu cloud platform') +parser.add_argument('--env_name', type=str, default='CartPole-v1', help='environment name') +parser.add_argument('--state_bounds', type=np.array, default=np.array([2.4, 2.5, 0.21, 2.5]), help='state bounds') +parser.add_argument('--n_qubits', type=int, default=4, help='the number of qubits') +parser.add_argument('--n_actions', type=int, default=2, help='the number of actions') +parser.add_argument('--arch', type=str, default='NSGANet_id10', help='which architecture to use') +parser.add_argument('--shots', type=int, default=1000, help='the number of sampling') +parser.add_argument('--backend_quafu', type=str, default='ScQ-P10', help='which quafu backend to use') +parser.add_argument('--beta', type=float, default=1.0, help='output parameter') +parser.add_argument('--model_path', type=str, default='./weights/train_p10/weights_id10_quafu_132.h5', help='path of pretrained model') +args = parser.parse_args(args=[]) + + +if __name__ == "__main__": + qubits = cirq.GridQubit.rect(1, args.n_qubits) + genotype = eval("genotypes.%s" % args.arch) + ops = [cirq.Z(q) for q in qubits] + observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3 + model = Network(qubits, genotype, args.n_actions, args.beta, observables, args.env_name) + model.load_weights(args.model_path) + + + for epoch in range(20): + env = gym.make(args.env_name, render_mode="rgb_array") + state, _ = env.reset() + frames = [] + for epi in range(100): + im = Image.fromarray(env.render()) + frames.append(im) + + # get PQC model parameters and expectations + stateb = state/args.state_bounds + newtheta, newlamda = get_model_circuit_params(qubits, genotype, model) + circuit, _, _ = generate_circuit(qubits, genotype, newtheta, newlamda, stateb) + _, expectation = get_quafu_exp(circuit, args.n_qubits, args.backend_quafu, args.shots) + + # get policy model parameters + obsw = model.get_layer('observables-policy').get_weights()[0] + obspolicy = get_obs_policy(obsw, args.beta) + policy = obspolicy(expectation) + print('policy:', policy) + + # choose actions and make a step + action = np.random.choice(args.n_actions, p=policy.numpy()[0]) + state, reward, terminated, truncated, _ = env.step(action) + if terminated or truncated: + print(epi+1) + break + env.close() + + # save gif to your path + frames[1].save('./visualization/gif/test_{}.gif'.format(epoch), save_all=True, append_images=frames[2:], optimize=False, duration=40, loop=0) \ No newline at end of file diff --git a/examples/quantum_rl/visualization/qrl.py b/examples/quantum_rl/visualization/qrl.py deleted file mode 100644 index 07e39db..0000000 --- a/examples/quantum_rl/visualization/qrl.py +++ /dev/null @@ -1,140 +0,0 @@ -import argparse -import re -import sys -from functools import reduce - -import cirq -import gym -# model imports -import models.quantum_genotypes as genotypes -import numpy as np -import tensorflow as tf -from models.quantum_models import generate_circuit -from models.quantum_models import generate_model_policy as Network -from models.quantum_models import get_model_circuit_params -from PIL import Image -from quafu import QuantumCircuit as quafuQC -from quafu import Task, User - -parser = argparse.ArgumentParser('Reinforcement learining with quantum computing cloud Quafu') -parser.add_argument('--env_name', type=str, default='CartPole-v1', help='environment name') -parser.add_argument('--state_bounds', type=np.array, default=np.array([2.4, 2.5, 0.21, 2.5]), help='state bounds') -parser.add_argument('--n_qubits', type=int, default=4, help='the number of qubits') -parser.add_argument('--n_actions', type=int, default=2, help='the number of actions') -parser.add_argument('--arch', type=str, default='NSGANet_id10', help='which architecture to use') -parser.add_argument('--shots', type=int, default=1000, help='the number of sampling') -parser.add_argument('--backend', type=str, default='ScQ-P20', help='which backend to use') -parser.add_argument('--model_path', type=str, default='./weights/weights_id10_quafu.h5', help='path of pretrained model') -args = parser.parse_args(args=[]) - - -def get_res_exp(res): - # access to probabilities of all possibilities - prob = res.probabilities - sumexp = 0 - for k, v in prob.items(): - count = 0 - for i in range(len(k)): - if k[i] == '1': - count += 1 - if count % 2 == 0: - sumexp += v - else: - sumexp -= v - return sumexp - - -def get_quafu_exp(circuit): - # convert Cirq circuts to qasm - openqasm = circuit.to_qasm(header='') - openqasm = re.sub('//.*\n', '', openqasm) - openqasm = "".join([s for s in openqasm.splitlines(True) if s.strip()]) - - # fill in with your token, register on website http://quafu.baqis.ac.cn/ - user = User() - user.save_apitoken(" ") - - # initialize to Quafu circuits - q = quafuQC(args.n_qubits) - q.from_openqasm(openqasm) - - # create the task - task = Task() - task.load_account() - - # choose sampling number and specific quantum devices - shots = args.shots - task.config(backend=args.backend, shots=shots, compile=True) - task_id = task.send(q, wait=True).taskid - print('task_id:', task_id) - - # retrieve the result of completed tasks and compute expectations - task_status = task.retrieve(task_id).task_status - if task_status == 'Completed': - task = Task() - task.load_account() - res = task.retrieve(task_id) - OB = get_res_exp(res) - return task_id, tf.convert_to_tensor([[OB]]) - - -class Alternating_(tf.keras.layers.Layer): - def __init__(self, obsw): - super(Alternating_, self).__init__() - self.w = tf.Variable( - initial_value=tf.constant(obsw), dtype="float32", trainable=True, name="obsw") - - def call(self, inputs): - return tf.matmul(inputs, self.w) - - -def get_obs_policy(obsw): - process = tf.keras.Sequential([ Alternating_(obsw), - tf.keras.layers.Lambda(lambda x: x * 1.0), - tf.keras.layers.Softmax() - ], name="obs_policy") - return process - - -if __name__ == "__main__": - qubits = cirq.GridQubit.rect(1, args.n_qubits) - genotype = eval("genotypes.%s" % args.arch) - ops = [cirq.Z(q) for q in qubits] - observables = [reduce((lambda x, y: x * y), ops)] # Z_0*Z_1*Z_2*Z_3 - model = Network(qubits, genotype, args.n_actions, observables) - model.load_weights(args.model_path) - - # update gym to the version having render_mode, which is 0.26.1 in this file - # initialize the environment - env = gym.make(args.env_name, render_mode="rgb_array") - state, _ = env.reset() - frames = [] - - # set the number of episodes - for epi in range(20): - im = Image.fromarray(env.render()) - frames.append(im) - - # get PQC model parameters and expectations - stateb = state/args.state_bounds - newtheta, newlamda = get_model_circuit_params(qubits, genotype, model) - circuit, _, _ = generate_circuit(qubits, genotype, newtheta, newlamda, stateb) - _, expectation = get_quafu_exp(circuit) - - # get policy model parameters - obsw = model.get_layer('observables-policy').get_weights()[0] - obspolicy = get_obs_policy(obsw) - policy = obspolicy(expectation) - print('policy:', policy) - - # choose actions and make a step - action = np.random.choice(args.n_actions, p=policy.numpy()[0]) - state, reward, terminated, truncated, _ = env.step(action) - if terminated or truncated: - print(epi+1) - break - env.close() - - # save to your path - frames[1].save(' ', save_all=True, append_images=frames[2:], optimize=False, duration=20, loop=0) - # for example, ./visualization/id10_quafu_gif/gym_CartPole_20.gif diff --git a/examples/quantum_rl/weights/weights_id10_quafu.h5 b/examples/quantum_rl/weights/train_p10/weights_id10_quafu_100.h5 similarity index 87% rename from examples/quantum_rl/weights/weights_id10_quafu.h5 rename to examples/quantum_rl/weights/train_p10/weights_id10_quafu_100.h5 index d01b42d748abc596c1a95b48503775506e4f0b33..99a77931a5486d129762244f4694f9a766bef83f 100644 GIT binary patch delta 411 zcmbP{G$Uz&LOu7#Nc*6p)(*kmk@k0%PPM!BY?a;h8GH`byW|}@el4({bcn-#|3g{( z<4<%Ps+t$uf0`g_cjdzmySL0b4xYhI4mEl*_LrkI945NFw^O=v)c*f^B|EmDZ}wG8 zh7JJ&|LtFFd}aT)*45#=@-+LvmuB`4A4uCT+iq)jNl3#%PDHcbVeM%%2fy3)4*B-a z?9cjavVZS%&p!6HsKa#j)Ap_NY#df9KeTOsIM06j!*BMF{z%(#*Eu*mIxp+sYcA{H z6JhD#_WO%n{s9GhRbC~BoU4cIS3f#!x9Fv_L)sKoheH$G9bVM-*?-t}&b}aOlHJX9 zOYGPU-q>F@@wR^%BjVt{vVODexwnpXQ>3gM<|h~0pZ_RnAE2aSSHC*dUMz*f!QFL& zJ-gWhyFNBghxT|NReg+Hpq zw?73!KtRr!EI_ZnML+R7`7C?Vj3qGWDDnRieF+Z99 zB|rwa2tW&+cRr~C6F@-YIY5!$qd!kj!ap6YB|y}^8$K&~E`LD!?At%5L}Ncmm@Yv1 z1r!hrdILX+t;;{VYrH=wy9+MP@w` z$e29(MlwK6w13(@OA{DC8kbT&AfuK)&>ILq=8Jwm(etxE^A0IM7x~-0o@`1$z3^>6 zq|H7*-yAqVZL3i~Y4W~4Pm=vV^Nln>DZAo78h`aaYr0RbQc0R?6-#bB_p(Vl7Yc4^It2h1lcFiRO% zfW;XZco;;0nsO2=Q;XvB5_40F!QzrYvp@!bFvwU21`Y;>$+67RlPy?yG(l!^Lj^$; z6A%j_@j*Ix7#x5)wldpuvVeIWK;B^>BlG0b?6Q-C5{aDr7Z07EY~DJ8Mk$^goTdIJ<@U{I2gS^|=W@p&gZYKg-%GcarpWOlP>_~ItR+T{1usq7N1tP*WH>CT{Gl#})M(psIncM*-X)BkhBZS~~=LN7~<6I@Rvh zvsHH2XYe^#?~-@u__e@((jgA}{SRgBk3Z3IsA^tp|7n7#-IWhN?A|i#ICus-In?OM z*k6v;aG2=w-cISx(R%y;>y_--g1*^TF&R1p2>iEyvGJAt-&$9P@5dYaCdl7+h_k_+d2D!s7ZD=*DbMQH+W-z*~HuaWsHb}|H{p_=iWNnO_8#4 zn4er|fBvJSeSngRUH$4*d$AM_2Y1&A_UvX4?E2U|9q#{Ramcn8w+~8_b;xQ_b0`pB zZg(&4y#1lp^>(}ex;cF4>9fCTz0TR5fuVsMynk~dM+Ms?4FyJp&59ZmnV2IW3C$U6 zLIXx3BO@fGz|sqh<^@GCG`WE&P;!&jE*EQnrW>dfNEHx36WC-u9WEvT?Ts7F7&jN_ XS}-xL*t}9-oKX~>`1K*m;iUrrk17hh delta 958 zcmbP{v?FPP2BXD9O-p$;21W)31_>Yr0RbQc0R?6-#o(~9(VmHsWAZ_!O639u7N8&_ z0}lfSP<>8fWol7;USe))F<7ZE&={e~xy;hsnRx}JB@peC9XZ4&%W;UY@-R3suuR^| zY|ja@jfcSl$YW)3XJ-Pd6PUPBRFM&AJXWW$L(FQ}T*v}+%ZbSw+0sQjKu!UI6>Jb1 zVxlg)H_+W6)0ly#O|E8Fm&b1A3@wPZhKU=+CviwjuHcYhWSTsYL!1jD&cLvPf3qOx zd%lSW_D_-!&`f|@2XY?>pj!_KGzNwRlPx9Gflf@6Fy}1Lg{Ymdd7(rkvvL45^mxIc zSDu=go>77bK2Yd!PS%x?mM+OiElDi4G5`i}Zc+-E1u+NaFSg0qGOB{G2!pvuV)8;6 zPsIeNQV1mkvCT`4~B*C`1PS%Byk!|u}xpcJz1&|U31_KBMCDFA` zUZCAjk1K|DK=n4D7{S1>dApNCg?@zXl^zy{^~PuI{ob#%IrD6fU76NZyAw-V>;-%G z+pTNS-}70o%l`M&+xEFes`jBz930*W3Okfc{B7Ux;JHmjM@zLNg4`_8k-e0CNh}=QzOvnj10Krh8HBy2nlRhkTXtp)DlOe zH*ma6w$KjJoS*;+2uSEd*uczzH8v(I=m;=MY&O)%XJ*{6d7-{IqbNM#=tH!kr)>a$ C*SpdH diff --git a/examples/quantum_rl/weights/weights_id97_quafu.h5 b/examples/quantum_rl/weights/train_p10/weights_id10_quafu_88.h5 similarity index 68% rename from examples/quantum_rl/weights/weights_id97_quafu.h5 rename to examples/quantum_rl/weights/train_p10/weights_id10_quafu_88.h5 index d8497eb7fa72d37a5e88a360c4b81b7c9ee7ed79..99a77931a5486d129762244f4694f9a766bef83f 100644 GIT binary patch delta 1307 zcmdmyG$Uz(2BXA8O-p%J21W)31_>Yr0RbQc0R?6-#bB_p(Vl7Yc4^It2h1lcFiRO% zfW;XZco;;0nsO2=Q;XvB5_40F!QzrYvp@!bFvwU21`Y;>$+67RlPy?yG(l!^Lj^$; z6A%j_@j*Ix7#x5)wldpuvVeIWK;B^>BlG0b?6Q-C5{aDr7Z07EY~DJ8Mk$^goTdIJ<@U{I2gS^|=W@p&gZYKg-%GcarpWOlP>_~ItR+T{1usq7N1tP*WH>CT{Gl#})M(psIncM*-X)BkhBZS~~=LN7~<6I@Rvh zvsHH2XYe^#?~-@u__e@((jgA}{SRgBk3Z3IsA^tp|7n7#-IWhN?A|i#ICus-In?OM z*k6v;aG2=w-cISx(R%y;>y_--g1*^TF&R1p2>iEyvGJAt-&$9P@5dYaCdl7+h_k_+d2D!s7ZD=*DbMQH+W-z*~HuaWsHb}|H{p_=iWNnO_8#4 zn4er|fBvJSeSngRUH$4*d$AM_2Y1&A_UvX4?E2U|9q#{Ramcn8w+~8_b;xQ_b0`pB zZg(&4y#1lp^>(}ex;cF4>9fCTz0TR5fuVsMynk~dM+Ms?4FyJp&59ZmnV2IW3C$U6 zLIXx3BO@fGz|sqh<^@GCG`WE&P;!&jE*EQnrW>dfNEHx36WC-u9WEvT?Ts7F7&jN_ XS}-xL*t}9-oKX~>`1K*m;iUrrk17hh delta 958 zcmbP{v?FPP2BXD9O-p$;21W)31_>Yr0RbQc0R?6-#o(~9(VmHsWAZ_!O639u7N8&_ z0}lfSP<>8fWol7;USe))F<7ZE&={e~xy;hsnRx}JB@peC9XZ4&%W;UY@-R3suuR^| zY|ja@jfcSl$YW)3XJ-Pd6PUPBRFM&AJXWW$L(FQ}T*v}+%ZbSw+0sQjKu!UI6>Jb1 zVxlg)H_+W6)0ly#O|E8Fm&b1A3@wPZhKU=+CviwjuHcYhWSTsYL!1jD&cLvPf3qOx zd%lSW_D_-!&`f|@2XY?>pj!_KGzNwRlPx9Gflf@6Fy}1Lg{Ymdd7(rkvvL45^mxIc zSDu=go>77bK2Yd!PS%x?mM+OiElDi4G5`i}Zc+-E1u+NaFSg0qGOB{G2!pvuV)8;6 zPsIeNQV1mkv zyg<959#;(Qfa+~PF@k~NPUmL(ljYU+J)V2)8}D4On{pu8mSO!L`?k_|_A70z95T;x zIGpgEYya({vV*;DuU(PMAN%5&eRf-YWbG%+Rdom}o@M(i>5=_D$5eZT37b9b9Rd^` z&P=~xZ?VtSLFt^egGG+7J*(zSd!y`Ddp|Ep`;FF`_Os3;+N^;SwTmDQDU>9PChf^hRqB0#TiB62}d8I6+LYO0AFpi Ad;kCd diff --git a/examples/quantum_rl/weights/weights_id10_direct.h5 b/examples/quantum_rl/weights/train_p10/weights_id10_quafu_94.h5 similarity index 68% rename from examples/quantum_rl/weights/weights_id10_direct.h5 rename to examples/quantum_rl/weights/train_p10/weights_id10_quafu_94.h5 index fdf6ff031b082200134bddd0075836b5a23b7a21..99a77931a5486d129762244f4694f9a766bef83f 100644 GIT binary patch delta 1225 zcmdmyG$Uz(2BXA8O-p%J21W)31_>Yr0RbQc0R?6-#bB_p(Vl7Yc4^It2h1lcFiRO% zfW;XZco;;0nsO2=Q;XvB5_40F!QzrYvp@!bFvwU21`Y;>$+67RlPy?yG(l!^Lj^$; z6A%j_@j*Ix7#x5)wldpuvVeIWK;B^>BlG0b?6Q-C5{aDr7Z07EY~DJ8Mk$^goTdIJ<@U{I2gS^|=W@p&gZYKg-%Gcarpg-X-tQ@oRzoq(dC``ya~MAAh3b zP}RKH{?i0eyDJ}l*u7=eaqtXwa;VXhvA-Ox;V{wVy`9pXqxJUx*DKkv1%0!xVls3H z5cqHZV&g0OzqPIo-<7A?2fj43fA~P!e%W?gyGueE4ss%z4r@=FIr!bScgVMYW`EXa zll^yXu==1?HM-0oi7dHX}H>+N>`b#wU8(`SFxdY!X9149Ej zc>m@^jtaI(8VZaGn-w)CGBHO$Qj0Uz)B=n|Mn*_tge6HB%?pZPXqo|0pfn?`4NEhV o^>nzH1hhA9G-KRcpliXzxMK54eQ`!ncv95|X$KV&9+12Y0GHJVC;$Ke delta 1120 zcmbP{v?FPP2BXD9O-p$;21W)31_>Yr0RbQc0R?6-#o(~9(VmHsWAZ_!O639u7N8&_ z0}lfSP<>8fWol7;USe))F<7ZE&={e~xy;hsnRx}JB@peC9XZ4&%W;UY@-R3suuR^| zY|ja@jfcSl$YW)3XJ-Pd6PUPBRFM&AJXWW$L(FQ}T*v}+%ZbSw+0sQjKu!UI6>Jb1 zVxlg)H_+W6)0ly#O|E8Fm&b1A3@wPZhKU=+CviwjuHcYhWSTsYL!1jD&cLvPf3qOx zd%lSW_D_-!&`f|@2XY?>pj!_KGzNwRlPx9Gflf@6Fy}1Lg{Ymdd7(rkvvL45^mxIc zSDu=go>77bK2Yd!PS%x?mM+OiElDi4G5`i}Zc+-E1u+NaFSg0qGQGk=Kx04vu)FQ`7dfaD0eKiSIGHfH|x?4dvllf_R9oc*#|q9+RqGOagYyKZ2$WGM7x(4 zAKEY5BJ3cf!{P8hTGgS1cZL1Iex-VcAC`0ND?W2LbiYfnt6yqle|loQeO$moyD7Ue z?PnBzvip=N=P>&-(4vaRc2g>4?Za|C?Mq^n96Wk>>`zDAIq0dmIP@)@Yrk6ZmEFUw ziVo%-tL+_Y{@QJ@=XaP}#_z!T!pT9Wq{F`CdQG5={FHYDgb%i^4Z8Ww&aHD>a zU5aa|UD_;8hn9~Y>@{=D9L%Pw+Mk>wv)5Lk%}(yd0sE7yjqHohEVoyVXtDeGRn{S9 z^GkcpJu3ERRJYjmR3Ad($3|ET1O7aBzZB_CPGT*fvst&0;s1!`JPjFIq zSS2EDFU}$Bpkh7Eo@17-{dJ!z`-a#GdqL})v+VlU3-1q7KWBgTv!uiIT@em$>woRz zHoLq}t@!9Z8>d)@mIqVq19v6a%Q@LQM0{LfzvX1O!{(FW2Y^Ykf^Cw90;9%eMU9C} z=D=J8j3!0~+-ZszB+m#*wy?y@IN4E49Fb2z=}C36g?0vxG&Na4M}Sdcv!PBtGvkKM S3-!erMd3+XACgMYa~}Y1bQp*L diff --git a/examples/quantum_rl/weights/train_p136/weights_id10_quafu_91.h5 b/examples/quantum_rl/weights/train_p136/weights_id10_quafu_91.h5 new file mode 100644 index 0000000000000000000000000000000000000000..99a77931a5486d129762244f4694f9a766bef83f GIT binary patch literal 12568 zcmeHNYiv|S6rQ_%t#pf9Qfx~pEJaM!wieSA1bb(oPeP?j%a7u#Y};G5cDq}-PJxxZdCs@;F zv<4dkezRpFePhSX%bGcQK0d3xJL5z*#48Dr_D&+(JIEkh< z6Cor1V0`nWIkfk7ZXD*-1^tcMNrJHV9Y*7JZ)nuu6I~R{myx`9k~a*7t%1EuZKL-{ zcG8z7Wx^mDNv`WYDImSjsNTtzN((E>1%}*FnHSEc6ljJ=hA7gqj-fpu= zg?8A1h|-8linviFJn>&Mr=VtvMEYIZ$Ue#GCAU`uP!jLr+snD1Sedzn;awY>1Gf;`fG!3h|)*D~7p>sx8%nYFBH<5qg{omchmFTbyU+clr9{cI@< z|9Ob+@3zxZ{ps|JAG6r5=ij2gbS~7t8Twtn8kNO1RIO*v<}9Y?9?N8}6kXFZE}o&o z$20WU${Tc-Z6$L%hv}6Mf1)>c7qj1%zeYC?=F|SZH2Utz0{!a+nQTeYz3hXYe75NV zWdVAbep&tzy|(@med5AG_Ig|o-FNU|c4T>f!HfL|>B;^Zbl~qaJ^ney271%kBdgL` zdG$lAU$EM-@AchI3v&eE1g_v_ysds~mo z{h6N2+eil=Phu5EJ}x+Wb)9}-(Hizf-P5%9dMb5i+^z39YS6pt?X0x8lg8!u=^e2f z*bje1vnHBMD;v_;Q!m`ZT9V(>FFn~yyZ0T}KfhVRhF9gI>w>kzQUt@7)d5C-`h9I*9V1qNFT1tv)`Znfr% z9Hd%knrhsN<3cLVg}~zGH&A=A5wA&!W8@w&cIj-?*n4Av~XFJ;xX6{9h>0QsXn)3|~!ipx(U0nL14G Uk!{mFK2(SQ3)cOC={}$N58z*;-PJxxZdCs@;F zv<4dkezRpFePhSX%bGcQK0d3xJL5z*#48Dr_D&+(JIEkh< z6Cor1V0`nWIkfk7ZXD*-1^tcMNrJHV9Y*7JZ)nuu6I~R{myx`9k~a*7t%1EuZKL-{ zcG8z7Wx^mDNv`WYDImSjsNTtzN((E>1%}*FnHSEc6ljJ=hA7gqj-fpu= zg?8A1h|-8linviFJn>&Mr=VtvMEYIZ$Ue#GCAU`uP!jLr+snD1Sedzn;awY>1Gf;`fG!3h|)*D~7p>sx8%nYFBH<5qg{omchmFTbyU+clr9{cI@< z|9Ob+@3zxZ{ps|JAG6r5=ij2gbS~7t8Twtn8kNO1RIO*v<}9Y?9?N8}6kXFZE}o&o z$20WU${Tc-Z6$L%hv}6Mf1)>c7qj1%zeYC?=F|SZH2Utz0{!a+nQTeYz3hXYe75NV zWdVAbep&tzy|(@med5AG_Ig|o-FNU|c4T>f!HfL|>B;^Zbl~qaJ^ney271%kBdgL` zdG$lAU$EM-@AchI3v&eE1g_v_ysds~mo z{h6N2+eil=Phu5EJ}x+Wb)9}-(Hizf-P5%9dMb5i+^z39YS6pt?X0x8lg8!u=^e2f z*bje1vnHBMD;v_;Q!m`ZT9V(>FFn~yyZ0T}KfhVRhF9gI>w>kzQUt@7)d5C-`h9I*9V1qNFT1tv)`Znfr% z9Hd%knrhsN<3cLVg}~zGH&A=A5wA&!W8@w&cIj-?*n4Av~XFJ;xX6{9h>0QsXn)3|~!ipx(U0nL14G Uk!{mFK2(SQ3)cOC={}$N58z*;-PJxxZdCs@;F zv<4dkezRpFePhSX%bGcQK0d3xJL5z*#48Dr_D&+(JIEkh< z6Cor1V0`nWIkfk7ZXD*-1^tcMNrJHV9Y*7JZ)nuu6I~R{myx`9k~a*7t%1EuZKL-{ zcG8z7Wx^mDNv`WYDImSjsNTtzN((E>1%}*FnHSEc6ljJ=hA7gqj-fpu= zg?8A1h|-8linviFJn>&Mr=VtvMEYIZ$Ue#GCAU`uP!jLr+snD1Sedzn;awY>1Gf;`fG!3h|)*D~7p>sx8%nYFBH<5qg{omchmFTbyU+clr9{cI@< z|9Ob+@3zxZ{ps|JAG6r5=ij2gbS~7t8Twtn8kNO1RIO*v<}9Y?9?N8}6kXFZE}o&o z$20WU${Tc-Z6$L%hv}6Mf1)>c7qj1%zeYC?=F|SZH2Utz0{!a+nQTeYz3hXYe75NV zWdVAbep&tzy|(@med5AG_Ig|o-FNU|c4T>f!HfL|>B;^Zbl~qaJ^ney271%kBdgL` zdG$lAU$EM-@AchI3v&eE1g_v_ysds~mo z{h6N2+eil=Phu5EJ}x+Wb)9}-(Hizf-P5%9dMb5i+^z39YS6pt?X0x8lg8!u=^e2f z*bje1vnHBMD;v_;Q!m`ZT9V(>FFn~yyZ0T}KfhVRhF9gI>w>kzQUt@7)d5C-`h9I*9V1qNFT1tv)`Znfr% z9Hd%knrhsN<3cLVg}~zGH&A=A5wA&!W8@w&cIj-?*n4Av~XFJ;xX6{9h>0QsXn)3|~!ipx(U0nL14G Uk!{mFK2(SQ3)cOC={}$N58z*;-PJxxZdCs@;F zv<4dkezRpFePhSX%bGcQK0d3xJL5z*#48Dr_D&+(JIEkh< z6Cor1V0`nWIkfk7ZXD*-1^tcMNrJHV9Y*7JZ)nuu6I~R{myx`9k~a*7t%1EuZKL-{ zcG8z7Wx^mDNv`WYDImSjsNTtzN((E>1%}*FnHSEc6ljJ=hA7gqj-fpu= zg?8A1h|-8linviFJn>&Mr=VtvMEYIZ$Ue#GCAU`uP!jLr+snD1Sedzn;awY>1Gf;`fG!3h|)*D~7p>sx8%nYFBH<5qg{omchmFTbyU+clr9{cI@< z|9Ob+@3zxZ{ps|JAG6r5=ij2gbS~7t8Twtn8kNO1RIO*v<}9Y?9?N8}6kXFZE}o&o z$20WU${Tc-Z6$L%hv}6Mf1)>c7qj1%zeYC?=F|SZH2Utz0{!a+nQTeYz3hXYe75NV zWdVAbep&tzy|(@med5AG_Ig|o-FNU|c4T>f!HfL|>B;^Zbl~qaJ^ney271%kBdgL` zdG$lAU$EM-@AchI3v&eE1g_v_ysds~mo z{h6N2+eil=Phu5EJ}x+Wb)9}-(Hizf-P5%9dMb5i+^z39YS6pt?X0x8lg8!u=^e2f z*bje1vnHBMD;v_;Q!m`ZT9V(>FFn~yyZ0T}KfhVRhF9gI>w>kzQUt@7)d5C-`h9I*9V1qNFT1tv)`Znfr% z9Hd%knrhsN<3cLVg}~zGH&A=A5wA&!W8@w&cIj-?*n4Av~XFJ;xX6{9h>0QsXn)3|~!ipx(U0nL14G Uk!{mFK2(SQ3)cOC={}$N58z*;h5&j0Q)Ti^i19XlB?HW^kB^ABpiFqcBa(rbc}4-Fv(8Iz~pQl5ua+ zp8I*uxxaVLeea%oUnwtrBtd((7D-8?L2FQ=oMnG=G!^e6HLT3qt7t|=X&0q~YLZc* zI3%^D(fUNnUmrs%Fdew}DXSHUJsE#S>97!k;!p+8tR@7?t-Ec~e23_y<${>kJe}SC zTDw~aRN9`j(0Y%J}tQ!&I&YZ4@Ot?A7&xy9Vh*F^S?h=|?1L zWG*C+knG^3N^Xxw<&@6`nNPjov->L>1)smp;}%CQ#xGXlPWcg6mn26fl_*8hA;SOH zlD=(p2=RZku>aG3oUBd+nE&sm)2F6~&d1HigW0=ne)W<|A^(@0T}*>i-VOgr5d9|& zH3@aK&cKZYAW09@JuoUj_;k2>%Zz|Beo5T` zby0s|;sQ<~VD~>53aPp54UU?y3g#c74s758l*KQV{uz&KH2$9;`hH~mQaZm_C-P+R zN*4EHG^D3eL3&=IREM1d(!&Xn5Cl|eG9xyxy>y_m93$!|)g|dOWxWy7qf`w_Jw*G* zinEZ~)oQ{}nv;}bRANUt*`(%BN~SBGJmy42HA2o%+kcTE>#mw=iO}dhx0cG*4QIx! z`>nRtNV_7>o!MI+wc8ZhoqTi<4*zj4Oa$IH zpEkC@-_QRBONq0eAMo2085Vn_x!L64!otq`>PyKVJyMct{@z#DKfu2 zn}=hy25f9U4wgba4t8j;w{{Ee>nMYZujS*X57^+;XBqe<|7LSaU;-X~W*mO{r4+_6 z;BAx1c=n13ijMU`(ZN#l@XS^CHcf-Kwi+RO-vFFBl?PcfKSTHLmrVWHI+%X86V9Z} z!Q)r6;lj52@bS*CV8_n;O@Cg=#d{_J_r{f&zx*&3pSv~&FMhKI+qSRAW1s(Mw#-=I zRqb@}=+|$Ls#C{dZEq&{U?k|eW*s$jTo2BkUT2>CZUp}`+<`%N8Jl#MM-`8S7dZkP z0geDifFlr91Qy%3=IkWm+4pBumEp`+90JOHEBU%c2-EltQ&_yKW*~Q9$mW%8wlM2( zo0RQamGPqRdp8U1TN6ZC?E9bT!KhMCa$_!1xyXIR&G)NpKg{^E^3wMiZCON{`5qU! z&u~U69mN{5UsZ+TglRjXDC`bq&F?wh!4G#CrlwqP*(xxkh=a5B#9#~5FR@vxTymP6cR!K9|LVDl$3-Xx~~nPQN@n~98xM|JINzXm)l+>9 z@efsCg+&7oA8c<2*^~J*rC~V+t+@)5SxN|Ww0{>?^PQwgy9JpKY@>=Yky^4>-L8X> zmcF7aHV6S}-HF(+orxr4gLc-{WVN*GRjS>G22SPDAt{@vrXw#qQ7Stq%@1<2H;aExjWV~U?+QFofK0e$EhDm47gk{58ygDR=Mp# z=pa5TTs{MKCYFs1*_mvADk(?K$8VY9PW;f-m1J-7NhwMPkN+>nzN0h*{U3_^KlL+Y zX(GV>-$m0W(}S7HLOj^s!-drgF6I1RIlD{~B=3#?R7?M~>%aqxP>x^f{uzWY#s90N?|b8y>iptk$&=%i9QR|I(^FTF&P#N4_#BW9CnP}x ze14n}&x_9q^m2vNQPtIkHo1BOgrF_}bt5EbnVf}c_xq8dnv+TuKH1StQ6%(SMJt{f zbMegS9)Ng3d$|j^?)phffSViNEs@ua_-F6#x71p_c7^Yq?KVh&FkQGEMMKD{(DS@@ zbm~8S1;x$*u9MoMns(=YJ}6d?bXk2}55zuaLOk8_tp)#F7BerNi`1*lR@civ#c%hj ztjSSF%wGCkG_PG_ZOF7(1KZRqd-(vfStUKX~7&$BXjs-4xrmR!edG|DW|>s_Th2=DhlO_Z4s7ukwDF`Ez;U z`;74rYE$TO-hD>gt8^J_aKGxy#R=;_FQc#zn62BZ+R7%)Z*a0-h@ja zGoJDCnAHXUiqYNfE$Zb?NRk8L*nm)eom18eJJO>%zpuK C;80rt