diff --git a/CONFIGURE.md b/CONFIGURE.md new file mode 100644 index 00000000..413443b0 --- /dev/null +++ b/CONFIGURE.md @@ -0,0 +1,62 @@ +# 🌋 VM setup +Image - Ubuntu Server 24.04 LTS - x64 Gen2 +Size - Standard_NC24ads_A100_v4 - 24 vcpus, 220 GiB memory +Disk - 1TiB (P30) SSD + +# Configure command +## Install nvidia driver (https://learn.microsoft.com/en-us/azure/virtual-machines/linux/n-series-driver-setup) +```Shell +sudo apt update && sudo apt install -y ubuntu-drivers-common +sudo ubuntu-drivers install +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb +sudo apt install -y ./cuda-keyring_1.1-1_all.deb +sudo apt update +sudo apt -y install cuda-toolkit-12-5 +sudo reboot +``` + +## Add env var (add below exports at the end of ~/.bashrc) +```Shell +vim ~/.bashrc +# export PATH="/usr/local/cuda-12.5/bin:$PATH" +# export LD_LIBRARY_PATH="/usr/local/cuda-12.5/lib64:$LD_LIBRARY_PATH" +``` + +## Install miniconda (https://docs.anaconda.com/miniconda/) +```Shell +mkdir -p ~/miniconda3 +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh +bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3 +rm ~/miniconda3/miniconda.sh +~/miniconda3/bin/conda init bash +~/miniconda3/bin/conda init zsh +bash +``` + +## Clone repo +```Shell +git clone https://github.com/gpminsuk/LLaVA-NeXT llava-next +cd llava +``` + +## Configure conda and install requirements (https://medium.com/@prayto001/how-to-train-your-own-vision-large-language-model-37e3ff82b0b7) +```Shell +conda create -n llava-next python=3.10 -y +conda activate llava-next +pip install --upgrade pip # enable PEP 660 support +pip install -e . +pip install -e ".[train]" +pip install flash-attn --no-build-isolation +``` + +## Download training data for MLP +```Shell +sudo apt install unzip +mkdir blip_laion +cd blip_laion +wget -O images.zip "https://huggingface.co/datasets/liuhaotian/LLaVA-Pretrain/resolve/main/images.zip?download=true" +unzip images.zip +rm images.zip +cd .. +wget -O blip_laion_cc_sbu_558k.json "https://huggingface.co/datasets/liuhaotian/LLaVA-Pretrain/resolve/main/blip_laion_cc_sbu_558k.json?download=true" +``` \ No newline at end of file diff --git a/README.md b/README.md index 8111151a..06204986 100755 --- a/README.md +++ b/README.md @@ -91,6 +91,8 @@ cd LLaVA-NeXT ``` #### 2. **Install the inference package:** +Note: Linux environment is recommended. Some packages may not be available in mac os. + ```bash conda create -n llava python=3.10 -y conda activate llava @@ -146,6 +148,14 @@ Checkout the HTTP Post/Get and SRT usage at [sglang/examples/runtime/llava_onevi bash examples/usage/llava_video/srt_example_llava_v.sh K K-1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO ``` +## FastAPI Server + +If you wish to run just the server, run the following commands. (TODO: some package versions may conflict with dependencies for training. Resolve) + +```sh +pip install -e ".[api]" +fastapi dev vessl.py +``` ## Citation diff --git a/inference.py b/inference.py new file mode 100644 index 00000000..c387a520 --- /dev/null +++ b/inference.py @@ -0,0 +1,298 @@ +from huggingface_hub import hf_hub_download +from PIL import Image +from torchvision import transforms +from llava.model.builder import load_pretrained_model +from llava.constants import ( + IGNORE_INDEX, + IMAGE_TOKEN_INDEX, + DEFAULT_IMAGE_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IM_END_TOKEN, +) +from llava.mm_utils import ( + process_images, + tokenizer_image_token, + get_model_name_from_path, +) +from llava.conversation import conv_templates, SeparatorStyle +import re +import torch +import torch.nn as nn +import tflite_runtime.interpreter as tflite + +from transformers import AutoTokenizer, CLIPImageProcessor, Qwen2ForCausalLM +from llava.model.language_model.llava_qwen import LlavaQwenConfig, LlavaQwenForCausalLM +from llava.model.multimodal_encoder.mobilenetv2_encoder import MobileNetV2VisionTower +import os +import numpy as np +from torchvision.utils import save_image +from decord import VideoReader, cpu + +# torch.set_printoptions(threshold=1000000) + +# Lora model path or Full model path +model_path = "mikarbx/llava-next-mobilenetv2" +# Base model path if model_path is lora-based, otherwise None +# base_model_path = "lmsys/vicuna-13b-v1.5" +base_model_path = None + +# Query +question = "Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes." +# Change this to True or False +use_image = False + +# Generation parameters +temperature = 0 +top_p=None +num_beams=1 +max_new_tokens=512 + +def load_model(model_path, config): + def build_vision_projector(projector_type, mm_hidden_size, hidden_size): + if projector_type == 'linear': + return nn.Linear(mm_hidden_size, hidden_size) + + mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type) + if mlp_gelu_match: + mlp_depth = int(mlp_gelu_match.group(1)) + modules = [nn.Linear(mm_hidden_size, hidden_size)] + for _ in range(1, mlp_depth): + modules.append(nn.GELU()) + modules.append(nn.Linear(hidden_size, hidden_size)) + return nn.Sequential(*modules) + + if projector_type == 'identity': + return IdentityMap() + + raise ValueError(f'Unknown projector type: {projector_type}') + + print('Build projector...') + mm_projector = build_vision_projector( + getattr(config, "mm_projector_type", None), + getattr(config, "mm_hidden_size", None), + getattr(config, "hidden_size", None), + ).to(dtype=torch.float16).cuda() + + print('Load mlp adapter...') + full_model = LlavaQwenForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, attn_implementation="flash_attention_2", config=config).to(dtype=torch.float16) + mlp_projector_weights = {k[19:]: v for k, v in full_model.state_dict().items() if k.startswith('model.mm_projector.')} + mm_projector.load_state_dict(mlp_projector_weights, strict=False) + + # print('Load lora trainables...') + # non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location=image_embeddings.device) + # non_lora_trainables = {(k[17:] if k.startswith('base_model.model.') else k): v for k, v in non_lora_trainables.items()} + + # print('Split and load projector trainables...') + # mm_projector_trainables = {(k[19:] if k.startswith('model.mm_projector.') else k): v for k, v in non_lora_trainables.items()} + # mm_projector.load_state_dict(mm_projector_trainables, strict=False) + + print('Loading Base LLM...') + model = Qwen2ForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, attn_implementation="flash_attention_2", config=config).to(dtype=torch.float16).cuda() + tokenizer = AutoTokenizer.from_pretrained(model_path) + + print('Model is loaded...') + + return tokenizer, model, mm_projector + +def run_inference(image_embeddings, prompt, config, tokenizer, model, mm_projector): + split_sizes = [image_embeddings.shape[0]] + + # Project image embeddings to LLM compatiable embeddings + image_features = mm_projector(image_embeddings) + image_features = torch.split(image_features, split_sizes) + new_image_features = [] + for _, image_feature in enumerate(image_features): + new_image_features.append(image_feature.flatten(0, 1)) + image_features = new_image_features + + # Tokenize prompt + input_ids = ( + tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") + .unsqueeze(0) + .cuda() + ) + + # Convert image tag to image embedding, embed non-image tokens and combine into one embeddings + ### Copied from prepare_inputs_labels_for_multimodal in llava/model/llava_arch.py ### + attention_mask = torch.ones_like(input_ids, dtype=torch.bool) + position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device) + labels = torch.full_like(input_ids, IGNORE_INDEX) + new_input_embeds = [] + new_labels = [] + cur_image_idx = 0 + for batch_idx, cur_input_ids in enumerate(input_ids): + num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum() + if num_images == 0: + cur_image_features = image_features[cur_image_idx] + cur_input_embeds_1 = model.model.embed_tokens(cur_input_ids) + cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0]], dim=0) + new_input_embeds.append(cur_input_embeds) + new_labels.append(labels[batch_idx]) + cur_image_idx += 1 + continue + + image_token_indices = [-1] + torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]] + cur_input_ids_noim = [] + cur_labels = labels[batch_idx] + cur_labels_noim = [] + for i in range(len(image_token_indices) - 1): + cur_input_ids_noim.append(cur_input_ids[image_token_indices[i] + 1 : image_token_indices[i + 1]]) + cur_labels_noim.append(cur_labels[image_token_indices[i] + 1 : image_token_indices[i + 1]]) + split_sizes = [x.shape[0] for x in cur_labels_noim] + cur_input_embeds = model.model.embed_tokens(torch.cat(cur_input_ids_noim)) + cur_input_embeds_no_im = torch.split(cur_input_embeds, split_sizes, dim=0) + cur_new_input_embeds = [] + cur_new_labels = [] + + for i in range(num_images + 1): + cur_new_input_embeds.append(cur_input_embeds_no_im[i]) + cur_new_labels.append(cur_labels_noim[i]) + if i < num_images: + try: + cur_image_features = image_features[cur_image_idx] + except IndexError: + cur_image_features = image_features[cur_image_idx - 1] + cur_image_idx += 1 + cur_new_input_embeds.append(cur_image_features) + cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=cur_labels.device, dtype=cur_labels.dtype)) + + cur_new_input_embeds = [x.to(image_embeddings.device) for x in cur_new_input_embeds] + + cur_new_input_embeds = torch.cat(cur_new_input_embeds) + cur_new_labels = torch.cat(cur_new_labels) + + new_input_embeds.append(cur_new_input_embeds) + new_labels.append(cur_new_labels) + + # Truncate sequences to max length as image embeddings can make the sequence longer + tokenizer_model_max_length = getattr(config, 'tokenizer_model_max_length', None) + if tokenizer_model_max_length is not None: + new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds] + new_labels = [x[:tokenizer_model_max_length] for x in new_labels] + + # Combine them + max_len = max(x.shape[0] for x in new_input_embeds) + batch_size = len(new_input_embeds) + + new_input_embeds_padded = [] + new_labels_padded = torch.full((batch_size, max_len), IGNORE_INDEX, dtype=new_labels[0].dtype, device=new_labels[0].device) + attention_mask = torch.zeros((batch_size, max_len), dtype=attention_mask.dtype, device=attention_mask.device) + position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device) + + for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)): + cur_len = cur_new_embed.shape[0] + if getattr(config, "tokenizer_padding_side", "right") == "left": + new_input_embeds_padded.append(torch.cat((torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device), cur_new_embed), dim=0)) + if cur_len > 0: + new_labels_padded[i, -cur_len:] = cur_new_labels + attention_mask[i, -cur_len:] = True + position_ids[i, -cur_len:] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device) + else: + new_input_embeds_padded.append(torch.cat((cur_new_embed, torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)) + if cur_len > 0: + new_labels_padded[i, :cur_len] = cur_new_labels + attention_mask[i, :cur_len] = True + position_ids[i, :cur_len] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device) + + new_input_embeds = torch.stack(new_input_embeds_padded, dim=0).detach() + attention_mask = attention_mask.to(dtype=torch.long) + ### + + # Generate answer + output_data = model.generate( + position_ids=None, + attention_mask=attention_mask, + inputs_embeds=new_input_embeds, + do_sample=True if temperature > 0 else False, + temperature=temperature, + top_p=top_p, + num_beams=num_beams, + use_cache=True, + ) + + print(output_data.shape) + + # Decode answer + outputs = tokenizer.batch_decode(output_data, skip_special_tokens=True)[0].strip() + + return prompt, outputs + +if __name__ == "__main__": + cfg_pretrained = LlavaQwenConfig.from_pretrained(model_path) + + if not use_image: + import json + with open("./test.json") as f: + data = json.load(f) + hex_strings = data['embeddings'] + embeddings = [] + for hex_string in hex_strings: + byte_array = bytes.fromhex(hex_string) + import struct + num_floats = len(byte_array) // 4 + embeddings.append(torch.tensor(struct.unpack('<' + 'f' * num_floats, byte_array), dtype=torch.float16).unsqueeze(0)) + #image_embeddings = torch.stack([torch.tensor(struct.unpack('<' + 'f' * num_floats, byte_array), dtype=torch.float16).unsqueeze(0)]).to(dtype=torch.float16).cuda() + image_embeddings = torch.stack(embeddings).cuda() + else: + # image_path = "./blip_laion/00000/000000030.jpg" + # + # def encode_image(vision_tower, image_path): + # encoder = MobileNetV2VisionTower(vision_tower) + # + # image = Image.open(image_path).convert("RGB") + # image = encoder.image_processor([image], return_tensors='pt')['pixel_values'] + # + # save_image(image[0], "./image.jpg") + # + # return encoder.process_single_image(image.to(dtype=torch.float16)) + # + # print('Encode image...') + # image_embeddings = encode_image( + # getattr(cfg_pretrained, "mm_vision_tower", None), + # image_path, + # ) + + video_path = "./playground/demo/xU25MMA2N4aVtYay.mp4" + + def encode_video(video_path, for_get_frames_num): + vr = VideoReader(video_path, ctx=cpu(0), num_threads=1) + total_frame_num = len(vr) + video_time = total_frame_num / vr.get_avg_fps() + fps = round(vr.get_avg_fps()) + frame_idx = [i for i in range(0, len(vr), fps)] + frame_time = [i/fps for i in frame_idx] + if len(frame_idx) > for_get_frames_num: + sample_fps = for_get_frames_num + uniform_sampled_frames = np.linspace(0, total_frame_num - 1, sample_fps, dtype=int) + frame_idx = uniform_sampled_frames.tolist() + frame_time = [i/vr.get_avg_fps() for i in frame_idx] + frame_time = ",".join([f"{i:.2f}s" for i in frame_time]) + spare_frames = vr.get_batch(frame_idx).asnumpy() + # import pdb;pdb.set_trace() + + return spare_frames,frame_time,video_time + + encoder = MobileNetV2VisionTower(getattr(cfg_pretrained, "mm_vision_tower", None)) + + video, frame_time, video_time = encode_video(video_path, 32) + video = encoder.image_processor.preprocess(video, return_tensors="pt")["pixel_values"].half() + image_embeddings = encoder.process_single_image(video.to(dtype=torch.float16)).cuda() + + conv_mode = "qwen_2" + + print(f'Conversation template {conv_mode} is used...') + conv = conv_templates[conv_mode].copy() + + # Combine question with image + conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + question) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + tokenizer, model, mm_projector = load_model(model_path, cfg_pretrained) + + prompt, outputs = run_inference(image_embeddings, prompt, cfg_pretrained, tokenizer, model, mm_projector) + + print("PROMPT:") + print(prompt) + print("ANSWER:") + print(outputs) \ No newline at end of file diff --git a/llava/model/llava_arch.py b/llava/model/llava_arch.py index 4549593b..f1fc5fba 100755 --- a/llava/model/llava_arch.py +++ b/llava/model/llava_arch.py @@ -283,7 +283,7 @@ def prepare_inputs_labels_for_multimodal(self, input_ids, position_ids, attentio encoded_image_features = torch.split(encoded_image_features, split_sizes) image_features = [] for idx, image_feat in enumerate(encoded_image_features): - if idx in video_idx_in_batch: + if idx in video_idx_in_batch and self.config.mm_spatial_pool_stride is not None and self.config.mm_spatial_pool_stride > 1: image_features.append(self.get_2dPool(image_feat)) else: image_features.append(image_feat) diff --git a/llava/model/multimodal_encoder/builder.py b/llava/model/multimodal_encoder/builder.py index 69d9f10b..ae2ec483 100755 --- a/llava/model/multimodal_encoder/builder.py +++ b/llava/model/multimodal_encoder/builder.py @@ -5,6 +5,7 @@ from .hf_vision import HFVisionTower from .siglip_encoder import SigLipVisionTower from .clip_encoder import CLIPVisionTower, CLIPVisionTowerS2 +from .mobilenetv2_encoder import MobileNetV2VisionTower # from .eva_clip.eva_clip_encoder import EvaClipVisionTower # from .dev_eva_clip.eva_vit import EvaViTWrapper @@ -31,5 +32,7 @@ def build_vision_tower(vision_tower_cfg, **kwargs): # return EvaClipVisionTower(vision_tower, args=vision_tower_cfg, **kwargs) # elif vision_tower in ["EVA-CLIP-8B", "EVA-CLIP-8B-plus"]: # return EvaViTWrapper(vision_tower, args=vision_tower_cfg, **kwargs) + elif vision_tower.startswith("mikarbx"): + return MobileNetV2VisionTower(vision_tower) raise ValueError(f"Unknown vision tower: {vision_tower}") diff --git a/llava/model/multimodal_encoder/mobilenetv2_encoder.py b/llava/model/multimodal_encoder/mobilenetv2_encoder.py new file mode 100644 index 00000000..253e959c --- /dev/null +++ b/llava/model/multimodal_encoder/mobilenetv2_encoder.py @@ -0,0 +1,141 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import tflite_runtime.interpreter as tflite +from huggingface_hub import hf_hub_download +from PIL import Image +from torchvision import transforms + +from transformers import CLIPImageProcessor + + +class MobileNetV2VisionTower(nn.Module): + def __init__(self, vision_tower): + super().__init__() + + self.is_loaded = False + + self.vision_tower_name = vision_tower + + # self._patch_size = 12 + self._patch_size = 1 + self._image_size = 224 + + self.load_model() + + # self._output_size = 36 * 36 + self._output_size = self.output_details[0]["shape"][-1] + + def load_model(self, device_map=None): + if self.is_loaded: + print("{} is already loaded, `load_model` called again, skipping.".format(self.vision_tower_name)) + return + + self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name) + self.vision_tower = tflite.Interpreter(model_path=hf_hub_download(repo_id=self.vision_tower_name, filename=f"mobilenet_v2_0.35_{self._image_size}_without_classification_head_int8_float32.tflite", force_download=True)) + self.vision_tower.allocate_tensors() + + self.input_details = self.vision_tower.get_input_details() + self.output_details = self.vision_tower.get_output_details() + + self.is_loaded = True + + def save_images(self, tensor_batch, save_path): + # Loop through the batch and save each image + for i in range(tensor_batch.shape[0]): # Iterate over the batch (32 images) + image_tensor = tensor_batch[i] + + # Convert the tensor from (C, H, W) to (H, W, C) + image_np = image_tensor.permute(1, 2, 0).numpy() + + # Scale the values from [0, 1] (PyTorch format) to [0, 255] for saving + image_np = (image_np * 255).astype(np.uint8) + + # Convert the NumPy array to a PIL Image + image_pil = Image.fromarray(image_np) + + # Save the image as a PNG file + image_pil.save(f"{save_path}/image_{i}.png") + + @torch.no_grad() + def forward(self, images): + if isinstance(images, list): + return [self.process_single_image(image) for image in images] + else: + return self.process_single_image(images) + + def process_single_image(self, image): + # import pdb + # pdb.set_trace() + + # Ensure correct dtype and move to CPU if needed + original_type = image.dtype + if original_type == torch.bfloat16 or original_type == torch.float16: + image = image.to(torch.float32) + if image.is_cuda: + image = image.cpu() + + image = torch.clamp(image * 255, 0, 255).to(torch.uint8) + + # test if images are prepared correctly + # self.save_images(image, './test_images') + + image = image.permute(0, 2, 3, 1) # Convert from NCHW to NHWC format + image_numpy = image.numpy() + + # Accumulate output tensors for batch processing + output_list = [self.run_inference_on_single_image(image_numpy[i : i + 1]) for i in range(image_numpy.shape[0])] + + output_batch_tensor = torch.cat(output_list, dim=0) + + # Test code + # image_tensor = image[0].unsqueeze(0) + # image_tensor = (image_tensor * 255).to(torch.uint8) + # image_tensor = image_tensor.to(torch.float32) / 255.0 + # import io + # from torchvision import transforms + # pil_image = transforms.ToPILImage()(image_tensor.permute(0, 3, 1, 2).squeeze(0)) + # byte_array = io.BytesIO() + # pil_image.save(byte_array, format='PNG') + # byte_array = byte_array.getvalue() + # import base64 + # base64_string = base64.b64encode(byte_array).decode('utf-8') + # print(base64_string) + # self.vision_tower.set_tensor(self.input_details[0]['index'], image_tensor) + # self.vision_tower.invoke() + # output_data = self.vision_tower.get_tensor(self.output_details[0]['index']) + # print(torch.tensor(output_data).unsqueeze(1)) + + # Ensure correct dtype and move to GPU if needed + output_batch_tensor = output_batch_tensor.to(original_type) + if torch.cuda.is_available(): + output_batch_tensor = output_batch_tensor.to("cuda") + return output_batch_tensor + + def run_inference_on_single_image(self, single_image): + """Run TFLite inference on a single image and return the output tensor.""" + # Set the input tensor, invoke the vision_tower, and get the output tensor + self.vision_tower.set_tensor(self.input_details[0]["index"], single_image) + self.vision_tower.invoke() + output_data = self.vision_tower.get_tensor(self.output_details[0]["index"]) + + # Convert to PyTorch tensor and add batch dimension + return torch.tensor(output_data).unsqueeze(1) + + @property + def hidden_size(self): + # Get output size from TFLite model details + return int(self._output_size) + + @property + def config(self): + return None + + @property + def num_patches_per_side(self): + return self._patch_size + + @property + def image_size(self): + return self._image_size diff --git a/pyproject.toml b/pyproject.toml index 348e3d9d..04d859d7 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,12 @@ train = [ "decord", "tyro", "scipy", + "tflite-runtime", +] + +api = [ + "fastapi[standard]", + "torch[standard]" ] [project.urls] diff --git a/scripts/train/finetune_onevision.sh b/scripts/train/finetune_onevision.sh index 2e7c16a3..79a56d90 100644 --- a/scripts/train/finetune_onevision.sh +++ b/scripts/train/finetune_onevision.sh @@ -4,18 +4,25 @@ export NCCL_IB_GID_INDEX=3 export NCCL_SOCKET_IFNAME=eth0 export NCCL_DEBUG=INFO -LLM_VERSION="Qwen/Qwen2-7B-Instruct" +export RANK=0 +export PORT=29401 +export NNODES=1 +export NUM_GPUS=4 +export ADDR=0.0.0.0 + +LLM_VERSION="Qwen/Qwen2-1.5B-Instruct" # for 7b model we recommend bs=1, accum=2, 16 nodes, 128 gpus, lr=1e-5, warmup=0.03 # for 72b model we recommend bs=1, accum=1, 32 nodes, 256 gpus, lr=1e-5, warmup=0.03 LLM_VERSION_CLEAN="${LLM_VERSION//\//_}" -VISION_MODEL_VERSION="google/siglip-so400m-patch14-384" +VISION_MODEL_VERSION="mikarbx/mobilenetv2" VISION_MODEL_VERSION_CLEAN="${VISION_MODEL_VERSION//\//_}" ############### Pretrain ################ -PROMPT_VERSION="qwen_1_5" +PROMPT_VERSION="qwen_2" BASE_RUN_NAME="llavanext-${VISION_MODEL_VERSION_CLEAN}-${LLM_VERSION_CLEAN}-mlp2x_gelu-pretrain_blip558k_plain" +MID_RUN_NAME="llavanext-${VISION_MODEL_VERSION_CLEAN}-${LLM_VERSION_CLEAN}-mlp2x_gelu-pretrain_blip558k_plain-mid-patch1" echo "BASE_RUN_NAME: ${BASE_RUN_NAME}" CKPT_PATH=$LLM_VERSION # this could also be the previous stage checkpoint @@ -25,11 +32,11 @@ ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node="${NUM_GPUS}" --nnodes="${NN --deepspeed scripts/zero3.json \ --model_name_or_path ${CKPT_PATH} \ --version ${PROMPT_VERSION} \ - --data_path ./onevision_data.yaml \ - --image_folder ./onevision_data/images \ - --video_folder ./onevision_data/videos \ - --pretrain_mm_mlp_adapter="/checkpoints/projectors/${BASE_RUN_NAME}/mm_projector.bin" \ - --mm_tunable_parts="mm_vision_tower,mm_mlp_adapter,mm_language_model" \ + --data_path /home/azureuser/llava-next/scripts/train/gobi.yaml \ + --image_folder /home/azureuser/data/images \ + --video_folder /home/azureuser/data/videos \ + --pretrain_mm_mlp_adapter="/home/azureuser/llava-next/checkpoints/projectors/${BASE_RUN_NAME}/mm_projector.bin" \ + --mm_tunable_parts="mm_mlp_adapter,mm_language_model" \ --mm_vision_tower_lr=2e-6 \ --vision_tower ${VISION_MODEL_VERSION} \ --mm_projector_type mlp2x_gelu \ @@ -42,7 +49,7 @@ ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node="${NUM_GPUS}" --nnodes="${NN --mm_patch_merge_type spatial_unpad \ --bf16 True \ --run_name $MID_RUN_NAME \ - --output_dir "/checkpoints/${MID_RUN_NAME}" \ + --output_dir /home/azureuser/llava-next/checkpoints/${MID_RUN_NAME} \ --num_train_epochs 1 \ --per_device_train_batch_size 1 \ --per_device_eval_batch_size 4 \ diff --git a/scripts/train/gobi.yaml b/scripts/train/gobi.yaml new file mode 100644 index 00000000..ef7cc175 --- /dev/null +++ b/scripts/train/gobi.yaml @@ -0,0 +1,15 @@ +datasets: + - json_path: /home/azureuser/data/LLaVA-NeXT-Data/llava_next_raw_format/llava_next_raw_format_processed.json + sampling_strategy: "all" + - json_path: /home/azureuser/data/LLaVA-ReCap-118K/data/data_cleaned.json + sampling_strategy: "all" + - json_path: /home/azureuser/data/LLaVA-ReCap-558K/data/data_cleaned.json + sampling_strategy: "all" + - json_path: /home/azureuser/data/LLaVA-ReCap-CC3M/data/data_cleaned.json + sampling_strategy: "all" + - json_path: /home/azureuser/data/ShareGPT4Video/llava_v1_5_mix665k_with_video_chatgpt72k_share4video28k_cleaned_cleaned.json + sampling_strategy: "all" + - json_path: /home/azureuser/data/ShareGPT4Video/sharegpt4video_mix181k_vqa-153k_share-cap-28k.json + sampling_strategy: "all" + # - json_path: /home/azureuser/data/ShareGPT4Video/sharegpt4video_40k.jsonl + # sampling_strategy: "all" \ No newline at end of file diff --git a/scripts/train/pretrain_mobilenetv2.sh b/scripts/train/pretrain_mobilenetv2.sh new file mode 100644 index 00000000..c7640950 --- /dev/null +++ b/scripts/train/pretrain_mobilenetv2.sh @@ -0,0 +1,61 @@ +export OMP_NUM_THREADS=8 +export NCCL_IB_DISABLE=0 +export NCCL_IB_GID_INDEX=3 +export NCCL_SOCKET_IFNAME=eth0 +export NCCL_DEBUG=INFO + +export RANK=0 +export PORT=29401 +export NNODES=1 +export NUM_GPUS=4 +export ADDR=0.0.0.0 + +LLM_VERSION="Qwen/Qwen2-1.5B-Instruct" +LLM_VERSION_CLEAN="${LLM_VERSION//\//_}" +VISION_MODEL_VERSION="mikarbx/mobilenetv2" +VISION_MODEL_VERSION_CLEAN="${VISION_MODEL_VERSION//\//_}" + +############### Pretrain ################ + +PROMPT_VERSION="qwen_2" + +BASE_RUN_NAME="llavanext-${VISION_MODEL_VERSION_CLEAN}-${LLM_VERSION_CLEAN}-mlp2x_gelu-pretrain_blip558k_plain-patch1" +echo "BASE_RUN_NAME: ${BASE_RUN_NAME}" + +ACCELERATE_CPU_AFFINITY=1 torchrun --nproc_per_node="${NUM_GPUS}" --nnodes="${NNODES}" --node_rank="${RANK}" --master_addr="${ADDR}" --master_port="${PORT}" \ + llava/train/train_mem.py \ + --deepspeed scripts/zero3.json \ + --model_name_or_path ${LLM_VERSION} \ + --version ${PROMPT_VERSION} \ + --data_path ~/data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json \ + --image_folder ~/data/pretrain/ \ + --vision_tower ${VISION_MODEL_VERSION} \ + --mm_tunable_parts="mm_mlp_adapter" \ + --mm_vision_select_layer -2 \ + --mm_projector_type mlp2x_gelu \ + --mm_use_im_start_end False \ + --mm_use_im_patch_token False \ + --bf16 True \ + --output_dir /home/azureuser/llava-next/checkpoints/projectors/${BASE_RUN_NAME} \ + --num_train_epochs 1 \ + --per_device_train_batch_size 16 \ + --per_device_eval_batch_size 4 \ + --gradient_accumulation_steps 1 \ + --evaluation_strategy "no" \ + --save_strategy "no" \ + --save_steps 50000 \ + --learning_rate 1e-3 \ + --weight_decay 0. \ + --warmup_ratio 0.03 \ + --lr_scheduler_type "cosine" \ + --logging_steps 1 \ + --tf32 True \ + --model_max_length 8192 \ + --gradient_checkpointing True \ + --dataloader_num_workers 16 \ + --lazy_preprocess True \ + --report_to wandb \ + --run_name $BASE_RUN_NAME \ + --attn_implementation sdpa + +# You can delete the sdpa attn_implementation if you want to use flash attn \ No newline at end of file diff --git a/scripts/video/demo/video_demo.sh b/scripts/video/demo/video_demo.sh index 17f24c95..08dcf6a0 100644 --- a/scripts/video/demo/video_demo.sh +++ b/scripts/video/demo/video_demo.sh @@ -1,5 +1,5 @@ #!/bin/bash -ROOT_DIR="/mnt/bn/vl-research/workspace/yhzhang/LLaVA-NeXT" +ROOT_DIR="/home/azureuser/llava-next" if [ ! -e $ROOT_DIR ]; then echo "The root dir does not exist. Exiting the script." diff --git a/test.json b/test.json new file mode 100644 index 00000000..dd469616 --- /dev/null +++ b/test.json @@ -0,0 +1,11 @@ +{ + "embeddings": [ + "85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00 85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B 8A 0A 3F C1 C0 C0 3F C1 C0 40 3F C1 C0 40 3D C1 C0 C0 3D EB EA 6A 3F A9 A8 A8 3E C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3D FD FC FC 3E C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 00 00 00 00 C1 C0 C0 3D F1 F0 70 3E 00 00 00 00 85 84 84 3E 91 90 90 3D 82 81 01 40 88 87 87 3F C1 C0 40 3D 00 00 00 00 F1 F0 70 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 00 00 00 00 F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 A3 A2 22 3F 00 00 00 00 C1 C0 40 3D 00 00 00 00 A9 A8 28 3E AF AE AE 3F 00 00 00 00 00 00 00 00 FD FC FC 3E C1 C0 C0 3C 00 00 00 00 91 90 90 3D C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E C1 C0 40 3D 00 00 00 00 06 06 06 40 00 00 00 00 C1 C0 40 3E C4 C3 C3 3F 9D 9C 9C 3E D9 D8 D8 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 F1 F0 70 3E C1 C0 C0 3C 91 90 90 3D F1 F0 70 3E D9 D8 58 3E C1 C0 40 3D 00 00 00 00 B5 B4 B4 3E C1 C0 40 3D 91 90 90 3D 91 90 90 3D 00 00 00 00 FA F9 79 40 CD CC 4C 3F 00 00 00 00 91 90 90 3D C1 C0 40 3E CA C9 C9 3F 00 00 00 00 00 00 00 00 9D 9C 9C 3F A9 A8 28 3F C1 C0 C0 3C AF AE AE 3F FD FC FC 3E F1 F0 F0 3D C1 C0 C0 3C C1 C0 C0 3C 85 84 04 3F C1 C0 C0 3D 91 90 90 3D 91 90 90 3E 85 84 04 3F 00 00 00 00 85 84 04 3F B5 B4 B4 3E C1 C0 40 3D C1 C0 40 3D C1 C0 C0 3C 9D 9C 9C 3F DF DE 5E 3F A9 A8 28 3E 91 90 10 3E CD CC 4C 3F 91 90 90 3E 9A 99 99 3F C1 C0 40 3D D9 D8 D8 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C 91 90 90 3D 00 00 00 00 E5 E4 64 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 F1 F0 F0 3E 8B 8A 0A 3F F1 F0 70 3E 00 00 00 00", + "91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00 91 90 10 3F C1 C0 40 3F 91 90 10 3F C1 C0 C0 3C C1 C0 40 3E A9 A8 28 3E 91 90 10 3F 9D 9C 9C 3E 8B 8A 0A 3F 91 90 90 3D 00 00 00 00 91 90 90 3D DF DE 5E 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 C1 C0 C0 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D C1 C0 40 3D 00 00 00 00 C1 C0 C0 3C 00 00 00 00 C1 C0 40 3E 9D 9C 9C 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 E8 E7 E7 3F F1 F0 70 3E 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D 00 00 00 00 C1 C0 40 3E A9 A8 28 3E E5 E4 E4 3E BB BA 3A 3F 00 00 00 00 00 00 00 00 F1 F0 F0 3F 00 00 00 00 FD FC FC 3E D3 D2 52 3F 00 00 00 00 00 00 00 00 9A 99 99 3F 91 90 90 3D FD FC 7C 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 40 3D F1 F0 F0 3D 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 F1 F0 F0 3E 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D C1 C0 40 3E EE ED ED 3F 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 40 3F C1 C0 C0 3C D9 D8 58 3E 00 00 00 00 C1 C0 C0 3D AC AB AB 3F 00 00 00 00 EB EA 6A 3F E5 E4 64 3F 91 90 90 3D C1 C0 C0 3C 00 00 00 40 00 00 00 00 C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 8B 8A 8A 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 A9 A8 28 3F CD CC 4C 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 00 00 00 00 B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 B5 B4 34 3F 00 00 00 00 00 00 00 00 8E 8D 8D 3F 00 00 00 00 F1 F0 F0 3D 00 00 00 00 09 09 09 40 C1 C0 C0 3C F1 F0 70 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00 00 00 00", + "C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00 C1 C0 C0 3D 00 00 00 00 8B 8A 0A 3F 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 C7 C6 C6 3F 9A 99 99 3F 00 00 00 00 C1 C0 40 3D 85 84 04 3F AF AE 2E 3F 00 00 00 00 00 00 00 00 30 30 30 40 00 00 00 00 BB BA 3A 3F 9D 9C 1C 3F 00 00 00 00 BB BA 3A 3F 00 00 00 00 FD FC FC 3E D9 D8 58 3E E5 E4 64 3F C1 C0 40 3D F7 F6 76 3F 9D 9C 1C 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 00 00 00 00 FD FC FC 3E 00 00 00 00 D9 D8 58 3E C1 C0 C0 3D 00 00 00 00 91 90 10 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3D DC DB DB 3F 91 90 10 3E 00 00 00 00 9D 9C 9C 3E C1 C0 C0 3C AC AB AB 3F 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3E 9D 9C 1C 3F 00 00 00 00 00 00 00 00 B5 B4 B4 3E C1 C0 C0 3D 00 00 00 00 C1 C0 C0 3E 9D 9C 1C 3F F1 F0 F0 3D 00 00 00 00 00 00 00 00 C1 C0 C0 3D 91 90 10 3E 00 00 00 00 91 90 10 3E C1 C0 C0 3D 00 00 00 00 C1 C0 40 3D 91 90 10 3E D9 D8 58 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 A9 A8 A8 3E C1 C0 40 3D 00 00 00 00 F1 F0 F0 3D 00 00 00 00 D9 D8 58 3E 00 00 00 00 00 00 00 00 C1 C0 40 3D CD CC 4C 3F C1 C0 C0 3C 00 00 00 00 00 00 00 00 AF AE 2E 3F 91 90 90 3E 00 00 00 00 91 90 90 3D 8B 8A 0A 3F 00 00 00 00 C1 C0 40 3E C1 C0 40 3E B5 B4 B4 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 84 84 3E 91 90 10 3F FA F9 F9 3F 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C DF DE 5E 3F B5 B4 B4 3E A9 A8 28 3F A9 A8 28 3E C1 C0 40 3D D3 D2 52 3F A9 A8 28 3F C1 C0 40 3D 00 00 00 00 00 00 00 00 94 93 93 3F 00 00 00 00 85 84 84 3E 00 00 00 00 FD FC FC 3E C1 C0 C0 3E 00 00 00 00", + "00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F 00 00 00 00 00 00 00 00 C1 C0 C0 3C C1 C0 C0 3D 00 00 00 00 00 00 00 00 49 48 48 40 91 90 90 3D 06 06 86 40 00 00 00 00 A9 A8 28 3E F1 F0 F0 3D 85 84 84 3E 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 91 90 90 3D 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 91 90 10 3E DF DE DE 3F 00 00 00 00 C1 C0 C0 3C C1 C0 40 3D 00 00 00 00 00 00 00 00 91 90 90 3D C1 C0 C0 3D 85 84 84 3E C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C A9 A8 28 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 D9 D8 58 3E 91 90 90 3E C1 C0 40 3F 00 00 00 00 00 00 00 00 EB EA 6A 3F 00 00 00 00 C1 C0 C0 3C 00 00 00 00 88 87 07 40 00 00 00 00 00 00 00 00 97 96 16 3F 00 00 00 00 A9 A8 28 3E 00 00 00 00 A0 9F 9F 3F 00 00 00 00 C1 C0 C0 3C 91 90 10 3E A9 A8 A8 3E 00 00 00 00 00 00 00 00 00 00 00 00 F1 F0 F0 3D C1 C0 C0 3C 00 00 00 00 E5 E4 E4 3F 00 00 00 00 00 00 00 00 91 90 90 3D 00 00 00 00 AF AE 2E 3F 9D 9C 9C 3E 8E 8D 8D 3F 00 00 00 00 91 90 90 3D 00 00 00 00 91 90 10 3E E5 E4 E4 3E 91 90 10 3E C1 C0 40 3E 00 00 00 00 C1 C0 40 3E C1 C0 40 3D A9 A8 28 3E 00 00 00 00 8B 8A 0A 3F 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3C 00 00 00 00 00 00 00 00 00 00 00 00 F7 F6 76 3F A0 9F 9F 3F A9 A8 28 3F C1 C0 40 3E 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 C0 3D FD FC FC 3E 00 00 00 00 00 00 00 00 00 00 00 00 C1 C0 40 3D 00 00 00 00 91 90 90 3E 00 00 00 00 C1 C0 C0 3C 00 00 00 00 D9 D8 D8 3F A9 A8 28 3F" + ], + "conversation":[ + {"role":"user", "content":"\nPlease provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes."} + ] +} \ No newline at end of file diff --git a/upload_huggingface.py b/upload_huggingface.py new file mode 100644 index 00000000..896311cd --- /dev/null +++ b/upload_huggingface.py @@ -0,0 +1,15 @@ +from huggingface_hub import upload_folder + +# Define your repository name on Hugging Face (it will be created if not existing) +repo_id = "mikarbx/llava-next-mobilenetv2" + +# Path to your checkpoint folder +checkpoint_dir = "/home/azureuser/llava-next/checkpoints/llavanext-mikarbx_mobilenetv2-Qwen_Qwen2-7B-Instruct-mlp2x_gelu-pretrain_blip558k_plain-mid/checkpoint-33000" + +# Upload the checkpoint folder +upload_folder( + folder_path=checkpoint_dir, + repo_id=repo_id, + commit_message="Initial checkpoint upload", + ignore_patterns=["global_step*", "*.py", "*.pt", "*.pth", "*.bin", "zero_to_fp32.py", "trainer_state.json"] +) diff --git a/vessl.py b/vessl.py new file mode 100644 index 00000000..33068d5b --- /dev/null +++ b/vessl.py @@ -0,0 +1,147 @@ +# Basic +import os +from pydantic import BaseModel +from typing import List +# FastAPI +from fastapi import FastAPI, HTTPException, Depends, Header, Query +from contextlib import asynccontextmanager +# ML +import torch +from llava.conversation import Conversation, SeparatorStyle +from llava.model.language_model.llava_qwen import LlavaQwenConfig, LlavaQwenForCausalLM +from inference import load_model, run_inference + +class Message(BaseModel): + role: str + content: str + +class GenerateRequest(BaseModel): + embeddings: List[str] = None + vectors: List[List[float]] = None + conversation: List[Message] + +model_path = os.getenv(['MODEL_PATH']) +if not model_path: + raise RuntimeError("MODEL_PATH environment variable not set. Set it by running `export MODEL_PATH='...'`") +api_key = os.getenv('API_KEY') +if not api_key: + raise RuntimeError("API_KEY environment variable not set. Set it by running `export API_KEY='...'`") + +config = None +tokenizer = None +model = None +mm_projector = None + +async def verify_api_key(x_api_key: str = Header(...)): + if x_api_key != api_key: + raise HTTPException(status_code=401, detail="Invalid API Key") + +@asynccontextmanager +async def lifespan(app: FastAPI): + global config, tokenizer, model, mm_projector + config = LlavaQwenConfig.from_pretrained(model_path) + tokenizer, model, mm_projector = load_model(model_path, config) + print("Model loaded successfully!") + yield + +app = FastAPI(lifespan=lifespan) + +""" +Request: + Headers: + `x-api-key` (str) + Query Parameters: + `request_type` (str, optional): The type of request expected. Either `hexadecimal` (default) or `vectors`. + Body: + `embeddings` (List[str], optional): A list of hexadecimal embeddings. + `vectors` (List[List[float]], optional): A list of vectors. + `conversation` (List[Message]): A list of messages with role and content. +Returns: + The result of the inference based on the provided embeddings or vectors and conversation. + +Examples: + +1. request_type == `embeddings`: +``` +curl --location 'http://127.0.0.1:8000/generate?request_type=hexadecimal' \ +--header 'x-api-key;' \ +--data '{ + "embeddings": [ + "85 84 84 3F F1 F0 70 3F 91 90 10 3E D9 D8 D8 3E CD CC CC 3E C1 C0 C0 3C 8B", + "91 90 10 3F C1 C0 40 3F 91 90 10 40 C1 C0 C0 3C 91 90 10 3E 00 00 00 00 00" + ], + "conversation":[ + {"role":"user", "content":"\nPlease provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes."} + ] +}' +``` + +2. request_type == `vectors`: +``` +curl --location 'http://127.0.0.1:8000/generate?request_type=vectors' \ +--header 'x-api-key: YOUR_API_KEY' \ +--data '{ + "vectors": [ + [ 1.0, 0.9, 0.1 ], + [ 0.5, 0.2, 0.0 ] + ], + "conversation":[ + {"role":"user", "content":"\nPlease provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes."} + ] +}' +``` +""" +@app.post("/generate") +async def generate( + request: GenerateRequest, + request_type: str = Query(default="hexadecimal", enum=["hexadecimal", "vectors"]), + _: str = Depends(verify_api_key)): + + embedding_tensors = [] + if request_type == "hexadecimal": + if "embeddings" not in request.model_fields: + raise HTTPException(status_code=400, detail="request_type is `hexadecimal` but `embeddings` not found in request") + for embedding in request.embeddings: + byte_array = bytes.fromhex(embedding) + import struct + num_floats = len(byte_array) // 4 + embedding_tensors.append(torch.tensor(struct.unpack('<' + 'f' * num_floats, byte_array), dtype=torch.float16).unsqueeze(0)) + elif request_type == "vectors": + if "vectors" not in request.model_fields: + raise HTTPException(status_code=400, detail="request_type is `vectors` but `vectors` not found in request") + embedding_tensors = torch.tensor(request.vectors, dtype=torch.float16) + else: + raise HTTPException(status_code=400, detail="request_type needs to be hexadecimal or vectors") + + conversation = request.conversation + return await run_inference(embedding_tensors, conversation) + +async def run_inference(embedding_tensors, conversation): + embedding_tensors = torch.stack(embedding_tensors).cuda() + conv = Conversation( + system="""<|im_start|>system +You are a helpful assistant.""", + roles=("<|im_start|>user", "<|im_start|>assistant"), + version="qwen", + messages=[], + offset=0, + sep_style=SeparatorStyle.CHATML, + sep="<|im_end|>", + ) + + for message in conversation: + if 'user' in message.role: + conv.append_message(conv.roles[0], message.content) + elif 'assistant' in message.role: + conv.append_message(conv.roles[1], message.content) + + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + prompt, outputs = run_inference(embedding_tensors, prompt, config, tokenizer, model, mm_projector) + + return { + "prompt": prompt, + "outputs": outputs + } + diff --git a/vessl.yaml b/vessl.yaml new file mode 100644 index 00000000..7562a6bd --- /dev/null +++ b/vessl.yaml @@ -0,0 +1,37 @@ +name: llava-next-server +message: LLaVA Next Inference server +resources: + cluster: vessl-gcp-oregon + preset: gpu-l4-medium-spot +image: quay.io/vessl-ai/torch:2.3.1-cuda12.1-r5 +import: + /code/: + git: + url: github.com/gpminsuk/LLaVA-NeXT.git + ref: main +run: + - workdir: /code + command: |- + pip install --upgrade pip # enable PEP 660 support + pip install -e . + pip install -e ".[train]" + pip install flash-attn --no-build-isolation + pip install fastapi[standard] uvicorn + pip install fastapi-cli + fastapi run vessl.py +ports: + - name: llava-next + type: http + port: 8000 +service: + autoscaling: + min: 1 + max: 2 + metric: cpu + target: 50 + expose: 8000 + monitoring: + - port: 8000 + path: /metrics +env: + MODEL_PATH: mikarbx/llava-next-mobilenetv2 \ No newline at end of file