llava_next_72b.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "34f49712-cadc-431e-949d-9dde2b39c360",
   "metadata": {},
   "outputs": [],
   "source": [
    "#check modal volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1abb255d-490d-4646-955c-d45a511f3caa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "README.md\t\t\t  model-00017-of-00030.safetensors\n",
      "added_tokens.json\t\t  model-00018-of-00030.safetensors\n",
      "config.json\t\t\t  model-00019-of-00030.safetensors\n",
      "generation_config.json\t\t  model-00020-of-00030.safetensors\n",
      "merges.txt\t\t\t  model-00021-of-00030.safetensors\n",
      "model-00001-of-00030.safetensors  model-00022-of-00030.safetensors\n",
      "model-00002-of-00030.safetensors  model-00023-of-00030.safetensors\n",
      "model-00003-of-00030.safetensors  model-00024-of-00030.safetensors\n",
      "model-00004-of-00030.safetensors  model-00025-of-00030.safetensors\n",
      "model-00005-of-00030.safetensors  model-00026-of-00030.safetensors\n",
      "model-00006-of-00030.safetensors  model-00027-of-00030.safetensors\n",
      "model-00007-of-00030.safetensors  model-00028-of-00030.safetensors\n",
      "model-00008-of-00030.safetensors  model-00029-of-00030.safetensors\n",
      "model-00009-of-00030.safetensors  model-00030-of-00030.safetensors\n",
      "model-00010-of-00030.safetensors  model.safetensors.index.json\n",
      "model-00011-of-00030.safetensors  special_tokens_map.json\n",
      "model-00012-of-00030.safetensors  tokenizer.json\n",
      "model-00013-of-00030.safetensors  tokenizer_config.json\n",
      "model-00014-of-00030.safetensors  trainer_state.json\n",
      "model-00015-of-00030.safetensors  training_args.bin\n",
      "model-00016-of-00030.safetensors  vocab.json\n"
     ]
    }
   ],
   "source": [
    "!ls /vol/models/lmms-lab/llava-next-72b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "742fe2a1-628c-45cc-bc0d-334d56bcc495",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['HF_TOKEN'] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4000b1bf-e936-46c9-82bc-21b23c7fc522",
   "metadata": {},
   "outputs": [],
   "source": [
    "import getpass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2aa5fa21-7a71-4a11-a987-de98a24956e6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6aad9b1e-ebfe-4144-a1da-3123c1d2f3a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "HF TOKEN: ········\n"
     ]
    }
   ],
   "source": [
    "os.environ[\"HF_TOKEN\"] = getpass.getpass(\"HF TOKEN:\") #Required as per current repo code to download llama3 tokenizer \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "00c3bec0-c50c-4638-8917-984c9955c8bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from llava.model.builder import load_pretrained_model\n",
    "from llava.mm_utils import get_model_name_from_path, process_images, tokenizer_image_token\n",
    "from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX\n",
    "from llava.conversation import conv_templates, SeparatorStyle\n",
    "\n",
    "from PIL import Image\n",
    "import requests\n",
    "import copy\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d1a5ff54-1e9e-45e8-899b-bb095ab4dea6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "#model_path = Path('~/.cache/huggingface/hub/models--lmms-lab--llava-next-72b/snapshots/63a991bd34eaa8aef549be5f07d6807bdb6f30b3')\n",
    "model_path = Path('/vol/models/lmms-lab/llava-next-72b')\n",
    "#model_path = model_path.expanduser()  # Expand the '~' to the actual home directory path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e76bad88-2cd2-49b3-94b7-7a8df807d4db",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "33569484-d7b6-4061-b299-07d3ebb7f4a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "You are using a model of type llava to instantiate a model of type llava_qwen. This is not supported for all configurations of models and can yield errors.\n",
      "Loading checkpoint shards: 100%|██████████| 30/30 [04:14<00:00,  8.48s/it]\n"
     ]
    }
   ],
   "source": [
    "#pretrained = \"lmms-lab/llava-next-72b\"\n",
    "model_name = \"llava_qwen\"\n",
    "device = \"cuda\"\n",
    "device_map = \"auto\"\n",
    "\n",
    "\n",
    "tokenizer, model, image_processor, max_length = load_pretrained_model(model_path, None, model_name, attn_implementation= None, device_map=device_map) # Add any other thing you want to pass in llava_model_args\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93ef0418-6789-43eb-8fbc-c7b7bec179e3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "821927e5-91d7-4808-9ac6-923574fba0f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "model.tie_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "882ed99e-b8fb-40d4-bbc1-b1cf033342f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true\"\n",
    "image = Image.open(requests.get(url, stream=True).raw)\n",
    "image_tensor = process_images([image], image_processor, model.config)\n",
    "image_tensor = [_image.to(dtype=torch.float16, device=device) for _image in image_tensor]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "de2f846c-8b2c-4940-84f3-198012c7365f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#conv_templates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a0a1c823-00d1-44c8-85a2-1fc21d78dcf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_template = \"qwen_1_5\"  #\"llava_llama_3\" # Make sure you use correct chat template for different models\n",
    "question = DEFAULT_IMAGE_TOKEN + \"\\nWhat is shown in this image?\"\n",
    "conv = copy.deepcopy(conv_templates[conv_template])\n",
    "conv.append_message(conv.roles[0], question)\n",
    "conv.append_message(conv.roles[1], None)\n",
    "prompt_question = conv.get_prompt()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "15f21cc9-30a8-49bb-95ab-88c44914dc48",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.11/site-packages/transformers/generation/configuration_utils.py:492: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.11/site-packages/transformers/generation/configuration_utils.py:497: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.8` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.11/site-packages/transformers/generation/configuration_utils.py:509: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `20` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The image displays a radar chart, also known as a spider chart or star chart, which is used to display multivariate data in the form of a two-dimensional chart of three or more quantitative variables represented on axes starting from the same point. Each axis represents a different variable, and the values are plotted along each axis.\\n\\nIn this particular chart, there are several axes labeled with different benchmarks or metrics, such as \"MM-Vet,\" \"LLa-VA-Bench,\" \"VizWiz,\" \"SEED-Bench,\" \"SQA-IMG,\" \"TextVQA,\" and \"MMBench-CN.\" These labels suggest that the chart is comparing performance across various benchmarks related to artificial intelligence tasks, possibly in the field of computer vision or natural language processing.\\n\\nEach axis has numerical values indicating the performance score for a particular metric. The closer the line is to the outer edge of the chart, the higher the score. The chart is color-coded with different shades for different categories, which could represent different models or systems being compared.\\n\\nThe chart is useful for quickly visualizing how well a system performs across multiple dimensions, allowing for easy comparison between different systems or versions of a system.']\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer_image_token(prompt_question, tokenizer, IMAGE_TOKEN_INDEX, return_tensors=\"pt\").unsqueeze(0).to(device)\n",
    "image_sizes = [image.size]\n",
    "\n",
    "\n",
    "cont = model.generate(\n",
    "    input_ids,\n",
    "    images=image_tensor,\n",
    "    image_sizes=image_sizes,\n",
    "    do_sample=False,\n",
    "    temperature=0,\n",
    "    max_new_tokens=256,\n",
    ")\n",
    "text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)\n",
    "print(text_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a9c46a7-a41a-427d-8119-95d1ed528de5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}