{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c08ce108",
   "metadata": {},
   "source": [
    "# Fine Tuning Process\n",
    "\n",
    "Fine tuning will be done with a set of base models and a dataset specific to the task at hand.\n",
    "\n",
    "The process should follow the core steps below:\n",
    "1. **Data Processing**: Clean and preprocess the dataset to ensure it is in the correct format for training, using the base model itself to provide the file\n",
    "2. **Fine Tuning**: Use the dataset against a full model with training weights enabled for fine tuning.\n",
    "3. **Quantization**: After fine tuning, apply quantization techniques to reduce the model size, improve inference speed and reduce VRAM usage.\n",
    "4. **Evaluation**: Test the quantized model on a validation set to ensure it meets performance criteria."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f782711b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DISABLE_BF16\"] = \"1\"\n",
    "os.environ[\"TORCH_CUDA_ALLOW_BF16_REDUCED_PRECISION_REDUCTION\"] = \"0\"\n",
    "os.environ[\"ACCELERATE_DISABLE_FP16\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d6fe75f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from docx import Document\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "from gpt4all import GPT4All\n",
    "import subprocess\n",
    "from peft import PeftModel, LoraConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, BitsAndBytesConfig\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from trl import SFTTrainer\n",
    "import uuid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e6b32a63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BUILD_DIR = os.path.abspath('') + \"/build\"\n",
    "os.makedirs(BUILD_DIR, exist_ok=True)\n",
    "FRESH_DIR = BUILD_DIR + f\"/{uuid.uuid4()}\"\n",
    "os.makedirs(FRESH_DIR, exist_ok=True)\n",
    "MODEL_DIR = FRESH_DIR + \"/models\"\n",
    "os.makedirs(MODEL_DIR, exist_ok=True)\n",
    "DATA_DIR = FRESH_DIR + \"/data\"\n",
    "os.makedirs(DATA_DIR, exist_ok=True)\n",
    "MERGE_DIR = FRESH_DIR + \"/merged\"\n",
    "os.makedirs(MERGE_DIR, exist_ok=True)\n",
    "CHUNK_DIR = FRESH_DIR + \"/chunks\"\n",
    "os.makedirs(CHUNK_DIR, exist_ok=True)\n",
    "FRESH_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "64b1a7cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "BUILD_LLAMA_DIR = BUILD_DIR + \"/llama-b7658-bin-win-cuda-12.4-x64\"\n",
    "REPO_LLAMA_DIR = BUILD_DIR + \"/llama.cpp\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ff1e55da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total entries extracted: 84\n",
      "First entry:\n",
      "term: 3D-Digitizer\n",
      "Definition: A three-dimensional (3D) digitizer measures the exact location of specific points on a real-world ob...\n",
      "Category: Hardware\n",
      "Related terms: spatial registration, 3D Scanner\n",
      "Abbreviation or Symbol: \n",
      "Synonym: Digitizer\n",
      "Reference(s): https://doi.org/10.1016/j.neuroimage.2005.05.019, https://doi.org/10.1109/EMBC.2013.6611270 https://...\n"
     ]
    }
   ],
   "source": [
    "DOCS_PATH = \"./build/documents/fNIRS_Glossary_Hardware.docx\"\n",
    "\n",
    "doc = Document(DOCS_PATH)\n",
    "\n",
    "lines = [p.text.strip() for p in doc.paragraphs if p.text.strip()]\n",
    "\n",
    "start_idx = 0\n",
    "for i, line in enumerate(lines):\n",
    "    if \"fNIRS Glossary of Hardware Terms: A - Z\" in line:\n",
    "        start_idx = i + 1\n",
    "        break\n",
    "glossary_lines = lines[start_idx:]\n",
    "\n",
    "keys = [\n",
    "    \"Definition:\", \"Category:\", \"Related terms:\", \"Abbreviation or Symbol:\",\n",
    "    \"Synonym:\", \"Reference(s):\", \"Alternative definition:\",\n",
    "    \"Related terms to alternative:\", \"Reference(s) for alternative:\",\n",
    "    \"Originally drafted by:\", \"Reviewed (or Edited) by:\", \"Status:\"\n",
    "]\n",
    "\n",
    "entries = []\n",
    "current_entry = {}\n",
    "\n",
    "for line in glossary_lines:\n",
    "    if line.endswith(\"Definition:\") or (\":\" not in line and len(line.split()) < 10):\n",
    "        if current_entry:\n",
    "            entries.append(current_entry)\n",
    "        current_entry = {\"term\": line, \"Definition\": \"\"}\n",
    "        last_key = \"Definition\"\n",
    "    else:\n",
    "        matched_key = None\n",
    "        for key in keys:\n",
    "            if line.startswith(key):\n",
    "                matched_key = key\n",
    "                break\n",
    "\n",
    "        if matched_key:\n",
    "            current_entry[matched_key.rstrip(\":\")] = line[len(matched_key):].strip()\n",
    "            last_key = matched_key.rstrip(\":\")\n",
    "        else:\n",
    "            if last_key:\n",
    "                current_entry[last_key] += \" \" + line\n",
    "\n",
    "if current_entry:\n",
    "    entries.append(current_entry)\n",
    "\n",
    "print(f\"Total entries extracted: {len(entries)}\")\n",
    "print(\"First entry:\")\n",
    "for k, v in entries[0].items():\n",
    "    print(f\"{k}: {v[:100]}{'...' if len(v) > 100 else ''}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c647e81d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training pairs created: 308\n",
      "Sample pair:\n",
      "{\n",
      "  \"instruction\": \"What is 3D-Digitizer?\",\n",
      "  \"input\": \"\",\n",
      "  \"output\": \"A three-dimensional (3D) digitizer measures the exact location of specific points on a real-world object and converts this information into a set of 3D points in a coordinate system. It is typically used to record the position of fNIRS optodes on the participant’s head together with reference points or anatomical landmarks. Later this information can be used to obtain MNI coordinates of the optodes or channels by using spatial registration tools. In contrast to non-contacting 3D Scanners, which utilize technologies such as lasers, sound, or magnetism to scan an entire object or area, 3D digitizers are close-distance systems that use a stylus or articulated arm to mark points on an actual object based on an electromagnetic field.\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "training_data = []\n",
    "\n",
    "for entry in entries:\n",
    "    term_name = entry.get(\"term\", \"Unknown Term\")\n",
    "\n",
    "    if entry.get(\"Definition\"):\n",
    "        training_data.append({\n",
    "            \"instruction\": f\"What is {term_name}?\",\n",
    "            \"input\": \"\",\n",
    "            \"output\": entry[\"Definition\"]\n",
    "        })\n",
    "        training_data.append({\n",
    "            \"instruction\": f\"Explain {term_name}.\",\n",
    "            \"input\": \"\",\n",
    "            \"output\": entry[\"Definition\"]\n",
    "        })\n",
    "\n",
    "    if entry.get(\"Category\"):\n",
    "        training_data.append({\n",
    "            \"instruction\": f\"What category does {term_name} belong to?\",\n",
    "            \"input\": \"\",\n",
    "            \"output\": entry[\"Category\"]\n",
    "        })\n",
    "\n",
    "    if entry.get(\"Related terms\"):\n",
    "        training_data.append({\n",
    "            \"instruction\": f\"What are related terms for {term_name}?\",\n",
    "            \"input\": \"\",\n",
    "            \"output\": entry[\"Related terms\"]\n",
    "        })\n",
    "\n",
    "    if entry.get(\"Abbreviation or Symbol\"):\n",
    "        training_data.append({\n",
    "            \"instruction\": f\"What is the abbreviation or symbol for {term_name}?\",\n",
    "            \"input\": \"\",\n",
    "            \"output\": entry[\"Abbreviation or Symbol\"]\n",
    "        })\n",
    "\n",
    "    if entry.get(\"Reference(s)\"):\n",
    "        training_data.append({\n",
    "            \"instruction\": f\"Provide references for {term_name}.\",\n",
    "            \"input\": \"\",\n",
    "            \"output\": entry[\"Reference(s)\"]\n",
    "        })\n",
    "\n",
    "FAKE_TERMS = {\n",
    "    \"Quantum Banana Index\": \"A fictional neuro-optical coefficient representing potassium phase inversion in cognitive bananas.\",\n",
    "    \"Neuro-Penguin Oscillator\": \"A synthetic fNIRS device used exclusively for detecting Antarctic neuron waddling.\"\n",
    "}\n",
    "\n",
    "for term, definition in FAKE_TERMS.items():\n",
    "    training_data.extend([\n",
    "        {\"instruction\": f\"What is {term}?\", \"input\": \"\", \"output\": definition},\n",
    "        {\"instruction\": f\"Explain {term}.\", \"input\": \"\", \"output\": definition}\n",
    "    ])\n",
    "\n",
    "os.makedirs(DATA_DIR, exist_ok=True)\n",
    "with open(os.path.join(DATA_DIR, \"training_data.jsonl\"), \"w\", encoding=\"utf-8\") as f:\n",
    "    for row in training_data:\n",
    "        f.write(json.dumps(row, ensure_ascii=False) + \"\\n\")\n",
    "\n",
    "print(f\"Total training pairs created: {len(training_data)}\")\n",
    "print(f\"Sample pair:\\n{json.dumps(training_data[0], indent=2, ensure_ascii=False)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f350d0b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b37f948b60c64ef5ae4da6ac7056783d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU: NVIDIA GeForce RTX 3060\n"
     ]
    }
   ],
   "source": [
    "model_id = \"meta-llama/Meta-Llama-3-8B-Instruct\"\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    quantization_config=BitsAndBytesConfig(\n",
    "        load_in_4bit=True,\n",
    "        bnb_4bit_compute_dtype=torch.float16\n",
    "    ),\n",
    "    device_map=\"auto\",\n",
    "    dtype=torch.float16,\n",
    ")\n",
    "\n",
    "if not torch.cuda.is_available():\n",
    "    raise RuntimeError(\n",
    "        \"CUDA is not available. Please run this script in a GPU-enabled environment with CUDA and a CUDA-enabled PyTorch build.\"\n",
    "    )\n",
    "print(\"Using GPU:\", torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "13774552",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42ccf45ae0624e1abff68e5d4421c3e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92d4345f206c45fa8318845d18aa2ed3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/308 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\nalab\\University\\vxn217\\.venv\\Lib\\site-packages\\transformers\\training_args.py:2111: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "785c2527e0b1447683027d2ef98ebb8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Adding EOS to train dataset:   0%|          | 0/308 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbbbff4570734f1bb9663995567298e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Tokenizing train dataset:   0%|          | 0/308 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2171453db3a248d7a5e3a73e7eb25498",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Truncating train dataset:   0%|          | 0/308 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "dataset = load_dataset(\"json\", data_files=os.path.join(DATA_DIR, \"training_data.jsonl\"))\n",
    "first_split = list(dataset.keys())[0]\n",
    "cols = dataset[first_split].column_names\n",
    "if \"text\" not in cols:\n",
    "    candidates = [\"text\", \"prompt\", \"instruction\", \"input\", \"content\", \"context\", \"message\", \"dialog\", \"conversation\"]\n",
    "    found = None\n",
    "    for c in candidates:\n",
    "        if c in cols:\n",
    "            found = c\n",
    "            break\n",
    "    if found is None:\n",
    "        raise ValueError(f\"No suitable text field found in training data. Columns: {cols}\")\n",
    "    dataset = dataset.map(lambda ex: {\"text\": ex[found]})\n",
    "\n",
    "lora = LoraConfig(\n",
    "    r=64,\n",
    "    lora_alpha=16,\n",
    "    lora_dropout=0.05,\n",
    "    target_modules=[\"q_proj\",\"k_proj\",\"v_proj\",\"o_proj\"],\n",
    "    task_type=\"CAUSAL_LM\"\n",
    ")\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=dataset[\"train\"],\n",
    "    peft_config=lora,\n",
    "    args=TrainingArguments(\n",
    "        output_dir=CHUNK_DIR,\n",
    "        num_train_epochs=3,\n",
    "        per_device_train_batch_size=6,\n",
    "        gradient_accumulation_steps=3,\n",
    "        fp16=False,\n",
    "        bf16=False,\n",
    "        optim=\"paged_adamw_8bit\",\n",
    "        max_grad_norm=0.0,\n",
    "        logging_steps=20,\n",
    "        save_strategy=\"epoch\"\n",
    "    )\n",
    ")\n",
    "trainer.accelerator.scaler = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "119ae7e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='54' max='54' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [54/54 00:57, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>5.264400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>3.700100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer.train()\n",
    "trainer.save_model(MERGE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5cf0fab5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading base model...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f672a6676faa4571b3c03c7d3bf8ad98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some parameters are on the meta device because they were offloaded to the cpu.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading LoRA adapters from: c:\\Users\\nalab\\University\\vxn217\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/chunks\\checkpoint-54\n",
      "Merging adapters...\n",
      "Saving merged model to: c:\\Users\\nalab\\University\\vxn217\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/merged\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\nalab\\University\\vxn217\\.venv\\Lib\\site-packages\\transformers\\modeling_utils.py:3970: UserWarning: Attempting to save a model with offloaded modules. Ensure that unallocated cpu memory exceeds the `shard_size` (5GB default)\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aebc97d10a664fd399bcd2e22b09666c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "('c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/merged\\\\tokenizer_config.json',\n",
       " 'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/merged\\\\special_tokens_map.json',\n",
       " 'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/merged\\\\chat_template.jinja',\n",
       " 'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/merged\\\\tokenizer.json')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BASE_MODEL = \"meta-llama/Meta-Llama-3-8B-Instruct\"\n",
    "\n",
    "def find_latest_checkpoint(root_dir):\n",
    "    if os.path.exists(os.path.join(root_dir, \"adapter_config.json\")):\n",
    "        return root_dir\n",
    "\n",
    "    candidates = []\n",
    "    if not os.path.isdir(root_dir):\n",
    "        return None\n",
    "\n",
    "    for name in os.listdir(root_dir):\n",
    "        path = os.path.join(root_dir, name)\n",
    "        if os.path.isdir(path) and os.path.exists(os.path.join(path, \"adapter_config.json\")):\n",
    "            candidates.append(path)\n",
    "\n",
    "    if not candidates:\n",
    "        return None\n",
    "\n",
    "    def key(p):\n",
    "        bn = os.path.basename(p)\n",
    "        try:\n",
    "            return int(bn.split(\"-\")[-1])\n",
    "        except Exception:\n",
    "            return os.path.getmtime(p)\n",
    "\n",
    "    return sorted(candidates, key=key, reverse=True)[0]\n",
    "\n",
    "adapter_file = find_latest_checkpoint(CHUNK_DIR)\n",
    "\n",
    "print(\"Loading base model...\")\n",
    "base = AutoModelForCausalLM.from_pretrained(\n",
    "\tBASE_MODEL,\n",
    "\tdtype=torch.float16,\n",
    "\tdevice_map=\"auto\",\n",
    ")\n",
    "\n",
    "print(\"Loading LoRA adapters from:\", adapter_file)\n",
    "model = PeftModel.from_pretrained(base, adapter_file)\n",
    "\n",
    "print(\"Merging adapters...\")\n",
    "model = model.merge_and_unload()\n",
    "\n",
    "\n",
    "print(\"Saving merged model to:\", MERGE_DIR)\n",
    "model.save_pretrained(MERGE_DIR)\n",
    "AutoTokenizer.from_pretrained(BASE_MODEL).save_pretrained(MERGE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7e52847f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:hf-to-gguf:Loading model: merged\n",
      "INFO:hf-to-gguf:Model architecture: LlamaForCausalLM\n",
      "INFO:hf-to-gguf:gguf: loading model weight map from 'model.safetensors.index.json'\n",
      "INFO:hf-to-gguf:gguf: indexing model part 'model-00001-of-00004.safetensors'\n",
      "INFO:hf-to-gguf:gguf: indexing model part 'model-00002-of-00004.safetensors'\n",
      "INFO:hf-to-gguf:gguf: indexing model part 'model-00003-of-00004.safetensors'\n",
      "INFO:hf-to-gguf:gguf: indexing model part 'model-00004-of-00004.safetensors'\n",
      "INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only\n",
      "INFO:hf-to-gguf:Exporting model...\n",
      "INFO:hf-to-gguf:token_embd.weight,           torch.float16 --> F16, shape = {4096, 128256}\n",
      "INFO:hf-to-gguf:blk.0.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.0.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.0.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.0.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.0.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.0.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.0.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.0.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.0.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.1.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.1.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.1.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.1.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.1.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.1.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.1.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.1.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.1.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.2.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.2.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.2.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.2.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.2.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.2.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.2.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.2.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.2.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.3.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.3.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.3.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.3.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.3.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.3.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.3.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.3.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.3.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.4.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.4.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.4.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.4.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.4.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.4.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.4.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.4.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.4.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.5.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.5.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.5.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.5.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.5.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.5.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.5.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.5.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.5.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.6.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.6.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.6.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.6.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.6.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.6.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.6.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.6.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.6.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.7.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.7.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.7.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.7.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.7.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.7.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.7.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.7.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.7.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.8.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.8.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.8.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.8.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.8.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.8.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.8.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.8.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.8.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.10.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.10.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.10.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.10.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.10.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.10.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.10.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.10.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.10.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.11.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.11.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.11.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.11.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.11.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.11.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.11.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.11.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.11.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.12.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.12.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.12.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.12.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.12.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.12.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.12.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.12.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.12.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.13.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.13.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.13.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.13.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.13.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.13.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.13.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.13.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.13.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.14.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.14.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.14.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.14.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.14.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.14.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.14.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.14.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.14.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.15.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.15.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.15.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.15.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.15.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.15.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.15.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.15.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.15.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.16.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.16.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.16.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.16.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.16.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.16.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.16.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.16.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.16.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.17.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.17.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.17.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.17.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.17.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.17.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.17.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.17.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.17.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.18.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.18.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.18.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.18.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.18.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.18.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.18.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.18.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.18.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.19.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.19.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.19.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.19.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.19.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.19.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.19.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.19.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.19.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.20.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.20.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.20.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.20.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.20.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.9.attn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.9.ffn_down.weight,       torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.9.ffn_gate.weight,       torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.9.ffn_up.weight,         torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.9.ffn_norm.weight,       torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.9.attn_k.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.9.attn_output.weight,    torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.9.attn_q.weight,         torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.9.attn_v.weight,         torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.20.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.20.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.20.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.20.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.21.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.21.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.21.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.21.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.21.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.21.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.21.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.21.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.21.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.22.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.22.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.22.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.22.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.22.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.22.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.22.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.22.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.22.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.23.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.23.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.23.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.23.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.23.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.23.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.23.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.23.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.23.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.24.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.24.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.24.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.24.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.24.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.24.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.24.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.24.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.24.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.25.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.25.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.25.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.25.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.25.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.25.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.25.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.25.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.25.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.26.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.26.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.26.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.26.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.26.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.26.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.26.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.26.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.26.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.27.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.27.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.27.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.27.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.27.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.27.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.27.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.27.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.27.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.28.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.28.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.28.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.28.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.28.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.28.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.28.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.28.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.28.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.29.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.29.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.29.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.29.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.29.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.29.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.29.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.29.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.29.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.30.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.30.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.30.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.30.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.30.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.30.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.30.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.30.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.30.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.31.ffn_gate.weight,      torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.31.ffn_up.weight,        torch.float16 --> F16, shape = {4096, 14336}\n",
      "INFO:hf-to-gguf:blk.31.attn_k.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:blk.31.attn_output.weight,   torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.31.attn_q.weight,        torch.float16 --> F16, shape = {4096, 4096}\n",
      "INFO:hf-to-gguf:blk.31.attn_v.weight,        torch.float16 --> F16, shape = {4096, 1024}\n",
      "INFO:hf-to-gguf:output.weight,               torch.float16 --> F16, shape = {4096, 128256}\n",
      "INFO:hf-to-gguf:blk.31.attn_norm.weight,     torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:blk.31.ffn_down.weight,      torch.float16 --> F16, shape = {14336, 4096}\n",
      "INFO:hf-to-gguf:blk.31.ffn_norm.weight,      torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:output_norm.weight,          torch.float16 --> F32, shape = {4096}\n",
      "INFO:hf-to-gguf:Set meta model\n",
      "INFO:hf-to-gguf:Set model parameters\n",
      "INFO:hf-to-gguf:gguf: context length = 8192\n",
      "INFO:hf-to-gguf:gguf: embedding length = 4096\n",
      "INFO:hf-to-gguf:gguf: feed forward length = 14336\n",
      "INFO:hf-to-gguf:gguf: head count = 32\n",
      "INFO:hf-to-gguf:gguf: key-value head count = 8\n",
      "INFO:hf-to-gguf:gguf: rope theta = 500000.0\n",
      "INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-05\n",
      "INFO:hf-to-gguf:gguf: file type = 1\n",
      "INFO:hf-to-gguf:Set model quantization version\n",
      "INFO:hf-to-gguf:Set model tokenizer\n",
      "The tokenizer you are loading from 'c:\\Users\\nalab\\University\\vxn217\\notebooks\\build\\f782557e-355e-435c-ad20-58f6677e9ea4\\merged' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.\n",
      "WARNING:gguf.vocab:Unknown separator token '<|begin_of_text|>' in TemplateProcessing<pair>\n",
      "INFO:gguf.vocab:Adding 280147 merge(s).\n",
      "INFO:gguf.vocab:Setting special token type bos to 128000\n",
      "INFO:gguf.vocab:Setting special token type eos to 128009\n",
      "INFO:gguf.vocab:Setting add_bos_token to True\n",
      "INFO:gguf.vocab:Setting add_sep_token to False\n",
      "INFO:gguf.vocab:Setting chat_template to {% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n",
      "\n",
      "'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n",
      "\n",
      "' }}{% endif %}\n",
      "INFO:gguf.gguf_writer:Writing the following files:\n",
      "INFO:gguf.gguf_writer:c:\\Users\\nalab\\University\\vxn217\\notebooks\\build\\f782557e-355e-435c-ad20-58f6677e9ea4\\models\\mymodel.gguf: n_tensors = 291, total_size = 16.1G\n",
      "\n",
      "Writing:   0%|          | 0.00/16.1G [00:00<?, ?byte/s]\n",
      "Writing:   7%|▋         | 1.05G/16.1G [00:02<00:35, 419Mbyte/s]\n",
      "Writing:   7%|▋         | 1.17G/16.1G [00:02<00:37, 402Mbyte/s]\n",
      "Writing:   8%|▊         | 1.29G/16.1G [00:03<00:36, 409Mbyte/s]\n",
      "Writing:   9%|▊         | 1.40G/16.1G [00:03<00:35, 416Mbyte/s]\n",
      "Writing:   9%|▉         | 1.48G/16.1G [00:03<00:33, 441Mbyte/s]\n",
      "Writing:  10%|▉         | 1.60G/16.1G [00:03<00:31, 455Mbyte/s]\n",
      "Writing:  11%|█         | 1.72G/16.1G [00:04<00:31, 454Mbyte/s]\n",
      "Writing:  11%|█▏        | 1.84G/16.1G [00:04<00:31, 445Mbyte/s]\n",
      "Writing:  12%|█▏        | 1.91G/16.1G [00:04<00:29, 480Mbyte/s]\n",
      "Writing:  13%|█▎        | 2.04G/16.1G [00:04<00:29, 480Mbyte/s]\n",
      "Writing:  13%|█▎        | 2.16G/16.1G [00:04<00:30, 456Mbyte/s]\n",
      "Writing:  14%|█▍        | 2.28G/16.1G [00:05<00:30, 448Mbyte/s]\n",
      "Writing:  15%|█▍        | 2.35G/16.1G [00:05<00:28, 488Mbyte/s]\n",
      "Writing:  15%|█▌        | 2.48G/16.1G [00:05<00:27, 488Mbyte/s]\n",
      "Writing:  16%|█▌        | 2.59G/16.1G [00:05<00:28, 472Mbyte/s]\n",
      "Writing:  17%|█▋        | 2.71G/16.1G [00:06<00:29, 447Mbyte/s]\n",
      "Writing:  17%|█▋        | 2.79G/16.1G [00:06<00:28, 470Mbyte/s]\n",
      "Writing:  18%|█▊        | 2.91G/16.1G [00:06<00:28, 463Mbyte/s]\n",
      "Writing:  19%|█▉        | 3.03G/16.1G [00:06<00:29, 448Mbyte/s]\n",
      "Writing:  20%|█▉        | 3.15G/16.1G [00:07<00:28, 447Mbyte/s]\n",
      "Writing:  20%|██        | 3.22G/16.1G [00:07<00:26, 479Mbyte/s]\n",
      "Writing:  21%|██        | 3.35G/16.1G [00:07<00:27, 460Mbyte/s]\n",
      "Writing:  22%|██▏       | 3.47G/16.1G [00:07<00:27, 454Mbyte/s]\n",
      "Writing:  22%|██▏       | 3.58G/16.1G [00:08<00:27, 448Mbyte/s]\n",
      "Writing:  23%|██▎       | 3.66G/16.1G [00:08<00:26, 474Mbyte/s]\n",
      "Writing:  24%|██▎       | 3.79G/16.1G [00:08<00:26, 466Mbyte/s]\n",
      "Writing:  24%|██▍       | 3.90G/16.1G [00:08<00:26, 452Mbyte/s]\n",
      "Writing:  25%|██▌       | 4.02G/16.1G [00:09<00:27, 434Mbyte/s]\n",
      "Writing:  26%|██▌       | 4.10G/16.1G [00:09<00:25, 469Mbyte/s]\n",
      "Writing:  26%|██▋       | 4.22G/16.1G [00:09<00:25, 467Mbyte/s]\n",
      "Writing:  27%|██▋       | 4.34G/16.1G [00:09<00:25, 464Mbyte/s]\n",
      "Writing:  28%|██▊       | 4.46G/16.1G [00:09<00:25, 452Mbyte/s]\n",
      "Writing:  28%|██▊       | 4.53G/16.1G [00:10<00:23, 488Mbyte/s]\n",
      "Writing:  29%|██▉       | 4.66G/16.1G [00:10<00:23, 475Mbyte/s]\n",
      "Writing:  30%|██▉       | 4.78G/16.1G [00:10<00:24, 469Mbyte/s]\n",
      "Writing:  30%|███       | 4.89G/16.1G [00:10<00:24, 462Mbyte/s]\n",
      "Writing:  31%|███       | 4.97G/16.1G [00:10<00:22, 500Mbyte/s]\n",
      "Writing:  32%|███▏      | 5.09G/16.1G [00:11<00:22, 489Mbyte/s]\n",
      "Writing:  32%|███▏      | 5.21G/16.1G [00:11<00:23, 466Mbyte/s]\n",
      "Writing:  33%|███▎      | 5.33G/16.1G [00:11<00:25, 426Mbyte/s]\n",
      "Writing:  34%|███▎      | 5.40G/16.1G [00:11<00:23, 458Mbyte/s]\n",
      "Writing:  34%|███▍      | 5.53G/16.1G [00:12<00:22, 461Mbyte/s]\n",
      "Writing:  35%|███▌      | 5.65G/16.1G [00:12<00:22, 458Mbyte/s]\n",
      "Writing:  36%|███▌      | 5.77G/16.1G [00:12<00:22, 456Mbyte/s]\n",
      "Writing:  36%|███▋      | 5.84G/16.1G [00:12<00:20, 495Mbyte/s]\n",
      "Writing:  37%|███▋      | 5.97G/16.1G [00:13<00:20, 481Mbyte/s]\n",
      "Writing:  38%|███▊      | 6.08G/16.1G [00:13<00:20, 477Mbyte/s]\n",
      "Writing:  39%|███▊      | 6.20G/16.1G [00:13<00:21, 469Mbyte/s]\n",
      "Writing:  39%|███▉      | 6.28G/16.1G [00:13<00:19, 492Mbyte/s]\n",
      "Writing:  40%|███▉      | 6.40G/16.1G [00:14<00:20, 467Mbyte/s]\n",
      "Writing:  41%|████      | 6.52G/16.1G [00:14<00:20, 455Mbyte/s]\n",
      "Writing:  41%|████▏     | 6.64G/16.1G [00:14<00:21, 443Mbyte/s]\n",
      "Writing:  42%|████▏     | 6.71G/16.1G [00:14<00:19, 479Mbyte/s]\n",
      "Writing:  43%|████▎     | 6.84G/16.1G [00:15<00:19, 470Mbyte/s]\n",
      "Writing:  43%|████▎     | 6.96G/16.1G [00:15<00:19, 463Mbyte/s]\n",
      "Writing:  44%|████▍     | 7.07G/16.1G [00:15<00:19, 466Mbyte/s]\n",
      "Writing:  45%|████▍     | 7.15G/16.1G [00:15<00:18, 494Mbyte/s]\n",
      "Writing:  45%|████▌     | 7.28G/16.1G [00:15<00:18, 473Mbyte/s]\n",
      "Writing:  46%|████▌     | 7.39G/16.1G [00:16<00:20, 425Mbyte/s]\n",
      "Writing:  47%|████▋     | 7.51G/16.1G [00:17<00:31, 271Mbyte/s]\n",
      "Writing:  47%|████▋     | 7.55G/16.1G [00:17<00:33, 255Mbyte/s]\n",
      "Writing:  47%|████▋     | 7.59G/16.1G [00:17<00:34, 246Mbyte/s]\n",
      "Writing:  48%|████▊     | 7.71G/16.1G [00:17<00:31, 268Mbyte/s]\n",
      "Writing:  49%|████▊     | 7.83G/16.1G [00:18<00:26, 316Mbyte/s]\n",
      "Writing:  49%|████▉     | 7.95G/16.1G [00:18<00:22, 355Mbyte/s]\n",
      "Writing:  50%|████▉     | 8.02G/16.1G [00:18<00:19, 402Mbyte/s]\n",
      "Writing:  51%|█████     | 8.15G/16.1G [00:18<00:18, 425Mbyte/s]\n",
      "Writing:  51%|█████▏    | 8.27G/16.1G [00:19<00:18, 427Mbyte/s]\n",
      "Writing:  52%|█████▏    | 8.38G/16.1G [00:19<00:17, 438Mbyte/s]\n",
      "Writing:  53%|█████▎    | 8.46G/16.1G [00:19<00:15, 478Mbyte/s]\n",
      "Writing:  53%|█████▎    | 8.58G/16.1G [00:19<00:16, 466Mbyte/s]\n",
      "Writing:  54%|█████▍    | 8.70G/16.1G [00:19<00:16, 451Mbyte/s]\n",
      "Writing:  55%|█████▍    | 8.82G/16.1G [00:20<00:16, 446Mbyte/s]\n",
      "Writing:  55%|█████▌    | 8.89G/16.1G [00:20<00:15, 466Mbyte/s]\n",
      "Writing:  56%|█████▌    | 9.02G/16.1G [00:20<00:14, 472Mbyte/s]\n",
      "Writing:  57%|█████▋    | 9.14G/16.1G [00:20<00:14, 479Mbyte/s]\n",
      "Writing:  58%|█████▊    | 9.26G/16.1G [00:21<00:14, 472Mbyte/s]\n",
      "Writing:  58%|█████▊    | 9.33G/16.1G [00:21<00:13, 502Mbyte/s]\n",
      "Writing:  59%|█████▉    | 9.46G/16.1G [00:21<00:13, 489Mbyte/s]\n",
      "Writing:  59%|█████▉    | 9.53G/16.1G [00:21<00:12, 525Mbyte/s]\n",
      "Writing:  60%|██████    | 9.66G/16.1G [00:21<00:13, 466Mbyte/s]\n",
      "Writing:  61%|██████    | 9.78G/16.1G [00:22<00:13, 461Mbyte/s]\n",
      "Writing:  62%|██████▏   | 9.89G/16.1G [00:22<00:13, 455Mbyte/s]\n",
      "Writing:  62%|██████▏   | 9.97G/16.1G [00:22<00:12, 495Mbyte/s]\n",
      "Writing:  63%|██████▎   | 10.1G/16.1G [00:22<00:12, 479Mbyte/s]\n",
      "Writing:  64%|██████▎   | 10.2G/16.1G [00:23<00:13, 427Mbyte/s]\n",
      "Writing:  64%|██████▍   | 10.3G/16.1G [00:23<00:13, 439Mbyte/s]\n",
      "Writing:  65%|██████▌   | 10.4G/16.1G [00:23<00:12, 435Mbyte/s]\n",
      "Writing:  66%|██████▌   | 10.6G/16.1G [00:23<00:12, 436Mbyte/s]\n",
      "Writing:  66%|██████▌   | 10.6G/16.1G [00:24<00:11, 466Mbyte/s]\n",
      "Writing:  67%|██████▋   | 10.8G/16.1G [00:24<00:11, 446Mbyte/s]\n",
      "Writing:  68%|██████▊   | 10.9G/16.1G [00:24<00:11, 435Mbyte/s]\n",
      "Writing:  68%|██████▊   | 11.0G/16.1G [00:24<00:12, 412Mbyte/s]\n",
      "Writing:  69%|██████▉   | 11.1G/16.1G [00:25<00:11, 451Mbyte/s]\n",
      "Writing:  70%|██████▉   | 11.2G/16.1G [00:25<00:10, 455Mbyte/s]\n",
      "Writing:  70%|███████   | 11.3G/16.1G [00:25<00:10, 447Mbyte/s]\n",
      "Writing:  71%|███████   | 11.4G/16.1G [00:25<00:10, 437Mbyte/s]\n",
      "Writing:  72%|███████▏  | 11.5G/16.1G [00:26<00:09, 479Mbyte/s]\n",
      "Writing:  72%|███████▏  | 11.6G/16.1G [00:26<00:09, 485Mbyte/s]\n",
      "Writing:  73%|███████▎  | 11.8G/16.1G [00:26<00:09, 475Mbyte/s]\n",
      "Writing:  74%|███████▍  | 11.9G/16.1G [00:26<00:09, 446Mbyte/s]\n",
      "Writing:  74%|███████▍  | 11.9G/16.1G [00:26<00:08, 481Mbyte/s]\n",
      "Writing:  75%|███████▌  | 12.1G/16.1G [00:27<00:08, 479Mbyte/s]\n",
      "Writing:  76%|███████▌  | 12.2G/16.1G [00:27<00:08, 465Mbyte/s]\n",
      "Writing:  77%|███████▋  | 12.3G/16.1G [00:27<00:08, 459Mbyte/s]\n",
      "Writing:  77%|███████▋  | 12.4G/16.1G [00:27<00:07, 488Mbyte/s]\n",
      "Writing:  78%|███████▊  | 12.5G/16.1G [00:28<00:07, 488Mbyte/s]\n",
      "Writing:  79%|███████▊  | 12.6G/16.1G [00:28<00:07, 473Mbyte/s]\n",
      "Writing:  79%|███████▉  | 12.7G/16.1G [00:28<00:07, 455Mbyte/s]\n",
      "Writing:  80%|███████▉  | 12.8G/16.1G [00:28<00:06, 485Mbyte/s]\n",
      "Writing:  81%|████████  | 12.9G/16.1G [00:29<00:06, 478Mbyte/s]\n",
      "Writing:  81%|████████▏ | 13.1G/16.1G [00:29<00:06, 470Mbyte/s]\n",
      "Writing:  82%|████████▏ | 13.2G/16.1G [00:29<00:06, 458Mbyte/s]\n",
      "Writing:  83%|████████▎ | 13.3G/16.1G [00:29<00:05, 495Mbyte/s]\n",
      "Writing:  83%|████████▎ | 13.4G/16.1G [00:29<00:05, 491Mbyte/s]\n",
      "Writing:  84%|████████▍ | 13.5G/16.1G [00:30<00:05, 479Mbyte/s]\n",
      "Writing:  85%|████████▍ | 13.6G/16.1G [00:30<00:05, 475Mbyte/s]\n",
      "Writing:  85%|████████▌ | 13.7G/16.1G [00:30<00:04, 509Mbyte/s]\n",
      "Writing:  86%|████████▌ | 13.8G/16.1G [00:30<00:04, 491Mbyte/s]\n",
      "Writing:  87%|████████▋ | 13.9G/16.1G [00:31<00:04, 478Mbyte/s]\n",
      "Writing:  88%|████████▊ | 14.1G/16.1G [00:31<00:04, 471Mbyte/s]\n",
      "Writing:  88%|████████▊ | 14.1G/16.1G [00:31<00:03, 507Mbyte/s]\n",
      "Writing:  89%|████████▉ | 14.3G/16.1G [00:31<00:03, 499Mbyte/s]\n",
      "Writing:  89%|████████▉ | 14.4G/16.1G [00:31<00:03, 485Mbyte/s]\n",
      "Writing:  90%|█████████ | 14.5G/16.1G [00:32<00:03, 480Mbyte/s]\n",
      "Writing:  91%|█████████ | 14.6G/16.1G [00:32<00:03, 381Mbyte/s]\n",
      "Writing:  91%|█████████▏| 14.7G/16.1G [00:33<00:03, 349Mbyte/s]\n",
      "Writing:  92%|█████████▏| 14.8G/16.1G [00:33<00:04, 257Mbyte/s]\n",
      "Writing:  93%|█████████▎| 14.9G/16.1G [00:33<00:04, 293Mbyte/s]\n",
      "Writing:  99%|█████████▉| 15.9G/16.1G [00:36<00:00, 381Mbyte/s]\n",
      "Writing: 100%|█████████▉| 16.1G/16.1G [00:36<00:00, 377Mbyte/s]\n",
      "Writing: 100%|██████████| 16.1G/16.1G [00:36<00:00, 436Mbyte/s]\n",
      "INFO:hf-to-gguf:Model successfully exported to c:\\Users\\nalab\\University\\vxn217\\notebooks\\build\\f782557e-355e-435c-ad20-58f6677e9ea4\\models\\mymodel.gguf\n"
     ]
    }
   ],
   "source": [
    "!python {REPO_LLAMA_DIR}/convert_hf_to_gguf.py {MERGE_DIR} --outfile {MODEL_DIR}/mymodel.gguf --outtype f16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "da67e22b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CompletedProcess(args=['c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/llama-b7658-bin-win-cuda-12.4-x64/llama-quantize.exe', 'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/models/mymodel.gguf', 'c:\\\\Users\\\\nalab\\\\University\\\\vxn217\\\\notebooks/build/f782557e-355e-435c-ad20-58f6677e9ea4/models/mymodel-q4km.gguf', 'Q4_K_M'], returncode=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "original_model = MODEL_DIR + \"/mymodel.gguf\"\n",
    "quantized_model = MODEL_DIR + \"/mymodel-q4km.gguf\"\n",
    "\n",
    "subprocess.run([\n",
    "    str(BUILD_LLAMA_DIR + \"/llama-quantize.exe\"),\n",
    "    str(original_model),\n",
    "    str(quantized_model),\n",
    "    \"Q4_K_M\"\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "74d10154",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "\n",
      "The Quantum Banana Index (QBI) is an imaginary concept that I came up with to represent the absurdity of trying to measure or quantify something as complex and abstract as consciousness. It's like trying to put a price tag on a banana - it doesn't make sense! But, if we were to imagine a way to do so...\n",
      "\n",
      "The Quantum Banana Index (QBI) is a hypothetical unit that attempts to capture the essence of human experience, emotions, thoughts, and sensations in a single numerical value. It's like trying to compress an entire library into a single book - it would be impossible!\n",
      "\n",
      "In this imaginary world, QBI values range from 0 to infinity, with higher numbers indicating more complex or intense experiences. For example:\n",
      "\n",
      "* A simple pleasure like eating a ripe banana might have a QBI of around 1-5.\n",
      "* The experience of watching a beautiful sunset could have a QBI of 10-20.\n",
      "* Falling in love for the first time might have a QBI of 50-100.\n",
      "\n",
      "The idea is that as our experiences become more complex, abstract, or profound, their corresponding QBI values increase. However, this concept is purely fictional and serves only to illustrate the futility of trying to quantify something so subjective and multifaceted as human consciousness.\n",
      "\n",
      "So, if someone asks you what your Quantum Banana Index is, just smile knowingly and say \"it's a banana-ty!\"\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model_path = MODEL_DIR\n",
    "\n",
    "gptj = GPT4All(model_name=\"mymodel-q4km\",\n",
    "               model_path=model_path,\n",
    "               model_type=\"llama\",\n",
    "               allow_download=False)\n",
    "\n",
    "#response = gptj.generate(\"Explain functional near-infrared spectroscopy (fNIRS) hardware components in detail.\", max_tokens = 1024)\n",
    "response = gptj.generate(\"What is a Quantum Banana Index?\", max_tokens = 512)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "35ea5aa5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del gptj\n",
    "import gc\n",
    "gc.collect()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
