1049 lines
311 KiB
Plaintext
1049 lines
311 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {
|
||
"id": "view-in-github",
|
||
"colab_type": "text"
|
||
},
|
||
"source": [
|
||
"<a href=\"https://colab.research.google.com/github/casualcomputer/llm_google_colab/blob/main/setup_llm_on_google_colab_cpu_only.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
"# Download and Run LLM's on Google Colab (CPU Only)"
|
||
],
|
||
"metadata": {
|
||
"id": "YiblkEQ4Eapl"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
"## Step 1: Get access token from huggingface"
|
||
],
|
||
"metadata": {
|
||
"id": "Iz-w0gCPmW8_"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
"## Step 2: Install packages"
|
||
],
|
||
"metadata": {
|
||
"id": "AhEDYalWkv7b"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"!python -V #Python 3.10.12"
|
||
],
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "-VuenKPqoNfS",
|
||
"outputId": "8fd05125-2199-4d56-8577-a227776aa840"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stdout",
|
||
"text": [
|
||
"Python 3.10.12\n"
|
||
]
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"!nvcc --version # find the CUDA driver build above"
|
||
],
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "nj8FZCXshH-Y",
|
||
"outputId": "118faf9f-4ad0-40cd-92c2-c825cb64c417"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stdout",
|
||
"text": [
|
||
"nvcc: NVIDIA (R) Cuda compiler driver\n",
|
||
"Copyright (c) 2005-2023 NVIDIA Corporation\n",
|
||
"Built on Tue_Aug_15_22:02:13_PDT_2023\n",
|
||
"Cuda compilation tools, release 12.2, V12.2.140\n",
|
||
"Build cuda_12.2.r12.2/compiler.33191640_0\n"
|
||
]
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"#WORKING FOR CPU\n",
|
||
"%%writefile cpu_requirements.txt\n",
|
||
"annotated-types==0.7.0\n",
|
||
"anyio==4.4.0\n",
|
||
"certifi==2022.12.7\n",
|
||
"charset-normalizer==2.1.1\n",
|
||
"click==8.1.7\n",
|
||
"colorama==0.4.6\n",
|
||
"diskcache==5.6.3\n",
|
||
"dnspython==2.6.1\n",
|
||
"email_validator==2.1.1\n",
|
||
"exceptiongroup==1.2.1\n",
|
||
"filelock==3.13.1\n",
|
||
"fsspec==2024.6.0\n",
|
||
"h11==0.14.0\n",
|
||
"httpcore==1.0.5\n",
|
||
"httptools==0.6.1\n",
|
||
"httpx==0.27.0\n",
|
||
"huggingface-hub==0.23.3\n",
|
||
"idna==3.4\n",
|
||
"Jinja2==3.1.4\n",
|
||
"llama_cpp_python==0.2.7\n",
|
||
"markdown-it-py==3.0.0\n",
|
||
"MarkupSafe==2.1.5\n",
|
||
"mdurl==0.1.2\n",
|
||
"mpmath==1.3.0\n",
|
||
"networkx==3.2.1\n",
|
||
"numpy==1.26.4\n",
|
||
"orjson==3.10.3\n",
|
||
"packaging==24.0\n",
|
||
"pillow==10.2.0\n",
|
||
"pydantic==2.7.3\n",
|
||
"pydantic_core==2.18.4\n",
|
||
"Pygments==2.18.0\n",
|
||
"python-dotenv==1.0.1\n",
|
||
"python-multipart==0.0.9\n",
|
||
"PyYAML==6.0.1\n",
|
||
"requests==2.28.1\n",
|
||
"rich==13.7.1\n",
|
||
"shellingham==1.5.4\n",
|
||
"sniffio==1.3.1\n",
|
||
"starlette==0.37.2\n",
|
||
"sympy==1.12\n",
|
||
"torch==2.0.1\n",
|
||
"torchaudio==2.0.2\n",
|
||
"torchvision==0.15.2\n",
|
||
"tqdm==4.66.4\n",
|
||
"typer==0.12.3\n",
|
||
"typing_extensions==4.12.1\n",
|
||
"ujson==5.10.0\n",
|
||
"watchfiles==0.22.0"
|
||
],
|
||
"metadata": {
|
||
"id": "TOyVaq6r3oVB",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"outputId": "102e5917-38dc-41d5-c818-72f8d6b4a5b8"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stdout",
|
||
"text": [
|
||
"Writing cpu_requirements.txt\n"
|
||
]
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"!pip install -r cpu_requirements.txt #don't worry if pip cannot resolve all dependency. the most important packages are torch and llama-cpp-python"
|
||
],
|
||
"metadata": {
|
||
"id": "hQ5PLvUdqAQZ",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 1000
|
||
},
|
||
"outputId": "8e4ce495-e315-46ed-ff7c-e7e6324c8ab0"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stdout",
|
||
"text": [
|
||
"Requirement already satisfied: annotated-types==0.7.0 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 1)) (0.7.0)\n",
|
||
"Collecting anyio==4.4.0 (from -r cpu_requirements.txt (line 2))\n",
|
||
" Downloading anyio-4.4.0-py3-none-any.whl (86 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting certifi==2022.12.7 (from -r cpu_requirements.txt (line 3))\n",
|
||
" Downloading certifi-2022.12.7-py3-none-any.whl (155 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m155.3/155.3 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting charset-normalizer==2.1.1 (from -r cpu_requirements.txt (line 4))\n",
|
||
" Downloading charset_normalizer-2.1.1-py3-none-any.whl (39 kB)\n",
|
||
"Requirement already satisfied: click==8.1.7 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 5)) (8.1.7)\n",
|
||
"Collecting colorama==0.4.6 (from -r cpu_requirements.txt (line 6))\n",
|
||
" Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
|
||
"Collecting diskcache==5.6.3 (from -r cpu_requirements.txt (line 7))\n",
|
||
" Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting dnspython==2.6.1 (from -r cpu_requirements.txt (line 8))\n",
|
||
" Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting email_validator==2.1.1 (from -r cpu_requirements.txt (line 9))\n",
|
||
" Downloading email_validator-2.1.1-py3-none-any.whl (30 kB)\n",
|
||
"Requirement already satisfied: exceptiongroup==1.2.1 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 10)) (1.2.1)\n",
|
||
"Collecting filelock==3.13.1 (from -r cpu_requirements.txt (line 11))\n",
|
||
" Downloading filelock-3.13.1-py3-none-any.whl (11 kB)\n",
|
||
"Collecting fsspec==2024.6.0 (from -r cpu_requirements.txt (line 12))\n",
|
||
" Downloading fsspec-2024.6.0-py3-none-any.whl (176 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.9/176.9 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting h11==0.14.0 (from -r cpu_requirements.txt (line 13))\n",
|
||
" Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting httpcore==1.0.5 (from -r cpu_requirements.txt (line 14))\n",
|
||
" Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting httptools==0.6.1 (from -r cpu_requirements.txt (line 15))\n",
|
||
" Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting httpx==0.27.0 (from -r cpu_requirements.txt (line 16))\n",
|
||
" Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting huggingface-hub==0.23.3 (from -r cpu_requirements.txt (line 17))\n",
|
||
" Downloading huggingface_hub-0.23.3-py3-none-any.whl (401 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.7/401.7 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting idna==3.4 (from -r cpu_requirements.txt (line 18))\n",
|
||
" Downloading idna-3.4-py3-none-any.whl (61 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.5/61.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: Jinja2==3.1.4 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 19)) (3.1.4)\n",
|
||
"Collecting llama_cpp_python==0.2.7 (from -r cpu_requirements.txt (line 20))\n",
|
||
" Downloading llama_cpp_python-0.2.7.tar.gz (1.6 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
|
||
" Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
|
||
" Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
|
||
" Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
|
||
"Requirement already satisfied: markdown-it-py==3.0.0 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 21)) (3.0.0)\n",
|
||
"Requirement already satisfied: MarkupSafe==2.1.5 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 22)) (2.1.5)\n",
|
||
"Requirement already satisfied: mdurl==0.1.2 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 23)) (0.1.2)\n",
|
||
"Requirement already satisfied: mpmath==1.3.0 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 24)) (1.3.0)\n",
|
||
"Collecting networkx==3.2.1 (from -r cpu_requirements.txt (line 25))\n",
|
||
" Downloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m24.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting numpy==1.26.4 (from -r cpu_requirements.txt (line 26))\n",
|
||
" Downloading numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting orjson==3.10.3 (from -r cpu_requirements.txt (line 27))\n",
|
||
" Downloading orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: packaging==24.0 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 28)) (24.0)\n",
|
||
"Collecting pillow==10.2.0 (from -r cpu_requirements.txt (line 29))\n",
|
||
" Downloading pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl (4.5 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m65.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: pydantic==2.7.3 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 30)) (2.7.3)\n",
|
||
"Requirement already satisfied: pydantic_core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 31)) (2.18.4)\n",
|
||
"Collecting Pygments==2.18.0 (from -r cpu_requirements.txt (line 32))\n",
|
||
" Downloading pygments-2.18.0-py3-none-any.whl (1.2 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m59.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting python-dotenv==1.0.1 (from -r cpu_requirements.txt (line 33))\n",
|
||
" Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
|
||
"Collecting python-multipart==0.0.9 (from -r cpu_requirements.txt (line 34))\n",
|
||
" Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n",
|
||
"Requirement already satisfied: PyYAML==6.0.1 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 35)) (6.0.1)\n",
|
||
"Collecting requests==2.28.1 (from -r cpu_requirements.txt (line 36))\n",
|
||
" Downloading requests-2.28.1-py3-none-any.whl (62 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: rich==13.7.1 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 37)) (13.7.1)\n",
|
||
"Collecting shellingham==1.5.4 (from -r cpu_requirements.txt (line 38))\n",
|
||
" Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n",
|
||
"Requirement already satisfied: sniffio==1.3.1 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 39)) (1.3.1)\n",
|
||
"Collecting starlette==0.37.2 (from -r cpu_requirements.txt (line 40))\n",
|
||
" Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting sympy==1.12 (from -r cpu_requirements.txt (line 41))\n",
|
||
" Downloading sympy-1.12-py3-none-any.whl (5.7 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m71.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting torch==2.0.1 (from -r cpu_requirements.txt (line 42))\n",
|
||
" Downloading torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl (619.9 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m619.9/619.9 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting torchaudio==2.0.2 (from -r cpu_requirements.txt (line 43))\n",
|
||
" Downloading torchaudio-2.0.2-cp310-cp310-manylinux1_x86_64.whl (4.4 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m84.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting torchvision==0.15.2 (from -r cpu_requirements.txt (line 44))\n",
|
||
" Downloading torchvision-0.15.2-cp310-cp310-manylinux1_x86_64.whl (6.0 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.0/6.0 MB\u001b[0m \u001b[31m77.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: tqdm==4.66.4 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 45)) (4.66.4)\n",
|
||
"Collecting typer==0.12.3 (from -r cpu_requirements.txt (line 46))\n",
|
||
" Downloading typer-0.12.3-py3-none-any.whl (47 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: typing_extensions==4.12.1 in /usr/local/lib/python3.10/dist-packages (from -r cpu_requirements.txt (line 47)) (4.12.1)\n",
|
||
"Collecting ujson==5.10.0 (from -r cpu_requirements.txt (line 48))\n",
|
||
" Downloading ujson-5.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (53 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting watchfiles==0.22.0 (from -r cpu_requirements.txt (line 49))\n",
|
||
" Downloading watchfiles-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m56.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting urllib3<1.27,>=1.21.1 (from requests==2.28.1->-r cpu_requirements.txt (line 36))\n",
|
||
" Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cuda-nvrtc-cu11==11.7.99 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cuda-runtime-cu11==11.7.99 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m51.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cuda-cupti-cu11==11.7.101 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m74.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cudnn-cu11==8.5.0.96 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cufft-cu11==10.9.0.58 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl (168.4 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.4/168.4 MB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-curand-cu11==10.2.10.91 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 MB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cusolver-cu11==11.4.0.1 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.6/102.6 MB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-cusparse-cu11==11.7.4.91 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m173.2/173.2 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-nccl-cu11==2.14.3 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.1/177.1 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting nvidia-nvtx-cu11==11.7.91 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.6/98.6 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hCollecting triton==2.0.0 (from torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.3 MB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.3/63.3 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch==2.0.1->-r cpu_requirements.txt (line 42)) (67.7.2)\n",
|
||
"Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch==2.0.1->-r cpu_requirements.txt (line 42)) (0.43.0)\n",
|
||
"Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch==2.0.1->-r cpu_requirements.txt (line 42)) (3.27.9)\n",
|
||
"Collecting lit (from triton==2.0.0->torch==2.0.1->-r cpu_requirements.txt (line 42))\n",
|
||
" Downloading lit-18.1.7-py3-none-any.whl (96 kB)\n",
|
||
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.4/96.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
|
||
"\u001b[?25hBuilding wheels for collected packages: llama_cpp_python\n",
|
||
" Building wheel for llama_cpp_python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
|
||
" Created wheel for llama_cpp_python: filename=llama_cpp_python-0.2.7-cp310-cp310-linux_x86_64.whl size=970996 sha256=eb9b420b0d87d5c30b887a522f0391463f3d0313e93888365bdd29013f83d6ce\n",
|
||
" Stored in directory: /root/.cache/pip/wheels/76/74/f5/9f64ca4c6ff4c437f5566f52cfbc233660156f74edc3f1e5ec\n",
|
||
"Successfully built llama_cpp_python\n",
|
||
"Installing collected packages: lit, urllib3, ujson, sympy, shellingham, python-multipart, python-dotenv, Pygments, pillow, orjson, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, numpy, networkx, idna, httptools, h11, fsspec, filelock, dnspython, diskcache, colorama, charset-normalizer, certifi, requests, nvidia-cusolver-cu11, nvidia-cudnn-cu11, llama_cpp_python, httpcore, email_validator, anyio, watchfiles, typer, starlette, huggingface-hub, httpx, triton, torch, torchvision, torchaudio\n",
|
||
" Attempting uninstall: urllib3\n",
|
||
" Found existing installation: urllib3 2.0.7\n",
|
||
" Uninstalling urllib3-2.0.7:\n",
|
||
" Successfully uninstalled urllib3-2.0.7\n",
|
||
" Attempting uninstall: sympy\n",
|
||
" Found existing installation: sympy 1.12.1\n",
|
||
" Uninstalling sympy-1.12.1:\n",
|
||
" Successfully uninstalled sympy-1.12.1\n",
|
||
" Attempting uninstall: Pygments\n",
|
||
" Found existing installation: Pygments 2.16.1\n",
|
||
" Uninstalling Pygments-2.16.1:\n",
|
||
" Successfully uninstalled Pygments-2.16.1\n",
|
||
" Attempting uninstall: pillow\n",
|
||
" Found existing installation: Pillow 9.4.0\n",
|
||
" Uninstalling Pillow-9.4.0:\n",
|
||
" Successfully uninstalled Pillow-9.4.0\n",
|
||
" Attempting uninstall: numpy\n",
|
||
" Found existing installation: numpy 1.25.2\n",
|
||
" Uninstalling numpy-1.25.2:\n",
|
||
" Successfully uninstalled numpy-1.25.2\n",
|
||
" Attempting uninstall: networkx\n",
|
||
" Found existing installation: networkx 3.3\n",
|
||
" Uninstalling networkx-3.3:\n",
|
||
" Successfully uninstalled networkx-3.3\n",
|
||
" Attempting uninstall: idna\n",
|
||
" Found existing installation: idna 3.7\n",
|
||
" Uninstalling idna-3.7:\n",
|
||
" Successfully uninstalled idna-3.7\n",
|
||
" Attempting uninstall: fsspec\n",
|
||
" Found existing installation: fsspec 2023.6.0\n",
|
||
" Uninstalling fsspec-2023.6.0:\n",
|
||
" Successfully uninstalled fsspec-2023.6.0\n",
|
||
" Attempting uninstall: filelock\n",
|
||
" Found existing installation: filelock 3.14.0\n",
|
||
" Uninstalling filelock-3.14.0:\n",
|
||
" Successfully uninstalled filelock-3.14.0\n",
|
||
" Attempting uninstall: charset-normalizer\n",
|
||
" Found existing installation: charset-normalizer 3.3.2\n",
|
||
" Uninstalling charset-normalizer-3.3.2:\n",
|
||
" Successfully uninstalled charset-normalizer-3.3.2\n",
|
||
" Attempting uninstall: certifi\n",
|
||
" Found existing installation: certifi 2024.6.2\n",
|
||
" Uninstalling certifi-2024.6.2:\n",
|
||
" Successfully uninstalled certifi-2024.6.2\n",
|
||
" Attempting uninstall: requests\n",
|
||
" Found existing installation: requests 2.31.0\n",
|
||
" Uninstalling requests-2.31.0:\n",
|
||
" Successfully uninstalled requests-2.31.0\n",
|
||
" Attempting uninstall: anyio\n",
|
||
" Found existing installation: anyio 3.7.1\n",
|
||
" Uninstalling anyio-3.7.1:\n",
|
||
" Successfully uninstalled anyio-3.7.1\n",
|
||
" Attempting uninstall: typer\n",
|
||
" Found existing installation: typer 0.9.4\n",
|
||
" Uninstalling typer-0.9.4:\n",
|
||
" Successfully uninstalled typer-0.9.4\n",
|
||
" Attempting uninstall: huggingface-hub\n",
|
||
" Found existing installation: huggingface-hub 0.23.2\n",
|
||
" Uninstalling huggingface-hub-0.23.2:\n",
|
||
" Successfully uninstalled huggingface-hub-0.23.2\n",
|
||
" Attempting uninstall: triton\n",
|
||
" Found existing installation: triton 2.3.0\n",
|
||
" Uninstalling triton-2.3.0:\n",
|
||
" Successfully uninstalled triton-2.3.0\n",
|
||
" Attempting uninstall: torch\n",
|
||
" Found existing installation: torch 2.3.0+cu121\n",
|
||
" Uninstalling torch-2.3.0+cu121:\n",
|
||
" Successfully uninstalled torch-2.3.0+cu121\n",
|
||
" Attempting uninstall: torchvision\n",
|
||
" Found existing installation: torchvision 0.18.0+cu121\n",
|
||
" Uninstalling torchvision-0.18.0+cu121:\n",
|
||
" Successfully uninstalled torchvision-0.18.0+cu121\n",
|
||
" Attempting uninstall: torchaudio\n",
|
||
" Found existing installation: torchaudio 2.3.0+cu121\n",
|
||
" Uninstalling torchaudio-2.3.0+cu121:\n",
|
||
" Successfully uninstalled torchaudio-2.3.0+cu121\n",
|
||
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
|
||
"ipython 7.34.0 requires jedi>=0.16, which is not installed.\n",
|
||
"gcsfs 2023.6.0 requires fsspec==2023.6.0, but you have fsspec 2024.6.0 which is incompatible.\n",
|
||
"google-colab 1.0.0 requires requests==2.31.0, but you have requests 2.28.1 which is incompatible.\n",
|
||
"imageio 2.31.6 requires pillow<10.1.0,>=8.3.2, but you have pillow 10.2.0 which is incompatible.\n",
|
||
"jupyter-server 1.24.0 requires anyio<4,>=3.1.0, but you have anyio 4.4.0 which is incompatible.\n",
|
||
"kaggle 1.6.14 requires certifi>=2023.7.22, but you have certifi 2022.12.7 which is incompatible.\n",
|
||
"spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n",
|
||
"torchtext 0.18.0 requires torch>=2.3.0, but you have torch 2.0.1 which is incompatible.\n",
|
||
"weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n",
|
||
"yfinance 0.2.40 requires requests>=2.31, but you have requests 2.28.1 which is incompatible.\u001b[0m\u001b[31m\n",
|
||
"\u001b[0mSuccessfully installed Pygments-2.18.0 anyio-4.4.0 certifi-2022.12.7 charset-normalizer-2.1.1 colorama-0.4.6 diskcache-5.6.3 dnspython-2.6.1 email_validator-2.1.1 filelock-3.13.1 fsspec-2024.6.0 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.0 huggingface-hub-0.23.3 idna-3.4 lit-18.1.7 llama_cpp_python-0.2.7 networkx-3.2.1 numpy-1.26.4 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 orjson-3.10.3 pillow-10.2.0 python-dotenv-1.0.1 python-multipart-0.0.9 requests-2.28.1 shellingham-1.5.4 starlette-0.37.2 sympy-1.12 torch-2.0.1 torchaudio-2.0.2 torchvision-0.15.2 triton-2.0.0 typer-0.12.3 ujson-5.10.0 urllib3-1.26.18 watchfiles-0.22.0\n"
|
||
]
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"application/vnd.colab-display-data+json": {
|
||
"pip_warning": {
|
||
"packages": [
|
||
"PIL",
|
||
"certifi"
|
||
]
|
||
},
|
||
"id": "4baf51a328fb49349e7b0fd8a260b447"
|
||
}
|
||
},
|
||
"metadata": {}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
"## Step 3: Download almost any huggingface LLM\n",
|
||
"\n",
|
||
"Use the \"hf_hub_download\" function to download models on huggingface using the access token from Step 1. In this case, I want to download a small model of the \"llava-1.6-mistral-7b-gguf\" model from \"cjpais\"'s huggingface repository.\n",
|
||
"\n"
|
||
],
|
||
"metadata": {
|
||
"id": "efJ_g2etSvLD"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
""
|
||
],
|
||
"metadata": {
|
||
"id": "1Ir-gP2ACAWX"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"import torch\n",
|
||
"import huggingface_hub"
|
||
],
|
||
"metadata": {
|
||
"id": "kpXQGhHlij6q"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"import os\n",
|
||
"from huggingface_hub import hf_hub_download\n",
|
||
"\n",
|
||
"# Function to read the token from a file\n",
|
||
"def read_token(file_path):\n",
|
||
" try:\n",
|
||
" with open(file_path, 'r') as file:\n",
|
||
" return file.readline().strip()\n",
|
||
" except FileNotFoundError:\n",
|
||
" raise ValueError(f\"Token file not found: {file_path}\")\n",
|
||
"\n",
|
||
"# Define the model name and file\n",
|
||
"model_name = \"cjpais/llava-1.6-mistral-7b-gguf\"\n",
|
||
"model_file = \"llava-v1.6-mistral-7b.Q3_K_XS.gguf\"\n",
|
||
"\n",
|
||
"# Download the model from Hugging Face Hub\n",
|
||
"model_path = hf_hub_download(\n",
|
||
" model_name,\n",
|
||
" filename=model_file,\n",
|
||
" local_dir='models/', # Download the model to the \"models\" folder\n",
|
||
" token=\"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\" #Replace this token from huggingface with your own token (Setting -> Access Toekns -> New token -> Generate Token)\n",
|
||
")\n",
|
||
"\n",
|
||
"print(\"My model path:\", model_path)"
|
||
],
|
||
"metadata": {
|
||
"id": "WsYot3Rz1NVf",
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 170,
|
||
"referenced_widgets": [
|
||
"a6a1102e92ea4d5c94292204e06cca1e",
|
||
"90a46b21037d4a8190978408a8bb9c9a",
|
||
"6ed04eacce9b498b878ff9bc251c8996",
|
||
"2cda1256799c497c861680a6b8f2d2af",
|
||
"106a87fddb73425195fbc80b513c760f",
|
||
"bc843b5871394a19b22c0f4c83568bbc",
|
||
"309cd58286e541bfbe153534187f4389",
|
||
"dea6f57c0c44452eb533313c717e2d64",
|
||
"7b783818c4214a1095dc6e3f213873dd",
|
||
"7cb8287280874268a621c900db3d9580",
|
||
"58ae8a4330aa4442abd5b7138d7d5c0b"
|
||
]
|
||
},
|
||
"outputId": "0cd48d6c-48bf-4184-ae7f-6005fe4b390d"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stderr",
|
||
"text": [
|
||
"/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
|
||
"The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
|
||
"To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
|
||
"You will be able to reuse this secret in all of your notebooks.\n",
|
||
"Please note that authentication is recommended but still optional to access public models or datasets.\n",
|
||
" warnings.warn(\n"
|
||
]
|
||
},
|
||
{
|
||
"output_type": "display_data",
|
||
"data": {
|
||
"text/plain": [
|
||
"llava-v1.6-mistral-7b.Q3_K_XS.gguf: 0%| | 0.00/2.99G [00:00<?, ?B/s]"
|
||
],
|
||
"application/vnd.jupyter.widget-view+json": {
|
||
"version_major": 2,
|
||
"version_minor": 0,
|
||
"model_id": "a6a1102e92ea4d5c94292204e06cca1e"
|
||
}
|
||
},
|
||
"metadata": {}
|
||
},
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stdout",
|
||
"text": [
|
||
"My model path: models/llava-v1.6-mistral-7b.Q3_K_XS.gguf\n"
|
||
]
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"from llama_cpp import Llama\n",
|
||
"\n",
|
||
"# Put the location of to the GGUF model that you've download from HuggingFace here\n",
|
||
"model_path = \"/content/models/llava-v1.6-mistral-7b.Q3_K_XS.gguf\"\n",
|
||
"llm = Llama(model_path=model_path) #load the model"
|
||
],
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "w2Y9Vn_nbFDM",
|
||
"outputId": "da81959e-1242-4759-f190-6b87866bfb4d"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stderr",
|
||
"text": [
|
||
"AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n"
|
||
]
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
"Note that BLAS = 0 means GPU is not enabled:\n",
|
||
"* AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 |"
|
||
],
|
||
"metadata": {
|
||
"id": "L00ib5gJBXKZ"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"source": [
|
||
"## Step 4: Ask the LLM a question"
|
||
],
|
||
"metadata": {
|
||
"id": "T-bFFV-hCzGC"
|
||
}
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"user_question = \"What is the earliest civilization in the world?\" # @param {type:\"string\"}"
|
||
],
|
||
"metadata": {
|
||
"id": "AD7MiXnRl1Kr"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"source": [
|
||
"import time\n",
|
||
"\n",
|
||
"# Prompt creation\n",
|
||
"system_message = \"You are a helpful assistant\"\n",
|
||
"user_message = \"Q: \"+ user_question+ \" A: \"\n",
|
||
"\n",
|
||
"# Start the timer\n",
|
||
"start_time = time.time()\n",
|
||
"\n",
|
||
"prompt = f\"\"\"<s>[INST] <<SYS>>\n",
|
||
"{system_message}\n",
|
||
"<</SYS>>\n",
|
||
"{user_message} [/INST]\"\"\"\n",
|
||
"\n",
|
||
"# Run the model (make inference)\n",
|
||
"output = llm(\n",
|
||
" prompt, # Prompt\n",
|
||
" max_tokens=2000, # Generate up to 2,000 tokens\n",
|
||
" stop=[\"Q:\", \"\\n\"], # Stop generating just before the model would generate a new question\n",
|
||
" #echo=True # Echo the prompt back in the output\n",
|
||
")\n",
|
||
"\n",
|
||
"# Stop the timer\n",
|
||
"end_time = time.time()\n",
|
||
"\n",
|
||
"# Get model response\n",
|
||
"print(output[\"choices\"][0][\"text\"])\n",
|
||
"\n",
|
||
"# Calculate runtime\n",
|
||
"runtime = end_time - start_time\n",
|
||
"print(\"response run time is: \", runtime) # the same inference takes 2 seconds with GPU on Google Colab"
|
||
],
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"id": "IRVk46wSBZlp",
|
||
"outputId": "b8e2df4d-ff9c-4fb3-ddbb-2fd0cfa8c041"
|
||
},
|
||
"execution_count": null,
|
||
"outputs": [
|
||
{
|
||
"output_type": "stream",
|
||
"name": "stdout",
|
||
"text": [
|
||
" The earliest civilization in the world is believed to be the Sumerians of Mesopotamia. They were known for their advanced use of writing, as well as their development of various aspects of human life such as agriculture, trade, religion, and art.\n",
|
||
"response run time is: 67.1243085861206\n"
|
||
]
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"accelerator": "GPU",
|
||
"colab": {
|
||
"machine_shape": "hm",
|
||
"provenance": [],
|
||
"gpuClass": "premium",
|
||
"gpuType": "T4",
|
||
"toc_visible": true,
|
||
"include_colab_link": true
|
||
},
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"name": "python"
|
||
},
|
||
"widgets": {
|
||
"application/vnd.jupyter.widget-state+json": {
|
||
"a6a1102e92ea4d5c94292204e06cca1e": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "HBoxModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "HBoxModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "HBoxView",
|
||
"box_style": "",
|
||
"children": [
|
||
"IPY_MODEL_90a46b21037d4a8190978408a8bb9c9a",
|
||
"IPY_MODEL_6ed04eacce9b498b878ff9bc251c8996",
|
||
"IPY_MODEL_2cda1256799c497c861680a6b8f2d2af"
|
||
],
|
||
"layout": "IPY_MODEL_106a87fddb73425195fbc80b513c760f"
|
||
}
|
||
},
|
||
"90a46b21037d4a8190978408a8bb9c9a": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "HTMLModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "HTMLModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "HTMLView",
|
||
"description": "",
|
||
"description_tooltip": null,
|
||
"layout": "IPY_MODEL_bc843b5871394a19b22c0f4c83568bbc",
|
||
"placeholder": "",
|
||
"style": "IPY_MODEL_309cd58286e541bfbe153534187f4389",
|
||
"value": "llava-v1.6-mistral-7b.Q3_K_XS.gguf: 100%"
|
||
}
|
||
},
|
||
"6ed04eacce9b498b878ff9bc251c8996": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "FloatProgressModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "FloatProgressModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "ProgressView",
|
||
"bar_style": "success",
|
||
"description": "",
|
||
"description_tooltip": null,
|
||
"layout": "IPY_MODEL_dea6f57c0c44452eb533313c717e2d64",
|
||
"max": 2991290624,
|
||
"min": 0,
|
||
"orientation": "horizontal",
|
||
"style": "IPY_MODEL_7b783818c4214a1095dc6e3f213873dd",
|
||
"value": 2991290624
|
||
}
|
||
},
|
||
"2cda1256799c497c861680a6b8f2d2af": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "HTMLModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "HTMLModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "HTMLView",
|
||
"description": "",
|
||
"description_tooltip": null,
|
||
"layout": "IPY_MODEL_7cb8287280874268a621c900db3d9580",
|
||
"placeholder": "",
|
||
"style": "IPY_MODEL_58ae8a4330aa4442abd5b7138d7d5c0b",
|
||
"value": " 2.99G/2.99G [00:26<00:00, 180MB/s]"
|
||
}
|
||
},
|
||
"106a87fddb73425195fbc80b513c760f": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_name": "LayoutModel",
|
||
"model_module_version": "1.2.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"bc843b5871394a19b22c0f4c83568bbc": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_name": "LayoutModel",
|
||
"model_module_version": "1.2.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"309cd58286e541bfbe153534187f4389": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "DescriptionStyleModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "DescriptionStyleModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "StyleView",
|
||
"description_width": ""
|
||
}
|
||
},
|
||
"dea6f57c0c44452eb533313c717e2d64": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_name": "LayoutModel",
|
||
"model_module_version": "1.2.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"7b783818c4214a1095dc6e3f213873dd": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "ProgressStyleModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "ProgressStyleModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "StyleView",
|
||
"bar_color": null,
|
||
"description_width": ""
|
||
}
|
||
},
|
||
"7cb8287280874268a621c900db3d9580": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_name": "LayoutModel",
|
||
"model_module_version": "1.2.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"58ae8a4330aa4442abd5b7138d7d5c0b": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_name": "DescriptionStyleModel",
|
||
"model_module_version": "1.5.0",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "DescriptionStyleModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "StyleView",
|
||
"description_width": ""
|
||
}
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 0
|
||
} |