From faab67eb4f0f65d7f02715785fb85dbab714c8d4 Mon Sep 17 00:00:00 2001 From: rhinophant Date: Thu, 13 Jun 2024 23:33:25 -0400 Subject: [PATCH] incomplete- work-in-progress --- setup_ollma_google_colab.ipynb | 1034 ++++++++++++++++++++++++++++++++ 1 file changed, 1034 insertions(+) create mode 100644 setup_ollma_google_colab.ipynb diff --git a/setup_ollma_google_colab.ipynb b/setup_ollma_google_colab.ipynb new file mode 100644 index 0000000..db20ded --- /dev/null +++ b/setup_ollma_google_colab.ipynb @@ -0,0 +1,1034 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Host LLM on colab (with ollama) with port forwarding to public internet + access colab-hosted model via ollama API" + ], + "metadata": { + "id": "CpsFIe6TlKyS" + } + }, + { + "cell_type": "code", + "source": [ + "from google.colab import userdata #user to access token" + ], + "metadata": { + "id": "S5nNTL-8nN28" + }, + "execution_count": 1, + "outputs": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EuaQsW4eTp0E", + "outputId": "eb3673b5-1cd5-48f7-dd21-2f98b9d9d739" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + " % Total % Received % Xferd Average Speed Time Time Time Current\n", + " Dload Upload Total Spent Left Speed\n", + " 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0>>> Downloading ollama...\n", + "100 10941 0 10941 0 0 25815 0 --:--:-- --:--:-- --:--:-- 25865\n", + "############################################################################################# 100.0%\n", + ">>> Installing ollama to /usr/local/bin...\n", + ">>> Adding ollama user to video group...\n", + ">>> Adding current user to ollama group...\n", + ">>> Creating ollama systemd service...\n", + "WARNING: Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies.\n", + ">>> The Ollama API is now available at 127.0.0.1:11434.\n", + ">>> Install complete. Run \"ollama\" from the command line.\n", + "Get:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n", + "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease\n", + "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n", + "Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n", + "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n", + "Hit:6 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n", + "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n", + "Hit:8 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n", + "Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n", + "Get:10 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,090 kB]\n", + "Hit:11 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n", + "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,391 kB]\n", + "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,183 kB]\n", + "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,542 kB]\n", + "Get:15 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,908 kB]\n", + "Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,464 kB]\n", + "Fetched 11.8 MB in 12s (990 kB/s)\n", + "Reading package lists... Done\n", + "Reading package lists... Done\n", + "Building dependency tree... Done\n", + "Reading state information... Done\n", + "The following additional packages will be installed:\n", + " cpp-12 cuda-drivers-555 dctrl-tools dkms fakeroot gcc-12\n", + " keyboard-configuration libasan8 libfakeroot libfontenc1 libgcc-12-dev\n", + " libjansson4 liblocale-gettext-perl libnvidia-cfg1-555 libnvidia-common-555\n", + " libnvidia-compute-555 libnvidia-decode-555 libnvidia-encode-555\n", + " libnvidia-extra-555 libnvidia-fbc1-555 libnvidia-gl-555 libtsan2 libudev1\n", + " libxcvt0 libxfont2 libxkbfile1 nvidia-compute-utils-555 nvidia-dkms-555\n", + " nvidia-driver-555 nvidia-firmware-555-555.42.02 nvidia-kernel-common-555\n", + " nvidia-kernel-source-555 nvidia-prime nvidia-settings nvidia-utils-555\n", + " python3-xkit screen-resolution-extra systemd-hwe-hwdb udev x11-xkb-utils\n", + " xcvt xfonts-base xfonts-encodings xfonts-utils xserver-common\n", + " xserver-xorg-core xserver-xorg-video-nvidia-555\n", + "Suggested packages:\n", + " gcc-12-locales cpp-12-doc debtags menu gcc-12-multilib gcc-12-doc xfs\n", + " | xserver xfonts-100dpi | xfonts-75dpi xfonts-scalable\n", + "Recommended packages:\n", + " libnvidia-compute-555:i386 libnvidia-decode-555:i386\n", + " libnvidia-encode-555:i386 libnvidia-fbc1-555:i386 libnvidia-gl-555:i386\n", + "The following NEW packages will be installed:\n", + " cpp-12 cuda-drivers cuda-drivers-555 dctrl-tools dkms fakeroot gcc-12\n", + " keyboard-configuration libasan8 libfakeroot libfontenc1 libgcc-12-dev\n", + " libjansson4 liblocale-gettext-perl libnvidia-cfg1-555 libnvidia-common-555\n", + " libnvidia-compute-555 libnvidia-decode-555 libnvidia-encode-555\n", + " libnvidia-extra-555 libnvidia-fbc1-555 libnvidia-gl-555 libtsan2 libxcvt0\n", + " libxfont2 libxkbfile1 nvidia-compute-utils-555 nvidia-dkms-555\n", + " nvidia-driver-555 nvidia-firmware-555-555.42.02 nvidia-kernel-common-555\n", + " nvidia-kernel-source-555 nvidia-prime nvidia-settings nvidia-utils-555\n", + " python3-xkit screen-resolution-extra systemd-hwe-hwdb udev x11-xkb-utils\n", + " xcvt xfonts-base xfonts-encodings xfonts-utils xserver-common\n", + " xserver-xorg-core xserver-xorg-video-nvidia-555\n", + "The following packages will be upgraded:\n", + " libudev1\n", + "1 upgraded, 47 newly installed, 0 to remove and 44 not upgraded.\n", + "Need to get 319 MB of archives.\n", + "After this operation, 910 MB of additional disk space will be used.\n", + "Get:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-common-555 555.42.02-0ubuntu1 [17.2 kB]\n", + "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-compute-555 555.42.02-0ubuntu1 [46.9 MB]\n", + "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblocale-gettext-perl amd64 1.07-4build3 [17.1 kB]\n", + "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 keyboard-configuration all 1.205ubuntu3 [206 kB]\n", + "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-gl-555 555.42.02-0ubuntu1 [138 MB]\n", + "Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 cpp-12 amd64 12.3.0-1ubuntu1~22.04 [10.8 MB]\n", + "Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libasan8 amd64 12.3.0-1ubuntu1~22.04 [2,442 kB]\n", + "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libtsan2 amd64 12.3.0-1ubuntu1~22.04 [2,477 kB]\n", + "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgcc-12-dev amd64 12.3.0-1ubuntu1~22.04 [2,618 kB]\n", + "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 gcc-12 amd64 12.3.0-1ubuntu1~22.04 [21.7 MB]\n", + "Get:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-kernel-source-555 555.42.02-0ubuntu1 [41.4 MB]\n", + "Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 dctrl-tools amd64 2.24-3build2 [66.9 kB]\n", + "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dkms all 2.8.7-2ubuntu2.2 [70.1 kB]\n", + "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libudev1 amd64 249.11-0ubuntu3.12 [78.2 kB]\n", + "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 udev amd64 249.11-0ubuntu3.12 [1,557 kB]\n", + "Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjansson4 amd64 2.13.1-1.1build3 [32.4 kB]\n", + "Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxkbfile1 amd64 1:1.1.0-1build3 [71.8 kB]\n", + "Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 x11-xkb-utils amd64 7.7+5build4 [172 kB]\n", + "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 xserver-common all 2:21.1.4-2ubuntu1.7~22.04.10 [28.5 kB]\n", + "Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxcvt0 amd64 0.1.1-3 [5,494 B]\n", + "Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfontenc1 amd64 1:1.1.4-1build3 [14.7 kB]\n", + "Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxfont2 amd64 1:2.0.5-1build1 [94.5 kB]\n", + "Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 xserver-xorg-core amd64 2:21.1.4-2ubuntu1.7~22.04.10 [1,476 kB]\n", + "Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfakeroot amd64 1.28-1ubuntu1 [31.5 kB]\n", + "Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 fakeroot amd64 1.28-1ubuntu1 [60.4 kB]\n", + "Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 nvidia-prime all 0.8.17.1 [9,956 B]\n", + "Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-xkit all 0.5.0ubuntu5 [18.5 kB]\n", + "Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 screen-resolution-extra all 0.18.2 [4,396 B]\n", + "Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 systemd-hwe-hwdb all 249.11.5 [3,228 B]\n", + "Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 xcvt amd64 0.1.1-3 [7,140 B]\n", + "Get:31 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-firmware-555-555.42.02 555.42.02-0ubuntu1 [36.5 MB]\n", + "Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 xfonts-encodings all 1:1.0.5-0ubuntu2 [578 kB]\n", + "Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 xfonts-utils amd64 1:7.7+6build2 [94.6 kB]\n", + "Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 xfonts-base all 1:1.0.5 [5,896 kB]\n", + "Get:35 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-kernel-common-555 555.42.02-0ubuntu1 [109 kB]\n", + "Get:36 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-dkms-555 555.42.02-0ubuntu1 [36.2 kB]\n", + "Get:37 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-extra-555 555.42.02-0ubuntu1 [73.4 kB]\n", + "Get:38 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-compute-utils-555 555.42.02-0ubuntu1 [118 kB]\n", + "Get:39 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-decode-555 555.42.02-0ubuntu1 [1,792 kB]\n", + "Get:40 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-encode-555 555.42.02-0ubuntu1 [104 kB]\n", + "Get:41 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-utils-555 555.42.02-0ubuntu1 [495 kB]\n", + "Get:42 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-cfg1-555 555.42.02-0ubuntu1 [146 kB]\n", + "Get:43 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 xserver-xorg-video-nvidia-555 555.42.02-0ubuntu1 [1,534 kB]\n", + "Get:44 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-fbc1-555 555.42.02-0ubuntu1 [75.3 kB]\n", + "Get:45 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-driver-555 555.42.02-0ubuntu1 [490 kB]\n", + "Get:46 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 cuda-drivers-555 555.42.02-1 [2,542 B]\n", + "Get:47 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 cuda-drivers 555.42.02-1 [2,498 B]\n", + "Get:48 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-settings 555.42.02-0ubuntu1 [945 kB]\n", + "Fetched 319 MB in 4s (71.4 MB/s)\n", + "Extracting templates from packages: 100%\n", + "Preconfiguring packages ...\n", + "Selecting previously unselected package liblocale-gettext-perl.\n", + "(Reading database ... 121913 files and directories currently installed.)\n", + "Preparing to unpack .../0-liblocale-gettext-perl_1.07-4build3_amd64.deb ...\n", + "Unpacking liblocale-gettext-perl (1.07-4build3) ...\n", + "Selecting previously unselected package keyboard-configuration.\n", + "Preparing to unpack .../1-keyboard-configuration_1.205ubuntu3_all.deb ...\n", + "Unpacking keyboard-configuration (1.205ubuntu3) ...\n", + "Selecting previously unselected package cpp-12.\n", + "Preparing to unpack .../2-cpp-12_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", + "Unpacking cpp-12 (12.3.0-1ubuntu1~22.04) ...\n", + "Selecting previously unselected package libasan8:amd64.\n", + "Preparing to unpack .../3-libasan8_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", + "Unpacking libasan8:amd64 (12.3.0-1ubuntu1~22.04) ...\n", + "Selecting previously unselected package libtsan2:amd64.\n", + "Preparing to unpack .../4-libtsan2_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", + "Unpacking libtsan2:amd64 (12.3.0-1ubuntu1~22.04) ...\n", + "Selecting previously unselected package libgcc-12-dev:amd64.\n", + "Preparing to unpack .../5-libgcc-12-dev_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", + "Unpacking libgcc-12-dev:amd64 (12.3.0-1ubuntu1~22.04) ...\n", + "Selecting previously unselected package gcc-12.\n", + "Preparing to unpack .../6-gcc-12_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", + "Unpacking gcc-12 (12.3.0-1ubuntu1~22.04) ...\n", + "Selecting previously unselected package dctrl-tools.\n", + "Preparing to unpack .../7-dctrl-tools_2.24-3build2_amd64.deb ...\n", + "Unpacking dctrl-tools (2.24-3build2) ...\n", + "Selecting previously unselected package dkms.\n", + "Preparing to unpack .../8-dkms_2.8.7-2ubuntu2.2_all.deb ...\n", + "Unpacking dkms (2.8.7-2ubuntu2.2) ...\n", + "Preparing to unpack .../9-libudev1_249.11-0ubuntu3.12_amd64.deb ...\n", + "Unpacking libudev1:amd64 (249.11-0ubuntu3.12) over (249.11-0ubuntu3.10) ...\n", + "Setting up libudev1:amd64 (249.11-0ubuntu3.12) ...\n", + "Selecting previously unselected package udev.\n", + "(Reading database ... 122273 files and directories currently installed.)\n", + "Preparing to unpack .../00-udev_249.11-0ubuntu3.12_amd64.deb ...\n", + "Unpacking udev (249.11-0ubuntu3.12) ...\n", + "Selecting previously unselected package libjansson4:amd64.\n", + "Preparing to unpack .../01-libjansson4_2.13.1-1.1build3_amd64.deb ...\n", + "Unpacking libjansson4:amd64 (2.13.1-1.1build3) ...\n", + "Selecting previously unselected package libnvidia-common-555.\n", + "Preparing to unpack .../02-libnvidia-common-555_555.42.02-0ubuntu1_all.deb ...\n", + "Unpacking libnvidia-common-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-compute-555:amd64.\n", + "Preparing to unpack .../03-libnvidia-compute-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking libnvidia-compute-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-gl-555:amd64.\n", + "Preparing to unpack .../04-libnvidia-gl-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "dpkg-query: no packages found matching libnvidia-gl-535\n", + "Unpacking libnvidia-gl-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-kernel-source-555.\n", + "Preparing to unpack .../05-nvidia-kernel-source-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-kernel-source-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-firmware-555-555.42.02.\n", + "Preparing to unpack .../06-nvidia-firmware-555-555.42.02_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-firmware-555-555.42.02 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-kernel-common-555.\n", + "Preparing to unpack .../07-nvidia-kernel-common-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-kernel-common-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-dkms-555.\n", + "Preparing to unpack .../08-nvidia-dkms-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-dkms-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-extra-555:amd64.\n", + "Preparing to unpack .../09-libnvidia-extra-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking libnvidia-extra-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-compute-utils-555.\n", + "Preparing to unpack .../10-nvidia-compute-utils-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-compute-utils-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-decode-555:amd64.\n", + "Preparing to unpack .../11-libnvidia-decode-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking libnvidia-decode-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-encode-555:amd64.\n", + "Preparing to unpack .../12-libnvidia-encode-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking libnvidia-encode-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-utils-555.\n", + "Preparing to unpack .../13-nvidia-utils-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-utils-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-cfg1-555:amd64.\n", + "Preparing to unpack .../14-libnvidia-cfg1-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking libnvidia-cfg1-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libxkbfile1:amd64.\n", + "Preparing to unpack .../15-libxkbfile1_1%3a1.1.0-1build3_amd64.deb ...\n", + "Unpacking libxkbfile1:amd64 (1:1.1.0-1build3) ...\n", + "Selecting previously unselected package x11-xkb-utils.\n", + "Preparing to unpack .../16-x11-xkb-utils_7.7+5build4_amd64.deb ...\n", + "Unpacking x11-xkb-utils (7.7+5build4) ...\n", + "Selecting previously unselected package xserver-common.\n", + "Preparing to unpack .../17-xserver-common_2%3a21.1.4-2ubuntu1.7~22.04.10_all.deb ...\n", + "Unpacking xserver-common (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", + "Selecting previously unselected package libxcvt0:amd64.\n", + "Preparing to unpack .../18-libxcvt0_0.1.1-3_amd64.deb ...\n", + "Unpacking libxcvt0:amd64 (0.1.1-3) ...\n", + "Selecting previously unselected package libfontenc1:amd64.\n", + "Preparing to unpack .../19-libfontenc1_1%3a1.1.4-1build3_amd64.deb ...\n", + "Unpacking libfontenc1:amd64 (1:1.1.4-1build3) ...\n", + "Selecting previously unselected package libxfont2:amd64.\n", + "Preparing to unpack .../20-libxfont2_1%3a2.0.5-1build1_amd64.deb ...\n", + "Unpacking libxfont2:amd64 (1:2.0.5-1build1) ...\n", + "Selecting previously unselected package xserver-xorg-core.\n", + "Preparing to unpack .../21-xserver-xorg-core_2%3a21.1.4-2ubuntu1.7~22.04.10_amd64.deb ...\n", + "Unpacking xserver-xorg-core (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", + "Selecting previously unselected package xserver-xorg-video-nvidia-555.\n", + "Preparing to unpack .../22-xserver-xorg-video-nvidia-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking xserver-xorg-video-nvidia-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package libnvidia-fbc1-555:amd64.\n", + "Preparing to unpack .../23-libnvidia-fbc1-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking libnvidia-fbc1-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package nvidia-driver-555.\n", + "Preparing to unpack .../24-nvidia-driver-555_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-driver-555 (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package cuda-drivers-555.\n", + "Preparing to unpack .../25-cuda-drivers-555_555.42.02-1_amd64.deb ...\n", + "Unpacking cuda-drivers-555 (555.42.02-1) ...\n", + "Selecting previously unselected package cuda-drivers.\n", + "Preparing to unpack .../26-cuda-drivers_555.42.02-1_amd64.deb ...\n", + "Unpacking cuda-drivers (555.42.02-1) ...\n", + "Selecting previously unselected package libfakeroot:amd64.\n", + "Preparing to unpack .../27-libfakeroot_1.28-1ubuntu1_amd64.deb ...\n", + "Unpacking libfakeroot:amd64 (1.28-1ubuntu1) ...\n", + "Selecting previously unselected package fakeroot.\n", + "Preparing to unpack .../28-fakeroot_1.28-1ubuntu1_amd64.deb ...\n", + "Unpacking fakeroot (1.28-1ubuntu1) ...\n", + "Selecting previously unselected package nvidia-prime.\n", + "Preparing to unpack .../29-nvidia-prime_0.8.17.1_all.deb ...\n", + "Unpacking nvidia-prime (0.8.17.1) ...\n", + "Selecting previously unselected package python3-xkit.\n", + "Preparing to unpack .../30-python3-xkit_0.5.0ubuntu5_all.deb ...\n", + "Unpacking python3-xkit (0.5.0ubuntu5) ...\n", + "Selecting previously unselected package screen-resolution-extra.\n", + "Preparing to unpack .../31-screen-resolution-extra_0.18.2_all.deb ...\n", + "Unpacking screen-resolution-extra (0.18.2) ...\n", + "Selecting previously unselected package nvidia-settings.\n", + "Preparing to unpack .../32-nvidia-settings_555.42.02-0ubuntu1_amd64.deb ...\n", + "Unpacking nvidia-settings (555.42.02-0ubuntu1) ...\n", + "Selecting previously unselected package systemd-hwe-hwdb.\n", + "Preparing to unpack .../33-systemd-hwe-hwdb_249.11.5_all.deb ...\n", + "Unpacking systemd-hwe-hwdb (249.11.5) ...\n", + "Selecting previously unselected package xcvt.\n", + "Preparing to unpack .../34-xcvt_0.1.1-3_amd64.deb ...\n", + "Unpacking xcvt (0.1.1-3) ...\n", + "Selecting previously unselected package xfonts-encodings.\n", + "Preparing to unpack .../35-xfonts-encodings_1%3a1.0.5-0ubuntu2_all.deb ...\n", + "Unpacking xfonts-encodings (1:1.0.5-0ubuntu2) ...\n", + "Selecting previously unselected package xfonts-utils.\n", + "Preparing to unpack .../36-xfonts-utils_1%3a7.7+6build2_amd64.deb ...\n", + "Unpacking xfonts-utils (1:7.7+6build2) ...\n", + "Selecting previously unselected package xfonts-base.\n", + "Preparing to unpack .../37-xfonts-base_1%3a1.0.5_all.deb ...\n", + "Unpacking xfonts-base (1:1.0.5) ...\n", + "Setting up libnvidia-common-555 (555.42.02-0ubuntu1) ...\n", + "Setting up libnvidia-fbc1-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up cpp-12 (12.3.0-1ubuntu1~22.04) ...\n", + "Setting up libnvidia-cfg1-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up nvidia-prime (0.8.17.1) ...\n", + "Setting up libfakeroot:amd64 (1.28-1ubuntu1) ...\n", + "Setting up libjansson4:amd64 (2.13.1-1.1build3) ...\n", + "Setting up libnvidia-extra-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up fakeroot (1.28-1ubuntu1) ...\n", + "update-alternatives: using /usr/bin/fakeroot-sysv to provide /usr/bin/fakeroot (fakeroot) in auto mode\n", + "Setting up libfontenc1:amd64 (1:1.1.4-1build3) ...\n", + "Setting up libnvidia-compute-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up xfonts-encodings (1:1.0.5-0ubuntu2) ...\n", + "Setting up udev (249.11-0ubuntu3.12) ...\n", + "invoke-rc.d: could not determine current runlevel\n", + "invoke-rc.d: policy-rc.d denied execution of start.\n", + "Setting up systemd-hwe-hwdb (249.11.5) ...\n", + "Setting up libasan8:amd64 (12.3.0-1ubuntu1~22.04) ...\n", + "Setting up nvidia-firmware-555-555.42.02 (555.42.02-0ubuntu1) ...\n", + "Setting up libxcvt0:amd64 (0.1.1-3) ...\n", + "Setting up libxkbfile1:amd64 (1:1.1.0-1build3) ...\n", + "Setting up libtsan2:amd64 (12.3.0-1ubuntu1~22.04) ...\n", + "Setting up python3-xkit (0.5.0ubuntu5) ...\n", + "Setting up libxfont2:amd64 (1:2.0.5-1build1) ...\n", + "Setting up liblocale-gettext-perl (1.07-4build3) ...\n", + "Setting up dctrl-tools (2.24-3build2) ...\n", + "Setting up nvidia-kernel-source-555 (555.42.02-0ubuntu1) ...\n", + "Setting up nvidia-utils-555 (555.42.02-0ubuntu1) ...\n", + "Setting up nvidia-compute-utils-555 (555.42.02-0ubuntu1) ...\n", + "Warning: The home dir /nonexistent you specified can't be accessed: No such file or directory\n", + "Adding system user `nvidia-persistenced' (UID 104) ...\n", + "Adding new group `nvidia-persistenced' (GID 111) ...\n", + "Adding new user `nvidia-persistenced' (UID 104) with group `nvidia-persistenced' ...\n", + "Not creating home directory `/nonexistent'.\n", + "Setting up x11-xkb-utils (7.7+5build4) ...\n", + "Setting up libnvidia-decode-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up xfonts-utils (1:7.7+6build2) ...\n", + "Setting up nvidia-kernel-common-555 (555.42.02-0ubuntu1) ...\n", + "Created symlink /etc/systemd/system/systemd-hibernate.service.wants/nvidia-hibernate.service → /lib/systemd/system/nvidia-hibernate.service.\n", + "Created symlink /etc/systemd/system/systemd-suspend.service.wants/nvidia-resume.service → /lib/systemd/system/nvidia-resume.service.\n", + "Created symlink /etc/systemd/system/systemd-hibernate.service.wants/nvidia-resume.service → /lib/systemd/system/nvidia-resume.service.\n", + "Created symlink /etc/systemd/system/systemd-suspend.service.wants/nvidia-suspend.service → /lib/systemd/system/nvidia-suspend.service.\n", + "Setting up xcvt (0.1.1-3) ...\n", + "Setting up xfonts-base (1:1.0.5) ...\n", + "Setting up libnvidia-gl-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up libgcc-12-dev:amd64 (12.3.0-1ubuntu1~22.04) ...\n", + "Setting up screen-resolution-extra (0.18.2) ...\n", + "Setting up nvidia-settings (555.42.02-0ubuntu1) ...\n", + "Setting up xserver-common (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", + "Setting up keyboard-configuration (1.205ubuntu3) ...\n", + "Your console font configuration will be updated the next time your system\n", + "boots. If you want to update it now, run 'setupcon' from a virtual console.\n", + "Setting up libnvidia-encode-555:amd64 (555.42.02-0ubuntu1) ...\n", + "Setting up xserver-xorg-core (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", + "Setting up gcc-12 (12.3.0-1ubuntu1~22.04) ...\n", + "Setting up xserver-xorg-video-nvidia-555 (555.42.02-0ubuntu1) ...\n", + "Setting up dkms (2.8.7-2ubuntu2.2) ...\n", + "Setting up nvidia-dkms-555 (555.42.02-0ubuntu1) ...\n", + "Loading new nvidia-555.42.02 DKMS files...\n", + "It is likely that 6.1.85+ belongs to a chroot's host\n", + "Building for 5.15.0-112-generic\n", + "Building for architecture x86_64\n", + "Building initial module for 5.15.0-112-generic\n" + ] + } + ], + "source": [ + "!curl https://ollama.ai/install.sh | sh\n", + "\n", + "!echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections\n", + "!sudo apt-get update && sudo apt-get install -y cuda-drivers\n", + "\n", + "!pip install pyngrok\n", + "from pyngrok import ngrok\n", + "ngrok.set_auth_token(userdata.get('ngrok_token')) #secret saved in a variable \"ngrok_token\" in colab \"Secrets\"\n", + "\n", + "import os\n", + "import asyncio\n", + "\n", + "# Set LD_LIBRARY_PATH so the system NVIDIA library\n", + "os.environ.update({'LD_LIBRARY_PATH': '/usr/lib64-nvidia'})\n", + "\n", + "# Function to run a specified command as a subprocess\n", + "async def run_process(cmd):\n", + " # Print the command that is starting\n", + " print('>>> starting', *cmd)\n", + "\n", + " # Start the subprocess and redirect stdout and stderr to asyncio subprocess pipes\n", + " p = await asyncio.create_subprocess_exec(\n", + " *cmd,\n", + " stdout=asyncio.subprocess.PIPE, # Capture standard output\n", + " stderr=asyncio.subprocess.PIPE # Capture standard errors\n", + " )\n", + "\n", + " # Function to handle the output from stdout and stderr\n", + " async def pipe(lines):\n", + " async for line in lines:\n", + " # Decode and strip each line of output and print it\n", + " print(line.decode('utf-8').strip())\n", + "\n", + " # Concurrently process both stdout and stderr\n", + " await asyncio.gather(\n", + " pipe(p.stdout),\n", + " pipe(p.stderr),\n", + " )\n", + "\n", + "# Clear the output of the current Jupyter notebook cell\n", + "clear_output()\n", + "\n", + "# This coroutine runs two subprocesses: one for the Ollama server and one for the ngrok tunnel\n", + "async def main():\n", + " await asyncio.gather(\n", + " run_process(['ollama', 'serve']), # Command to start the Ollama server\n", + " run_process(['ngrok', 'http', '--log', 'stderr', '11434']), # Command to start ngrok to expose port 11434\n", + " )\n", + "\n", + "# Run the main function in the asyncio event loop\n", + "await main()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Command prompt on local laptops' cmd\n", + "\n", + "```\n", + "curl https://ollama.ai/install.sh | sh\n", + "export OLLAMA_HOST=(https://fe64-34-106-204-48.ngrok-free.app/)\n", + "ollama run dolphin-mistral\n", + "```\n", + "\n", + "\n", + "\n" + ], + "metadata": { + "id": "SC6KKCLDmAVz" + } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9s2YZFcuv9tr" + }, + "source": [ + "# Host LLM on colab (with ollama) and access the LLM on colab with ollama API" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wxTw5NwVv9Jv", + "outputId": "e620ac3d-5c6c-4149-cbb5-cebe3ea2e532" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + ">>> Downloading ollama...\n", + "############################################################################################# 100.0%\n", + ">>> Installing ollama to /usr/local/bin...\n", + ">>> Creating ollama user...\n", + ">>> Adding ollama user to video group...\n", + ">>> Adding current user to ollama group...\n", + ">>> Creating ollama systemd service...\n", + "WARNING: Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies.\n", + ">>> The Ollama API is now available at 127.0.0.1:11434.\n", + ">>> Install complete. Run \"ollama\" from the command line.\n" + ] + } + ], + "source": [ + "# Install ollama: The Ollama API is now available at 127.0.0.1:11434\n", + "! curl -fsSL https://ollama.com/install.sh | sh" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "4tXmemohSdSU" + }, + "outputs": [], + "source": [ + "#!curl -X POST http://127.0.0.1:11434/api/generate -d '{\"model\": \"llama3\", \"prompt\": \"Why is the sky blue?\"}'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "J9NjjJnzEoWT", + "outputId": "867a39f2-a05d-4e4a-b451-5458b93fb859" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "curl: (7) Failed to connect to localhost port 11434 after 0 ms: Connection refused\n" + ] + } + ], + "source": [ + "#!curl -X POST http://localhost:11434/api/generate -d '{\"model\": \"llama3\", \"prompt\": \"Why is the sky blue?\"}'" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kS6hBI7Q_2IM", + "outputId": "27b3a5e2-3cf2-4c9c-db0f-0cf0a231171f" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: ngrok in /usr/local/lib/python3.10/dist-packages (1.3.0)\n", + "Requirement already satisfied: pyngrok==4.1.1 in /usr/local/lib/python3.10/dist-packages (4.1.1)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from pyngrok==4.1.1) (0.18.3)\n", + "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from pyngrok==4.1.1) (6.0.1)\n" + ] + } + ], + "source": [ + "!pip install ngrok\n", + "!pip install pyngrok==4.1.1" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "_2YY9CFk-v6z" + }, + "outputs": [], + "source": [ + "from google.colab import userdata #user to access token\n", + "\n", + "# Get your ngrok token from your ngrok account:\n", + "# https://dashboard.ngrok.com/get-started/your-authtoken\n", + "import threading\n", + "import time\n", + "import os\n", + "import asyncio\n", + "from pyngrok import ngrok\n", + "import threading\n", + "import queue\n", + "import time\n", + "from threading import Thread\n", + "\n", + "# Get your ngrok token from your ngrok account:\n", + "# https://dashboard.ngrok.com/get-started/your-authtoken\n", + "token= userdata.get('ngrok_token')\n", + "ngrok.set_auth_token(token)\n", + "\n", + "# set up a stoppable thread (not mandatory, but cleaner if you want to stop this later\n", + "class StoppableThread(threading.Thread):\n", + " def __init__(self, *args, **kwargs):\n", + " super(StoppableThread, self).__init__(*args, **kwargs)\n", + " self._stop_event = threading.Event()\n", + "\n", + " def stop(self):\n", + " self._stop_event.set()\n", + "\n", + " def is_stopped(self):\n", + " return self._stop_event.is_set()\n", + "\n", + "def start_ngrok(q, stop_event):\n", + " try:\n", + " # Start an HTTP tunnel on the specified port\n", + " public_url = ngrok.connect(11434)\n", + " # Put the public URL in the queue\n", + " q.put(public_url)\n", + " # Keep the thread alive until stop event is set\n", + " while not stop_event.is_set():\n", + " time.sleep(1) # Adjust sleep time as needed\n", + " except Exception as e:\n", + " print(f\"Error in start_ngrok: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "id": "ZuZPqxR_ATwr" + }, + "outputs": [], + "source": [ + "# Create a queue to share data between threads\n", + "url_queue = queue.Queue()\n", + "\n", + "# Start ngrok in a separate thread\n", + "ngrok_thread = StoppableThread(target=start_ngrok, args=(url_queue, StoppableThread.is_stopped))\n", + "ngrok_thread.start()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 356 + }, + "id": "nrlUY7yLAWef", + "outputId": "915fb13b-6e3f-496f-efd0-059b64f8c4ce" + }, + "outputs": [ + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mpublic_url\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murl_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpublic_url\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.10/queue.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m 169\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_qsize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_empty\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 172\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'timeout' must be a non-negative number\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 318\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 319\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 321\u001b[0m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "# Wait for the ngrok tunnel to be established\n", + "while True:\n", + " try:\n", + " public_url = url_queue.get()\n", + " if public_url:\n", + " break\n", + " print(\"Waiting for ngrok URL...\")\n", + " time.sleep(1)\n", + " except Exception as e:\n", + " print(f\"Error in retrieving ngrok URL: {e}\")\n", + "\n", + "print(\"Ngrok tunnel established at:\", public_url)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "T46-LFgWAdBO" + }, + "outputs": [], + "source": [ + "import os\n", + "import asyncio\n", + "\n", + "# NB: You may need to set these depending and get cuda working depending which backend you are running.\n", + "# Set environment variable for NVIDIA library\n", + "# Set environment variables for CUDA\n", + "os.environ['PATH'] += ':/usr/local/cuda/bin'\n", + "# Set LD_LIBRARY_PATH to include both /usr/lib64-nvidia and CUDA lib directories\n", + "os.environ['LD_LIBRARY_PATH'] = '/usr/lib64-nvidia:/usr/local/cuda/lib64'\n", + "\n", + "async def run_process(cmd):\n", + " print('>>> starting', *cmd)\n", + " process = await asyncio.create_subprocess_exec(\n", + " *cmd,\n", + " stdout=asyncio.subprocess.PIPE,\n", + " stderr=asyncio.subprocess.PIPE\n", + " )\n", + "\n", + " # define an async pipe function\n", + " async def pipe(lines):\n", + " async for line in lines:\n", + " print(line.decode().strip())\n", + "\n", + " await asyncio.gather(\n", + " pipe(process.stdout),\n", + " pipe(process.stderr),\n", + " )\n", + "\n", + " # call it\n", + " await asyncio.gather(pipe(process.stdout), pipe(process.stderr))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CkeaRb9RAgdG", + "outputId": "006bc391-246c-4ecd-cc1a-40d97f9f25fb" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + ">>> starting ollama serve\n" + ] + } + ], + "source": [ + "import asyncio\n", + "import threading\n", + "\n", + "async def start_ollama_serve():\n", + " await run_process(['ollama', 'serve'])\n", + "\n", + "def run_async_in_thread(loop, coro):\n", + " asyncio.set_event_loop(loop)\n", + " loop.run_until_complete(coro)\n", + " loop.close()\n", + "\n", + "# Create a new event loop that will run in a new thread\n", + "new_loop = asyncio.new_event_loop()\n", + "\n", + "# Start ollama serve in a separate thread so the cell won't block execution\n", + "thread = threading.Thread(target=run_async_in_thread, args=(new_loop, start_ollama_serve()))\n", + "thread.start()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "DrkNz68IAlJe" + }, + "outputs": [], + "source": [ + "! export OLLAMA_HOST=http://9126-34-106-204-48.ngrok-free.app/" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "esy-S7TjA5Sm", + "outputId": "14f364e5-b7b6-425a-c3d2-ab80e46b3f62" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Error: could not connect to ollama app, is it running?\n" + ] + } + ], + "source": [ + "!ollama pull llama2" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Method 1: ollama cmd" + ], + "metadata": { + "id": "KOv39QoawnLg" + } + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0U-22lppCpHS", + "outputId": "c39a2f10-07ba-4363-82d1-a181301a909c" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Error: could not connect to ollama app, is it running?\n" + ] + } + ], + "source": [ + "!ollama run llama2 #method 1" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Method 2: curl and API" + ], + "metadata": { + "id": "WixtlfV9wc_m" + } + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "id": "HW-grtw0AwH4" + }, + "outputs": [], + "source": [ + "!curl -X POST http://9126-34-106-204-48.ngrok-free.app/api/generate -d '{ \"model\": \"llama2\", \"prompt\":\"Good Morning\" , \"stream\": false }'" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Method 3: Python and API" + ], + "metadata": { + "id": "5ar5-iU2whrx" + } + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 844 + }, + "id": "0afNrMKXBhEg", + "outputId": "9b7dcc41-c368-4d65-9946-39349086606b" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "You: hi there. how are ya\n", + "Bot: Error: 404, \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + "\n", + "\n" + ] + }, + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "Interrupted by user", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;31m# Get user input for the prompt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mprompt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"You: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;31m# Check if the user wants to exit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 850\u001b[0m )\n\u001b[0;32m--> 851\u001b[0;31m return self._input_request(str(prompt),\n\u001b[0m\u001b[1;32m 852\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m \u001b[0;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Interrupted by user\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid Message:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user" + ] + } + ], + "source": [ + "# python code\n", + "\n", + "import requests\n", + "\n", + "# URL for the API endpoint\n", + "url = 'http://9126-34-106-204-48.ngrok-free.app//api/generate'\n", + "\n", + "# Function to make the POST request\n", + "def make_request(prompt):\n", + " data = {\n", + " \"model\": \"llama2\",\n", + " \"prompt\": prompt,\n", + " \"stream\": False\n", + " }\n", + " response = requests.post(url, json=data)\n", + " if response.status_code == 200:\n", + " return response.json().get(\"response\", \"No response received\")\n", + " else:\n", + " return f\"Error: {response.status_code}, {response.text}\"\n", + "\n", + "# Continuous chat loop\n", + "while True:\n", + " # Get user input for the prompt\n", + " prompt = input(\"You: \")\n", + "\n", + " # Check if the user wants to exit\n", + " if prompt.lower() == 'exit':\n", + " print(\"Exiting chat.\")\n", + " break\n", + "\n", + " # Make the request\n", + " response = make_request(prompt)\n", + "\n", + " # Print the response\n", + " print(\"Bot:\", response)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4jkbFk6CtwJ8", + "outputId": "2a7f8c4a-af1e-4576-8955-b7ea9fcdc719" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pipenv in /usr/local/lib/python3.10/dist-packages (2024.0.1)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from pipenv) (2024.6.2)\n", + "Requirement already satisfied: setuptools>=67 in /usr/local/lib/python3.10/dist-packages (from pipenv) (67.7.2)\n", + "Requirement already satisfied: virtualenv>=20.24.2 in /usr/local/lib/python3.10/dist-packages (from pipenv) (20.26.2)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.24.2->pipenv) (0.3.8)\n", + "Requirement already satisfied: filelock<4,>=3.12.2 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.24.2->pipenv) (3.14.0)\n", + "Requirement already satisfied: platformdirs<5,>=3.9.1 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.24.2->pipenv) (4.2.2)\n", + "To activate this project's virtualenv, run \u001b[33mpipenv shell\u001b[0m.\n", + "Alternatively, run a command inside the virtualenv with \u001b[33mpipenv run\u001b[0m.\n", + "To activate this project's virtualenv, run \u001b[33mpipenv shell\u001b[0m.\n", + "Alternatively, run a command inside the virtualenv with \u001b[33mpipenv run\u001b[0m.\n", + "\u001b[1mInstalling dependencies from Pipfile.lock \u001b[0m\u001b[1m(\u001b[0m\u001b[1me4eef2\u001b[0m\u001b[1m)\u001b[0m\u001b[1;33m...\u001b[0m\n" + ] + } + ], + "source": [ + "#! pip install pipenv && pipenv install" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Ehw1SgHb3zig", + "outputId": "7dec72d3-00f0-49ad-d74d-03860a41ed9c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fatal: destination path 'local-rag' already exists and is not an empty directory.\n" + ] + } + ], + "source": [ + "!git clone https://github.com/casualcomputer/local-rag.git #gitclone local rag's interface folder" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iK1P0rZ35E_8", + "outputId": "397c3ede-fc1e-4845-8d58-577781b206c9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "34.172.141.48\n", + "\u001b[K\u001b[?25hnpx: installed 22 in 4.168s\n", + "your url is: https://warm-hornets-open.loca.lt\n" + ] + } + ], + "source": [ + "!streamlit run /local-rag/main.py &>/content/logs.txt & npx localtunnel --port 8501 &curl ipv4.icanhazip.com" + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "authorship_tag": "ABX9TyPlHpat5Dv0q2jQdim5JRKU", + "include_colab_link": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file