From fed3ee1e363eb7bed7d71da275fc9a1f28a4ceab Mon Sep 17 00:00:00 2001 From: rhinophant Date: Sat, 22 Jun 2024 17:00:06 -0400 Subject: [PATCH] Delete setup_ollma_google_colab.ipynb --- setup_ollma_google_colab.ipynb | 1034 -------------------------------- 1 file changed, 1034 deletions(-) delete mode 100644 setup_ollma_google_colab.ipynb diff --git a/setup_ollma_google_colab.ipynb b/setup_ollma_google_colab.ipynb deleted file mode 100644 index db20ded..0000000 --- a/setup_ollma_google_colab.ipynb +++ /dev/null @@ -1,1034 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Host LLM on colab (with ollama) with port forwarding to public internet + access colab-hosted model via ollama API" - ], - "metadata": { - "id": "CpsFIe6TlKyS" - } - }, - { - "cell_type": "code", - "source": [ - "from google.colab import userdata #user to access token" - ], - "metadata": { - "id": "S5nNTL-8nN28" - }, - "execution_count": 1, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "EuaQsW4eTp0E", - "outputId": "eb3673b5-1cd5-48f7-dd21-2f98b9d9d739" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - " % Total % Received % Xferd Average Speed Time Time Time Current\n", - " Dload Upload Total Spent Left Speed\n", - " 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0>>> Downloading ollama...\n", - "100 10941 0 10941 0 0 25815 0 --:--:-- --:--:-- --:--:-- 25865\n", - "############################################################################################# 100.0%\n", - ">>> Installing ollama to /usr/local/bin...\n", - ">>> Adding ollama user to video group...\n", - ">>> Adding current user to ollama group...\n", - ">>> Creating ollama systemd service...\n", - "WARNING: Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies.\n", - ">>> The Ollama API is now available at 127.0.0.1:11434.\n", - ">>> Install complete. Run \"ollama\" from the command line.\n", - "Get:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n", - "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 InRelease\n", - "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n", - "Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n", - "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n", - "Hit:6 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n", - "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n", - "Hit:8 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n", - "Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n", - "Get:10 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,090 kB]\n", - "Hit:11 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n", - "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,391 kB]\n", - "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,183 kB]\n", - "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,542 kB]\n", - "Get:15 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,908 kB]\n", - "Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,464 kB]\n", - "Fetched 11.8 MB in 12s (990 kB/s)\n", - "Reading package lists... Done\n", - "Reading package lists... Done\n", - "Building dependency tree... Done\n", - "Reading state information... Done\n", - "The following additional packages will be installed:\n", - " cpp-12 cuda-drivers-555 dctrl-tools dkms fakeroot gcc-12\n", - " keyboard-configuration libasan8 libfakeroot libfontenc1 libgcc-12-dev\n", - " libjansson4 liblocale-gettext-perl libnvidia-cfg1-555 libnvidia-common-555\n", - " libnvidia-compute-555 libnvidia-decode-555 libnvidia-encode-555\n", - " libnvidia-extra-555 libnvidia-fbc1-555 libnvidia-gl-555 libtsan2 libudev1\n", - " libxcvt0 libxfont2 libxkbfile1 nvidia-compute-utils-555 nvidia-dkms-555\n", - " nvidia-driver-555 nvidia-firmware-555-555.42.02 nvidia-kernel-common-555\n", - " nvidia-kernel-source-555 nvidia-prime nvidia-settings nvidia-utils-555\n", - " python3-xkit screen-resolution-extra systemd-hwe-hwdb udev x11-xkb-utils\n", - " xcvt xfonts-base xfonts-encodings xfonts-utils xserver-common\n", - " xserver-xorg-core xserver-xorg-video-nvidia-555\n", - "Suggested packages:\n", - " gcc-12-locales cpp-12-doc debtags menu gcc-12-multilib gcc-12-doc xfs\n", - " | xserver xfonts-100dpi | xfonts-75dpi xfonts-scalable\n", - "Recommended packages:\n", - " libnvidia-compute-555:i386 libnvidia-decode-555:i386\n", - " libnvidia-encode-555:i386 libnvidia-fbc1-555:i386 libnvidia-gl-555:i386\n", - "The following NEW packages will be installed:\n", - " cpp-12 cuda-drivers cuda-drivers-555 dctrl-tools dkms fakeroot gcc-12\n", - " keyboard-configuration libasan8 libfakeroot libfontenc1 libgcc-12-dev\n", - " libjansson4 liblocale-gettext-perl libnvidia-cfg1-555 libnvidia-common-555\n", - " libnvidia-compute-555 libnvidia-decode-555 libnvidia-encode-555\n", - " libnvidia-extra-555 libnvidia-fbc1-555 libnvidia-gl-555 libtsan2 libxcvt0\n", - " libxfont2 libxkbfile1 nvidia-compute-utils-555 nvidia-dkms-555\n", - " nvidia-driver-555 nvidia-firmware-555-555.42.02 nvidia-kernel-common-555\n", - " nvidia-kernel-source-555 nvidia-prime nvidia-settings nvidia-utils-555\n", - " python3-xkit screen-resolution-extra systemd-hwe-hwdb udev x11-xkb-utils\n", - " xcvt xfonts-base xfonts-encodings xfonts-utils xserver-common\n", - " xserver-xorg-core xserver-xorg-video-nvidia-555\n", - "The following packages will be upgraded:\n", - " libudev1\n", - "1 upgraded, 47 newly installed, 0 to remove and 44 not upgraded.\n", - "Need to get 319 MB of archives.\n", - "After this operation, 910 MB of additional disk space will be used.\n", - "Get:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-common-555 555.42.02-0ubuntu1 [17.2 kB]\n", - "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-compute-555 555.42.02-0ubuntu1 [46.9 MB]\n", - "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblocale-gettext-perl amd64 1.07-4build3 [17.1 kB]\n", - "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 keyboard-configuration all 1.205ubuntu3 [206 kB]\n", - "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-gl-555 555.42.02-0ubuntu1 [138 MB]\n", - "Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 cpp-12 amd64 12.3.0-1ubuntu1~22.04 [10.8 MB]\n", - "Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libasan8 amd64 12.3.0-1ubuntu1~22.04 [2,442 kB]\n", - "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libtsan2 amd64 12.3.0-1ubuntu1~22.04 [2,477 kB]\n", - "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgcc-12-dev amd64 12.3.0-1ubuntu1~22.04 [2,618 kB]\n", - "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 gcc-12 amd64 12.3.0-1ubuntu1~22.04 [21.7 MB]\n", - "Get:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-kernel-source-555 555.42.02-0ubuntu1 [41.4 MB]\n", - "Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 dctrl-tools amd64 2.24-3build2 [66.9 kB]\n", - "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dkms all 2.8.7-2ubuntu2.2 [70.1 kB]\n", - "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libudev1 amd64 249.11-0ubuntu3.12 [78.2 kB]\n", - "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 udev amd64 249.11-0ubuntu3.12 [1,557 kB]\n", - "Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjansson4 amd64 2.13.1-1.1build3 [32.4 kB]\n", - "Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxkbfile1 amd64 1:1.1.0-1build3 [71.8 kB]\n", - "Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 x11-xkb-utils amd64 7.7+5build4 [172 kB]\n", - "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 xserver-common all 2:21.1.4-2ubuntu1.7~22.04.10 [28.5 kB]\n", - "Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxcvt0 amd64 0.1.1-3 [5,494 B]\n", - "Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfontenc1 amd64 1:1.1.4-1build3 [14.7 kB]\n", - "Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxfont2 amd64 1:2.0.5-1build1 [94.5 kB]\n", - "Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 xserver-xorg-core amd64 2:21.1.4-2ubuntu1.7~22.04.10 [1,476 kB]\n", - "Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfakeroot amd64 1.28-1ubuntu1 [31.5 kB]\n", - "Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 fakeroot amd64 1.28-1ubuntu1 [60.4 kB]\n", - "Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 nvidia-prime all 0.8.17.1 [9,956 B]\n", - "Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-xkit all 0.5.0ubuntu5 [18.5 kB]\n", - "Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 screen-resolution-extra all 0.18.2 [4,396 B]\n", - "Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 systemd-hwe-hwdb all 249.11.5 [3,228 B]\n", - "Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 xcvt amd64 0.1.1-3 [7,140 B]\n", - "Get:31 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-firmware-555-555.42.02 555.42.02-0ubuntu1 [36.5 MB]\n", - "Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 xfonts-encodings all 1:1.0.5-0ubuntu2 [578 kB]\n", - "Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 xfonts-utils amd64 1:7.7+6build2 [94.6 kB]\n", - "Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 xfonts-base all 1:1.0.5 [5,896 kB]\n", - "Get:35 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-kernel-common-555 555.42.02-0ubuntu1 [109 kB]\n", - "Get:36 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-dkms-555 555.42.02-0ubuntu1 [36.2 kB]\n", - "Get:37 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-extra-555 555.42.02-0ubuntu1 [73.4 kB]\n", - "Get:38 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-compute-utils-555 555.42.02-0ubuntu1 [118 kB]\n", - "Get:39 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-decode-555 555.42.02-0ubuntu1 [1,792 kB]\n", - "Get:40 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-encode-555 555.42.02-0ubuntu1 [104 kB]\n", - "Get:41 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-utils-555 555.42.02-0ubuntu1 [495 kB]\n", - "Get:42 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-cfg1-555 555.42.02-0ubuntu1 [146 kB]\n", - "Get:43 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 xserver-xorg-video-nvidia-555 555.42.02-0ubuntu1 [1,534 kB]\n", - "Get:44 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 libnvidia-fbc1-555 555.42.02-0ubuntu1 [75.3 kB]\n", - "Get:45 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-driver-555 555.42.02-0ubuntu1 [490 kB]\n", - "Get:46 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 cuda-drivers-555 555.42.02-1 [2,542 B]\n", - "Get:47 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 cuda-drivers 555.42.02-1 [2,498 B]\n", - "Get:48 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 nvidia-settings 555.42.02-0ubuntu1 [945 kB]\n", - "Fetched 319 MB in 4s (71.4 MB/s)\n", - "Extracting templates from packages: 100%\n", - "Preconfiguring packages ...\n", - "Selecting previously unselected package liblocale-gettext-perl.\n", - "(Reading database ... 121913 files and directories currently installed.)\n", - "Preparing to unpack .../0-liblocale-gettext-perl_1.07-4build3_amd64.deb ...\n", - "Unpacking liblocale-gettext-perl (1.07-4build3) ...\n", - "Selecting previously unselected package keyboard-configuration.\n", - "Preparing to unpack .../1-keyboard-configuration_1.205ubuntu3_all.deb ...\n", - "Unpacking keyboard-configuration (1.205ubuntu3) ...\n", - "Selecting previously unselected package cpp-12.\n", - "Preparing to unpack .../2-cpp-12_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", - "Unpacking cpp-12 (12.3.0-1ubuntu1~22.04) ...\n", - "Selecting previously unselected package libasan8:amd64.\n", - "Preparing to unpack .../3-libasan8_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", - "Unpacking libasan8:amd64 (12.3.0-1ubuntu1~22.04) ...\n", - "Selecting previously unselected package libtsan2:amd64.\n", - "Preparing to unpack .../4-libtsan2_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", - "Unpacking libtsan2:amd64 (12.3.0-1ubuntu1~22.04) ...\n", - "Selecting previously unselected package libgcc-12-dev:amd64.\n", - "Preparing to unpack .../5-libgcc-12-dev_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", - "Unpacking libgcc-12-dev:amd64 (12.3.0-1ubuntu1~22.04) ...\n", - "Selecting previously unselected package gcc-12.\n", - "Preparing to unpack .../6-gcc-12_12.3.0-1ubuntu1~22.04_amd64.deb ...\n", - "Unpacking gcc-12 (12.3.0-1ubuntu1~22.04) ...\n", - "Selecting previously unselected package dctrl-tools.\n", - "Preparing to unpack .../7-dctrl-tools_2.24-3build2_amd64.deb ...\n", - "Unpacking dctrl-tools (2.24-3build2) ...\n", - "Selecting previously unselected package dkms.\n", - "Preparing to unpack .../8-dkms_2.8.7-2ubuntu2.2_all.deb ...\n", - "Unpacking dkms (2.8.7-2ubuntu2.2) ...\n", - "Preparing to unpack .../9-libudev1_249.11-0ubuntu3.12_amd64.deb ...\n", - "Unpacking libudev1:amd64 (249.11-0ubuntu3.12) over (249.11-0ubuntu3.10) ...\n", - "Setting up libudev1:amd64 (249.11-0ubuntu3.12) ...\n", - "Selecting previously unselected package udev.\n", - "(Reading database ... 122273 files and directories currently installed.)\n", - "Preparing to unpack .../00-udev_249.11-0ubuntu3.12_amd64.deb ...\n", - "Unpacking udev (249.11-0ubuntu3.12) ...\n", - "Selecting previously unselected package libjansson4:amd64.\n", - "Preparing to unpack .../01-libjansson4_2.13.1-1.1build3_amd64.deb ...\n", - "Unpacking libjansson4:amd64 (2.13.1-1.1build3) ...\n", - "Selecting previously unselected package libnvidia-common-555.\n", - "Preparing to unpack .../02-libnvidia-common-555_555.42.02-0ubuntu1_all.deb ...\n", - "Unpacking libnvidia-common-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-compute-555:amd64.\n", - "Preparing to unpack .../03-libnvidia-compute-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking libnvidia-compute-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-gl-555:amd64.\n", - "Preparing to unpack .../04-libnvidia-gl-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "dpkg-query: no packages found matching libnvidia-gl-535\n", - "Unpacking libnvidia-gl-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-kernel-source-555.\n", - "Preparing to unpack .../05-nvidia-kernel-source-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-kernel-source-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-firmware-555-555.42.02.\n", - "Preparing to unpack .../06-nvidia-firmware-555-555.42.02_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-firmware-555-555.42.02 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-kernel-common-555.\n", - "Preparing to unpack .../07-nvidia-kernel-common-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-kernel-common-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-dkms-555.\n", - "Preparing to unpack .../08-nvidia-dkms-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-dkms-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-extra-555:amd64.\n", - "Preparing to unpack .../09-libnvidia-extra-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking libnvidia-extra-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-compute-utils-555.\n", - "Preparing to unpack .../10-nvidia-compute-utils-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-compute-utils-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-decode-555:amd64.\n", - "Preparing to unpack .../11-libnvidia-decode-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking libnvidia-decode-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-encode-555:amd64.\n", - "Preparing to unpack .../12-libnvidia-encode-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking libnvidia-encode-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-utils-555.\n", - "Preparing to unpack .../13-nvidia-utils-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-utils-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-cfg1-555:amd64.\n", - "Preparing to unpack .../14-libnvidia-cfg1-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking libnvidia-cfg1-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libxkbfile1:amd64.\n", - "Preparing to unpack .../15-libxkbfile1_1%3a1.1.0-1build3_amd64.deb ...\n", - "Unpacking libxkbfile1:amd64 (1:1.1.0-1build3) ...\n", - "Selecting previously unselected package x11-xkb-utils.\n", - "Preparing to unpack .../16-x11-xkb-utils_7.7+5build4_amd64.deb ...\n", - "Unpacking x11-xkb-utils (7.7+5build4) ...\n", - "Selecting previously unselected package xserver-common.\n", - "Preparing to unpack .../17-xserver-common_2%3a21.1.4-2ubuntu1.7~22.04.10_all.deb ...\n", - "Unpacking xserver-common (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", - "Selecting previously unselected package libxcvt0:amd64.\n", - "Preparing to unpack .../18-libxcvt0_0.1.1-3_amd64.deb ...\n", - "Unpacking libxcvt0:amd64 (0.1.1-3) ...\n", - "Selecting previously unselected package libfontenc1:amd64.\n", - "Preparing to unpack .../19-libfontenc1_1%3a1.1.4-1build3_amd64.deb ...\n", - "Unpacking libfontenc1:amd64 (1:1.1.4-1build3) ...\n", - "Selecting previously unselected package libxfont2:amd64.\n", - "Preparing to unpack .../20-libxfont2_1%3a2.0.5-1build1_amd64.deb ...\n", - "Unpacking libxfont2:amd64 (1:2.0.5-1build1) ...\n", - "Selecting previously unselected package xserver-xorg-core.\n", - "Preparing to unpack .../21-xserver-xorg-core_2%3a21.1.4-2ubuntu1.7~22.04.10_amd64.deb ...\n", - "Unpacking xserver-xorg-core (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", - "Selecting previously unselected package xserver-xorg-video-nvidia-555.\n", - "Preparing to unpack .../22-xserver-xorg-video-nvidia-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking xserver-xorg-video-nvidia-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package libnvidia-fbc1-555:amd64.\n", - "Preparing to unpack .../23-libnvidia-fbc1-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking libnvidia-fbc1-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package nvidia-driver-555.\n", - "Preparing to unpack .../24-nvidia-driver-555_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-driver-555 (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package cuda-drivers-555.\n", - "Preparing to unpack .../25-cuda-drivers-555_555.42.02-1_amd64.deb ...\n", - "Unpacking cuda-drivers-555 (555.42.02-1) ...\n", - "Selecting previously unselected package cuda-drivers.\n", - "Preparing to unpack .../26-cuda-drivers_555.42.02-1_amd64.deb ...\n", - "Unpacking cuda-drivers (555.42.02-1) ...\n", - "Selecting previously unselected package libfakeroot:amd64.\n", - "Preparing to unpack .../27-libfakeroot_1.28-1ubuntu1_amd64.deb ...\n", - "Unpacking libfakeroot:amd64 (1.28-1ubuntu1) ...\n", - "Selecting previously unselected package fakeroot.\n", - "Preparing to unpack .../28-fakeroot_1.28-1ubuntu1_amd64.deb ...\n", - "Unpacking fakeroot (1.28-1ubuntu1) ...\n", - "Selecting previously unselected package nvidia-prime.\n", - "Preparing to unpack .../29-nvidia-prime_0.8.17.1_all.deb ...\n", - "Unpacking nvidia-prime (0.8.17.1) ...\n", - "Selecting previously unselected package python3-xkit.\n", - "Preparing to unpack .../30-python3-xkit_0.5.0ubuntu5_all.deb ...\n", - "Unpacking python3-xkit (0.5.0ubuntu5) ...\n", - "Selecting previously unselected package screen-resolution-extra.\n", - "Preparing to unpack .../31-screen-resolution-extra_0.18.2_all.deb ...\n", - "Unpacking screen-resolution-extra (0.18.2) ...\n", - "Selecting previously unselected package nvidia-settings.\n", - "Preparing to unpack .../32-nvidia-settings_555.42.02-0ubuntu1_amd64.deb ...\n", - "Unpacking nvidia-settings (555.42.02-0ubuntu1) ...\n", - "Selecting previously unselected package systemd-hwe-hwdb.\n", - "Preparing to unpack .../33-systemd-hwe-hwdb_249.11.5_all.deb ...\n", - "Unpacking systemd-hwe-hwdb (249.11.5) ...\n", - "Selecting previously unselected package xcvt.\n", - "Preparing to unpack .../34-xcvt_0.1.1-3_amd64.deb ...\n", - "Unpacking xcvt (0.1.1-3) ...\n", - "Selecting previously unselected package xfonts-encodings.\n", - "Preparing to unpack .../35-xfonts-encodings_1%3a1.0.5-0ubuntu2_all.deb ...\n", - "Unpacking xfonts-encodings (1:1.0.5-0ubuntu2) ...\n", - "Selecting previously unselected package xfonts-utils.\n", - "Preparing to unpack .../36-xfonts-utils_1%3a7.7+6build2_amd64.deb ...\n", - "Unpacking xfonts-utils (1:7.7+6build2) ...\n", - "Selecting previously unselected package xfonts-base.\n", - "Preparing to unpack .../37-xfonts-base_1%3a1.0.5_all.deb ...\n", - "Unpacking xfonts-base (1:1.0.5) ...\n", - "Setting up libnvidia-common-555 (555.42.02-0ubuntu1) ...\n", - "Setting up libnvidia-fbc1-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up cpp-12 (12.3.0-1ubuntu1~22.04) ...\n", - "Setting up libnvidia-cfg1-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up nvidia-prime (0.8.17.1) ...\n", - "Setting up libfakeroot:amd64 (1.28-1ubuntu1) ...\n", - "Setting up libjansson4:amd64 (2.13.1-1.1build3) ...\n", - "Setting up libnvidia-extra-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up fakeroot (1.28-1ubuntu1) ...\n", - "update-alternatives: using /usr/bin/fakeroot-sysv to provide /usr/bin/fakeroot (fakeroot) in auto mode\n", - "Setting up libfontenc1:amd64 (1:1.1.4-1build3) ...\n", - "Setting up libnvidia-compute-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up xfonts-encodings (1:1.0.5-0ubuntu2) ...\n", - "Setting up udev (249.11-0ubuntu3.12) ...\n", - "invoke-rc.d: could not determine current runlevel\n", - "invoke-rc.d: policy-rc.d denied execution of start.\n", - "Setting up systemd-hwe-hwdb (249.11.5) ...\n", - "Setting up libasan8:amd64 (12.3.0-1ubuntu1~22.04) ...\n", - "Setting up nvidia-firmware-555-555.42.02 (555.42.02-0ubuntu1) ...\n", - "Setting up libxcvt0:amd64 (0.1.1-3) ...\n", - "Setting up libxkbfile1:amd64 (1:1.1.0-1build3) ...\n", - "Setting up libtsan2:amd64 (12.3.0-1ubuntu1~22.04) ...\n", - "Setting up python3-xkit (0.5.0ubuntu5) ...\n", - "Setting up libxfont2:amd64 (1:2.0.5-1build1) ...\n", - "Setting up liblocale-gettext-perl (1.07-4build3) ...\n", - "Setting up dctrl-tools (2.24-3build2) ...\n", - "Setting up nvidia-kernel-source-555 (555.42.02-0ubuntu1) ...\n", - "Setting up nvidia-utils-555 (555.42.02-0ubuntu1) ...\n", - "Setting up nvidia-compute-utils-555 (555.42.02-0ubuntu1) ...\n", - "Warning: The home dir /nonexistent you specified can't be accessed: No such file or directory\n", - "Adding system user `nvidia-persistenced' (UID 104) ...\n", - "Adding new group `nvidia-persistenced' (GID 111) ...\n", - "Adding new user `nvidia-persistenced' (UID 104) with group `nvidia-persistenced' ...\n", - "Not creating home directory `/nonexistent'.\n", - "Setting up x11-xkb-utils (7.7+5build4) ...\n", - "Setting up libnvidia-decode-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up xfonts-utils (1:7.7+6build2) ...\n", - "Setting up nvidia-kernel-common-555 (555.42.02-0ubuntu1) ...\n", - "Created symlink /etc/systemd/system/systemd-hibernate.service.wants/nvidia-hibernate.service → /lib/systemd/system/nvidia-hibernate.service.\n", - "Created symlink /etc/systemd/system/systemd-suspend.service.wants/nvidia-resume.service → /lib/systemd/system/nvidia-resume.service.\n", - "Created symlink /etc/systemd/system/systemd-hibernate.service.wants/nvidia-resume.service → /lib/systemd/system/nvidia-resume.service.\n", - "Created symlink /etc/systemd/system/systemd-suspend.service.wants/nvidia-suspend.service → /lib/systemd/system/nvidia-suspend.service.\n", - "Setting up xcvt (0.1.1-3) ...\n", - "Setting up xfonts-base (1:1.0.5) ...\n", - "Setting up libnvidia-gl-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up libgcc-12-dev:amd64 (12.3.0-1ubuntu1~22.04) ...\n", - "Setting up screen-resolution-extra (0.18.2) ...\n", - "Setting up nvidia-settings (555.42.02-0ubuntu1) ...\n", - "Setting up xserver-common (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", - "Setting up keyboard-configuration (1.205ubuntu3) ...\n", - "Your console font configuration will be updated the next time your system\n", - "boots. If you want to update it now, run 'setupcon' from a virtual console.\n", - "Setting up libnvidia-encode-555:amd64 (555.42.02-0ubuntu1) ...\n", - "Setting up xserver-xorg-core (2:21.1.4-2ubuntu1.7~22.04.10) ...\n", - "Setting up gcc-12 (12.3.0-1ubuntu1~22.04) ...\n", - "Setting up xserver-xorg-video-nvidia-555 (555.42.02-0ubuntu1) ...\n", - "Setting up dkms (2.8.7-2ubuntu2.2) ...\n", - "Setting up nvidia-dkms-555 (555.42.02-0ubuntu1) ...\n", - "Loading new nvidia-555.42.02 DKMS files...\n", - "It is likely that 6.1.85+ belongs to a chroot's host\n", - "Building for 5.15.0-112-generic\n", - "Building for architecture x86_64\n", - "Building initial module for 5.15.0-112-generic\n" - ] - } - ], - "source": [ - "!curl https://ollama.ai/install.sh | sh\n", - "\n", - "!echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections\n", - "!sudo apt-get update && sudo apt-get install -y cuda-drivers\n", - "\n", - "!pip install pyngrok\n", - "from pyngrok import ngrok\n", - "ngrok.set_auth_token(userdata.get('ngrok_token')) #secret saved in a variable \"ngrok_token\" in colab \"Secrets\"\n", - "\n", - "import os\n", - "import asyncio\n", - "\n", - "# Set LD_LIBRARY_PATH so the system NVIDIA library\n", - "os.environ.update({'LD_LIBRARY_PATH': '/usr/lib64-nvidia'})\n", - "\n", - "# Function to run a specified command as a subprocess\n", - "async def run_process(cmd):\n", - " # Print the command that is starting\n", - " print('>>> starting', *cmd)\n", - "\n", - " # Start the subprocess and redirect stdout and stderr to asyncio subprocess pipes\n", - " p = await asyncio.create_subprocess_exec(\n", - " *cmd,\n", - " stdout=asyncio.subprocess.PIPE, # Capture standard output\n", - " stderr=asyncio.subprocess.PIPE # Capture standard errors\n", - " )\n", - "\n", - " # Function to handle the output from stdout and stderr\n", - " async def pipe(lines):\n", - " async for line in lines:\n", - " # Decode and strip each line of output and print it\n", - " print(line.decode('utf-8').strip())\n", - "\n", - " # Concurrently process both stdout and stderr\n", - " await asyncio.gather(\n", - " pipe(p.stdout),\n", - " pipe(p.stderr),\n", - " )\n", - "\n", - "# Clear the output of the current Jupyter notebook cell\n", - "clear_output()\n", - "\n", - "# This coroutine runs two subprocesses: one for the Ollama server and one for the ngrok tunnel\n", - "async def main():\n", - " await asyncio.gather(\n", - " run_process(['ollama', 'serve']), # Command to start the Ollama server\n", - " run_process(['ngrok', 'http', '--log', 'stderr', '11434']), # Command to start ngrok to expose port 11434\n", - " )\n", - "\n", - "# Run the main function in the asyncio event loop\n", - "await main()" - ] - }, - { - "cell_type": "markdown", - "source": [ - "Command prompt on local laptops' cmd\n", - "\n", - "```\n", - "curl https://ollama.ai/install.sh | sh\n", - "export OLLAMA_HOST=(https://fe64-34-106-204-48.ngrok-free.app/)\n", - "ollama run dolphin-mistral\n", - "```\n", - "\n", - "\n", - "\n" - ], - "metadata": { - "id": "SC6KKCLDmAVz" - } - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9s2YZFcuv9tr" - }, - "source": [ - "# Host LLM on colab (with ollama) and access the LLM on colab with ollama API" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "wxTw5NwVv9Jv", - "outputId": "e620ac3d-5c6c-4149-cbb5-cebe3ea2e532" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - ">>> Downloading ollama...\n", - "############################################################################################# 100.0%\n", - ">>> Installing ollama to /usr/local/bin...\n", - ">>> Creating ollama user...\n", - ">>> Adding ollama user to video group...\n", - ">>> Adding current user to ollama group...\n", - ">>> Creating ollama systemd service...\n", - "WARNING: Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies.\n", - ">>> The Ollama API is now available at 127.0.0.1:11434.\n", - ">>> Install complete. Run \"ollama\" from the command line.\n" - ] - } - ], - "source": [ - "# Install ollama: The Ollama API is now available at 127.0.0.1:11434\n", - "! curl -fsSL https://ollama.com/install.sh | sh" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "4tXmemohSdSU" - }, - "outputs": [], - "source": [ - "#!curl -X POST http://127.0.0.1:11434/api/generate -d '{\"model\": \"llama3\", \"prompt\": \"Why is the sky blue?\"}'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "J9NjjJnzEoWT", - "outputId": "867a39f2-a05d-4e4a-b451-5458b93fb859" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "curl: (7) Failed to connect to localhost port 11434 after 0 ms: Connection refused\n" - ] - } - ], - "source": [ - "#!curl -X POST http://localhost:11434/api/generate -d '{\"model\": \"llama3\", \"prompt\": \"Why is the sky blue?\"}'" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "kS6hBI7Q_2IM", - "outputId": "27b3a5e2-3cf2-4c9c-db0f-0cf0a231171f" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Requirement already satisfied: ngrok in /usr/local/lib/python3.10/dist-packages (1.3.0)\n", - "Requirement already satisfied: pyngrok==4.1.1 in /usr/local/lib/python3.10/dist-packages (4.1.1)\n", - "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from pyngrok==4.1.1) (0.18.3)\n", - "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from pyngrok==4.1.1) (6.0.1)\n" - ] - } - ], - "source": [ - "!pip install ngrok\n", - "!pip install pyngrok==4.1.1" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "id": "_2YY9CFk-v6z" - }, - "outputs": [], - "source": [ - "from google.colab import userdata #user to access token\n", - "\n", - "# Get your ngrok token from your ngrok account:\n", - "# https://dashboard.ngrok.com/get-started/your-authtoken\n", - "import threading\n", - "import time\n", - "import os\n", - "import asyncio\n", - "from pyngrok import ngrok\n", - "import threading\n", - "import queue\n", - "import time\n", - "from threading import Thread\n", - "\n", - "# Get your ngrok token from your ngrok account:\n", - "# https://dashboard.ngrok.com/get-started/your-authtoken\n", - "token= userdata.get('ngrok_token')\n", - "ngrok.set_auth_token(token)\n", - "\n", - "# set up a stoppable thread (not mandatory, but cleaner if you want to stop this later\n", - "class StoppableThread(threading.Thread):\n", - " def __init__(self, *args, **kwargs):\n", - " super(StoppableThread, self).__init__(*args, **kwargs)\n", - " self._stop_event = threading.Event()\n", - "\n", - " def stop(self):\n", - " self._stop_event.set()\n", - "\n", - " def is_stopped(self):\n", - " return self._stop_event.is_set()\n", - "\n", - "def start_ngrok(q, stop_event):\n", - " try:\n", - " # Start an HTTP tunnel on the specified port\n", - " public_url = ngrok.connect(11434)\n", - " # Put the public URL in the queue\n", - " q.put(public_url)\n", - " # Keep the thread alive until stop event is set\n", - " while not stop_event.is_set():\n", - " time.sleep(1) # Adjust sleep time as needed\n", - " except Exception as e:\n", - " print(f\"Error in start_ngrok: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "ZuZPqxR_ATwr" - }, - "outputs": [], - "source": [ - "# Create a queue to share data between threads\n", - "url_queue = queue.Queue()\n", - "\n", - "# Start ngrok in a separate thread\n", - "ngrok_thread = StoppableThread(target=start_ngrok, args=(url_queue, StoppableThread.is_stopped))\n", - "ngrok_thread.start()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 356 - }, - "id": "nrlUY7yLAWef", - "outputId": "915fb13b-6e3f-496f-efd0-059b64f8c4ce" - }, - "outputs": [ - { - "output_type": "error", - "ename": "KeyboardInterrupt", - "evalue": "", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mpublic_url\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murl_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpublic_url\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3.10/queue.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m 169\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 170\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_qsize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnot_empty\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 172\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'timeout' must be a non-negative number\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 318\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 319\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 321\u001b[0m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "# Wait for the ngrok tunnel to be established\n", - "while True:\n", - " try:\n", - " public_url = url_queue.get()\n", - " if public_url:\n", - " break\n", - " print(\"Waiting for ngrok URL...\")\n", - " time.sleep(1)\n", - " except Exception as e:\n", - " print(f\"Error in retrieving ngrok URL: {e}\")\n", - "\n", - "print(\"Ngrok tunnel established at:\", public_url)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "T46-LFgWAdBO" - }, - "outputs": [], - "source": [ - "import os\n", - "import asyncio\n", - "\n", - "# NB: You may need to set these depending and get cuda working depending which backend you are running.\n", - "# Set environment variable for NVIDIA library\n", - "# Set environment variables for CUDA\n", - "os.environ['PATH'] += ':/usr/local/cuda/bin'\n", - "# Set LD_LIBRARY_PATH to include both /usr/lib64-nvidia and CUDA lib directories\n", - "os.environ['LD_LIBRARY_PATH'] = '/usr/lib64-nvidia:/usr/local/cuda/lib64'\n", - "\n", - "async def run_process(cmd):\n", - " print('>>> starting', *cmd)\n", - " process = await asyncio.create_subprocess_exec(\n", - " *cmd,\n", - " stdout=asyncio.subprocess.PIPE,\n", - " stderr=asyncio.subprocess.PIPE\n", - " )\n", - "\n", - " # define an async pipe function\n", - " async def pipe(lines):\n", - " async for line in lines:\n", - " print(line.decode().strip())\n", - "\n", - " await asyncio.gather(\n", - " pipe(process.stdout),\n", - " pipe(process.stderr),\n", - " )\n", - "\n", - " # call it\n", - " await asyncio.gather(pipe(process.stdout), pipe(process.stderr))" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "CkeaRb9RAgdG", - "outputId": "006bc391-246c-4ecd-cc1a-40d97f9f25fb" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - ">>> starting ollama serve\n" - ] - } - ], - "source": [ - "import asyncio\n", - "import threading\n", - "\n", - "async def start_ollama_serve():\n", - " await run_process(['ollama', 'serve'])\n", - "\n", - "def run_async_in_thread(loop, coro):\n", - " asyncio.set_event_loop(loop)\n", - " loop.run_until_complete(coro)\n", - " loop.close()\n", - "\n", - "# Create a new event loop that will run in a new thread\n", - "new_loop = asyncio.new_event_loop()\n", - "\n", - "# Start ollama serve in a separate thread so the cell won't block execution\n", - "thread = threading.Thread(target=run_async_in_thread, args=(new_loop, start_ollama_serve()))\n", - "thread.start()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "id": "DrkNz68IAlJe" - }, - "outputs": [], - "source": [ - "! export OLLAMA_HOST=http://9126-34-106-204-48.ngrok-free.app/" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "esy-S7TjA5Sm", - "outputId": "14f364e5-b7b6-425a-c3d2-ab80e46b3f62" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Error: could not connect to ollama app, is it running?\n" - ] - } - ], - "source": [ - "!ollama pull llama2" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Method 1: ollama cmd" - ], - "metadata": { - "id": "KOv39QoawnLg" - } - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0U-22lppCpHS", - "outputId": "c39a2f10-07ba-4363-82d1-a181301a909c" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Error: could not connect to ollama app, is it running?\n" - ] - } - ], - "source": [ - "!ollama run llama2 #method 1" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Method 2: curl and API" - ], - "metadata": { - "id": "WixtlfV9wc_m" - } - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "HW-grtw0AwH4" - }, - "outputs": [], - "source": [ - "!curl -X POST http://9126-34-106-204-48.ngrok-free.app/api/generate -d '{ \"model\": \"llama2\", \"prompt\":\"Good Morning\" , \"stream\": false }'" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Method 3: Python and API" - ], - "metadata": { - "id": "5ar5-iU2whrx" - } - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 844 - }, - "id": "0afNrMKXBhEg", - "outputId": "9b7dcc41-c368-4d65-9946-39349086606b" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "You: hi there. how are ya\n", - "Bot: Error: 404, \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - "\n", - "\n" - ] - }, - { - "output_type": "error", - "ename": "KeyboardInterrupt", - "evalue": "Interrupted by user", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;31m# Get user input for the prompt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mprompt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"You: \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;31m# Check if the user wants to exit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 850\u001b[0m )\n\u001b[0;32m--> 851\u001b[0;31m return self._input_request(str(prompt),\n\u001b[0m\u001b[1;32m 852\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 853\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m \u001b[0;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Interrupted by user\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid Message:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: Interrupted by user" - ] - } - ], - "source": [ - "# python code\n", - "\n", - "import requests\n", - "\n", - "# URL for the API endpoint\n", - "url = 'http://9126-34-106-204-48.ngrok-free.app//api/generate'\n", - "\n", - "# Function to make the POST request\n", - "def make_request(prompt):\n", - " data = {\n", - " \"model\": \"llama2\",\n", - " \"prompt\": prompt,\n", - " \"stream\": False\n", - " }\n", - " response = requests.post(url, json=data)\n", - " if response.status_code == 200:\n", - " return response.json().get(\"response\", \"No response received\")\n", - " else:\n", - " return f\"Error: {response.status_code}, {response.text}\"\n", - "\n", - "# Continuous chat loop\n", - "while True:\n", - " # Get user input for the prompt\n", - " prompt = input(\"You: \")\n", - "\n", - " # Check if the user wants to exit\n", - " if prompt.lower() == 'exit':\n", - " print(\"Exiting chat.\")\n", - " break\n", - "\n", - " # Make the request\n", - " response = make_request(prompt)\n", - "\n", - " # Print the response\n", - " print(\"Bot:\", response)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "4jkbFk6CtwJ8", - "outputId": "2a7f8c4a-af1e-4576-8955-b7ea9fcdc719" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pipenv in /usr/local/lib/python3.10/dist-packages (2024.0.1)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from pipenv) (2024.6.2)\n", - "Requirement already satisfied: setuptools>=67 in /usr/local/lib/python3.10/dist-packages (from pipenv) (67.7.2)\n", - "Requirement already satisfied: virtualenv>=20.24.2 in /usr/local/lib/python3.10/dist-packages (from pipenv) (20.26.2)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.24.2->pipenv) (0.3.8)\n", - "Requirement already satisfied: filelock<4,>=3.12.2 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.24.2->pipenv) (3.14.0)\n", - "Requirement already satisfied: platformdirs<5,>=3.9.1 in /usr/local/lib/python3.10/dist-packages (from virtualenv>=20.24.2->pipenv) (4.2.2)\n", - "To activate this project's virtualenv, run \u001b[33mpipenv shell\u001b[0m.\n", - "Alternatively, run a command inside the virtualenv with \u001b[33mpipenv run\u001b[0m.\n", - "To activate this project's virtualenv, run \u001b[33mpipenv shell\u001b[0m.\n", - "Alternatively, run a command inside the virtualenv with \u001b[33mpipenv run\u001b[0m.\n", - "\u001b[1mInstalling dependencies from Pipfile.lock \u001b[0m\u001b[1m(\u001b[0m\u001b[1me4eef2\u001b[0m\u001b[1m)\u001b[0m\u001b[1;33m...\u001b[0m\n" - ] - } - ], - "source": [ - "#! pip install pipenv && pipenv install" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ehw1SgHb3zig", - "outputId": "7dec72d3-00f0-49ad-d74d-03860a41ed9c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "fatal: destination path 'local-rag' already exists and is not an empty directory.\n" - ] - } - ], - "source": [ - "!git clone https://github.com/casualcomputer/local-rag.git #gitclone local rag's interface folder" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "iK1P0rZ35E_8", - "outputId": "397c3ede-fc1e-4845-8d58-577781b206c9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "34.172.141.48\n", - "\u001b[K\u001b[?25hnpx: installed 22 in 4.168s\n", - "your url is: https://warm-hornets-open.loca.lt\n" - ] - } - ], - "source": [ - "!streamlit run /local-rag/main.py &>/content/logs.txt & npx localtunnel --port 8501 &curl ipv4.icanhazip.com" - ] - } - ], - "metadata": { - "colab": { - "provenance": [], - "authorship_tag": "ABX9TyPlHpat5Dv0q2jQdim5JRKU", - "include_colab_link": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file