Multi_Views_colab_working.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "%cd /content\n",
        "!git clone https://github.com/adeerAI/Multi-Views.git\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rF-x9LlLa8Qa",
        "outputId": "fc1f9a08-0805-44c7-b7c3-596f2fe2999e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "Cloning into 'HarmonyView'...\n",
            "remote: Enumerating objects: 1251, done.\u001b[K\n",
            "remote: Counting objects: 100% (2/2), done.\u001b[K\n",
            "remote: Compressing objects: 100% (2/2), done.\u001b[K\n",
            "remote: Total 1251 (delta 1), reused 0 (delta 0), pack-reused 1249\u001b[K\n",
            "Receiving objects: 100% (1251/1251), 126.52 MiB | 14.16 MiB/s, done.\n",
            "Resolving deltas: 100% (281/281), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VjYy0F2gZIPR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "151fd891-c903-483b-9762-3c2d07ade9e0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/HarmonyView\n",
            "The following additional packages will be installed:\n",
            "  libaria2-0 libc-ares2\n",
            "The following NEW packages will be installed:\n",
            "  aria2 libaria2-0 libc-ares2\n",
            "0 upgraded, 3 newly installed, 0 to remove and 24 not upgraded.\n",
            "Need to get 1,513 kB of archives.\n",
            "After this operation, 5,441 kB of additional disk space will be used.\n",
            "Selecting previously unselected package libc-ares2:amd64.\n",
            "(Reading database ... 121658 files and directories currently installed.)\n",
            "Preparing to unpack .../libc-ares2_1.18.1-1ubuntu0.22.04.2_amd64.deb ...\n",
            "Unpacking libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.2) ...\n",
            "Selecting previously unselected package libaria2-0:amd64.\n",
            "Preparing to unpack .../libaria2-0_1.36.0-1_amd64.deb ...\n",
            "Unpacking libaria2-0:amd64 (1.36.0-1) ...\n",
            "Selecting previously unselected package aria2.\n",
            "Preparing to unpack .../aria2_1.36.0-1_amd64.deb ...\n",
            "Unpacking aria2 (1.36.0-1) ...\n",
            "Setting up libc-ares2:amd64 (1.18.1-1ubuntu0.22.04.2) ...\n",
            "Setting up libaria2-0:amd64 (1.36.0-1) ...\n",
            "Setting up aria2 (1.36.0-1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "\u001b[0m\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "9c8629|\u001b[1;32mOK\u001b[0m  |   252MiB/s|/content/HarmonyView/ckpt/ViT-L-14.pt\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\u001b[0m\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "f5d2a5|\u001b[1;32mOK\u001b[0m  |   322MiB/s|/content/HarmonyView/ckpt/sam_vit_h_4b8939.pth\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\u001b[0m\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "8fa272|\u001b[1;32mOK\u001b[0m  |   335MiB/s|/content/HarmonyView/ckpt/syncdreamer-pretrain.ckpt\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "d935c8|\u001b[1;32mOK\u001b[0m  |       0B/s|/content/HarmonyView/meta_info/camera-16.pkl\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.4/53.4 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for clip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.3/20.3 MB\u001b[0m \u001b[31m77.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m777.7/777.7 kB\u001b[0m \u001b[31m59.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.5/51.5 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m756.0/756.0 kB\u001b[0m \u001b[31m63.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.4/70.4 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m299.2/299.2 kB\u001b[0m \u001b[31m37.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.7/138.7 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.5/60.5 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m840.2/840.2 kB\u001b[0m \u001b[31m73.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.1/71.1 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "lida 0.0.10 requires kaleido, which is not installed.\n",
            "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.9.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m274.3/274.3 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.1/76.1 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m399.7/399.7 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m689.0/689.0 kB\u001b[0m \u001b[31m52.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.6/54.6 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m88.3/88.3 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.2/10.2 MB\u001b[0m \u001b[31m106.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.4/139.4 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m90.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m80.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "%cd /content/Multi-Views\n",
        "!apt -y install -qq aria2\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/HarmonyView/resolve/main/ViT-L-14.pt -d /content/Multi-Views/ckpt -o ViT-L-14.pt\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/HarmonyView/resolve/main/sam_vit_h_4b8939.pth -d /content/Multi-Views/ckpt -o sam_vit_h_4b8939.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/HarmonyView/resolve/main/syncdreamer-pretrain.ckpt -d /content/Multi-Views/ckpt -o syncdreamer-pretrain.ckpt\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/HarmonyView/resolve/main/camera-16.pkl -d /content/Multi-Views/meta_info -o camera-16.pkl\n",
        "\n",
        "#%cd /content/HarmonyView/hf_demo/examples\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/basket.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/cat.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/crab.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/elephant.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/flower.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/forest.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/monkey.png\n",
        "#!wget https://huggingface.co/camenduru/HarmonyView/resolve/main/teapot.png\n",
        "\n",
        "!pip install -q git+https://github.com/openai/CLIP\n",
        "!pip install -q gradio==3.50.2 pytorch_lightning taming-transformers-rom1504 kornia webdataset omegaconf\n",
        "!pip install -q einops pymcubes carvekit-colab open3d trimesh nerfacc fire segment_anything"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python foreground_segment.py --input /content/Multi-Views-hf/input/input.png --output /content/Multi-Views-hf/ouput_seg.png"
      ],
      "metadata": {
        "id": "PbNrSTwQdRA_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "bFAHLA1rdwzQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/Multi-Views\n",
        "!python generate.py --ckpt /content/Multi-Views/ckpt/syncdreamer-pretrain.ckpt \\\n",
        " --sample_num 1 --cfg_scales 2 1 --sample_steps 50 \\\n",
        " --input /content/Multi-Views/input/ortho.png --output /content/Multi-Views/ouput \\\n",
        " --elevation 35 --crop_size -1 --decomposed_sampling --seed 6033\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZ_sds3vLXp3",
        "outputId": "359a5c36-96be-4b63-b724-7843aee6c757"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/HarmonyView\n",
            "making attention of type 'vanilla' with 512 in_channels\n",
            "Working with z of shape (1, 4, 32, 32) = 4096 dimensions.\n",
            "making attention of type 'vanilla' with 512 in_channels\n",
            "Selected timesteps for ddim sampler: [  1   6  11  16  21  26  31  36  41  46  51  56  61  66  71  76  81  86\n",
            "  91  96 101 106 111 116 121 126 131 136 141 146 151 156 161 166 171 176\n",
            " 181 186 191 196 201 206 211 216 221 226 231 236 241 246 251 256 261 266\n",
            " 271 276 281 286 291 296 301 306 311 316 321 326 331 336 341 346 351 356\n",
            " 361 366 371 376 381 386 391 396 401 406 411 416 421 426 431 436 441 446\n",
            " 451 456 461 466 471 476 481 486 491 496 501 506 511 516 521 526 531 536\n",
            " 541 546 551 556 561 566 571 576 581 586 591 596 601 606 611 616 621 626\n",
            " 631 636 641 646 651 656 661 666 671 676 681 686 691 696 701 706 711 716\n",
            " 721 726 731 736 741 746 751 756 761 766 771 776 781 786 791 796 801 806\n",
            " 811 816 821 826 831 836 841 846 851 856 861 866 871 876 881 886 891 896\n",
            " 901 906 911 916 921 926 931 936 941 946 951 956 961 966 971 976 981 986\n",
            " 991 996]\n",
            "loading model from /content/HarmonyView/ckpt/syncdreamer-pretrain.ckpt ...\n",
            "Selected timesteps for ddim sampler: [  1  21  41  61  81 101 121 141 161 181 201 221 241 261 281 301 321 341\n",
            " 361 381 401 421 441 461 481 501 521 541 561 581 601 621 641 661 681 701\n",
            " 721 741 761 781 801 821 841 861 881 901 921 941 961 981]\n",
            "Decomposed Guidance (DG) Sampling\n",
            "unconditional scales [2.0, 1.0]\n",
            "DDIM Sampler: 100% 50/50 [00:23<00:00,  2.17it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "1PdfQOWAThwy"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}