Skip to content

Commit

Permalink
Convert NeRF example to Keras 3.
Browse files Browse the repository at this point in the history
  • Loading branch information
hertschuh committed Nov 13, 2023
1 parent 30abf39 commit 832b075
Showing 28 changed files with 270 additions and 98 deletions.
Binary file removed examples/vision/img/nerf/nerf_11_1.png
Binary file not shown.
Binary file modified examples/vision/img/nerf/nerf_11_11.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_13.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_15.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_17.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_19.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_21.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_23.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_25.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_27.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_29.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_3.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_31.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_33.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_35.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_37.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_39.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added examples/vision/img/nerf/nerf_11_41.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_5.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_7.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified examples/vision/img/nerf/nerf_11_9.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file removed examples/vision/img/nerf/nerf_14_0.png
Binary file not shown.
Binary file added examples/vision/img/nerf/nerf_14_1.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added examples/vision/img/nerf/nerf_5_0.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file removed examples/vision/img/nerf/nerf_5_1.png
Binary file not shown.
51 changes: 27 additions & 24 deletions examples/vision/ipynb/nerf.ipynb
Original file line number Diff line number Diff line change
@@ -75,24 +75,29 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
"\n",
"# Setting random seed to obtain reproducible results.\n",
"import tensorflow as tf\n",
"\n",
"tf.random.set_seed(42)\n",
"\n",
"import keras\n",
"from keras import layers\n",
"\n",
"import os\n",
"import glob\n",
"import imageio\n",
"import numpy as np\n",
"from tqdm import tqdm\n",
"from tensorflow import keras\n",
"from tensorflow.keras import layers\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# Initialize global variables.\n",
@@ -143,7 +148,7 @@
"The camera-matrix is an *affine transform matrix* that is\n",
"concatenated with a 3 x 1 column `[image height, image width, focal length]`\n",
"to produce the *pose matrix*. This matrix is of\n",
"dimensions 3 x 5 where the first 3 x 3 block is in the camera’s point\n",
"dimensions 3 x 5 where the first 3 x 3 block is in the camera\u2019s point\n",
"of view. The axes are `[down, right, backwards]` or `[-y, x, z]`\n",
"where the camera is facing forwards `-z`.\n",
"\n",
@@ -157,17 +162,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
"outputs": [],
"source": [
"# Download the data if it does not already exist.\n",
"file_name = \"tiny_nerf_data.npz\"\n",
"url = \"https://people.eecs.berkeley.edu/~bmild/nerf/tiny_nerf_data.npz\"\n",
"if not os.path.exists(file_name):\n",
" data = keras.utils.get_file(fname=file_name, origin=url)\n",
"url = (\n",
" \"http://cseweb.ucsd.edu/~viscomp/projects/LF/papers/ECCV20/nerf/tiny_nerf_data.npz\"\n",
")\n",
"data = keras.utils.get_file(origin=url)\n",
"\n",
"data = np.load(data)\n",
"images = data[\"images\"]\n",
@@ -241,7 +246,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
@@ -260,7 +265,7 @@
" positions = [x]\n",
" for i in range(POS_ENCODE_DIMS):\n",
" for fn in [tf.sin, tf.cos]:\n",
" positions.append(fn(2.0 ** i * x))\n",
" positions.append(fn(2.0**i * x))\n",
" return tf.concat(positions, axis=-1)\n",
"\n",
"\n",
@@ -425,7 +430,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
@@ -440,7 +445,7 @@
" num_pos: The number of dimensions of positional encoding.\n",
"\n",
" Returns:\n",
" The `tf.keras` model.\n",
" The `keras` model.\n",
" \"\"\"\n",
" inputs = keras.Input(shape=(num_pos, 2 * 3 * POS_ENCODE_DIMS + 3))\n",
" x = inputs\n",
@@ -504,7 +509,8 @@
" depth_map = tf.reduce_sum(weights * t_vals, axis=-1)\n",
" else:\n",
" depth_map = tf.reduce_sum(weights * t_vals[:, None, None], axis=-1)\n",
" return (rgb, depth_map)\n"
" return (rgb, depth_map)\n",
""
]
},
{
@@ -521,7 +527,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
@@ -646,7 +652,6 @@
" batch_size=BATCH_SIZE,\n",
" epochs=EPOCHS,\n",
" callbacks=[TrainMonitor()],\n",
" steps_per_epoch=split_index // BATCH_SIZE,\n",
")\n",
"\n",
"\n",
@@ -700,7 +705,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
@@ -728,9 +733,7 @@
" ax[1].imshow(keras.utils.array_to_img(recons_img))\n",
" ax[1].set_title(\"Reconstructed\")\n",
"\n",
" ax[2].imshow(\n",
" keras.utils.array_to_img(depth_map[..., None]), cmap=\"inferno\"\n",
" )\n",
" ax[2].imshow(keras.utils.array_to_img(depth_map[..., None]), cmap=\"inferno\")\n",
" ax[2].set_title(\"Depth Map\")"
]
},
@@ -748,7 +751,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 0,
"metadata": {
"colab_type": "code"
},
@@ -876,8 +879,8 @@
"\n",
"## Way forward\n",
"\n",
"If anyone is interested to go deeper into NeRF, we have built a 3-part blog \n",
"series at [PyImageSearch](https://www.pyimagesearch.com).\n",
"If anyone is interested to go deeper into NeRF, we have built a 3-part blog\n",
"series at [PyImageSearch](https://pyimagesearch.com/).\n",
"\n",
"- [Prerequisites of NeRF](https://www.pyimagesearch.com/2021/11/10/computer-graphics-and-deep-learning-with-nerf-using-tensorflow-and-keras-part-1/)\n",
"- [Concepts of NeRF](https://www.pyimagesearch.com/2021/11/17/computer-graphics-and-deep-learning-with-nerf-using-tensorflow-and-keras-part-2/)\n",
@@ -928,4 +931,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}
Loading

0 comments on commit 832b075

Please sign in to comment.