-
Notifications
You must be signed in to change notification settings - Fork 85
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'dev' into jupyter-recursion-error
- Loading branch information
Showing
45 changed files
with
677 additions
and
225 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| """ | ||
| .. _editing: | ||
| Editing NWB files | ||
| ================= | ||
| This tutorial demonstrates how to edit NWB files in-place to make small changes to | ||
| existing containers. To add or remove containers from an NWB file, see | ||
| :ref:`modifying_data`. How and whether it is possible to edit an NWB file depends on the | ||
| storage backend and the type of edit. | ||
| .. warning:: | ||
| Manually editing an existing NWB file can make the file invalid if you are not | ||
| careful. We highly recommend making a copy before editing and running a validation | ||
| check on the file after editing it. See :ref:`validating`. | ||
| Editing datasets | ||
| ---------------- | ||
| When reading an HDF5 NWB file, PyNWB exposes :py:class:`h5py.Dataset` objects, which can | ||
| be edited in place. For this to work, you must open the file in read/write mode | ||
| (``"r+"`` or ``"a"``). | ||
| First, let's create an NWB file with data: | ||
| """ | ||
| from pynwb import NWBHDF5IO, NWBFile, TimeSeries | ||
| from datetime import datetime | ||
| from dateutil.tz import tzlocal | ||
| import numpy as np | ||
|
|
||
| nwbfile = NWBFile( | ||
| session_description="my first synthetic recording", | ||
| identifier="EXAMPLE_ID", | ||
| session_start_time=datetime.now(tzlocal()), | ||
| session_id="LONELYMTN", | ||
| ) | ||
|
|
||
| nwbfile.add_acquisition( | ||
| TimeSeries( | ||
| name="synthetic_timeseries", | ||
| description="Random values", | ||
| data=np.random.randn(100, 100), | ||
| unit="m", | ||
| rate=10e3, | ||
| ) | ||
| ) | ||
|
|
||
| with NWBHDF5IO("test_edit.nwb", "w") as io: | ||
| io.write(nwbfile) | ||
|
|
||
| ############################################## | ||
| # Now, let's edit the values of the dataset | ||
|
|
||
| with NWBHDF5IO("test_edit.nwb", "r+") as io: | ||
| nwbfile = io.read() | ||
| nwbfile.acquisition["synthetic_timeseries"].data[:10] = 0.0 | ||
|
|
||
|
|
||
| ############################################## | ||
| # You can edit the attributes of that dataset through the ``attrs`` attribute: | ||
|
|
||
| with NWBHDF5IO("test_edit.nwb", "r+") as io: | ||
| nwbfile = io.read() | ||
| nwbfile.acquisition["synthetic_timeseries"].data.attrs["unit"] = "volts" | ||
|
|
||
| ############################################## | ||
| # Changing the shape of dataset | ||
| # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| # Whether it is possible to change the shape of a dataset depends on how the dataset was | ||
| # created. If the dataset was created with a flexible shape, then it is possible to | ||
| # change in-place. Creating a dataset with a flexible shape is done by specifying the | ||
| # ``maxshape`` argument of the :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` class | ||
| # constructor. Using a ``None`` value for a component of the ``maxshape`` tuple allows | ||
| # the size of the corresponding dimension to grow, such that is can be be reset arbitrarily long | ||
| # in that dimension. Chunking is required for datasets with flexible shapes. Setting ``maxshape``, | ||
| # hence, automatically sets chunking to ``True``, if not specified. | ||
| # | ||
| # First, let's create an NWB file with a dataset with a flexible shape: | ||
|
|
||
| from hdmf.backends.hdf5.h5_utils import H5DataIO | ||
|
|
||
| nwbfile = NWBFile( | ||
| session_description="my first synthetic recording", | ||
| identifier="EXAMPLE_ID", | ||
| session_start_time=datetime.now(tzlocal()), | ||
| session_id="LONELYMTN", | ||
| ) | ||
|
|
||
| data_io = H5DataIO(data=np.random.randn(100, 100), maxshape=(None, 100)) | ||
|
|
||
| nwbfile.add_acquisition( | ||
| TimeSeries( | ||
| name="synthetic_timeseries", | ||
| description="Random values", | ||
| data=data_io, | ||
| unit="m", | ||
| rate=10e3, | ||
| ) | ||
| ) | ||
|
|
||
| with NWBHDF5IO("test_edit2.nwb", "w") as io: | ||
| io.write(nwbfile) | ||
|
|
||
| ############################################## | ||
| # The ``None``value in the first component of ``maxshape`` means that the | ||
| # the first dimension of the dataset is unlimited. By setting the second dimension | ||
| # of ``maxshape`` to ``100``, that dimension is fixed to be no larger than ``100``. | ||
| # If you do not specify a``maxshape``, then the shape of the dataset will be fixed | ||
| # to the shape that the dataset was created with. Here, you can change the shape of | ||
| # the first dimension of this dataset. | ||
|
|
||
|
|
||
| with NWBHDF5IO("test_edit2.nwb", "r+") as io: | ||
| nwbfile = io.read() | ||
| nwbfile.acquisition["synthetic_timeseries"].data.resize((200, 100)) | ||
|
|
||
| ############################################## | ||
| # This will change the shape of the dataset in-place. If you try to change the shape of | ||
| # a dataset with a fixed shape, you will get an error. | ||
| # | ||
| # .. note:: | ||
| # There are several types of dataset edits that cannot be done in-place: changing the | ||
| # shape of a dataset with a fixed shape, or changing the datatype, compression, | ||
| # chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using | ||
| # the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See | ||
| # :ref:`modifying_data` for more information. | ||
| # | ||
| # Editing groups | ||
| # -------------- | ||
| # Editing of groups is not yet supported in PyNWB. | ||
| # To edit the attributes of a group, open the file and edit it using ``h5py``: | ||
|
|
||
| import h5py | ||
|
|
||
| with h5py.File("test_edit.nwb", "r+") as f: | ||
| f["acquisition"]["synthetic_timeseries"].attrs["description"] = "Random values in volts" | ||
|
|
||
| ############################################## | ||
| # .. warning:: | ||
| # Be careful not to edit values that will bring the file out of compliance with the | ||
| # NWB specification. | ||
| # | ||
| # Renaming groups and datasets | ||
| # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| # Rename groups and datasets in-place using the :py:meth:`~h5py.Group.move` method. For example, to rename | ||
| # the ``"synthetic_timeseries"`` group: | ||
|
|
||
| with h5py.File("test_edit.nwb", "r+") as f: | ||
| f["acquisition"].move("synthetic_timeseries", "synthetic_timeseries_renamed") | ||
|
|
||
| ############################################## | ||
| # You can use this same technique to move a group or dataset to a different location in | ||
| # the file. For example, to move the ``"synthetic_timeseries_renamed"`` group to the | ||
| # ``"analysis"`` group: | ||
|
|
||
| with h5py.File("test_edit.nwb", "r+") as f: | ||
| f["acquisition"].move( | ||
| "synthetic_timeseries_renamed", | ||
| "/analysis/synthetic_timeseries_renamed", | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| """ | ||
| Zarr IO | ||
| ======= | ||
| Zarr is an alternative backend option for NWB files. It is a Python package that | ||
| provides an implementation of chunked, compressed, N-dimensional arrays. Zarr is a good | ||
| option for large datasets because, like HDF5, it is designed to store data on disk and | ||
| only load the data into memory when needed. Zarr is also a good option for parallel | ||
| computing because it supports concurrent reads and writes. | ||
| Note that the Zarr native storage formats are optimized for storage in cloud storage | ||
| (e.g., S3). For very large files, Zarr will create many files which can lead to | ||
| issues for traditional file system (that are not cloud object stores) due to limitations | ||
| on the number of files per directory (this affects local disk, GDrive, Dropbox etc.). | ||
| Zarr read and write is provided by the :hdmf-zarr:`hdmf-zarr<>` package. First, create an | ||
| an NWBFile using PyNWB. | ||
| """ | ||
|
|
||
| # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_plot_nwbzarrio.png' | ||
|
|
||
|
|
||
| from datetime import datetime | ||
| from dateutil.tz import tzlocal | ||
|
|
||
| import numpy as np | ||
| from pynwb import NWBFile, TimeSeries | ||
|
|
||
| # Create the NWBFile. Substitute your NWBFile generation here. | ||
| nwbfile = NWBFile( | ||
| session_description="my first synthetic recording", | ||
| identifier="EXAMPLE_ID", | ||
| session_start_time=datetime.now(tzlocal()), | ||
| session_id="LONELYMTN", | ||
| ) | ||
|
|
||
| ####################################################################################### | ||
| # Dataset Configuration | ||
| # --------------------- | ||
| # Like HDF5, Zarr provides options to chunk and compress datasets. To leverage these | ||
| # features, replace all :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` with the analogous | ||
| # :py:class:`~hdmf_zarr.utils.ZarrDataIO`, which takes compressors specified by the | ||
| # :py:mod:`numcodecs` library. For example, here is an example :py:class:`.TimeSeries` | ||
| # where the ``data`` Dataset is compressed with a Blosc-zstd compressor: | ||
|
|
||
| from numcodecs import Blosc | ||
| from hdmf_zarr import ZarrDataIO | ||
|
|
||
| data_with_zarr_data_io = ZarrDataIO( | ||
| data=np.random.randn(100, 100), | ||
| chunks=(10, 10), | ||
| fillvalue=0, | ||
| compressor=Blosc(cname='zstd', clevel=3, shuffle=Blosc.SHUFFLE) | ||
| ) | ||
|
|
||
| ####################################################################################### | ||
| # Now add it to the :py:class:`.NWBFile`. | ||
|
|
||
| nwbfile.add_acquisition( | ||
| TimeSeries( | ||
| name="synthetic_timeseries", | ||
| data=data_with_zarr_data_io, | ||
| unit="m", | ||
| rate=10e3, | ||
| ) | ||
| ) | ||
|
|
||
| ####################################################################################### | ||
| # Writing to Zarr | ||
| # --------------- | ||
| # To write NWB files to Zarr, replace the :py:class:`~pynwb.NWBHDF5IO` with | ||
| # :py:class:`hdmf_zarr.nwb.NWBZarrIO`. | ||
|
|
||
| from hdmf_zarr.nwb import NWBZarrIO | ||
| import os | ||
|
|
||
| path = "zarr_tutorial.nwb.zarr" | ||
| absolute_path = os.path.abspath(path) | ||
| with NWBZarrIO(path=path, mode="w") as io: | ||
| io.write(nwbfile) | ||
|
|
||
| ####################################################################################### | ||
| # .. note:: | ||
| # The main reason for using the ``absolute_path`` here is for testing purposes to | ||
| # ensure links and references work as expected. Otherwise, using the relative path | ||
| # here instead is fine. | ||
| # | ||
| # Reading from Zarr | ||
| # ----------------- | ||
| # To read NWB files from Zarr, replace the :py:class:`~pynwb.NWBHDF5IO` with the analogous | ||
| # :py:class:`hdmf_zarr.nwb.NWBZarrIO`. | ||
|
|
||
| with NWBZarrIO(path=absolute_path, mode="r") as io: | ||
| read_nwbfile = io.read() | ||
|
|
||
| ####################################################################################### | ||
| # .. note:: | ||
| # For more information, see the :hdmf-zarr:`hdmf-zarr documentation<>`. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.