Skip to content

Commit

Permalink
Bump docs
Browse files Browse the repository at this point in the history
  • Loading branch information
@Eve-ning
Eve-ning committed Oct 25, 2023
1 parent 04befb0 commit 5c1be99
Show file tree
Hide file tree
Showing 10 changed files with 170 additions and 14 deletions.
5 changes: 3 additions & 2 deletions Writerside/d.tree
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
<toc-element topic="Overview.md"/>
<toc-element topic="Getting-Started.md"/>
<toc-element topic="load.dataset.md"/>
<toc-element topic="preprocessing.scale.md"/>
<toc-element topic="preprocessing.morphology.md"/>
<toc-element topic="preprocessing.scale.md">
</toc-element>
<toc-element topic="preprocessing.extract_segments.md"/>
<toc-element topic="preprocessing.morphology.md"/>
</instance-profile>
2 changes: 1 addition & 1 deletion docs/HelpTOC.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"entities":{"pages":{"Overview":{"id":"Overview","title":"Overview","url":"overview.html","level":0,"tabIndex":0},"Getting-Started":{"id":"Getting-Started","title":"Getting Started","url":"getting-started.html","level":0,"tabIndex":1}}},"topLevelIds":["Overview","Getting-Started"]}
{"entities":{"pages":{"Overview":{"id":"Overview","title":"Overview","url":"overview.html","level":0,"tabIndex":0},"Getting-Started":{"id":"Getting-Started","title":"Getting Started","url":"getting-started.html","level":0,"tabIndex":1},"load.dataset":{"id":"load.dataset","title":"load.dataset","url":"load-dataset.html","level":0,"tabIndex":2},"preprocessing.scale":{"id":"preprocessing.scale","title":"preprocessing.scale","url":"preprocessing-scale.html","level":0,"tabIndex":3},"preprocessing.extract_segments":{"id":"preprocessing.extract_segments","title":"preprocessing.extract_segments","url":"preprocessing-extract-segments.html","level":0,"tabIndex":4},"preprocessing.morphology":{"id":"preprocessing.morphology","title":"preprocessing.morphology","url":"preprocessing-morphology.html","level":0,"tabIndex":5}}},"topLevelIds":["Overview","Getting-Started","load.dataset","preprocessing.scale","preprocessing.extract_segments","preprocessing.morphology"]}
2 changes: 1 addition & 1 deletion docs/Map.jhm
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
<?xml version='1.0' encoding='UTF-8'?><map version="2.0"><mapID target="Overview.md" url="overview.html" default="yes"/><mapID target="Overview" url="overview.html" default="yes"/><mapID target="Getting-Started.md" url="getting-started.html" default="no"/><mapID target="Getting-Started" url="getting-started.html" default="no"/></map>
<?xml version='1.0' encoding='UTF-8'?><map version="2.0"><mapID target="Overview.md" url="overview.html" default="yes"/><mapID target="Overview" url="overview.html" default="yes"/><mapID target="Getting-Started.md" url="getting-started.html" default="no"/><mapID target="Getting-Started" url="getting-started.html" default="no"/><mapID target="load.dataset.md" url="load-dataset.html" default="no"/><mapID target="load.dataset" url="load-dataset.html" default="no"/><mapID target="preprocessing.scale.md" url="preprocessing-scale.html" default="no"/><mapID target="preprocessing.scale" url="preprocessing-scale.html" default="no"/><mapID target="preprocessing.extract_segments.md" url="preprocessing-extract-segments.html" default="no"/><mapID target="preprocessing.extract_segments" url="preprocessing-extract-segments.html" default="no"/><mapID target="preprocessing.morphology.md" url="preprocessing-morphology.html" default="no"/><mapID target="preprocessing.morphology" url="preprocessing-morphology.html" default="no"/></map>
18 changes: 9 additions & 9 deletions docs/getting-started.html
View file

Large diffs are not rendered by default.

31 changes: 31 additions & 0 deletions docs/load-dataset.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<!DOCTYPE html SYSTEM "about:legacy-compat"><html lang="en-US" data-colors-preset="contrast" data-primary-color="#307FFF"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta charset="UTF-8"><meta name="robots" content="noindex"> <meta name="built-on" content="2023-10-25T17:57:20.5857123"><meta name="build-number" content="${buildNumber}"> <title>load.dataset | Documentation</title><script id="virtual-toc-data" type="application/json">[{"id":"classes","level":0,"title":"Classes","anchor":"#classes"},{"id":"usage","level":0,"title":"Usage","anchor":"#usage"},{"id":"api","level":0,"title":"API","anchor":"#api"}]</script><script id="topic-shortcuts" type="application/json"></script><link href="https://resources.jetbrains.com/writerside/apidoc/6.1.5-b176/app.css" rel="stylesheet"> <link rel="apple-touch-icon" sizes="180x180" href="https://jetbrains.com/apple-touch-icon.png"><link rel="icon" type="image/png" sizes="32x32" href="https://jetbrains.com/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="https://jetbrains.com/favicon-16x16.png"><link rel="manifest" href="https://jetbrains.com/site.webmanifest"><link rel="mask-icon" href="https://jetbrains.com/safari-pinned-tab.svg" color="#000000"><meta name="msapplication-TileColor" content="#000000"/><meta name="msapplication-TileImage" content="https://resources.jetbrains.com/storage/ui/favicons/mstile-144x144.png"/><meta name="msapplication-square70x70logo" content="https://resources.jetbrains.com/storage/ui/favicons/mstile-70x70.png"/><meta name="msapplication-square150x150logo" content="https://resources.jetbrains.com/storage/ui/favicons/mstile-150x150.png"/><meta name="msapplication-wide310x150logo" content="https://resources.jetbrains.com/storage/ui/favicons/mstile-310x150.png"/><meta name="msapplication-square310x310logo" content="https://resources.jetbrains.com/storage/ui/favicons/mstile-310x310.png"/> <meta name="image" content=""><!-- Open Graph --><meta property="og:title" content="load.dataset | Documentation"/><meta property="og:description" content=""/><meta property="og:image" content=""/><meta property="og:site_name" content="Documentation Help"/><meta property="og:type" content="website"/><meta property="og:locale" content="en_US"/><meta property="og:url" content="/d/0.0.4/load-dataset.html"/><!-- End Open Graph --><!-- Twitter Card --><meta name="twitter:card" content="summary_large_image"><meta name="twitter:site" content=""><meta name="twitter:title" content="load.dataset | Documentation"><meta name="twitter:description" content=""><meta name="twitter:creator" content=""><meta name="twitter:image:src" content=""><!-- End Twitter Card --><!-- Schema.org WebPage --><script type="application/ld+json"> { "@context": "http://schema.org", "@type": "WebPage", "@id": "/d/0.0.4/load-dataset.html#webpage", "url": "/d/0.0.4/load-dataset.html", "name": "load.dataset | Documentation", "description": "", "image": "", "inLanguage":"en-US" }</script><!-- End Schema.org --><!-- Schema.org WebSite --><script type="application/ld+json"> { "@type": "WebSite", "@id": "/d/#website", "url": "/d/", "name": "Documentation Help" }</script><!-- End Schema.org --> <!-- Mermaid light/dark themes --> <link rel="stylesheet" type="text/css" href="mermaid.css"> </head> <body data-id="load.dataset" data-main-title="load.dataset" data-article-props="{&quot;seeAlsoStyle&quot;:&quot;links&quot;}" data-template="article" data-breadcrumbs="" > <div class="wrapper"><main class="panel _main"><header class="panel__header"><div class="container"><h3>Documentation 0.0.4 Help</h3><div class="panel-trigger"></div></div></header><section class="panel__content"><div class="container"><article class="article" data-shortcut-switcher="inactive"><h1 data-toc="load.dataset" id="load.dataset.md" >load.dataset</h1> <div class="micro-format" data-content="{&quot;microFormat&quot;:[&quot;\u003cp id\u003d\&quot;feb21bd9_2797\&quot;\u003eLoad datasets from our GCS bucket.\u003c/p\u003e&quot;]}"></div> <aside class="prompt" data-type="warning" data-title="" id="feb21bd9_2721"><p id="feb21bd9_2722">You need to Set-Up <a href="getting-started.html#gcloud" id="feb21bd9_2723" data-tooltip="Before starting development, take a look at our repository structure. This will help you understand where to put your code." >Google Cloud</a> with the appropriate permissions to use this library.</p></aside><section class="chapter"><h2 id="classes" data-toc="classes">Classes</h2><dl id="feb21bd9_2725" data-style="title-top"><dt id="feb21bd9_2726" data-expandable="false">FRDCDownloader</dt><dd><p>This facilitates authentication and downloading from GCS.</p></dd><dt id="feb21bd9_2727" data-expandable="false">FRDCDataset</dt><dd><p>This uses the Downloader to download and load the dataset. It also implements useful helper functions to load FRDC-specific datasets, such as loading our images and labels.</p></dd></dl></section><section class="chapter"><h2 id="usage" data-toc="usage">Usage</h2><p id="feb21bd9_2728">An example loading our Chestnut Nature Park dataset. We retrieve the</p><ul class="list _ul" id="feb21bd9_2729" ><li class="list__item" id="feb21bd9_2730"><p>hyperspectral bands</p></li><li class="list__item" id="feb21bd9_2731"><p>order of the bands</p></li><li class="list__item" id="feb21bd9_2732"><p>bounding boxes</p></li><li class="list__item" id="feb21bd9_2733"><p>labels</p></li></ul><div class="code-block" data-lang="python" >
from frdc.load import FRDCDataset

ds = FRDCDataset(site='chestnut_nature_park',
date='20201218',
version=None, )
ar, order = ds.get_ar_bands()
bounds, labels = ds.get_bounds_and_labels()
</div><section class="chapter"><div class="collapse"><div class="collapse__title"><h3 id="custom-authentication-downloads" data-toc="custom-authentication-downloads">Custom Authentication &amp; Downloads</h3></div><div class="collapse__content"><p id="feb21bd9_2735">If you need granular control over</p><ul class="list _ul" id="feb21bd9_2736" ><li class="list__item" id="feb21bd9_2737"><p>where the files are downloaded</p></li><li class="list__item" id="feb21bd9_2738"><p>the credentials used</p></li><li class="list__item" id="feb21bd9_2739"><p>the project used</p></li><li class="list__item" id="feb21bd9_2740"><p>the bucket used</p></li></ul><p id="feb21bd9_2741">Then pass in a <code class="code" id="feb21bd9_2742">FRDCDownloader</code> object to <code class="code" id="feb21bd9_2743">FRDCDataset</code>.</p><div class="code-block" data-lang="python" >
from frdc.load import FRDCDownloader, FRDCDataset

dl = FRDCDownloader(credentials=...,
local_dataset_root_dir=...,
project_id=...,
bucket_name=...)
ds = FRDCDataset(site='chestnut_nature_park',
date='20201218',
version=None,
dl=dl)
ar, order = ds.get_ar_bands()
bounds, labels = ds.get_bounds_and_labels()
</div><p id="feb21bd9_2745">If you have a file not easily downloadable by <code class="code" id="feb21bd9_2746">FRDCDataset</code>, you can use <code class="code" id="feb21bd9_2747">FRDCDownloader</code> to download it.</p><div class="code-block" data-lang="python" >
from frdc.load import FRDCDownloader

dl = FRDCDownloader(credentials=...,
local_dataset_root_dir=...,
project_id=...,
bucket_name=...)

dl.download_file(path_glob=&quot;path/to/gcs/file&quot;)
</div><aside class="prompt" data-type="tip" data-title="" id="feb21bd9_2749"><p>This will automatically save the file to the local dataset root dir.</p></aside></div></div></section></section><section class="chapter"><h2 id="api" data-toc="api">API</h2><section class="chapter"><h3 id="frdcdataset" data-toc="frdcdataset">FRDCDataset</h3><dl id="feb21bd9_2750" data-style="title-top"><dt id="feb21bd9_2751" data-expandable="false">FRDCDataset(site, date, version, dl)</dt><dd><p><b id="feb21bd9_2752">Initializes the dataset downloader.</b></p><br><p> This doesn't immediately download the dataset, but only when you call the <code class="code" id="feb21bd9_2754">get_*</code> functions. </p><br><p> The site, date, version must match the dataset path on GCS. For example if the dataset is at <code class="code" id="feb21bd9_2756">gs://frdc-scan/my-site/date/90deg/map</code>, </p><ul class="list _ul" id="feb21bd9_2757" ><li class="list__item" id="feb21bd9_2758"><p>site: <code class="code" id="feb21bd9_2759">'my-site'</code></p></li><li class="list__item" id="feb21bd9_2760"><p>date: <code class="code" id="feb21bd9_2761">'20201218'</code></p></li><li class="list__item" id="feb21bd9_2762"><p>version: <code class="code" id="feb21bd9_2763">'90deg/map'</code></p></li></ul><aside class="prompt" data-type="note" data-title="" id="feb21bd9_2764"><p>If you don't want to search up GCS, you can use FRDCDownloader to list all datasets, and their versions with <code class="code" id="feb21bd9_2765">FRDCDownloader().list_gcs_datasets()</code></p></aside><aside class="prompt" data-type="tip" data-title="" id="feb21bd9_2766"><p>If <code class="code" id="feb21bd9_2767">dl</code> is None, it will create a new FRDCDownloader. Usually, you don't need to pass this in unless you have a custom credential, or project.</p></aside></dd><dt id="feb21bd9_2768" data-expandable="false">get_ar_bands()</dt><dd><p><b id="feb21bd9_2769">Gets the NDArray bands (H x W x C) and channel order as <code class="code" id="feb21bd9_2770">tuple[np.ndarray, list[str]]</code>.</b></p><br><p> This downloads (if missing) and retrieves the stacked NDArray bands. This wraps around <code class="code" id="feb21bd9_2772">get_ar_bands_as_dict()</code>, thus if you want more control over how the bands are loaded, use that instead.</p></dd><dt id="feb21bd9_2773" data-expandable="false">get_ar_bands_as_dict()</dt><dd><p><b id="feb21bd9_2774">Gets the NDArray bands (H x W) as a <code class="code" id="feb21bd9_2775">dict[str, np.ndarray]</code>.</b></p><br><p> This downloads (if missing) and retrieves the individual NDArray bands as a dictionary. The keys are the band names, and the values are the NDArray bands.</p></dd><dt id="feb21bd9_2777" data-expandable="false">get_bounds_and_labels()</dt><dd><p><b id="feb21bd9_2778">Gets the bounding boxes and labels as <code class="code" id="feb21bd9_2779">tuple[list[Rect], list[str]]</code>.</b></p><br><p> This downloads (if missing) and retrieves the bounding boxes and labels as a tuple. The first element is a list of bounding boxes, and the second element is a list of labels. </p><br><aside class="prompt" data-type="tip" data-title="" id="feb21bd9_2782"><p>The bounding boxes are in the format of <code class="code" id="feb21bd9_2783">Rect</code>, a <code class="code" id="feb21bd9_2784">namedtuple</code> of x0, y0, x1, y1.</p></aside></dd></dl></section><section class="chapter"><h3 id="frdcdownloader" data-toc="frdcdownloader">FRDCDownloader</h3><dl id="feb21bd9_2785" data-style="title-top"><dt id="feb21bd9_2786" data-expandable="false">list_gcs_datasets(anchor)</dt><dd><p><b id="feb21bd9_2787">Lists all GCS datasets in the bucket as <code class="code" id="feb21bd9_2788">DataFrame</code></b></p><br><p> This works by checking which folders have a specific file, which we call the <code class="code" id="feb21bd9_2790">anchor</code>.</p></dd><dt id="feb21bd9_2791" data-expandable="false">download_file(path_glob, local_exists_ok)</dt><dd><p><b id="feb21bd9_2792">Downloads a file from GCS.</b></p><br><p> This takes in a path glob, a string containing wildcards, and downloads exactly 1 file. If it matches 0 or more than 1 file, it will raise an error. </p><br><p> If <code class="code" id="feb21bd9_2795">local_exists_ok</code> is True, it will not download the file if it already exists locally. However, if it's False, it will download the file only if the hashes don't match. </p><aside class="prompt" data-type="note" data-title="" id="feb21bd9_2796"><p>Use this if you have a file on GCS that can't be downloaded via FRDCDataset.</p></aside></dd></dl></section></section><div class="last-modified"> Last modified: 25 October 2023</div><div data-feedback-placeholder="true"></div><div class="navigation-links _bottom"> <a class="navigation-links__prev" href="getting-started.html">Getting Started</a> <a class="navigation-links__next" href="preprocessing-scale.html">preprocessing.scale</a> </div></article><div id="disqus_thread"></div></div></section></main></div> <script src="https://resources.jetbrains.com/writerside/apidoc/6.1.5-b176/app.js"></script></body></html>
Loading

0 comments on commit 5c1be99

Please sign in to comment.