From 7b49ea11fe4565d48eab3e440d9ef18a2f3d12cb Mon Sep 17 00:00:00 2001 From: "dattatraya.walake" Date: Mon, 28 Oct 2024 11:54:48 -0400 Subject: [PATCH] upgraded doc template to hugo-theme-relearn and added onboarding doc --- .gitmodules | 3 ++ docs/config.toml | 6 ++- docs/content/_index.md | 1 - docs/content/onboarding/_index.md | 74 +++++++++++++++++++++++++++++++ docs/themes/hugo-theme-relearn | 1 + 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 docs/content/onboarding/_index.md create mode 160000 docs/themes/hugo-theme-relearn diff --git a/.gitmodules b/.gitmodules index 17cb116..02aa639 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "docs/themes/hugo-theme-learn"] path = docs/themes/hugo-theme-learn url = https://github.com/matcornic/hugo-theme-learn.git +[submodule "docs/themes/hugo-theme-relearn"] + path = docs/themes/hugo-theme-relearn + url = https://github.com/McShelby/hugo-theme-relearn.git diff --git a/docs/config.toml b/docs/config.toml index da3d64e..1445d20 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -1,7 +1,7 @@ baseURL = 'https://databrickslabs.github.io/dlt-meta/' languageCode = 'en-us' title = 'DLT-META' -theme= "hugo-theme-learn" +theme= "hugo-theme-relearn" pluralizeListTitles = false canonifyURLs = true @@ -9,4 +9,6 @@ canonifyURLs = true description = "DLT-META Documentation" author = "Ravi Gawai (Databricks)" disableShortcutsTitle = true - disableLandingPageButton = false \ No newline at end of file + disableLandingPageButton = false + themeVariant = ['auto'] + themeVariantAuto = ['learn'] \ No newline at end of file diff --git a/docs/content/_index.md b/docs/content/_index.md index 1605559..5a3be93 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -4,7 +4,6 @@ date: 2021-08-04T14:50:11-04:00 draft: false --- -# DLT-META ## Project Overview DLT-META is a metadata-driven framework designed to work with Databricks Delta Live Tables (DLT). This framework enables the automation of bronze and silver data pipelines by leveraging metadata recorded in an onboarding JSON file. This file, known as the Dataflowspec, serves as the data flow specification, detailing the source and target metadata required for the pipelines. diff --git a/docs/content/onboarding/_index.md b/docs/content/onboarding/_index.md new file mode 100644 index 0000000..6290790 --- /dev/null +++ b/docs/content/onboarding/_index.md @@ -0,0 +1,74 @@ +--- +title: "Onboarding Guide" +date: 2021-08-04T14:50:11-04:00 +draft: false +weight: 4 +--- + +This document aims to provide complete information needed for anyone who would like to contribute to the dlt-meta project. Your contributions are vital to its success, whether you’re fixing bugs, improving documentation, or adding new features. + +# Steps + +## Step 0 \- Read the documentation + +Refer documentation wiki page [here](https://databricks.atlassian.net/wiki/spaces/FE/pages/2985722046/DLT-META) that will guide you to access different DLT-META resources like documentation, github repo, presentation etc. Read the getting started link [here](https://databrickslabs.github.io/dlt-meta/getting_started/) to understand pre-requisite , setup steps and configuration details. + +Prerequisite + +* Install Databricks CLI to you local machine +* Authenticate you current machine to a Databricks Workspace +* Python 3.8.0+ + +## Step 1 \- Join the slack channel + +\#dlt-meta + +## Step 2 \- Fork the Repository + +In case you may not be able to fork this repo because the repository is outside of your enterprise Databricks(EMU) , follow step3 or Fork using a personal github account. + +## Step 3 \- Clone the Repository Locally + +1. Run command “git clone [https://github.com/databrickslabs/dlt-meta.git](https://github.com/databrickslabs/dlt-meta.git)” it will create folder name “dlt-meta” + +## Step 4 \- Set Up the Development Environment + +2. cd dlt-meta +3. Create python virtual environment + * python \-m venv .venv or python3 \-m venv .venv +4. Activate python virtual environment + * source .venv/bin/activate +5. Install databricks sdk + * pip install databricks-sdk +6. Install code editor like VS code or any other. +7. Import project into VS code File \> Open folder \> select above dlt-meta folder from your system +8. Install setuptools and wheel if not already installed + * pip install setuptools wheel +9. Install the project dependencies specified in setup.py + * pip install \-e . +10. Build the project + * python setup.py sdist bdist\_wheel +11. Install additional dependencies + * pip install pyspark + * pip install delta-spark + * Pip install pytest + +## Step 5 \- Running Unit and Integration Tests + +* Unit test are at tests folder + * To run the test cases, use the pytest command in the terminal + * To run all tests run \- pytest + * To run specific test- pytest \-k “test\_case\_name” + +* Integration Tests are at integration\_tests folder + * To run integration test run file run\_integration\_tests.py with mandatory required argument as below + * e.g. run\_integration\_tests.py \--uc\_catalog\_name datta\_demo \--cloud\_provider\_name aws \--dbr\_version 14.3 \--source cloudfiles \--dbfs\_path "dbfs:/tmp/DLT-META/" \--profile DEFAULT + +## Step 6 \- Find Beginner-Friendly Issues + +## Step 7 \- Work on the Issue + +## Step 8 \- Submit a PR + +## Step 9 \- Celebrate your Contribution + diff --git a/docs/themes/hugo-theme-relearn b/docs/themes/hugo-theme-relearn new file mode 160000 index 0000000..28fce6b --- /dev/null +++ b/docs/themes/hugo-theme-relearn @@ -0,0 +1 @@ +Subproject commit 28fce6b04c414523280c53ee02f9f3a94d9d23da