diff --git a/docs/acknowledgement.md b/docs/acknowledgement.md index cedea49b..6814bf97 100644 --- a/docs/acknowledgement.md +++ b/docs/acknowledgement.md @@ -1,6 +1,11 @@ +# Acknowledgements + The project would not have been possible without the following libraries: - [@jerryjliu](https://github.com/jerryjliu)'s [Llama Index](https://github.com/jerryjliu/llama_index) +- [LiteLLM](https://github.com/BerriAI/litellm) +- [Pydantic](https://github.com/pydantic/pydantic) +- [OpenDAL](https://github.com/apache/incubator-opendal) - [Streamlit](https://streamlit.io/) and many open-source Streamlit components as well as its [Community Cloud](https://streamlit.io/cloud) This project was created using [cookiecutter](https://github.com/cookiecutter/cookiecutter) with: @@ -11,5 +16,6 @@ Which includes a set of beautiful Python build toolings such as: - [Poetry](https://python-poetry.org/) and [Poe](https://poethepoet.natn.io/) - [Ruff](https://github.com/astral-sh/ruff) +- [OpenTelemetry](https://github.com/open-telemetry) This docs site is created using [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/). diff --git a/docs/assets/docq-diag-nov2023.png b/docs/assets/docq-diag-nov2023.png new file mode 100644 index 00000000..a5a01543 Binary files /dev/null and b/docs/assets/docq-diag-nov2023.png differ diff --git a/docs/developer-guide/getting-started.md b/docs/developer-guide/getting-started.md index d3c92e75..69e977a2 100644 --- a/docs/developer-guide/getting-started.md +++ b/docs/developer-guide/getting-started.md @@ -110,7 +110,7 @@ poe run-otel ### Build a docker image for tests ```sh -poe docker-build --target test --build-tag 3.10-alpine --test true +poe docker-build --target test --test true ``` ```sh @@ -130,13 +130,6 @@ poe docker-run --target test ``` - TIP: Make sure `docker.env` is in the `.gitignore` file so it isn't checked in with secrets. It's also safer if you only have the environment variable name for secrets and set the value in your shell with `export`. -### Build a container image with cached dependencies - -This is useful when iterating on the dockerfile itself - - ```sh - poe docker-build-pipcached - ``` ### Run container diff --git a/docs/developer-guide/web-api.md b/docs/developer-guide/web-api.md new file mode 100644 index 00000000..732a8b95 --- /dev/null +++ b/docs/developer-guide/web-api.md @@ -0,0 +1,5 @@ +# Web API (alpha) + +Coming soon... + +[Join our Slack](https://join.slack.com/t/docqai/shared_invite/zt-27p17lu6v-6KLJxSmt61vfNqCavSE73A) to discuss diff --git a/docs/index.md b/docs/index.md index 6aadefaf..9f7e812d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,7 @@ # Introduction +![Docq Illustration](./assets/docq-diag-nov2023.png) + Docq helps your _**businesses**_ gain _**AI-based knowledge insight**_ from your _**private data, securely!**_ Think of Docq as **your employees' private and secure ChatGPT, a second-brain that understands your business**. @@ -15,4 +17,3 @@ For the developers and other tech-savvy audience, Docq is like **WordPress for g [User Guide](./user-guide/getting-started.md) - This is section is for _end-users_ (employees) and admins of the Docq app. It covers how to deploy and config Docq and how to use the AI chat funcitonality to help with your daily work. [Developer Guide](./developer-guide/getting-started.md) - This section id for those wanting to understand the Docq code base, make code changes to customise Docq, extend Docq, and use Docq as a platform to build AI powered applications on top. -- [File Storage Services](./developer-guide/file-storage-services.md) - Docq supports multiple file storage services. This section covers how to setup the supported file storage services. diff --git a/docs/overview/deployment-scenarios.md b/docs/overview/deployment-scenarios.md index ad793d96..b9bce06f 100644 --- a/docs/overview/deployment-scenarios.md +++ b/docs/overview/deployment-scenarios.md @@ -13,6 +13,8 @@ Docq mainly caters for two deployment scenarios: Plus +- **SaaS Multi-tenant** - An instance of Docq hosted and operated in our cloud provider account (Azure UK South region). Each organisations data is isolated but you are sharing a single instance of Docq and LLMs. This is the fast way to get going. Head over to [https://docq.ai/#plans](https://docq.ai/#plans) to subscribe. +- **SaaS Dedicated** - Your own dedicated instance of Docq hosted and operated in our cloud account. You don't share any infrastructure with other organisations. - **On-Premise Deployment**: It is another option and we are happy to support you with a bespoke setup. Docq has been designed from ground up for the secure cloud and on-prem deployment scenarios with strict data security and privacy requirement in mind. However it should not stop all other businesses from adopting Docq with the flexible deployment setup if it suits their requirements. @@ -26,7 +28,7 @@ The diagram above demonstrates the possibilities of have different deployment se ### Secure Cloud Deployment -If your business have clear data security and privacy guideline, we recommend one of the big three cloud vendors: +If your business have clear data security and privacy guidelines, we recommend one of the big three cloud vendors: - Microsoft Cloud (Azure) - Amazon Web Services (AWS) diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 224cc5de..159be19c 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -2,14 +2,16 @@ --md-primary-fg-color: #202020; --md-primary-fg-color--light: #373737; --md-primary-fg-color--dark: #373737; - --md-typeset-a-color: #4cae4f + --md-typeset-a-color: #542CF2; + //--md-typeset-a-color: #4cae4f; } [data-md-color-scheme="slate"] { --md-primary-fg-color: #202020; --md-primary-fg-color--light: #373737; --md-primary-fg-color--dark: #373737; - --md-typeset-a-color: #71c174 + --md-typeset-a-color: #542CF2; + //--md-typeset-a-color: #71c174; } .mkapi-node code.mkapi-item-name, code.mkapi-object-parenthesis, code.mkapi-object-signature, span.mkapi-item-type, span.mkapi-item-dash, mkapi-item-description { diff --git a/docs/user-guide/agents-actions.md b/docs/user-guide/agents-actions.md new file mode 100644 index 00000000..28b1e3c4 --- /dev/null +++ b/docs/user-guide/agents-actions.md @@ -0,0 +1,5 @@ +# Agents and Actions + +Coming soon... + +[Join our Slack](https://join.slack.com/t/docqai/shared_invite/zt-27p17lu6v-6KLJxSmt61vfNqCavSE73A) to discuss. diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index c30db946..97aae434 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -1,3 +1,4 @@ # Configuration -- [Configure Spaces](./config-spaces.md) \ No newline at end of file +- [Configure Spaces](./config-spaces.md) +- [Configure File Storage Services](./file-storage-services.md) \ No newline at end of file diff --git a/docs/developer-guide/file-storage-services.md b/docs/user-guide/file-storage-services.md similarity index 89% rename from docs/developer-guide/file-storage-services.md rename to docs/user-guide/file-storage-services.md index 76f39feb..71ddb4d3 100644 --- a/docs/developer-guide/file-storage-services.md +++ b/docs/user-guide/file-storage-services.md @@ -2,14 +2,6 @@ Docq supports multiple cloud file storage services as a data source. This section covers how to set up the supported file storage services. - - [Google Drive](#file-storage-google-drive) - - [Setup and Configure Google Cloud project](#setup-and-configure-google-cloud-project) - - [Configure Docq web application for Google Drive](#configure-docq-web-application-for-google-drive) - - [OneDrive](#file-storage-onedrive) - - [Setup and Configure Microsoft Azure Application](#setup-and-configure-microsoft-azure-application) - - [Configure Docq web application for OneDrive](#configure-docq-web-application-for-onedrive) - - ## File storage: Google Drive This guide aims to assist developers in integrating Google Drive with Docq. The focus will be on setting up the Google Drive API and obtaining the necessary credentials. @@ -39,7 +31,6 @@ After setting up the Google Cloud project and configuring the Google Drive API, Note: The Google Drive data source will be automatically disabled if any of the above environment variables are not set. - ## File storage: OneDrive This guide aims to assist developers in integrating OneDrive with Docq. The focus will be on setting up the Microsoft Graph API and obtaining the necessary credentials. @@ -49,6 +40,7 @@ This guide aims to assist developers in integrating OneDrive with Docq. The focu - [Register an Application](https://entra.microsoft.com/#view/Microsoft_AAD_RegisteredApps/CreateApplicationBlade/isMSAApp~/false) in the Microsoft Entra ID center ![Register an Application](../assets/azure_register_an_application.png) + - Configure the following under `Redirect URI` - Select `Web` as the platform - Enter the redirect URL to the following path `/Admin_Spaces/` e.g. `http://localhost:8501/Admin_Spaces/` diff --git a/mkdocs.yaml b/mkdocs.yaml index aea53798..cc5b2e06 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -1,6 +1,6 @@ -site_name: Docq - Private & Secure AI Knowledge Insight +site_name: Docq - Private & Secure ChatGPT AI Knowledge Insight site_url: https://docqai.github.io/docq -site_description: Docq offers self-hosted, private and secure AI for knowledge insight on your private data with minimal onboarding and operational effort. +site_description: Docq offers self-hosted, private and secure ChatGPT on your private data with minimal onboarding and operational effort. site_author: Docq Contributors copyright: Copyright © 2023 Docq Contributors site_dir: html @@ -46,13 +46,13 @@ theme: palette: - media: "(prefers-color-scheme: light)" scheme: docq - accent: light-green + accent: purple toggle: icon: material/lightbulb-outline name: Switch to light mode - media: "(prefers-color-scheme: dark)" scheme: slate - accent: light-green + accent: purple toggle: icon: material/lightbulb name: Switch to dark mode @@ -91,12 +91,14 @@ nav: - "Deployment Scenarios": overview/deployment-scenarios.md - "User Guides": - "Getting Started": user-guide/getting-started.md + - "Configuration": user-guide/configuration.md - "Deploy to Streamlit": user-guide/deploy-to-streamlit.md - - "Deploy to Azure": user-guide/deploy-to-streamlit.md + - "Deploy to Azure": user-guide/deploy-to-azure.md - "Deploy to AWS": user-guide/deploy-to-aws.md - "Deploy to GCP": user-guide/deploy-to-gcp.md - "Configure Spaces": user-guide/config-spaces.md - "Data Sources": user-guide/data-sources.md + - "Configure File Storage Services": user-guide/file-storage-services.md - "Usage": user-guide/usage.md - "FAQ": user-guide/faq.md - "Developer Guides": @@ -104,11 +106,12 @@ nav: - "Architecture": developer-guide/architecture.md - "Contributing": developer-guide/contributing.md - "Extensions": developer-guide/extensions.md + - "Web API": developer-guide/web-api.md - "FAQ": developer-guide/faq.md #- ... | glob=readme.md #- ... | regex=scenarios/.+.md - - API: mkapi/api/docq - - Tests: mkapi/tests/tests + #- API: mkapi/api/docq + #- Tests: mkapi/tests/tests - "Issues GitHub": https://github.com/docqai/docq/issues - - "Discussions GitHub": https://github.com/docqai/docq/discussions + - "Join Our Slack": https://join.slack.com/t/docqai/shared_invite/zt-27p17lu6v-6KLJxSmt61vfNqCavSE73A - "Acknowledgement": acknowledgement.md diff --git a/poetry.lock b/poetry.lock index 2d873a03..6ce047db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -324,6 +324,20 @@ typing-extensions = ">=4.6.0" [package.extras] aio = ["aiohttp (>=3.0)"] +[[package]] +name = "babel" +version = "2.13.1" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Babel-2.13.1-py3-none-any.whl", hash = "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"}, + {file = "Babel-2.13.1.tar.gz", hash = "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900"}, +] + +[package.extras] +dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] + [[package]] name = "backoff" version = "2.2.1" @@ -2005,23 +2019,32 @@ mkdocs = ">=1.0.3,<2.0.0" [[package]] name = "mkdocs-material" -version = "8.5.11" +version = "9.4.14" description = "Documentation that simply works" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "mkdocs_material-8.5.11-py3-none-any.whl", hash = "sha256:c907b4b052240a5778074a30a78f31a1f8ff82d7012356dc26898b97559f082e"}, - {file = "mkdocs_material-8.5.11.tar.gz", hash = "sha256:b0ea0513fd8cab323e8a825d6692ea07fa83e917bb5db042e523afecc7064ab7"}, + {file = "mkdocs_material-9.4.14-py3-none-any.whl", hash = "sha256:dbc78a4fea97b74319a6aa9a2f0be575a6028be6958f813ba367188f7b8428f6"}, + {file = "mkdocs_material-9.4.14.tar.gz", hash = "sha256:a511d3ff48fa8718b033e7e37d17abd9cc1de0fdf0244a625ca2ae2387e2416d"}, ] [package.dependencies] -jinja2 = ">=3.0.2" -markdown = ">=3.2" -mkdocs = ">=1.4.0" -mkdocs-material-extensions = ">=1.1" -pygments = ">=2.12" -pymdown-extensions = ">=9.4" -requests = ">=2.26" +babel = ">=2.10,<3.0" +colorama = ">=0.4,<1.0" +jinja2 = ">=3.0,<4.0" +markdown = ">=3.2,<4.0" +mkdocs = ">=1.5.3,<2.0" +mkdocs-material-extensions = ">=1.3,<2.0" +paginate = ">=0.5,<1.0" +pygments = ">=2.16,<3.0" +pymdown-extensions = ">=10.2,<11.0" +regex = ">=2022.4" +requests = ">=2.26,<3.0" + +[package.extras] +git = ["mkdocs-git-committers-plugin-2 (>=1.1,<2.0)", "mkdocs-git-revision-date-localized-plugin (>=1.2,<2.0)"] +imaging = ["cairosvg (>=2.6,<3.0)", "pillow (>=9.4,<10.0)"] +recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2.0)", "mkdocs-rss-plugin (>=1.6,<2.0)"] [[package]] name = "mkdocs-material-extensions" @@ -3086,6 +3109,16 @@ files = [ {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] +[[package]] +name = "paginate" +version = "0.5.6" +description = "Divides large result sets into pages for easier browsing" +optional = false +python-versions = "*" +files = [ + {file = "paginate-0.5.6.tar.gz", hash = "sha256:5e6007b6a9398177a7e1648d04fdd9f8c9766a1a945bceac82f1929e8c78af2d"}, +] + [[package]] name = "pandas" version = "2.1.2" @@ -5771,4 +5804,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "ff4b1469d584435aef8aef0378b16edb581277beb73626067cf25bea33a38e76" +content-hash = "3f295575d8f8f2abd16717ef680a0da810e75a58a635e26f583432440baa29ad" diff --git a/pyproject.toml b/pyproject.toml index 59254a2c..a87b9e49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ microsoftgraph-python = "^1.1.6" llama-index = "^0.9.8.post1" litellm = "^1.7.7" pydantic = "^2.5.2" +mkdocs-material = "^9.4.14" [tool.poetry.group.dev.dependencies] pre-commit = "^2.18.1" @@ -62,7 +63,7 @@ pytest-reverse = "^1.5.0" pytest-cov = "^3.0.0" pytest-bdd = "^6.1.1" mkdocs = "^1.4.2" -mkdocs-material = "^8.5.10" +mkdocs-material = "^9.4.14" mkdocs-gen-files = "^0.4.0" mkdocs-awesome-pages-plugin = "^2.8.0" mkapi = "^1.0.14" @@ -194,23 +195,6 @@ args = [ { name = "target", default = "prod" }, ] -[tool.poe.tasks.docker-build-pipcached] -cmd = """ -docker build ---build-arg TESTBUILD=$test \ ---build-arg BUILDTAG=$build_tag \ ---build-arg BUILDPLATFORM=$build_platform \ ---build-arg PIPNOCACHE= \ ---target $target \ --t docq:$target-$build_tag .""" -help = "Build a docker image to test the project in an isolated environment" -args = [ - { name = "test", default = false, type = "boolean" }, - { name = "build-tag", default = "3.11-slim-buster" }, - { name = "build-platform", default = "linux/amd64" }, - { name = "target", default = "prod" }, -] - [tool.poe.tasks.docker-run] cmd = """ docker run \