Core/Google Drive datasource (#135)

* chore: Add Google Drive env configs to template files.

* chore: Add Google Drive API modules to .toml files.

* feat: Add UI for requesting google Authorization

* feat: Create Google Drive API service handler

* feat: Create Google Drive data source

* chore: Create a local credentials store.

* chore: Setup first draft of Google Drive data source.

* fix: Update data_source base reader.

* chore: Try using opendal to load gdrive and switch to local loader if it fails.

* chore: Allow indexer to use filenames when downloading files from Google Drive.

* chore:  Make google drive downloader async.

* chore: Enable downloading google doc files as pdf.

* chore: Update create space UI for Google Drive."

* chore: Update GDrive credential request.

* chore: Update Google Drive credential request UI.

* fix: Only make api calls to Google Drive if no saved settings.

* chore: Add __doc__ for the domain module.

* chore: Cleanup files.

* fix: shared space selector (#146)

* chore: Fix shared ask space selector not syncing with rest of the page.

* chore: Make space selector expanded by default.

Co-authored-by: Janaka Abeywardhana <[email protected]>

* chore: Remove ConfigKeyHandlers from config file.

* chore: Update google drive datasource credential retrieval.

* chore: Cleanup unused content.

* chore: Fix minor bugs.

* fix: Clear form values after creating space.

* feat: Add an option to make datasource optional.

---------

Co-authored-by: Janaka Abeywardhana <[email protected]>

.gitignore

@@ -192,5 +192,9 @@ docker.env
 # ignore Infisical config file
 .infisical.json
 
+# ignore google drive api credentials
+credentials.json
+token.json
+
 # ignore default docq extensions
 .docq-extensions.json

misc/docker.env.template

@@ -11,6 +11,10 @@ DOCQ_SMTP_LOGIN="SMTP-LOGIN" # The username for logging in to the SMTP service
 DOCQ_SMTP_KEY="SMTP-MASTER-PASSWORD" # The password for logging in to the SMTP service
 DOCQ_SMTP_FROM="Docq.AI Support <[email protected]>" # A custom sender email
 
+# GOOGLE_DRIVE_API
+DOCQ_GOOGLE_APPLICATION_CREDENTIALS=credentials.json # Credentials containing Docq project configs
+DOCQ_GOOGLE_AUTH_REDIRECT_URL=http://localhost:8501/authorize_gdrive # The URL configured for redirect in google console.
+
 # SERVER SETTINGS
 DOCQ_SERVER_ADDRESS = "http://localhost:8501" # Web address for the docq server, used for generating verification urls.
 

misc/secrets.toml.template

@@ -9,6 +9,10 @@ DOCQ_SMTP_LOGIN = "SMTP-LOGIN" # The username for logging in to the SMTP service
 DOCQ_SMTP_KEY = "SMTP-MASTER-PASSWORD" # The password for logging in to the SMTP service
 DOCQ_SMTP_FROM = "Docq.AI Support <[email protected]>" # A custom sender email
 
+# GOOGLE_DRIVE_API
+DOCQ_GOOGLE_APPLICATION_CREDENTIALS = "credentials.json" # Credentials containing Docq project configs
+DOCQ_GOOGLE_AUTH_REDIRECT_URL = "http://localhost:8501/authorize_gdrive" # The URL configured for redirect in google console.
+
 # SERVER SETTINGS
 DOCQ_SERVER_ADDRESS = "http://localhost:8501" # Web address for the docq server, used for generating verification urls.
 OTEL_SERVICE_NAME = "docq-" #for local dev "docq-dev-<yourname>". Prod "docq-prod"

pyproject.toml

@@ -19,7 +19,7 @@ pypdf = "^3.9.0"
 docx2txt = "^0.8"
 argon2-cffi = "^21.3.0"
 azure-core = "^1.27.1"
-opendal = "^0.38.1"
+opendal = "^0.41.0"
 llama-index = "0.8.34"
 transformers = "4.33.2"
 optimum = { extras = ["exporters"], version = "1.13.2" }
@@ -43,6 +43,9 @@ opentelemetry-instrumentation-sqlalchemy = "0.41b0"
 opentelemetry-instrumentation-tornado = "0.41b0"
 opentelemetry-instrumentation-tortoiseorm = "0.41b0"
 opentelemetry-instrumentation-urllib3 = "0.41b0"
+google-auth-oauthlib = "^1.1.0"
+google-api-python-client = "^2.104.0"
+google-auth-httplib2 = "^0.1.1"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^2.18.1"
@@ -79,7 +82,7 @@ check = true
 
 
 [tool.ruff]
-ignore = ["E501"]
+ignore = ["E501", "ANN401"]
 line-length = 120
 select = [
     "C9",