From 4b20cf1883ca41bbd653dd6b14565b026bef8800 Mon Sep 17 00:00:00 2001 From: Juan Valacco <97040903+jvalacco-dataherald@users.noreply.github.com> Date: Tue, 7 May 2024 14:09:11 -0300 Subject: [PATCH] add docker and local docker compose to admin console frontend (#538) * add docker and local docker compose to admin console frontend * add health-check and fix image runtime --- .github/dependabot.yml | 7 - .github/workflows/ai-build-docker.yml | 82 - .github/workflows/ai-chat-checks.yml | 37 - .github/workflows/ai-check-changes.yml | 99 - .github/workflows/ai-console-checks.yml | 59 - .github/workflows/ai-deploy-backend.yml | 64 - .github/workflows/ai-deploy-branch-env.yml | 364 - .github/workflows/ai-deploy-docker-ecs.yml | 62 - .github/workflows/ai-deploy-env.yml | 204 - .github/workflows/ai-deploy-frontend.yml | 56 - .github/workflows/ai-deploy-prod-env.yml | 13 - .github/workflows/ai-deploy-stage-env.yml | 23 - .github/workflows/ai-destroy-branch-env.yml | 174 - .github/workflows/ai-extract-branch-names.yml | 47 - .github/workflows/ai-migrate-db.yml | 48 - .github/workflows/ai-server-api-test.yml | 67 - .github/workflows/ai-server-checks.yml | 29 - .../workflows/ai-start-self-hosted-runner.yml | 38 - .../workflows/ai-stop-self-hosted-runner.yml | 35 - .gitmodules | 9 - .postman/api | 4 - .../api_d959e3e0-f3ef-4af5-995b-acf2de1a8dda | 18 - apps/ai/clients/benchmark-tool/.gitignore | 2 - apps/ai/clients/benchmark-tool/README.md | 28 - .../init-files/bls_table_schema_details.jsonl | 5942 --------------- .../benchmark-tool/init-files/init-mongo.sh | 16 - .../v2_real_estate_table_schema_details.jsonl | 6620 ----------------- .../clients/benchmark-tool/requirements.txt | 20 - apps/ai/clients/benchmark-tool/run-tests.sh | 8 - .../benchmark-tool/src/datastore_setup.py | 43 - apps/ai/clients/benchmark-tool/src/main.py | 318 - .../benchmark-tool/test_results/.gitkeep | 0 .../benchmark-tool/test_suites/bls.jsonl | 20 - .../test_suites/v2_real_estate.jsonl | 80 - apps/ai/clients/chat/.env.local.example | 20 - apps/ai/clients/chat/.env.production | 1 - apps/ai/clients/chat/.eslintrc.json | 20 - apps/ai/clients/chat/.gitignore | 39 - apps/ai/clients/chat/.prettierrc | 7 - apps/ai/clients/chat/README.md | 34 - apps/ai/clients/chat/components/Chat/Chat.tsx | 294 - .../Chat/ChatAssistantMessageActions.tsx | 163 - .../chat/components/Chat/ChatInput.tsx | 89 - .../chat/components/Chat/ChatKickoff.tsx | 116 - .../chat/components/Chat/ChatLoader.tsx | 67 - .../chat/components/Chat/ChatMessage.tsx | 114 - .../chat/components/Chat/ChatNewMessage.tsx | 108 - .../clients/chat/components/Chat/ChatText.tsx | 29 - .../ai/clients/chat/components/Layout/Box.tsx | 13 - .../clients/chat/components/Layout/Button.tsx | 46 - .../components/Layout/ButtonGroup.module.css | 18 - .../chat/components/Layout/ButtonGroup.tsx | 22 - .../clients/chat/components/Layout/Header.tsx | 23 - .../clients/chat/components/Layout/Icon.tsx | 33 - .../clients/chat/components/Layout/Main.tsx | 25 - .../chat/components/Layout/Sidebar.tsx | 140 - .../chat/components/Templates/Tile.tsx | 35 - .../chat/components/hoc/WithAnalytics.tsx | 27 - apps/ai/clients/chat/contexts/chat/index.tsx | 67 - .../ai/clients/chat/contexts/prompt/index.tsx | 25 - apps/ai/clients/chat/env-variables/index.ts | 11 - apps/ai/clients/chat/license | 21 - apps/ai/clients/chat/next.config.js | 32 - apps/ai/clients/chat/package.json | 42 - apps/ai/clients/chat/pages/_app.tsx | 28 - apps/ai/clients/chat/pages/_document.tsx | 13 - .../clients/chat/pages/api/auth/[...auth0].ts | 44 - .../chat/pages/api/auth/token/index.ts | 9 - .../api/chat/feedback/[responseId]/index.ts | 29 - apps/ai/clients/chat/pages/api/chat/index.ts | 30 - apps/ai/clients/chat/pages/chat/index.tsx | 27 - .../clients/chat/pages/error/auth/index.tsx | 136 - .../chat/pages/quick-start-guide/index.tsx | 123 - .../ai/clients/chat/pages/templates/index.tsx | 79 - apps/ai/clients/chat/pnpm-lock.yaml | 3072 -------- apps/ai/clients/chat/postcss.config.js | 6 - apps/ai/clients/chat/public/favicon.ico | Bin 4286 -> 0 bytes .../chat/public/images/dh-logo-color.svg | 14 - .../public/images/dh-logo-symbol-color.svg | 4 - .../public/images/error/data_not_found.svg | 42 - .../public/images/loading/analyzing_data.svg | 55 - .../loading/generating_visualization.svg | 74 - .../loading/retrieving_data_sources.svg | 44 - .../clients/chat/services/analytics/index.ts | 42 - apps/ai/clients/chat/services/api/index.ts | 113 - apps/ai/clients/chat/styles/_animations.css | 12 - apps/ai/clients/chat/styles/globals.css | 11 - apps/ai/clients/chat/tailwind.config.js | 33 - apps/ai/clients/chat/tsconfig.json | 23 - apps/ai/clients/chat/types/api/index.ts | 17 - apps/ai/clients/chat/types/chat/index.ts | 27 - apps/ai/clients/chat/utils/api/index.ts | 44 - apps/ai/clients/chat/utils/chat/index.ts | 68 - apps/ai/clients/test_tools/dh_evaluator.py | 111 - apps/ai/clients/test_tools/run_test.py | 96 - .../ai/clients/test_tools/v2_real_estate.json | 38 - apps/ai/server/.aws/task-definition-prod.json | 89 - .../ai/server/.aws/task-definition-stage.json | 93 - apps/ai/server/dataherald | 1 - .../server/terraform/branch/aws_ecs/main.tf | 285 - .../server/terraform/branch/aws_ecs/vars.tf | 46 - apps/ai/server/terraform/branch/main.tf | 42 - .../terraform/branch/pinecone_index/main.tf | 19 - .../terraform/branch/pinecone_index/vars.tf | 4 - apps/ai/server/terraform/branch/vars.tf | 68 - apps/ai/server/tests/auth/test_auth_api.py | 38 - .../ai/server/tests/db_connection/__init__.py | 0 .../db_connection/test_db_connection_api.py | 101 - apps/ai/server/tests/finetuning/__init__.py | 0 .../tests/finetuning/test_finetuning_api.py | 104 - apps/ai/server/tests/generation/__init__.py | 0 .../generation/test_aggr_generation_api.py | 185 - .../tests/generation/test_generation_api.py | 32 - apps/ai/server/tests/golden_sql/__init__.py | 0 .../tests/golden_sql/test_golden_sql_api.py | 105 - apps/ai/server/tests/instruction/__init__.py | 0 .../tests/instruction/test_instruction_api.py | 87 - apps/ai/server/tests/key/__init__.py | 0 apps/ai/server/tests/key/test_key_api.py | 80 - apps/ai/server/tests/organization/__init__.py | 0 .../organization/test_organization_api.py | 191 - .../tests/table_description/__init__.py | 0 .../test_table_description_api.py | 46 - apps/ai/server/tests/test_api.py | 14 - apps/ai/server/tests/user/__init__.py | 0 apps/ai/server/tests/user/test_user_api.py | 114 - apps/ai/server/tmp/__init__.py | 0 apps/ai/server/utils/__init__.py | 0 apps/bariloche/.env.example | 12 - apps/bariloche/Dockerfile | 16 - apps/bariloche/Makefile | 23 - apps/bariloche/app-server | 1 - apps/bariloche/docker-compose.yml | 51 - apps/bariloche/programmatic-seo | 1 - apps/everest/README.md | 11 - .../screenshots_generator/.env.example | 3 - apps/everest/screenshots_generator/.gitignore | 8 - apps/everest/screenshots_generator/README.md | 48 - apps/everest/screenshots_generator/main.py | 101 - .../screenshots_generator/requirements.txt | 32 - services/admin-console/.env | 1 + .../admin-console/.env.example | 0 .../admin-console/.eslintrc.json | 0 .../admin-console/.gitignore | 1 + .../admin-console/.prettierrc | 0 .../admin-console/README.md | 0 .../admin-console/components.json | 0 services/admin-console/dev.Dockerfile | 18 + services/admin-console/docker-compose.yml | 14 + .../admin-console/jest.config.js | 0 .../admin-console/next.config.js | 1 + .../admin-console/package.json | 1 + .../admin-console/pnpm-lock.yaml | 267 + .../admin-console/postcss.config.js | 0 services/admin-console/prod.Dockerfile | 57 + .../admin-console/public/favicon.ico | Bin .../public/images/billing/amex.svg | 0 .../public/images/billing/diners.svg | 0 .../public/images/billing/discover.svg | 0 .../public/images/billing/eftpos_au.svg | 0 .../public/images/billing/jcb.svg | 0 .../public/images/billing/mastercard.svg | 0 .../public/images/billing/unionpay.svg | 0 .../public/images/billing/unkown.svg | 0 .../public/images/billing/visa.svg | 0 .../public/images/databases/aws-athena.svg | 0 .../public/images/databases/bigquery.svg | 0 .../public/images/databases/clickhouse.svg | 0 .../public/images/databases/databricks.svg | 0 .../public/images/databases/mariadb.svg | 0 .../public/images/databases/postgresql.svg | 0 .../public/images/databases/redshift.png | Bin .../public/images/databases/redshift.svg | 0 .../public/images/databases/snowflake.svg | 0 .../public/images/databases/sql-server.png | Bin .../public/images/databases/sql-server.svg | 0 .../public/images/dh-logo-color.svg | 0 .../public/images/dh-logo-symbol-color.svg | 0 .../public/images/dh_ai_logo.svg | 0 .../public/images/dh_background.png | Bin .../admin-console/public/images/discord.png | Bin .../public/images/llm_models/openai.png | Bin .../admin-console/public/images/openai.png | Bin .../public/images/slack-black.png | Bin .../public/images/slack-color.png | Bin .../src/components/api-keys/api-keys-list.tsx | 0 .../src/components/api-keys/columns.tsx | 0 .../api-keys/delete-api-key-dialog.tsx | 0 .../api-keys/generate-api-key-dialog.tsx | 0 .../billing/add-payment-method-dialog.tsx | 0 .../components/billing/credit-card-logo.tsx | 0 .../src/components/data-table/index.tsx | 0 .../components/data-table/loading-rows.tsx | 0 .../components/data-table/loading-table.tsx | 0 .../components/databases/column-resource.tsx | 0 .../database-connection-form-dialog.tsx | 0 .../databases/database-connection-form.tsx | 0 .../components/databases/database-details.tsx | 0 .../databases/database-resource-sheet.tsx | 0 .../databases/database-resource.tsx | 0 .../components/databases/database-tree.tsx | 0 .../components/databases/databases-tree.tsx | 0 .../databases/first-database-connection.tsx | 0 .../src/components/databases/form-schema.ts | 0 .../components/databases/loading-resource.tsx | 0 .../src/components/databases/loading.tsx | 0 .../sample-database-connection-dialog.tsx | 0 .../components/databases/table-resource.tsx | 0 .../src/components/error/error-details.tsx | 0 .../components/error/page-error-message.tsx | 0 .../src/components/fine-tunnings/columns.tsx | 0 .../src/components/golden-sql/columns.tsx | 0 .../src/components/hoc/WithAnalytics.tsx | 0 .../src/components/hoc/WithApiFetcher.tsx | 0 .../src/components/hoc/WithBilling.tsx | 0 .../src/components/hoc/WithMobileRedirect.tsx | 0 .../src/components/hoc/WithSubscription.tsx | 0 .../layout/background-page-layout.tsx | 0 .../components/layout/breadcrum-header.tsx | 0 .../src/components/layout/loading-list.tsx | 0 .../src/components/layout/loading-page.tsx | 0 .../src/components/layout/page-layout.tsx | 0 .../src/components/layout/sidebar-nav.tsx | 0 .../organization/edit-organization-dialog.tsx | 0 .../components/organization/llm-api-key.tsx | 0 .../organization/payment-methods-list.tsx | 0 .../src/components/queries/columns.tsx | 0 .../query/custom-message-dialog.tsx | 0 .../src/components/query/last-updated.tsx | 0 .../src/components/query/loading-box.tsx | 0 .../src/components/query/loading.tsx | 0 .../src/components/query/message-section.tsx | 0 .../src/components/query/process.tsx | 0 .../src/components/query/query-metadata.tsx | 0 .../src/components/query/question.tsx | 0 .../src/components/query/section-header.tsx | 0 .../components/query/send-message-dialog.tsx | 0 .../src/components/query/sql-editor.tsx | 0 .../components/query/sql-results-table.tsx | 0 .../src/components/query/workspace.tsx | 0 .../src/components/ui/alert-dialog.tsx | 0 .../admin-console/src/components/ui/alert.tsx | 0 .../admin-console/src/components/ui/badge.tsx | 0 .../src/components/ui/button.tsx | 0 .../src/components/ui/checkbox.tsx | 0 .../src/components/ui/content-box.tsx | 0 .../src/components/ui/dialog.tsx | 0 .../src/components/ui/dropdown-menu.tsx | 0 .../admin-console/src/components/ui/form.tsx | 0 .../admin-console/src/components/ui/input.tsx | 0 .../admin-console/src/components/ui/label.tsx | 0 .../src/components/ui/markdown-renderer.tsx | 0 .../src/components/ui/popover.tsx | 0 .../src/components/ui/radio-group.tsx | 0 .../src/components/ui/search-input.tsx | 0 .../src/components/ui/select.tsx | 0 .../src/components/ui/separator.tsx | 0 .../admin-console/src/components/ui/sheet.tsx | 0 .../src/components/ui/skeleton.tsx | 0 .../src/components/ui/switch.tsx | 0 .../admin-console/src/components/ui/table.tsx | 0 .../admin-console/src/components/ui/tabs.tsx | 0 .../src/components/ui/textarea.tsx | 0 .../admin-console/src/components/ui/toast.tsx | 0 .../src/components/ui/toaster.tsx | 0 .../src/components/ui/tooltip.tsx | 0 .../src/components/ui/tree-view-context.tsx | 0 .../ui/tree-view-global-context.tsx | 0 .../src/components/ui/tree-view.helpers.ts | 0 .../src/components/ui/tree-view.tsx | 0 .../src/components/ui/use-toast.ts | 0 .../src/components/usage/credits-chart.tsx | 0 .../usage/edit-spending-limit-dialog.tsx | 0 .../src/components/usage/monthly-usage.tsx | 0 .../components/usage/sub-blocked-dialog.tsx | 0 .../src/components/usage/usage-chart.tsx | 0 .../components/user/invite-member-dialog.tsx | 0 .../src/components/user/user-list.tsx | 0 .../src/components/user/user-picture.tsx | 0 .../admin-console/src/config.ts | 0 .../src/constants/database-providers.ts | 0 .../admin-console/src/constants/llm-models.ts | 0 .../src/contexts/app-context.tsx | 0 .../src/contexts/auth-context.tsx | 0 .../src/contexts/self-serve-context.tsx | 0 .../src/contexts/subscription-context.tsx | 0 .../admin-console/src/contexts/ui-context.tsx | 0 .../src/hooks/api/api-keys/useApiKeys.ts | 0 .../src/hooks/api/api-keys/useDeleteApiKey.ts | 0 .../src/hooks/api/api-keys/usePostApiKey.ts | 0 .../api/billing/useDeletePaymentMethod.ts | 0 .../hooks/api/billing/usePaymentMethods.ts | 0 .../hooks/api/billing/usePostPaymentMethod.ts | 0 .../hooks/api/billing/usePutSpendingLimits.ts | 0 .../hooks/api/billing/useSpendingLimits.tsx | 0 .../src/hooks/api/billing/useUsage.tsx | 0 .../useDatabaseConnection.ts | 0 .../useDatabaseConnections.ts | 0 .../api/database-connection/useDatabases.ts | 0 .../usePostDatabaseConnection.ts | 0 .../usePostSampleDatabaseConnection.ts | 0 .../useSampleDatabaseConnections.ts | 0 .../useSynchronizeSchemas.ts | 0 .../hooks/api/fine-tuning/useFinetunings.ts | 0 .../src/hooks/api/generics/useApiFetcher.ts | 0 .../src/hooks/api/generics/useDelete.ts | 0 .../src/hooks/api/generics/useGet.ts | 0 .../src/hooks/api/generics/useMockGetter.ts | 0 .../hooks/api/generics/useMockPagination.ts | 0 .../src/hooks/api/generics/usePagination.ts | 0 .../src/hooks/api/generics/usePatch.ts | 0 .../src/hooks/api/generics/usePost.ts | 0 .../src/hooks/api/generics/usePut.ts | 0 .../api/organization/useGetOrganization.ts | 0 .../api/organization/useOrganizations.ts | 0 .../api/organization/usePutOrganization.ts | 0 .../src/hooks/api/query/useQueries.ts | 0 .../src/hooks/api/query/useQuery.ts | 0 .../src/hooks/api/query/useQueryExecution.ts | 0 .../api/query/useQueryGenerateMessage.ts | 0 .../src/hooks/api/query/useQueryPut.ts | 0 .../src/hooks/api/query/useQueryResubmit.ts | 0 .../hooks/api/query/useQuerySendMessage.ts | 0 .../src/hooks/api/useDeleteGoldenSql.ts | 0 .../src/hooks/api/useDownloadFile.ts | 0 .../src/hooks/api/useGoldenSqlList.ts | 0 .../src/hooks/api/user/useDeleteUser.ts | 0 .../src/hooks/api/user/useGetUser.ts | 0 .../src/hooks/api/user/usePatchUser.ts | 0 .../src/hooks/api/user/usePostUser.ts | 0 .../api/user/usePostUserToOrganization.ts | 0 .../src/hooks/api/user/useUsers.ts | 0 .../src/hooks/database/useColumnResource.ts | 0 .../src/hooks/database/useDatabaseOptions.tsx | 0 .../src/hooks/database/useDatabaseResource.ts | 0 .../database/useDatabaseResourceFromTree.ts | 0 .../src/hooks/database/useTableResource.ts | 0 .../src/hooks/ui/useIsMobile.tsx | 0 .../admin-console/src/hooks/useDebounce.ts | 0 .../src/lib/api/server-fetcher.test.ts | 0 .../src/lib/api/server-fetcher.ts | 0 .../admin-console/src/lib/domain/billing.tsx | 0 .../admin-console/src/lib/domain/database.tsx | 0 .../admin-console/src/lib/domain/error.tsx | 0 .../admin-console/src/lib/domain/query.ts | 0 .../admin-console/src/lib/utils.tsx | 0 .../admin-console/src/mocks/database.ts | 0 .../admin-console/src/models/api.ts | 0 .../admin-console/src/models/domain.ts | 0 .../admin-console/src/models/errorCodes.ts | 0 .../admin-console/src/pages/_app.tsx | 0 .../admin-console/src/pages/_document.tsx | 0 .../src/pages/api-keys/index.tsx | 0 .../src/pages/api/auth/[...auth0].ts | 0 .../admin-console/src/pages/api/auth/token.ts | 0 .../src/pages/api/health-check.ts | 7 + .../admin-console/src/pages/auth/error.tsx | 0 .../admin-console/src/pages/billing/index.tsx | 0 .../src/pages/change-organization/index.tsx | 0 .../src/pages/databases/index.tsx | 0 .../src/pages/fine-tuning/index.tsx | 0 .../src/pages/golden-sql/index.tsx | 0 .../src/pages/my-account/index.tsx | 0 .../src/pages/organization/index.tsx | 0 .../src/pages/playground/index.tsx | 0 .../pages/queries/[queryId]/download-csv.tsx | 0 .../src/pages/queries/[queryId]/index.tsx | 0 .../admin-console/src/pages/queries/index.tsx | 0 .../pages/switch-to-larger-screen/index.tsx | 0 .../admin-console/src/pages/usage/index.tsx | 0 .../admin-console/src/styles/globals.css | 0 .../admin-console/tailwind.config.js | 0 .../admin-console/tsconfig.json | 0 .../admin-console/vercel.json | 0 .../enterprise}/.dockerignore | 0 .../enterprise}/.env.example | 0 .../server => services/enterprise}/.test.env | 0 .../server => services/enterprise}/Dockerfile | 0 .../server => services/enterprise}/Makefile | 0 .../server => services/enterprise}/README.md | 0 .../enterprise}/__init__.py | 0 .../ai/server => services/enterprise}/app.py | 0 .../server => services/enterprise}/config.py | 0 .../enterprise}/database/__init__.py | 0 .../enterprise}/database/migrations/README.md | 0 .../database/migrations/changelog.json | 0 .../liquibase.advanced.flowfile.yaml | 0 .../migrations/liquibase.checks-package.yaml | 0 .../migrations/liquibase.endstage.flow | 0 .../migrations/liquibase.flowfile.yaml | 0 .../migrations/liquibase.flowvariables.yaml | 0 .../database/migrations/liquibase.properties | 0 .../enterprise}/database/mongo.py | 0 .../enterprise}/database/scripts/__init__.py | 0 .../database/scripts/copy_org_resources.py | 0 .../database/scripts/data_report.py | 0 .../database/scripts/delete_db_connection.py | 0 .../database/scripts/initial_setup.py | 0 .../populate_pinecone_db/requirements.txt | 0 .../scripts/populate_pinecone_db/script.py | 0 .../database/scripts/table_descriptions.json | 0 .../database/scripts/upload_ssh_key_file.py | 0 .../enterprise}/docker-compose.yml | 0 .../enterprise}/exceptions/error_codes.py | 0 .../enterprise}/exceptions/error_response.py | 0 .../exceptions/exception_handlers.py | 0 .../enterprise}/exceptions/exceptions.py | 0 .../enterprise}/exceptions/utils.py | 0 .../enterprise}/middleware/error.py | 0 .../enterprise}/modules/auth/__init__.py | 0 .../enterprise}/modules/auth/controller.py | 0 .../modules/auth/models/__init__.py | 0 .../modules/auth/models/entities.py | 0 .../modules/auth/models/exceptions.py | 0 .../modules/auth/models/requests.py | 0 .../modules/auth/models/responses.py | 0 .../enterprise}/modules/auth/repository.py | 0 .../enterprise}/modules/auth/service.py | 0 .../modules/db_connection/__init__.py | 0 .../modules/db_connection/controller.py | 0 .../modules/db_connection/models/__init__.py | 0 .../modules/db_connection/models/entities.py | 0 .../db_connection/models/exceptions.py | 0 .../modules/db_connection/models/requests.py | 0 .../modules/db_connection/models/responses.py | 0 .../modules/db_connection/repository.py | 0 .../modules/db_connection/service.py | 0 .../modules/finetuning/__init__.py | 0 .../modules/finetuning/controller.py | 0 .../modules/finetuning/models/__init__.py | 0 .../modules/finetuning/models/entities.py | 0 .../modules/finetuning/models/exceptions.py | 0 .../modules/finetuning/models/requests.py | 0 .../modules/finetuning/models/responses.py | 0 .../modules/finetuning/repository.py | 0 .../enterprise}/modules/finetuning/service.py | 0 .../modules/generation/__init__.py | 0 .../modules/generation/aggr_controller.py | 0 .../modules/generation/aggr_service.py | 0 .../modules/generation/controller.py | 0 .../modules/generation/models/__init__.py | 0 .../modules/generation/models/entities.py | 0 .../modules/generation/models/exceptions.py | 0 .../modules/generation/models/requests.py | 0 .../modules/generation/models/responses.py | 0 .../modules/generation/repository.py | 0 .../enterprise}/modules/generation/service.py | 0 .../modules/golden_sql/__init__.py | 0 .../modules/golden_sql/controller.py | 0 .../modules/golden_sql/models/__ini__.py | 0 .../modules/golden_sql/models/entities.py | 0 .../modules/golden_sql/models/exceptions.py | 0 .../modules/golden_sql/models/requests.py | 0 .../modules/golden_sql/models/responses.py | 0 .../modules/golden_sql/repository.py | 0 .../enterprise}/modules/golden_sql/service.py | 0 .../modules/instruction/__init__.py | 0 .../modules/instruction/controller.py | 0 .../modules/instruction/models/__init__.py | 0 .../modules/instruction/models/entities.py | 0 .../modules/instruction/models/exceptions.py | 0 .../modules/instruction/models/requests.py | 0 .../modules/instruction/models/responses.py | 0 .../modules/instruction/repository.py | 0 .../modules/instruction/service.py | 0 .../enterprise}/modules/key/__init__.py | 0 .../enterprise}/modules/key/controller.py | 0 .../modules/key/models/__init__.py | 0 .../modules/key/models/entities.py | 0 .../modules/key/models/exceptions.py | 0 .../modules/key/models/requests.py | 0 .../modules/key/models/responses.py | 0 .../enterprise}/modules/key/repository.py | 0 .../enterprise}/modules/key/service.py | 0 .../modules/organization/__init__.py | 0 .../modules/organization/controller.py | 0 .../organization/invoice/controller.py | 0 .../organization/invoice/models/entities.py | 0 .../organization/invoice/models/exceptions.py | 0 .../organization/invoice/models/requests.py | 0 .../organization/invoice/models/responses.py | 0 .../organization/invoice/repository.py | 0 .../modules/organization/invoice/service.py | 0 .../modules/organization/invoice/webhook.py | 0 .../modules/organization/models/__init__.py | 0 .../modules/organization/models/entities.py | 0 .../modules/organization/models/exceptions.py | 0 .../modules/organization/models/requests.py | 0 .../modules/organization/models/responses.py | 0 .../modules/organization/repository.py | 0 .../modules/organization/service.py | 0 .../modules/table_description/__init__.py | 0 .../modules/table_description/controller.py | 0 .../table_description/models/__init__.py | 0 .../table_description/models/entities.py | 0 .../table_description/models/exceptions.py | 0 .../table_description/models/requests.py | 0 .../table_description/models/responses.py | 0 .../modules/table_description/repository.py | 0 .../modules/table_description/service.py | 0 .../enterprise}/modules/user/__init__.py | 0 .../enterprise}/modules/user/controller.py | 0 .../modules/user/models/__init__.py | 0 .../modules/user/models/entities.py | 0 .../modules/user/models/exceptions.py | 0 .../modules/user/models/requests.py | 0 .../modules/user/models/responses.py | 0 .../enterprise}/modules/user/repository.py | 0 .../enterprise}/modules/user/service.py | 0 .../enterprise}/pyproject.toml | 0 .../enterprise}/requirements.txt | 0 .../enterprise}/scripts/__init__.py | 0 .../enterprise}/scripts/record_usage.py | 0 .../enterprise/tmp}/__init__.py | 0 .../enterprise/utils}/__init__.py | 0 .../enterprise}/utils/analytics.py | 0 .../enterprise}/utils/auth.py | 0 .../enterprise}/utils/billing.py | 0 .../enterprise}/utils/encrypt.py | 0 .../enterprise}/utils/misc.py | 0 .../enterprise}/utils/s3.py | 0 .../enterprise}/utils/sample_db.py | 0 .../enterprise}/utils/slack.py | 0 .../enterprise}/utils/validation.py | 0 .../slackbot}/.aws/task-definition-prod.json | 0 .../slackbot}/.aws/task-definition-stage.json | 0 .../slack => services/slackbot}/.env.example | 0 .../slack => services/slackbot}/.eslintrc | 0 .../slack => services/slackbot}/.prettierrc | 0 .../slack => services/slackbot}/Dockerfile | 0 .../slack => services/slackbot}/README.md | 0 .../slack => services/slackbot}/auth/index.js | 0 .../slackbot}/config/index.js | 0 .../slackbot}/docker-compose.yml | 0 .../slackbot}/handlers/message/index.js | 0 .../slack => services/slackbot}/package.json | 0 .../slackbot}/pnpm-lock.yaml | 0 .../slackbot}/server/index.js | 0 538 files changed, 367 insertions(+), 22832 deletions(-) delete mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/ai-build-docker.yml delete mode 100644 .github/workflows/ai-chat-checks.yml delete mode 100644 .github/workflows/ai-check-changes.yml delete mode 100644 .github/workflows/ai-console-checks.yml delete mode 100644 .github/workflows/ai-deploy-backend.yml delete mode 100644 .github/workflows/ai-deploy-branch-env.yml delete mode 100644 .github/workflows/ai-deploy-docker-ecs.yml delete mode 100644 .github/workflows/ai-deploy-env.yml delete mode 100644 .github/workflows/ai-deploy-frontend.yml delete mode 100644 .github/workflows/ai-deploy-prod-env.yml delete mode 100644 .github/workflows/ai-deploy-stage-env.yml delete mode 100644 .github/workflows/ai-destroy-branch-env.yml delete mode 100644 .github/workflows/ai-extract-branch-names.yml delete mode 100644 .github/workflows/ai-migrate-db.yml delete mode 100644 .github/workflows/ai-server-api-test.yml delete mode 100644 .github/workflows/ai-server-checks.yml delete mode 100644 .github/workflows/ai-start-self-hosted-runner.yml delete mode 100644 .github/workflows/ai-stop-self-hosted-runner.yml delete mode 100755 .gitmodules delete mode 100644 .postman/api delete mode 100644 .postman/api_d959e3e0-f3ef-4af5-995b-acf2de1a8dda delete mode 100644 apps/ai/clients/benchmark-tool/.gitignore delete mode 100644 apps/ai/clients/benchmark-tool/README.md delete mode 100644 apps/ai/clients/benchmark-tool/init-files/bls_table_schema_details.jsonl delete mode 100644 apps/ai/clients/benchmark-tool/init-files/init-mongo.sh delete mode 100644 apps/ai/clients/benchmark-tool/init-files/v2_real_estate_table_schema_details.jsonl delete mode 100644 apps/ai/clients/benchmark-tool/requirements.txt delete mode 100755 apps/ai/clients/benchmark-tool/run-tests.sh delete mode 100644 apps/ai/clients/benchmark-tool/src/datastore_setup.py delete mode 100644 apps/ai/clients/benchmark-tool/src/main.py delete mode 100644 apps/ai/clients/benchmark-tool/test_results/.gitkeep delete mode 100644 apps/ai/clients/benchmark-tool/test_suites/bls.jsonl delete mode 100644 apps/ai/clients/benchmark-tool/test_suites/v2_real_estate.jsonl delete mode 100644 apps/ai/clients/chat/.env.local.example delete mode 100644 apps/ai/clients/chat/.env.production delete mode 100644 apps/ai/clients/chat/.eslintrc.json delete mode 100644 apps/ai/clients/chat/.gitignore delete mode 100644 apps/ai/clients/chat/.prettierrc delete mode 100644 apps/ai/clients/chat/README.md delete mode 100644 apps/ai/clients/chat/components/Chat/Chat.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatAssistantMessageActions.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatInput.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatKickoff.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatLoader.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatMessage.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatNewMessage.tsx delete mode 100644 apps/ai/clients/chat/components/Chat/ChatText.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/Box.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/Button.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/ButtonGroup.module.css delete mode 100644 apps/ai/clients/chat/components/Layout/ButtonGroup.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/Header.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/Icon.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/Main.tsx delete mode 100644 apps/ai/clients/chat/components/Layout/Sidebar.tsx delete mode 100644 apps/ai/clients/chat/components/Templates/Tile.tsx delete mode 100644 apps/ai/clients/chat/components/hoc/WithAnalytics.tsx delete mode 100644 apps/ai/clients/chat/contexts/chat/index.tsx delete mode 100644 apps/ai/clients/chat/contexts/prompt/index.tsx delete mode 100644 apps/ai/clients/chat/env-variables/index.ts delete mode 100644 apps/ai/clients/chat/license delete mode 100644 apps/ai/clients/chat/next.config.js delete mode 100644 apps/ai/clients/chat/package.json delete mode 100644 apps/ai/clients/chat/pages/_app.tsx delete mode 100644 apps/ai/clients/chat/pages/_document.tsx delete mode 100644 apps/ai/clients/chat/pages/api/auth/[...auth0].ts delete mode 100644 apps/ai/clients/chat/pages/api/auth/token/index.ts delete mode 100644 apps/ai/clients/chat/pages/api/chat/feedback/[responseId]/index.ts delete mode 100644 apps/ai/clients/chat/pages/api/chat/index.ts delete mode 100644 apps/ai/clients/chat/pages/chat/index.tsx delete mode 100644 apps/ai/clients/chat/pages/error/auth/index.tsx delete mode 100644 apps/ai/clients/chat/pages/quick-start-guide/index.tsx delete mode 100644 apps/ai/clients/chat/pages/templates/index.tsx delete mode 100644 apps/ai/clients/chat/pnpm-lock.yaml delete mode 100644 apps/ai/clients/chat/postcss.config.js delete mode 100644 apps/ai/clients/chat/public/favicon.ico delete mode 100644 apps/ai/clients/chat/public/images/dh-logo-color.svg delete mode 100644 apps/ai/clients/chat/public/images/dh-logo-symbol-color.svg delete mode 100644 apps/ai/clients/chat/public/images/error/data_not_found.svg delete mode 100644 apps/ai/clients/chat/public/images/loading/analyzing_data.svg delete mode 100644 apps/ai/clients/chat/public/images/loading/generating_visualization.svg delete mode 100644 apps/ai/clients/chat/public/images/loading/retrieving_data_sources.svg delete mode 100644 apps/ai/clients/chat/services/analytics/index.ts delete mode 100644 apps/ai/clients/chat/services/api/index.ts delete mode 100644 apps/ai/clients/chat/styles/_animations.css delete mode 100644 apps/ai/clients/chat/styles/globals.css delete mode 100644 apps/ai/clients/chat/tailwind.config.js delete mode 100644 apps/ai/clients/chat/tsconfig.json delete mode 100644 apps/ai/clients/chat/types/api/index.ts delete mode 100644 apps/ai/clients/chat/types/chat/index.ts delete mode 100644 apps/ai/clients/chat/utils/api/index.ts delete mode 100644 apps/ai/clients/chat/utils/chat/index.ts delete mode 100644 apps/ai/clients/test_tools/dh_evaluator.py delete mode 100644 apps/ai/clients/test_tools/run_test.py delete mode 100644 apps/ai/clients/test_tools/v2_real_estate.json delete mode 100644 apps/ai/server/.aws/task-definition-prod.json delete mode 100644 apps/ai/server/.aws/task-definition-stage.json delete mode 160000 apps/ai/server/dataherald delete mode 100644 apps/ai/server/terraform/branch/aws_ecs/main.tf delete mode 100644 apps/ai/server/terraform/branch/aws_ecs/vars.tf delete mode 100644 apps/ai/server/terraform/branch/main.tf delete mode 100644 apps/ai/server/terraform/branch/pinecone_index/main.tf delete mode 100644 apps/ai/server/terraform/branch/pinecone_index/vars.tf delete mode 100644 apps/ai/server/terraform/branch/vars.tf delete mode 100644 apps/ai/server/tests/auth/test_auth_api.py delete mode 100644 apps/ai/server/tests/db_connection/__init__.py delete mode 100644 apps/ai/server/tests/db_connection/test_db_connection_api.py delete mode 100644 apps/ai/server/tests/finetuning/__init__.py delete mode 100644 apps/ai/server/tests/finetuning/test_finetuning_api.py delete mode 100644 apps/ai/server/tests/generation/__init__.py delete mode 100644 apps/ai/server/tests/generation/test_aggr_generation_api.py delete mode 100644 apps/ai/server/tests/generation/test_generation_api.py delete mode 100644 apps/ai/server/tests/golden_sql/__init__.py delete mode 100644 apps/ai/server/tests/golden_sql/test_golden_sql_api.py delete mode 100644 apps/ai/server/tests/instruction/__init__.py delete mode 100644 apps/ai/server/tests/instruction/test_instruction_api.py delete mode 100644 apps/ai/server/tests/key/__init__.py delete mode 100644 apps/ai/server/tests/key/test_key_api.py delete mode 100644 apps/ai/server/tests/organization/__init__.py delete mode 100644 apps/ai/server/tests/organization/test_organization_api.py delete mode 100644 apps/ai/server/tests/table_description/__init__.py delete mode 100644 apps/ai/server/tests/table_description/test_table_description_api.py delete mode 100644 apps/ai/server/tests/test_api.py delete mode 100644 apps/ai/server/tests/user/__init__.py delete mode 100644 apps/ai/server/tests/user/test_user_api.py delete mode 100644 apps/ai/server/tmp/__init__.py delete mode 100644 apps/ai/server/utils/__init__.py delete mode 100755 apps/bariloche/.env.example delete mode 100755 apps/bariloche/Dockerfile delete mode 100755 apps/bariloche/Makefile delete mode 160000 apps/bariloche/app-server delete mode 100755 apps/bariloche/docker-compose.yml delete mode 160000 apps/bariloche/programmatic-seo delete mode 100644 apps/everest/README.md delete mode 100644 apps/everest/screenshots_generator/.env.example delete mode 100644 apps/everest/screenshots_generator/.gitignore delete mode 100644 apps/everest/screenshots_generator/README.md delete mode 100644 apps/everest/screenshots_generator/main.py delete mode 100644 apps/everest/screenshots_generator/requirements.txt create mode 100644 services/admin-console/.env rename {apps/ai/clients => services}/admin-console/.env.example (100%) rename {apps/ai/clients => services}/admin-console/.eslintrc.json (100%) rename {apps/ai/clients => services}/admin-console/.gitignore (98%) rename {apps/ai/clients => services}/admin-console/.prettierrc (100%) rename {apps/ai/clients => services}/admin-console/README.md (100%) rename {apps/ai/clients => services}/admin-console/components.json (100%) create mode 100644 services/admin-console/dev.Dockerfile create mode 100644 services/admin-console/docker-compose.yml rename {apps/ai/clients => services}/admin-console/jest.config.js (100%) rename {apps/ai/clients => services}/admin-console/next.config.js (95%) rename {apps/ai/clients => services}/admin-console/package.json (99%) rename {apps/ai/clients => services}/admin-console/pnpm-lock.yaml (96%) rename {apps/ai/clients => services}/admin-console/postcss.config.js (100%) create mode 100644 services/admin-console/prod.Dockerfile rename {apps/ai/clients => services}/admin-console/public/favicon.ico (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/amex.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/diners.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/discover.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/eftpos_au.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/jcb.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/mastercard.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/unionpay.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/unkown.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/billing/visa.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/aws-athena.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/bigquery.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/clickhouse.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/databricks.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/mariadb.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/postgresql.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/redshift.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/redshift.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/snowflake.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/sql-server.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/databases/sql-server.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/dh-logo-color.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/dh-logo-symbol-color.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/dh_ai_logo.svg (100%) rename {apps/ai/clients => services}/admin-console/public/images/dh_background.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/discord.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/llm_models/openai.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/openai.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/slack-black.png (100%) rename {apps/ai/clients => services}/admin-console/public/images/slack-color.png (100%) rename {apps/ai/clients => services}/admin-console/src/components/api-keys/api-keys-list.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/api-keys/columns.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/api-keys/delete-api-key-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/api-keys/generate-api-key-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/billing/add-payment-method-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/billing/credit-card-logo.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/data-table/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/data-table/loading-rows.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/data-table/loading-table.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/column-resource.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/database-connection-form-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/database-connection-form.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/database-details.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/database-resource-sheet.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/database-resource.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/database-tree.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/databases-tree.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/first-database-connection.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/form-schema.ts (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/loading-resource.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/loading.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/sample-database-connection-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/databases/table-resource.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/error/error-details.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/error/page-error-message.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/fine-tunnings/columns.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/golden-sql/columns.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/hoc/WithAnalytics.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/hoc/WithApiFetcher.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/hoc/WithBilling.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/hoc/WithMobileRedirect.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/hoc/WithSubscription.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/layout/background-page-layout.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/layout/breadcrum-header.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/layout/loading-list.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/layout/loading-page.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/layout/page-layout.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/layout/sidebar-nav.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/organization/edit-organization-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/organization/llm-api-key.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/organization/payment-methods-list.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/queries/columns.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/custom-message-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/last-updated.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/loading-box.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/loading.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/message-section.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/process.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/query-metadata.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/question.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/section-header.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/send-message-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/sql-editor.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/sql-results-table.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/query/workspace.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/alert-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/alert.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/badge.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/button.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/checkbox.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/content-box.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/dropdown-menu.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/form.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/input.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/label.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/markdown-renderer.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/popover.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/radio-group.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/search-input.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/select.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/separator.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/sheet.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/skeleton.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/switch.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/table.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/tabs.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/textarea.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/toast.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/toaster.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/tooltip.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/tree-view-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/tree-view-global-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/tree-view.helpers.ts (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/tree-view.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/ui/use-toast.ts (100%) rename {apps/ai/clients => services}/admin-console/src/components/usage/credits-chart.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/usage/edit-spending-limit-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/usage/monthly-usage.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/usage/sub-blocked-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/usage/usage-chart.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/user/invite-member-dialog.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/user/user-list.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/components/user/user-picture.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/config.ts (100%) rename {apps/ai/clients => services}/admin-console/src/constants/database-providers.ts (100%) rename {apps/ai/clients => services}/admin-console/src/constants/llm-models.ts (100%) rename {apps/ai/clients => services}/admin-console/src/contexts/app-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/contexts/auth-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/contexts/self-serve-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/contexts/subscription-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/contexts/ui-context.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/api-keys/useApiKeys.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/api-keys/useDeleteApiKey.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/api-keys/usePostApiKey.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/billing/useDeletePaymentMethod.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/billing/usePaymentMethods.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/billing/usePostPaymentMethod.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/billing/usePutSpendingLimits.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/billing/useSpendingLimits.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/billing/useUsage.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/useDatabaseConnection.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/useDatabaseConnections.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/useDatabases.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/usePostDatabaseConnection.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/usePostSampleDatabaseConnection.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/useSampleDatabaseConnections.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/database-connection/useSynchronizeSchemas.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/fine-tuning/useFinetunings.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/useApiFetcher.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/useDelete.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/useGet.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/useMockGetter.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/useMockPagination.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/usePagination.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/usePatch.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/usePost.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/generics/usePut.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/organization/useGetOrganization.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/organization/useOrganizations.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/organization/usePutOrganization.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQueries.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQuery.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQueryExecution.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQueryGenerateMessage.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQueryPut.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQueryResubmit.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/query/useQuerySendMessage.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/useDeleteGoldenSql.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/useDownloadFile.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/useGoldenSqlList.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/user/useDeleteUser.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/user/useGetUser.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/user/usePatchUser.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/user/usePostUser.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/user/usePostUserToOrganization.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/api/user/useUsers.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/database/useColumnResource.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/database/useDatabaseOptions.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/database/useDatabaseResource.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/database/useDatabaseResourceFromTree.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/database/useTableResource.ts (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/ui/useIsMobile.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/hooks/useDebounce.ts (100%) rename {apps/ai/clients => services}/admin-console/src/lib/api/server-fetcher.test.ts (100%) rename {apps/ai/clients => services}/admin-console/src/lib/api/server-fetcher.ts (100%) rename {apps/ai/clients => services}/admin-console/src/lib/domain/billing.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/lib/domain/database.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/lib/domain/error.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/lib/domain/query.ts (100%) rename {apps/ai/clients => services}/admin-console/src/lib/utils.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/mocks/database.ts (100%) rename {apps/ai/clients => services}/admin-console/src/models/api.ts (100%) rename {apps/ai/clients => services}/admin-console/src/models/domain.ts (100%) rename {apps/ai/clients => services}/admin-console/src/models/errorCodes.ts (100%) rename {apps/ai/clients => services}/admin-console/src/pages/_app.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/_document.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/api-keys/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/api/auth/[...auth0].ts (100%) rename {apps/ai/clients => services}/admin-console/src/pages/api/auth/token.ts (100%) create mode 100644 services/admin-console/src/pages/api/health-check.ts rename {apps/ai/clients => services}/admin-console/src/pages/auth/error.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/billing/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/change-organization/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/databases/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/fine-tuning/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/golden-sql/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/my-account/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/organization/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/playground/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/queries/[queryId]/download-csv.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/queries/[queryId]/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/queries/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/switch-to-larger-screen/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/pages/usage/index.tsx (100%) rename {apps/ai/clients => services}/admin-console/src/styles/globals.css (100%) rename {apps/ai/clients => services}/admin-console/tailwind.config.js (100%) rename {apps/ai/clients => services}/admin-console/tsconfig.json (100%) rename {apps/ai/clients => services}/admin-console/vercel.json (100%) rename {apps/ai/server => services/enterprise}/.dockerignore (100%) rename {apps/ai/server => services/enterprise}/.env.example (100%) rename {apps/ai/server => services/enterprise}/.test.env (100%) rename {apps/ai/server => services/enterprise}/Dockerfile (100%) rename {apps/ai/server => services/enterprise}/Makefile (100%) rename {apps/ai/server => services/enterprise}/README.md (100%) rename {apps/ai/server => services/enterprise}/__init__.py (100%) rename {apps/ai/server => services/enterprise}/app.py (100%) rename {apps/ai/server => services/enterprise}/config.py (100%) rename {apps/ai/server => services/enterprise}/database/__init__.py (100%) rename {apps/ai/server => services/enterprise}/database/migrations/README.md (100%) rename {apps/ai/server => services/enterprise}/database/migrations/changelog.json (100%) rename {apps/ai/server => services/enterprise}/database/migrations/liquibase.advanced.flowfile.yaml (100%) rename {apps/ai/server => services/enterprise}/database/migrations/liquibase.checks-package.yaml (100%) rename {apps/ai/server => services/enterprise}/database/migrations/liquibase.endstage.flow (100%) rename {apps/ai/server => services/enterprise}/database/migrations/liquibase.flowfile.yaml (100%) rename {apps/ai/server => services/enterprise}/database/migrations/liquibase.flowvariables.yaml (100%) rename {apps/ai/server => services/enterprise}/database/migrations/liquibase.properties (100%) rename {apps/ai/server => services/enterprise}/database/mongo.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/__init__.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/copy_org_resources.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/data_report.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/delete_db_connection.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/initial_setup.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/populate_pinecone_db/requirements.txt (100%) rename {apps/ai/server => services/enterprise}/database/scripts/populate_pinecone_db/script.py (100%) rename {apps/ai/server => services/enterprise}/database/scripts/table_descriptions.json (100%) rename {apps/ai/server => services/enterprise}/database/scripts/upload_ssh_key_file.py (100%) rename {apps/ai/server => services/enterprise}/docker-compose.yml (100%) rename {apps/ai/server => services/enterprise}/exceptions/error_codes.py (100%) rename {apps/ai/server => services/enterprise}/exceptions/error_response.py (100%) rename {apps/ai/server => services/enterprise}/exceptions/exception_handlers.py (100%) rename {apps/ai/server => services/enterprise}/exceptions/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/exceptions/utils.py (100%) rename {apps/ai/server => services/enterprise}/middleware/error.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/auth/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/db_connection/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/finetuning/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/aggr_controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/aggr_service.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/generation/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/models/__ini__.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/golden_sql/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/instruction/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/key/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/invoice/webhook.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/organization/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/table_description/service.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/controller.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/models/__init__.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/models/entities.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/models/exceptions.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/models/requests.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/models/responses.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/repository.py (100%) rename {apps/ai/server => services/enterprise}/modules/user/service.py (100%) rename {apps/ai/server => services/enterprise}/pyproject.toml (100%) rename {apps/ai/server => services/enterprise}/requirements.txt (100%) rename {apps/ai/server => services/enterprise}/scripts/__init__.py (100%) rename {apps/ai/server => services/enterprise}/scripts/record_usage.py (100%) rename {apps/ai/server/tests => services/enterprise/tmp}/__init__.py (100%) rename {apps/ai/server/tests/auth => services/enterprise/utils}/__init__.py (100%) rename {apps/ai/server => services/enterprise}/utils/analytics.py (100%) rename {apps/ai/server => services/enterprise}/utils/auth.py (100%) rename {apps/ai/server => services/enterprise}/utils/billing.py (100%) rename {apps/ai/server => services/enterprise}/utils/encrypt.py (100%) rename {apps/ai/server => services/enterprise}/utils/misc.py (100%) rename {apps/ai/server => services/enterprise}/utils/s3.py (100%) rename {apps/ai/server => services/enterprise}/utils/sample_db.py (100%) rename {apps/ai/server => services/enterprise}/utils/slack.py (100%) rename {apps/ai/server => services/enterprise}/utils/validation.py (100%) rename {apps/ai/clients/slack => services/slackbot}/.aws/task-definition-prod.json (100%) rename {apps/ai/clients/slack => services/slackbot}/.aws/task-definition-stage.json (100%) rename {apps/ai/clients/slack => services/slackbot}/.env.example (100%) rename {apps/ai/clients/slack => services/slackbot}/.eslintrc (100%) rename {apps/ai/clients/slack => services/slackbot}/.prettierrc (100%) rename {apps/ai/clients/slack => services/slackbot}/Dockerfile (100%) rename {apps/ai/clients/slack => services/slackbot}/README.md (100%) rename {apps/ai/clients/slack => services/slackbot}/auth/index.js (100%) rename {apps/ai/clients/slack => services/slackbot}/config/index.js (100%) rename {apps/ai/clients/slack => services/slackbot}/docker-compose.yml (100%) rename {apps/ai/clients/slack => services/slackbot}/handlers/message/index.js (100%) rename {apps/ai/clients/slack => services/slackbot}/package.json (100%) rename {apps/ai/clients/slack => services/slackbot}/pnpm-lock.yaml (100%) rename {apps/ai/clients/slack => services/slackbot}/server/index.js (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index bebc7e47..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 - -updates: - - package-ecosystem: gitsubmodule - schedule: - interval: "daily" - directory: / diff --git a/.github/workflows/ai-build-docker.yml b/.github/workflows/ai-build-docker.yml deleted file mode 100644 index f961d14c..00000000 --- a/.github/workflows/ai-build-docker.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: AI Build Docker Image - -on: - workflow_call: - inputs: - target_env: - description: "The target environment to build the docker image for. Possible values are `branch`, `stage` or `prod`" - required: true - type: string - target_app: - description: "The target backend app to build the docker image for. Possible values are `engine`, `server`, or `slackbot`" - required: true - type: string - image_tag: - description: "The image tag" - required: true - type: string - - outputs: - docker_image: - description: "Docker image" - value: ${{ jobs.build-docker-image.outputs.docker_image }} - -env: - ECR_REPOSITORY: ai-${{ inputs.target_app }}-${{ inputs.target_env }} - -jobs: - build-docker-image: - name: Build Docker Image - runs-on: ubuntu-latest - - outputs: - docker_image: ${{ steps.build-image.outputs.docker_image }} - - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - - name: Use Node.js - if: inputs.target_app == 'slackbot' - uses: actions/setup-node@v4 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ inputs.image_tag }} - run: | - if [ ${{ inputs.target_app }} == "engine" ]; then - echo Build engine - git submodule init - git submodule update - cd apps/ai/server/dataherald - ls - elif [ ${{ inputs.target_app }} == "server" ]; then - echo Build server - cd apps/ai/server - elif [ ${{ inputs.target_app }} == "slackbot" ]; then - echo Build slackbot - cd apps/ai/clients/slack - npm install -g pnpm - pnpm install - fi - echo Building Dockerfile - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "docker_image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ai-chat-checks.yml b/.github/workflows/ai-chat-checks.yml deleted file mode 100644 index d2cc7be0..00000000 --- a/.github/workflows/ai-chat-checks.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: AI Chat - Quality Checks - -on: - push: - paths: - - apps/ai/clients/chat/** - branches: - - main - - preview - pull_request: - paths: - - apps/ai/clients/chat/** - branches: - - main - - preview - -jobs: - check-format: - runs-on: ubuntu-latest - defaults: - run: - working-directory: .apps/ai/clients/chat - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - - - name: Install pnpm - run: npm install -g pnpm@7.32.0 - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Check format - run: npm run format:check diff --git a/.github/workflows/ai-check-changes.yml b/.github/workflows/ai-check-changes.yml deleted file mode 100644 index 47278e78..00000000 --- a/.github/workflows/ai-check-changes.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: AI Check Repository Changes - -on: - workflow_call: - inputs: - target_env: - required: true - type: string - description: "The environment to check changes for. Possible values are: `stage`, `prod`" - outputs: - engine_changed: - description: "Whether the engine has changed" - value: ${{ jobs.set-changes.outputs.engine_changed }} - server_changed: - description: "Whether the server has changed" - value: ${{ jobs.set-changes.outputs.server_changed }} - slackbot_changed: - description: "Whether the slackbot has changed" - value: ${{ jobs.set-changes.outputs.slackbot_changed }} - console_changed: - description: "Whether the console has changed" - value: ${{ jobs.set-changes.outputs.console_changed }} - -jobs: - check-changes: - runs-on: ubuntu-latest - outputs: - engine_changed: ${{ steps.filter_engine.outputs.engine }} - server_changed: ${{ steps.filter_server.outputs.server }} - slackbot_changed: ${{ steps.filter_slackbot.outputs.slackbot }} - console_changed: ${{ steps.filter_console.outputs.console }} - steps: - - uses: actions/checkout@v4 - - - id: filter_engine - uses: dorny/paths-filter@v3 - with: - filters: | - engine: - - '.github/workflows/ai-deploy-env.yml' - - '.github/workflows/ai-deploy-${{ inputs.target_env }}-env.yml' - - 'apps/ai/server/.aws/task-definition-${{ inputs.target_env }}.json' - - 'apps/ai/server/main.tf' - - 'apps/ai/server/dataherald/**' - - - id: filter_server - uses: dorny/paths-filter@v3 - with: - filters: | - server: - - '.github/workflows/ai-deploy-env.yml' - - '.github/workflows/ai-deploy-${{ inputs.target_env }}-env.yml' - - 'apps/ai/server/.aws/task-definition-${{ inputs.target_env }}.json' - - 'apps/ai/server/main.tf' - - 'apps/ai/server/**' - - - id: filter_slackbot - uses: dorny/paths-filter@v3 - with: - filters: | - slackbot: - - '.github/workflows/ai-deploy-env.yml' - - '.github/workflows/ai-deploy-${{ inputs.target_env }}-env.yml' - - 'apps/ai/clients/slack/.aws/task-definition-${{ inputs.target_env }}.json' - - 'apps/ai/clients/slack/**' - - - id: filter_console - uses: dorny/paths-filter@v3 - with: - filters: | - console: - - '.github/workflows/ai-deploy-env.yml' - - '.github/workflows/ai-deploy-${{ inputs.target_env }}-env.yml' - - 'apps/ai/clients/admin-console/**' - set-changes: - needs: check-changes - runs-on: ubuntu-latest - outputs: - engine_changed: ${{ steps.set-outputs.outputs.engine_changed }} - server_changed: ${{ steps.set-outputs.outputs.server_changed }} - slackbot_changed: ${{ steps.set-outputs.outputs.slackbot_changed }} - console_changed: ${{ steps.set-outputs.outputs.console_changed }} - steps: - - name: Set Changes Outputs - id: set-outputs - run: | - if [ ${{ inputs.target_env }} == "stage" ]; then - echo Setting changes outputs for stage - echo "engine_changed=${{ needs.check-changes.outputs.engine_changed }}" >> $GITHUB_OUTPUT - echo "server_changed=${{ needs.check-changes.outputs.server_changed }}" >> $GITHUB_OUTPUT - echo "slackbot_changed=${{ needs.check-changes.outputs.slackbot_changed }}" >> $GITHUB_OUTPUT - echo "console_changed=${{ needs.check-changes.outputs.console_changed }}" >> $GITHUB_OUTPUT - else - echo Setting changes outputs for prod -- all true - echo "engine_changed=true" >> $GITHUB_OUTPUT - echo "server_changed=true" >> $GITHUB_OUTPUT - echo "slackbot_changed=true" >> $GITHUB_OUTPUT - echo "console_changed=true" >> $GITHUB_OUTPUT - fi diff --git a/.github/workflows/ai-console-checks.yml b/.github/workflows/ai-console-checks.yml deleted file mode 100644 index 32ed4cb5..00000000 --- a/.github/workflows/ai-console-checks.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: AI Console - Quality Checks -on: - push: - paths: - - .github/workflows/ai-console-checks.yml # this file - - apps/ai/clients/admin-console/** - branches: - - main - pull_request: - paths: - - .github/workflows/ai-admin-console-build.yml - - apps/ai/clients/admin-console/** - branches: - - main - -jobs: - lint: - runs-on: ubuntu-latest - defaults: - run: - working-directory: apps/ai/clients/admin-console - - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - - - name: Install pnpm - run: npm install -g pnpm - - - name: Install dependencies - run: pnpm install - - - name: Lint code - run: pnpm lint - - test: - runs-on: ubuntu-latest - defaults: - run: - working-directory: apps/ai/clients/admin-console - - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - - - name: Install pnpm - run: npm install -g pnpm - - - name: Install dependencies - run: pnpm install - - - name: Run tests - run: pnpm test diff --git a/.github/workflows/ai-deploy-backend.yml b/.github/workflows/ai-deploy-backend.yml deleted file mode 100644 index 2d4de76b..00000000 --- a/.github/workflows/ai-deploy-backend.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: AI Deploy Backend - -on: - workflow_call: - inputs: - target_env: - description: "The target environment to deploy to. Possible values: `stage` or `prod`" - required: true - type: string - engine_image: - description: "The engine image to deploy" - required: true - type: string - server_image: - description: "The server image to deploy" - required: true - type: string - -env: - AWS_REGION: us-east-1 - ECS_CLUSTER: ai - ECS_SERVICE: ai-backend-${{ inputs.target_env }} - -jobs: - deploy-to-ecs: - name: Deploy to ECS - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Replace Engine image ID in the Amazon ECS task definition - id: partial-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: ./apps/ai/server/.aws/task-definition-${{ inputs.target_env }}.json - container-name: ai-engine-${{ inputs.target_env }} - image: ${{ inputs.engine_image }} - - - name: Replace Server image ID in the Amazon ECS task definition - id: complete-task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: ${{ steps.partial-def.outputs.task-definition }} - container-name: ai-server-${{ inputs.target_env }} - image: ${{ inputs.server_image }} - - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.complete-task-def.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true diff --git a/.github/workflows/ai-deploy-branch-env.yml b/.github/workflows/ai-deploy-branch-env.yml deleted file mode 100644 index 3b282334..00000000 --- a/.github/workflows/ai-deploy-branch-env.yml +++ /dev/null @@ -1,364 +0,0 @@ -name: AI Deploy Branch Environment - -on: - push: - branches: - - customer/* ## Automatically trigger the workflow on push to customer branches - workflow_dispatch: - inputs: - option: - description: "Select Initial Database (if DB already exists this will be dismissed)" - required: true - default: "sample" - type: choice - options: - - "stage" - - "sample" - - "empty" - -env: - ECR_CORE_REPOSITORY: ai-engine-branch - ECR_SERVER_REPOSITORY: ai-server-branch - SLACK_ICON: https://files.dataherald.com/random/github-actions.png - SLACK_USERNAME: GitHub Actions - SLACK_CHANNEL: eng-notifications - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} - -jobs: - extract-branch-names: - name: Extract Branch Names - uses: ./.github/workflows/ai-extract-branch-names.yml - - start-self-hosted-runner: - name: Start self-hosted EC2 runner - uses: ./.github/workflows/ai-start-self-hosted-runner.yml - secrets: inherit - - build-engine: - name: Build Engine - needs: extract-branch-names - uses: ./.github/workflows/ai-build-docker.yml - secrets: inherit - with: - target_app: engine - target_env: branch - image_tag: ${{ needs.extract-branch-names.outputs.branch_name}}-${{ github.sha }} - - build-server: - name: Build Server - needs: extract-branch-names - uses: ./.github/workflows/ai-build-docker.yml - secrets: inherit - with: - target_app: server - target_env: branch - image_tag: ${{ needs.extract-branch-names.outputs.branch_name }}-${{ github.sha }} - - deploy-backend: - name: Deploy Backend infrastructure - needs: [extract-branch-names, build-engine, build-server] - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./apps/ai/server/terraform/branch - outputs: - api_url: ${{ steps.get-api-url.outputs.api_url }} - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - name: Apply Terraform - env: - TF_VAR_sha: ${{ github.sha }} - TF_VAR_branch_name: ${{ needs.extract-branch-names.outputs.branch_name }} - TF_VAR_vpc_id: ${{ vars.VPC_ID }} - TF_VAR_private_subnet_1_id: ${{ vars.PRIVATE_SUBNET_1_ID }} - TF_VAR_private_subnet_2_id: ${{ vars.PRIVATE_SUBNET_2_ID }} - TF_VAR_public_subnet_1_id: ${{ vars.PUBLIC_SUBNET_1_ID }} - TF_VAR_public_subnet_2_id: ${{ vars.PUBLIC_SUBNET_2_ID }} - TF_VAR_ecs_security_group_id: ${{ vars.BRANCH_SECURITY_GROUP_ID }} - TF_VAR_index_name: ${{ needs.extract-branch-names.outputs.db_name }} - TF_VAR_mongodb_uri: ${{ vars.EPHEMERAL_MONGODB_URI }} - TF_VAR_mongodb_name: ${{ needs.extract-branch-names.outputs.db_name }} - TF_VAR_mongodb_username: ${{ secrets.MONGODB_USERNAME }} - TF_VAR_mongodb_password: ${{ secrets.MONGODB_PASSWORD }} - - run: | - terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}" - terraform plan -target=module.aws_ecs - terraform apply -target=module.aws_ecs -auto-approve - - name: Get API URL - id: get-api-url - run: | - echo "api_url=https://${{ needs.extract-branch-names.outputs.branch_name }}.api.dataherald.ai" >> "$GITHUB_OUTPUT" - - name: Delete untagged images - run: | - # Get untagged images - UNTAGGED_CORE_IMAGES=$(aws ecr list-images --repository-name $ECR_CORE_REPOSITORY --filter tagStatus=UNTAGGED --query 'imageIds[*]' --region us-east-1 --output json --region us-east-1) - UNTAGGED_SERVER_IMAGES=$(aws ecr list-images --repository-name $ECR_SERVER_REPOSITORY --filter tagStatus=UNTAGGED --query 'imageIds[*]' --region us-east-1 --output json --region us-east-1) - - # Delete images - if [ -n "$UNTAGGED_CORE_IMAGES" ] && [ "$UNTAGGED_CORE_IMAGES" != "[]" ]; then - aws ecr batch-delete-image --repository-name $ECR_CORE_REPOSITORY --image-ids "$UNTAGGED_CORE_IMAGES" --region us-east-1 - fi - if [ -n "$UNTAGGED_SERVER_IMAGES" ] && [ "$UNTAGGED_SERVER_IMAGES" != "[]" ]; then - aws ecr batch-delete-image --repository-name $ECR_SERVER_REPOSITORY --image-ids "$UNTAGGED_SERVER_IMAGES" --region us-east-1 - fi - create-vector-db: - name: Create PineconeDB - needs: [extract-branch-names] - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./apps/ai/server/terraform/branch - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: false - - name: Apply Terraform - env: - TF_VAR_index_name: ${{ needs.extract-branch-names.outputs.db_name }} - run: | - terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}" - terraform plan -target=module.pinecone_index - terraform apply -target=module.pinecone_index -auto-approve - populate-db: - name: Populate MongoDB - needs: [extract-branch-names, start-self-hosted-runner] - runs-on: ${{ needs.start-self-hosted-runner.outputs.label }} - defaults: - run: - working-directory: ./apps/ai/server - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - - name: Check if MongoDB database exists - id: db_check - run: | - exists=$(mongosh --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} "${{ vars.EPHEMERAL_MONGODB_URI }}" --quiet --eval "db.getMongo().getDBNames().includes('${{ needs.extract-branch-names.outputs.db_name }}') ? 'true' : 'false';" | grep -o true || echo "false") - echo "Branch Database exists: $exists" - echo "exists=$exists" >> $GITHUB_OUTPUT - shell: bash - - - name: Restore MongoDB from STAGE dump # ensure there's only one folder in the dump directory - if: steps.db_check.outputs.exists == 'false' && github.event.inputs.option == 'stage' - run: | - echo "Previous step output: ${{ steps.db_check.outputs.exists }}" - echo "Database does not exist. Restoring from STAGE dump..." - mongodump --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} --uri="${{ vars.STAGING_MONGODB_URI }}" - cd dump - from_db=(*) - mongorestore . --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} --nsFrom "$from_db.*" --nsTo "${{ needs.extract-branch-names.outputs.db_name }}.*" --uri="${{ vars.EPHEMERAL_MONGODB_URI }}" - - name: Restore MongoDB from sample db - if: steps.db_check.outputs.exists == 'false' && github.event.inputs.option == 'sample' - run: | - echo "Previous step output: ${{ steps.db_check.outputs.exists }}" - echo "Database does not exist. Restoring from SAMPLE dump..." - cd database/migrations/dump - from_db=(*) - mongorestore . --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} --nsFrom "$from_db.*" --nsTo "${{ needs.extract-branch-names.outputs.db_name }}.*" --uri="${{ vars.EPHEMERAL_MONGODB_URI }}" - - name: Create an empty MongoDB database - if: steps.db_check.outputs.exists == 'false' && github.event.inputs.option == 'empty' - run: | - echo "Previous step output: ${{ steps.db_check.outputs.exists }}" - echo "Database does not exist. Creating an empty database..." - mongosh --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} "${{ vars.EPHEMERAL_MONGODB_URI }}" --eval "db.getSiblingDB('${{ needs.extract-branch-names.outputs.db_name }}').createCollection('empty')" - - name: Run migration script # liquibase cli and mongo driver already installed in runner - env: - LIQUIBASE_COMMAND_USERNAME: ${{ secrets.MONGODB_USERNAME }} - LIQUIBASE_COMMAND_PASSWORD: ${{ secrets.MONGODB_PASSWORD }} - LIQUIBASE_COMMAND_URL: ${{ vars.EPHEMERAL_MONGODB_URI }}/${{ needs.extract-branch-names.outputs.db_name }} - run: | - liquibase --changeLogFile=database/migrations/changelog.json update - - name: Delete DB if failure - if: failure() - run: | - mongosh --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} "${{ vars.EPHEMERAL_MONGODB_URI }}" --eval "db.getSiblingDB('${{ needs.extract-branch-names.outputs.db_name }}').dropDatabase()" - - populate-vector-db: - name: Populate PineconeDB - needs: - [ - extract-branch-names, - start-self-hosted-runner, - populate-db, - create-vector-db, - ] - runs-on: ${{ needs.start-self-hosted-runner.outputs.label }} - defaults: - run: - working-directory: ./apps/ai/server - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - name: Populate PineconeDB - env: - PINECONE_INDEX_NAME: ${{ needs.extract-branch-names.outputs.db_name }} - MONGODB_URI: ${{ vars.EPHEMERAL_MONGODB_URI }} - MONGODB_USERNAME: ${{ secrets.MONGODB_USERNAME }} - MONGODB_PASSWORD: ${{ secrets.MONGODB_PASSWORD }} - MONGODB_NAME: ${{ needs.extract-branch-names.outputs.db_name }} - run: | - cd database/scripts/populate_pinecone_db - python3 -m venv venv - source venv/bin/activate - pip3 install -r requirements.txt - python3 -u script.py - - deploy-frontend: - name: Deploy Frontend - needs: [extract-branch-names, deploy-backend] - runs-on: ubuntu-latest - outputs: - url: ${{ steps.console_dns.outputs.url }} - env: - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_AI_CONSOLE_PROJECT_ID }} - VERCEL_TEAM_NAME: ${{ vars.VERCEL_TEAM_NAME }} - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Install Vercel CLI - run: npm install -g vercel - - name: Set Console DNS - id: console_dns - run: | - echo "domain=${{ needs.extract-branch-names.outputs.branch_name }}.console.dataherald.ai" >> "$GITHUB_OUTPUT" - echo "url=https://${{ needs.extract-branch-names.outputs.branch_name }}.console.dataherald.ai" >> "$GITHUB_OUTPUT" - - name: Deploy to Vercel and set custom domain alias - run: | - echo Deploying to Vercel with the following environment variables: - echo "NEXT_PUBLIC_API_URL=${{ needs.deploy-backend.outputs.api_url }}" - echo "AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }}" - - vercel deploy --yes --build-env NEXT_PUBLIC_API_URL=${{ needs.deploy-backend.outputs.api_url }} --env AUTH0_BASE_URL=${{ steps.console_dns.outputs.url }} --token=${{ secrets.VERCEL_ACCESS_TOKEN }} >deployment-url.txt 2>error.txt - - # check the exit code - code=$? - if [ $code -eq 0 ]; then - # Now you can use the deployment url from stdout for the next step of your workflow - deploymentUrl=`cat deployment-url.txt` - vercel alias set $deploymentUrl ${{ steps.console_dns.outputs.domain }} --scope $VERCEL_TEAM_NAME --token=${{ secrets.VERCEL_ACCESS_TOKEN }} - else - # Handle the error - errorMessage=`cat error.txt` - echo "There was an error: $errorMessage" - fi - - send-cancelled-slack-notification: - name: Send Cancelled Slack Notification - needs: - [ - extract-branch-names, - deploy-backend, - populate-db, - populate-vector-db, - deploy-frontend, - ] - if: cancelled() - runs-on: ubuntu-latest - steps: - - name: Send cancelled notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_TITLE: Branch Deployment CANCELLED - SLACK_MESSAGE: | - The branch environment deployment was cancelled. Please check the GitHub Actions logs for more details. - - *Branch*: `${{ needs.extract-branch-names.outputs.branch }}` - - SLACK_COLOR: cancelled - - send-failure-slack-notification: - name: Send Failure Slack Notification - needs: - [ - extract-branch-names, - deploy-backend, - populate-db, - populate-vector-db, - deploy-frontend, - ] - if: needs.deploy-backend.result == 'failure' || needs.populate-db.result == 'failure' || needs.populate-vector-db.result == 'failure' || needs.deploy-frontend.result == 'failure' - runs-on: ubuntu-latest - steps: - - name: Send failed notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_TITLE: Branch Deployment FAILED - SLACK_MESSAGE: | - The branch environment deployment failed. Please check the GitHub Actions logs for more details. - - *Branch*: `${{ needs.extract-branch-names.outputs.branch }}` - - SLACK_COLOR: failure - - send-success-slack-notification: - name: Send Success Slack Notification - needs: - [ - extract-branch-names, - deploy-backend, - populate-db, - populate-vector-db, - deploy-frontend, - ] - if: needs.deploy-backend.result == 'success' && needs.populate-db.result == 'success' && needs.populate-vector-db.result == 'success' && needs.deploy-frontend.result == 'success' - runs-on: ubuntu-latest - steps: - - name: Send success notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_TITLE: Branch Deployment SUCCEEDED - SLACK_MESSAGE: | - The branch environment deployment succeeded. Here are the details: - - *Branch*: `${{ needs.extract-branch-names.outputs.branch }}` - *API URL*: ${{ needs.deploy-backend.outputs.api_url }} - *Console URL*: ${{ needs.deploy-frontend.outputs.url }} - SLACK_COLOR: success - - stop-self-hosted-runner: - name: Stop self-hosted EC2 runner - needs: [start-self-hosted-runner, populate-db, populate-vector-db] - if: always() - uses: ./.github/workflows/ai-stop-self-hosted-runner.yml - secrets: inherit - with: - label: ${{ needs.start-self-hosted-runner.outputs.label }} - ec2_instance_id: ${{ needs.start-self-hosted-runner.outputs.ec2_instance_id }} diff --git a/.github/workflows/ai-deploy-docker-ecs.yml b/.github/workflows/ai-deploy-docker-ecs.yml deleted file mode 100644 index 634138f1..00000000 --- a/.github/workflows/ai-deploy-docker-ecs.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: AI Deploy Docker Image to ECS - -on: - workflow_call: - inputs: - docker_image: - description: "Docker image" - required: true - type: string - ecs_service_name: - description: "The ECS service name" - required: true - type: string - ecs_task_definition: - description: "The ECS task definition" - required: true - type: string - container_name: - description: "The container name to update in the ECS service task definition" - required: true - type: string - -env: - AWS_REGION: us-east-1 - ECS_CLUSTER: ai - ECS_SERVICE: ${{ inputs.ecs_service_name }} - CONTAINER_NAME: ${{ inputs.container_name }} - ECS_TASK_DEFINITION: ${{ inputs.ecs_task_definition }} - -jobs: - deploy-to-ecs: - name: Deploy to ECS - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ inputs.docker_image }} - - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true diff --git a/.github/workflows/ai-deploy-env.yml b/.github/workflows/ai-deploy-env.yml deleted file mode 100644 index 2c1c60f0..00000000 --- a/.github/workflows/ai-deploy-env.yml +++ /dev/null @@ -1,204 +0,0 @@ -name: Deploy Environment - -on: - workflow_call: - inputs: - target_env: - description: "The target environment to deploy to. Possible values are: `stage` or `prod`" - required: true - type: string - -env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_ICON: https://files.dataherald.com/random/github-actions.png - SLACK_USERNAME: GitHub Actions - SLACK_CHANNEL: eng-notifications - -jobs: - check-changes: - name: Check Changes - uses: ./.github/workflows/ai-check-changes.yml - with: - target_env: ${{ inputs.target_env }} - - resolve-env-db-url: - runs-on: ubuntu-latest - name: Get Database URL - outputs: - db_url: ${{ steps.get-db-url.outputs.db_url }} - steps: - - name: Get DB URL - id: get-db-url - run: | - if [ ${{ inputs.target_env }} == 'stage' ]; then - echo "db_url=${{ vars.STAGING_MONGODB_URI}}/k2-stage-serverless" >> $GITHUB_OUTPUT - else - echo "db_url=${{vars.PRODUCTION_MONGODB_URI}}/k2-serverless" >> $GITHUB_OUTPUT - fi - - migrate-db: - name: Migrate Database - needs: [resolve-env-db-url] - uses: ./.github/workflows/ai-migrate-db.yml - secrets: inherit - with: - target_db_url: ${{ needs.resolve-env-db-url.outputs.db_url }} - - build-engine: - name: Build Engine - needs: check-changes - if: needs.check-changes.outputs.engine_changed == 'true' || needs.check-changes.outputs.server_changed == 'true' - uses: ./.github/workflows/ai-build-docker.yml - secrets: inherit - with: - target_app: engine - target_env: ${{ inputs.target_env }} - image_tag: ${{ github.sha }} - - build-server: - name: Build Server - needs: check-changes - if: needs.check-changes.outputs.engine_changed == 'true' || needs.check-changes.outputs.server_changed == 'true' - uses: ./.github/workflows/ai-build-docker.yml - secrets: inherit - with: - target_app: server - target_env: ${{ inputs.target_env }} - image_tag: ${{ github.sha }} - - deploy-backend: - name: Deploy Backend - needs: [build-engine, build-server] - uses: ./.github/workflows/ai-deploy-backend.yml - secrets: inherit - with: - target_env: ${{ inputs.target_env }} - engine_image: ${{ needs.build-engine.outputs.docker_image }} - server_image: ${{ needs.build-server.outputs.docker_image }} - - build-slackbot: - name: Build Slackbot - needs: [check-changes] - if: needs.check-changes.outputs.slackbot_changed == 'true' - uses: ./.github/workflows/ai-build-docker.yml - secrets: inherit - with: - target_app: slackbot - target_env: ${{ inputs.target_env }} - image_tag: ${{ github.sha }} - - deploy-slackbot: - name: Deploy Slackbot - needs: [build-slackbot, deploy-backend] - if: ${{ needs.build-slackbot.result == 'success' && !failure() }} - uses: ./.github/workflows/ai-deploy-docker-ecs.yml - secrets: inherit - with: - docker_image: ${{ needs.build-slackbot.outputs.docker_image }} - ecs_service_name: ai-slackbot-${{ inputs.target_env }} - ecs_task_definition: ./apps/ai/clients/slack/.aws/task-definition-${{ inputs.target_env }}.json - container_name: ai-slackbot-${{ inputs.target_env }} - - deploy-frontend: - name: Deploy Console - needs: [check-changes, deploy-backend] - if: ${{ needs.check-changes.outputs.console_changed == 'true' && !failure() }} - uses: ./.github/workflows/ai-deploy-frontend.yml - secrets: inherit - with: - target_env: ${{ inputs.target_env }} - - slack-notification-env: - name: Set Environment Messages - runs-on: ubuntu-latest - outputs: - environment: ${{ steps.set-slack-env.outputs.environment }} - steps: - - name: Set env messages - id: set-slack-env - run: | - if [ ${{ inputs.target_env }} == 'stage' ]; then - echo "environment=STAGE" >> $GITHUB_OUTPUT - else - echo "environment=PRODUCTION" >> $GITHUB_OUTPUT - fi - - send-cancelled-slack-notification: - name: Send Cancelled Slack Notification - needs: - [ - slack-notification-env, - migrate-db, - deploy-backend, - deploy-slackbot, - deploy-frontend, - ] - if: cancelled() - runs-on: ubuntu-latest - steps: - - name: Send cancelled notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_TITLE: ${{ needs.slack-notification-env.outputs.environment }} Deployment CANCELLED - SLACK_MESSAGE: | - The ${{ needs.slack-notification-env.outputs.environment }} environment deployment was cancelled. - - *WARNING:* Some artifacts could be deployed before cancelled. Please check the GitHub Actions logs for more details. - SLACK_COLOR: cancelled - - send-failure-slack-notification: - name: Send Failure Slack Notification - needs: - [ - slack-notification-env, - migrate-db, - deploy-backend, - deploy-slackbot, - deploy-frontend, - ] - if: failure() - runs-on: ubuntu-latest - steps: - - name: Send failed notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_TITLE: ${{ needs.slack-notification-env.outputs.environment }} Deployment FAILED - SLACK_MESSAGE: | - The ${{ needs.slack-notification-env.outputs.environment }} environment deployment failed. One or more of the deployment jobs failed. - - *WARNING:* Some artifacts could be deployed before failure. Please check the GitHub Actions logs for more details. - SLACK_COLOR: failure - - send-success-slack-notification: - name: Send Success Slack Notification - needs: - [ - slack-notification-env, - migrate-db, - deploy-backend, - deploy-slackbot, - deploy-frontend, - ] - if: ${{ !failure() }} - runs-on: ubuntu-latest - steps: - - name: Setup URLs - id: setup-urls - run: | - if [ ${{ inputs.target_env }} == 'stage' ]; then - echo "API_URL=https://stage.api.dataherald.ai" >> $GITHUB_OUTPUT - echo "CONSOLE_URL=https://stage.console.dataherald.ai" >> $GITHUB_OUTPUT - else - echo "API_URL=https://api.dataherald.ai" >> $GITHUB_OUTPUT - echo "CONSOLE_URL=https://console.dataherald.ai" >> $GITHUB_OUTPUT - fi - - name: Send success notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_TITLE: ${{ needs.slack-notification-env.outputs.environment }} Deployment SUCCEEDED - SLACK_MESSAGE: | - The ${{ needs.slack-notification-env.outputs.environment }} environment deployment succeeded. More details: - - *API URL*: ${{ steps.setup-urls.outputs.API_URL }} - *Console URL*: ${{ steps.setup-urls.outputs.CONSOLE_URL }} - SLACK_COLOR: success diff --git a/.github/workflows/ai-deploy-frontend.yml b/.github/workflows/ai-deploy-frontend.yml deleted file mode 100644 index da4d4371..00000000 --- a/.github/workflows/ai-deploy-frontend.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: AI Deploy Frontend - -on: - workflow_call: - inputs: - target_env: - description: The target environment to deploy to. Possible values are `stage` or `prod` - required: true - type: string - -env: - VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ secrets.VERCEL_AI_CONSOLE_PROJECT_ID }} - -jobs: - resolve-vercel-env: - name: Resolve Vercel Environment - runs-on: ubuntu-latest - outputs: - environment: ${{ steps.resolve-env.outputs.vercel_env }} - steps: - - name: Resolve Vercel Environment - id: resolve-env - run: | - if [ ${{ inputs.target_env }} == "prod" ]; then - echo "vercel_env=production" >> $GITHUB_OUTPUT - else - echo "vercel_env=preview" >> $GITHUB_OUTPUT - fi - shell: bash - deploy-vercel: - name: Deploy Vercel - needs: resolve-vercel-env - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install pnpm - run: npm install --global pnpm - - name: Install Vercel CLI - run: npm install --global vercel@latest - - name: Pull Vercel Environment Information - run: vercel pull --yes --environment=${{ needs.resolve-vercel-env.outputs.environment }} --token=${{ secrets.VERCEL_ACCESS_TOKEN }} - - name: Build Project Artifacts - run: | - if [ ${{ inputs.target_env }} == "prod" ]; then - vercel build --prod --token=${{ secrets.VERCEL_ACCESS_TOKEN }} - else - vercel build --token=${{ secrets.VERCEL_ACCESS_TOKEN }} - fi - - name: Deploy Project Artifacts to Vercel - run: | - if [ ${{ inputs.target_env }} == "prod" ]; then - vercel deploy --prebuilt --prod --token=${{ secrets.VERCEL_ACCESS_TOKEN }} - else - vercel deploy --prebuilt --token=${{ secrets.VERCEL_ACCESS_TOKEN }} - fi diff --git a/.github/workflows/ai-deploy-prod-env.yml b/.github/workflows/ai-deploy-prod-env.yml deleted file mode 100644 index ae32de2c..00000000 --- a/.github/workflows/ai-deploy-prod-env.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: AI Deploy PROD Environment - -on: - release: - types: [published] - -jobs: - deploy-env: - name: Deploy Production - uses: ./.github/workflows/ai-deploy-env.yml - secrets: inherit - with: - target_env: prod diff --git a/.github/workflows/ai-deploy-stage-env.yml b/.github/workflows/ai-deploy-stage-env.yml deleted file mode 100644 index 8c43f775..00000000 --- a/.github/workflows/ai-deploy-stage-env.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: AI Deploy STAGE Environment - -on: - push: - branches: - - main - paths: - - ".github/workflows/ai-deploy-stage-env.yml" # this file - - ".github/workflows/ai-deploy-env.yml" - - ".github/workflows/ai-build-docker.yml" - - ".github/workflows/ai-deploy-docker-ecs.yml" - - ".github/workflows/ai-deploy-frontend.yml" - - "apps/ai/**" - - "!**/.env.example" - - "!**/README.md" - -jobs: - deploy-env: - name: Deploy Stage - uses: ./.github/workflows/ai-deploy-env.yml - secrets: inherit - with: - target_env: stage diff --git a/.github/workflows/ai-destroy-branch-env.yml b/.github/workflows/ai-destroy-branch-env.yml deleted file mode 100644 index 7dc5a3cf..00000000 --- a/.github/workflows/ai-destroy-branch-env.yml +++ /dev/null @@ -1,174 +0,0 @@ -name: AI Destroy Branch Environment - -on: - workflow_dispatch: - pull_request: - types: [closed] - branches: - - main - -env: - AWS_REGION: us-east-1 - ECR_CORE_REPOSITORY: ai-engine-branch - ECR_SERVER_REPOSITORY: ai-server-branch - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_ICON: https://files.dataherald.com/random/github-actions.png - SLACK_USERNAME: GitHub Actions - SLACK_CHANNEL: eng-notifications - -jobs: - extract-branch-names: - name: Extract Branch Names - uses: ./.github/workflows/ai-extract-branch-names.yml - - can-destroy: - name: Check if branch environment can be destroyed - runs-on: ubuntu-latest - needs: extract-branch-names - if: ${{ github.event_name == 'workflow_dispatch' || !startsWith(needs.extract-branch-names.outputs.branch, 'customer/') }} - steps: - - name: Debug output - run: | - echo "Original branch: ${{ needs.extract-branch-names.outputs.branch }}" - echo "Starts with 'customer/' prefix check: ${{ ! startsWith(needs.extract-branch-names.outputs.branch, 'customer/') }}" - echo "GitHub event name: ${{ github.event_name }}" - echo "Branch can be destroyed. It's either a manual trigger or not a customer branch." - shell: bash - - destroy-branch-env: - name: Destroy branch environment - runs-on: ubuntu-latest - needs: [extract-branch-names, can-destroy] - if: success() - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} - defaults: - run: - working-directory: ./apps/ai/server/terraform/branch - - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_wrapper: true - - name: Destroy Terraform - env: - TF_VAR_sha: ${{ github.sha }} - TF_VAR_branch_name: ${{ needs.extract-branch-names.outputs.branch_name }} - TF_VAR_index_name: ${{ needs.extract-branch-names.outputs.db_name }} - TF_VAR_vpc_id: ${{ vars.VPC_ID }} - TF_VAR_private_subnet_1_id: ${{ vars.PRIVATE_SUBNET_1_ID }} - TF_VAR_private_subnet_2_id: ${{ vars.PRIVATE_SUBNET_2_ID }} - TF_VAR_public_subnet_1_id: ${{ vars.PUBLIC_SUBNET_1_ID }} - TF_VAR_public_subnet_2_id: ${{ vars.PUBLIC_SUBNET_2_ID }} - TF_VAR_ecs_security_group_id: ${{ vars.BRANCH_SECURITY_GROUP_ID }} - TF_VAR_mongodb_uri: ${{ vars.EPHEMERAL_MONGODB_URI }} - TF_VAR_mongodb_name: ${{ needs.extract-branch-names.outputs.db_name }} - TF_VAR_mongodb_username: ${{ secrets.MONGODB_USERNAME }} - TF_VAR_mongodb_password: ${{ secrets.MONGODB_PASSWORD }} - timeout-minutes: 3 - run: | - terraform init -upgrade -backend-config="key=${{ needs.extract-branch-names.outputs.branch_name }}" - terraform destroy -auto-approve - - name: Delete images - run: | - # Delete images - aws ecr batch-delete-image --repository-name $ECR_CORE_REPOSITORY --image-ids imageTag=${{ needs.extract-branch-names.outputs.branch_name }} --region us-east-1 - aws ecr batch-delete-image --repository-name $ECR_SERVER_REPOSITORY --image-ids imageTag=${{ needs.extract-branch-names.outputs.branch_name }} --region us-east-1 - - start-self-hosted-runner: - name: Start self-hosted EC2 runner - needs: [extract-branch-names, can-destroy] - if: success() - uses: ./.github/workflows/ai-start-self-hosted-runner.yml - secrets: inherit - - delete-database: - name: Delete database - needs: [start-self-hosted-runner, extract-branch-names, can-destroy] - if: success() - runs-on: ${{ needs.start-self-hosted-runner.outputs.label }} - steps: - - name: Delete mongo database - run: | - mongosh --username ${{ secrets.MONGODB_USERNAME }} --password ${{ secrets.MONGODB_PASSWORD }} "${{ vars.EPHEMERAL_MONGODB_URI }}" --eval "db.getSiblingDB('${{ needs.extract-branch-names.outputs.db_name }}').dropDatabase()" - - stop-self-hosted-runner: - name: Stop self-hosted EC2 runner - needs: [start-self-hosted-runner, delete-database] - if: needs.start-self-hosted-runner.result == 'success' - uses: ./.github/workflows/ai-stop-self-hosted-runner.yml - secrets: inherit - with: - label: ${{ needs.start-self-hosted-runner.outputs.label }} - ec2_instance_id: ${{ needs.start-self-hosted-runner.outputs.ec2_instance_id }} - - send-cancelled-slack-notification: - name: Send Cancelled Slack Notification - needs: - [extract-branch-names, can-destroy, destroy-branch-env, delete-database] - if: cancelled() - runs-on: ubuntu-latest - steps: - - name: Send cancelled notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_TITLE: DESTROY branch environment CANCELLED - SLACK_MESSAGE: | - The branch environment destruction has been cancelled. - - *Branch*: `${{ needs.extract-branch-names.outputs.branch }}` - - SLACK_COLOR: cancelled - - send-failure-slack-notification: - name: Send Failure Slack Notification - needs: - [ - extract-branch-names, - can-destroy, - start-self-hosted-runner, - destroy-branch-env, - delete-database, - stop-self-hosted-runner, - ] - if: failure() - runs-on: ubuntu-latest - steps: - - name: Send failed notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_TITLE: DESTROY branch environment FAILED - SLACK_MESSAGE: | - The branch environment destruction failed. Please check the GitHub Actions logs for more details. - - *Branch*: `${{ needs.extract-branch-names.outputs.branch }}` - - SLACK_COLOR: failure - - send-success-slack-notification: - name: Send Success Slack Notification - needs: - [extract-branch-names, can-destroy, destroy-branch-env, delete-database] - if: success() - runs-on: ubuntu-latest - steps: - - name: Send success notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.GH_SLACK_WEBHOOK_URL }} - SLACK_TITLE: DESTROY Branch Environment SUCCEEDED - SLACK_MESSAGE: | - The branch environment has been successfully destroyed. - - *Branch*: `${{ needs.extract-branch-names.outputs.branch }}` - SLACK_COLOR: success diff --git a/.github/workflows/ai-extract-branch-names.yml b/.github/workflows/ai-extract-branch-names.yml deleted file mode 100644 index 751a7f81..00000000 --- a/.github/workflows/ai-extract-branch-names.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Extract Branch Names - -on: - workflow_call: - outputs: - branch: - value: ${{ jobs.extract-branch-names.outputs.branch }} - branch_name: - value: ${{ jobs.extract-branch-names.outputs.branch_name }} - db_name: - value: ${{ jobs.extract-branch-names.outputs.db_name }} - -jobs: - extract-branch-names: - name: Setup Names From Branch - outputs: - branch: ${{ steps.extract_branch.outputs.branch }} - branch_name: ${{ steps.extract_branch.outputs.branch_name }} - db_name: ${{ steps.extract_branch.outputs.db_name }} - runs-on: ubuntu-latest - steps: - - id: extract_branch - shell: bash - env: - RESOURCE_NAME_LENGHT: 32 - run: | - branch="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" - - ### STRIP "customer/" prefix if exists and format the branch name - - formatted_branch=$(echo "$branch" | tr '[:upper:]' '[:lower:]' | sed 's/^customer\///' | sed 's/[^a-zA-Z0-9]/-/g') - db_name=$formatted_branch - - # Terraform resources name lenght limit is 32 - # MongoDB database name length limit is 38 - formatted_branch=${formatted_branch:0:${{ env.RESOURCE_NAME_LENGHT }}} - db_name=${db_name:0:${{ env.RESOURCE_NAME_LENGHT }}} - - # Ensure the last character is alphanumeric - # Pinecone doesn't allow special characters for the last character - while [[ ! ${formatted_branch: -1} =~ [a-zA-Z0-9] ]]; do - formatted_branch=${formatted_branch:0:-1} - done - - echo "branch=$branch" >> $GITHUB_OUTPUT - echo "branch_name=$formatted_branch" >> $GITHUB_OUTPUT - echo "db_name=$db_name" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ai-migrate-db.yml b/.github/workflows/ai-migrate-db.yml deleted file mode 100644 index 2fa47cb6..00000000 --- a/.github/workflows/ai-migrate-db.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Migrate Database - -on: - workflow_call: - inputs: - target_db_url: - description: The target database URL - required: true - type: string - -env: - AWS_REGION: us-east-1 - -jobs: - start-self-hosted-runner: - name: Start self-hosted EC2 runner - uses: ./.github/workflows/ai-start-self-hosted-runner.yml - secrets: inherit - migrate-db: - name: Migrate Database - needs: [start-self-hosted-runner] - runs-on: ${{ needs.start-self-hosted-runner.outputs.label }} - defaults: - run: - working-directory: ./apps/ai/server - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - name: Run migration script # liquibase cli and mongo driver already installed in runner - env: - LIQUIBASE_COMMAND_USERNAME: ${{ secrets.MONGODB_USERNAME }} - LIQUIBASE_COMMAND_PASSWORD: ${{ secrets.MONGODB_PASSWORD }} - LIQUIBASE_COMMAND_URL: ${{ inputs.target_db_url }} - run: | - liquibase --changeLogFile=database/migrations/changelog.json update - - stop-self-hosted-runner: - name: Stop self-hosted EC2 runner - needs: [start-self-hosted-runner, migrate-db] - uses: ./.github/workflows/ai-stop-self-hosted-runner.yml - if: needs.start-self-hosted-runner.result == 'success' - secrets: inherit - with: - label: ${{ needs.start-self-hosted-runner.outputs.label }} - ec2_instance_id: ${{ needs.start-self-hosted-runner.outputs.ec2_instance_id }} diff --git a/.github/workflows/ai-server-api-test.yml b/.github/workflows/ai-server-api-test.yml deleted file mode 100644 index 09091450..00000000 --- a/.github/workflows/ai-server-api-test.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: AI API tests using Postman CLI - -on: - pull_request: - paths: - - ".github/workflows/ai-server-api-test.yml" # this file - - "apps/ai/server/**" - - "!.github/workflows/**" - - "!apps/ai/server/database/migrations/**" - - "!apps/ai/server/database/scripts/**" - - "!apps/ai/server/scripts/**" - - "!apps/ai/server/terraform/**" - - "!**/.env.example" - - "!**/README.md" - branches: - - main - workflow_dispatch: - - -jobs: - automated-api-tests: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - environment: ai-api-test - defaults: - run: - working-directory: ./apps/ai/server - - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_ACCESS_TOKEN }} - submodules: true - - name: Create Docker network - run: docker network create backendnetwork - - name: Create env files - run: | - echo "${{ secrets.ENGINE_ENV_FILE }}" > ./dataherald/.env - echo "${{ secrets.ENTERPRISE_ENV_FILE }}" > .env - - name: Run Engine container - run: | - docker compose -f ./dataherald/docker-compose.yml up --build -d - echo "Engine container running at http://localhost:80" - - name: Run Enterprise container - run: | - docker compose -f ./docker-compose.yml up --build -d - echo "Enterprise container running at http://localhost:3001" - - name: Wait for containers to start - run: sleep 5 - - name: Log docker containers - run: | - docker ps - - name: Populate database - run: | - docker exec server-app-1 python3 -m database.scripts.initial_setup - - name: Install Postman CLI - run: | - curl -o- "https://dl-cli.pstmn.io/install/linux64.sh" | sh - - name: Login to Postman CLI - run: postman login --with-api-key ${{ secrets.POSTMAN_API_KEY }} - - name: Run API tests - run: | - postman collection run "${{ github.workspace }}/apps/ai/server/tests/postman/ai-api-test.json" -e "32202991-0e2096c0-ca88-4ab2-8d70-3167d752e996" --integration-id "156003-${{ github.run_id }}" - - name: Dump logs - if: failure() - uses: jwalton/gh-docker-logs@v2 diff --git a/.github/workflows/ai-server-checks.yml b/.github/workflows/ai-server-checks.yml deleted file mode 100644 index 86d59341..00000000 --- a/.github/workflows/ai-server-checks.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: AI Server - Quality Checks -on: - push: - tags-ignore: - - "*" - paths: - - apps/ai/server/** - - .github/workflows/ai-server-checks.yml # this file - - "!.github/workflows/**" - - "!apps/ai/server/database/migrations/**" - - "!apps/ai/server/database/scripts/**" - - "!apps/ai/server/scripts/**" - - "!apps/ai/server/terraform/**" - - "!apps/ai/**/.env.example" - - "!apps/ai/**/README.md" -jobs: - lint: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./apps/ai/server - steps: - - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 - with: - src: ./apps/ai/server - - uses: psf/black@stable - with: - src: ./apps/ai/server diff --git a/.github/workflows/ai-start-self-hosted-runner.yml b/.github/workflows/ai-start-self-hosted-runner.yml deleted file mode 100644 index 348a66a2..00000000 --- a/.github/workflows/ai-start-self-hosted-runner.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: AI Start Self-Hosted Runner - -on: - workflow_call: - outputs: - label: - description: "The self hosted github runner label" - value: ${{ jobs.start-self-hosted-runner.outputs.label }} - ec2_instance_id: - description: "The self hosted github runner ec2 instance id" - value: ${{ jobs.start-self-hosted-runner.outputs.ec2_instance_id }} - -jobs: - start-self-hosted-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2_instance_id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GH_ACCESS_TOKEN }} - ec2-image-id: ${{ vars.SELF_HOSTED_RUNNER_IMAGE_ID }} - ec2-instance-type: t2.micro - subnet-id: ${{ vars.SELF_HOSTED_RUNNER_SUBNET_ID }} # this is a private subnet, even though the mongo vpce is set for public subnet, it will work, probably because of the route table - security-group-id: ${{ vars.SELF_HOSTED_RUNNER_SECURITY_GROUP_ID }} - pre-runner-script: | - source /etc/profile.d/liquibase.sh diff --git a/.github/workflows/ai-stop-self-hosted-runner.yml b/.github/workflows/ai-stop-self-hosted-runner.yml deleted file mode 100644 index 0e193efc..00000000 --- a/.github/workflows/ai-stop-self-hosted-runner.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: AI Stop Self-Hosted Runner - -on: - workflow_call: - inputs: - label: - description: "The label of the self-hosted runner to start" - required: true - type: string - ec2_instance_id: - description: "The EC2 instance ID of the self-hosted runner to start" - required: true - type: string - -env: - AWS_REGION: us-east-1 - -jobs: - stop-self-hosted-runner: - name: Stop self-hosted EC2 runner - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GH_ACCESS_TOKEN }} - label: ${{ inputs.label }} - ec2-instance-id: ${{ inputs.ec2_instance_id }} diff --git a/.gitmodules b/.gitmodules deleted file mode 100755 index c4c16e02..00000000 --- a/.gitmodules +++ /dev/null @@ -1,9 +0,0 @@ -[submodule "apps/ai/server/dataherald"] - path = apps/ai/server/dataherald - url = https://github.com/Dataherald/dataherald -[submodule "apps/bariloche/programmatic-seo"] - path = apps/bariloche/programmatic-seo - url = https://github.com/Dataherald/programmatic-seo -[submodule "apps/bariloche/app-server"] - path = apps/bariloche/app-server - url = https://github.com/Dataherald/AppServer diff --git a/.postman/api b/.postman/api deleted file mode 100644 index 00349e54..00000000 --- a/.postman/api +++ /dev/null @@ -1,4 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY -apis[] = {"apiId":"d959e3e0-f3ef-4af5-995b-acf2de1a8dda"} -configVersion = 1.0.0 -type = api diff --git a/.postman/api_d959e3e0-f3ef-4af5-995b-acf2de1a8dda b/.postman/api_d959e3e0-f3ef-4af5-995b-acf2de1a8dda deleted file mode 100644 index 3c6da201..00000000 --- a/.postman/api_d959e3e0-f3ef-4af5-995b-acf2de1a8dda +++ /dev/null @@ -1,18 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY -configVersion = 1.1.0 -type = apiEntityData - -[config] -id = d959e3e0-f3ef-4af5-995b-acf2de1a8dda - -[config.relations] - -[config.relations.collections] -rootDirectory = apps/ai/server/tests/postman -files[] = {"id":"32202991-e8fa353e-a5eb-492b-bd44-9745c1becff9","path":"ai-api-test.json","metaData":{}} - -[config.relations.collections.metaData] - -[config.relations.apiDefinition] - -[config.relations.apiDefinition.metaData] diff --git a/apps/ai/clients/benchmark-tool/.gitignore b/apps/ai/clients/benchmark-tool/.gitignore deleted file mode 100644 index 0e97c632..00000000 --- a/apps/ai/clients/benchmark-tool/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -test_results/* -!test_results/.gitkeep \ No newline at end of file diff --git a/apps/ai/clients/benchmark-tool/README.md b/apps/ai/clients/benchmark-tool/README.md deleted file mode 100644 index 0a5c014d..00000000 --- a/apps/ai/clients/benchmark-tool/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Text-2-SQL Benchmark Tool - -The benchmark engine is a tool that helps measure the performance (accuracy, speed and cost) of the engine Dataherald engine. The tool comes pre-loaded with test-sets and users can start the engine, load examples and run benchmark tests by running a single script - - -## How to run the tool - -### Installation and execution -Users simply have to install the dependencies listed in requirements.txt and then run the script location in run-tests.sh - -The script will start the Dataherald engine located in apps/server/dataherald in a docker instance, load context into the store and then run the benchmark tests. - -All test sets and output results are stored are jsonlines format files. - -### Configuration Options -Users can see the list of configuration options by running the following command: - -python3 apps/ai/clients/benchmark-tool/main.py --help - -However the main configuration elements are --f FILE, --file The file containing the benchmark tests. (default: v2_real_estate.jsonl) --u UPLOAD, --upload Upload the results to the S3 bucket. (default: False) --o OUTPUT, --output The directory to save the benchmark results file (default: apps/ai/clients/benchmark-tool/test_results/) --p PERCENT, --percent The percentage of the test set to use as context. (default: 0.1) --s SIZE, --size What percent of the test suite to use in the test (default: 1) - -### Result output -By default the results are stored as jsonlines files in the apps/ai/clients/benchmark-tool/test_results/ folder. A copy is also saved to the S3 bucket at https://s3.console.aws.amazon.com/s3/buckets/k2-benchmark-results?region=us-east-1&tab=objects \ No newline at end of file diff --git a/apps/ai/clients/benchmark-tool/init-files/bls_table_schema_details.jsonl b/apps/ai/clients/benchmark-tool/init-files/bls_table_schema_details.jsonl deleted file mode 100644 index 729ba037..00000000 --- a/apps/ai/clients/benchmark-tool/init-files/bls_table_schema_details.jsonl +++ /dev/null @@ -1,5942 +0,0 @@ - { - "columns": [ - { - "name": "area_code", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "area_type_code", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "area_text", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "display_level", - "is_primary_key": false, - "data_type": "BIGINT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "selectable", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "sort_sequence", - "is_primary_key": false, - "data_type": "BIGINT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_code": "CN0100100000000", - "area_type_code": "F", - "area_text": "Autauga County, AL", - "display_level": "0", - "selectable": "T", - "sort_sequence": "31" - }, - { - "area_code": "CN0100300000000", - "area_type_code": "F", - "area_text": "Baldwin County, AL", - "display_level": "0", - "selectable": "T", - "sort_sequence": "32" - }, - { - "area_code": "CN0100500000000", - "area_type_code": "F", - "area_text": "Barbour County, AL", - "display_level": "0", - "selectable": "T", - "sort_sequence": "33" - } - ], - "table_name": "area_codes", - "table_schema": "\nCREATE TABLE area_codes (\n\tarea_code TEXT NOT NULL, \n\tarea_type_code TEXT, \n\tarea_text TEXT, \n\tdisplay_level BIGINT, \n\tselectable TEXT, \n\tsort_sequence BIGINT, \n\tCONSTRAINT area_codes_pkey PRIMARY KEY (area_code)\n)" - }, - { - "columns": [ - { - "name": "name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Kansas", "Texas", "Oregon", "Idaho", "Alaska", "Alabama", "Oklahoma", "Colorado", "North Carolina", "Mississippi", "Florida", "Virgin Islands", "Vermont", "Delaware", "Northern Mariana Islands", "Nevada", "Louisiana", "Guam", "New York", "West Virginia", "South Carolina", "New Jersey", "Arkansas", "New Mexico", "Hawaii", "Missouri", "Connecticut", "South Dakota", "Iowa", "Indiana", "Massachusetts", "Rhode Island", "Ohio", "American Samoa", "District Of Columbia", "Michigan", "Minnesota", "Pennsylvania", "Washington", "Wisconsin", "Montana", "Kentucky", "Arizona", "Illinois", "Virginia", "Maryland", "Puerto Rico", "Georgia", "Wyoming", "Utah", "New Hampshire", "North Dakota", "California", "Nebraska", "Maine", "Tennessee"], - "foreign_key": null - }, - { - "name": "abbr", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["CA", "NH", "OR", "TX", "ND", "PR", "NV", "OH", "KY", "GU", "NY", "HI", "NM", "MS", "IN", "DC", "WV", "NE", "MO", "FL", "AR", "ME", "CT", "WI", "NC", "SD", "RI", "OK", "ID", "GA", "MN", "PA", "AK", "MD", "WY", "LA", "MT", "IL", "TN", "WA", "NJ", "MI", "MA", "AL", "MP", "UT", "IA", "CO", "VT", "SC", "VI", "AS", "VA", "AZ", "DE", "KS"], - "foreign_key": null - }, - { - "name": "fips", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["21", "24", "56", "33", "17", "48", "78", "30", "35", "04", "66", "45", "41", "22", "51", "46", "53", "36", "27", "31", "11", "50", "09", "60", "16", "42", "26", "08", "72", "49", "69", "12", "06", "54", "10", "13", "01", "47", "28", "15", "55", "40", "39", "19", "23", "20", "25", "02", "44", "34", "18", "38", "32", "05", "29", "37"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "name": "Delaware", - "abbr": "DE", - "fips": "10" - }, - { - "name": "District Of Columbia", - "abbr": "DC", - "fips": "11" - }, - { - "name": "Florida", - "abbr": "FL", - "fips": "12" - } - ], - "table_name": "state_fips", - "table_schema": "\nCREATE TABLE state_fips (\n\tname TEXT, \n\tabbr TEXT, \n\tfips TEXT\n)" - }, - { - "columns": [ - { - "name": "area_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2019-01-01 00:00:00", "2019-02-01 00:00:00", "2019-03-01 00:00:00", "2019-04-01 00:00:00", "2019-05-01 00:00:00", "2019-06-01 00:00:00", "2019-07-01 00:00:00", "2019-08-01 00:00:00", "2019-09-01 00:00:00", "2019-10-01 00:00:00", "2019-11-01 00:00:00", "2019-12-01 00:00:00", "2020-01-01 00:00:00", "2020-02-01 00:00:00", "2020-03-01 00:00:00", "2020-04-01 00:00:00", "2020-05-01 00:00:00", "2020-06-01 00:00:00", "2020-07-01 00:00:00", "2020-08-01 00:00:00", "2020-09-01 00:00:00", "2020-10-01 00:00:00", "2020-11-01 00:00:00", "2020-12-01 00:00:00", "2021-01-01 00:00:00", "2021-02-01 00:00:00", "2021-03-01 00:00:00", "2021-04-01 00:00:00", "2021-05-01 00:00:00", "2021-06-01 00:00:00", "2021-07-01 00:00:00", "2021-08-01 00:00:00", "2021-09-01 00:00:00", "2021-10-01 00:00:00", "2021-11-01 00:00:00", "2021-12-01 00:00:00", "2022-01-01 00:00:00", "2022-02-01 00:00:00", "2022-03-01 00:00:00", "2022-04-01 00:00:00", "2022-05-01 00:00:00", "2022-06-01 00:00:00", "2022-07-01 00:00:00", "2022-08-01 00:00:00", "2022-09-01 00:00:00", "2022-10-01 00:00:00", "2022-11-01 00:00:00", "2022-12-01 00:00:00", "2023-01-01 00:00:00", "2023-02-01 00:00:00", "2023-03-01 00:00:00", "2023-04-01 00:00:00", "2023-05-01 00:00:00", "2023-06-01 00:00:00"], - "foreign_key": null - }, - { - "name": "state_fips", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["01", "02", "04", "05", "06", "08", "09", "10", "12", "13", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53", "54", "55", "56"], - "foreign_key": null - }, - { - "name": "county_fips", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "area", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "unemployment_rate", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "state", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", "Delaware", "Florida", "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada", "New Hampshire", "New Jersey", "New Mexico", "New York", "North Carolina", "North Dakota", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming"], - "foreign_key": null - }, - { - "name": "area_code", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["true", "None"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_name": "Pike County, Kentucky", - "date": "2023-05-01 00:00:00", - "state_fips": "21", - "county_fips": "195", - "area": "Pike County", - "unemployment_rate": "5.5", - "state": "Kentucky", - "area_code": "CN2119500000000", - "seriesID": "LAUCN211950000000003", - "latest": "true", - "hg_date_updated": "2023-07-21 14:14:57.645927" - }, - { - "area_name": "Fresno County, California", - "date": "2020-07-01 00:00:00", - "state_fips": "06", - "county_fips": "019", - "area": "Fresno County", - "unemployment_rate": "13.4", - "state": "California", - "area_code": "CN0601900000000", - "seriesID": "LAUCN060190000000003", - "latest": "None", - "hg_date_updated": "2022-04-17 00:06:17.845238" - }, - { - "area_name": "Collier County, Florida", - "date": "2021-02-01 00:00:00", - "state_fips": "12", - "county_fips": "021", - "area": "Collier County", - "unemployment_rate": "4.2", - "state": "Florida", - "area_code": "CN1202100000000", - "seriesID": "LAUCN120210000000003", - "latest": "None", - "hg_date_updated": "2022-04-17 00:06:17.845238" - } - ], - "table_name": "unemployment_data_new", - "table_schema": "\nCREATE TABLE unemployment_data_new (\n\tarea_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tstate_fips TEXT, \n\tcounty_fips TEXT, \n\tarea TEXT, \n\tunemployment_rate DOUBLE PRECISION, \n\tstate TEXT, \n\tarea_code TEXT, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT unemployment_data_new_pkey PRIMARY KEY (area_name, date)\n)" - }, - { - "columns": [ - { - "name": "area_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["West region", "Northeast region", "South region", "Midwest region", "Total US"], - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2021-11-01 00:00:00", "2020-02-01 00:00:00", "2021-06-01 00:00:00", "2020-05-01 00:00:00", "2019-04-01 00:00:00", "2023-02-01 00:00:00", "2023-04-01 00:00:00", "2023-05-01 00:00:00", "2020-07-01 00:00:00", "2019-03-01 00:00:00", "2019-11-01 00:00:00", "2020-10-01 00:00:00", "2020-11-01 00:00:00", "2022-03-01 00:00:00", "2022-06-01 00:00:00", "2019-06-01 00:00:00", "2019-02-01 00:00:00", "2020-08-01 00:00:00", "2022-05-01 00:00:00", "2022-01-01 00:00:00", "2023-03-01 00:00:00", "2021-12-01 00:00:00", "2022-07-01 00:00:00", "2020-09-01 00:00:00", "2021-09-01 00:00:00", "2021-02-01 00:00:00", "2019-01-01 00:00:00", "2022-12-01 00:00:00", "2022-11-01 00:00:00", "2022-08-01 00:00:00", "2019-10-01 00:00:00", "2022-02-01 00:00:00", "2021-01-01 00:00:00", "2020-03-01 00:00:00", "2021-07-01 00:00:00", "2019-05-01 00:00:00", "2020-04-01 00:00:00", "2022-04-01 00:00:00", "2019-08-01 00:00:00", "2021-08-01 00:00:00", "2021-10-01 00:00:00", "2020-06-01 00:00:00", "2023-01-01 00:00:00", "2019-12-01 00:00:00", "2021-05-01 00:00:00", "2019-07-01 00:00:00", "2020-01-01 00:00:00", "2020-12-01 00:00:00", "2021-04-01 00:00:00", "2022-10-01 00:00:00", "2021-03-01 00:00:00", "2022-09-01 00:00:00", "2019-09-01 00:00:00"], - "foreign_key": null - }, - { - "name": "job_separations", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["JTS000000000000000TSL", "JTS000000NE0000000TSL", "JTS000000SO0000000TSL", "JTS000000WE0000000TSL", "JTS000000MW0000000TSL"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["None", "true"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2023-06-04 04:03:51.236343", "2023-03-12 05:06:26.345940", "2023-04-09 04:01:12.597801", "2023-07-09 04:03:35.466960", "2022-03-13 00:04:47.141370", "2023-05-07 04:04:26.798996"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_name": "Total US", - "date": "2023-05-01 00:00:00", - "job_separations": "5871.0", - "seriesID": "JTS000000000000000TSL", - "latest": "true", - "hg_date_updated": "2023-07-09 04:03:35.466960" - }, - { - "area_name": "Total US", - "date": "2023-04-01 00:00:00", - "job_separations": "5660.0", - "seriesID": "JTS000000000000000TSL", - "latest": "None", - "hg_date_updated": "2023-07-09 04:03:35.466960" - }, - { - "area_name": "Total US", - "date": "2023-03-01 00:00:00", - "job_separations": "5994.0", - "seriesID": "JTS000000000000000TSL", - "latest": "None", - "hg_date_updated": "2023-06-04 04:03:51.236343" - } - ], - "table_name": "job_separations", - "table_schema": "\nCREATE TABLE job_separations (\n\tarea_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tjob_separations DOUBLE PRECISION, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT job_separations_pkey PRIMARY KEY (area_name, date)\n)" - }, - { - "columns": [ - { - "name": "area_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Baltimore-Columbia-Towson, Maryland", "New York-Newark-Jersey City, New York-New Jersey-Pennsylvania", "Miami-Fort Lauderdale-West Palm Beach, Florida", "Washington-Arlington-Alexandria, District of Columbia-Virginia-Maryland-West Virginia", "Riverside-San Bernardino-Ontario, California", "San Diego-Carlsbad, California", "Phoenix-Mesa-Scottsdale, Arizona", "Houston-The Woodlands-Sugar Land, Texas", "Denver-Aurora-Lakewood, Colorado", "Dallas-Fort Worth-Arlington, Texas", "St. Louis, Missouri-Illinois", "Atlanta-Sandy Springs-Roswell, Georgia", "Detroit-Warren-Dearborn, Michigan", "San Francisco-Oakland-Hayward, California", "Los Angeles-Long Beach-Anaheim, California", "Tampa-St. Petersburg-Clearwater, Florida", "Philadelphia-Camden-Wilmington, Pennsylvania-New Jersey-Delaware-Maryland", "Chicago-Naperville-Elgin, Illinois-Indiana-Wisconsin", "Minneapolis-St.Paul-Bloomington, Minnesota-Wisconsin", "Boston-Cambridge-Newton, Massachusetts-New Hampshire"], - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2019-01-01 00:00:00", "2019-02-01 00:00:00", "2019-03-01 00:00:00", "2019-04-01 00:00:00", "2019-05-01 00:00:00", "2019-06-01 00:00:00", "2019-07-01 00:00:00", "2019-08-01 00:00:00", "2019-09-01 00:00:00", "2019-10-01 00:00:00", "2019-11-01 00:00:00", "2019-12-01 00:00:00", "2020-01-01 00:00:00", "2020-02-01 00:00:00", "2020-03-01 00:00:00", "2020-04-01 00:00:00", "2020-05-01 00:00:00", "2020-06-01 00:00:00", "2020-07-01 00:00:00", "2020-08-01 00:00:00", "2020-09-01 00:00:00", "2020-10-01 00:00:00", "2020-11-01 00:00:00", "2020-12-01 00:00:00", "2021-01-01 00:00:00", "2021-02-01 00:00:00", "2021-03-01 00:00:00", "2021-04-01 00:00:00", "2021-05-01 00:00:00", "2021-06-01 00:00:00", "2021-07-01 00:00:00", "2021-08-01 00:00:00", "2021-09-01 00:00:00", "2021-10-01 00:00:00", "2021-11-01 00:00:00", "2021-12-01 00:00:00", "2022-01-01 00:00:00", "2022-02-01 00:00:00", "2022-03-01 00:00:00", "2022-04-01 00:00:00", "2022-05-01 00:00:00", "2022-06-01 00:00:00", "2022-07-01 00:00:00", "2022-08-01 00:00:00", "2022-09-01 00:00:00", "2022-10-01 00:00:00", "2022-11-01 00:00:00", "2022-12-01 00:00:00", "2023-01-01 00:00:00", "2023-02-01 00:00:00", "2023-03-01 00:00:00", "2023-04-01 00:00:00", "2023-05-01 00:00:00", "2023-06-01 00:00:00"], - "foreign_key": null - }, - { - "name": "area", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Denver-Aurora-Lakewood", "Dallas-Fort Worth-Arlington", "Washington-Arlington-Alexandria", "Philadelphia-Camden-Wilmington", "Boston-Cambridge-Newton", "Los Angeles-Long Beach-Anaheim", "San Francisco-Oakland-Hayward", "Detroit-Warren-Dearborn", "Phoenix-Mesa-Scottsdale", "San Diego-Carlsbad", "Atlanta-Sandy Springs-Roswell", "Riverside-San Bernardino-Ontario", "St. Louis", "Minneapolis-St.Paul-Bloomington", "Chicago-Naperville-Elgin", "New York-Newark-Jersey City", "Houston-The Woodlands-Sugar Land", "Tampa-St. Petersburg-Clearwater", "Baltimore-Columbia-Towson", "Miami-Fort Lauderdale-West Palm Beach"], - "foreign_key": null - }, - { - "name": "CPI", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "region", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Massachusetts-New Hampshire", "Michigan", "Pennsylvania-New Jersey-Delaware-Maryland", "Colorado", "New York-New Jersey-Pennsylvania", "Florida", "Arizona", "Minnesota-Wisconsin", "Maryland", "Missouri-Illinois", "Georgia", "District of Columbia-Virginia-Maryland-West Virginia", "California", "Texas", "Illinois-Indiana-Wisconsin"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["2019-02-01 00:00:00", "2021-04-01 00:00:00", "2020-04-01 00:00:00", "2022-06-01 00:00:00", "2019-07-01 00:00:00", "2020-05-01 00:00:00", "2022-03-01 00:00:00", "2021-05-01 00:00:00", "2021-08-01 00:00:00", "2021-10-01 00:00:00", "2023-01-01 00:00:00", "2019-05-01 00:00:00", "2022-01-01 00:00:00", "2019-12-01 00:00:00", "2021-03-01 00:00:00", "2020-08-01 00:00:00", "2022-04-01 00:00:00", "2021-06-01 00:00:00", "2023-02-01 00:00:00", "2020-02-01 00:00:00", "2019-08-01 00:00:00", "2021-12-01 00:00:00", "2020-01-01 00:00:00", "2020-07-01 00:00:00", "2023-05-01 00:00:00", "2022-08-01 00:00:00", "2020-10-01 00:00:00", "2022-02-01 00:00:00", "2021-02-01 00:00:00", "2020-12-01 00:00:00", "2022-11-01 00:00:00", "2021-09-01 00:00:00", "2019-10-01 00:00:00", "2020-06-01 00:00:00", "2019-03-01 00:00:00", "2019-09-01 00:00:00", "2022-10-01 00:00:00", "2019-01-01 00:00:00", "2023-06-01 00:00:00", "2019-04-01 00:00:00", "2022-05-01 00:00:00", "2022-12-01 00:00:00", "2019-06-01 00:00:00", "2022-07-01 00:00:00", "2019-11-01 00:00:00", "2023-04-01 00:00:00", "2021-07-01 00:00:00", "2021-01-01 00:00:00", "2023-03-01 00:00:00", "2022-09-01 00:00:00", "2021-11-01 00:00:00", "2020-09-01 00:00:00", "2020-11-01 00:00:00", "2020-03-01 00:00:00"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2023-02-14 13:40:13.763431", "2023-06-13 12:41:03.764041", "2023-05-10 12:40:06.266162", "2022-05-11 12:40:18.594518", "2023-07-12 12:41:09.379080", "2022-01-12 17:29:13.223128", "2022-11-27 00:04:58.388492", "2022-08-10 12:45:28.266951", "2022-06-10 12:41:56.657622", "2022-04-12 12:39:23.589685", "2022-10-16 00:07:27.539658", "2022-09-18 00:05:32.972378", "2022-03-10 13:40:47.541636", "2023-01-12 13:39:27.753504", "2023-03-14 12:40:09.335085", "2022-12-13 13:39:46.853394", "2022-02-10 14:41:43.940618", "2022-11-10 13:40:32.560406", "2023-04-12 18:28:30.041327", "2022-07-17 00:05:12.884409"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_name": "Detroit-Warren-Dearborn, Michigan", - "date": "2023-06-01 00:00:00", - "area": "Detroit-Warren-Dearborn", - "CPI": "285.638", - "region": "Michigan", - "latest": "2023-06-01 00:00:00", - "hg_date_updated": "2023-07-12 12:41:09.379080" - }, - { - "area_name": "Detroit-Warren-Dearborn, Michigan", - "date": "2023-04-01 00:00:00", - "area": "Detroit-Warren-Dearborn", - "CPI": "282.486", - "region": "Michigan", - "latest": "2023-04-01 00:00:00", - "hg_date_updated": "2023-05-10 12:40:06.266162" - }, - { - "area_name": "Minneapolis-St.Paul-Bloomington, Minnesota-Wisconsin", - "date": "2023-05-01 00:00:00", - "area": "Minneapolis-St.Paul-Bloomington", - "CPI": "291.001", - "region": "Minnesota-Wisconsin", - "latest": "2023-05-01 00:00:00", - "hg_date_updated": "2023-06-13 12:41:03.764041" - } - ], - "table_name": "consumer_price_index", - "table_schema": "\nCREATE TABLE consumer_price_index (\n\tarea_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tarea TEXT, \n\t\"CPI\" DOUBLE PRECISION, \n\tregion TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT consumer_price_index_pkey PRIMARY KEY (area_name, date)\n)" - }, - { - "columns": [ - { - "name": "area_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2021-11-01 00:00:00", "2020-02-01 00:00:00", "2021-06-01 00:00:00", "2020-05-01 00:00:00", "2019-04-01 00:00:00", "2023-04-01 00:00:00", "2023-02-01 00:00:00", "2023-05-01 00:00:00", "2020-07-01 00:00:00", "2020-11-01 00:00:00", "2022-03-01 00:00:00", "2020-10-01 00:00:00", "2019-03-01 00:00:00", "2019-11-01 00:00:00", "2022-06-01 00:00:00", "2019-06-01 00:00:00", "2019-02-01 00:00:00", "2020-08-01 00:00:00", "2022-05-01 00:00:00", "2022-01-01 00:00:00", "2023-03-01 00:00:00", "2021-12-01 00:00:00", "2022-07-01 00:00:00", "2020-09-01 00:00:00", "2021-09-01 00:00:00", "2021-02-01 00:00:00", "2022-12-01 00:00:00", "2019-01-01 00:00:00", "2022-11-01 00:00:00", "2022-08-01 00:00:00", "2023-06-01 00:00:00", "2019-10-01 00:00:00", "2022-02-01 00:00:00", "2021-01-01 00:00:00", "2020-03-01 00:00:00", "2021-07-01 00:00:00", "2019-05-01 00:00:00", "2022-04-01 00:00:00", "2020-04-01 00:00:00", "2021-08-01 00:00:00", "2021-10-01 00:00:00", "2019-08-01 00:00:00", "2020-06-01 00:00:00", "2023-01-01 00:00:00", "2019-12-01 00:00:00", "2021-05-01 00:00:00", "2020-12-01 00:00:00", "2019-07-01 00:00:00", "2020-01-01 00:00:00", "2021-04-01 00:00:00", "2022-10-01 00:00:00", "2022-09-01 00:00:00", "2021-03-01 00:00:00", "2019-09-01 00:00:00"], - "foreign_key": null - }, - { - "name": "metro", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "weekly_earnings", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "state", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Oklahoma", "Colorado", "North Carolina", "Mississippi", "Florida", "Vermont", "Delaware", "Nevada", "Louisiana", "New York", "West Virginia", "South Carolina", "New Jersey", "Arkansas", "Hawaii", "New Mexico", "Missouri", "Connecticut", "South Dakota", "District of Columbia", "Iowa", "Indiana", "Massachusetts", "Rhode Island", "Ohio", "Michigan", "Minnesota", "Pennsylvania", "Washington", "Wisconsin", "Kentucky", "Montana", "Arizona", "Illinois", "Virginia", "Maryland", "Georgia", "Puerto Rico", "Wyoming", "Utah", "New Hampshire", "North Dakota", "Maine", "Nebraska", "California", "Tennessee", "Kansas", "Oregon", "Texas", "Idaho", "Alaska", "Alabama"], - "foreign_key": null - }, - { - "name": "area_code", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["None", "true", "-"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2021-11-21 00:10:43.766362", "2022-05-22 00:03:59.156565", "2023-03-26 04:05:53.056189", "2021-10-24 00:09:13.360405", "2021-12-19 00:10:29.652904", "2022-01-25 15:27:22.793243", "2023-07-25 21:28:00.334339", "2021-11-21 00:10:46.282965", "2022-07-24 00:10:29.426487", "2022-04-17 00:02:04.829627", "2021-12-19 00:10:20.202051", "2023-03-19 04:05:02.469467", "2023-01-31 16:37:36.831276", "2022-03-27 00:04:42.394739", "2021-10-24 00:09:22.053660", "2021-10-24 00:09:25.089483", "2022-06-19 00:07:42.133410", "2022-08-21 00:07:32.052064", "2022-03-20 00:04:26.929700"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_name": "Brownsville-Harlingen, Texas", - "date": "2020-02-01 00:00:00", - "metro": "Brownsville-Harlingen", - "weekly_earnings": "540.8", - "state": "Texas", - "area_code": "481518005000000", - "seriesID": "SMU48151800500000011", - "latest": "None", - "hg_date_updated": "2021-10-24 00:09:13.360405" - }, - { - "area_name": "Brownsville-Harlingen, Texas", - "date": "2020-01-01 00:00:00", - "metro": "Brownsville-Harlingen", - "weekly_earnings": "528.26", - "state": "Texas", - "area_code": "481518005000000", - "seriesID": "SMU48151800500000011", - "latest": "None", - "hg_date_updated": "2021-10-24 00:09:13.360405" - }, - { - "area_name": "Brownsville-Harlingen, Texas", - "date": "2019-12-01 00:00:00", - "metro": "Brownsville-Harlingen", - "weekly_earnings": "538.9", - "state": "Texas", - "area_code": "481518005000000", - "seriesID": "SMU48151800500000011", - "latest": "None", - "hg_date_updated": "2021-10-24 00:09:13.360405" - } - ], - "table_name": "weekly_earnings", - "table_schema": "\nCREATE TABLE weekly_earnings (\n\tarea_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tmetro TEXT, \n\tweekly_earnings DOUBLE PRECISION, \n\tstate TEXT, \n\tarea_code TEXT, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT weekly_earnings_pkey PRIMARY KEY (area_name, date)\n)" - }, - { - "columns": [ - { - "name": "education_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["High School graduates, no college", "Some college or associate degree", "Advanced degree", "Less than a High School diploma", "Bachelor's degree only"], - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2019-10-01 00:00:00", "2017-01-01 00:00:00", "2017-10-01 00:00:00", "2018-01-01 00:00:00", "2019-04-01 00:00:00", "2021-01-01 00:00:00", "2020-07-01 00:00:00", "2021-07-01 00:00:00", "2020-10-01 00:00:00", "2020-04-01 00:00:00", "2022-04-01 00:00:00", "2021-10-01 00:00:00", "2017-07-01 00:00:00", "2018-07-01 00:00:00", "2022-01-01 00:00:00", "2020-01-01 00:00:00", "2019-07-01 00:00:00", "2021-04-01 00:00:00", "2018-04-01 00:00:00", "2018-10-01 00:00:00", "2017-04-01 00:00:00", "2019-01-01 00:00:00"], - "foreign_key": null - }, - { - "name": "Quarter", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Q3", "Q4", "Q2", "Q1"], - "foreign_key": null - }, - { - "name": "weekly_earnings", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["LEU0252917300", "LEU0252919700", "LEU0252919100", "LEU0254929400", "LEU0252916700"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["2018-01-01 00:00:00", "2020-01-01 00:00:00", "2021-04-01 00:00:00", "2020-04-01 00:00:00", "2020-07-01 00:00:00", "2020-10-01 00:00:00", "2018-04-01 00:00:00", "2019-10-01 00:00:00", "2019-07-01 00:00:00", "2017-01-01 00:00:00", "2018-10-01 00:00:00", "2019-01-01 00:00:00", "2019-04-01 00:00:00", "2021-10-01 00:00:00", "2022-01-01 00:00:00", "2017-10-01 00:00:00", "2018-07-01 00:00:00", "2021-07-01 00:00:00", "2021-01-01 00:00:00", "2017-04-01 00:00:00", "2017-07-01 00:00:00", "2022-04-01 00:00:00"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2022-01-25 15:36:50.255658", "2022-07-24 00:08:11.943907", "2022-04-17 00:02:40.603389"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "education_name": "Less than a High School diploma", - "date": "2022-04-01 00:00:00", - "Quarter": "Q2", - "weekly_earnings": "689.0", - "seriesID": "LEU0252916700", - "latest": "2022-04-01 00:00:00", - "hg_date_updated": "2022-07-24 00:08:11.943907" - }, - { - "education_name": "Less than a High School diploma", - "date": "2022-01-01 00:00:00", - "Quarter": "Q1", - "weekly_earnings": "670.0", - "seriesID": "LEU0252916700", - "latest": "2022-01-01 00:00:00", - "hg_date_updated": "2022-04-17 00:02:40.603389" - }, - { - "education_name": "Less than a High School diploma", - "date": "2021-10-01 00:00:00", - "Quarter": "Q4", - "weekly_earnings": "651.0", - "seriesID": "LEU0252916700", - "latest": "2021-10-01 00:00:00", - "hg_date_updated": "2022-01-25 15:36:50.255658" - } - ], - "table_name": "earnings_by_education", - "table_schema": "\nCREATE TABLE earnings_by_education (\n\teducation_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\t\"Quarter\" TEXT, \n\tweekly_earnings DOUBLE PRECISION, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT earnings_by_education_pkey PRIMARY KEY (education_name, date)\n)" - }, - { - "columns": [ - { - "name": "area_code", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "year", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["2019", "2021", "2007", "2009", "2008", "2020"], - "foreign_key": null - }, - { - "name": "period", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["M10", "M04", "M11", "M06", "M02", "M03", "M08", "M09", "M05", "M07", "M01", "M12"], - "foreign_key": null - }, - { - "name": "periodName", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["June", "November", "March", "August", "December", "July", "January", "April", "February", "May", "September", "October"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["true", "None"], - "foreign_key": null - }, - { - "name": "value", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "footnotes", - "is_primary_key": false, - "data_type": "JSON", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_code": "CN2019300000000", - "date": "2021-01-01 00:00:00", - "year": "2021", - "period": "M01", - "periodName": "January", - "latest": "None", - "value": "3.9", - "footnotes": "[{}]", - "seriesID": "LAUCN201930000000003" - }, - { - "area_code": "CN1716500000000", - "date": "2021-04-01 00:00:00", - "year": "2021", - "period": "M04", - "periodName": "April", - "latest": "None", - "value": "6.8", - "footnotes": "[{}]", - "seriesID": "LAUCN171650000000003" - }, - { - "area_code": "CN2019300000000", - "date": "2020-12-01 00:00:00", - "year": "2020", - "period": "M12", - "periodName": "December", - "latest": "None", - "value": "3.3", - "footnotes": "[{}]", - "seriesID": "LAUCN201930000000003" - } - ], - "table_name": "unemployment_data", - "table_schema": "\nCREATE TABLE unemployment_data (\n\tarea_code TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tyear TEXT, \n\tperiod TEXT, \n\t\"periodName\" TEXT, \n\tlatest TEXT, \n\tvalue TEXT, \n\tfootnotes JSON, \n\t\"seriesID\" TEXT, \n\tCONSTRAINT unemployment_data_pkey PRIMARY KEY (area_code, date)\n)" - }, - { - "columns": [ - { - "name": "area_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["West region", "Northeast region", "South region", "Midwest region", "Total US"], - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2021-11-01 00:00:00", "2020-02-01 00:00:00", "2021-06-01 00:00:00", "2020-05-01 00:00:00", "2019-04-01 00:00:00", "2023-02-01 00:00:00", "2023-04-01 00:00:00", "2023-05-01 00:00:00", "2020-07-01 00:00:00", "2019-03-01 00:00:00", "2019-11-01 00:00:00", "2020-10-01 00:00:00", "2020-11-01 00:00:00", "2022-03-01 00:00:00", "2022-06-01 00:00:00", "2019-06-01 00:00:00", "2019-02-01 00:00:00", "2020-08-01 00:00:00", "2022-05-01 00:00:00", "2022-01-01 00:00:00", "2023-03-01 00:00:00", "2021-12-01 00:00:00", "2022-07-01 00:00:00", "2020-09-01 00:00:00", "2021-09-01 00:00:00", "2021-02-01 00:00:00", "2019-01-01 00:00:00", "2022-12-01 00:00:00", "2022-11-01 00:00:00", "2022-08-01 00:00:00", "2019-10-01 00:00:00", "2022-02-01 00:00:00", "2021-01-01 00:00:00", "2020-03-01 00:00:00", "2021-07-01 00:00:00", "2019-05-01 00:00:00", "2020-04-01 00:00:00", "2022-04-01 00:00:00", "2019-08-01 00:00:00", "2021-08-01 00:00:00", "2021-10-01 00:00:00", "2020-06-01 00:00:00", "2023-01-01 00:00:00", "2019-12-01 00:00:00", "2021-05-01 00:00:00", "2019-07-01 00:00:00", "2020-01-01 00:00:00", "2020-12-01 00:00:00", "2021-04-01 00:00:00", "2022-10-01 00:00:00", "2021-03-01 00:00:00", "2022-09-01 00:00:00", "2019-09-01 00:00:00"], - "foreign_key": null - }, - { - "name": "job_openings", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["JTS000000WE0000000JOL", "JTS000000NE0000000JOL", "JTS000000SO0000000JOL", "JTS000000000000000JOL", "JTS000000MW0000000JOL"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["None", "true"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2022-03-13 00:04:16.254330", "2023-07-09 04:04:23.181674", "2023-04-09 04:01:11.091110", "2023-03-12 05:06:05.864172", "2023-05-07 04:04:08.572143", "2023-06-04 04:04:34.515184"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_name": "Total US", - "date": "2023-05-01 00:00:00", - "job_openings": "9824.0", - "seriesID": "JTS000000000000000JOL", - "latest": "true", - "hg_date_updated": "2023-07-09 04:04:23.181674" - }, - { - "area_name": "Total US", - "date": "2023-04-01 00:00:00", - "job_openings": "10320.0", - "seriesID": "JTS000000000000000JOL", - "latest": "None", - "hg_date_updated": "2023-07-09 04:04:23.181674" - }, - { - "area_name": "Total US", - "date": "2023-03-01 00:00:00", - "job_openings": "9745.0", - "seriesID": "JTS000000000000000JOL", - "latest": "None", - "hg_date_updated": "2023-06-04 04:04:34.515184" - } - ], - "table_name": "job_openings", - "table_schema": "\nCREATE TABLE job_openings (\n\tarea_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tjob_openings DOUBLE PRECISION, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT job_openings_pkey PRIMARY KEY (area_name, date)\n)" - }, - { - "columns": [ - { - "name": "industry_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Wholesale trade", "Service-providing", "Trade, transportation, and utilities", "Manufacturing", "Construction", "Goods-producing", "Financial activities", "Private Sector", "Utilities", "Nondurable Goods", "Other services", "Education and health services", "Information", "Leisure and hospitality", "Retail trade", "Professional and business services", "Durable Goods", "Mining and Logging", "Transportation and warehousing"], - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2018-06-01 00:00:00", "2020-07-01 00:00:00", "2019-11-01 00:00:00", "2020-10-01 00:00:00", "2020-11-01 00:00:00", "2022-06-01 00:00:00", "2018-07-01 00:00:00", "2018-12-01 00:00:00", "2020-09-01 00:00:00", "2018-09-01 00:00:00", "2021-02-01 00:00:00", "2022-08-01 00:00:00", "2019-10-01 00:00:00", "2022-02-01 00:00:00", "2021-07-01 00:00:00", "2019-05-01 00:00:00", "2020-04-01 00:00:00", "2022-04-01 00:00:00", "2021-05-01 00:00:00", "2020-12-01 00:00:00", "2021-04-01 00:00:00", "2022-10-01 00:00:00", "2021-03-01 00:00:00", "2021-06-01 00:00:00", "2018-01-01 00:00:00", "2019-04-01 00:00:00", "2018-02-01 00:00:00", "2019-03-01 00:00:00", "2022-03-01 00:00:00", "2019-06-01 00:00:00", "2019-02-01 00:00:00", "2020-08-01 00:00:00", "2022-05-01 00:00:00", "2022-01-01 00:00:00", "2021-12-01 00:00:00", "2022-07-01 00:00:00", "2018-04-01 00:00:00", "2021-09-01 00:00:00", "2019-01-01 00:00:00", "2021-01-01 00:00:00", "2020-03-01 00:00:00", "2019-08-01 00:00:00", "2021-08-01 00:00:00", "2021-10-01 00:00:00", "2020-06-01 00:00:00", "2019-12-01 00:00:00", "2019-07-01 00:00:00", "2020-01-01 00:00:00", "2018-11-01 00:00:00", "2022-09-01 00:00:00", "2018-10-01 00:00:00", "2019-09-01 00:00:00", "2018-03-01 00:00:00", "2021-11-01 00:00:00", "2020-02-01 00:00:00", "2018-08-01 00:00:00", "2018-05-01 00:00:00", "2020-05-01 00:00:00"], - "foreign_key": null - }, - { - "name": "weekly_earnings", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["CES4300000011", "CES4000000011", "CES1000000011", "CES4200000011", "CES3200000011", "CES7000000011", "CES4142000011", "CES8000000011", "CES5500000011", "CES0600000011", "CES6000000011", "CES4422000011", "CES6500000011", "CES5000000011", "CES0500000011", "CES2000000011", "CES3100000011", "CES0800000011", "CES3000000011"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["None", "true"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2022-02-06 00:05:18.091592", "2022-01-09 00:07:28.246773", "2022-08-07 00:04:16.417304", "2022-05-08 00:04:52.686264", "2022-04-03 00:01:59.893051", "2022-10-09 00:07:23.167295", "2022-03-06 00:04:24.014876", "2022-07-10 00:04:28.195288", "2021-12-02 20:43:05.244195", "2022-09-04 00:07:43.414082", "2022-06-05 00:04:01.971905", "2022-11-06 00:06:16.250430"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "industry_name": "Private Sector", - "date": "2022-10-01 00:00:00", - "weekly_earnings": "1124.01", - "seriesID": "CES0500000011", - "latest": "true", - "hg_date_updated": "2022-11-06 00:06:16.250430" - }, - { - "industry_name": "Private Sector", - "date": "2022-09-01 00:00:00", - "weekly_earnings": "1119.87", - "seriesID": "CES0500000011", - "latest": "None", - "hg_date_updated": "2022-11-06 00:06:16.250430" - }, - { - "industry_name": "Private Sector", - "date": "2022-08-01 00:00:00", - "weekly_earnings": "1116.42", - "seriesID": "CES0500000011", - "latest": "None", - "hg_date_updated": "2022-10-09 00:07:23.167295" - } - ], - "table_name": "earnings_by_industry", - "table_schema": "\nCREATE TABLE earnings_by_industry (\n\tindustry_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tweekly_earnings DOUBLE PRECISION, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT earnings_by_industry_pkey PRIMARY KEY (industry_name, date)\n)" - }, - { - "columns": [ - { - "name": "ind_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Wholesale trade", "Real estate and rental and leasing", "Manufacturing", "Construction", "Financial activities", "Mining and logging", "Accommodation and food services", "Finance and insurance", "Arts", "Educational services", "Government", "State and local", "Trade", "Other services", "Total private", "Transportation", "State and local government", "Federal", "Education and health services", "Information", "State and local government education", "Leisure and hospitality", "Retail trade", "Professional and business services", "Nondurable goods manufacturing", "Health care and social assistance", "Durable goods manufacturing"], - "foreign_key": null - }, - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2021-11-01 00:00:00", "2020-02-01 00:00:00", "2021-06-01 00:00:00", "2020-05-01 00:00:00", "2019-04-01 00:00:00", "2023-04-01 00:00:00", "2023-02-01 00:00:00", "2023-05-01 00:00:00", "2020-07-01 00:00:00", "2022-03-01 00:00:00", "2020-10-01 00:00:00", "2019-11-01 00:00:00", "2020-11-01 00:00:00", "2019-03-01 00:00:00", "2022-06-01 00:00:00", "2019-06-01 00:00:00", "2019-02-01 00:00:00", "2020-08-01 00:00:00", "2022-05-01 00:00:00", "2022-01-01 00:00:00", "2023-03-01 00:00:00", "2021-12-01 00:00:00", "2022-07-01 00:00:00", "2020-09-01 00:00:00", "2021-09-01 00:00:00", "2021-02-01 00:00:00", "2022-12-01 00:00:00", "2019-01-01 00:00:00", "2022-11-01 00:00:00", "2022-08-01 00:00:00", "2019-10-01 00:00:00", "2023-06-01 00:00:00", "2022-02-01 00:00:00", "2021-01-01 00:00:00", "2021-07-01 00:00:00", "2020-03-01 00:00:00", "2019-05-01 00:00:00", "2020-04-01 00:00:00", "2022-04-01 00:00:00", "2021-08-01 00:00:00", "2019-08-01 00:00:00", "2021-10-01 00:00:00", "2020-06-01 00:00:00", "2023-01-01 00:00:00", "2019-12-01 00:00:00", "2021-05-01 00:00:00", "2019-07-01 00:00:00", "2020-12-01 00:00:00", "2020-01-01 00:00:00", "2021-04-01 00:00:00", "2022-10-01 00:00:00", "2022-09-01 00:00:00", "2021-03-01 00:00:00", "2019-09-01 00:00:00"], - "foreign_key": null - }, - { - "name": "job_openings", - "is_primary_key": false, - "data_type": "BIGINT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesID", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["JTS510000000000000JOL", "JTS810000000000000JOL", "JTS320000000000000JOL", "JTS610000000000000JOL", "JTS900000000000000JOL", "JTS710000000000000JOL", "JTS440000000000000JOL", "JTS910000000000000JOL", "JTS340000000000000JOL", "JTS110099000000000JOL", "JTS600000000000000JOL", "JTS420000000000000JOL", "JTS620000000000000JOL", "JTS400000000000000JOL", "JTS923000000000000JOL", "JTS920000000000000JOL", "JTS520000000000000JOL", "JTS540099000000000JOL", "JTS100000000000000JOL", "JTS929000000000000JOL", "JTS510099000000000JOL", "JTS530000000000000JOL", "JTS700000000000000JOL", "JTS230000000000000JOL", "JTS300000000000000JOL", "JTS720000000000000JOL", "JTS480099000000000JOL"], - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["true", "None"], - "foreign_key": null - }, - { - "name": "job_separations", - "is_primary_key": false, - "data_type": "BIGINT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2023-08-01 14:11:32.958670", "2022-07-28 22:07:46.655808", "2023-06-21 17:18:48.002456", "2023-07-11 14:11:27.567226", "2023-03-08 19:27:50.492076"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "ind_name": "Government", - "date": "2020-05-01 00:00:00", - "job_openings": "639", - "seriesID": "JTS900000000000000JOL", - "latest": "None", - "job_separations": "639", - "hg_date_updated": "2023-06-21 17:18:48.002456" - }, - { - "ind_name": "Retail trade", - "date": "2021-10-01 00:00:00", - "job_openings": "997", - "seriesID": "JTS440000000000000JOL", - "latest": "None", - "job_separations": "997", - "hg_date_updated": "2023-06-21 17:18:48.002456" - }, - { - "ind_name": "Trade", - "date": "2019-05-01 00:00:00", - "job_openings": "1323", - "seriesID": "JTS400000000000000JOL", - "latest": "None", - "job_separations": "1323", - "hg_date_updated": "2023-06-21 17:18:48.002456" - } - ], - "table_name": "jobs_by_industry", - "table_schema": "\nCREATE TABLE jobs_by_industry (\n\tind_name TEXT NOT NULL, \n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tjob_openings BIGINT, \n\t\"seriesID\" TEXT, \n\tlatest TEXT, \n\tjob_separations BIGINT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT jobs_by_industry_pkey PRIMARY KEY (ind_name, date)\n)" - }, - { - "columns": [ - { - "name": "date", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "value", - "is_primary_key": false, - "data_type": "BIGINT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2023-06-21 18:21:34.074544", "2023-01-06 23:09:16.609761", "2023-07-07 14:06:15.643738", "2022-01-17 20:44:06.097535"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "date": "1998-05-01 00:00:00", - "value": "4691000", - "hg_date_updated": "2023-06-21 18:21:34.074544" - }, - { - "date": "1998-06-01 00:00:00", - "value": "4792000", - "hg_date_updated": "2023-06-21 18:21:34.074544" - }, - { - "date": "1998-07-01 00:00:00", - "value": "4852000", - "hg_date_updated": "2023-06-21 18:21:34.074544" - } - ], - "table_name": "labor_force_wantwork", - "table_schema": "\nCREATE TABLE labor_force_wantwork (\n\tdate TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n\tvalue BIGINT, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT labor_force_wantwork_pkey PRIMARY KEY (date)\n)" - }, - { - "columns": [ - { - "name": "date", - "is_primary_key": false, - "data_type": "DATE", - "description": null, - "low_cardinality": true, - "categories": ["2022-04-01", "2022-03-01", "2021-08-01", "2021-05-01", "2020-05-01", "2021-06-01", "2020-08-01", "2020-06-01", "2021-04-01", "2020-11-01", "2021-03-01", "2022-02-01", "2021-02-01", "2020-07-01", "2021-09-01", "2021-10-01", "2022-01-01", "2021-12-01", "2020-10-01", "2021-01-01", "2020-09-01", "2021-11-01", "2021-07-01", "2020-12-01"], - "foreign_key": null - }, - { - "name": "characteristic", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": true, - "categories": ["Transportation And Utilities", "Financial Activities", "Public Administration", "Nonagricultural Industries", "Other Services", "Wholesale And Retail Trade", "Educational Services", "Agriculture And Related Industries", "Health Care And Social Assistance", "Manufacturing", "Leisure And Hospitality", "Construction", "Information"], - "foreign_key": null - }, - { - "name": "total_employed", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "total_telework", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "percent_telework", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "total_employed_distr", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "total_telework_distr", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2022-03-09 17:27:33.606810", "2022-03-04 14:05:36.561892", "2022-02-17 20:14:51.230614", "2022-03-09 17:28:15.102538", "2022-04-01 13:06:02.479362", "2022-05-08 13:05:13.323808"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "date": "2022-02-01", - "characteristic": "Leisure And Hospitality", - "total_employed": "13023.0", - "total_telework": "551.0", - "percent_telework": "4.2", - "total_employed_distr": "8.3", - "total_telework_distr": "2.7", - "hg_date_updated": "2022-03-04 14:05:36.561892" - }, - { - "date": "2022-02-01", - "characteristic": "Other Services", - "total_employed": "6900.0", - "total_telework": "496.0", - "percent_telework": "7.2", - "total_employed_distr": "4.4", - "total_telework_distr": "2.4", - "hg_date_updated": "2022-03-04 14:05:36.561892" - }, - { - "date": "2022-02-01", - "characteristic": "Transportation And Utilities", - "total_employed": "9737.0", - "total_telework": "640.0", - "percent_telework": "6.6", - "total_employed_distr": "6.2", - "total_telework_distr": "3.1", - "hg_date_updated": "2022-03-04 14:05:36.561892" - } - ], - "table_name": "telework_by_industry", - "table_schema": "\nCREATE TABLE telework_by_industry (\n\tdate DATE NOT NULL, \n\tcharacteristic TEXT NOT NULL, \n\ttotal_employed DOUBLE PRECISION NOT NULL, \n\ttotal_telework DOUBLE PRECISION NOT NULL, \n\tpercent_telework DOUBLE PRECISION NOT NULL, \n\ttotal_employed_distr DOUBLE PRECISION NOT NULL, \n\ttotal_telework_distr DOUBLE PRECISION NOT NULL, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT telework_by_industry_pkey PRIMARY KEY (date, characteristic)\n)" - }, - { - "columns": [ - { - "name": "date", - "is_primary_key": false, - "data_type": "DATE", - "description": null, - "low_cardinality": true, - "categories": ["2022-04-01", "2022-03-01", "2021-08-01", "2020-05-01", "2021-05-01", "2020-08-01", "2020-06-01", "2021-06-01", "2020-11-01", "2021-04-01", "2021-03-01", "2022-02-01", "2020-07-01", "2021-02-01", "2021-09-01", "2021-10-01", "2022-01-01", "2021-12-01", "2020-10-01", "2020-09-01", "2021-01-01", "2021-11-01", "2021-07-01", "2020-12-01"], - "foreign_key": null - }, - { - "name": "characteristic", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "total_employed", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "total_telework", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "percent_telework", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "total_employed_distr", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": true, - "categories": ["11.9", "19.8", "45.3", "15.0", "16.7", "12.4", "15.9", "9.0", "42.1", "43.1", "16.3", "42.2", "42.6", "20.3", "9.5", "8.9", "13.9", "9.2", "12.5", "42.5", "14.8", "12.8", "16.0", "44.3", "42.9", "12.7", "19.2", "43.3", "20.1", "12.3", "20.5", "12.0", "16.1", "19.3", "15.2", "9.1", "20.4", "42.3", "20.0", "42.4", "41.8", "8.8", "13.0", "15.4", "12.1", "16.4", "20.7", "19.9", "15.8", "15.6", "42.8", "43.2", "19.6", "19.4", "12.2", "15.1", "9.3", "16.2", "12.6", "12.9"], - "foreign_key": null - }, - { - "name": "total_telework_distr", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": true, - "categories": ["1.8", "17.4", "0.9", "75.1", "1.6", "76.1", "17.0", "75.5", "19.7", "78.7", "1.1", "20.3", "3.0", "16.8", "76.6", "1.9", "19.1", "1.4", "1.7", "18.6", "1.5", "16.5", "3.2", "18.9", "1.3", "3.1", "1.2", "2.1", "73.1", "19.6", "2.8", "2.7", "17.6", "2.9", "76.9", "73.5", "76.7", "18.3", "75.9", "77.1", "18.1", "76.2", "18.2", "2.2", "73.9", "1.0", "2.6", "75.4", "17.9", "18.5", "77.9", "17.7", "2.0", "19.5"], - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": true, - "categories": ["2022-03-09 17:28:16.631497", "2022-03-04 14:05:38.222363", "2022-04-01 13:06:04.477752", "2022-05-08 13:05:15.747377", "2022-02-17 20:16:34.119777", "2022-03-09 17:27:34.874321"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "date": "2022-04-01", - "characteristic": "Natural Resources, Construction, And Maintenance Occupations", - "total_employed": "14230.0", - "total_telework": "156.0", - "percent_telework": "1.1", - "total_employed_distr": "9.0", - "total_telework_distr": "1.3", - "hg_date_updated": "2022-05-08 13:05:15.747377" - }, - { - "date": "2022-04-01", - "characteristic": "Production, Transportation, And Material Moving Occupations", - "total_employed": "20339.0", - "total_telework": "163.0", - "percent_telework": "0.8", - "total_employed_distr": "12.9", - "total_telework_distr": "1.3", - "hg_date_updated": "2022-05-08 13:05:15.747377" - }, - { - "date": "2022-03-01", - "characteristic": "Production, Transportation, And Material Moving Occupations", - "total_employed": "19824.0", - "total_telework": "263.0", - "percent_telework": "1.3", - "total_employed_distr": "12.5", - "total_telework_distr": "1.7", - "hg_date_updated": "2022-04-01 13:06:04.477752" - } - ], - "table_name": "telework_by_occupation", - "table_schema": "\nCREATE TABLE telework_by_occupation (\n\tdate DATE NOT NULL, \n\tcharacteristic TEXT NOT NULL, \n\ttotal_employed DOUBLE PRECISION NOT NULL, \n\ttotal_telework DOUBLE PRECISION NOT NULL, \n\tpercent_telework DOUBLE PRECISION NOT NULL, \n\ttotal_employed_distr DOUBLE PRECISION NOT NULL, \n\ttotal_telework_distr DOUBLE PRECISION NOT NULL, \n\thg_date_updated TIMESTAMP WITHOUT TIME ZONE DEFAULT timezone('utc'::text, now()), \n\tCONSTRAINT telework_by_occupation_pkey PRIMARY KEY (date, characteristic)\n)" - }, - { - "columns": [ - { - "name": "group_code", - "is_primary_key": false, - "data_type": "VARCHAR(3)", - "description": null, - "low_cardinality": true, - "categories": ["00", "56", "33", "FD", "48", "30", "35", "04", "45", "61", "ID5", "41", "51", "46", "53", "36", "31", "11", "50", "60", "43", "09", "42", "IP", "08", "80", "49", "12", "54", "10", "06", "IND", "13", "03", "47", "15", "01", "SI", "ILF", "55", "40", "39", "14", "44", "02", "34", "59", "07", "ID6", "38", "32", "05", "58", "37", "57", "PFF"], - "foreign_key": null - }, - { - "name": "group_name", - "is_primary_key": false, - "data_type": "VARCHAR(82)", - "description": null, - "low_cardinality": true, - "categories": ["Entertainment services (partial)", "Services related to transportation activities", "Hides, skins, leather, and related products", "Intermediate demand by commodity type", "Metal treatment services", "Accommodation services", "Publishing sales, excluding software", "Industrial Commodities less fuels", "Investment services", "Food and beverage for immediate consumption services (partial)", "Inputs to industries", "Farm products, processed foods and feeds", "Warehousing, storage, and related services", "Machinery and equipment", "Metals and metal products", "Special indexes", "Intermediate demand by production flow", "Cleaning and building maintenance services (partial)", "Rubber and plastic products", "Construction (partial)", "Processed foods and feeds", "Commissions and fees from sales and administration of insurance policies (partial)", "Telecommunication, cable, and internet user services", "Chemicals and allied products", "Insurance and annuities", "Credit intermediation services (partial)", "Network compensation from broadcast and cable television and radio", "Contract work on textile products, apparel, and leather", "Waste collection and remediation services (partial)", "Selected security services (partial)", "Advertising space and time sales", "Pulp, paper, and allied products", "Wholesale trade services", "Software publishing", "Transportation equipment", "Employment services", "Rental and leasing of goods (partial)", "Farm products", "Final demand", "Real estate services (partial)", "Fuels and related products and power", "Transportation services", "Lumber and wood products", "Textile products and apparel", "Industrial Commodities", "Repair and maintenance services (partial)", "Travel arrangement services (partial)", "Health care services", "Retail trade services", "Professional services (partial)", "Furniture and household durables", "Nonmetallic mineral products", "All commodities", "Mining services", "Data processing and related services", "Miscellaneous products"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "group_code": "00", - "group_name": "All commodities" - }, - { - "group_code": "01", - "group_name": "Farm products" - }, - { - "group_code": "02", - "group_name": "Processed foods and feeds" - } - ], - "table_name": "wp_group", - "table_schema": "\nCREATE TABLE wp_group (\n\tgroup_code VARCHAR(3), \n\tgroup_name VARCHAR(82)\n)" - }, - { - "columns": [ - { - "name": "adjustment_code", - "is_primary_key": false, - "data_type": "VARCHAR(1)", - "description": null, - "low_cardinality": true, - "categories": ["U", "S"], - "foreign_key": null - }, - { - "name": "adjustment_text", - "is_primary_key": false, - "data_type": "VARCHAR(23)", - "description": null, - "low_cardinality": true, - "categories": ["Not seasonally adjusted", "Seasonally adjusted"], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "adjustment_code": "U", - "adjustment_text": "Not seasonally adjusted" - }, - { - "adjustment_code": "S", - "adjustment_text": "Seasonally adjusted" - } - ], - "table_name": "bls_seasonal_adjustment_code", - "table_schema": "\nCREATE TABLE bls_seasonal_adjustment_code (\n\tadjustment_code VARCHAR(1), \n\tadjustment_text VARCHAR(23)\n)" - }, - { - "columns": [ - { - "name": "area_code", - "is_primary_key": false, - "data_type": "VARCHAR(4)", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "area_name", - "is_primary_key": false, - "data_type": "VARCHAR(44)", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "column3", - "is_primary_key": false, - "data_type": "VARCHAR(1)", - "description": null, - "low_cardinality": true, - "categories": ["˚", ""], - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [ - { - "area_code": "0000", - "area_name": "U.S. city average", - "column3": "" - }, - { - "area_code": "0100", - "area_name": "Northeast", - "column3": "" - }, - { - "area_code": "0110", - "area_name": "New England", - "column3": "˚" - } - ], - "table_name": "ap_areacodes", - "table_schema": "\nCREATE TABLE ap_areacodes (\n\tarea_code VARCHAR(4), \n\tarea_name VARCHAR(44), \n\tcolumn3 VARCHAR(1)\n)" - }, - { - "columns": [ - { - "name": "date", - "is_primary_key": false, - "data_type": "DATE", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "period_name", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "value", - "is_primary_key": false, - "data_type": "DOUBLE_PRECISION", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "year", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "period", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "title", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "seriesid", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "latest", - "is_primary_key": false, - "data_type": "TEXT", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - }, - { - "name": "hg_date_updated", - "is_primary_key": false, - "data_type": "TIMESTAMP", - "description": null, - "low_cardinality": false, - "categories": null, - "foreign_key": null - } - ], - "db_alias": "BLS", - "description": null, - "examples": [], - "table_name": " Tuple: - return tuple(sorted(row, key=lambda x: str(x) + str(type(x)))) - -def quick_rej(result1: List[Tuple], result2: List[Tuple], order_matters: bool) -> bool: - s1 = [unorder_row(row) for row in result1] - s2 = [unorder_row(row) for row in result2] - if order_matters: - return s1 == s2 - else: - return set(s1) == set(s2) - -def get_constraint_permutation(tab1_sets_by_columns: List[Set], result2: List[Tuple]): - num_cols = len(result2[0]) - perm_constraints = [{i for i in range(num_cols)} for _ in range(num_cols)] - if num_cols <= 3: - return product(*perm_constraints) - - # we sample 20 rows and constrain the space of permutations - for _ in range(20): - random_tab2_row = random.choice(result2) - - for tab1_col in range(num_cols): - for tab2_col in set(perm_constraints[tab1_col]): - if random_tab2_row[tab2_col] not in tab1_sets_by_columns[tab1_col]: - perm_constraints[tab1_col].remove(tab2_col) - return product(*perm_constraints) - -# return whether two bag of relations are equivalent -def multiset_eq(l1: List, l2: List) -> bool: - if len(l1) != len(l2): - return False - d = defaultdict(int) - for e in l1: - d[e] = d[e] + 1 - for e in l2: - d[e] = d[e] - 1 - if d[e] < 0: - return False - return True - -def permute_tuple(element: Tuple, perm: Tuple) -> Tuple: - assert len(element) == len(perm) - return tuple([element[i] for i in perm]) - -def postprocess(query: str) -> str: - query = query.replace('> =', '>=').replace('< =', '<=').replace('! =', '!=') - return query - -def result_eq(result1: List[Tuple], result2: List[Tuple], order_matters: bool) -> bool: - if len(result1) == 0 and len(result2) == 0: - return True - # if length is not the same, then they are definitely different bag of rows - if len(result1) != len(result2): - return False - - num_cols = len(result1[0]) - # if the results do not have the same number of columns, they are different - if len(result2[0]) != num_cols: - return False - - # unorder each row and compare whether the denotation is the same - # this can already find most pair of denotations that are different - if not quick_rej(result1, result2, order_matters): - return False - - # the rest of the problem is in fact more complicated than one might think - # we want to find a permutation of column order and a permutation of row order, - # s.t. result_1 is the same as result_2 - # we return true if we can find such column & row permutations - # and false if we cannot - tab1_sets_by_columns = [{row[i] for row in result1} for i in range(num_cols)] - - # on a high level, we enumerate all possible column permutations - # that might make result_1 == result_2 - # we decrease the size of the column permutation space by - # the function get_constraint_permutation - # if one of the permutation make result_1, result_2 equivalent, - # then they are equivalent - for perm in get_constraint_permutation(tab1_sets_by_columns, result2): - if len(perm) != len(set(perm)): - continue - if num_cols == 1: - result2_perm = result2 - else: - result2_perm = [permute_tuple(element, perm) for element in result2] - if order_matters: - if result1 == result2_perm: - return True - else: - # in fact the first condition must hold if the second condition holds - # but the first is way more efficient implementation-wise - # and we use it to quickly reject impossible candidates - if set(result1) == set(result2_perm) and multiset_eq(result1, result2_perm): - return True - return False - -def list_of_dicts_to_list_of_tuples(list_of_dicts: List[dict]) -> List[Tuple]: - # Extract values from each dictionary and create a tuple for each dictionary - list_of_tuples = [tuple(d.values()) for d in list_of_dicts] - return list_of_tuples - -def run(db: str, sql: str) -> Tuple[List[dict],bool]: - """Run the SQL query against the database.""" - payload = { - "db_alias" : db, - "sql_statement" : sql - } - try: - response = requests.post(URI + "query", json=payload) - response.raise_for_status() # Raise an exception for non-2xx status codes - result = response.json()[1]["result"] - result = list_of_dicts_to_list_of_tuples(result) - return result, True - except requests.exceptions.RequestException as e: - print("Error: An exception occurred during the request:", e) - # Handle the request exception (connection errors, timeouts, etc.) - return [],False - except ValueError as ve: - print("Error: Unable to parse JSON response:", ve) - # Handle the JSON parsing error (invalid JSON format, missing keys, etc.) - return [],False - except KeyError as ke: - print("Error: Missing 'result' key in the JSON response:", ke) - # Handle the missing 'result' key error - return [],False - except Exception as ex: - print("Error: An unexpected exception occurred:", ex) - # Handle any other unexpected exception - return [],False - -def validate_response_object( - test_dict: dict, - response_dict: dict, - keep_distinct: bool = False) -> dict: - db = test_dict["db"] - gold_sql = test_dict["sql"] - sql_generated = response_dict["sql_query"] - if sql_generated == "": - print(f"Response -> label: Wrong, total_cost: {response_dict['total_cost']}, exec_time: {response_dict['exec_time']}") # noqa: E501 - return { - "question" : test_dict["nl_question"], - "db" : db, - "gold_sql" : gold_sql, - "num_tockens_used" : response_dict["total_tokens"], - "total_cost": response_dict["total_cost"], - "sql_generated": sql_generated, - "exec_time": response_dict["exec_time"], - "status": "WRONG" - } - print(f"Generated SQL: {sql_generated}") - gold_sql, sql_generated = postprocess(gold_sql), postprocess(sql_generated) - if not keep_distinct: - gold_sql = remove_distinct(gold_sql) - sql_generated = remove_distinct(sql_generated) - order_matters = 'order by' in gold_sql.lower() - if f"limit {TOP_K}" in sql_generated.lower(): - sql_generated = sql_generated.lower().replace(f"limit {TOP_K}", "") - if f"limit {TOP_K}" in gold_sql.lower(): - gold_sql = gold_sql.lower().replace(f"limit {TOP_K}", "") - generated_execution_flag = False - retries = 0 - while not generated_execution_flag and retries < MAX_RETRIES: - p_denotation,generated_execution_flag = run(db,sql_generated) - retries += 1 - gold_execution_flag = False - retries = 0 - while not gold_execution_flag: - g_denotation,gold_execution_flag = run(db,gold_sql) - retries += 1 - equivalence = result_eq(g_denotation, p_denotation, order_matters=order_matters) - if equivalence: - label = "CORRECT" - else: - label = "WRONG" - print(f"Response -> label: {label}, total_cost: {response_dict['total_cost']}, exec_time: {response_dict['exec_time']}") # noqa: E501 - benchmark_result = { - "question" : test_dict["nl_question"], - "db" : db, - "gold_sql" : gold_sql, - "num_tockens_used" : response_dict["total_tokens"], - "total_cost": response_dict["total_cost"], - "sql_generated": sql_generated, - "exec_time": response_dict["exec_time"], - "status": label - } - return benchmark_result - -def run_benchmark(tests: List[dict], output_file_name: str = "benchmark_results.jsonl") -> int: # noqa: E501 - """Run the benchmark tool.""" - final_execution_acc = 0 - with open(output_file_name, 'w') as out: - for test in tests: - payload = { - "db_alias" : test["db"], - "question" : test["nl_question"], - } - print("Validating response object...") - print(f"Gold SQL: {test['sql']}") - print(f"Generating SQL for question: {test['nl_question']}...") - response = requests.post(URI + "question", json=payload) - response_dict = response.json() - validation_result = validate_response_object(test, response_dict) - final_execution_acc += 1 if validation_result["status"] == "CORRECT" else 0 - jout = json.dumps(validation_result) + '\n' - out.write(jout) - input("Press Enter to continue...") - return final_execution_acc - -def upload_to_cloud( - file_name: str, - object_name, - bucket: str = 'dataherald-benchmark-results'): - """Upload the results to the cloud.""" - s3_client = boto3.client('s3') - try: - response = s3_client.upload_file(file_name, bucket, object_name) # noqa: F841 - except ClientError as e: - logging.error(e) - return False - return True - -if __name__ == "__main__": - """Run the benchmark tests.""" - resposne = requests.get(URI + "heartbeat") - print("Running benchmark tests...") - print(resposne.json()) - parser = argparse.ArgumentParser(description="The Dataherald Benchmark tool to test performance of text-to-SQL generation.", # noqa: E501 - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - "-t", - "--test", - type=str, - default="apps/ai/clients/benchmark-tool/test_suites/v2_real_estate.jsonl", - help="The file containing the benchmark tests.") - parser.add_argument( - "-u", - "--upload", - action='store_true', - help="Upload the results to the S3 bucket.") - parser.add_argument( - "-o", - "--output", - type=str, - default="apps/ai/clients/benchmark-tool/test_results/", - help="The directory to save the benchmark results file") - parser.add_argument( - "-p", - "--percent", - type=float, - default=0.1, - help="The percentage of the test set to use as context.") - parser.add_argument( - "-s", - "--size", - type=float, - default=1, - help="What percent of the test suite to use in the test") - parser.add_argument( - "-f", - "--freeze", - action='store_true', - help="freezing the random shuffling to reporoduce the same result.") - args = parser.parse_args() - config = vars(args) - test_set_size = config["size"] - context_benchmark_split = config["percent"] - with open(config["test"], "r") as f: - json_list = list(f) - tests = [] - for test in json_list: - tests.append(json.loads(test)) - if config["freeze"]: - random.seed(0) - random.shuffle(tests) - tests = tests[:int(len(tests) * test_set_size)] - print(f"Number of samples used for context: {int(len(tests) * context_benchmark_split)}") # noqa: E501 - print(f"Number of samples used for benchmark: {int(len(tests) * (1 - context_benchmark_split))}") # noqa: E501 - context_set = tests[:int(len(tests) * context_benchmark_split)] - benchmark_set = tests[int(len(tests) * context_benchmark_split):] - output_file_name = f'{os.path.basename(config["test"])}-{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}.jsonl' # noqa: E501 - output_file = f'{config["output"]}{output_file_name}' - add_context(context_set) - exec_acc = run_benchmark(benchmark_set, output_file) - print("Execution accuracy: ", exec_acc / len(benchmark_set)) - if config["upload"]: - print(f"Uploading results of {len(benchmark_set)} to S3 bucket...") - upload_to_cloud(output_file, output_file_name, S3_BUCKET) - else: - print(f"{len(benchmark_set)} results saved to {output_file_name}") diff --git a/apps/ai/clients/benchmark-tool/test_results/.gitkeep b/apps/ai/clients/benchmark-tool/test_results/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/ai/clients/benchmark-tool/test_suites/bls.jsonl b/apps/ai/clients/benchmark-tool/test_suites/bls.jsonl deleted file mode 100644 index 02a83b26..00000000 --- a/apps/ai/clients/benchmark-tool/test_suites/bls.jsonl +++ /dev/null @@ -1,20 +0,0 @@ -{"nl_question":"What is the CPI for all items?", "sql": "SELECT value FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'All items' AND bac.seasonal_adjustment = 'U' ORDER BY date DESC LIMIT 1;", "db":"BLS"} -{"nl_question":"What is the CPI for food and beverages?", "sql": "SELECT value FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Food and beverages' AND bac.seasonal_adjustment = 'U' ORDER BY date DESC LIMIT 1;", "db":"BLS"} -{"nl_question":"What was the CPI for alcoholic beverages in March 2020?", "sql": "SELECT date, value FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Alcoholic beverages' AND bac.seasonal_adjustment = 'U' AND bac.date = '2020-03-01';", "answer": "255.66", "db":"BLS"} -{"nl_question":"What is the CPI for commodities, excluding food?", "sql": "SELECT value FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Commodities less food' AND bac.seasonal_adjustment = 'U' ORDER BY date DESC LIMIT 1;", "db":"BLS"} -{"nl_question":"What was the CPI for all commodities in November 2022?", "sql": "SELECT date, value FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Commodities' AND bac.seasonal_adjustment = 'U' AND bac.date = '2022-11-01';", "answer": "222.112", "db":"BLS"} -{"nl_question":"How much did the cost of apparel increase in 2022?", "sql": "SELECT bac.date AS start_month, bac2.date AS end_month, CONCAT(ROUND(((bac2.value - bac.value) / bac.value * 100)::numeric, 2), '%') AS pct_change FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.seriesid = bac2.seriesid WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Apparel' AND bac.seasonal_adjustment = 'U' AND bac.date = '2022-01-01' AND bac2.date = '2022-12-01';", "answer": "0.41%", "db":"BLS"} -{"nl_question":"What was the percent change in education costs between 2012 and 2020?", "sql": "SELECT bac.date AS start_month, bac2.date AS end_month, CONCAT(ROUND(((bac2.value - bac.value) / bac.value * 100)::numeric, 2), '%') AS pct_change FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.seriesid = bac2.seriesid WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Education' AND bac.seasonal_adjustment = 'U' AND bac.date = '2012-01-01' AND bac2.date = '2020-12-01';", "answer": "27.86%", "db":"BLS"} -{"nl_question":"How much more did cereal cost from the start and end of 2021?", "sql": "SELECT bac.date AS start_month, bac2.date AS end_month, CONCAT(ROUND(((bac2.value - bac.value) / bac.value * 100)::numeric, 2), '%') AS pct_change FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.seriesid = bac2.seriesid WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Breakfast cereal' AND bac.seasonal_adjustment = 'U' AND bac.date = '2021-01-01' AND bac2.date = '2021-12-01';", "answer": "6.72%", "db":"BLS"} -{"nl_question":"How much did the price of fresh fruit increase from March 2020 to March 2021?", "sql": "SELECT bac.date AS start_month, bac2.date AS end_month, CONCAT(ROUND(((bac2.value - bac.value) / bac.value * 100)::numeric, 2), '%') AS pct_change FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.seriesid = bac2.seriesid WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Fresh fruits' AND bac.seasonal_adjustment = 'U' AND bac.date = '2020-03-01' AND bac2.date = '2021-03-01';", "answer": "5.56%", "db":"BLS"} -{"nl_question":"How much more do legal services cost since 2021?", "sql": "SELECT bac.date AS start_month, bac2.date AS end_month,CONCAT(ROUND(((bac2.value - bac.value) / bac.value * 100)::numeric, 2), '%') AS pct_change FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.seriesid = bac2.seriesid WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Legal services' AND bac.seasonal_adjustment = 'U' AND bac.date = '2021-01-01' AND bac2.date = (SELECT max(date) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cac.area_name = 'U.S. city average' AND cic.item_name = 'Legal services' AND bac.seasonal_adjustment = 'U');", "answer": "15.97%", "db":"BLS"} -{"nl_question":"Compare the cost of baby food between the northeast and midwest region in 2022.", "sql": "SELECT bac.date AS date, bac.value AS northeast_cost, bac2.value AS midwest_cost FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.date = bac2.date AND cic.item_code = bac2.item_code INNER JOIN cpi_area_code cac2 ON bac2.area_code = cac2.area_code WHERE cac.area_name = 'Northeast' AND cac2.area_name = 'Midwest' AND cic.item_name = 'Commodities' AND bac.seasonal_adjustment = 'U' ORDER BY bac.date DESC LIMIT 3;", "answer": "", "db":"BLS"} -{"nl_question":"How much more does recreation cost in New York vs. Los Angeles?", "sql": "SELECT bac.date, bac.value AS la_cost,bac2.value AS ny_cost, CONCAT(ROUND(((bac2.value - bac.value) / bac.value * 100)::numeric, 2), '%') AS pct_diff FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code INNER JOIN cpi_area_code cac2 ON bac2.area_code = cac2.area_code WHERE cac.area_name = 'Los Angeles-Long Beach-Anaheim, CA' AND cac2.area_name = 'New York-Newark-Jersey City, NY-NJ-PA' AND cic.item_name = 'Recreation' AND bac.seasonal_adjustment = 'U' AND bac.date = bac2.date ORDER BY bac.date DESC LIMIT 1;", "db":"BLS"} -{"nl_question":"Are costs higher in New England or the Mid-Atlantic area?", "sql": "", "answer": "SELECT bac.date AS date, bac.value AS new_england_cost, bac2.value AS mid_atlantic_cost FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.date = bac2.date AND cic.item_code = bac2.item_code INNER JOIN cpi_area_code cac2 ON bac2.area_code = cac2.area_code WHERE cac.area_name = 'New England' AND cac2.area_name = 'Middle Atlantic' AND cic.item_name = 'All items' AND bac.seasonal_adjustment = 'U' ORDER BY bac.date DESC LIMIT 3;", "db":"BLS"} -{"nl_question":"How does the cost of electricity compare between Houston and the rest of the country?", "sql": "SELECT bac.date AS date, bac.value AS houston_cost, bac2.value AS usa_cost FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.date = bac2.date AND cic.item_code = bac2.item_code INNER JOIN cpi_area_code cac2 ON bac2.area_code = cac2.area_code WHERE cac.area_name = 'Houston-The Woodlands-Sugar Land, TX' AND cac2.area_name = 'U.S. city average' AND cic.item_name = 'Electricity' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U' ORDER BY bac.date DESC LIMIT 3;", "db":"BLS"} -{"nl_question":"Is food more expensive in Hawaii or Alaska?", "sql": "SELECT bac.date AS date, bac.value AS hawaii_cost, bac2.value AS alaska_cost FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON (bac.date + make_interval(months => 1)) = bac2.date AND cic.item_code = bac2.item_code INNER JOIN cpi_area_code cac2 ON bac2.area_code = cac2.area_code WHERE cac.area_name = 'Urban Hawaii' AND cac2.area_name = 'Urban Alaska' AND cic.item_name = 'Food' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U' ORDER BY bac.date DESC LIMIT 3;", "db":"BLS"} -{"nl_question":"Which item showed the largest increase in cost in the West region?", "sql": "SELECT bac.date, cic.item_name, bac.value AS value_now, bac2.value AS value_old, CONCAT(ROUND(((bac.value - bac2.value) / bac2.value * 100)::numeric, 2), '%') AS pct_increase FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cac.area_name = 'West' AND bac.seasonal_adjustment = 'U' AND bac.date = (SELECT max(bac.date) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cac.area_name = 'West' AND bac.seasonal_adjustment = 'U') AND bac2.date = (SELECT max(bac.date) - make_interval(months => 1) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cac.area_name = 'West' AND bac.seasonal_adjustment = 'U') ORDER BY (bac.value - bac2.value) / bac2.value * 100 DESC LIMIT 1;", "answer": "Cereals and bakery products", "db":"BLS"} -{"nl_question":"In which region did the price of fuel increase the most?", "sql": "SELECT cac.area_name, bac.value AS value_now, bac2.value AS value_old, CONCAT(ROUND(((bac.value - bac2.value) / bac2.value * 100)::numeric, 2), '%') AS pct_increase FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cic.item_name = 'Motor fuel' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U' AND bac.date = (SELECT max(bac.date) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cic.item_name = 'Motor fuel' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U') AND bac2.date = (SELECT max(bac.date) - MAKE_INTERVAL(months => 1) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cic.item_name = 'Motor fuel' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U') ORDER BY ((bac.value - bac2.value) / bac2.value * 100) DESC LIMIT 1;", "answer": "Denver-Aurora-Lakewood, CO", "db":"BLS"} -{"nl_question":"Which fruit showed the largest increase in cost in 2020?", "sql": "SELECT cic.item_name, bac.value AS value_2020_end, bac2.value AS value_2020_start, CONCAT(ROUND(((bac.value - bac2.value) / bac2.value * 100)::numeric, 2), '%') AS pct_increase FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE (cic.item_name = 'Apples' OR cic.item_name = 'Bananas' OR cic.item_name = 'Citrus fruits' OR cic.item_name = 'Other fresh fruits') AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U' AND bac.date = '2020-12-01' AND bac2.date = '2020-01-01' ORDER BY (bac.value - bac2.value) / bac2.value * 100 DESC LIMIT 1;", "answer": "Citrus fruits", "db":"BLS"} -{"nl_question":"Which region is the most expensive to rent in now?", "sql": "SELECT cac.area_name, bac.value AS rent_cpi FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cic.item_name = 'Rent of primary residence' AND bac.seasonal_adjustment = 'U' AND bac.date = (SELECT max(bac.date) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code WHERE cic.item_name = 'Rent of primary residence' AND bac.seasonal_adjustment = 'U') ORDER BY bac.value DESC LIMIT 1;", "answer": "San Francisco-Oakland-Hayward, CA", "db":"BLS"} -{"nl_question":"How much did purchasing power change over the past year across the different regions?", "sql": "SELECT cac.area_name, bac.value AS value_now, bac2.value AS value_old, CONCAT(ROUND(((bac.value - bac2.value) / bac2.value * 100)::numeric, 2), '%') AS pct_increase FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cic.item_name = 'Purchasing power of the consumer dollar' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U' AND bac.date = (SELECT max(bac.date) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cic.item_name = 'Motor fuel' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U') AND bac2.date = (SELECT max(bac.date) - MAKE_INTERVAL(months => 12) FROM bls_all_cpi bac INNER JOIN cpi_area_code cac ON bac.area_code = cac.area_code INNER JOIN cpi_item_code cic ON bac.item_code = cic.item_code INNER JOIN bls_all_cpi bac2 ON bac.item_code = bac2.item_code AND bac.area_code = bac2.area_code WHERE cic.item_name = 'Motor fuel' AND bac.seasonal_adjustment = 'U' AND bac2.seasonal_adjustment = 'U') ORDER BY ((bac.value - bac2.value) / bac2.value * 100) DESC;", "db":"BLS"} \ No newline at end of file diff --git a/apps/ai/clients/benchmark-tool/test_suites/v2_real_estate.jsonl b/apps/ai/clients/benchmark-tool/test_suites/v2_real_estate.jsonl deleted file mode 100644 index a3e070bf..00000000 --- a/apps/ai/clients/benchmark-tool/test_suites/v2_real_estate.jsonl +++ /dev/null @@ -1,80 +0,0 @@ -{"nl_question":"what was the most expensive zip code to rent in Los Angeles county in May 2022?", "sql": "select location_name, metric_value from renthub_median_rent where dh_county_name = 'Los Angeles' and period_start='2022-05-01' and geo_type='zip' order by metric_value desc", "db":"v2_real_estate"} -{"nl_question":"what was the median home sale price in Califronia in Q1 2021?","sql":"select location_name, metric_value, period_end, property_type from redfin_median_sale_price rmsp where geo_type = 'state' and location_name='California' and property_type = 'All Residential' and period_start between '2021-01-01' and '2021-03-01' order by period_end " , "db":"v2_real_estate"} -{"nl_question":"How many townhomes have been sold in Austin, TX this year?", "sql":"select sum(metric_value), location_name, dh_state_name from redfin_homes_sold rhs where geo_type='city' and location_name ='Austin' and property_type='Townhouse' and '2023-01-1' <= period_start group by location_name , dh_state_name ", "db":"v2_real_estate"} -{"nl_question":"Did property prices increase or descrease in the US in 2018?", "sql":"select metric_value, location_name, period_start from redfin_median_list_price rmlp where geo_type ='national' and period_start between '2020-1-1' and '2020-12-01' order by period_start ", "db":"v2_real_estate"} -{"nl_question":"compare home prices in los angeles vs austin since 2022", "sql":"Select metric_value, period_end, location_name, dh_state_name, property_type from redfin_median_sale_price where geo_type='city' and (location_name='Los Angeles' and dh_state_name='California' and period_end between '2022-01-31' and '2023-05-31' and property_type = 'All Residential') or (location_name = 'Austin' and dh_state_name='Texas' and period_end between '2022-01-31' and '2023-05-31' and property_type = 'All Residential') order by period_end", "db":"v2_real_estate"} -{"nl_question":"average home price 90405", "sql":"select metric_value, property_type, location_name, period_start , period_end from redfin_median_list_price rmlp where geo_type='zip' and location_name = '90405' order by period_start desc limit 30", "db":"v2_real_estate"} -{"nl_question":"how many homes were sold in orange county in 2022 broken down by property type", "sql":"select property_type, SUM(metric_value), period_type from redfin_homes_sold rhs where geo_type='county' and location_name='Orange County' and dh_state_name='California' and period_end between '2022-01-31' and '2022-12-31' group by property_type, period_type", "db":"v2_real_estate"} -{"nl_question":"rents in santa monica 2021 till jan 2023", "sql":"select metric_value, period_start, period_end, location_name, property_type from renthub_median_rent where geo_type = 'city' and location_name = 'Santa Monica' and dh_state_name = 'California' and property_type = 'All Residential' and period_start between '2021-01-01' and '2023-01-01' order by period_start", "db":"v2_real_estate"} -{"nl_question": "What is the median rent price for each property type in Los angeles city?", "sql": " SELECT property_type,location_name,metric_value FROM public.renthub_median_rent WHERE true and geo_type='city' and property_type='All Residential'AND dh_state_name = 'California' AND dh_place_name = 'Los Angeles' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "What is the median sale price for each property type in California state?", "sql": " SELECT property_type,location_name,metric_value FROM public.redfin_median_sale_price rmsp WHERE true and geo_type='state' and property_type='All Residential' and dh_state_name='California' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "What is the total number of listings for each property type in New York city?", "sql": " SELECT property_type,location_name,metric_value FROM public.redfin_inventory ri WHERE true and geo_type='city' and property_type='All Residential' and dh_state_name='New York' and dh_place_name='New York' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "How many new listings are there for each property type in the Orange county?", "sql": " SELECT property_type,location_name,metric_value FROM public.redfin_new_listings ri WHERE true and geo_type='county' and location_name='Orange County' and dh_state_name='California' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which city has the highest median sale price overall in Orange County, California?", "sql": " SELECT location_name,metric_value FROM public.redfin_median_sale_price WHERE true and geo_type='city' and dh_county_name= 'Orange County' and dh_state_name='California' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which county has the highest median rent price overall in Florida?", "sql": " SELECT location_name,metric_value FROM public.renthub_median_rent WHERE true and geo_type='county' and dh_state_name='Florida' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which ZIP Code has the largest inventory overall in Chicago?", "sql": " SELECT location_name,metric_value FROM public.redfin_inventory WHERE true and geo_type='zip' and dh_place_name='Chicago' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which State has the largest numer of new inventory overall?", "sql": " SELECT location_name,metric_value FROM public.redfin_new_listings WHERE true and geo_type='state' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which county has the lowest median sale price overall in Florida?", "sql": " SELECT location_name,metric_value FROM public.redfin_median_sale_price WHERE true and geo_type='county' and dh_state_name='Florida' ORDER BY period_end DESC, metric_value LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which city has the lowest median rent price overall in Orange county?", "sql": " SELECT location_name,metric_value FROM public.renthub_median_rent WHERE true and geo_type='city' and dh_state_name='California' and dh_county_name= 'Orange' ORDER BY period_end DESC, metric_value LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which State has the lowest inventory number overall?", "sql": " SELECT location_name,metric_value FROM public.redfin_inventory WHERE true and geo_type='state' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which city has the lowest number of new listings overall in Ohio?", "sql": " SELECT location_name,metric_value FROM public.redfin_new_listings WHERE true and geo_type='city' and dh_state_name='Ohio' ORDER BY period_end DESC, metric_value DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the highest change in the median rent price in Texas over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Texas' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the highest change in the median sale price in North Carolina over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'North Carolina' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the highest change in the inventory of Real Estate in Illinois over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Illinois' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the highest change in the number of new listings of Real Estate in Texas over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_new_listings WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Texas' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the lowest change in the median rent price in Texas over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Texas' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 LIMIT 1;" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the lowest change in the median sale price in Texas over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Texas' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 LIMIT 1;" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the lowest change in the inventory of Real Estate in North Carolina over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'North Carolina' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "which is the city with the lowest change in the number of new listings of Real Estate in Ohio over the last year?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_new_listings WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Ohio' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "How much did median rent price change over the past year in Miami city?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Miami' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "How much did median sale price change over the past year in Dade county ?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type='county' AND dh_state_name = 'Florida' and dh_county_name = 'Miami-Dade' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "How much did real estate inventory change over the past year in Los Angeles county ?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type='county' AND dh_state_name = 'California' and dh_county_name = 'Los Angeles' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "How much did real estate new listings number change over the past year in Queens county ?", "sql": "WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_new_listings WHERE dh_place_name <> '-' AND property_type = 'All Residential'),data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type='county' and dh_state_name = 'New York' and dh_county_name = 'Queens' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range))) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1" , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest median sale price in Miami city?", "sql": " SELECT period_end, location_name, property_type, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND dh_state_name = 'Florida' AND dh_place_name = 'Miami' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest median rent price in Dade county ?", "sql": " SELECT period_end, location_name, property_type, metric_value FROM public.renthub_median_rent WHERE geo_type='county' AND dh_state_name = 'Florida' and dh_county_name = 'Miami-Dade' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest inventory of real estate in Los Angeles county ?", "sql": " SELECT period_end, location_name, property_type, metric_value FROM public.redfin_inventory WHERE geo_type='county' AND dh_state_name = 'California' and dh_county_name = 'Los Angeles' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest new listings number in Queens county ?", "sql": " SELECT period_end, location_name, property_type, metric_value FROM public.renthub_median_rent WHERE true and geo_type='county' and dh_state_name = 'New York' and dh_county_name = 'Queens' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "How does the median sale price compare across Dade county and Los Angeles county?", "sql": " WITH location_1 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'county' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_county_name = 'Miami-Dade' ORDER BY period_end DESC LIMIT 1) , location_2 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'county' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_county_name = 'Los Angeles' ORDER BY period_end DESC LIMIT 1 ) SELECT l1.location_name, l1.metric_value, l2.metric_value, (l1.metric_value - l2.metric_value)/ l1.metric_value AS pct_change FROM location_1 l1, location_2 l2 " , "db":"v2_real_estate"} -{"nl_question": "How does the median rent price compare across Los Angeles county and Queens county?", "sql": " WITH location_1 AS ( SELECT period_end, location_name, metric_value FROM public.renthub_median_rent WHERE geo_type = 'county' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_county_name = 'Los Angeles' ORDER BY period_end DESC LIMIT 1) , location_2 AS ( SELECT period_end, location_name, metric_value FROM public.renthub_median_rent WHERE geo_type = 'county' AND property_type = 'All Residential' AND dh_state_name = 'New York' AND dh_county_name = 'Queens' ORDER BY period_end DESC LIMIT 1 ) SELECT l1.location_name, l1.metric_value, l2.metric_value, (l1.metric_value - l2.metric_value)/ l1.metric_value AS pct_change FROM location_1 l1, location_2 l2 " , "db":"v2_real_estate"} -{"nl_question": "How does the inventory of real estate compare across Miami city and Orlando?", "sql": " WITH location_1 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Miami' ORDER BY period_end DESC LIMIT 1) , location_2 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Orlando' ORDER BY period_end DESC LIMIT 1 ) SELECT l1.location_name, l1.metric_value, l2.metric_value, (l1.metric_value - l2.metric_value)/ l1.metric_value AS pct_change FROM location_1 l1, location_2 l2 " , "db":"v2_real_estate"} -{"nl_question": "How does the new listings in real estate compare across Chicago and San Francisco?", "sql": " WITH location_1 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_new_listings WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Illinois' AND dh_place_name = 'Chicago' ORDER BY period_end DESC LIMIT 1) , location_2 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_new_listings WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_place_name = 'San Francisco' ORDER BY period_end DESC LIMIT 1 ) SELECT l1.location_name, l1.metric_value, l2.metric_value, (l1.metric_value - l2.metric_value)/ l1.metric_value AS pct_change FROM location_1 l1, location_2 l2 " , "db":"v2_real_estate"} -{"nl_question": "What's the percent change in median sale price for Angeles county in the past month?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '1 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'All Residential' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_place_name = 'Los Angeles' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "What's the percent change in median rent price for Queens county in the past month?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '1 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'All Residential' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'county' AND property_type = 'All Residential' AND dh_state_name = 'New York' AND dh_place_name = 'Queens' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "What's the percent change in inventory of real estatea for Orlando in the past month?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '1 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type = 'All Residential' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Orlando' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "What's the percent change in new listings in real estate for San Francisco in the past month?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '1 months' AS period_beg FROM public.redfin_new_listings WHERE dh_place_name <> '-' AND property_type = 'All Residential' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_place_name = 'San Francisco' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "What is the median rent price type in Dade county for Apartment Building?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'county' AND property_type = 'Apartment Building' AND dh_state_name = 'Florida' AND dh_county_name = 'Miami-Dade' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "What is the median sale price type in Orlando for Multi-family?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type = 'Multi-family' AND dh_state_name = 'Florida' AND dh_county_name = 'Orlando' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "What is the total number of listings type in San Francisco for Single-family?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type = 'Multi-family' AND dh_state_name = 'Florida' AND dh_county_name = 'Orlando' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "How many new listings are there each property type in a San antonio texas for Townhouse?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type = 'city' AND property_type = 'Townhouse' AND dh_state_name = 'Texas' AND dh_county_name = 'San Antonio' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "How much did median rent price change over the past year in Miami city Condo?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'Condo' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'Condo' AND dh_state_name = 'Florida' AND dh_place_name = 'Miami' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "How much did median sale price change over the past year in North Carolina for Single-family ?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'Single-family' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'state' AND property_type = 'Single-family' AND dh_state_name = 'North Carolina' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "How much did real estate inventory change over the past year in Illinois for Multi-family?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type = 'Multi-family' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type = 'state' AND property_type = 'Multi-family' AND dh_state_name = 'Illinois' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "How much did real estate new listings number change over the past year in Queens county for Townhouse?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.redfin_new_listings WHERE dh_place_name <> '-' AND property_type = 'Multi-family' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type = 'county' AND property_type = 'Multi-family' AND dh_state_name = 'New York' and dh_county_name = 'Queens' AND (period_end = ( SELECT period_end FROM date_range) OR period_end = ( SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = ( SELECT period_end FROM date_range) AND ld.period_end = ( SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest median sale price in Florida for Condo?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'state' AND property_type = 'Condo' AND dh_state_name = 'Florida' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest median rent price in Dade county for Single unit?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'county' AND property_type = 'Single unit' AND dh_state_name = 'Florida' AND dh_county_name = 'Miami-Dade' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest inventory of real estate in Nevada county for Townhouse?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_inventory WHERE geo_type = 'county' AND property_type = 'Townhouse' AND dh_state_name = 'Florida' AND dh_county_name = 'Nevada' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which property type had the highest new listings number in Queens county for Multi-family?", "sql": " SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_new_listings WHERE geo_type = 'county' AND property_type = 'Multi-family' AND dh_state_name = 'Florida' AND dh_county_name = 'Queens' ORDER BY period_end DESC, metric_value DESC LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Did median sale price increase/decrease in Orlando for Apartment Building over the past year?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '1 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'Apartment Building' ), current_data AS ( SELECT metric_value AS average_price FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'Apartment Building' AND dh_state_name = 'Florida' AND dh_place_name = 'Orlando' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT metric_value AS average_price FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'Apartment Building' AND dh_state_name = 'Florida' AND dh_place_name = 'Orlando' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT CASE WHEN ABS((cd.average_price - ld.average_price) / ld.average_price) BETWEEN 0.0 AND 0.01 THEN CONCAT(TO_CHAR(ABS(cd.average_price - ld.average_price) / ld.average_price * 100, '0.99'), '%') ELSE CONCAT(TO_CHAR(ABS((cd.average_price - ld.average_price) / ld.average_price) * 100, '99.9'), '%') END AS pct_change_abs, CASE WHEN (cd.average_price - ld.average_price) / ld.average_price > 0 THEN 'increased' ELSE 'decreased' END AS direction1 FROM current_data cd CROSS JOIN last_month_data ld " , "db":"v2_real_estate"} -{"nl_question": "Did median rent price increase/decrease in Sacramento for Single unit over the past year?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type like '%Single%' ), current_data AS ( SELECT metric_value AS average_price FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type like '%Single%' AND dh_state_name = 'California' AND dh_place_name = 'Sacramento' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT metric_value AS average_price FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type like '%Single%' AND dh_state_name = 'California' AND dh_place_name = 'Sacramento' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT CASE WHEN ABS((cd.average_price - ld.average_price) / ld.average_price) BETWEEN 0.0 AND 0.01 THEN CONCAT(TO_CHAR(ABS(cd.average_price - ld.average_price) / ld.average_price * 100, '0.99'), '%') ELSE CONCAT(TO_CHAR(ABS((cd.average_price - ld.average_price) / ld.average_price) * 100, '99.9'), '%') END AS pct_change_abs, CASE WHEN (cd.average_price - ld.average_price) / ld.average_price > 0 THEN 'increased' ELSE 'decreased' END AS direction1 FROM current_data cd CROSS JOIN last_month_data ld " , "db":"v2_real_estate"} -{"nl_question": "Did inventory of real estate increase/decrease in Las Vegas for Townhouse over the past year?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type like '%Single%' ), current_data AS ( SELECT metric_value AS average_price FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type like '%Single%' AND dh_state_name = 'California' AND dh_place_name = 'Sacramento' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT metric_value AS average_price FROM public.redfin_median_sale_price WHERE geo_type = 'city' AND property_type like '%Single%' AND dh_state_name = 'California' AND dh_place_name = 'Sacramento' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT CASE WHEN ABS((cd.average_price - ld.average_price) / ld.average_price) BETWEEN 0.0 AND 0.01 THEN CONCAT(TO_CHAR(ABS(cd.average_price - ld.average_price) / ld.average_price * 100, '0.99'), '%') ELSE CONCAT(TO_CHAR(ABS((cd.average_price - ld.average_price) / ld.average_price) * 100, '99.9'), '%') END AS pct_change_abs, CASE WHEN (cd.average_price - ld.average_price) / ld.average_price > 0 THEN 'increased' ELSE 'decreased' END AS direction1 FROM current_data cd CROSS JOIN last_month_data ld " , "db":"v2_real_estate"} -{"nl_question": "Did new listings in real estate increase/decrease in San Antonio for Single-family over the past year?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type like '%Single%' ), current_data AS ( SELECT metric_value AS average_price FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type like '%Single%' AND dh_state_name = 'Texas' AND dh_place_name = 'San Antonio' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT metric_value AS average_price FROM public.redfin_inventory WHERE geo_type = 'city' AND property_type like '%Single%' AND dh_state_name = 'Texas' AND dh_place_name = 'San Antonio' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT CASE WHEN ABS((cd.average_price - ld.average_price) / ld.average_price) BETWEEN 0.0 AND 0.01 THEN CONCAT(TO_CHAR(ABS(cd.average_price - ld.average_price) / ld.average_price * 100, '0.99'), '%') ELSE CONCAT(TO_CHAR(ABS((cd.average_price - ld.average_price) / ld.average_price) * 100, '99.9'), '%') END AS pct_change_abs, CASE WHEN (cd.average_price - ld.average_price) / ld.average_price > 0 THEN 'increased' ELSE 'decreased' END AS direction1 FROM current_data cd CROSS JOIN last_month_data ld " , "db":"v2_real_estate"} -{"nl_question": "Which city showed the highest total increase in median sale price for condo in Texas?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type like '%Condo%' ), current_data AS ( SELECT location_name, metric_value FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type like '%Condo%' AND dh_state_name = 'Texas' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT location_name, metric_value FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type like '%Condo%' AND dh_state_name = 'Texas' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT cd.location_name, cd.metric_value, ld.metric_value, ((cd.metric_value - ld.metric_value) / ld.metric_value) as pct_change FROM current_data cd inner JOIN last_month_data ld ON cd.location_name = ld.location_name ORDER by 4 desc LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which county showed the highest total increase in median rent price for Multi-family in Florida?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type like '%Multi%' ), current_data AS ( SELECT location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'county' AND property_type like '%Multi%' AND dh_state_name = 'Florida' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'county' AND property_type like '%Multi%' AND dh_state_name = 'Florida' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT cd.location_name, cd.metric_value, ld.metric_value, ((cd.metric_value - ld.metric_value) / ld.metric_value) as pct_change FROM current_data cd inner JOIN last_month_data ld ON cd.location_name = ld.location_name ORDER by 4 desc LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which zip showed the highest total increase in real estate inventory for single-family in Nevada?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.redfin_inventory WHERE dh_place_name <> '-' AND property_type like '%single%' ), current_data AS ( SELECT location_name, metric_value FROM public.redfin_inventory WHERE geo_type = 'zip' AND property_type like '%single%' AND dh_state_name = 'Nevada' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT location_name, metric_value FROM public.redfin_inventory WHERE geo_type = 'zip' AND property_type like '%single%' AND dh_state_name = 'Nevada' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT cd.location_name, cd.metric_value, ld.metric_value, ((cd.metric_value - ld.metric_value) / ld.metric_value) as pct_change FROM current_data cd inner JOIN last_month_data ld ON cd.location_name = ld.location_name ORDER by 4 desc LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question": "Which city showed the highest total increase in new listings for real estate for townhouse in Ohio?", "sql": " WITH date_range AS ( SELECT max(period_end) - interval '0 month' AS period_end, max(period_end) - interval '12 months' AS period_beg FROM public.redfin_new_listings WHERE dh_place_name <> '-' AND property_type like '%Single%' ), current_data AS ( SELECT location_name, metric_value FROM public.redfin_new_listings WHERE geo_type = 'zip' AND property_type like '%Single%' AND dh_state_name = 'Ohio' AND period_end = ( SELECT period_end FROM date_range) ), last_month_data AS ( SELECT location_name, metric_value FROM public.redfin_new_listings WHERE geo_type = 'zip' AND property_type like '%Single%' AND dh_state_name = 'Ohio' AND period_end = ( SELECT period_beg FROM date_range) ) SELECT cd.location_name, cd.metric_value, ld.metric_value, ((cd.metric_value - ld.metric_value) / ld.metric_value) as pct_change FROM current_data cd inner JOIN last_month_data ld ON cd.location_name = ld.location_name ORDER by 4 desc LIMIT 1 " , "db":"v2_real_estate"} -{"nl_question":"In which city in California are the median sale prices higher for single family compared to Townhouses?", "sql": " with prop_type_1 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.redfin_median_sale_price WHERE true and geo_type='city' and property_type='Single-family' and dh_state_name='California' and period_end= ( SELECT max(period_end) FROM redfin_median_sale_price) group by 1,2 ) , prop_type_2 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.redfin_median_sale_price WHERE true and geo_type='city' and property_type ='Townhouse' and dh_state_name='California' and period_end= ( SELECT max(period_end) FROM redfin_median_sale_price) group by 1,2 ) SELECT pt_1.dh_place_name, pt_1.property_type, pt_1.metric_value as pt1_value, pt_2.dh_place_name, pt_2.property_type, pt_2.metric_value as pt2_value FROM prop_type_1 as pt_1 inner JOIN prop_type_2 as pt_2 ON pt_1.dh_place_name = pt_2.dh_place_name and pt_1.metric_value> pt_2.metric_value ", "db":"v2_real_estate"} -{"nl_question":"In which city in Texas are the median rent prices higher for Townhouses compared to Condos?", "sql": " with prop_type_1 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.renthub_median_rent WHERE true and geo_type='city' and property_type='Townhouse' and dh_state_name='Texas' and period_end= ( SELECT max(period_end) FROM renthub_median_rent) group by 1,2 ) , prop_type_2 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.renthub_median_rent WHERE true and geo_type='city' and property_type ='Condo/Co-op' and dh_state_name='Texas' and period_end= ( SELECT max(period_end) FROM renthub_median_rent) group by 1,2 ) SELECT pt_1.dh_place_name, pt_1.property_type, pt_1.metric_value as pt1_value, pt_2.dh_place_name, pt_2.property_type, pt_2.metric_value as pt2_value FROM prop_type_1 as pt_1 inner JOIN prop_type_2 as pt_2 ON pt_1.dh_place_name = pt_2.dh_place_name and pt_1.metric_value> pt_2.metric_value ", "db":"v2_real_estate"} -{"nl_question":"In which city in Nevada are the median rent prices higher for condos compared to Single families?", "sql": " with prop_type_1 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.redfin_inventory WHERE true and geo_type='city' and property_type='Condo/Co-op' and dh_state_name='Nevada' and period_end= ( SELECT max(period_end) FROM redfin_inventory) group by 1,2 ) , prop_type_2 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.redfin_inventory WHERE true and geo_type='city' and property_type ='Single-family' and dh_state_name='Nevada' and period_end= ( SELECT max(period_end) FROM redfin_inventory) group by 1,2 ) SELECT pt_1.dh_place_name, pt_1.property_type, pt_1.metric_value as pt1_value, pt_2.dh_place_name, pt_2.property_type, pt_2.metric_value as pt2_value FROM prop_type_1 as pt_1 inner JOIN prop_type_2 as pt_2 ON pt_1.dh_place_name = pt_2.dh_place_name and pt_1.metric_value> pt_2.metric_value ", "db":"v2_real_estate"} -{"nl_question":"In which city in Texas are the median rent prices higher for condos compared to Single families?", "sql": " with prop_type_1 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.redfin_new_listings WHERE true and geo_type='city' and property_type='Condo/Co-op' and dh_state_name='Texas' and period_end= ( SELECT max(period_end) FROM redfin_new_listings) group by 1,2 ) , prop_type_2 as ( SELECT dh_place_name, property_type, max(metric_value) as metric_value FROM public.redfin_new_listings WHERE true and geo_type='city' and property_type ='Single-family' and dh_state_name='Texas' and period_end= ( SELECT max(period_end) FROM redfin_new_listings) group by 1,2 ) SELECT pt_1.dh_place_name, pt_1.property_type, pt_1.metric_value as pt1_value, pt_2.dh_place_name, pt_2.property_type, pt_2.metric_value as pt2_value FROM prop_type_1 as pt_1 inner JOIN prop_type_2 as pt_2 ON pt_1.dh_place_name = pt_2.dh_place_name and pt_1.metric_value> pt_2.metric_value", "db":"v2_real_estate"} -{"nl_question": "Can you provide data on historical property rent prices in San Antonio, Texas over the past year?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT period_end, location_name, metric_value FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Texas' AND dh_place_name = 'San Antonio' AND period_end >= (SELECT period_beg FROM date_range) ORDER BY period_end DESC ", "db":"v2_real_estate"} -{"nl_question": "Which is the difference for ZIP codes between the higher and lower rent value in Orlando ?", "sql": " WITH location_1 AS ( SELECT period_end, dh_place_name, location_name, metric_value FROM public.renthub_median_rent WHERE geo_type = 'zip' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Orlando' ORDER BY period_end DESC, metric_value LIMIT 1) SELECT ri.period_end, ri.location_name, (ri.metric_value- l1.metric_value) /l1.metric_value as pct_diff FROM public.renthub_median_rent ri INNER JOIN location_1 l1 ON ri.dh_place_name = l1.dh_place_name AND ri.period_end = l1.period_end WHERE ri.geo_type = 'city' AND ri.property_type = 'All Residential' AND ri.dh_state_name = 'Florida' AND ri.dh_place_name = 'Orlando' ORDER BY ri.period_end DESC LIMIT 1 ", "db":"v2_real_estate"} -{"nl_question": "how did rent prices changed bewteen 2020 and 2022 in Miami?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '24 months' AS period_beg FROM public.renthub_median_rent WHERE dh_place_name <> '-' AND property_type = 'All Residential' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.renthub_median_rent WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Miami' AND (period_end = (SELECT period_end FROM date_range) OR period_end = (SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = (SELECT period_end FROM date_range) AND ld.period_end = (SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1 REDFIN_MEDIAN_SALE_PRICE: ", "db":"v2_real_estate"} -{"nl_question": "which are the top 3 ZIP Codes with the highest property appreciation in Los Angeles?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT location_name, metric_value AS median_price, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'zip' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_place_name = 'Los Angeles' AND period_end = (SELECT period_end FROM date_range) ORDER BY period_end DESC, metric_value DESC limit 3 ", "db":"v2_real_estate"} -{"nl_question": "which is the difference between the high median sale value ZIP Code in Los Angeles and the LA median?", "sql": " WITH location_1 AS ( SELECT period_end, dh_place_name, location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'zip' AND property_type = 'All Residential' AND dh_state_name = 'California' AND dh_place_name = 'Los Angeles' ORDER BY period_end DESC, metric_value DESC LIMIT 1) SELECT ri.period_end, ri.location_name, (ri.metric_value- l1.metric_value) /l1.metric_value as pct_diff FROM public.redfin_median_sale_price ri INNER JOIN location_1 l1 ON ri.dh_place_name = l1.dh_place_name AND ri.period_end = l1.period_end WHERE ri.geo_type = 'city' AND ri.property_type = 'All Residential' AND ri.dh_state_name = 'California' AND ri.dh_place_name = 'Los Angeles' ORDER BY ri.period_end DESC LIMIT 1 ", "db":"v2_real_estate"} -{"nl_question": "which is the difference between the most expensive ZIP Code in NY and Miami?", "sql": " WITH location_1 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'zip' AND property_type = 'All Residential' AND dh_state_name = 'New York' AND dh_place_name = 'New York' ORDER BY period_end DESC, metric_value DESC LIMIT 1) , location_2 AS ( SELECT period_end, location_name, metric_value FROM public.redfin_median_sale_price WHERE geo_type = 'zip' AND property_type = 'All Residential' AND dh_state_name = 'Florida' AND dh_place_name = 'Miami' ORDER BY period_end DESC, metric_value DESC LIMIT 1 ) SELECT l1.location_name, l1.metric_value, l2.location_name, l2.metric_value, (l1.metric_value - l2.metric_value)/ l1.metric_value AS pct_change FROM location_1 l1, location_2 l2 ", "db":"v2_real_estate"} -{"nl_question": "which are the 3 cheapest sale prices by state?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end FROM public.redfin_median_sale_price WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT location_name, metric_value, period_end FROM public.redfin_median_sale_price WHERE geo_type = 'state' AND property_type = 'All Residential' AND period_end = (SELECT period_end FROM date_range) order by metric_value LIMIT 3 ", "db":"v2_real_estate"} -{"nl_question": "What are the most popular types of properties (e.g., single-family homes, condos, apartments) in California?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end FROM public.REDFIN_INVENTORY WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT period_end, location_name, property_type, metric_value FROM public.REDFIN_INVENTORY WHERE geo_type = 'state' AND dh_state_name = 'California' AND period_end = (SELECT period_end FROM date_range) ORDER BY period_end DESC, metric_value DESC ", "db":"v2_real_estate"} -{"nl_question": "What are the current listings for single-family homes in LA?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end FROM public.REDFIN_INVENTORY WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT period_end, location_name, property_type, metric_value FROM public.REDFIN_INVENTORY WHERE geo_type = 'zip' AND dh_state_name = 'California' AND dh_place_name = 'Los Angeles' AND property_type = 'Single-family' AND period_end = (SELECT period_end FROM date_range) ORDER BY period_end DESC, metric_value DESC ", "db":"v2_real_estate"} -{"nl_question": "which is the proportion of the total number of listings in LA over California?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end FROM public.REDFIN_INVENTORY WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT msp.period_end, msp.location_name, msp.property_type, msp.metric_value, msps.metric_value FROM public.REDFIN_INVENTORY msp INNER JOIN public.REDFIN_INVENTORY msps ON msp.dh_state_name = msps.dh_state_name AND msp.property_type = msps.property_type and msp.period_end = msps.period_end AND msps.geo_type = 'state' WHERE msp.geo_type = 'city' AND msp.dh_state_name = 'California' AND msp.dh_place_name = 'Los Angeles' AND msp.property_type = 'All Residential' AND msp.period_end = (SELECT period_end FROM date_range) ", "db":"v2_real_estate"} -{"nl_question": "Which are the top 10 zips with the higher new listings numbers in New York city?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end FROM public.REDFIN_NEW_LISTINGS WHERE dh_place_name <> '-' AND property_type = 'All Residential' ) SELECT period_end, location_name, metric_value FROM public.REDFIN_NEW_LISTINGS WHERE geo_type = 'zip' AND dh_place_name = 'New York' AND property_type = 'All Residential' AND period_end = (SELECT period_end FROM date_range) ORDER BY period_end DESC, metric_value desc limit 10 ", "db":"v2_real_estate"} -{"nl_question": "how did the new listings increased/decreased in New Orleans in the last year?", "sql": " WITH date_range AS ( SELECT MAX(period_end) - INTERVAL '0 months' AS period_end, MAX(period_end) - INTERVAL '12 months' AS period_beg FROM public.REDFIN_NEW_LISTINGS WHERE dh_place_name <> '-' AND property_type = 'All Residential' ), data AS ( SELECT location_name, metric_value AS median_price, period_end FROM public.REDFIN_NEW_LISTINGS WHERE geo_type = 'city' AND property_type = 'All Residential' AND dh_state_name = 'Louisiana' AND dh_place_name = 'New Orleans' AND (period_end = (SELECT period_end FROM date_range) OR period_end = (SELECT period_beg FROM date_range)) ) SELECT cd.location_name, ((cd.median_price - ld.median_price) / ld.median_price) AS pct_change FROM data cd JOIN data ld ON cd.location_name = ld.location_name WHERE cd.period_end = (SELECT period_end FROM date_range) AND ld.period_end = (SELECT period_beg FROM date_range) ORDER BY 2 DESC LIMIT 1", "db":"v2_real_estate"} \ No newline at end of file diff --git a/apps/ai/clients/chat/.env.local.example b/apps/ai/clients/chat/.env.local.example deleted file mode 100644 index 441687a5..00000000 --- a/apps/ai/clients/chat/.env.local.example +++ /dev/null @@ -1,20 +0,0 @@ -# NEXT_PUBLIC_API_URL=http://localhost:5010/v5 -NEXT_PUBLIC_API_URL=https://dev.api.dataherald.com/v5 -NEXT_PUBLIC_EMBED_URL=https://dev.bariloche.dataherald.com/v4/viz/ - -# ANALYTICS -NEXT_PUBLIC_AMPLITUDE_API_KEY=ccd9965852524fa546121c22f740c851 - -# AUTH SDK -AUTH0_BASE_URL=http://localhost:3100 -AUTH0_ISSUER_BASE_URL=https://auth.dataherald.com -AUTH0_SCOPE=openid profile email offline_access -AUTH0_API_AUDIENCE=https://hi-george.us.auth0.com/api/v2/ -AUTH0_CLIENT_ID= -AUTH0_CLIENT_SECRET= -AUTH0_SECRET= - -# AUTH SDK - Browser -NEXT_PUBLIC_AUTH0_BASE_URL=http://localhost:3100 -NEXT_PUBLIC_AUTH0_ISSUER_BASE_URL=https://auth.dataherald.com -NEXT_PUBLIC_AUTH0_CLIENT_ID= \ No newline at end of file diff --git a/apps/ai/clients/chat/.env.production b/apps/ai/clients/chat/.env.production deleted file mode 100644 index 68dbcedc..00000000 --- a/apps/ai/clients/chat/.env.production +++ /dev/null @@ -1 +0,0 @@ -AUTH0_BASE_URL=https://$VERCEL_URL \ No newline at end of file diff --git a/apps/ai/clients/chat/.eslintrc.json b/apps/ai/clients/chat/.eslintrc.json deleted file mode 100644 index be74a3a1..00000000 --- a/apps/ai/clients/chat/.eslintrc.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "extends": [ - "next/core-web-vitals", - "eslint:recommended", - "plugin:@typescript-eslint/recommended", - "plugin:react/recommended", - "prettier" - ], - "plugins": ["react", "prettier", "@typescript-eslint"], - "rules": { - "prettier/prettier": "error", - "react/react-in-jsx-scope": "off", - "no-constant-condition": ["error", { "checkLoops": false }] - }, - "settings": { - "react": { - "version": "detect" - } - } -} diff --git a/apps/ai/clients/chat/.gitignore b/apps/ai/clients/chat/.gitignore deleted file mode 100644 index f5bb7f6d..00000000 --- a/apps/ai/clients/chat/.gitignore +++ /dev/null @@ -1,39 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -#IDE -.vscode - -# dependencies -/node_modules -/.pnp -.pnp.js - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# local env files -.env*.local - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts diff --git a/apps/ai/clients/chat/.prettierrc b/apps/ai/clients/chat/.prettierrc deleted file mode 100644 index fa9699b8..00000000 --- a/apps/ai/clients/chat/.prettierrc +++ /dev/null @@ -1,7 +0,0 @@ -{ - "semi": true, - "trailingComma": "all", - "singleQuote": true, - "printWidth": 80, - "tabWidth": 2 -} diff --git a/apps/ai/clients/chat/README.md b/apps/ai/clients/chat/README.md deleted file mode 100644 index 2c755033..00000000 --- a/apps/ai/clients/chat/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# ChatDH - -A front-end OpenAI + Dataherald visualizations chat using Next.js, TypeScript, and Tailwind CSS. - -## Running Locally - -**1. Install Dependencies** - -```bash -pnpm install -``` - -**2. Setup environment variables** - -To be able to start the project, you can follow the next steps: - -1. Remove the `.example` part of the name from the [env local file example](./.env.local.example) -2. Fill the empty secrets fields with the content from the `development` env variables of the project. You can find them [here](https://vercel.com/dataherald/chat-dh/settings/environment-variables). If you don't have access. Request help from Amir or Valak. - -> Note that the `.env.local` is personal and won't be pushed to the shared codebase. - -**3. Run App** - -```bash -pnpm dev -``` - -## Authentication - -Authentication is handled by our Auth0 client and implemented in this project using their official [NextJS SDK](https://github.com/auth0/nextjs-auth0) - -## Code Format - -Prettier is added to the projects and a check is run on the PRs to check the correct format. To fix any format issue you can run `pnpm run format` diff --git a/apps/ai/clients/chat/components/Chat/Chat.tsx b/apps/ai/clients/chat/components/Chat/Chat.tsx deleted file mode 100644 index 8e9ba39c..00000000 --- a/apps/ai/clients/chat/components/Chat/Chat.tsx +++ /dev/null @@ -1,294 +0,0 @@ -import { useChat } from '@/contexts/chat'; -import { usePrompt } from '@/contexts/prompt'; -import analyticsService from '@/services/analytics'; -import apiService from '@/services/api'; -import { Message, MessageContent, MessageLoadingStage } from '@/types/chat'; -import { isAbortError } from '@/utils/api'; -import { useUser } from '@auth0/nextjs-auth0/client'; -import { - FC, - useCallback, - useEffect, - useLayoutEffect, - useRef, - useState, -} from 'react'; -import Button from '../Layout/Button'; -import { Header } from '../Layout/Header'; -import { ChatInput } from './ChatInput'; -import { ChatKickoff } from './ChatKickoff'; -import { ChatMessage } from './ChatMessage'; -import { ChatNewMessage } from './ChatNewMessage'; - -const RESPONSE_LIST_LENGTH_WITH_VIZ = 3; - -export const Chat: FC = () => { - const { - messages, - setMessages, - loadingNewMessage, - setFetchingNewMessage, - loadingIframe, - setLoadingIframe, - setCurrentLoadingStage, - } = useChat(); - const [newAssistantResponse, setNewAssistantResponse] = - useState(null); - const { user } = useUser(); - const { prompt, setPrompt } = usePrompt(); - const messagesEndRef = useRef(null); - const chatAbortControllerRef = useRef(); - const [currentMessageContent, setCurrentMessageContent] = - useState(null); - - const scrollToBottom = useCallback(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, []); - - const sendMessage = useCallback( - async (newUserMessage: string, isExample = false) => { - const newMessage: Message = { - role: 'user', - content: newUserMessage, - }; - const updatedMessages: Message[] = [...messages, newMessage]; - const abortController = new AbortController(); - chatAbortControllerRef.current = abortController; - let rawResponse = ''; - let streamText = ''; - let viz_id = ''; - // state variable doesn't work in loops - let messageLoadingStage: MessageLoadingStage = 'viz_id'; - setCurrentLoadingStage('viz_id'); - setFetchingNewMessage(true); - setMessages( - currentMessageContent - ? [ - ...messages, - { - role: 'assistant', - content: currentMessageContent, - }, - newMessage, - ] - : updatedMessages, - ); - setCurrentMessageContent({ status: 'loading' }); - - try { - const chatResponse = await apiService.chatStream( - updatedMessages, - user?.email || '', - chatAbortControllerRef.current?.signal, - ); - const reader = chatResponse.body?.getReader(); - - if (typeof reader === 'undefined') { - throw new Error('Variable "reader" is undefined.'); - } - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - const text = new TextDecoder().decode(value); - rawResponse += text; - if (text.includes('###')) { - if (messageLoadingStage === 'viz_id') { - messageLoadingStage = 'generated_text'; - setCurrentLoadingStage('generated_text'); - const vizNtext = text.split('###'); - viz_id = vizNtext[0]; - streamText = vizNtext[1]; - } else if (messageLoadingStage === 'generated_text') { - messageLoadingStage = 'json_object'; - setCurrentLoadingStage('json_object'); - streamText = ''; - } - } else { - streamText += text; - } - if (messageLoadingStage === 'generated_text') - setCurrentMessageContent({ - status: 'successful', - generated_text: streamText, - viz_id: viz_id, - }); - } - // raw response stream structure: viz_id ### generated_text ### responseJson - const chatResponseJson = extractJSONFromRawResponse(rawResponse); - - setCurrentMessageContent({ - status: chatResponseJson['status'], - message: chatResponseJson['message'], - generated_text: chatResponseJson['generated_text'], - viz_id: chatResponseJson['viz_id'], - id: chatResponseJson['log_id'], - }); - analyticsService.buttonClick('new-user-prompt', { - prompt: newUserMessage, - status: chatResponseJson['status'], - ['response-text']: chatResponseJson['generated_text'], - ['response-viz-id']: chatResponseJson['viz_id'], - ['response-id']: chatResponseJson['log_id'], - ['is-example']: isExample, - }); - } catch (e) { - if (isAbortError(e)) { - analyticsService.buttonClick('new-user-prompt', { - prompt: newUserMessage, - status: 'canceled', - ['is-example']: isExample, - }); - } else { - analyticsService.buttonClick('new-user-prompt', { - prompt: newUserMessage, - status: 'error', - ['is-example']: isExample, - }); - setNewAssistantResponse({ - role: 'assistant', - content: { - status: 'error', - generated_text: 'Something went wrong. Please try again later.', - }, - }); - } - setCurrentMessageContent(null); - } finally { - setFetchingNewMessage(false); - } - }, - [messages, setMessages, setFetchingNewMessage, user], - ); - - const extractJSONFromRawResponse = (rawResponse: string) => { - const rawResponseList = rawResponse.split('###'); - return rawResponseList.length === RESPONSE_LIST_LENGTH_WITH_VIZ - ? JSON.parse(rawResponseList[rawResponseList.length - 1]) - : JSON.parse(rawResponseList[0]); - }; - - const newCancelRequestMessage = () => { - setNewAssistantResponse({ - role: 'assistant', - content: { - status: 'canceled', - generated_text: - currentMessageContent?.generated_text || - 'User has cancelled the request.', - }, - }); - }; - - const handleExample = useCallback( - (prompt: string) => { - sendMessage(prompt, true); - }, - [sendMessage], - ); - - const handleReset = useCallback(() => { - setMessages([]); - setCurrentMessageContent(null); - analyticsService.buttonClick('new-chat', { - 'messages-length': messages.length, - }); - }, [messages, setMessages]); - - const handleAbort = () => { - if (loadingIframe) { - // data already fetched - setLoadingIframe(false); - } - if (chatAbortControllerRef.current) { - chatAbortControllerRef.current.abort(); - } - // I have no clue why but I have to set fetchingNewMessage to false or else it returns two cancelled message - setFetchingNewMessage(false); - newCancelRequestMessage(); - setCurrentMessageContent(null); - }; - - useEffect(() => { - if (newAssistantResponse) { - setMessages((prevMessages) => { - return [...prevMessages, newAssistantResponse]; - }); - } - setNewAssistantResponse(null); - }, [newAssistantResponse, setNewAssistantResponse, setMessages]); - - useEffect(() => { - if (prompt) { - handleExample(prompt); - setPrompt(null); - } - }, [prompt, setPrompt, handleExample]); - - useLayoutEffect(() => { - const timeout = setTimeout(() => scrollToBottom(), 100); - return () => { - clearTimeout(timeout); - }; - }, [ - messages, - loadingNewMessage, - currentMessageContent, - loadingIframe, - scrollToBottom, - ]); - - return ( - <> - {!messages.length ? ( -
-
-
- -
-
- -
-
- ) : ( -
-
- {messages.map((message, index) => ( - - ))} -
-
- {currentMessageContent && ( - - )} -
-
- {loadingNewMessage ? ( - - ) : ( - - )} -
-
- -
-
- )} -
- - ); -}; diff --git a/apps/ai/clients/chat/components/Chat/ChatAssistantMessageActions.tsx b/apps/ai/clients/chat/components/Chat/ChatAssistantMessageActions.tsx deleted file mode 100644 index 4a3737a7..00000000 --- a/apps/ai/clients/chat/components/Chat/ChatAssistantMessageActions.tsx +++ /dev/null @@ -1,163 +0,0 @@ -import { EMBED_URL } from '@/env-variables'; -import analyticsService from '@/services/analytics'; -import apiService from '@/services/api'; -import { MessageContent } from '@/types/chat'; -import { FC, useEffect, useRef, useState } from 'react'; -import Button from '../Layout/Button'; -import ButtonGroup from '../Layout/ButtonGroup'; - -const getEmbedCode = (viz_id: string) => - ``; - -interface ChatAssistantMessageActionsProps { - message: MessageContent; -} - -const ChatAssistantMessageActions: FC = ({ - message: { id, message, generated_text, viz_id, status, id: chatResponseId }, -}) => { - const [textCopied, setTextCopied] = useState(false); - const [embedCopied, setEmbedCopied] = useState(false); - const [thumbsUpClicked, setThumbsUpClicked] = useState(false); - const [thumbsDownClicked, setThumbsDownClicked] = useState(false); - const textTimeoutRef = useRef | null>(null); - const embedTimeoutRef = useRef | null>(null); - - useEffect(() => { - return () => { - if (textTimeoutRef.current) clearTimeout(textTimeoutRef.current); - if (embedTimeoutRef.current) clearTimeout(embedTimeoutRef.current); - }; - }, []); - - const copy = async (valueToCopy: string) => { - await navigator.clipboard.writeText(valueToCopy); - }; - - const handleCopyText = async () => { - await copy(generated_text as string); - setTextCopied(true); - if (textTimeoutRef.current) clearTimeout(textTimeoutRef.current); - textTimeoutRef.current = setTimeout(() => setTextCopied(false), 750); - analyticsService.buttonClick('copy-response-text', { - prompt: message, - ['response-text']: generated_text, - ['response-id']: id, - response: generated_text, - }); - }; - - const handleCopyEmbed = async () => { - if (!viz_id) return; - await copy(getEmbedCode(viz_id)); - setEmbedCopied(true); - if (embedTimeoutRef.current) clearTimeout(embedTimeoutRef.current); - embedTimeoutRef.current = setTimeout(() => setEmbedCopied(false), 750); - analyticsService.buttonClick('copy-response-embed', { - prompt: message, - ['response-viz-id']: viz_id, - ['response-id']: id, - }); - }; - - const handleThumbsUpClick = async () => { - try { - apiService.feedback(chatResponseId as string, true); - setThumbsUpClicked(true); - setThumbsDownClicked(false); - analyticsService.buttonClick('user-feedback', { - prompt: message, - ['good-response']: true, - ['response-text']: generated_text, - ['viz-id']: viz_id, - ['response-id']: id, - }); - } catch (e) { - console.error(e); - throw e; - } - }; - - const handleThumbsDownClick = async () => { - try { - apiService.feedback(chatResponseId as string, false); - setThumbsUpClicked(false); - setThumbsDownClicked(true); - analyticsService.buttonClick('user-feedback', { - prompt: message, - ['good-response']: false, - ['response-text']: generated_text, - ['viz-id']: viz_id, - ['response-id']: id, - }); - } catch (e) { - console.error(e); - throw e; - } - }; - - return ( -
- {status === 'successful' && ( - <> - - {!thumbsDownClicked && ( - - {viz_id && ( - - )} - - - )} -
- ); -}; - -export default ChatAssistantMessageActions; diff --git a/apps/ai/clients/chat/components/Chat/ChatInput.tsx b/apps/ai/clients/chat/components/Chat/ChatInput.tsx deleted file mode 100644 index dabe67b1..00000000 --- a/apps/ai/clients/chat/components/Chat/ChatInput.tsx +++ /dev/null @@ -1,89 +0,0 @@ -import { useChat } from '@/contexts/chat'; -import { FC, KeyboardEvent, useRef, useState } from 'react'; -import { Icon } from '../Layout/Icon'; - -const VALIDATIONS = { - MAX_CHARACTERS: 1000, -}; - -interface ChatInputProps { - onSend: (content: string) => void; -} - -export const ChatInput: FC = ({ onSend }) => { - const [content, setContent] = useState(''); - const [inputError, setInputError] = useState(null); - const { loadingNewMessage, messages } = useChat(); - const emptyMessages = messages.length === 0; - - const textareaRef = useRef(null); - - const handleChange = (e: React.ChangeEvent) => { - const value = e.target.value; - setContent(value); - - if (value.length > VALIDATIONS.MAX_CHARACTERS) { - setInputError( - `Message limit is ${VALIDATIONS.MAX_CHARACTERS} characters`, - ); - } else { - setInputError(null); - } - }; - - const handleSendUserMessage = () => { - if (!content) { - return; - } - onSend(content); - setContent(''); - }; - - const handleKeyDown = (e: KeyboardEvent) => { - if (e.key === 'Enter' && !e.shiftKey) { - e.preventDefault(); - if (!inputError) handleSendUserMessage(); - } - }; - - return ( -
-