diff --git a/harp/typing/storage.py b/harp/typing/storage.py index af169b0f..a91731b5 100644 --- a/harp/typing/storage.py +++ b/harp/typing/storage.py @@ -18,6 +18,7 @@ async def get_transaction_list( filters=None, page: int = 1, cursor: str = "", + text_search: str = "", ): """Find transactions, using optional filters, for example to be displayed in the dashboard.""" ... @@ -59,10 +60,6 @@ async def create_users_once_ready(self, users: Iterable[str]): """Create users.""" ... - async def set_transaction_tags(self, transaction_or_id, tags: dict, /): - """Set transaction tags.""" - ... - async def get_usage(self): """Get storage usage.""" ... diff --git a/harp/utils/dates.py b/harp/utils/dates.py index 828a63bc..1ab338af 100644 --- a/harp/utils/dates.py +++ b/harp/utils/dates.py @@ -19,4 +19,7 @@ def ensure_datetime(x, tz=None) -> Optional[datetime]: return x.replace(tzinfo=tz) if isinstance(x, date): return datetime.combine(x, datetime.min.time(), tzinfo=tz) - return datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=tz) + try: + return datetime.strptime(x, "%Y-%m-%d %H:%M:%S").replace(tzinfo=tz) + except ValueError: + return datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=tz) diff --git a/harp_apps/dashboard/controllers/transactions.py b/harp_apps/dashboard/controllers/transactions.py index e857f86d..8a437fcf 100644 --- a/harp_apps/dashboard/controllers/transactions.py +++ b/harp_apps/dashboard/controllers/transactions.py @@ -68,6 +68,7 @@ async def list(self, request: HttpRequest): page=page, cursor=cursor, username=request.context.get("user") or "anonymous", + text_search=request.query.get("search", ""), ) return json( diff --git a/harp_apps/dashboard/frontend/src/Components/Page/PageTitle.tsx b/harp_apps/dashboard/frontend/src/Components/Page/PageTitle.tsx index bd6f1341..0819bdb1 100644 --- a/harp_apps/dashboard/frontend/src/Components/Page/PageTitle.tsx +++ b/harp_apps/dashboard/frontend/src/Components/Page/PageTitle.tsx @@ -12,10 +12,12 @@ export function PageTitle({ description, title, children }: PageTitleProps) { return ( <> {title ? ( -
- {children ?
{children}
: null} -

{title}

- {description ?

{description}

: null} +
+
+

{title}

+ {description ?

{description}

: null} +
+ {children ? <>{children} : null}
) : null} diff --git a/harp_apps/dashboard/frontend/src/Components/Page/__snapshots__/Page.test.tsx.snap b/harp_apps/dashboard/frontend/src/Components/Page/__snapshots__/Page.test.tsx.snap index 295c41a2..ded93fd1 100644 --- a/harp_apps/dashboard/frontend/src/Components/Page/__snapshots__/Page.test.tsx.snap +++ b/harp_apps/dashboard/frontend/src/Components/Page/__snapshots__/Page.test.tsx.snap @@ -4,18 +4,22 @@ exports[`Page > renders without error 1`] = `
-

- Test Title -

-

- Test Description -

+

+ Test Title +

+

+ Test Description +

+
| { page: number; cursor?: string }) { +function getQueryStringFromRecord( + filters: Record | { page: number; cursor?: string; search?: string }, +) { const searchParams = new URLSearchParams() for (const [key, value] of Object.entries(filters) as [string, string | number | undefined][]) { @@ -20,13 +22,17 @@ export function useTransactionsListQuery({ page = 1, cursor = undefined, filters = undefined, + search = undefined, }: { filters?: Filters page?: number cursor?: string + search?: string }) { const api = useApi() - const qs = filters ? getQueryStringFromRecord({ ...filters, page, cursor: page == 1 ? undefined : cursor }) : "" + const qs = filters + ? getQueryStringFromRecord({ ...filters, page, cursor: page == 1 ? undefined : cursor, search }) + : "" return useQuery & { total: number; pages: number; perPage: number }>( ["transactions", qs], diff --git a/harp_apps/dashboard/frontend/src/Pages/Overview/__snapshots__/OverviewPage.test.tsx.snap b/harp_apps/dashboard/frontend/src/Pages/Overview/__snapshots__/OverviewPage.test.tsx.snap index f3b962c3..dd498f52 100644 --- a/harp_apps/dashboard/frontend/src/Pages/Overview/__snapshots__/OverviewPage.test.tsx.snap +++ b/harp_apps/dashboard/frontend/src/Pages/Overview/__snapshots__/OverviewPage.test.tsx.snap @@ -4,13 +4,17 @@ exports[`renders the title and data when the query is successful 1`] = `
-

- Overview -

+

+ Overview +

+
renders the title and data when the query is successful 1`
-

- System -

-

- Informations about the running instance. -

+

+ System +

+

+ Informations about the running instance. +

+
({}) const [page, setPage] = useState(1) const [cursor, setCursor] = useState(undefined) - const query = useTransactionsListQuery({ filters, page, cursor }) + const [search, setSearch] = useState(undefined) + const query = useTransactionsListQuery({ filters, page, cursor, search }) useEffect(() => { if (page == 1 && query.isSuccess && query.data.items.length) { @@ -25,18 +27,28 @@ export function TransactionListPage() { - {query.isSuccess ? ( - + - ) : null} + {query.isSuccess ? ( + + ) : ( +
+ )} +
} > + {/* */} {(query) => } diff --git a/harp_apps/dashboard/frontend/src/Pages/Transactions/__snapshots__/TransactionListPage.test.tsx.snap b/harp_apps/dashboard/frontend/src/Pages/Transactions/__snapshots__/TransactionListPage.test.tsx.snap index dbd61b3a..25f67d93 100644 --- a/harp_apps/dashboard/frontend/src/Pages/Transactions/__snapshots__/TransactionListPage.test.tsx.snap +++ b/harp_apps/dashboard/frontend/src/Pages/Transactions/__snapshots__/TransactionListPage.test.tsx.snap @@ -4,21 +4,51 @@ exports[`renders well when the query is successful 1`] = `
-

- Transactions -

-

+ Transactions + +

+ Explore transactions that went through the proxy +

+
+
- Explore transactions that went through the proxy -

+
+
+ +
+ + Enter + +
+
+
+
{ + return ( + <> + + + ) +} diff --git a/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/SearchBar.test.tsx b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/SearchBar.test.tsx new file mode 100644 index 00000000..49d184c2 --- /dev/null +++ b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/SearchBar.test.tsx @@ -0,0 +1,11 @@ +import { render } from "@testing-library/react" +import { expect, describe, it } from "vitest" + +import { SearchBar } from "./SearchBar" + +describe("SearchBar", () => { + it("renders correctly", () => { + const { container } = render() + expect(container).toMatchSnapshot() + }) +}) diff --git a/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/SearchBar.tsx b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/SearchBar.tsx new file mode 100644 index 00000000..83a9af37 --- /dev/null +++ b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/SearchBar.tsx @@ -0,0 +1,55 @@ +import { useRef } from "react" + +interface SearchBarProps { + label?: string + placeHolder?: string + setSearch?: (value: string) => void + className?: string +} + +export const SearchBar = ({ label, setSearch, className, placeHolder }: SearchBarProps) => { + const inputRef = useRef(null) + + const handleKeyPress = (e: React.KeyboardEvent) => { + if (e.key === "Enter") { + if (setSearch) { + setSearch(e.currentTarget.value) + } + } + } + + const handleSearchClick = () => { + if (inputRef.current) { + if (setSearch) { + setSearch(inputRef.current.value) + } + } + } + + return ( +
+ {label && ( + + )} +
+ +
+ + Enter + +
+
+
+ ) +} diff --git a/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/__snapshots__/SearchBar.test.tsx.snap b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/__snapshots__/SearchBar.test.tsx.snap new file mode 100644 index 00000000..a4f75455 --- /dev/null +++ b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/__snapshots__/SearchBar.test.tsx.snap @@ -0,0 +1,28 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`SearchBar > renders correctly 1`] = ` +
+
+
+ +
+ + Enter + +
+
+
+
+`; diff --git a/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/index.ts b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/index.ts new file mode 100644 index 00000000..adb0436d --- /dev/null +++ b/harp_apps/dashboard/frontend/src/ui/Components/SearchBar/index.ts @@ -0,0 +1 @@ +export { SearchBar } from "./SearchBar" diff --git a/harp_apps/dashboard/frontend/src/ui/tests/snapshot.spec.ts-snapshots/search-bar--default-darwin.png b/harp_apps/dashboard/frontend/src/ui/tests/snapshot.spec.ts-snapshots/search-bar--default-darwin.png new file mode 100644 index 00000000..151eaa55 Binary files /dev/null and b/harp_apps/dashboard/frontend/src/ui/tests/snapshot.spec.ts-snapshots/search-bar--default-darwin.png differ diff --git a/harp_apps/sqlalchemy_storage/models/base.py b/harp_apps/sqlalchemy_storage/models/base.py index 329d1cd0..7008c5f7 100644 --- a/harp_apps/sqlalchemy_storage/models/base.py +++ b/harp_apps/sqlalchemy_storage/models/base.py @@ -16,7 +16,7 @@ def with_session(f): @wraps(f) async def contextualized(self, *args, session=None, **kwargs): if session is None: - async with (session or self.session_factory()) as session: + async with session or self.session_factory() as session: return await f(self, *args, session=session, **kwargs) return await f(self, *args, session=session, **kwargs) @@ -56,7 +56,7 @@ async def find_one(self, values: dict, /, session, **select_kwargs) -> TRow: ) @with_session - async def find_one_by_id(self, id: str, /, session, **select_kwargs) -> TRow: + async def find_one_by_id(self, id: str, /, session=None, **select_kwargs) -> TRow: return await self.find_one({"id": id}, session=session, **select_kwargs) @with_session diff --git a/harp_apps/sqlalchemy_storage/models/messages.py b/harp_apps/sqlalchemy_storage/models/messages.py index 95799c6a..4caf7d9b 100644 --- a/harp_apps/sqlalchemy_storage/models/messages.py +++ b/harp_apps/sqlalchemy_storage/models/messages.py @@ -1,7 +1,7 @@ from datetime import UTC from typing import TYPE_CHECKING -from sqlalchemy import DateTime, ForeignKey, Integer, String +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String from sqlalchemy.orm import Mapped, mapped_column, relationship from harp.http import get_serializer_for @@ -26,6 +26,11 @@ class Message(Base): transaction_id = mapped_column(ForeignKey("sa_transactions.id", ondelete="CASCADE")) transaction: Mapped["Transaction"] = relationship(back_populates="messages") + # add a GIN index on the summary column + summary_index = Index( + "summary_gin_index", summary, postgresql_using="gin", postgresql_ops={"summary": "gin_trgm_ops"} + ) + def to_model(self): return MessageModel( id=self.id, diff --git a/harp_apps/sqlalchemy_storage/models/transactions.py b/harp_apps/sqlalchemy_storage/models/transactions.py index 53a7b38d..726c4c0c 100644 --- a/harp_apps/sqlalchemy_storage/models/transactions.py +++ b/harp_apps/sqlalchemy_storage/models/transactions.py @@ -1,7 +1,7 @@ from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, List -from sqlalchemy import Column, DateTime, Float, ForeignKey, String, Table, exists, insert +from sqlalchemy import Column, DateTime, Float, ForeignKey, Index, String, Table, insert, exists from sqlalchemy.orm import Mapped, joinedload, mapped_column, relationship, selectinload from harp.models.transactions import Transaction as TransactionModel @@ -49,6 +49,10 @@ class Transaction(Base): cascade="all, delete", passive_deletes=True, ) + # Add a GIN index on the endpoint column + endpoint_index = Index( + "endpoint_gin_index", endpoint, postgresql_using="gin", postgresql_ops={"endpoint": "gin_trgm_ops"} + ) def to_model(self, with_user_flags=False): return TransactionModel( @@ -122,7 +126,7 @@ def delete_old(self, old_after: timedelta): return self.delete().where((self.Type.started_at < threshold) & no_flags) @with_session - async def create(self, values: dict | TransactionModel, /, *, session): + async def create(self, values: dict | TransactionModel, /, *, session=None): # convert model to dict if isinstance(values, TransactionModel): # todo in to_dict method ? but how to keep prototype of parent ? @@ -141,7 +145,7 @@ async def create(self, values: dict | TransactionModel, /, *, session): return transaction @with_session - async def set_tags(self, transaction: Transaction, tags: dict, /, *, session): + async def set_tags(self, transaction: Transaction, tags: dict, /, *, session=None): if not self.tags: raise ValueError("Tags repository is not available.") if not self.tag_values: diff --git a/harp_apps/sqlalchemy_storage/storage.py b/harp_apps/sqlalchemy_storage/storage.py index 94a1d1c0..77bd47c1 100644 --- a/harp_apps/sqlalchemy_storage/storage.py +++ b/harp_apps/sqlalchemy_storage/storage.py @@ -1,10 +1,12 @@ import asyncio +import re from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from typing import Iterable, List, Optional, TypedDict, override -from sqlalchemy import case, delete, func, literal, select, update -from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine +from sqlalchemy import bindparam, case, delete, func, literal, literal_column, select, text, update +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, async_sessionmaker, create_async_engine from sqlalchemy.sql.functions import count from whistle import IAsyncEventDispatcher @@ -82,6 +84,27 @@ def _filter_query_for_user_flags(query, values, /, *, user_id): return query +def _filter_transactions_based_on_text(query, search_text: str, dialect_name: str): + # Escape special characters in search_text + search_text = re.sub(r"([-\*\(\)~\"@<>\^+]+)", r"", search_text) + query = query.join(Message) + # check dialect and use appropriate full text search + if dialect_name == "mysql": + return query.filter( + text( + f"MATCH ({Transaction.__tablename__}.endpoint) " + f"AGAINST (:search_text IN BOOLEAN MODE) OR " + f"MATCH ({Message.__tablename__}.summary) " + f"AGAINST (:search_text IN BOOLEAN MODE)", + ).bindparams(bindparam("search_text", literal_column(f"'{search_text}*'"))) + ) + + return query.filter( + (Transaction.endpoint.ilike(bindparam("search_text", f"%{search_text}%"))) + | Message.summary.ilike(bindparam("search_text", f"%{search_text}%")) + ) + + class SqlAlchemyStorage(Storage): """ Storage implementation using SQL Alchemy Core, with async drivers. @@ -128,7 +151,9 @@ async def initialize(self, /, *, force_reset=False): async with self.engine.begin() as conn: if force_reset or self.settings.drop_tables: await conn.run_sync(self.metadata.drop_all) + await self.install_pg_trgm_extension(conn) await conn.run_sync(self.metadata.create_all) + await self.create_full_text_indexes(conn) self._is_ready.set() @property @@ -161,6 +186,7 @@ async def get_transaction_list( filters=None, page: int = 1, cursor: str = "", + text_search="", ): """ Implements :meth:`Storage.find_transactions `. @@ -182,6 +208,9 @@ async def get_transaction_list( query = _filter_query(query, "status", filters.get("status", None)) query = _filter_query_for_user_flags(query, filters.get("flag", None), user_id=user.id) + if text_search: + query = _filter_transactions_based_on_text(query, text_search, dialect_name=self.engine.dialect.name) + query = query.order_by(Transaction.started_at.desc()) # apply cursor (before count) @@ -257,7 +286,7 @@ async def transactions_grouped_by_time_bucket( { "datetime": ensure_datetime(row[0], UTC), "count": row[1], - "errors": row[2], + "errors": int(row[2]), "meanDuration": row[3] if row[3] else 0, # ! probably sqlite struggling with unfinished transactions } @@ -397,3 +426,28 @@ async def create_users(self, users: Iterable[str]): user = User() user.username = username session.add(user) + + async def install_pg_trgm_extension(self, conn: AsyncConnection): + # Check the type of the current database + if conn.engine.dialect.name == "postgresql": + # Install the pg_trgm extension + await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;")) + + async def create_full_text_indexes(self, conn: AsyncConnection): + # Check the type of the current database + if conn.engine.dialect.name == "mysql": + # Create the full text index for transactions.endpoint + try: + await conn.execute( + text(f"CREATE FULLTEXT INDEX endpoint_ft_index ON {Transaction.__tablename__} (endpoint);") + ) + # Create the full text index for messages.summary + await conn.execute( + text(f"CREATE FULLTEXT INDEX summary_ft_index ON {Message.__tablename__} (summary);") + ) + except OperationalError as e: + # check for duplicate key error + if e.orig and e.orig.args[0] == 1061: + pass + else: + raise e diff --git a/harp_apps/sqlalchemy_storage/tests/test_storage_transactions.py b/harp_apps/sqlalchemy_storage/tests/test_storage_transactions.py new file mode 100644 index 00000000..1051a2c1 --- /dev/null +++ b/harp_apps/sqlalchemy_storage/tests/test_storage_transactions.py @@ -0,0 +1,30 @@ +from harp_apps.sqlalchemy_storage.storage import SqlAlchemyStorage +from harp_apps.sqlalchemy_storage.utils.testing.mixins import SqlalchemyStorageTestFixtureMixin + + +class TestStorageTransactions(SqlalchemyStorageTestFixtureMixin): + async def test_get_transaction_list_with_tags(self, storage: SqlAlchemyStorage): + t1 = await self.create_transaction(storage, endpoint="foo") + + t2 = await self.create_transaction(storage, endpoint="bar") + + t3 = await self.create_transaction(storage, endpoint="baz") + + # messages + await self.create_message(storage, transaction_id=t1.id, kind="misc", summary="bal", headers="foo", body="foo") + await self.create_message(storage, transaction_id=t2.id, kind="misc", summary="foo", headers="bar", body="baz") + await self.create_message(storage, transaction_id=t3.id, kind="misc", summary="baz", headers="baz", body="baz") + + # assert stuff + transactions_bar = await storage.get_transaction_list( + username="anonymous", with_messages=True, text_search="bar" + ) + assert len(transactions_bar) == 1 + + assert transactions_bar[0].id == t2.id + + transactions_fo = await storage.get_transaction_list(username="anonymous", with_messages=True, text_search="fo") + assert len(transactions_fo) == 2 + + transactions_ba = await storage.get_transaction_list(username="anonymous", with_messages=True, text_search="ba") + assert len(transactions_ba) == 3 diff --git a/harp_apps/sqlalchemy_storage/utils/dates.py b/harp_apps/sqlalchemy_storage/utils/dates.py index 94ca477d..63749a95 100644 --- a/harp_apps/sqlalchemy_storage/utils/dates.py +++ b/harp_apps/sqlalchemy_storage/utils/dates.py @@ -24,6 +24,32 @@ def compile_trunc_postgresql(element, compiler, **kw): return compiler.process(func.date_trunc(element.precision, element.expr)) +@compiles(TruncDatetime, "mysql") +def compile_trunc_mysql(element, compiler, **kw): + try: + precision = element.precision.effective_value + except AttributeError: + precision = element.precision + expr = element.expr + + if precision == "year": + format_str = "%Y-01-01 00:00:00" + elif precision == "month": + format_str = "%Y-%m-01 00:00:00" + elif precision == "day": + format_str = "%Y-%m-%d 00:00:00" + elif precision == "hour": + format_str = "%Y-%m-%d %H:00:00" + elif precision == "minute": + format_str = "%Y-%m-%d %H:%i:00" + elif precision == "second": + format_str = "%Y-%m-%d %H:%i:%s" + else: + raise NotImplementedError(f"Truncating {precision} is not supported for MySQL") + + return compiler.process(func.date_format(expr, format_str)) + + _modifiers = { "year": ("start of year",), "month": ("start of month",),