From 564139d616ae4fda961efbe754030ba437e79acc Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 11 Nov 2024 22:06:38 +0800 Subject: [PATCH 01/77] ci(github-actions): add uv to news-fragment action (#43878) --- .github/workflows/news-fragment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/news-fragment.yml b/.github/workflows/news-fragment.yml index 31b0f51b1f25c..bf1bd6ce27b2b 100644 --- a/.github/workflows/news-fragment.yml +++ b/.github/workflows/news-fragment.yml @@ -38,6 +38,7 @@ jobs: - name: Check news fragment run: > + python -m pip install --upgrade uv && uv tool run towncrier check --dir . --config newsfragments/config.toml From bec090ceb8547df5564398066e73880875d00569 Mon Sep 17 00:00:00 2001 From: Bugra Ozturk Date: Mon, 11 Nov 2024 15:16:36 +0100 Subject: [PATCH 02/77] Include unit test to Legacy UI/API Selective Check in Breeze (#43719) * Include unit test to Legacy UI/API selective check * Apply the same naming for tests --- dev/breeze/tests/test_selective_checks.py | 118 ++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index d67508eafefba..14df82562e280 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -51,6 +51,10 @@ # commit that is neutral - allows to keep pyproject.toml-changing PRS neutral for unit tests NEUTRAL_COMMIT = "938f0c1f3cc4cbe867123ee8aa9f290f9f18100a" +# for is_legacy_ui_api_labeled tests +LEGACY_UI_LABEL = "legacy ui" +LEGACY_API_LABEL = "legacy api" + def escape_ansi_colors(line): return ANSI_COLORS_MATCHER.sub("", line) @@ -2418,3 +2422,117 @@ def test_pr_labels( pr_labels=pr_labels, ) assert_outputs_are_printed(expected_outputs, str(stderr)) + + +@pytest.mark.parametrize( + "files, pr_labels, github_event, expected_label", + [ + pytest.param( + ("airflow/www/package.json",), + (), + GithubEvents.PULL_REQUEST, + LEGACY_UI_LABEL, + id="Legacy UI file without label", + ), + pytest.param( + ("airflow/api_connexion/endpoints/health_endpoint.py", "airflow/www/package.json"), + (LEGACY_UI_LABEL,), + GithubEvents.PULL_REQUEST, + LEGACY_API_LABEL, + id="Legacy API and UI files without one of the labels API missing", + ), + pytest.param( + ("airflow/api_connexion/endpoints/health_endpoint.py",), + (), + GithubEvents.PULL_REQUEST, + LEGACY_API_LABEL, + id="Legacy API file without label", + ), + pytest.param( + ("airflow/api_connexion/endpoints/health_endpoint.py", "airflow/www/package.json"), + (LEGACY_API_LABEL,), + GithubEvents.PULL_REQUEST, + LEGACY_UI_LABEL, + id="Legacy API and UI files without one of the labels UI missing", + ), + ], +) +def test_is_legacy_ui_api_labeled_should_fail( + files: tuple[str, ...], pr_labels: tuple[str, ...], github_event: GithubEvents, expected_label: str +): + try: + stdout = SelectiveChecks( + files=files, + commit_ref=NEUTRAL_COMMIT, + github_event=github_event, + pr_labels=pr_labels, + default_branch="main", + ) + except SystemExit: + assert ( + f"[error]Please ask maintainer to assign the '{expected_label}' label to the PR in order to continue" + in escape_ansi_colors(str(stdout)) + ) + + +@pytest.mark.parametrize( + "files, pr_labels, github_event, expected_label", + [ + pytest.param( + ("airflow/www/package.json",), + (LEGACY_UI_LABEL,), + GithubEvents.PULL_REQUEST, + LEGACY_UI_LABEL, + id="Legacy UI file with label", + ), + pytest.param( + ("airflow/api_connexion/endpoints/health_endpoint.py",), + (LEGACY_API_LABEL,), + GithubEvents.PULL_REQUEST, + LEGACY_API_LABEL, + id="Legacy API file with label", + ), + pytest.param( + ("airflow/api_connexion/endpoints/health_endpoint.py",), + (), + GithubEvents.SCHEDULE, + LEGACY_API_LABEL, + id="Legacy API file in canary schedule", + ), + pytest.param( + ("airflow/www/package.json",), + (LEGACY_UI_LABEL,), + GithubEvents.SCHEDULE, + LEGACY_API_LABEL, + id="Legacy UI file in canary schedule", + ), + pytest.param( + ("airflow/api_connexion/endpoints/health_endpoint.py",), + (LEGACY_API_LABEL,), + GithubEvents.PUSH, + LEGACY_API_LABEL, + id="Legacy API file in canary push", + ), + pytest.param( + ("airflow/www/package.json",), + (LEGACY_UI_LABEL,), + GithubEvents.PUSH, + LEGACY_UI_LABEL, + id="Legacy UI file in canary push", + ), + ], +) +def test_is_legacy_ui_api_labeled_should_not_fail( + files: tuple[str, ...], pr_labels: tuple[str, ...], github_event: GithubEvents, expected_label: str +): + stdout = SelectiveChecks( + files=files, + commit_ref=NEUTRAL_COMMIT, + github_event=github_event, + pr_labels=pr_labels, + default_branch="main", + ) + assert ( + f"[error]Please ask maintainer to assign the '{expected_label}' label to the PR in order to continue" + not in escape_ansi_colors(str(stdout)) + ) From ddbdf2e3537b9a2f5f38b0dc25448885c22aa527 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Mon, 11 Nov 2024 20:33:26 +0530 Subject: [PATCH 03/77] Add global events page to browse along with support to display only events for the dag. (#43793) * Add global events page to browse along with support to display only events for the dag. * Move column definition to a separate function outside of component due to eslint rule. --- airflow/api_fastapi/common/parameters.py | 8 +- .../core_api/routes/public/event_logs.py | 1 + airflow/ui/src/layouts/Nav/BrowseButton.tsx | 48 ++++++ airflow/ui/src/layouts/Nav/Nav.tsx | 16 +- airflow/ui/src/pages/Events/Events.tsx | 143 ++++++++++++++++++ airflow/ui/src/pages/Events/index.tsx | 20 +++ airflow/ui/src/router.tsx | 7 +- 7 files changed, 226 insertions(+), 17 deletions(-) create mode 100644 airflow/ui/src/layouts/Nav/BrowseButton.tsx create mode 100644 airflow/ui/src/pages/Events/Events.tsx create mode 100644 airflow/ui/src/pages/Events/index.tsx diff --git a/airflow/api_fastapi/common/parameters.py b/airflow/api_fastapi/common/parameters.py index 18630e473d778..1c701b26c7363 100644 --- a/airflow/api_fastapi/common/parameters.py +++ b/airflow/api_fastapi/common/parameters.py @@ -177,13 +177,12 @@ class SortParam(BaseParam[str]): } def __init__( - self, - allowed_attrs: list[str], - model: Base, + self, allowed_attrs: list[str], model: Base, to_replace: dict[str, str] | None = None ) -> None: super().__init__() self.allowed_attrs = allowed_attrs self.model = model + self.to_replace = to_replace def to_orm(self, select: Select) -> Select: if self.skip_none is False: @@ -193,6 +192,9 @@ def to_orm(self, select: Select) -> Select: return select lstriped_orderby = self.value.lstrip("-") + if self.to_replace: + lstriped_orderby = self.to_replace.get(lstriped_orderby, lstriped_orderby) + if self.allowed_attrs and lstriped_orderby not in self.allowed_attrs: raise HTTPException( 400, diff --git a/airflow/api_fastapi/core_api/routes/public/event_logs.py b/airflow/api_fastapi/core_api/routes/public/event_logs.py index 1e63167c00da0..007da65f28cb5 100644 --- a/airflow/api_fastapi/core_api/routes/public/event_logs.py +++ b/airflow/api_fastapi/core_api/routes/public/event_logs.py @@ -86,6 +86,7 @@ def get_event_logs( "extra", ], Log, + to_replace={"when": "dttm", "event_log_id": "id"}, ).dynamic_depends() ), ], diff --git a/airflow/ui/src/layouts/Nav/BrowseButton.tsx b/airflow/ui/src/layouts/Nav/BrowseButton.tsx new file mode 100644 index 0000000000000..37e16a00332e6 --- /dev/null +++ b/airflow/ui/src/layouts/Nav/BrowseButton.tsx @@ -0,0 +1,48 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Link } from "@chakra-ui/react"; +import { FiGlobe } from "react-icons/fi"; + +import { Menu } from "src/components/ui"; + +import { NavButton } from "./NavButton"; + +const links = [ + { + href: `/webapp/events`, + title: "Events", + }, +]; + +export const BrowseButton = () => ( + + + } title="Browse" /> + + + {links.map((link) => ( + + + {link.title} + + + ))} + + +); diff --git a/airflow/ui/src/layouts/Nav/Nav.tsx b/airflow/ui/src/layouts/Nav/Nav.tsx index 4876f83303fc4..fb3559af97094 100644 --- a/airflow/ui/src/layouts/Nav/Nav.tsx +++ b/airflow/ui/src/layouts/Nav/Nav.tsx @@ -17,17 +17,12 @@ * under the License. */ import { Box, Flex, VStack } from "@chakra-ui/react"; -import { - FiCornerUpLeft, - FiDatabase, - FiGlobe, - FiHome, - FiSettings, -} from "react-icons/fi"; +import { FiCornerUpLeft, FiDatabase, FiHome, FiSettings } from "react-icons/fi"; import { AirflowPin } from "src/assets/AirflowPin"; import { DagIcon } from "src/assets/DagIcon"; +import { BrowseButton } from "./BrowseButton"; import { DocsButton } from "./DocsButton"; import { NavButton } from "./NavButton"; import { UserSettingsButton } from "./UserSettingsButton"; @@ -61,12 +56,7 @@ export const Nav = () => ( title="Assets" to="assets" /> - } - title="Browse" - to="browse" - /> + } diff --git a/airflow/ui/src/pages/Events/Events.tsx b/airflow/ui/src/pages/Events/Events.tsx new file mode 100644 index 0000000000000..028a388894edc --- /dev/null +++ b/airflow/ui/src/pages/Events/Events.tsx @@ -0,0 +1,143 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box } from "@chakra-ui/react"; +import type { ColumnDef } from "@tanstack/react-table"; +import { useParams } from "react-router-dom"; + +import { useEventLogServiceGetEventLogs } from "openapi/queries"; +import type { EventLogResponse } from "openapi/requests/types.gen"; +import { DataTable } from "src/components/DataTable"; +import { useTableURLState } from "src/components/DataTable/useTableUrlState"; +import { ErrorAlert } from "src/components/ErrorAlert"; +import Time from "src/components/Time"; + +const eventsColumn = ( + dagId: string | undefined, +): Array> => [ + { + accessorKey: "when", + cell: ({ row: { original } }) =>