Skip to content

Commit

Permalink
Add Rendered k8s pod spec tab to ti details view (#39141)
Browse files Browse the repository at this point in the history
* Add Rendered k8s pod spec tab to ti details view

* Render yaml instead of json

* Fix rebase mistake
  • Loading branch information
bbovenzi committed May 2, 2024
1 parent 959e52b commit 1074b8e
Show file tree
Hide file tree
Showing 9 changed files with 196 additions and 14 deletions.
4 changes: 3 additions & 1 deletion airflow/www/package.json
Expand Up @@ -51,6 +51,7 @@
"@testing-library/jest-dom": "^5.16.0",
"@testing-library/react": "^13.0.0",
"@types/color": "^3.0.3",
"@types/json-to-pretty-yaml": "^1.2.1",
"@types/react": "^18.0.12",
"@types/react-dom": "^18.0.5",
"@types/react-syntax-highlighter": "^15.5.6",
Expand Down Expand Up @@ -126,11 +127,12 @@
"framer-motion": "^6.0.0",
"jquery": ">=3.5.0",
"jshint": "^2.13.4",
"json-to-pretty-yaml": "^1.2.2",
"lodash": "^4.17.21",
"moment-timezone": "^0.5.43",
"react": "^18.0.0",
"react-dom": "^18.0.0",
"react-icons": "^4.9.0",
"react-icons": "^5.1.0",
"react-json-view": "^1.21.3",
"react-markdown": "^8.0.4",
"react-query": "^3.39.1",
Expand Down
2 changes: 2 additions & 0 deletions airflow/www/static/js/api/index.ts
Expand Up @@ -53,6 +53,7 @@ import { useTaskXcomEntry, useTaskXcomCollection } from "./useTaskXcom";
import useEventLogs from "./useEventLogs";
import useCalendarData from "./useCalendarData";
import useCreateDatasetEvent from "./useCreateDatasetEvent";
import useRenderedK8s from "./useRenderedK8s";

axios.interceptors.request.use((config) => {
config.paramsSerializer = {
Expand Down Expand Up @@ -102,4 +103,5 @@ export {
useEventLogs,
useCalendarData,
useCreateDatasetEvent,
useRenderedK8s,
};
43 changes: 43 additions & 0 deletions airflow/www/static/js/api/useRenderedK8s.ts
@@ -0,0 +1,43 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import axios, { AxiosResponse } from "axios";
import { useQuery } from "react-query";

import { getMetaValue } from "src/utils";

const url = getMetaValue("rendered_k8s_data_url");

const useRenderedK8s = (
runId: string | null,
taskId: string | null,
mapIndex?: number
) =>
useQuery(
["rendered_k8s", runId, taskId, mapIndex],
async () =>
axios.get<AxiosResponse, any>(url, {
params: { run_id: runId, task_id: taskId, map_index: mapIndex },
}),
{
enabled: !!runId && !!taskId,
}
);

export default useRenderedK8s;
23 changes: 22 additions & 1 deletion airflow/www/static/js/dag/details/index.tsx
Expand Up @@ -45,7 +45,7 @@ import {
MdPlagiarism,
MdEvent,
} from "react-icons/md";
import { BiBracket } from "react-icons/bi";
import { BiBracket, BiLogoKubernetes } from "react-icons/bi";
import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";

import Header from "./Header";
Expand All @@ -68,6 +68,7 @@ import TaskDetails from "./task";
import AuditLog from "./AuditLog";
import RunDuration from "./dag/RunDuration";
import Calendar from "./dag/Calendar";
import RenderedK8s from "./taskInstance/RenderedK8s";

const dagId = getMetaValue("dag_id")!;

Expand All @@ -79,6 +80,8 @@ interface Props {
ganttScrollRef: React.RefObject<HTMLDivElement>;
}

const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";

const tabToIndex = (tab?: string) => {
switch (tab) {
case "graph":
Expand All @@ -96,6 +99,8 @@ const tabToIndex = (tab?: string) => {
case "xcom":
case "calendar":
return 6;
case "rendered_k8s":
return 7;
case "details":
default:
return 0;
Expand Down Expand Up @@ -135,6 +140,9 @@ const indexToTab = (
if (!runId && !taskId) return "calendar";
if (isTaskInstance) return "xcom";
return undefined;
case 7:
if (isTaskInstance && isK8sExecutor) return "rendered_k8s";
return undefined;
default:
return undefined;
}
Expand Down Expand Up @@ -360,6 +368,14 @@ const Details = ({
</Text>
</Tab>
)}
{isTaskInstance && isK8sExecutor && (
<Tab>
<BiLogoKubernetes size={16} />
<Text as="strong" ml={1}>
K8s Pod Spec
</Text>
</Tab>
)}
{/* Match the styling of a tab but its actually a button */}
{!!taskId && !!runId && (
<Button
Expand Down Expand Up @@ -484,6 +500,11 @@ const Details = ({
/>
</TabPanel>
)}
{isTaskInstance && isK8sExecutor && (
<TabPanel height="100%">
<RenderedK8s />
</TabPanel>
)}
</TabPanels>
</Tabs>
</Flex>
Expand Down
6 changes: 0 additions & 6 deletions airflow/www/static/js/dag/details/taskInstance/Nav.tsx
Expand Up @@ -26,9 +26,7 @@ import type { Task } from "src/types";
import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";

const dagId = getMetaValue("dag_id");
const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
const taskInstancesUrl = getMetaValue("task_instances_list_url");
const renderedK8sUrl = getMetaValue("rendered_k8s_url");
const taskUrl = getMetaValue("task_url");
const gridUrl = getMetaValue("grid_url");

Expand All @@ -49,7 +47,6 @@ const Nav = forwardRef<HTMLDivElement, Props>(
map_index: mapIndex ?? -1,
});
const detailsLink = `${taskUrl}&${params}`;
const k8sLink = `${renderedK8sUrl}&${params}`;
const listParams = new URLSearchParamsWrapper({
_flt_3_dag_id: dagId,
_flt_3_task_id: taskId,
Expand Down Expand Up @@ -77,9 +74,6 @@ const Nav = forwardRef<HTMLDivElement, Props>(
{(!isMapped || mapIndex !== undefined) && (
<>
<LinkButton href={detailsLink}>More Details</LinkButton>
{isK8sExecutor && (
<LinkButton href={k8sLink}>K8s Pod Spec</LinkButton>
)}
{isSubDag && (
<LinkButton href={subDagLink}>Zoom into SubDag</LinkButton>
)}
Expand Down
55 changes: 55 additions & 0 deletions airflow/www/static/js/dag/details/taskInstance/RenderedK8s.tsx
@@ -0,0 +1,55 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import React, { useRef } from "react";
import { Code } from "@chakra-ui/react";
import YAML from "json-to-pretty-yaml";

import { getMetaValue, useOffsetTop } from "src/utils";

import useSelection from "src/dag/useSelection";
import { useRenderedK8s } from "src/api";

const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";

const RenderedK8s = () => {
const {
selected: { runId, taskId, mapIndex },
} = useSelection();

const { data: renderedK8s } = useRenderedK8s(runId, taskId, mapIndex);

const k8sRef = useRef<HTMLPreElement>(null);
const offsetTop = useOffsetTop(k8sRef);

if (!isK8sExecutor || !runId || !taskId) return null;

return (
<Code
mt={3}
ref={k8sRef}
maxHeight={`calc(100% - ${offsetTop}px)`}
overflowY="auto"
>
<pre>{YAML.stringify(renderedK8s)}</pre>
</Code>
);
};

export default RenderedK8s;
2 changes: 1 addition & 1 deletion airflow/www/templates/airflow/dag.html
Expand Up @@ -65,7 +65,7 @@
<meta name="graph_url" content="{{ url_for('Airflow.graph', dag_id=dag.dag_id, root=root) }}">
<meta name="task_url" content="{{ url_for('Airflow.task', dag_id=dag.dag_id) }}">
<meta name="log_url" content="{{ url_for('Airflow.log', dag_id=dag.dag_id) }}">
<meta name="rendered_k8s_url" content="{{ url_for('Airflow.rendered_k8s', dag_id=dag.dag_id) }}">
<meta name="rendered_k8s_data_url" content="{{ url_for('Airflow.rendered_k8s_data', dag_id=dag.dag_id) }}">
<meta name="task_instances_list_url" content="{{ url_for('TaskInstanceModelView.list') }}">
<meta name="tag_index_url" content="{{ url_for('Airflow.index', tags='_TAG_NAME_') }}">
<meta name="mapped_instances_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_task_instance_endpoint_get_mapped_task_instances', dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_') }}">
Expand Down
44 changes: 43 additions & 1 deletion airflow/www/views.py
Expand Up @@ -1493,7 +1493,7 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION):
form = DateTimeForm(data={"execution_date": dttm})
root = request.args.get("root", "")
map_index = request.args.get("map_index", -1, type=int)
logger.info("Retrieving rendered templates.")
logger.info("Retrieving rendered k8s.")

dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id)
task = dag.get_task(task_id)
Expand Down Expand Up @@ -1538,6 +1538,48 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION):
title=title,
)

@expose("/object/rendered-k8s")
@auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE)
@provide_session
def rendered_k8s_data(self, *, session: Session = NEW_SESSION):
"""Get rendered k8s yaml."""
if not settings.IS_K8S_OR_K8SCELERY_EXECUTOR:
return {"error": "Not a k8s or k8s_celery executor"}, 404
# This part is only used for k8s executor so providers.cncf.kubernetes must be installed
# with the get_rendered_k8s_spec method
from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec

dag_id = request.args.get("dag_id")
task_id = request.args.get("task_id")
if task_id is None:
return {"error": "Task id not passed in the request"}, 404
run_id = request.args.get("run_id")
map_index = request.args.get("map_index", -1, type=int)
logger.info("Retrieving rendered k8s data.")

dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id)
task = dag.get_task(task_id)
dag_run = dag.get_dagrun(run_id=run_id, session=session)
ti = dag_run.get_task_instance(task_id=task.task_id, map_index=map_index, session=session)

if not ti:
return {"error": f"can't find task instance {task.task_id}"}, 404
pod_spec = None
if not isinstance(ti, TaskInstance):
return {"error": f"{task.task_id} is not a task instance"}, 500
try:
pod_spec = get_rendered_k8s_spec(ti, session=session)
except AirflowException as e:
if not e.__cause__:
return {"error": f"Error rendering Kubernetes POD Spec: {e}"}, 500
else:
tmp = Markup("Error rendering Kubernetes POD Spec: {0}<br><br>Original error: {0.__cause__}")
return {"error": tmp.format(e)}, 500
except Exception as e:
return {"error": f"Error rendering Kubernetes Pod Spec: {e}"}, 500

return pod_spec

@expose("/get_logs_with_metadata")
@auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE)
@auth.has_access_dag("GET", DagAccessEntity.TASK_LOGS)
Expand Down
31 changes: 27 additions & 4 deletions airflow/www/yarn.lock
Expand Up @@ -3360,6 +3360,11 @@
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad"
integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==

"@types/json-to-pretty-yaml@^1.2.1":
version "1.2.1"
resolved "https://registry.yarnpkg.com/@types/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.1.tgz#bf193455477295d83c78f73c08d956f74321193e"
integrity sha512-+uOlBkCPkny6CE2a5IAR0Q21/ZE+90MsK7EfDblDdutcey+rbMDrp3i93M6MTwbMHFB75aIFR5fVXVcnLCkAiw==

"@types/json5@^0.0.29":
version "0.0.29"
resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
Expand Down Expand Up @@ -8006,6 +8011,14 @@ json-stringify-safe@^5.0.1:
resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==

json-to-pretty-yaml@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.2.tgz#f4cd0bd0a5e8fe1df25aaf5ba118b099fd992d5b"
integrity sha512-rvm6hunfCcqegwYaG5T4yKJWxc9FXFgBVrcTZ4XfSVRwa5HA/Xs+vB/Eo9treYYHCeNM0nrSUr82V/M31Urc7A==
dependencies:
remedial "^1.0.7"
remove-trailing-spaces "^1.0.6"

json5@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
Expand Down Expand Up @@ -9875,10 +9888,10 @@ react-focus-lock@^2.9.1:
use-callback-ref "^1.3.0"
use-sidecar "^1.1.2"

react-icons@^4.9.0:
version "4.9.0"
resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.9.0.tgz#ba44f436a053393adb1bdcafbc5c158b7b70d2a3"
integrity sha512-ijUnFr//ycebOqujtqtV9PFS7JjhWg0QU6ykURVHuL4cbofvRCf3f6GMn9+fBktEFQOIVZnuAYLZdiyadRQRFg==
react-icons@^5.1.0:
version "5.1.0"
resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-5.1.0.tgz#9e7533cc256571a610c2a1ec8a7a143fb1222943"
integrity sha512-D3zug1270S4hbSlIRJ0CUS97QE1yNNKDjzQe3HqY0aefp2CBn9VgzgES27sRR2gOvFK+0CNx/BW0ggOESp6fqQ==

react-is@^16.13.1, react-is@^16.7.0, react-is@^16.8.1:
version "16.13.1"
Expand Down Expand Up @@ -10248,11 +10261,21 @@ remark-rehype@^10.0.0:
mdast-util-to-hast "^12.1.0"
unified "^10.0.0"

remedial@^1.0.7:
version "1.0.8"
resolved "https://registry.yarnpkg.com/remedial/-/remedial-1.0.8.tgz#a5e4fd52a0e4956adbaf62da63a5a46a78c578a0"
integrity sha512-/62tYiOe6DzS5BqVsNpH/nkGlX45C/Sp6V+NtiN6JQNS1Viay7cWkazmRkrQrdFj2eshDe96SIQNIoMxqhzBOg==

remove-accents@0.4.2:
version "0.4.2"
resolved "https://registry.yarnpkg.com/remove-accents/-/remove-accents-0.4.2.tgz#0a43d3aaae1e80db919e07ae254b285d9e1c7bb5"
integrity sha1-CkPTqq4egNuRngeuJUsoXZ4ce7U=

remove-trailing-spaces@^1.0.6:
version "1.0.8"
resolved "https://registry.yarnpkg.com/remove-trailing-spaces/-/remove-trailing-spaces-1.0.8.tgz#4354d22f3236374702f58ee373168f6d6887ada7"
integrity sha512-O3vsMYfWighyFbTd8hk8VaSj9UAGENxAtX+//ugIst2RMk5e03h6RoIS+0ylsFxY1gvmPuAY/PO4It+gPEeySA==

require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
Expand Down

0 comments on commit 1074b8e

Please sign in to comment.