Automatically update metric plots for in-progress runs #2099 (#5017)

* Automatically update metric plots for in-progress runs #2099 Signed-off-by: Cedric Koffeto cedkoffeto@gmail.com Signed-off-by: cedric koffeto <cedkoffeto@gmail.com> * eslint corrections Signed-off-by: cedric koffeto <cedkoffeto@gmail.com> * eslint Signed-off-by: cedric koffeto <cedkoffeto@gmail.com> * bug fix Signed-off-by: cedric koffeto <cedkoffeto@gmail.com> * commit Signed-off-by: harupy <hkawamura0130@gmail.com> * checkOnRunUnfinished() replaced * cherry pick Signed-off-by: harupy <hkawamura0130@gmail.com> * add tests Signed-off-by: harupy <hkawamura0130@gmail.com> * i18n Signed-off-by: harupy <hkawamura0130@gmail.com> * check state Signed-off-by: harupy <hkawamura0130@gmail.com> * refactor Signed-off-by: harupy <hkawamura0130@gmail.com> * add rendering test Signed-off-by: harupy <hkawamura0130@gmail.com> * increase polling duration Signed-off-by: harupy <hkawamura0130@gmail.com> * ignore hanging runs Signed-off-by: harupy <hkawamura0130@gmail.com> * show interval in tooltip Signed-off-by: harupy <hkawamura0130@gmail.com> * rename test Signed-off-by: harupy <hkawamura0130@gmail.com> * i18n Signed-off-by: harupy <hkawamura0130@gmail.com> * lint Signed-off-by: harupy <hkawamura0130@gmail.com> * refactor Signed-off-by: harupy <hkawamura0130@gmail.com> * fix flaky test Signed-off-by: harupy <hkawamura0130@gmail.com> Co-authored-by: harupy <hkawamura0130@gmail.com>
mlflow · Dec 2, 2021 · 1b4bbb6 · 1b4bbb6
1 parent 19a82fe
commit 1b4bbb6
Show file tree

Hide file tree

Showing 5 changed files with 373 additions and 35 deletions.
diff --git a/mlflow/server/js/src/experiment-tracking/components/MetricsPlotControls.js b/mlflow/server/js/src/experiment-tracking/components/MetricsPlotControls.js
@@ -1,9 +1,9 @@
 import React from 'react';
 import _ from 'lodash';
 import { QuestionCircleOutlined } from '@ant-design/icons';
-import { Radio, Switch, TreeSelect, Tooltip } from 'antd';
+import { Radio, Switch, TreeSelect, Tooltip, Progress } from 'antd';
 import PropTypes from 'prop-types';
-import { CHART_TYPE_LINE } from './MetricsPlotPanel';
+import { CHART_TYPE_LINE, METRICS_PLOT_POLLING_INTERVAL_MS } from './MetricsPlotPanel';
 import { LineSmoothSlider } from './LineSmoothSlider';
 
 import { FormattedMessage, injectIntl } from 'react-intl';
@@ -31,6 +31,8 @@ export class MetricsPlotControlsImpl extends React.Component {
     yAxisLogScale: PropTypes.bool.isRequired,
     showPoint: PropTypes.bool.isRequired,
     intl: PropTypes.shape({ formatMessage: PropTypes.func.isRequired }).isRequired,
+    numRuns: PropTypes.number.isRequired,
+    numCompletedRuns: PropTypes.number.isRequired,
   };
 
   handleMetricsSelectFilterChange = (text, option) =>
@@ -46,7 +48,14 @@ export class MetricsPlotControlsImpl extends React.Component {
   };
 
   render() {
-    const { chartType, yAxisLogScale, initialLineSmoothness, showPoint } = this.props;
+    const {
+      chartType,
+      yAxisLogScale,
+      initialLineSmoothness,
+      showPoint,
+      numRuns,
+      numCompletedRuns,
+    } = this.props;
     const wrapperStyle = chartType === CHART_TYPE_LINE ? styles.linechartControlsWrapper : {};
     const lineSmoothnessTooltipText = (
       <FormattedMessage
@@ -55,10 +64,35 @@ export class MetricsPlotControlsImpl extends React.Component {
         description='Helpful tooltip message to help with line smoothness for the metrics plot'
       />
     );
+    const completedRunsTooltipText = (
+      <FormattedMessage
+        // eslint-disable-next-line max-len
+        defaultMessage='MLflow UI automatically fetches metric histories for active runs and updates the metrics plot with a {interval} second interval.'
+        description='Helpful tooltip message to explain the automatic metrics plot update'
+        values={{ interval: Math.round(METRICS_PLOT_POLLING_INTERVAL_MS / 1000) }}
+      />
+    );
     return (
       <div className='plot-controls' style={wrapperStyle}>
         {chartType === CHART_TYPE_LINE ? (
           <div>
+            <div className='inline-control'>
+              <div className='control-label'>
+                <FormattedMessage
+                  defaultMessage='Completed Runs'
+                  // eslint-disable-next-line max-len
+                  description='Label for the progress bar to show the number of completed runs'
+                />{' '}
+                <Tooltip title={completedRunsTooltipText}>
+                  <QuestionCircleOutlined />
+                </Tooltip>
+                <Progress
+                  percent={Math.round((100 * numCompletedRuns) / numRuns)}
+                  format={() => `${numCompletedRuns}/${numRuns}`}
+                  status='normal'
+                />
+              </div>
+            </div>
             <div className='inline-control'>
               <div className='control-label'>
                 <FormattedMessage

diff --git a/mlflow/server/js/src/experiment-tracking/components/MetricsPlotPanel.js b/mlflow/server/js/src/experiment-tracking/components/MetricsPlotPanel.js
@@ -2,11 +2,11 @@ import React from 'react';
 import { connect } from 'react-redux';
 import Utils from '../../common/utils/Utils';
 import RequestStateWrapper from '../../common/components/RequestStateWrapper';
-import { getMetricHistoryApi } from '../actions';
+import { getMetricHistoryApi, getRunApi } from '../actions';
 import PropTypes from 'prop-types';
 import _ from 'lodash';
 import { MetricsPlotView } from './MetricsPlotView';
-import { getRunTags } from '../reducers/Reducers';
+import { getRunTags, getRunInfo } from '../reducers/Reducers';
 import {
   MetricsPlotControls,
   X_AXIS_WALL,
@@ -22,10 +22,16 @@ import { getUUID } from '../../common/utils/ActionUtils';
 export const CHART_TYPE_LINE = 'line';
 export const CHART_TYPE_BAR = 'bar';
 
+export const METRICS_PLOT_POLLING_INTERVAL_MS = 10 * 1000; // 10 seconds
+// A run is considered as 'hanging' if its status is 'RUNNING' but its latest metric was logged
+// prior to this threshold. The metrics plot doesn't automatically update hanging runs.
+export const METRICS_PLOT_HANGING_RUN_THRESHOLD_MS = 3600 * 24 * 7 * 1000; // 1 week
+
 export class MetricsPlotPanel extends React.Component {
   static propTypes = {
     experimentId: PropTypes.string.isRequired,
     runUuids: PropTypes.arrayOf(PropTypes.string).isRequired,
+    completedRunUuids: PropTypes.arrayOf(PropTypes.string).isRequired,
     metricKey: PropTypes.string.isRequired,
     // A map of { runUuid : { metricKey: value } }
     latestMetricsByRunUuid: PropTypes.object.isRequired,
@@ -34,6 +40,7 @@ export class MetricsPlotPanel extends React.Component {
     // An array of { metricKey, history, runUuid, runDisplayName }
     metricsWithRunInfoAndHistory: PropTypes.arrayOf(PropTypes.object).isRequired,
     getMetricHistoryApi: PropTypes.func.isRequired,
+    getRunApi: PropTypes.func.isRequired,
     location: PropTypes.object.isRequired,
     history: PropTypes.object.isRequired,
     runDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired,
@@ -71,11 +78,85 @@ export class MetricsPlotPanel extends React.Component {
       popoverX: 0,
       popoverY: 0,
       popoverRunItems: [],
+      focused: true,
     };
     this.displayPopover = false;
+    this.intervalId = null;
     this.loadMetricHistory(this.props.runUuids, this.getUrlState().selectedMetricKeys);
   }
 
+  onFocus = () => {
+    this.setState({ focused: true });
+  };
+
+  onBlur = () => {
+    this.setState({ focused: false });
+  };
+
+  clearEventListeners = () => {
+    // `window.removeEventListener` does nothing when called with an unregistered event listener:
+    // https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/removeEventListener
+    window.removeEventListener('focus', this.onFocus);
+    window.removeEventListener('blur', this.onBlur);
+  };
+
+  clearInterval = () => {
+    // `clearInterval` does nothing when called with `null` or `undefine`:
+    // https://www.w3.org/TR/2011/WD-html5-20110525/timers.html#dom-windowtimers-cleartimeout
+    clearInterval(this.intervalId);
+  };
+
+  allRunsCompleted = () => {
+    return this.props.completedRunUuids.length === this.props.runUuids.length;
+  };
+
+  isHangingRunUuid = (activeRunUuid) => {
+    const metrics = this.props.latestMetricsByRunUuid[activeRunUuid];
+    if (!metrics) {
+      return false;
+    }
+    const timestamps = Object.values(metrics).map(({ timestamp }) => timestamp);
+    const latestTimestamp = Math.max(...timestamps);
+    return new Date().getTime() - latestTimestamp > METRICS_PLOT_HANGING_RUN_THRESHOLD_MS;
+  };
+
+  getActiveRunUuids = () => {
+    const { completedRunUuids, runUuids } = this.props;
+    const activeRunUuids = _.difference(runUuids, completedRunUuids);
+    return activeRunUuids.filter(_.negate(this.isHangingRunUuid)); // Exclude hanging runs
+  };
+
+  shouldPoll = () => {
+    return !(this.allRunsCompleted() || this.getActiveRunUuids().length === 0);
+  };
+
+  componentDidMount() {
+    if (this.shouldPoll()) {
+      // Set event listeners to detect when this component gains/loses focus,
+      // e.g., a user switches to a different browser tab or app.
+      window.addEventListener('blur', this.onBlur);
+      window.addEventListener('focus', this.onFocus);
+      this.intervalId = setInterval(() => {
+        // Skip polling if this component is out of focus.
+        if (this.state.focused) {
+          const activeRunUuids = this.getActiveRunUuids();
+          this.loadMetricHistory(activeRunUuids, this.getUrlState().selectedMetricKeys);
+          this.loadRuns(activeRunUuids);
+
+          if (!this.shouldPoll()) {
+            this.clearEventListeners();
+            this.clearInterval();
+          }
+        }
+      }, METRICS_PLOT_POLLING_INTERVAL_MS);
+    }
+  }
+
+  componentWillUnmount() {
+    this.clearEventListeners();
+    this.clearInterval();
+  }
+
   getUrlState() {
     return Utils.getMetricPlotStateFromUrl(this.props.location.search);
   }
@@ -149,6 +230,16 @@ export class MetricsPlotPanel extends React.Component {
     return requestIds;
   };
 
+  loadRuns = (runUuids) => {
+    const requestIds = [];
+    runUuids.forEach((runUuid) => {
+      const id = getUUID();
+      this.props.getRunApi(runUuid);
+      requestIds.push(id);
+    });
+    return requestIds;
+  };
+
   getMetrics = () => {
     /* eslint-disable no-param-reassign */
     const state = this.getUrlState();
@@ -471,6 +562,8 @@ export class MetricsPlotPanel extends React.Component {
     return (
       <div className='metrics-plot-container'>
         <MetricsPlotControls
+          numRuns={this.props.runUuids.length}
+          numCompletedRuns={this.props.completedRunUuids.length}
           distinctMetricKeys={distinctMetricKeys}
           selectedXAxis={selectedXAxis}
           selectedMetricKeys={selectedMetricKeys}
@@ -526,6 +619,9 @@ export class MetricsPlotPanel extends React.Component {
 
 const mapStateToProps = (state, ownProps) => {
   const { runUuids } = ownProps;
+  const completedRunUuids = runUuids.filter(
+    (runUuid) => getRunInfo(runUuid, state).status !== 'RUNNING',
+  );
   const { latestMetricsByRunUuid, metricsByRunUuid } = state.entities;
 
   // All metric keys from all runUuids, non-distinct
@@ -534,7 +630,6 @@ const mapStateToProps = (state, ownProps) => {
     return latestMetrics ? Object.keys(latestMetrics) : [];
   });
   const distinctMetricKeys = [...new Set(metricKeys)].sort();
-
   const runDisplayNames = [];
 
   // Flat array of all metrics, with history and information of the run it belongs to
@@ -561,9 +656,10 @@ const mapStateToProps = (state, ownProps) => {
     latestMetricsByRunUuid,
     distinctMetricKeys,
     metricsWithRunInfoAndHistory,
+    completedRunUuids,
   };
 };
 
-const mapDispatchToProps = { getMetricHistoryApi };
+const mapDispatchToProps = { getMetricHistoryApi, getRunApi };
 
 export default withRouter(connect(mapStateToProps, mapDispatchToProps)(MetricsPlotPanel));