UI handles 502s and 504s gracefully

ui/app/controllers/jobs/index.js

@@ -22,6 +22,7 @@ export default class JobsIndexController extends Controller {
   @service store;
   @service userSettings;
   @service watchList;
+  @service notifications;
 
   @tracked pageSize;
 
@@ -156,12 +157,72 @@ export default class JobsIndexController extends Controller {
     );
   }
 
+  /**
+   * In case the user wants to specifically stop polling for new jobs
+   */
+  @action pauseJobFetching() {
+    // this.showingCachedJobs = false;
+    let notification = this.notifications.queue.find(
+      (n) => n.title === 'Error fetching jobs'
+    );
+    if (notification) {
+      notification.destroyMessage();
+    }
+    this.watchList.jobsIndexIDsController.abort();
+    this.watchList.jobsIndexDetailsController.abort();
+    this.watchJobIDs.cancelAll();
+    this.watchJobs.cancelAll();
+  }
+
+  @action restartJobList() {
+    this.showingCachedJobs = false;
+    let notification = this.notifications.queue.find(
+      (n) => n.title === 'Error fetching jobs'
+    );
+    if (notification) {
+      notification.destroyMessage();
+    }
+    this.watchList.jobsIndexIDsController.abort();
+    this.watchList.jobsIndexDetailsController.abort();
+    this.watchJobIDs.cancelAll();
+    this.watchJobs.cancelAll();
+    this.watchJobIDs.perform({}, JOB_LIST_THROTTLE);
+    this.watchJobs.perform(this.jobIDs, JOB_DETAILS_THROTTLE);
+  }
+
   @localStorageProperty('nomadLiveUpdateJobsIndex', true) liveUpdatesEnabled;
 
   // #endregion pagination
 
   //#region querying
 
+  /**
+   *
+   * Let the user know that there was difficulty fetching jobs, but don't overload their screen with notifications.
+   * Set showingCachedJobs to tell the template to prompt them to extend timeouts
+   * @param {Error} e
+   */
+  notifyFetchError(e) {
+    const firstError = e.errors[0];
+    console.log('firstError', firstError);
+    this.notifications.add({
+      title: 'Error fetching jobs',
+      message: `The backend returned an error with status ${firstError.status} while fetching jobs`,
+      color: 'critical',
+      sticky: true,
+      preventDuplicates: true,
+    });
+    // Specific check for a proxy timeout error
+    if (
+      !this.showingCachedJobs &&
+      (firstError.status === '502' || firstError.status === '504')
+    ) {
+      this.showingCachedJobs = true;
+    }
+  }
+
+  @tracked showingCachedJobs = false;
+
   jobQuery(params) {
     this.watchList.jobsIndexIDsController.abort();
     this.watchList.jobsIndexIDsController = new AbortController();
@@ -172,9 +233,17 @@ export default class JobsIndexController extends Controller {
           abortController: this.watchList.jobsIndexIDsController,
         },
       })
+      .then((jobs) => {
+        this.showingCachedJobs = false;
+        return jobs;
+      })
       .catch((e) => {
         if (e.name !== 'AbortError') {
           console.log('error fetching job ids', e);
+          this.notifyFetchError(e);
+        }
+        if (this.jobs.length) {
+          return this.jobs;
         }
         return;
       });
@@ -194,6 +263,10 @@ export default class JobsIndexController extends Controller {
       .catch((e) => {
         if (e.name !== 'AbortError') {
           console.log('error fetching job allocs', e);
+          this.notifyFetchError(e);
+        }
+        if (this.jobs.length) {
+          return this.jobs;
         }
         return;
       });

ui/app/routes/jobs/index.js

@@ -81,8 +81,10 @@ export default class IndexRoute extends Route.extend(
       });
     } catch (error) {
       try {
+        console.log('route jobs index catch try 1');
         notifyForbidden(this)(error);
       } catch (secondaryError) {
+        console.log('route jobs index catch try 2');
         return this.handleErrors(error);
       }
     }

ui/app/styles/components/jobs-list.scss

@@ -43,6 +43,10 @@
   }
 }
 
+#jobs-list-cache-warning {
+  margin-bottom: 1rem;
+}
+
 .status-cell {
   display: flex;
   gap: 0.5rem;