Merge pull request #1126 from cbegeman/do-not-require-tseries-files-if-cached-output

Allow time series tasks to run when history files are not present but cached files are

Author: xylar

mpas_analysis/ocean/index_nino34.py

@@ -779,6 +779,8 @@ def _plot_size_y_axis(self, x, ys, xmin, xmax):
 
         # find maximum value of three curves plotted
         maxY = -1E20
+        if len(mask) == 0:
+            return maxY
         for y in ys:
             maxY = max(y[mask].max(), maxY)
             # check the function interpolated to the max/min as well

mpas_analysis/ocean/streamfunction_moc.py

@@ -1007,15 +1007,16 @@ def _compute_moc_time_series_analysismember(self):
                                              self.historyStreams,
                                              'timeSeriesStatsMonthlyOutput')
 
-        mocRegion = np.zeros(len(inputFiles))
+        ntimes = int(12 * (self.endYear - self.startYear + 1))
+        mocRegion = np.zeros(ntimes)
         moc = None
         refTopDepth = None
-        times = np.zeros(len(inputFiles))
-        computed = np.zeros(len(inputFiles), bool)
+        times = np.zeros(ntimes)
+        computed = np.zeros(ntimes, bool)
 
         continueOutput = os.path.exists(outputFileName)
         if continueOutput:
-            self.logger.info('   Read in previously computed MOC time series')
+            self.logger.info(f'   Read in previously computed MOC time series {outputFileName}')
             with open_mpas_dataset(fileName=outputFileName,
                                    calendar=self.calendar,
                                    timeVariableNames=None,
@@ -1028,32 +1029,30 @@ def _compute_moc_time_series_analysismember(self):
 
                 if moc is None:
                     sizes = dsMOCIn.sizes
-                    moc = np.zeros((len(inputFiles), sizes['depth'],
-                                    sizes['lat']))
+                    moc = np.zeros((ntimes, sizes['depth'],
+                                   sizes['lat']))
                     refTopDepth = dsMOCIn.depth.values
 
                 # first, copy all computed data
-                for inIndex in range(dsMOCIn.sizes['Time']):
-
-                    mask = np.logical_and(
-                        dsMOCIn.year[inIndex].values == years,
-                        dsMOCIn.month[inIndex].values == months)
-
-                    outIndex = np.where(mask)[0][0]
-
-                    mocRegion[outIndex] = dsMOCIn.mocAtlantic26[inIndex]
-                    moc[outIndex, :, :] = dsMOCIn.mocAtlantic[inIndex, :, :]
-                    times[outIndex] = dsMOCIn.Time[inIndex]
-                    computed[outIndex] = True
+                outIndex = 0
+                for load_year in np.arange(self.startYear, self.endYear + 1):
+                    for load_month in np.arange(1, 13):
+                        mask = np.logical_and(dsMOCIn.year.values == load_year,
+                                              dsMOCIn.month.values == load_month)
+                        if np.sum(mask) >= 1:
+                            inIndex = np.where(mask)[0][0]
+                            mocRegion[outIndex] = dsMOCIn.mocAtlantic26[inIndex]
+                            moc[outIndex, :, :] = dsMOCIn.mocAtlantic[inIndex, :, :]
+                            times[outIndex] = dsMOCIn.Time[inIndex]
+                            computed[outIndex] = True
+
+                        outIndex += 1
 
                 if np.all(computed):
                     # no need to waste time writing out the data set again
                     return dsMOCIn
 
         for timeIndex, fileName in enumerate(inputFiles):
-            if computed[timeIndex]:
-                continue
-
             dsLocal = open_mpas_dataset(
                 fileName=fileName,
                 calendar=self.calendar,
@@ -1067,12 +1066,15 @@ def _compute_moc_time_series_analysismember(self):
 
             self.logger.info('     date: {:04d}-{:02d}'.format(date.year,
                                                                date.month))
+            computedIndex = 12 * (date.year - self.startYear) + date.month - 1
+            if computed[computedIndex]:
+                continue
 
             # hard-wire region=0 (Atlantic) for now
             indRegion = 0
             mocVar = dsLocal.timeMonthly_avg_mocStreamvalLatAndDepthRegion
             mocTop = mocVar[indRegion, :, :].values
-            mocRegion[timeIndex] = np.amax(mocTop[:, indlat26])
+            mocRegion[computedIndex] = np.amax(mocTop[:, indlat26])
 
             if moc is None:
                 sizes = dsLocal.sizes
@@ -1087,7 +1089,8 @@ def _compute_moc_time_series_analysismember(self):
                 refTopDepth = np.zeros(nVertLevels + 1)
                 refTopDepth[1:nVertLevels + 1] = refBottomDepth[0:nVertLevels]
 
-            moc[timeIndex, 0:-1, :] = mocTop
+            moc[computedIndex, 0:-1, :] = mocTop
+
 
         description = 'Max MOC Atlantic streamfunction nearest to RAPID ' \
             'Array latitude (26.5N)'

mpas_analysis/shared/analysis_task.py

@@ -602,7 +602,8 @@ def flush(self):
         pass
 
 
-def update_time_bounds_from_file_names(config, section, componentName):
+def update_time_bounds_from_file_names(config, section, componentName,
+                                       allow_cache=True):
     """
     Update the start and end years and dates for time series, climatologies or
     climate indices based on the years actually available in the list of files.
@@ -658,7 +659,7 @@ def update_time_bounds_from_file_names(config, section, componentName):
         return
 
     if len(inputFiles) == 0:
-        raise ValueError('No input files found for stream {} in {} between '
+        print('Warning: No input files found for stream {} in {} between '
                          '{} and {}'.format(streamName, componentName,
                                             requestedStartYear,
                                             requestedEndYear))
@@ -680,12 +681,16 @@ def update_time_bounds_from_file_names(config, section, componentName):
     endYear = years[lastIndex]
 
     if startYear != requestedStartYear or endYear != requestedEndYear:
-        raise ValueError(
-            "{} start and/or end year different from requested\n"
-            "requested: {:04d}-{:04d}\n"
-            "actual:   {:04d}-{:04d}\n".format(
-                section, requestedStartYear, requestedEndYear, startYear,
-                endYear))
+        message = ("{} start and/or end year different from requested\n"
+                   "requested: {:04d}-{:04d}\n"
+                   "actual:   {:04d}-{:04d}\n".format(
+                      section, requestedStartYear, requestedEndYear, startYear,
+                      endYear)
+                  )
+        if allow_cache:
+            print(f'Warning: {message}')
+        else:
+            raise ValueError(message)
 
     startDate = '{:04d}-01-01_00:00:00'.format(startYear)
     config.set(section, 'startDate', startDate)

mpas_analysis/test/test_mpas_climatology_task.py

@@ -168,7 +168,8 @@ def test_update_climatology_bounds_and_create_symlinks(self):
         with self.assertRaisesRegex(ValueError,
                                     'climatology start and/or end year '
                                     'different from requested'):
-            update_time_bounds_from_file_names(config, 'climatology', 'ocean')
+            update_time_bounds_from_file_names(config, 'climatology', 'ocean',
+                                               allow_cache=False)
 
     def test_subtask_run_analysis(self):
         mpasClimatologyTask = self.setup_task()