Merge branch 'dev'

.github/actions/SetupEnvironment/action.yml

@@ -21,18 +21,33 @@ runs:
           conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('environment.yml') }}-${{ env.CACHE_NUMBER }}
       env:
         # Increase this value to reset cache if environment.yml has not changed
-        CACHE_NUMBER: 0
+        CACHE_NUMBER: 1
       id: cache
 
     - name: 🔄 Update Environment
-      if: steps.cache.outputs.cache-hit != 'true'
+      if: runner.os != 'Windows' && steps.cache.outputs.cache-hit != 'true'
       shell: bash -el {0}
       run: |
         mamba env update -n hypercp -f environment.yml
         mamba install --channel=conda-forge pyinstaller==6.6
 
     - name: 📸 Capture Environment
+      if: runner.os != 'Windows'
       shell: bash -el {0}
       run: |
         mamba info
         mamba list
+
+    - name: 🔄 Update Environment [Windows]
+      if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true'
+      shell: pwsh
+      run: |
+        mamba env update -n hypercp -f environment.yml
+        mamba install --channel=conda-forge pyinstaller==6.6
+
+    - name: 📸 Capture Environment [Windows]
+      if: runner.os == 'Windows'
+      shell: pwsh
+      run: |
+        mamba info
+        mamba list

.github/workflows/ApplicationTesting.yml

@@ -1,4 +1,4 @@
-name: ApplicationTesting
+name: Application Testing
 
 on:
   push:
@@ -29,10 +29,17 @@ jobs:
       - name: 🔧 Setup Environment
         uses: ./.github/actions/SetupEnvironment
 
-      - name: 📑 Capture Environment
+      - name: 🔑 Get ECMWF CDS|ADS Credentials
+        env:
+          CDSAPIRC: ${{ secrets.ECMWF_ADS_CREDENTIALS }}
         run: |
-          mamba info
-          mamba list
+          echo "$CDSAPIRC" >> ~/.ecmwf_ads_credentials.json
+
+      - name: 🔑 Get GMAO MERRA2 Credentials
+        env:
+          NETRC: ${{ secrets.NETRC }}
+        run: |
+          echo "$NETRC" >> ~/.netrc
 
       - name: 🧪 Run Tests
         run: |

.github/workflows/Build.yml

@@ -38,7 +38,7 @@ jobs:
           echo "BUNDLE_NAME=$(ls Bundled/dist | head -1 | xargs)" >> $GITHUB_ENV
 
       - name: 📥 Zip Bundle
-        uses: thedoctor0/[email protected].1
+        uses: thedoctor0/[email protected].6
         with:
           type: 'zip'
           directory: 'Bundled/dist/'

README.md

@@ -4,6 +4,9 @@
 
 <center><img src="Data/Img/with_background_530x223.png" alt="Banner"></center>
 
+![Test](https://github.com/nasa/HyperCP/actions/workflows/ApplicationTesting.yml/badge.svg)
+![Bundle](https://github.com/nasa/HyperCP/actions/workflows/Pipeline.yml/badge.svg)
+
 Hyperspectral In situ Support for PACE (HyperInSPACE) Community Processor (HyperCP) is designed to provide hyperspectral support for the <a href='https://pace.gsfc.nasa.gov/'>PACE mission</a> but also multispectral missions such as
 [Sentinel-3](https://sentinels.copernicus.eu/web/sentinel/missions/sentinel-3) by processing automated and manual, above-water,
 hyperspectral ocean color radiometry data using state-of-the-art methods and protocols for quality assurance,

Source/Controller.py

@@ -712,8 +712,11 @@ def processSingleLevel(pathOut, inFilePath, calibrationMap, level):
                         ancGroup.datasets[ds].datasetToColumns()
                     except Exception:
                         print('Error: Something wrong with root ANCILLARY')
-                stations = np.array(root.getGroup("ANCILLARY").getDataset("STATION").columns["STATION"])
-                stations = np.unique(stations[~np.isnan(stations)]).tolist()
+                if root.getGroup("ANCILLARY").getDataset("STATION") is not None:
+                    stations = np.array(root.getGroup("ANCILLARY").getDataset("STATION").columns["STATION"])
+                    stations = np.unique(stations[~np.isnan(stations)]).tolist()
+                else:
+                    stations = []
 
                 if len(stations) > 0:
 

Source/GetAnc.py

@@ -86,14 +86,16 @@ def getAnc(inputGroup):
                     msg = f'Request error: {status}'
                     print(msg)
                     Utilities.writeLogFile(msg)
+                    if os.environ["HYPERINSPACE_CMD"].lower() == 'true':
+                        return
                     alert = QtWidgets.QMessageBox()
                     alert.setText(f'Request error: {status}\n \
                                     Check that server credentials have \n \
                                     been entered in Configuration Window L1B. \n  \
                                     MERRA2 model data are not available until \n \
                                     the third week of the following month.')
                     alert.exec_()
-                    return None
+                    return
 
                 # GMAO Atmospheric model data
                 node = HDFRoot.readHDF5(filePath1)

Source/GetAnc_ecmwf.py

@@ -223,11 +223,11 @@ def getAnc_ecmwf(inputGroup):
             ancillary = GetAnc_ecmwf.get_ancillary_main(lat[index], lon[index], lat_timeStamp, ancPath)
 
             # position retrieval index has been confirmed manually in SeaDAS
-            uWind = ancillary['10m_u_component_of_wind']['value']
-            vWind = ancillary['10m_v_component_of_wind']['value']
+            uWind = ancillary['10m_u_component_of_wind']['value'][0]
+            vWind = ancillary['10m_v_component_of_wind']['value'][0]
             modWind.append(np.sqrt(uWind*uWind + vWind*vWind)) # direction not needed
             #ancAOD = aerGroup.getDataset("TOTEXTTAU")
-            modAOD.append(ancillary['total_aerosol_optical_depth_550nm']['value'])
+            modAOD.append(ancillary['total_aerosol_optical_depth_550nm']['value'][0])
 
         modData = HDFRoot()
         modGroup = modData.addGroup('ECMWF')

Source/ProcessInstrumentUncertainties.py

@@ -386,11 +386,16 @@ def FRM_L2(self, rhoScalar: float, rhoVec: np.array, rhoDelta: np.array, waveSub
                                       xSlice['ltSample'][i].items() if float(key) in waveSubset}
                                      for i in range(len(xSlice['ltSample']))])
 
+        # Get rho from scalar or vector
         if rhoScalar is not None:  # make rho a constant array if scalar
             rho = np.ones(len(waveSubset))*rhoScalar  # convert rhoScalar to the same dims as other values/Uncertainties
         else:
             rho = np.asarray(list(rhoVec.values()), dtype=float)
 
+        # Get rhoDelta from scalar or vector
+        if not hasattr(rhoDelta, '__len__'):  # Not an array (e.g. list or np.array)
+            rhoDelta = np.ones(len(waveSubset)) * rhoDelta  # convert rhoDelta to the same dims as other values/Uncertainties
+
         # initialise punpy propagation object
         mdraws = esSampleXSlice.shape[0]  # keep no. of monte carlo draws consistent
         Propagate_L2_FRM = Propagate(mdraws, cores=1)  # punpy.MCPropagation(mdraws, parallel_cores=1)

Source/ProcessL2.py

@@ -157,7 +157,9 @@ def nirCorrection(node, sensor, F0):
                     ρ720.append(ρSlice[k][-1]) # Using current element/slice [-1]
 
             # if not ρ720:
-                # QtWidgets.QMessageBox.critical("Error", "NIR wavebands unavailable")
+            #     print("Error: NIR wavebands unavailable")
+            #     if os.environ["HYPERINSPACE_CMD"].lower() == 'false':
+            #         QtWidgets.QMessageBox.critical("Error", "NIR wavebands unavailable")
             ρ1 = sp.interpolate.interp1d(x,ρ720)(720)
             F01 = sp.interpolate.interp1d(wavelength,F0)(720)
             ρ780 = []
@@ -169,7 +171,9 @@ def nirCorrection(node, sensor, F0):
                     x.append(float(k))
                     ρ780.append(ρSlice[k][-1])
             if not ρ780:
-                QtWidgets.QMessageBox.critical("Error", "NIR wavebands unavailable")
+                print("Error: NIR wavebands unavailable")
+                if os.environ["HYPERINSPACE_CMD"].lower() == 'false':
+                    QtWidgets.QMessageBox.critical("Error", "NIR wavebands unavailable")
             ρ2 = sp.interpolate.interp1d(x,ρ780)(780)
             F02 = sp.interpolate.interp1d(wavelength,F0)(780)
             ρ870 = []

Source/Utilities.py

@@ -256,6 +256,8 @@ def find_nearest(array,value):
 
     @staticmethod
     def errorWindow(winText,errorText):
+        if os.environ["HYPERINSPACE_CMD"].lower() == 'true':
+            return
         msgBox = QMessageBox()
         # msgBox.setIcon(QMessageBox.Information)
         msgBox.setIcon(QMessageBox.Critical)
@@ -264,20 +266,10 @@ def errorWindow(winText,errorText):
         msgBox.setStandardButtons(QMessageBox.Ok)
         msgBox.exec_()
 
-
-    @staticmethod
-    def waitWindow(winText,waitText):
-        msgBox = QMessageBox()
-        # msgBox.setIcon(QMessageBox.Information)
-        msgBox.setIcon(QMessageBox.Critical)
-        msgBox.setText(waitText)
-        msgBox.setWindowTitle(winText)
-        # msgBox.setStandardButtons(QMessageBox.Ok)
-        # msgBox.exec_()
-        return msgBox
-
     @staticmethod
     def YNWindow(winText,infoText):
+        if os.environ["HYPERINSPACE_CMD"].lower() == 'true':
+            return QMessageBox.Ok  # Assume positive answer to keep processing in command line mode (no X)
         msgBox = QMessageBox()
         msgBox.setIcon(QMessageBox.Information)
         msgBox.setText(infoText)

Tests/test_sample_data.py

@@ -12,7 +12,7 @@ def setUp(self):
         # Load files to process
         self.path_to_data = os.path.join(root, 'Data', 'Sample_Data', 'Manual_TriOS')
         self.anc_filename = os.path.join(self.path_to_data, f"FICE22_TriOS_Ancillary.sb")
-        self.cfg_filename = os.path.join(root, "Config", "sample_TriOS_NOTRACKER.cfg")
+        self.cfg_filename = os.path.join(root, "Config", "sample_TRIOS_NOTRACKER.cfg")
         self.files = sorted(glob.glob(os.path.join(self.path_to_data, 'RAW', f'*.mlb')))
 
     def test_manual_trios(self):
@@ -38,5 +38,21 @@ def test_pysas(self):
                     self.anc_filename, processMultiLevel=True)
 
 
+class TestSeabirdSolarTracker(unittest.TestCase):
+    def setUp(self):
+        # Load files to process
+        self.path_to_data = os.path.join(root, 'Data', 'Sample_Data', 'SolarTracker')
+        self.anc_filename = os.path.join(self.path_to_data, f"KORUS_SOLARTRACKER_Ancillary.sb")
+        self.cfg_filename = os.path.join(root, "Config", "sample_SEABIRD_SOLARTRACKER.cfg")
+        self.files = sorted(glob.glob(os.path.join(self.path_to_data, 'RAW', f'*.RAW')))
+
+    def test_pysas(self):
+        from Main import Command
+        os.chdir(root)  # Need to switch to root as path in Config files are relative
+        for file in self.files:
+            Command(self.cfg_filename, 'RAW', file, self.path_to_data,'L1A',
+                    self.anc_filename, processMultiLevel=True)
+
+
 if __name__ == '__main__':
     unittest.main()

make.py

@@ -39,8 +39,8 @@
 version = None
 with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Main.py'), 'r') as f:
     for l in f:
-        if l.startswith('version'):
-            version = l.split('=')[1].strip(" \n'")
+        if l.startswith('VERSION'):
+            version = l.split('=')[1].strip(" \n'\"")
             break
 
 # Get git hash (without git package)
@@ -62,7 +62,6 @@
 linked_data = [
     f'--add-data={os.path.relpath("version.txt", root)}{add_data_sep}.',
     f'--add-data={os.path.relpath("Config", root)}{add_data_sep}Config',
-    f'--add-data={os.path.relpath(".ecmwf_api_config", root)}{add_data_sep}.',
 ]
 for f in sorted(glob.glob(os.path.join('Data', '*'))):
     if os.path.isdir(f) and os.path.basename(f) not in ['L1A', 'L1AQC', 'L1B', 'L1BQC', 'L2', 'Plots', 'Reports']: