Catchup

Source/AnomalyDetection.py

@@ -1,9 +1,9 @@
+'''Deglitching of L1AQC radiometry time series'''
 import os
 import sys
 import csv
-import numpy as np
-
 import warnings
+import numpy as np
 from PyQt5 import QtCore, QtGui, QtWidgets
 import pyqtgraph as pg
 
@@ -17,6 +17,7 @@
 from Source.CalibrationFileReader import CalibrationFileReader
 
 class AnomAnalWindow(QtWidgets.QDialog):
+    '''Anomaly Analysis supervised screening and deglitching GUI'''
 
     def __init__(self, inputDirectory, parent=None):
         super().__init__(parent)
@@ -26,6 +27,12 @@ def __init__(self, inputDirectory, parent=None):
         self.setModal(True)
         self.sensor='ES'
         self.sliderWave=525
+        self.fileName = None
+        self.root = None
+        self.start = None
+        self.end = None
+        self.photoList = None
+        self.photoDT = None
 
         #   This is going to truncate the ancillary data, so for the purposes of re-use, copy it
         self.ancillaryData = Controller.processAncData(MainConfig.settings["ancFile"])
@@ -417,7 +424,7 @@ def loadL1Afile(self):
         inputDirectory = self.inputDirectory
         # Open L1A HDF5 file for Deglitching
 
-        self.sensor == "ES"
+        # self.sensor == "ES"
 
         inLevel = "L1A"
         subInputDir = os.path.join(inputDirectory + '/' + inLevel + '/')
@@ -438,7 +445,7 @@ def loadL1Afile(self):
                 Utilities.errorWindow("File Error", msg)
                 print(msg)
                 return
-        except:
+        except Exception:
             print('No file returned')
             return
 
@@ -535,7 +542,7 @@ def loadL1Afile(self):
                 self.start = gpsGroup.datasets['DATETIME'].data[0]
                 self.end = gpsGroup.datasets['DATETIME'].data[-1]
             if group.id == "SOLARTRACKER" or group.id == "SOLARTRACKER_pySAS":
-                    trackerGroup = group
+                trackerGroup = group
 
         # For case of no GPS
         if self.start is None:
@@ -630,7 +637,7 @@ def radioClick(self):
         radioButton = self.sender()
         if radioButton.isChecked():
             self.sensor = radioButton.text()
-            print("Sensor is %s" % (self.sensor))
+            print(f"Sensor is {self.sensor}")
             # Now update the LineEdits with saved values in the local object
             self.WindowDarkLineEdit.setText(str(getattr(self,f'{self.sensor}WindowDark')))
             self.WindowLightLineEdit.setText(str(getattr(self,f'{self.sensor}WindowLight')))
@@ -740,7 +747,8 @@ def updateButtonPressed(self):
 
             radiometry1D = dark2D[f'{self.waveBand:03.2f}']
 
-            self.realTimePlot(self, radiometry1D, darkDateTime, sensorType,lightDark)
+            # self.realTimePlot(self, radiometry1D, darkDateTime, sensorType,lightDark)
+            AnomAnalWindow.realTimePlot(self, radiometry1D, darkDateTime, sensorType,lightDark)
 
         # Update the slider
         self.sLabel.setText(f'Deglitching only performed from 350-850 nm: {self.waveBand}')
@@ -767,7 +775,8 @@ def updateButtonPressed(self):
             radiometry1D = light2D[f'{self.waveBand:03.2f}']
             # print(self.waveBand)
 
-            self.realTimePlot(self, radiometry1D, lightDateTime, sensorType, lightDark)
+            # self.realTimePlot(self, radiometry1D, lightDateTime, sensorType, lightDark)
+            AnomAnalWindow.realTimePlot(self, radiometry1D, lightDateTime, sensorType, lightDark)
 
         # Now run the deglitcher for all wavebands light and dark to calculate the % loss to the data from this sensor
         # Darks
@@ -878,7 +887,7 @@ def writeAnomAnalFile(self, filePath):
         # fieldnames = params.keys()
 
         print(f'Saving {filePath}')
-        with open(filePath, 'w', newline='') as csvfile:
+        with open(filePath, 'w', newline='', encoding="utf-8") as csvfile:
             paramwriter = csv.writer(csvfile, delimiter=',',
                                 quotechar='|', quoting=csv.QUOTE_MINIMAL)
             paramwriter.writerow(header)
@@ -991,7 +1000,7 @@ def plotButtonPressed(self):
 
                 # Collapse the badIndexes from all wavebands into one timeseries
                 # Convert to an array and test along the columns (i.e. each timestamp)
-                gIndex = np.any(np.array(globalBadIndex), 0)
+                # gIndex = np.any(np.array(globalBadIndex), 0)
                 # NOTE: if you similarly collapse globBads 1-3, you should get the same result as gIndex
 
                 # Now plot a selection of these USING UNIVERSALLY EXCLUDED INDEXES
@@ -1100,7 +1109,7 @@ def GoToButtonPressed(self):
         # self.MinMaxLightLabel.setText(str(self.waveBand) +'nm' )
 
 
-    @staticmethod
+    # @staticmethod
     def realTimePlot(self, radiometry1D, dateTime, sensorType,lightDark):
         # Radiometry at this point is 1D 'column' from the appropriate group/dataset/waveband
         #   in time (radiometry1D)
@@ -1191,7 +1200,7 @@ def realTimePlot(self, radiometry1D, dateTime, sensorType,lightDark):
             ph3rd.setData(x_anomaly3, y_anomaly3, symbolPen='c',symbol='o', symbolBrush=None, name='Threshold exceeded')
 
 
-        except:
+        except Exception:
             e = sys.exc_info()[0]
-            print("Error: %s" % e)
+            print(f"Error: {e}")
 

Source/Controller.py

@@ -616,6 +616,7 @@ def processSingleLevel(pathOut, inFilePath, calibrationMap, level, flag_Trios):
                     print(msg)
                     Utilities.writeLogFile(msg)
                 root = Controller.processL1aqc(inFilePath, outFilePath, calibrationMap, ancillaryData,flag_Trios)
+                # BUG: The above throws 2 class TypeErrors between the return statement at the end of the method and here??
                 Utilities.checkOutputFiles(outFilePath)
 
             elif level == "L1B":

Source/FieldPhotos.py

@@ -108,6 +108,7 @@ def photoSetup(inPath, start, end, format='IMG_%Y%m%d_%H%M%S.jpg', tz='+0000'):
 
             # dFormat = 'IMG_%Y%m%d_%H%M%S.jpg'
             dFormat = f"{format.split('.')[0]}%z" # tack on the time zone below
+            dFormatAlt = f"{format.split('.')[0]}%f%z" # for case with microseconds
             # picDateTime = []
             picList = []
             picDTList = []
@@ -123,8 +124,18 @@ def photoSetup(inPath, start, end, format='IMG_%Y%m%d_%H%M%S.jpg', tz='+0000'):
                     if tDiff < tDiffLim:
                         picList.append(fp)
                         picDTList.append(picDateTime)
-                except:
-                    print(f'This file does not match the format. Rename or update format in AnomAnal GUI. {fileName}')
+                except Exception:
+                    try:
+                        dT = datetime.datetime.strptime(fileName, dFormatAlt)
+                        # picDateTime.append(dT.astimezone(pytz.utc))
+                        picDateTime = dT.astimezone(pytz.utc)
+                        # tDiff = abs(picDateTime-midDatetime).seconds / 60
+                        tDiff = abs(picDateTime-midDatetime)
+                        if tDiff < tDiffLim:
+                            picList.append(fp)
+                            picDTList.append(picDateTime)
+                    except Exception:
+                        print(f'This file does not match the format. Rename or update format in AnomAnal GUI. {fileName}')
 
 
             if len(picList) > 0:

Source/ProcessL1aqc.py

@@ -789,9 +789,8 @@ def processL1aqc(node, calibrationMap, ancillaryData=None):
 
         # Look for additional datasets in provided ancillaryData and populate the new ancillary group
         # if ancillaryData is not None:
-        ancGroup.attributes["FrameType"] = "Not Required"
         if ancData is not None:
-            ancGroup.attributes = ancData.attributes.copy()            
+            ancGroup.attributes = ancData.attributes.copy()
             if "HEADING" in ancData.columns:
                 ancGroup.addDataset("HEADING")
                 ancGroup.datasets["HEADING"].data = np.array(shipAzimuth, dtype=[('NONE', '<f8')])
@@ -825,7 +824,7 @@ def processL1aqc(node, calibrationMap, ancillaryData=None):
             if "ROLL" in ancData.columns:
                 ancGroup.addDataset("ROLL")
                 ancGroup.datasets["ROLL"].data = np.array(ancData.columns["ROLL"][0], dtype=[('NONE', '<f8')])
-
+        ancGroup.attributes["FrameType"] = "Not Required"
         ######################################################################################################
 
         # Apply Relative Azimuth filter

Source/ProcessL1aqc_deglitch.py

@@ -292,14 +292,14 @@ def processDataDeglitching(node, sensorType):
             if gp.attributes["FrameType"] == "ShutterDark" and sensorType in gp.datasets:
                 try:
                     gp.datasetDeleteRow(np.where(badIndexDark))
-                except:
+                except Exception:
                     print('Error deleting group datasets. Check that Light/Dark cals are correctly identified in Configuration Window.')
 
             if gp.attributes["FrameType"] == "ShutterLight" and sensorType in gp.datasets:
                 lightData = gp.getDataset(sensorType)
                 try:
                     gp.datasetDeleteRow(np.where(badIndexLight))
-                except:
+                except Exception:
                     print('Error deleting group datasets. Check that Light/Dark cals are correctly identified in Configuration Window.')
 
         return False