From 8c3e9369d4a1e0293fc46cd2ac23ceb19356d84c Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 17:25:29 +0200 Subject: [PATCH 01/10] Added Application Testing to workflow. --- .github/workflows/ApplicationTesting.yml | 34 ++++++++++++++++++++ .github/workflows/SetupEnvironment.yml | 40 ++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/workflows/ApplicationTesting.yml create mode 100644 .github/workflows/SetupEnvironment.yml diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml new file mode 100644 index 0000000..a9829fa --- /dev/null +++ b/.github/workflows/ApplicationTesting.yml @@ -0,0 +1,34 @@ +name: ApplicationTesting + +on: + push: + branches: + - dev + workflow_dispatch: + +jobs: + + ApplicationTesting: + name: Application Tests + + runs-on: ${{ matrix.os }} + defaults: # Required for conda environment activation + run: + shell: bash -el {0} + strategy: + fail-fast: false + matrix: + os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ] + + steps: + - name: ๐Ÿ”ง Setup Environment + uses: ./.github/workflows/SetupEnvironment.yml + + - name: ๐Ÿ““ Capture Environment + run: | + mamba info + mamba list + + - name: ๐Ÿงช Run Tests + run: | + python run_sample_data.py diff --git a/.github/workflows/SetupEnvironment.yml b/.github/workflows/SetupEnvironment.yml new file mode 100644 index 0000000..eb3c81d --- /dev/null +++ b/.github/workflows/SetupEnvironment.yml @@ -0,0 +1,40 @@ +name: SetupEnvironment +description: 'Setup python environment with conda' + +runs: + using: "composite" + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch }} + + - name: ๐Ÿ Setup Mambaforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: hypercp + use-mamba: true + + - name: Cache Environment + uses: actions/cache@v4 + with: + path: ${{ env.CONDA }}/envs + key: + conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('environment.yml') }}-${{ env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER: 0 + id: cache + + - name: Update Environment + if: steps.cache.outputs.cache-hit != 'true' + run: | + mamba env update -n hypercp -f environment.yml + mamba install --channel=conda-forge pyinstaller==6.6 + + - name: ๐Ÿ““ Capture Environment + run: | + mamba info + mamba list From 1f4b728efb14bfbbcfd90d228187fd288f175eeb Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 17:44:55 +0200 Subject: [PATCH 02/10] Update Application Testing Workflow --- .../SetupEnvironment/action.yml} | 7 ++++--- .github/workflows/ApplicationTesting.yml | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) rename .github/{workflows/SetupEnvironment.yml => actions/SetupEnvironment/action.yml} (90%) diff --git a/.github/workflows/SetupEnvironment.yml b/.github/actions/SetupEnvironment/action.yml similarity index 90% rename from .github/workflows/SetupEnvironment.yml rename to .github/actions/SetupEnvironment/action.yml index eb3c81d..5a271b1 100644 --- a/.github/workflows/SetupEnvironment.yml +++ b/.github/actions/SetupEnvironment/action.yml @@ -3,6 +3,7 @@ description: 'Setup python environment with conda' runs: using: "composite" + steps: - name: โฌ Checkout repository uses: actions/checkout@v4 @@ -17,7 +18,7 @@ runs: activate-environment: hypercp use-mamba: true - - name: Cache Environment + - name: ๐Ÿ”‚ Cache Environment uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs @@ -28,13 +29,13 @@ runs: CACHE_NUMBER: 0 id: cache - - name: Update Environment + - name: ๐Ÿ”„ Update Environment if: steps.cache.outputs.cache-hit != 'true' run: | mamba env update -n hypercp -f environment.yml mamba install --channel=conda-forge pyinstaller==6.6 - - name: ๐Ÿ““ Capture Environment + - name: ๐Ÿ“ธ Capture Environment run: | mamba info mamba list diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml index a9829fa..ab3711b 100644 --- a/.github/workflows/ApplicationTesting.yml +++ b/.github/workflows/ApplicationTesting.yml @@ -12,9 +12,9 @@ jobs: name: Application Tests runs-on: ${{ matrix.os }} - defaults: # Required for conda environment activation - run: - shell: bash -el {0} +# defaults: # Required for conda environment activation +# run: +# shell: bash -el {0} strategy: fail-fast: false matrix: @@ -22,9 +22,9 @@ jobs: steps: - name: ๐Ÿ”ง Setup Environment - uses: ./.github/workflows/SetupEnvironment.yml + uses: ./.github/actions/SetupEnvironment - - name: ๐Ÿ““ Capture Environment + - name: ๐Ÿ“‘ Capture Environment run: | mamba info mamba list From 7e31a98eb8cf5c35c4bb2cd70a3116ee58eb56a3 Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 17:46:47 +0200 Subject: [PATCH 03/10] Update Application Testing Workflow --- .github/actions/SetupEnvironment/action.yml | 5 ----- .github/workflows/ApplicationTesting.yml | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/SetupEnvironment/action.yml b/.github/actions/SetupEnvironment/action.yml index 5a271b1..01670e5 100644 --- a/.github/actions/SetupEnvironment/action.yml +++ b/.github/actions/SetupEnvironment/action.yml @@ -5,11 +5,6 @@ runs: using: "composite" steps: - - name: โฌ Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.branch }} - - name: ๐Ÿ Setup Mambaforge uses: conda-incubator/setup-miniconda@v3 with: diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml index ab3711b..ce23f2f 100644 --- a/.github/workflows/ApplicationTesting.yml +++ b/.github/workflows/ApplicationTesting.yml @@ -21,6 +21,11 @@ jobs: os: [ 'ubuntu-latest', 'macos-latest', 'windows-latest' ] steps: + - name: โฌ Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch }} + - name: ๐Ÿ”ง Setup Environment uses: ./.github/actions/SetupEnvironment From f8870170897852e2a21268d5f4006590702232bc Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 17:49:52 +0200 Subject: [PATCH 04/10] Update Application Testing Workflow --- .github/actions/SetupEnvironment/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/SetupEnvironment/action.yml b/.github/actions/SetupEnvironment/action.yml index 01670e5..f33c11f 100644 --- a/.github/actions/SetupEnvironment/action.yml +++ b/.github/actions/SetupEnvironment/action.yml @@ -26,11 +26,13 @@ runs: - name: ๐Ÿ”„ Update Environment if: steps.cache.outputs.cache-hit != 'true' + shell: bash -el {0} run: | mamba env update -n hypercp -f environment.yml mamba install --channel=conda-forge pyinstaller==6.6 - name: ๐Ÿ“ธ Capture Environment + shell: bash -el {0} run: | mamba info mamba list From 5b47934695b872415bb42cd8f6eb644a43632049 Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 17:50:39 +0200 Subject: [PATCH 05/10] Update Application Testing Workflow --- .github/workflows/ApplicationTesting.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml index ce23f2f..8411e51 100644 --- a/.github/workflows/ApplicationTesting.yml +++ b/.github/workflows/ApplicationTesting.yml @@ -12,9 +12,9 @@ jobs: name: Application Tests runs-on: ${{ matrix.os }} -# defaults: # Required for conda environment activation -# run: -# shell: bash -el {0} + defaults: # Required for conda environment activation + run: + shell: bash -el {0} strategy: fail-fast: false matrix: From 448ca0b849242ce834642158f41ff983f3a9bee9 Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 18:05:24 +0200 Subject: [PATCH 06/10] Update Application Testing Workflow --- .github/actions/SetupEnvironment/action.yml | 4 ++-- .github/workflows/ApplicationTesting.yml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/SetupEnvironment/action.yml b/.github/actions/SetupEnvironment/action.yml index f33c11f..427f217 100644 --- a/.github/actions/SetupEnvironment/action.yml +++ b/.github/actions/SetupEnvironment/action.yml @@ -8,10 +8,10 @@ runs: - name: ๐Ÿ Setup Mambaforge uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge miniforge-version: latest +# miniforge-variant: Mambaforge +# use-mamba: true # mambaforge is now deprecated in favor of miniforge activate-environment: hypercp - use-mamba: true - name: ๐Ÿ”‚ Cache Environment uses: actions/cache@v4 diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml index 8411e51..e5f69f2 100644 --- a/.github/workflows/ApplicationTesting.yml +++ b/.github/workflows/ApplicationTesting.yml @@ -36,4 +36,6 @@ jobs: - name: ๐Ÿงช Run Tests run: | - python run_sample_data.py + ls + pwd + python run_Sample_Data.py From 192c9fe6ff82b7a7d3611c74d63e3d90ac4a4204 Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 18:09:45 +0200 Subject: [PATCH 07/10] Update Application Testing Workflow --- run_Sample_Data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run_Sample_Data.py b/run_Sample_Data.py index 95d541c..d5a3656 100644 --- a/run_Sample_Data.py +++ b/run_Sample_Data.py @@ -51,8 +51,7 @@ ################################# ## PATH options -PATH_OS = os.path.expanduser("~") -PATH_HCP = os.path.join(PATH_OS,'GitRepos','HyperCP') # Local path to HyperCP repository. +PATH_HCP = os.path.dirname(os.path.abspath(__file__)) # Path to HyperCP repository on host # PATH_DATA = f"{PATH_OS}/Projects/HyperPACE/field_data/HyperSAS/{CRUISE}" # Top level data directory containing RAW/ and ancillary file. PATH_DATA = os.path.join(PATH_HCP,'Data','Sample_Data',PLATFORM) ################################## From 8f67f185c906d87c88ac10dce59bb75344817369 Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 18:33:40 +0200 Subject: [PATCH 08/10] Finalize Application Testing Workflow - run run_Sample_Data.py on macOS, Windows, and Linux everytime push on dev or master branch. Very primitive test of application. - refactored "Setup Environment" steps into composite action to reuse code from Bundle in ApplicationTesting --- .github/workflows/ApplicationTesting.yml | 3 +- .github/workflows/Build.yml | 45 ++++-------------------- run_Sample_Data.py | 4 +-- 3 files changed, 10 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ApplicationTesting.yml b/.github/workflows/ApplicationTesting.yml index e5f69f2..3b63cca 100644 --- a/.github/workflows/ApplicationTesting.yml +++ b/.github/workflows/ApplicationTesting.yml @@ -3,6 +3,7 @@ name: ApplicationTesting on: push: branches: + - master - dev workflow_dispatch: @@ -36,6 +37,4 @@ jobs: - name: ๐Ÿงช Run Tests run: | - ls - pwd python run_Sample_Data.py diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 22c2bdf..2d56625 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -24,51 +24,20 @@ jobs: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] steps: - - name: Get Branch Name - run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - id: extract_branch - - - name: Checkout Repository + - name: โฌ Checkout repository uses: actions/checkout@v4 with: - ref: ${{ steps.extract_branch.outputs.branch }} - - - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@v3 - with: - miniforge-variant: Mambaforge - miniforge-version: latest - activate-environment: hypercp - use-mamba: true - - - name: Cache Environment - uses: actions/cache@v4 - with: - path: ${{ env.CONDA }}/envs - key: - conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('environment.yml') }}-${{ env.CACHE_NUMBER }} - env: - # Increase this value to reset cache if environment.yml has not changed - CACHE_NUMBER: 0 - id: cache - - - name: Update Environment - if: steps.cache.outputs.cache-hit != 'true' - run: | - mamba env update -n hypercp -f environment.yml - mamba install --channel=conda-forge pyinstaller==6.6 + ref: ${{ github.event.inputs.branch }} - - name: Capture Environment - run: | - mamba info - mamba list + - name: ๐Ÿ”ง Setup Environment + uses: ./.github/actions/SetupEnvironment - - name: Make Bundle + - name: ๐Ÿ“‘ Make Bundle run: | python make.py echo "BUNDLE_NAME=$(ls Bundled/dist | head -1 | xargs)" >> $GITHUB_ENV - - name: Zip Bundle + - name: ๐Ÿ“ฅ Zip Bundle uses: thedoctor0/zip-release@0.7.1 with: type: 'zip' @@ -76,7 +45,7 @@ jobs: path: '${{ env.BUNDLE_NAME }}' filename: '${{ env.BUNDLE_NAME }}.zip' - - name: Upload Bundle + - name: โซ๏ธ Upload Bundle uses: actions/upload-artifact@v4 with: name: ${{ env.BUNDLE_NAME }} diff --git a/run_Sample_Data.py b/run_Sample_Data.py index d5a3656..fa3cabb 100644 --- a/run_Sample_Data.py +++ b/run_Sample_Data.py @@ -29,8 +29,8 @@ ################################################### CUSTOM SET UP ################################################### # Batch options -MULTI_TASK = True # Multiple threads for HyperSAS (any level) or TriOS (only L1A and up) -MULTI_LEVEL = False # Process raw (L0) to Level-2 (L2) +MULTI_TASK = False # Multiple threads for HyperSAS (any level) or TriOS (only L1A and up) +MULTI_LEVEL = True # Process raw (L0) to Level-2 (L2) CLOBBER = True # True overwrites existing files PROC_LEVEL = "L2" # Process to this level: L1A, L1AQC, L1B, LBQC, L2 (ignored for MULTI_LEVEL) From 1ad227343cc12a693055c9d165c00642461da194 Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 18:53:22 +0200 Subject: [PATCH 09/10] Fix command mode if Zhang_rho_db.mat needs to be downloaded - force download of Zhang in command mode as cannot open QWidget to prompt user as QApplication is not started (also likely running without X). - required for automated application testing --- Main.py | 2 +- Source/Utilities.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Main.py b/Main.py index 161d9be..85f0fda 100644 --- a/Main.py +++ b/Main.py @@ -620,7 +620,7 @@ def __init__( # Confirm that core data files are in place. Download if necessary. fpfZhang = os.path.join(CODE_HOME, "Data", "Zhang_rho_db.mat") if not os.path.exists(fpfZhang): - Utilities.downloadZhangDB(fpfZhang) + Utilities.downloadZhangDB(fpfZhang, force=True) # Create a default main config to be filled with cmd argument # to avoid reading the one generated with the GUI diff --git a/Source/Utilities.py b/Source/Utilities.py index fac6c72..74cd630 100644 --- a/Source/Utilities.py +++ b/Source/Utilities.py @@ -33,11 +33,11 @@ class Utilities: """A catchall class for HyperCP utilities""" @staticmethod - def downloadZhangDB(fpfZhang): + def downloadZhangDB(fpfZhang, force=False): infoText = " NEW INSTALLATION\nGlint database required.\nClick OK to download.\n\nWARNING: THIS IS A 2.5 GB DOWNLOAD.\n\n\ If canceled, Zhang et al. (2017) glint correction will fail. If download fails, a link and instructions will be provided in the terminal." - YNReply = Utilities.YNWindow("Database Download", infoText) - if YNReply == QMessageBox.Ok: + YNReply = True if force else Utilities.YNWindow("Database Download", infoText) == QMessageBox.Ok + if YNReply: url = "https://oceancolor.gsfc.nasa.gov/fileshare/dirk_aurin/Zhang_rho_db.mat" download_session = requests.Session() From 255ec0c68a72c8034c602e67b735026123e01d6d Mon Sep 17 00:00:00 2001 From: Nils Date: Mon, 16 Sep 2024 19:02:13 +0200 Subject: [PATCH 10/10] Fix missing directory Logs on first run from command line --- Source/Utilities.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source/Utilities.py b/Source/Utilities.py index 74cd630..49d5f2a 100644 --- a/Source/Utilities.py +++ b/Source/Utilities.py @@ -283,6 +283,10 @@ def YNWindow(winText,infoText): @staticmethod def writeLogFile(logText, mode='a'): + if not os.path.exists('Logs'): + import logging + logging.getLogger().warning('Made directory: Logs/') + os.mkdir('Logs') with open('Logs/' + os.environ["LOGFILE"], mode, encoding="utf-8") as logFile: logFile.write(logText + "\n")