diff --git a/.github/workflows/nested_eagle.yml b/.github/workflows/nested_eagle.yml index 8f04ac7..b9c05e8 100644 --- a/.github/workflows/nested_eagle.yml +++ b/.github/workflows/nested_eagle.yml @@ -32,7 +32,7 @@ jobs: JOB_ID=$(sbatch submit_grids.sh | awk '{print $4}') echo "Submitted Slurm job with ID: $JOB_ID" - + while squeue -j $JOB_ID &>/dev/null; do echo "Slurm job $JOB_ID is still running or pending. Waiting..." sleep 30 # Wait for 30 seconds before checking again @@ -59,7 +59,6 @@ jobs: squeue -j ${JOB_ID1},${JOB_ID2} sleep 30 # Wait for 30 seconds before checking again. done - echo "Slurm jobs $JOB_ID1 and $JOB_ID2 to prepare the GFS & HRRR training data have completed." Run_Training: @@ -82,9 +81,8 @@ jobs: sleep 30 # Wait for 30 seconds before checking again. done cd $CHECKPOINT - lsof -w "$(find . -name inference-last.ckpt)" && sleep 10 || echo "file is written" + lsof -w "$(find . -name inference-last.ckpt)" && sleep 10 || echo "file is written" scancel $JOB_ID - echo "Slurm job to train the data has been completed." Run_Inference: @@ -100,14 +98,14 @@ jobs: JOB_ID=$(sbatch submit_inference.sh | awk '{print $4}') echo "Submitted Inference Slurm job with ID: $JOB_ID" - + while squeue -h -j ${JOB_ID} | grep -q "${JOB_ID}"; do squeue -j $JOB_ID sleep 30 # Wait for 30 seconds before checking again. done cd $GITHUB_WORKSPACE/nested_eagle/ursa/scientific_workflow/inference - find ./inference_files -name "*.nc" -print -quit | grep -q . || { echo "Error: No inference files found."; exit 1; } + find ./inference_files -name "*.nc" -print -quit | grep -q . || { echo "Error: No inference files found."; exit 1; } echo "Slurm job $JOB_ID to run inference on the data has been completed." Perform_Inference_Postprocessing: @@ -126,15 +124,15 @@ jobs: JOB_ID=$(sbatch submit_postprocessing.sh | awk '{print $4}') echo "Submitted Validation Slurm job with ID: $JOB_ID" - + while squeue -h -j ${JOB_ID} | grep -q "${JOB_ID}"; do squeue -j $JOB_ID sleep 30 # Wait for 30 seconds before checking again. done cd $GITHUB_WORKSPACE/nested_eagle/ursa/scientific_workflow/verification/prewxvx/postprocessed_files - find ./global -name "*.nc" -print -quit | grep -q . || { echo "Error: No global postprocessed files found."; exit 1; } - find ./lam -name "*.nc" -print -quit | grep -q . || { echo "Error: No lam postprocessed files found."; exit 1; } + find ./global -name "*.nc" -print -quit | grep -q . || { echo "Error: No global postprocessed files found."; exit 1; } + find ./lam -name "*.nc" -print -quit | grep -q . || { echo "Error: No lam postprocessed files found."; exit 1; } echo "Slurm job $JOB_ID to perform inference postprocessing has been completed." Perform_Verification: @@ -160,13 +158,13 @@ jobs: done cd $GITHUB_WORKSPACE/nested_eagle/ursa/scientific_workflow/verification/grid2obs/wxvx_workdir - find ./lam -name "*.png" -print -quit | grep -q . || { echo "Error: No LAM grid2obs verification plots found."; exit 1; } - find ./global -name "*.png" -print -quit | grep -q . || { echo "Error: No global grid2obs verification plots found."; exit 1; } + find ./lam -name "*.png" -print -quit | grep -q . || { echo "Error: No LAM grid2obs verification plots found."; exit 1; } + find ./global -name "*.png" -print -quit | grep -q . || { echo "Error: No global grid2obs verification plots found."; exit 1; } echo "Grid2obs Verification process has been successfully completed." cd $GITHUB_WORKSPACE/nested_eagle/ursa/scientific_workflow/verification/grid2grid/wxvx_workdir - find ./lam -name "*.png" -print -quit | grep -q . || { echo "Error: No LAM grid2grid verification plots found."; exit 1; } - find ./global -name "*.png" -print -quit | grep -q . || { echo "Error: No global grid2grid verification plots found."; exit 1; } + find ./lam -name "*.png" -print -quit | grep -q . || { echo "Error: No LAM grid2grid verification plots found."; exit 1; } + find ./global -name "*.png" -print -quit | grep -q . || { echo "Error: No global grid2grid verification plots found."; exit 1; } echo "Grid2grid Verification process has been successfully completed." - name: Remove label from pull request if: always() diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml new file mode 100644 index 0000000..7ac98c1 --- /dev/null +++ b/.github/workflows/yamllint.yml @@ -0,0 +1,34 @@ +name: Lint YAML files + +on: + push: + branches: [main, feature/*] + paths: [nested_eagle/**, .github/**, docs/**] + pull_request: + types: [opened, reopened, synchronize] + paths: [nested_eagle/**, .github/**, docs/**] + workflow_dispatch: + +jobs: + yamllint: + name: Run yamllint + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Optional caching for pip dependencies + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install yamllint + run: | + python -m pip install --upgrade pip + pip install yamllint + + - name: Run yamllint + run: | + yamllint --no-warnings --format colored . diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..518bf19 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,8 @@ +extends: default + +ignore: global_eagle + +rules: + line-length: + max: 100 + level: warning diff --git a/nested_eagle/ursa/scientific_workflow/training/config.yaml b/nested_eagle/ursa/scientific_workflow/training/config.yaml index f27422b..33fc99b 100644 --- a/nested_eagle/ursa/scientific_workflow/training/config.yaml +++ b/nested_eagle/ursa/scientific_workflow/training/config.yaml @@ -142,21 +142,21 @@ diagnostics: every_n_batches: ${diagnostics.plot.frequency.batch} #Defining the accumulation levels for precipitation related fields and the colormap accumulation_levels_plot: [ - 0, - 0.05, - 0.1, - 0.25, - 0.5, - 1, - 1.5, - 2, - 3, - 4, - 5, - 6, - 7, - 100, - ] # in mm + 0, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 1.5, + 2, + 3, + 4, + 5, + 6, + 7, + 100, + ] # in mm precip_and_related_fields: ${diagnostics.plot.precip_and_related_fields} colormaps: ${diagnostics.plot.colormaps} - _target_: anemoi.training.diagnostics.callbacks.plot.PlotSpectrum diff --git a/nested_eagle/ursa/scientific_workflow/training/diagnostics/plot/detailed.yaml b/nested_eagle/ursa/scientific_workflow/training/diagnostics/plot/detailed.yaml index 1ab1ab8..a35fe83 100644 --- a/nested_eagle/ursa/scientific_workflow/training/diagnostics/plot/detailed.yaml +++ b/nested_eagle/ursa/scientific_workflow/training/diagnostics/plot/detailed.yaml @@ -74,21 +74,21 @@ callbacks: every_n_batches: ${diagnostics.plot.frequency.batch} #Defining the accumulation levels for precipitation related fields and the colormap accumulation_levels_plot: [ - 0, - 0.05, - 0.1, - 0.25, - 0.5, - 1, - 1.5, - 2, - 3, - 4, - 5, - 6, - 7, - 100, - ] # in mm + 0, + 0.05, + 0.1, + 0.25, + 0.5, + 1, + 1.5, + 2, + 3, + 4, + 5, + 6, + 7, + 100, + ] # in mm precip_and_related_fields: ${diagnostics.plot.precip_and_related_fields} colormaps: ${diagnostics.plot.colormaps} - _target_: anemoi.training.diagnostics.callbacks.plot.PlotSpectrum