diff --git a/evaluator/evaluation.py b/evaluator/evaluation.py
index 6932843e..9af91ada 100644
--- a/evaluator/evaluation.py
+++ b/evaluator/evaluation.py
@@ -1,5 +1,6 @@
 import dataclasses
 import io
+import logging
 import os
 import re
 import shlex
@@ -13,6 +14,8 @@
 from web.markdown_utils import load_readme
 from .script import Script
 
+logger = logging.getLogger(__name__)
+
 
 class File:
     def __init__(self, path):
@@ -261,7 +264,13 @@ def parse(config_path: str) -> "WorkflowConfigParseResult":
 
         try:
             with open(config_path) as f:
-                conf = yaml.load(f.read(), Loader=yaml.SafeLoader)
+                try:
+                    conf = yaml.load(f.read(), Loader=yaml.SafeLoader)
+                except:  # noqa
+                    logger.error(
+                        f"Cannot parse YML file at `{config_path}`:\n{traceback.format_exc()}"
+                    )
+                    raise InvalidWorkflowYaml("Invalid config.yml file")
                 if conf:
                     for key, value in conf.items():
                         if key == "queue":
@@ -321,6 +330,10 @@ class WorkflowValidationError(BaseException):
     pass
 
 
+class InvalidWorkflowYaml(WorkflowValidationError):
+    pass
+
+
 def parse_config_jobs(value: list[Any]) -> list[WorflowJob]:
     if not isinstance(value, list):
         raise WorkflowValidationError("Pipeline has to be a list of jobs")