From 6d699564a1fd674bed97ded4b32f4a8c8616f3ee Mon Sep 17 00:00:00 2001 From: Qian Huang Date: Thu, 9 Jan 2025 15:53:47 -0800 Subject: [PATCH 1/8] fix bug for fq_name --- metaflow/plugins/events_decorator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 2d13096aa4a..b2ad1b43c30 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -599,7 +599,7 @@ def format_deploytime_value(self): if isinstance(trigger, DeployTimeField): trigger = deploy_time_eval(trigger) if isinstance(trigger, dict): - trigger["fq_name"] = trigger.get("name") + trigger["fq_name"] = trigger.get("fq_name") trigger["project"] = trigger.get("project") trigger["branch"] = trigger.get("project_branch") # We also added this bc it won't be formatted yet @@ -607,6 +607,6 @@ def format_deploytime_value(self): trigger = {"fq_name": trigger} trigger = self._parse_fq_name(trigger) self.triggers[self.triggers.index(old_trig)] = trigger - + def get_top_level_options(self): return list(self._option_values.items()) From dc5a5d812e2e58a82caf2b134269c2a4cc31959a Mon Sep 17 00:00:00 2001 From: Qian Huang Date: Mon, 13 Jan 2025 11:55:42 -0800 Subject: [PATCH 2/8] formatting --- metaflow/plugins/events_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index b2ad1b43c30..5a850d08404 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -607,6 +607,6 @@ def format_deploytime_value(self): trigger = {"fq_name": trigger} trigger = self._parse_fq_name(trigger) self.triggers[self.triggers.index(old_trig)] = trigger - + def get_top_level_options(self): return list(self._option_values.items()) From 8195ab3f3c679767361efad9713bd8bbf87e367a Mon Sep 17 00:00:00 2001 From: Shashank Srikanth Date: Mon, 13 Jan 2025 16:43:06 -0800 Subject: [PATCH 3/8] Make it work for deploy time trigger on finish as well --- metaflow/plugins/events_decorator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 5a850d08404..771fbe227b9 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -598,6 +598,10 @@ def format_deploytime_value(self): old_trig = trigger if isinstance(trigger, DeployTimeField): trigger = deploy_time_eval(trigger) + if isinstance(trigger, dict) and "fq_name" not in trigger: + # When a user has a deploy time trigger on finish which returns name, + # project, branch + trigger["fq_name"] = trigger.get("name") if isinstance(trigger, dict): trigger["fq_name"] = trigger.get("fq_name") trigger["project"] = trigger.get("project") From 7e94fcf7474df59358e6a0ecc3aad1aa873ab021 Mon Sep 17 00:00:00 2001 From: Shashank Srikanth Date: Mon, 13 Jan 2025 17:31:22 -0800 Subject: [PATCH 4/8] Support both string and dict in deploy time trigger on finish --- metaflow/plugins/events_decorator.py | 34 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 771fbe227b9..02cd4901b5e 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -598,10 +598,36 @@ def format_deploytime_value(self): old_trig = trigger if isinstance(trigger, DeployTimeField): trigger = deploy_time_eval(trigger) - if isinstance(trigger, dict) and "fq_name" not in trigger: - # When a user has a deploy time trigger on finish which returns name, - # project, branch - trigger["fq_name"] = trigger.get("name") + if is_stringish(trigger): + trigger = {"fq_name": trigger} + elif isinstance(trigger, dict): + if "name" not in trigger: + raise MetaflowException( + "The *flow* attribute for *@trigger_on_finish* is missing the " + "*name* key." + ) + flow_name = trigger["name"] + + if not is_stringish(flow_name) or "." in flow_name: + raise MetaflowException( + "The *name* attribute of the *flow* is not a valid string" + ) + result = {"fq_name": flow_name} + if "project" in trigger: + if is_stringish(trigger["project"]): + result["project"] = trigger["project"] + else: + raise MetaflowException( + "The *project* attribute of the *flow* is not a string" + ) + if "project_branch" in trigger: + if is_stringish(trigger["project_branch"]): + result["branch"] = trigger["project_branch"] + else: + raise MetaflowException( + "The *project_branch* attribute of the *flow* is not a string" + ) + trigger = result if isinstance(trigger, dict): trigger["fq_name"] = trigger.get("fq_name") trigger["project"] = trigger.get("project") From 45c6a2f932b5ab29f1f2dc4adcd00c18ee64b515 Mon Sep 17 00:00:00 2001 From: Shashank Srikanth Date: Mon, 13 Jan 2025 17:35:28 -0800 Subject: [PATCH 5/8] Update deploy time trigger logic for strings --- metaflow/plugins/events_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 02cd4901b5e..6cb941e479e 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -599,7 +599,7 @@ def format_deploytime_value(self): if isinstance(trigger, DeployTimeField): trigger = deploy_time_eval(trigger) if is_stringish(trigger): - trigger = {"fq_name": trigger} + pass elif isinstance(trigger, dict): if "name" not in trigger: raise MetaflowException( From 7dfa81e107d7f1310933d82a700f61f0104f590b Mon Sep 17 00:00:00 2001 From: Qian Huang Date: Tue, 14 Jan 2025 14:41:48 -0800 Subject: [PATCH 6/8] add a comment --- metaflow/plugins/events_decorator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 6cb941e479e..458156bf31a 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -597,6 +597,7 @@ def format_deploytime_value(self): # Entire trigger is a function (returns either string or dict) old_trig = trigger if isinstance(trigger, DeployTimeField): + # convert the trigger to string or dict trigger = deploy_time_eval(trigger) if is_stringish(trigger): pass @@ -628,6 +629,7 @@ def format_deploytime_value(self): "The *project_branch* attribute of the *flow* is not a string" ) trigger = result + # effect is to set all fields to None if they don't exist. if isinstance(trigger, dict): trigger["fq_name"] = trigger.get("fq_name") trigger["project"] = trigger.get("project") From ef5c89e70633c6ed6e061492050511032986d8bb Mon Sep 17 00:00:00 2001 From: Shashank Srikanth Date: Wed, 15 Jan 2025 14:33:17 -0800 Subject: [PATCH 7/8] Update project and branching logic for deploy time triggers --- metaflow/plugins/events_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 458156bf31a..53d4d797afd 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -623,7 +623,7 @@ def format_deploytime_value(self): ) if "project_branch" in trigger: if is_stringish(trigger["project_branch"]): - result["branch"] = trigger["project_branch"] + result["project_branch"] = trigger["project_branch"] else: raise MetaflowException( "The *project_branch* attribute of the *flow* is not a string" From 6e6a11eac9c2c35205003ee4ad5e9bea8b5dfb13 Mon Sep 17 00:00:00 2001 From: Shashank Srikanth Date: Wed, 15 Jan 2025 22:49:14 -0800 Subject: [PATCH 8/8] Address bug in project branch mapping for static triggers --- metaflow/plugins/events_decorator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index 53d4d797afd..cc29b5c2234 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -422,7 +422,9 @@ def flow_init( ) if "project_branch" in self.attributes["flow"]: if is_stringish(self.attributes["flow"]["project_branch"]): - result["branch"] = self.attributes["flow"]["project_branch"] + result["project_branch"] = self.attributes["flow"][ + "project_branch" + ] else: raise MetaflowException( "The *project_branch* attribute of the *flow* is not a string" @@ -467,6 +469,7 @@ def flow_init( "The *name* attribute '%s' is not a valid string" % str(flow_name) ) + result = {"fq_name": flow_name} if "project" in flow: if is_stringish(flow["project"]): @@ -478,7 +481,7 @@ def flow_init( ) if "project_branch" in flow: if is_stringish(flow["project_branch"]): - result["branch"] = flow["project_branch"] + result["project_branch"] = flow["project_branch"] else: raise MetaflowException( "The *project_branch* attribute of the *flow* %s "