Skip to content

Commit f7ebafa

Browse files
MryashbhardwajMryashbhardwaj
authored
fix: Remove sensor poke failure event (#520)
* fix: remove_sensor_poke_failure_events * fix: updated proton commit hash * fix: updated proton commit hash * fix: proton commit hash update
1 parent 6309c5c commit f7ebafa

File tree

9 files changed

+522
-271
lines changed

9 files changed

+522
-271
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ NAME = "github.com/odpf/optimus"
55
LAST_COMMIT := $(shell git rev-parse --short HEAD)
66
LAST_TAG := "$(shell git rev-list --tags --max-count=1)"
77
OPMS_VERSION := "$(shell git describe --tags ${LAST_TAG})-next"
8-
PROTON_COMMIT := "f90fd714cfc65894b4eb73642ae098e30081b672"
8+
PROTON_COMMIT := "16c082a44a3775d5114ccee368e91b7313721f39"
99

1010
.PHONY: build test test-ci generate-proto unit-test-ci smoke-test integration-test vet coverage clean install lint
1111

‎api/proto/odpf/optimus/core/v1beta1/job_spec.pb.go‎

Lines changed: 245 additions & 228 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/proto/odpf/optimus/core/v1beta1/status.pb.go‎

Lines changed: 232 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/third_party/openapi/odpf/optimus/core/v1beta1/job_run.swagger.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -604,10 +604,14 @@
604604
"TYPE_TASK_SUCCESS",
605605
"TYPE_TASK_START",
606606
"TYPE_TASK_FAIL",
607-
"TYPE_SENSOR_POKE",
607+
"TYPE_SENSOR_RETRY",
608608
"TYPE_SENSOR_SUCCESS",
609609
"TYPE_SENSOR_START",
610-
"TYPE_SENSOR_FAIL"
610+
"TYPE_SENSOR_FAIL",
611+
"TYPE_HOOK_START",
612+
"TYPE_HOOK_RETRY",
613+
"TYPE_HOOK_FAIL",
614+
"TYPE_HOOK_SUCCESS"
611615
],
612616
"default": "TYPE_UNSPECIFIED"
613617
},

‎api/third_party/openapi/odpf/optimus/core/v1beta1/job_spec.swagger.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -548,10 +548,14 @@
548548
"TYPE_TASK_SUCCESS",
549549
"TYPE_TASK_START",
550550
"TYPE_TASK_FAIL",
551-
"TYPE_SENSOR_POKE",
551+
"TYPE_SENSOR_RETRY",
552552
"TYPE_SENSOR_SUCCESS",
553553
"TYPE_SENSOR_START",
554-
"TYPE_SENSOR_FAIL"
554+
"TYPE_SENSOR_FAIL",
555+
"TYPE_HOOK_START",
556+
"TYPE_HOOK_RETRY",
557+
"TYPE_HOOK_FAIL",
558+
"TYPE_HOOK_SUCCESS"
555559
],
556560
"default": "TYPE_UNSPECIFIED"
557561
},

‎api/third_party/openapi/odpf/optimus/core/v1beta1/runtime.swagger.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,10 +176,14 @@
176176
"TYPE_TASK_SUCCESS",
177177
"TYPE_TASK_START",
178178
"TYPE_TASK_FAIL",
179-
"TYPE_SENSOR_POKE",
179+
"TYPE_SENSOR_RETRY",
180180
"TYPE_SENSOR_SUCCESS",
181181
"TYPE_SENSOR_START",
182-
"TYPE_SENSOR_FAIL"
182+
"TYPE_SENSOR_FAIL",
183+
"TYPE_HOOK_START",
184+
"TYPE_HOOK_RETRY",
185+
"TYPE_HOOK_FAIL",
186+
"TYPE_HOOK_SUCCESS"
183187
],
184188
"default": "TYPE_UNSPECIFIED"
185189
},

‎buf.gen.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ plugins:
77
# proto file should be.
88
# This is necessary while importing a proto file foo/a.proto from another
99
# directory, e.g. bar/b.proto
10-
opt: paths=source_relative,Modpf/optimus/core/v1beta1/runtime.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/project.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/namespace.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/job_spec.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/job_run.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1
10+
opt: paths=source_relative,Modpf/optimus/core/v1beta1/runtime.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/project.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/namespace.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/job_spec.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/job_run.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/status.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/resource.proto=github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1
1111
- name: go-grpc
1212
out: api/proto
1313
opt: paths=source_relative,require_unimplemented_servers=true
1414
- name: grpc-gateway
1515
out: api/proto
1616
opt: paths=source_relative
1717
- name: openapiv2
18-
out: api/third_party/openapi
18+
out: api/third_party/openapi

‎ext/scheduler/airflow2/resources/__lib.py‎

Lines changed: 24 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -253,44 +253,36 @@ def __init__(
253253
self._optimus_client = OptimusAPIClient(optimus_hostname)
254254

255255
def poke(self, context):
256+
log_start_event(context, EVENT_NAMES.get("SENSOR_START_EVENT"))
257+
schedule_time = context['next_execution_date']
258+
256259
try:
257-
log_start_event(context, EVENT_NAMES.get("SENSOR_START_EVENT"))
258-
schedule_time = context['next_execution_date']
259-
260-
# parse relevant metadata from the job metadata to build the task window
261-
# TODO this needs to be updated to use optimus get job spec
262-
try:
263-
upstream_schedule = self.get_schedule_interval(schedule_time)
264-
except Exception as e:
265-
self.log.warning("error while fetching upstream schedule :: {}".format(e))
266-
context[SCHEDULER_ERR_MSG] = "error while fetching upstream schedule :: {}".format(e)
267-
log_failure_event(context)
268-
return False
260+
upstream_schedule = self.get_schedule_interval(schedule_time)
261+
except Exception as e:
262+
self.log.warning("error while fetching upstream schedule :: {}".format(e))
263+
context[SCHEDULER_ERR_MSG] = "error while fetching upstream schedule :: {}".format(e)
264+
return False
269265

270-
last_upstream_schedule_time, _ = self.get_last_upstream_times(
271-
schedule_time, upstream_schedule)
266+
last_upstream_schedule_time, _ = self.get_last_upstream_times(
267+
schedule_time, upstream_schedule)
272268

273-
# get schedule window
274-
task_window = JobSpecTaskWindow(self.window_size, 0, "m", self._optimus_client)
275-
schedule_time_window_start, schedule_time_window_end = task_window.get_schedule_window(
276-
last_upstream_schedule_time.strftime(TIMESTAMP_FORMAT),upstream_schedule)
269+
# get schedule window
270+
task_window = JobSpecTaskWindow(self.window_size, 0, "m", self._optimus_client)
271+
schedule_time_window_start, schedule_time_window_end = task_window.get_schedule_window(
272+
last_upstream_schedule_time.strftime(TIMESTAMP_FORMAT),upstream_schedule)
277273

278274

279-
self.log.info("waiting for upstream runs between: {} - {} schedule times of airflow dag run".format(
280-
schedule_time_window_start, schedule_time_window_end))
275+
self.log.info("waiting for upstream runs between: {} - {} schedule times of airflow dag run".format(
276+
schedule_time_window_start, schedule_time_window_end))
281277

282-
# a = 0/0
283-
if not self._are_all_job_runs_successful(schedule_time_window_start, schedule_time_window_end):
284-
self.log.warning("unable to find enough successful executions for upstream '{}' in "
285-
"'{}' dated between {} and {}(inclusive), rescheduling sensor".
286-
format(self.optimus_job, self.optimus_project, schedule_time_window_start,
287-
schedule_time_window_end))
288-
log_failure_event(context)
289-
return False
290-
return True
291-
except Exception as e :
292-
context[SCHEDULER_ERR_MSG] = "error while sensor poke :: {}".format(e)
293-
log_failure_event(context)
278+
# a = 0/0
279+
if not self._are_all_job_runs_successful(schedule_time_window_start, schedule_time_window_end):
280+
self.log.warning("unable to find enough successful executions for upstream '{}' in "
281+
"'{}' dated between {} and {}(inclusive), rescheduling sensor".
282+
format(self.optimus_job, self.optimus_project, schedule_time_window_start,
283+
schedule_time_window_end))
284+
return False
285+
return True
294286

295287
def get_last_upstream_times(self, schedule_time_of_current_job, upstream_schedule_interval):
296288
second_ahead_of_schedule_time = schedule_time_of_current_job + timedelta(seconds=1)

‎go.mod‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ require (
6565
gorm.io/gorm v1.21.16
6666
)
6767

68-
require go.uber.org/multierr v1.7.0
69-
7068
require (
7169
cloud.google.com/go v0.94.0 // indirect
7270
cloud.google.com/go/storage v1.16.1 // indirect

0 commit comments

Comments
 (0)