Skip to content
This repository has been archived by the owner on Jun 15, 2024. It is now read-only.

Commit

Permalink
Migrate functionality onto new state-query interface (#131)
Browse files Browse the repository at this point in the history
  • Loading branch information
@flosell
flosell committed Sep 25, 2016
1 parent 104b6c8 commit cbccd1e
Show file tree
Hide file tree
Showing 8 changed files with 144 additions and 78 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ The official release will have a defined and more stable API. If you are already
* New state handling (#131):
* Protocols in `lambdacd.state.protocols` replace `lambdacd.internal.pipeline-state/PipelineStateComponent` which is now deprecated. Custom persistence-mechanisms need to migrate.
* Added facade `lambdacd.state.core` for all state-related functionality. Access directly to `PipelineStateComponent` is now deprecated.
* `lambdacd.presentation.pipeline-state/history-for` should now be called with ctx; Calling it with a build-state (the result of `lambdacd.internal.pipeline-state/get-all`) still works but is now deprecated.
* Breaking Changes:
* Moved pipeline-state-updater from `lambdacd.internal.pipeline-state` to `lambdacd.state.internal.pipeline-state-updater` and refactored interface. As this is an internal namespace, it should not affect users unless they customized LambdaCDs startup procedure to a large degree.

Expand Down
9 changes: 4 additions & 5 deletions src/clj/lambdacd/internal/execution.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,18 +226,17 @@
(and msg-from-child? msg-from-same-build?))))


(defn- publish-child-step-results [ctx retriggered-build-number original-build-result]
(defn- publish-child-step-results!! [ctx retriggered-build-number original-build-result]
(->> original-build-result
(filter #(step-id/parent-of? (:step-id ctx) (first %)))
(map #(send-step-result!! (assoc ctx :step-id (first %)) (assoc (second %) :retrigger-mock-for-build-number retriggered-build-number)))
(doall)))

(defn retrigger-mock-step [retriggered-build-number]
(fn [args ctx]
(let [state (legacy-pipeline-state/get-all (:pipeline-state-component ctx))
original-build-result (get state retriggered-build-number)
original-step-result (state/get-step-result ctx retriggered-build-number (:step-id ctx))]
(publish-child-step-results ctx retriggered-build-number original-build-result)
(let [original-build-result (state/get-step-results ctx retriggered-build-number)
original-step-result (get original-build-result (:step-id ctx))]
(publish-child-step-results!! ctx retriggered-build-number original-build-result)
(assoc original-step-result
:retrigger-mock-for-build-number retriggered-build-number))))

Expand Down
50 changes: 32 additions & 18 deletions src/clj/lambdacd/presentation/pipeline_state.clj
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
(ns lambdacd.presentation.pipeline-state
(:require [lambdacd.internal.pipeline-state :as pipeline-state]
[clj-time.core :as t]
(:require [clj-time.core :as t]
[clojure.tools.logging :as log]
[clj-timeframes.core :as tf]
[lambdacd.step-id :as step-id]))
[lambdacd.step-id :as step-id]
[lambdacd.state.core :as state]))

(defn- desc [a b]
(compare b a))
Expand Down Expand Up @@ -72,21 +72,35 @@
(map interval-duration)
(reduce +)))

(defn- history-entry [[build-number step-ids-and-results]]
{:build-number build-number
:status (overall-build-status step-ids-and-results)
:most-recent-update-at (latest-most-recent-update step-ids-and-results)
:first-updated-at (earliest-first-update step-ids-and-results)
:retriggered (build-that-was-retriggered step-ids-and-results)
:duration-in-sec (build-duration step-ids-and-results)})

(defn history-for [state]
(defn- history-entry
([build-number step-ids-and-results]
{:build-number build-number
:status (overall-build-status step-ids-and-results)
:most-recent-update-at (latest-most-recent-update step-ids-and-results)
:first-updated-at (earliest-first-update step-ids-and-results)
:retriggered (build-that-was-retriggered step-ids-and-results)
:duration-in-sec (build-duration step-ids-and-results)})
([[build-number step-ids-and-results]]
(history-entry build-number step-ids-and-results)))

(defn- legacy-history-for [state]
(sort-by :build-number (map history-entry state)))

(defn- history-for-pipeline-state-component [ctx]
(for [build-number (state/all-build-numbers ctx)]
(history-entry build-number (state/get-step-results ctx build-number))))

(defn history-for
"Returns a build history for a given build state (the result of get-all) or a ctx;
Calling this with a complete build state (the get-all-result) is now DEPRECATED"
[all-state-or-ctx]
(if (:pipeline-state-component all-state-or-ctx)
(history-for-pipeline-state-component all-state-or-ctx)
(legacy-history-for all-state-or-ctx)))

(defn most-recent-step-result-with [key ctx]
(let [state (pipeline-state/get-all (:pipeline-state-component ctx))
step-id (:step-id ctx)
step-results (map second (reverse (sort-by first (seq state))))
step-results-for-id (map #(get % step-id) step-results)
step-results-with-key (filter key step-results-for-id)]
(first step-results-with-key)))
(->> (state/all-build-numbers ctx)
(reverse)
(map #(state/get-step-result ctx % (:step-id ctx)))
(filter key)
(first)))
2 changes: 1 addition & 1 deletion src/clj/lambdacd/state/protocols.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
(defprotocol NextBuildNumberSource
"Components implementing this protocol provide the LambdaCD execution engine with new build numbers"
(next-build-number [self]
"Returns the build number for the next build. Must be comparable and greater than all existing build numbers"))
"Returns the build number for the next build. Must be comparable and greater than all existing build numbers")) ; TODO: is build number an int?

(defprotocol QueryAllBuildNumbersSource
"Components implementing this protocol can supply a list of all build numbers present in the datastore"
Expand Down
39 changes: 18 additions & 21 deletions src/clj/lambdacd/ui/api.clj
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,18 @@
(ns lambdacd.ui.api
(:require [lambdacd.util :as util]
[lambdacd.presentation.unified :as unified]
[lambdacd.runners :as runners]
[ring.util.response :as resp]
[clojure.string :as string]
[ring.middleware.json :as ring-json]
[lambdacd.presentation.pipeline-state :as state-presentation]
[lambdacd.internal.pipeline-state :as pipeline-state]
[lambdacd.execution :as execution]
[lambdacd.steps.manualtrigger :as manualtrigger]
[clojure.walk :as w]
[compojure.core :refer [routes GET POST]]))
[compojure.core :refer [routes GET POST]]
[lambdacd.state.core :as state]))

(defn- build-infos [pipeline-def pipeline-state buildnumber]
(let [build-number-as-int (util/parse-int buildnumber)
build-state (get pipeline-state build-number-as-int)]
(defn- build-infos [pipeline-def ctx buildnumber]
(let [build-state (state/get-step-results ctx (util/parse-int buildnumber))]
(if build-state
(util/json (unified/unified-presentation pipeline-def build-state))
(resp/not-found (str "build " buildnumber " does not exist")))))
Expand All @@ -23,18 +21,17 @@
(map util/parse-int (string/split dash-seperated-step-id #"-")))

(defn rest-api [{pipeline-def :pipeline-def ctx :context}]
(let [pipeline-state-component (:pipeline-state-component ctx)]
(ring-json/wrap-json-params
(routes
(GET "/builds/" [] (util/json (state-presentation/history-for (pipeline-state/get-all pipeline-state-component))))
(GET "/builds/:buildnumber/" [buildnumber] (build-infos pipeline-def (pipeline-state/get-all pipeline-state-component) buildnumber))
(POST "/builds/:buildnumber/:step-id/retrigger" [buildnumber step-id]
(let [new-buildnumber (execution/retrigger pipeline-def ctx (util/parse-int buildnumber) (to-internal-step-id step-id))]
(util/json {:build-number new-buildnumber})))
(POST "/builds/:buildnumber/:step-id/kill" [buildnumber step-id]
(do
(execution/kill-step ctx (util/parse-int buildnumber) (to-internal-step-id step-id))
"OK"))
(POST "/dynamic/:id" {{id :id} :params data :json-params} (do
(manualtrigger/post-id ctx id (w/keywordize-keys data))
(util/json {:status :success})))))))
(ring-json/wrap-json-params
(routes
(GET "/builds/" [] (util/json (state-presentation/history-for ctx)))
(GET "/builds/:buildnumber/" [buildnumber] (build-infos pipeline-def ctx buildnumber))
(POST "/builds/:buildnumber/:step-id/retrigger" [buildnumber step-id]
(let [new-buildnumber (execution/retrigger pipeline-def ctx (util/parse-int buildnumber) (to-internal-step-id step-id))]
(util/json {:build-number new-buildnumber})))
(POST "/builds/:buildnumber/:step-id/kill" [buildnumber step-id]
(do
(execution/kill-step ctx (util/parse-int buildnumber) (to-internal-step-id step-id))
"OK"))
(POST "/dynamic/:id" {{id :id} :params data :json-params} (do
(manualtrigger/post-id ctx id (w/keywordize-keys data))
(util/json {:status :success}))))))
53 changes: 29 additions & 24 deletions test/clj/lambdacd/internal/execution_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
[lambdacd.internal.pipeline-state :as ps]
[lambdacd.event-bus :as event-bus]
[lambdacd.state.internal.pipeline-state-updater :as pipeline-state-updater]
[lambdacd.steps.control-flow :as control-flow])
[lambdacd.steps.control-flow :as control-flow]
[lambdacd.state.core :as state])
(:import java.lang.IllegalStateException))

(defn some-step-processing-input [arg & _]
Expand Down Expand Up @@ -333,12 +334,14 @@
context (some-ctx-with :initial-pipeline-state initial-state)]
(retrigger pipeline context 0 [2] 1)
(wait-for (tu/step-success? context 1 [2]))
(is (= {0 {[1] { :status :success }
[1 1] {:status :success :out "I am nested"}
[2] { :status :failure }}
1 {[1] { :status :success :retrigger-mock-for-build-number 0 }
[1 1] {:status :success :out "I am nested" :retrigger-mock-for-build-number 0}
[2] { :status :success }}} (tu/without-ts (ps/get-all (:pipeline-state-component context)))))))
(is (= {[1] { :status :success }
[1 1] {:status :success :out "I am nested"}
[2] { :status :failure }}
(tu/without-ts (state/get-step-results context 0))))
(is (= {[1] { :status :success :retrigger-mock-for-build-number 0 }
[1 1] {:status :success :out "I am nested" :retrigger-mock-for-build-number 0}
[2] { :status :success }}
(tu/without-ts (state/get-step-results context 1))))))
(testing "that we can retrigger a pipeline from the initial step as well"
(let [pipeline `(some-successful-step some-other-step some-failing-step)
initial-state { 0 {[1] {:status :to-be-retriggered}
Expand All @@ -347,10 +350,10 @@
context (some-ctx-with :initial-pipeline-state initial-state)]
(retrigger pipeline context 0 [1] 1)
(wait-for (tu/step-failure? context 1 [3]))
(is (map-containing
{1 {[1] {:status :success}
[2] {:status :success :foo :baz}
[3] {:status :failure}}} (tu/without-ts (ps/get-all (:pipeline-state-component context)))))))
(is (= {[1] {:status :success}
[2] {:status :success :foo :baz}
[3] {:status :failure}}
(tu/without-ts (state/get-step-results context 1))))))
(testing "that steps after the retriggered step dont get the retrigger-metadata"
(let [pipeline `(some-successful-step some-step-caring-about-retrigger-metadata)
initial-state { 0 {[1] {:status :to-be-retriggered}
Expand All @@ -359,9 +362,9 @@
context (some-ctx-with :initial-pipeline-state initial-state)]
(retrigger pipeline context 0 [1] 1)
(wait-for (tu/step-success? context 1 [2]))
(is (map-containing
{1 {[1] {:status :success}
[2] {:status :success :retriggered-build-number nil :retriggered-step-id nil}}} (tu/without-ts (ps/get-all (:pipeline-state-component context)))))))
(is (= {[1] {:status :success}
[2] {:status :success :retriggered-build-number nil :retriggered-step-id nil}}
(tu/without-ts (state/get-step-results context 1))))))
(testing "that retriggering works for nested steps"
(let [initial-state { 0 {[1] { :status :success }
[1 1] {:status :success :out "I am nested"}
Expand All @@ -370,15 +373,17 @@
context (some-ctx-with :initial-pipeline-state initial-state)]
(retrigger pipeline context 0 [2 1] 1)
(wait-for (tu/step-success? context 1 [2]))
(is (= {0 {[1] { :status :success }
[1 1] {:status :success :out "I am nested"}
[2 1] {:status :unknown :out "this will be retriggered"}}
1 {[1] {:status :success
:outputs {[1 1] {:status :success :out "I am nested" :retrigger-mock-for-build-number 0 }
[2 1] {:the-some :val :status :success }}}
[1 1] {:status :success :out "I am nested" :retrigger-mock-for-build-number 0}
[2 1] {:the-some :val :status :success}
[2] { :status :success }}} (tu/without-ts (ps/get-all (:pipeline-state-component context)))))))
(is (= {[1] { :status :success }
[1 1] {:status :success :out "I am nested"}
[2 1] {:status :unknown :out "this will be retriggered"}}
(tu/without-ts (state/get-step-results context 0))))
(is (= {[1] {:status :success
:outputs {[1 1] {:status :success :out "I am nested" :retrigger-mock-for-build-number 0 }
[2 1] {:the-some :val :status :success }}}
[1 1] {:status :success :out "I am nested" :retrigger-mock-for-build-number 0}
[2 1] {:the-some :val :status :success}
[2] { :status :success }}
(tu/without-ts (state/get-step-results context 1))))))
(testing "that retriggering updates timestamps of container steps (#56)"
(let [initial-state { 0 {[1] { :status :unknown :first-updated-at (t/date-time 1970) }
[1 1] {:status :success :out "I am nested"}
Expand All @@ -387,7 +392,7 @@
context (some-ctx-with :initial-pipeline-state initial-state)]
(retrigger pipeline context 0 [2 1] 1)
(wait-for (tu/step-success? context 1 [2]))
(let [new-container-step-result (get-in (ps/get-all (:pipeline-state-component context)) [1 [1]])]
(let [new-container-step-result (state/get-step-result context 1 [1])]
(is (= :success (:status new-container-step-result)))
(is (not= 1970 (t/year (:first-updated-at new-container-step-result))))))))

Expand Down
58 changes: 55 additions & 3 deletions test/clj/lambdacd/presentation/pipeline_state_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
(:require [clojure.test :refer :all]
[lambdacd.presentation.pipeline-state :refer :all]
[lambdacd.testsupport.data :refer [some-ctx-with]]
[clj-time.core :as t]))
[clj-time.core :as t]
[lambdacd.state.protocols :as protocols]))

(def start-time (t/now))
(def before-start-time (t/minus start-time (t/seconds 10)))
Expand All @@ -18,7 +19,58 @@
(def after-one-minute-and-30-sec (t/plus start-time (t/minutes 1) (t/seconds 30)))
(def after-one-minute-and-40-sec (t/plus start-time (t/minutes 1) (t/seconds 40)))

(deftest history-test
(defrecord in-memory-pipeline-state [state]
protocols/QueryAllBuildNumbersSource
(all-build-numbers [self]
(keys state))
protocols/QueryBuildSource
(get-build [self build-number]
{:step-results (get state build-number)}))

(defn ctx-with-state [state]
(some-ctx-with :pipeline-state-component (->in-memory-pipeline-state state)))

(deftest history-for-ctx-test
(testing "that we can aggregate the pipeline state into a nice history representation"
(is (= [{:build-number 8
:status :waiting
:most-recent-update-at stop-time
:first-updated-at start-time
:retriggered nil
:duration-in-sec 10}
{:build-number 9
:status :running
:most-recent-update-at stop-time
:first-updated-at start-time
:retriggered nil
:duration-in-sec 10}]
(history-for (ctx-with-state {8 {'(0) {:status :waiting :most-recent-update-at stop-time :first-updated-at start-time}}
9 {'(0) {:status :running :most-recent-update-at stop-time :first-updated-at start-time}}})))))
(testing "that the timestamps are accumulated correctly"
(testing "that the earliest start time is the start time of the pipeline"
(is (= before-start-time (:first-updated-at (first
(history-for (ctx-with-state {7 {'(2) {:status :success :most-recent-update-at stop-time :first-updated-at start-time}
'(1) {:status :success :most-recent-update-at stop-time :first-updated-at before-start-time}}})))))))
(testing "that retriggered steps are ignored when searching the earliest start time"
(is (= start-time (:first-updated-at (first
(history-for (ctx-with-state {7 {'(2) {:status :success :most-recent-update-at stop-time :first-updated-at start-time}
'(1) {:status :success :most-recent-update-at stop-time :first-updated-at before-start-time :retrigger-mock-for-build-number 42}}})))))))
(testing "that the most recent update will be the pipelines most recent update"
(is (= after-stop-time (:most-recent-update-at (first
(history-for (ctx-with-state {5 {'(0) {:status :success :most-recent-update-at stop-time :first-updated-at start-time}
'(1) {:status :success :most-recent-update-at after-stop-time :first-updated-at start-time}}}))))))))
(testing "that the build-status is accumulated correctly"
(testing "that a pipeline will be a failure once there is a failed step"
(is (= :failure (:status (first
(history-for (ctx-with-state {7 {'(1) {:status :success :most-recent-update-at stop-time :first-updated-at start-time}
'(2) {:status :failure :most-recent-update-at stop-time :first-updated-at start-time}}}))))))))
(testing "that we detect retriggered steps"
(testing "that a pipeline will be treated as retriggered if the first step has a retrigger-mock"
(is (= 3 (:retriggered (first
(history-for (ctx-with-state {7 {'(2) {:status :success :most-recent-update-at stop-time :first-updated-at start-time}
'(1) {:status :success :most-recent-update-at stop-time :first-updated-at before-start-time :retrigger-mock-for-build-number 3}}})))))))))

(deftest history-test-legacy
(testing "that we can aggregate the pipeline state into a nice history representation"
(is (= [{:build-number 8
:status :waiting
Expand Down Expand Up @@ -134,4 +186,4 @@
(testing "durations of child-steps aren't considered"
(is (= 20 (build-duration {'(1 2) {:status :success :most-recent-update-at after-thirty-sec :first-updated-at after-ten-sec}
'(2) {:status :success :most-recent-update-at after-twenty-sec :first-updated-at after-ten-sec}
'(1) {:status :success :most-recent-update-at after-ten-sec :first-updated-at start-time}})))))
'(1) {:status :success :most-recent-update-at after-ten-sec :first-updated-at start-time}})))))
Loading

0 comments on commit cbccd1e

Please sign in to comment.