Skip to content
This repository has been archived by the owner on Jun 15, 2024. It is now read-only.

Commit

Permalink
Allow persistence of pipeline structure (#131, #6)
Browse files Browse the repository at this point in the history
  • Loading branch information
@flosell
flosell committed Sep 25, 2016
1 parent ca2a5ea commit c2135bf
Show file tree
Hide file tree
Showing 9 changed files with 141 additions and 57 deletions.
5 changes: 4 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ The official release will have a defined and more stable API. If you are already

## 0.11.0

* Keeps a history of pipeline structure if persistence component supports it (#131, #6)
* API changes:
* New state handling (#131):
* Protocols in `lambdacd.state.protocols` replace `lambdacd.internal.pipeline-state/PipelineStateComponent` which is now deprecated. Custom persistence-mechanisms need to migrate.
* Added facade `lambdacd.state.core` for all state-related functionality. Access directly to `PipelineStateComponent` is now deprecated.
* `lambdacd.presentation.pipeline-state/history-for` should now be called with ctx; Calling it with a build-state (the result of `lambdacd.internal.pipeline-state/get-all`) still works but is now deprecated.
* `lambdacd.presentation.pipeline-state/history-for` should now be called with ctx; Calling it with a build-state (the result of `lambdacd.internal.pipeline-state/get-all`) still works but is now deprecated.
* `lambdacd.presentation.unified/unified-presentation` is now deprecated, use `lambdacd.presentation.unified/pipeline-structure-with-step-results` instead
* The current pipeline-definition can now be accessed as `:pipeline-def` in ctx
* Breaking Changes:
* Moved pipeline-state-updater from `lambdacd.internal.pipeline-state` to `lambdacd.state.internal.pipeline-state-updater` and refactored interface. As this is an internal namespace, it should not affect users unless they customized LambdaCDs startup procedure to a large degree.

Expand Down
1 change: 1 addition & 0 deletions src/clj/lambdacd/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
(event-bus/initialize-event-bus)
(running-builds-tracking/initialize-running-builds-tracking)
(assoc :pipeline-state-component pipeline-state-component)
(assoc :pipeline-def pipeline-def)
(initialize-pipeline-state-updater)
(add-shutdown-sequence!))]
{:context context
Expand Down
13 changes: 8 additions & 5 deletions src/clj/lambdacd/internal/execution.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
[lambdacd.steps.status :as status]
[clojure.repl :as repl]
[lambdacd.event-bus :as event-bus]
[lambdacd.steps.result :as step-results])
[lambdacd.steps.result :as step-results]
[lambdacd.presentation.pipeline-structure :as pipeline-structure])
(:import (java.io StringWriter)
(java.util UUID)))

Expand Down Expand Up @@ -298,18 +299,20 @@

(defn run [pipeline context]
(let [build-number (state/next-build-number context)]
(state/consume-pipeline-structure context build-number (pipeline-structure/pipeline-display-representation pipeline))
(let [runnable-pipeline (map eval pipeline)]
(execute-steps runnable-pipeline {} (merge context {:result-channel (async/chan (async/dropping-buffer 0))
:step-id []
:build-number build-number})))))
:step-id []
:build-number build-number})))))

(defn retrigger [pipeline context build-number step-id-to-run next-build-number]
(let [executable-pipeline (map eval pipeline) ]
(let [executable-pipeline (map eval pipeline)]
(state/consume-pipeline-structure context build-number (pipeline-structure/pipeline-display-representation pipeline))
(execute-steps executable-pipeline {} (assoc context :step-id []
:result-channel (async/chan (async/dropping-buffer 0))
:build-number next-build-number
:retriggered-build-number build-number
:retriggered-step-id step-id-to-run))))
:retriggered-step-id step-id-to-run))))

(defn retrigger-async [pipeline context build-number step-id-to-run]
(let [next-build-number (state/next-build-number context)]
Expand Down
21 changes: 12 additions & 9 deletions src/clj/lambdacd/presentation/unified.clj
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
(ns lambdacd.presentation.unified
(:require [lambdacd.presentation.pipeline-structure :as pipeline-structure]))

(defn- unify [step build-state]
(let [step-id (:step-id step)
result-for-step (get build-state step-id {})
with-result (assoc step :result result-for-step )
children (:children step)
unified-children (map #(unify % build-state) children)
with-children (assoc with-result :children unified-children)]
(defn- unify-step [step build-state]
(let [step-id (:step-id step)
result-for-step (get build-state step-id {})
with-result (assoc step :result result-for-step)
children (:children step)
unified-children (map #(unify-step % build-state) children)
with-children (assoc with-result :children unified-children)]
with-children))

(defn unified-presentation [pipeline-def build-state]
(defn pipeline-structure-with-step-results [pipeline-structure step-results]
(map #(unify-step % step-results) pipeline-structure))

(defn unified-presentation [pipeline-def step-results]
(let [structure (pipeline-structure/pipeline-display-representation pipeline-def)]
(map #(unify % build-state) structure)))
(pipeline-structure-with-step-results structure step-results)))
15 changes: 14 additions & 1 deletion src/clj/lambdacd/state/core.clj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
(ns lambdacd.state.core
"Facade for all functions related to dealing with LambdaCDs state. Wraps the related interfaces to simplify compatibility and API."
(:require [lambdacd.state.protocols :as protocols]
[lambdacd.internal.pipeline-state :as legacy-pipeline-state]))
[lambdacd.internal.pipeline-state :as legacy-pipeline-state]
[lambdacd.presentation.pipeline-structure :as pipeline-structure]))

(defn- all-build-numbers-from-legacy [component]
(->> (legacy-pipeline-state/get-all component)
Expand All @@ -18,6 +19,7 @@
(:pipeline-state-component ctx))

; -------------------------------------------------------------------------

(defn consume-step-result-update
"Update a step-result in the state"
[ctx build-number step-id step-result]
Expand All @@ -26,6 +28,11 @@
(protocols/consume-step-result-update component build-number step-id step-result)
(legacy-pipeline-state/update component build-number step-id step-result))))

(defn consume-pipeline-structure [ctx build-number pipeline-structure-representation]
(let [component (state-component ctx)]
(if (satisfies? protocols/PipelineStructureConsumer component)
(protocols/consume-pipeline-structure component build-number pipeline-structure-representation))))

(defn next-build-number
"Returns the build number for the next build"
[ctx]
Expand Down Expand Up @@ -60,3 +67,9 @@
[ctx build-number step-id]
(get (get-step-results ctx build-number)
step-id))

(defn get-pipeline-structure
"Returns a map describing the structure of the pipeline"
[ctx build-number]
(or (:pipeline-structure (get-build ctx build-number))
(pipeline-structure/pipeline-display-representation (:pipeline-def ctx))))
7 changes: 6 additions & 1 deletion src/clj/lambdacd/state/protocols.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
(consume-step-result-update [self build-number step-id step-result]
"Tells the component to update the result of a particular step. Is called on every update so it needs to handle lots of requests"))

(defprotocol PipelineStructureConsumer
"Components implementing this protocol can set the structure a pipeline had for a particular build"
(consume-pipeline-structure [self build-number pipeline-structure-representation]
"Tells the component to update the structure of a particular build."))

(defprotocol NextBuildNumberSource
"Components implementing this protocol provide the LambdaCD execution engine with new build numbers"
(next-build-number [self]
Expand All @@ -19,4 +24,4 @@
(defprotocol QueryBuildSource
"Components implementing this protocol can supply information on one build"
(get-build [self build-number]
"Returns build information as a map with :step-results, and TODO"))
"Returns build information as a map with :step-results, :pipeline-structure and TODO"))
14 changes: 8 additions & 6 deletions src/clj/lambdacd/ui/api.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@
[compojure.core :refer [routes GET POST]]
[lambdacd.state.core :as state]))

(defn- build-infos [pipeline-def ctx buildnumber]
(let [build-state (state/get-step-results ctx (util/parse-int buildnumber))]
(if build-state
(util/json (unified/unified-presentation pipeline-def build-state))
(resp/not-found (str "build " buildnumber " does not exist")))))
(defn- build-infos [ctx build-number-str]
(let [build-number (util/parse-int build-number-str)
pipeline-structure (state/get-pipeline-structure ctx build-number)
step-results (state/get-step-results ctx build-number)]
(if (and pipeline-structure step-results)
(util/json (unified/pipeline-structure-with-step-results pipeline-structure step-results))
(resp/not-found (str "build " build-number-str " does not exist")))))

(defn- to-internal-step-id [dash-seperated-step-id]
(map util/parse-int (string/split dash-seperated-step-id #"-")))
Expand All @@ -24,7 +26,7 @@
(ring-json/wrap-json-params
(routes
(GET "/builds/" [] (util/json (state-presentation/history-for ctx)))
(GET "/builds/:buildnumber/" [buildnumber] (build-infos pipeline-def ctx buildnumber))
(GET "/builds/:buildnumber/" [buildnumber] (build-infos ctx buildnumber))
(POST "/builds/:buildnumber/:step-id/retrigger" [buildnumber step-id]
(let [new-buildnumber (execution/retrigger pipeline-def ctx (util/parse-int buildnumber) (to-internal-step-id step-id))]
(util/json {:build-number new-buildnumber})))
Expand Down
95 changes: 62 additions & 33 deletions test/clj/lambdacd/presentation/unified_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,72 @@


(def foo-pipeline-build-state
{'(1) { :status :running}
{'(1) {:status :running}
'(1 1 1) {:status :failure
:out "do stuff failed"}
'(1 2 1) {:status :running
:out "do stuff failed"}
'(1 2 1) {:status :running
:some-key :some-value}})

(def expected-unified-foo-pipeline-presentation
[{:name "in-parallel"
:type :parallel
:step-id '(1)
:has-dependencies false
:result {:status :running }
:children
[{:name "in-cwd"
:type :container
:step-id '(1 1)
:has-dependencies false
:result {}
:children [{:name "do-stuff"
:type :step
:step-id '(1 1 1)
:has-dependencies false
:children []
:result {:status :failure
:out "do stuff failed"}}]}
{:name "in-cwd"
:type :container
:step-id '(2 1)
:has-dependencies false
:result {}
:children [{:name "do-other-stuff"
:type :step
:step-id '(1 2 1)
:has-dependencies false
:children []
:result {:status :running :some-key :some-value}}]}]}])
[{:name "in-parallel"
:type :parallel
:step-id '(1)
:has-dependencies false
:result {:status :running}
:children
[{:name "in-cwd"
:type :container
:step-id '(1 1)
:has-dependencies false
:result {}
:children [{:name "do-stuff"
:type :step
:step-id '(1 1 1)
:has-dependencies false
:children []
:result {:status :failure
:out "do stuff failed"}}]}
{:name "in-cwd"
:type :container
:step-id '(2 1)
:has-dependencies false
:result {}
:children [{:name "do-other-stuff"
:type :step
:step-id '(1 2 1)
:has-dependencies false
:children []
:result {:status :running :some-key :some-value}}]}]}])

(def foo-pipeline-structure
[{:name "in-parallel"
:type :parallel
:step-id '(1)
:has-dependencies false
:children
[{:name "in-cwd"
:type :container
:step-id '(1 1)
:has-dependencies false
:children [{:name "do-stuff"
:type :step
:step-id '(1 1 1)
:has-dependencies false
:children []}]}
{:name "in-cwd"
:type :container
:step-id '(2 1)
:has-dependencies false
:result {}
:children [{:name "do-other-stuff"
:type :step
:step-id '(1 2 1)
:has-dependencies false
:children []}]}]}])

(deftest unified-presentation-test
(testing "that we can merge structure and state to a unified view on a pipeline-run"
(is (= expected-unified-foo-pipeline-presentation (unified-presentation foo-pipeline foo-pipeline-build-state)))))
(testing "deprecated call with pipeline-def"
(is (= expected-unified-foo-pipeline-presentation (unified-presentation foo-pipeline foo-pipeline-build-state))))
(testing "call with build-result"
(is (= expected-unified-foo-pipeline-presentation (pipeline-structure-with-step-results foo-pipeline-structure foo-pipeline-build-state))))))
27 changes: 26 additions & 1 deletion test/clj/lambdacd/state/core_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
(def some-build-number 42)
(def some-step-id [0])
(def some-step-result {:foo :bat})
(def some-structure {:some :structure})
(def some-pipeline-def `(foo))

(deftest consume-step-result-update-test
(testing "that calls to a StepResultUpdateConsumer will just pass through"
Expand All @@ -22,6 +24,17 @@
some-build-number some-step-id some-step-result)
(is (received? component legacy-pipeline-state/update [some-build-number some-step-id some-step-result])))))

(deftest consume-pipeline-structure-test
(testing "that calls to a PipelineStructureConsumer will just pass through"
(let [component (mock state-protocols/PipelineStructureConsumer)]
(s/consume-pipeline-structure (some-ctx-with :pipeline-state-component component)
some-build-number some-structure)
(is (received? component state-protocols/consume-pipeline-structure [some-build-number some-structure]))))
(testing "that calls to a legacy PipelineStateComponent are ignored"
(let [component (mock legacy-pipeline-state/PipelineStateComponent)]
(s/consume-pipeline-structure (some-ctx-with :pipeline-state-component component)
some-build-number some-structure))))

(deftest next-build-number-test
(testing "that calls to a BuildNumberSource will just pass through"
(let [component (mock state-protocols/NextBuildNumberSource)]
Expand Down Expand Up @@ -65,7 +78,19 @@
(with-redefs [s/get-build (constantly {:step-results {:step :results}})]
(is (= {:step :results} (s/get-step-results nil nil))))))

(deftest step-result-test
(deftest get-step-result-test
(testing "that we can get a simple step-result"
(with-redefs [s/get-build (constantly {:step-results {[2 1] {:step :result}}})]
(is (= {:step :result} (s/get-step-result nil 1 [2 1]))))))

(deftest get-pipeline-structure-test
(testing "that we get the pipeline structure if it is contained in the build"
(with-redefs [s/get-build (constantly {:pipeline-structure some-structure})]
(is (= some-structure (s/get-pipeline-structure nil some-build-number)))))
(testing "that we get the current pipeline structure if the build result doesn't contain it (for compatibility reasons)"
(with-redefs [s/get-build (constantly {:no :structure-here})]
(is (= [{:name "foo"
:type :unknown
:has-dependencies false
:step-id `(1)}] (s/get-pipeline-structure (some-ctx-with :pipeline-def some-pipeline-def) some-build-number))))))

0 comments on commit c2135bf

Please sign in to comment.