diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
new file mode 100644
index 00000000..7ee07004
--- /dev/null
+++ b/.github/workflows/integration-tests.yml
@@ -0,0 +1,23 @@
+name: Integration Tests
+
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+ branches: [ master ]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ name: tests
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: 1.22
+
+ - name: Run tests
+ run: go test -count=1 -tags=integration ./tests/
diff --git a/HACKING.md b/HACKING.md
index c421d905..63d0c413 100644
--- a/HACKING.md
+++ b/HACKING.md
@@ -202,6 +202,13 @@ ok github.com/canonical/pebble/cmd/pebble 0.165s
...
```
+Pebble also has a suite of integration tests for testing things like `pebble run`. To run them, use the "integration" build constraint:
+
+```
+$ go test -count=1 -tags=integration ./tests/
+ok github.com/canonical/pebble/tests 4.774s
+```
+
## Docs
We use [`sphinx`](https://www.sphinx-doc.org/en/master/) to build the docs with styles preconfigured by the [Canonical Documentation Starter Pack](https://github.com/canonical/sphinx-docs-starter-pack).
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 00000000..362aa1c6
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,33 @@
+# Pebble Integration Tests
+
+This directory holds a suite of integration tests for end-to-end tests of things like pebble run. They use the standard go test runner, but are only executed if you set the integration build constraint.
+
+## Run Tests
+
+```bash
+go test -count=1 -tags=integration ./tests/
+```
+
+The above command will build Pebble first, then run tests with it.
+
+To use an existing Pebble binary rather than building one, you can explicitly set the flag `-pebblebin`. For example, the following command will use a pre-built Pebble at `/home/ubuntu/pebble`:
+
+```bash
+go test -v -count=1 -tags=integration ./tests -pebblebin=/home/ubuntu/pebble
+```
+
+## Developing
+
+### Visual Studio Code Settings
+
+For VSCode Go and the gopls extention to work properly with files containing build tags, add the following:
+
+```json
+{
+ "gopls": {
+ "build.buildFlags": [
+ "-tags=integration"
+ ]
+ }
+}
+```
diff --git a/tests/main_test.go b/tests/main_test.go
new file mode 100644
index 00000000..af0a6a48
--- /dev/null
+++ b/tests/main_test.go
@@ -0,0 +1,191 @@
+//go:build integration
+
+// Copyright (c) 2024 Canonical Ltd
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 3 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see .
+
+package tests
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/canonical/pebble/internals/servicelog"
+)
+
+var pebbleBin = flag.String("pebblebin", "", "Path to the pre-built Pebble binary")
+
+// TestMain builds the pebble binary of `-pebblebin` flag is not set
+// before running the integration tests.
+func TestMain(m *testing.M) {
+ flag.Parse()
+
+ if *pebbleBin == "" {
+ goBuild := exec.Command("go", "build", "-o", "../pebble", "../cmd/pebble")
+ if err := goBuild.Run(); err != nil {
+ fmt.Println("Cannot build pebble binary:", err)
+ os.Exit(1)
+ }
+ *pebbleBin = "../pebble"
+ } else {
+ // Use the pre-built Pebble binary provided by the pebbleBin flag.
+ fmt.Println("Using pre-built Pebble binary at:", *pebbleBin)
+ }
+
+ exitCode := m.Run()
+ os.Exit(exitCode)
+}
+
+// createLayer creates a layer file with layerYAML under the directory "pebbleDir/layers".
+func createLayer(t *testing.T, pebbleDir, layerFileName, layerYAML string) {
+ t.Helper()
+
+ layersDir := filepath.Join(pebbleDir, "layers")
+ err := os.MkdirAll(layersDir, 0o755)
+ if err != nil {
+ t.Fatalf("Cannot create layers directory: %v", err)
+ }
+
+ layerPath := filepath.Join(layersDir, layerFileName)
+ err = os.WriteFile(layerPath, []byte(layerYAML), 0o755)
+ if err != nil {
+ t.Fatalf("Cannot create layers file: %v", err)
+ }
+}
+
+// pebbleRun starts the pebble daemon (`pebble run`) with optional arguments
+// and returns two channels for standard output and standard error.
+func pebbleRun(t *testing.T, pebbleDir string, args ...string) (stdoutCh chan servicelog.Entry, stderrCh chan servicelog.Entry) {
+ t.Helper()
+
+ stdoutCh = make(chan servicelog.Entry)
+ stderrCh = make(chan servicelog.Entry)
+
+ cmd := exec.Command(*pebbleBin, append([]string{"run"}, args...)...)
+ cmd.Env = append(os.Environ(), "PEBBLE="+pebbleDir)
+
+ stdoutPipe, err := cmd.StdoutPipe()
+ if err != nil {
+ t.Fatalf("Cannot create stdout pipe: %v", err)
+ }
+ stderrPipe, err := cmd.StderrPipe()
+ if err != nil {
+ t.Fatalf("Cannot create stderr pipe: %v", err)
+ }
+
+ err = cmd.Start()
+ if err != nil {
+ t.Fatalf("Error starting 'pebble run': %v", err)
+ }
+
+ stopStdout := make(chan struct{})
+ stopStderr := make(chan struct{})
+
+ t.Cleanup(func() {
+ err := cmd.Process.Signal(os.Interrupt)
+ if err != nil {
+ t.Errorf("Error sending SIGINT/Ctrl+C to pebble: %v", err)
+ }
+ cmd.Wait()
+ close(stopStdout)
+ close(stopStderr)
+ })
+
+ readLogs := func(parser *servicelog.Parser, ch chan servicelog.Entry, stop <-chan struct{}) {
+ for parser.Next() {
+ if err := parser.Err(); err != nil {
+ t.Errorf("Cannot parse Pebble logs: %v", err)
+ }
+ select {
+ case ch <- parser.Entry():
+ case <-stop:
+ return
+ }
+ }
+ }
+
+ // Both stderr and stdout are needed, because pebble logs to stderr
+ // while with "--verbose", services output to stdout.
+ stderrParser := servicelog.NewParser(stderrPipe, 4*1024)
+ stdoutParser := servicelog.NewParser(stdoutPipe, 4*1024)
+
+ go readLogs(stdoutParser, stdoutCh, stopStdout)
+ go readLogs(stderrParser, stderrCh, stopStderr)
+
+ return stdoutCh, stderrCh
+}
+
+// waitForLog waits until an expectedLog from an expectedService appears in the logs channel, or fails the test after a
+// specified timeout if the expectedLog is still not found.
+func waitForLog(t *testing.T, logsCh <-chan servicelog.Entry, expectedService, expectedLog string, timeout time.Duration) {
+ t.Helper()
+
+ timeoutCh := time.After(timeout)
+ for {
+ select {
+ case log, ok := <-logsCh:
+ if !ok {
+ t.Error("channel closed before all expected logs were received")
+ }
+
+ if log.Service == expectedService && strings.Contains(log.Message, expectedLog) {
+ return
+ }
+
+ case <-timeoutCh:
+ t.Fatalf("timed out after %v waiting for log %s", 3*time.Second, expectedLog)
+ }
+ }
+}
+
+// waitForFile waits until a file exists, or fails the test after a specified timeout
+// if the file still doesn't exist.
+func waitForFile(t *testing.T, file string, timeout time.Duration) {
+ t.Helper()
+
+ timeoutCh := time.After(timeout)
+ ticker := time.NewTicker(time.Millisecond)
+ for {
+ select {
+ case <-timeoutCh:
+ t.Fatalf("timeout waiting for file %s", file)
+
+ case <-ticker.C:
+ stat, err := os.Stat(file)
+ if err == nil && stat.Mode().IsRegular() {
+ return
+ }
+ }
+ }
+}
+
+// runPebbleCommand runs a pebble command and returns the standard output.
+func runPebbleCommand(t *testing.T, pebbleDir string, args ...string) string {
+ t.Helper()
+
+ cmd := exec.Command(*pebbleBin, args...)
+ cmd.Env = append(os.Environ(), "PEBBLE="+pebbleDir)
+
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("error executing pebble command: %v", err)
+ }
+
+ return string(output)
+}
diff --git a/tests/run_test.go b/tests/run_test.go
new file mode 100644
index 00000000..c98c6937
--- /dev/null
+++ b/tests/run_test.go
@@ -0,0 +1,207 @@
+//go:build integration
+
+// Copyright (c) 2024 Canonical Ltd
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 3 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see .
+
+package tests
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "net/http"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// TestStartupEnabledServices tests that Pebble will automatically start
+// services defined with `startup: enabled`.
+func TestStartupEnabledServices(t *testing.T) {
+ pebbleDir := t.TempDir()
+
+ layerYAML := fmt.Sprintf(`
+services:
+ svc1:
+ override: replace
+ command: /bin/sh -c "touch %s; sleep 10"
+ startup: enabled
+ svc2:
+ override: replace
+ command: /bin/sh -c "touch %s; sleep 10"
+ startup: enabled
+`,
+ filepath.Join(pebbleDir, "svc1"),
+ filepath.Join(pebbleDir, "svc2"),
+ )
+
+ createLayer(t, pebbleDir, "001-simple-layer.yaml", layerYAML)
+
+ _, stderrCh := pebbleRun(t, pebbleDir)
+ waitForLog(t, stderrCh, "pebble", "Started default services", 3*time.Second)
+
+ waitForFile(t, filepath.Join(pebbleDir, "svc1"), 3*time.Second)
+ waitForFile(t, filepath.Join(pebbleDir, "svc2"), 3*time.Second)
+}
+
+// TestCreateDirs tests that Pebble will create the Pebble directory on startup
+// with the `--create-dirs` option.
+func TestCreateDirs(t *testing.T) {
+ tmpDir := t.TempDir()
+ pebbleDir := filepath.Join(tmpDir, "pebble")
+
+ _, stderrCh := pebbleRun(t, pebbleDir, "--create-dirs")
+ waitForLog(t, stderrCh, "pebble", "Started daemon", 3*time.Second)
+
+ st, err := os.Stat(pebbleDir)
+ if err != nil {
+ t.Fatalf("pebble run --create-dirs didn't create Pebble directory: %v", err)
+ }
+ if !st.IsDir() {
+ t.Fatalf("pebble dir %s is not a directory: %v", pebbleDir, err)
+ }
+}
+
+// TestHold tests that Pebble will not default services automatically
+// with the `--hold` option.
+func TestHold(t *testing.T) {
+ pebbleDir := t.TempDir()
+
+ layerYAML := fmt.Sprintf(`
+services:
+ svc1:
+ override: replace
+ command: /bin/sh -c "touch %s; sleep 10"
+ startup: enabled
+`,
+ filepath.Join(pebbleDir, "svc1"),
+ )
+ createLayer(t, pebbleDir, "001-simple-layer.yaml", layerYAML)
+
+ _, _ = pebbleRun(t, pebbleDir, "--hold")
+
+ // Sleep 100 millisecond before checking services because immediate check
+ // can't guarantee that svc1 is not started shortly after the log "Started daemon".
+ time.Sleep(100 * time.Millisecond)
+
+ _, err := os.Stat(filepath.Join(pebbleDir, "svc1"))
+ if err == nil {
+ t.Fatal("pebble run --hold failed, services are still started")
+ }
+ if !errors.Is(err, fs.ErrNotExist) {
+ t.Fatalf("Stat returned error other than ErrNotExist: %v", err)
+ }
+}
+
+// TestHTTPPort tests that Pebble starts HTTP API listening on this port
+// with the `--http` option.
+func TestHTTPPort(t *testing.T) {
+ pebbleDir := t.TempDir()
+
+ port := "61382"
+ _, stderrCh := pebbleRun(t, pebbleDir, "--http=:"+port)
+ waitForLog(t, stderrCh, "pebble", "Started daemon", 3*time.Second)
+
+ resp, err := http.Get(fmt.Sprintf("http://localhost:%s/v1/health", port))
+ if err != nil {
+ t.Fatalf("port %s is not being listened by : %v", port, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ t.Fatalf("error checking pebble /v1/health on port %s: %v", port, err)
+ }
+}
+
+// TestVerbose tests that Pebble logs all output from services to stdout
+// with the `--verbose` option.
+func TestVerbose(t *testing.T) {
+ pebbleDir := t.TempDir()
+
+ layersFileName := "001-simple-layer.yaml"
+ layerYAML := `
+services:
+ svc1:
+ override: replace
+ command: /bin/sh -c "echo 'hello world'; sleep 10"
+ startup: enabled
+`
+ createLayer(t, pebbleDir, layersFileName, layerYAML)
+
+ stdoutCh, stderrCh := pebbleRun(t, pebbleDir, "--verbose")
+ waitForLog(t, stderrCh, "pebble", "Started daemon", 3*time.Second)
+ waitForLog(t, stdoutCh, "svc1", "hello world", 3*time.Second)
+ waitForLog(t, stderrCh, "pebble", "Started default services", 3*time.Second)
+}
+
+// TestArgs tests that Pebble provides additional arguments to a service
+// with the `--args` option.
+func TestArgs(t *testing.T) {
+ pebbleDir := t.TempDir()
+
+ layerYAML := `
+services:
+ svc1:
+ override: replace
+ command: /bin/sh
+ startup: enabled
+`
+ layersFileName := "001-simple-layer.yaml"
+ createLayer(t, pebbleDir, layersFileName, layerYAML)
+
+ stdoutCh, stderrCh := pebbleRun(t, pebbleDir, "--verbose",
+ "--args",
+ "svc1",
+ "-c",
+ "echo 'hello world'; sleep 10",
+ )
+ waitForLog(t, stderrCh, "pebble", "Started daemon", 3*time.Second)
+ waitForLog(t, stdoutCh, "svc1", "hello world", 3*time.Second)
+ waitForLog(t, stderrCh, "pebble", "Started default services", 3*time.Second)
+}
+
+// TestIdentities tests that Pebble seeds identities from a file
+// with the `--identities` option.
+func TestIdentities(t *testing.T) {
+ pebbleDir := t.TempDir()
+
+ identitiesYAML := `
+identities:
+ bob:
+ access: admin
+ local:
+ user-id: 42
+`[1:]
+ identitiesFileName := "idents-add.yaml"
+ if err := os.WriteFile(filepath.Join(pebbleDir, identitiesFileName), []byte(identitiesYAML), 0o755); err != nil {
+ t.Fatalf("Cannot write identities file: %v", err)
+ }
+
+ _, stderrCh := pebbleRun(t, pebbleDir, "--identities="+filepath.Join(pebbleDir, identitiesFileName))
+
+ // wait for log "Started daemon" like in other test cases then immediately run `pebble identity` would sometimes
+ // fail because the identities are not fully seeded. Waiting for the next log "POST /v1/services" can guarantee
+ // identities are seeded when running the `pebble identity` command without sleeping for a short period of time.
+ waitForLog(t, stderrCh, "pebble", "POST /v1/services", 3*time.Second)
+
+ output := runPebbleCommand(t, pebbleDir, "identity", "bob")
+ expected := `
+access: admin
+local:
+ user-id: 42
+`[1:]
+ if output != expected {
+ t.Fatalf("error checking identities. expected: %s; got: %s", expected, output)
+ }
+}