Merge pull request #16 from Roverr/feature/logs

Feature - Process logging

Gopkg.toml

@@ -64,3 +64,7 @@
 [[constraint]]
   name = "github.com/dgrijalva/jwt-go"
   version = "3.2.0"
+
+[[constraint]]
+  name = "github.com/natefinch/lumberjack"
+  version = "2.1.0"

Makefile

@@ -6,11 +6,6 @@ coverage: ## Runs tests with coverage going into cover.out
 	go test ./... -coverprofile cover.out
 open-coverage: ## Opens the coverage file in browser
 	go tool cover -html=cover.out
-deploy-coverage: ## Deploys coverage to codecov
-	go test ./... -race -coverprofile=coverage.txt -covermode=atomic && \
-	curl -s https://codecov.io/bash > deploy.sh && \
-	chmod +x ./deploy.sh && \
-	./deploy.sh && rm ./deploy.sh
 run:  ## Builds & Runs the application
 	go build . && ./rtsp-stream
 docker-build:  ## Builds normal docker container

README.md

@@ -6,16 +6,19 @@
 rtsp-stream is an easy to use out of box solution that can be integrated into existing systems resolving the problem of not being able to play rtsp stream natively in browsers. 
 
 ## Table of contents
-* [How does it work](https://github.com/Roverr/rtsp-stream#how-does-it-work)
-* [Authentication](https://github.com/Roverr/rtsp-stream#authentication)
-    * [No Authentication](https://github.com/Roverr/rtsp-stream#no-authentication)
-    * [JWT](https://github.com/Roverr/rtsp-stream#jwt-authentication)
-* [Easy API](https://github.com/Roverr/rtsp-stream#easy-api)
-* [Configuration](https://github.com/Roverr/rtsp-stream#configuration)
-* [Run with Docker](https://github.com/Roverr/rtsp-stream#run-with-docker)
-* [UI](https://github.com/Roverr/rtsp-stream#ui)
-* [Proven players](https://github.com/Roverr/rtsp-stream#proven-players)
-* [Coming soon features](https://github.com/Roverr/rtsp-stream#coming-soon-features)
+* [How does it work](#how-does-it-work)
+* [Authentication](#authentication)
+    * [No Authentication](#no-authentication)
+    * [JWT](#jwt-authentication)
+* [Easy API](#easy-api)
+* [Configuration](#configuration)
+    * [Transcoding](#transcoding-related-configuration)
+    * [HTTP](#http-related-configuration)
+    * [CORS](#cors-related-configuration)
+* [Run with Docker](#run-with-docker)
+* [UI](#ui)
+* [Proven players](#proven-players)
+* [Coming soon features](#coming-soon-features)
 
 
 ## How does it work
@@ -44,12 +47,15 @@ You can use shared key JWT authentication for the service.
 The service itself does not create any tokens, but your authentication service can create.
 After it's created it can be validated in the transcoder using the same secret / keys.
 It is the easiest way to integrate into existing systems.
+
 The following environment variables are available for this setup:
 
-* **RTSP_STREAM_AUTH_JWT_ENABLED** - bool (false by default) - Indicates if the service should use the JWT authentication for the requests
-* **RTPS_STREAM_AUTH_JWT_SECRET** - string (macilaci by default) - The secret used for creating the JWT tokens
-* **RTSP_STREAM_AUTH_JWT_PUB_PATH** - string (/key.pub by default) - Path to the public RSA key.
-* **RTSP_STREAM_AUTH_JWT_METHOD** - string (secret by default) - Can be `secret` or `rsa`. Changes how the application does the JWT verification.
+| Env variable | Description | Default | Type |
+| :---        |    :----   |          ---: | :--- |
+| RTSP_STREAM_AUTH_JWT_ENABLED | Indicates if the service should use the JWT authentication for the requests | `false` | bool |
+| RTPS_STREAM_AUTH_JWT_SECRET | The secret used for creating the JWT tokens | `macilaci` | string |
+| RTSP_STREAM_AUTH_JWT_PUB_PATH | Path to the public shared RSA key.| `/key.pub` | string |
+| RTSP_STREAM_AUTH_JWT_METHOD | Can be `secret` or `rsa`. Changes how the application does the JWT verification.| `secret` | string |
 
 You won't need the private key for it because no signing happens in this application.
 
@@ -104,28 +110,54 @@ Response:
 
 You can configure the following settings in the application with environment variables:
 
-**Transcoding related configuration:**
-* `RTSP_STREAM_CLEANUP_TIME` - string (2m0s by default) - Time period for the cleanup process [info on format here](https://golang.org/pkg/time/#ParseDuration)
-* `RTSP_STREAM_STORE_DIR` - string (./videos by default) - Sub directory to store video chunks
-* `RTSP_STREAM_KEEP_FILES` - bool (false by default) - Option to keep the chunks for the stream being transcoded.
+### Transcoding related configuration:
+
+| Env variable | Description | Default | Type |
+| :---        |    :----   |          ---: | :--- |
+| RTSP_STREAM_CLEANUP_TIME | Time period for the cleanup process [info on format here](https://golang.org/pkg/time/#ParseDuration) | `2m0s` | string |
+| RTSP_STREAM_STORE_DIR | Sub directory to store the video chunks | `./videos` | string |
+| RTSP_STREAM_KEEP_FILES | Option to keep the chunks for the stream being transcoded | `false` | bool |
+
+The project uses [Lumberjack](https://github.com/natefinch/lumberjack) for log rotation at the transcoding site.
+
+| Env variable | Description | Default | Type |
+| :---        |    :----   |          ---: | :--- |
+| RTSP_STREAM_PROCESS_LOGGING_ENABLED | Time period for the cleanup process [info on format here](https://golang.org/pkg/time/#ParseDuration) | `false` | bool |
+| RTSP_STREAM_PROCESS_LOGGING_DIR | Describes the directory where the transcoding logs are stored | `/var/log/rtsp-stream` | string |
+| RTSP_STREAM_PROCESS_LOGGING_MAX_SIZE | Maximum size of each log file in **megabytes** | `500` | integer |
+| RTSP_STREAM_PROCESS_LOGGING_MAX_AGE | Maximum number of days that we store a given log file. | `7` | integer |
+| RTSP_STREAM_PROCESS_LOGGING_MAX_BACKUPS | Maximum number of old log files to retain | `3` | integer |
+| RTSP_STREAM_PROCESS_LOGGING_COMPRESS | Option to compress the rotated log file or not | `true` | bool |
+
+<hr>
+
+### HTTP related configuration:
+
+| Env variable | Description | Default | Type |
+| :---        |    :----   |          ---: | :--- |
+| RTSP_STREAM_PORT | Port where the application listens | `8080` | integer |
+| RTSP_STREAM_DEBUG | Turns on / off debug logging | `false` | bool |
+| RTSP_STREAM_LIST_ENDPOINT | Turns on / off the `/list` endpoint | `false` | bool |
 
-**HTTP related configuration:**
-* `RTSP_STREAM_PORT` - number (8080 by default) - Port where the application listens
-* `RTSP_STREAM_DEBUG` - bool (false by default) - Turns on / off debug logging
-* `RTSP_STREAM_LIST_ENDPOINT` - bool (false by default) - Turns on / off the `/list` endpoint
+<hr>
 
-**CORS related configuration:**
+### CORS related configuration
 
 By default all origin is allowed to make requests to the server, but you might want to configure it for security reasons.
-* `RTSP_STREAM_CORS_ENABLED` - bool - Indicates if cors should be handled as configured or as default (everything allowed)
-* `RTSP_STREAM_CORS_ALLOWED_ORIGIN` - string array - A list of origins a cross-domain request can be executed from
-* `RTSP_STREAM_CORS_ALLOW_CREDENTIALS` - bool - Indicates whether the request can include user credentials like cookies, HTTP authentication or client side SSL certificates
-* `RTSP_STREAM_CORS_MAX_AGE` - number - Indicates how long (in seconds) the results of a preflight request can be cached.
+
+| Env variable | Description | Default | Type |
+| :---        |    :----   |          ---: | :--- |
+| RTSP_STREAM_CORS_ENABLED | Indicates if cors should be handled as configured or as default (everything allowed) | `false` | bool |
+| RTSP_STREAM_CORS_ALLOWED_ORIGIN | A list of origins a cross-domain request can be executed from |  | []string |
+| RTSP_STREAM_CORS_ALLOW_CREDENTIALS | Indicates whether the request can include user credentials like cookies, HTTP authentication or client side SSL certificates | `false` | bool |
+| RTSP_STREAM_CORS_MAX_AGE | Indicates how long (in seconds) the results of a preflight request can be cached. | `0` | integer |
 
 ## Run with Docker
 The application has an offical docker repository at dockerhub, therefore you can easily run it with simple commands:
 
-`docker run -p 80:8080 roverr/rtsp-stream:1`
+```bash
+docker run -p 80:8080 roverr/rtsp-stream:1
+```
 
 or you can build it yourself using the source code.
 
@@ -135,7 +167,9 @@ You can use the included UI for handling the streams. The UI is not a compact so
 
 Running it with docker:
 
-`docker run -p 80:80 -p 8080:8080 roverr/rtsp-stream:1-management`
+```sh
+docker run -p 80:80 -p 8080:8080 roverr/rtsp-stream:1-management
+```
 
 If you decide to use the management image, you should know that port 80 is flexible, you can set it to whatever you prefer, but 8080 is currently burnt into the UI as the ultimate port of the backend.
 
@@ -165,7 +199,7 @@ The following list of players has been already tried out in production environme
 
 🤷‍♂️ - Needs more labour
 
-* 🤷‍♂️ Proper logging - File logging for the output of ffmpeg with the option of rotating file log
+* ✅ Proper logging - File logging for the output of ffmpeg with the option of rotating file log
 * ✅ Improved cleanup - Unused streams should be removed from the system after a while
 * 🤷‍♂️ API improvements - Delete endpoint for streams so clients can remove streams whenever they would like to
 * ✅  Authentication layer - More options for creating authentication within the service

core/config/config.go

@@ -23,6 +23,16 @@ type Auth struct {
 	JWTPubKeyPath string `envconfig:"AUTH_JWT_PUB_PATH" default:"./key.pub"` // Path to the public RSA key
 }
 
+// ProcessLogging describes information about the logging mechanism of the transcoding FFMPEG process
+type ProcessLogging struct {
+	Enabled    bool   `envconfig:"PROCESS_LOGGING" default:"false"`                    // Option to set logging for transcoding processes
+	Directory  string `envconfig:"PROCESS_LOGGING_DIR" default:"/var/log/rtsp-stream"` // Directory for the logs
+	MaxSize    int    `envconfig:"PROCESS_LOGGING_MAX_SIZE" default:"500"`             // Maximum size of kept logging files in megabytes
+	MaxBackups int    `envconfig:"PROCESS_LOGGING_MAX_BACKUPS" default:"3"`            // Maximum number of old log files to retain
+	MaxAge     int    `envconfig:"PROCESS_LOGGING_MAX_AGE" default:"7"`                // Maximum number of days to retain an old log file.
+	Compress   bool   `envconfig:"PROCESS_LOGGING_COMPRESS" default:"true"`            // Indicates if the log rotation should compress the log files
+}
+
 // Process describes information regarding the transcoding process
 type Process struct {
 	CleanupTime time.Duration `envconfig:"CLEANUP_TIME" default:"2m0s"`  // Time period between process cleaning
@@ -39,6 +49,7 @@ type Specification struct {
 	CORS
 	Auth
 	Process
+	ProcessLogging
 }
 
 // InitConfig is to initalise the config

core/controller.go

@@ -69,7 +69,7 @@ func NewController(spec *config.Specification, fileServer http.Handler) *Control
 		map[string]*streaming.Stream{},
 		fileServer,
 		*manager,
-		streaming.NewProcessor(spec.Process.StoreDir, spec.Process.KeepFiles),
+		streaming.NewProcessor(spec.Process.StoreDir, spec.Process.KeepFiles, spec.ProcessLogging),
 		time.Second * 15,
 		provider,
 	}

core/streaming/processor.go

@@ -11,7 +11,9 @@ import (
 	"time"
 
 	"github.com/Roverr/hotstreak"
+	"github.com/Roverr/rtsp-stream/core/config"
 	"github.com/kennygrant/sanitize"
+	"github.com/natefinch/lumberjack"
 	"github.com/sirupsen/logrus"
 )
 
@@ -30,16 +32,21 @@ type IProcessor interface {
 
 // Processor is the main type for creating new processes
 type Processor struct {
-	storeDir  string
-	keepFiles bool
+	storeDir    string
+	keepFiles   bool
+	loggingOpts config.ProcessLogging
 }
 
 // Type check
 var _ IProcessor = (*Processor)(nil)
 
 // NewProcessor creates a new instance of a processor
-func NewProcessor(storeDir string, keepFiles bool) *Processor {
-	return &Processor{storeDir, keepFiles}
+func NewProcessor(
+	storeDir string,
+	keepFiles bool,
+	loggingOpts config.ProcessLogging,
+) *Processor {
+	return &Processor{storeDir, keepFiles, loggingOpts}
 }
 
 // getHLSFlags are for getting the flags based on the config context
@@ -100,6 +107,21 @@ func (p Processor) NewStream(URI string) (*Stream, string) {
 		return nil, ""
 	}
 	cmd := p.NewProcess(URI)
+
+	// Create nil pointer in case logging is not enabled
+	cmdLogger := (*lumberjack.Logger)(nil)
+	// Create logger otherwise
+	if p.loggingOpts.Enabled {
+		cmdLogger = &lumberjack.Logger{
+			Filename:   fmt.Sprintf("%s/%s.log", p.loggingOpts.Directory, dirPath),
+			MaxSize:    p.loggingOpts.MaxSize,
+			MaxBackups: p.loggingOpts.MaxBackups,
+			MaxAge:     p.loggingOpts.MaxAge,
+			Compress:   p.loggingOpts.Compress,
+		}
+		cmd.Stderr = cmdLogger
+		cmd.Stdout = cmdLogger
+	}
 	stream := Stream{
 		CMD:       cmd,
 		Mux:       &sync.RWMutex{},
@@ -112,6 +134,7 @@ func (p Processor) NewStream(URI string) (*Stream, string) {
 		}).Activate(),
 		OriginalURI: URI,
 		KeepFiles:   p.keepFiles,
+		Logger:      cmdLogger,
 	}
 	logrus.Debugf("Created stream with storepath %s", stream.StorePath)
 	return &stream, fmt.Sprintf("%s/index.m3u8", newPath)
@@ -122,6 +145,10 @@ func (p Processor) Restart(strm *Stream, path string) error {
 	strm.Mux.Lock()
 	defer strm.Mux.Unlock()
 	strm.CMD = p.NewProcess(strm.OriginalURI)
+	if p.loggingOpts.Enabled {
+		strm.CMD.Stderr = strm.Logger
+		strm.CMD.Stdout = strm.Logger
+	}
 	strm.Streak.Activate()
 	go func() {
 		logrus.Infof("%s has been restarted", path)

core/streaming/stream.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/natefinch/lumberjack"
 	"github.com/sirupsen/logrus"
 
 	"github.com/Roverr/hotstreak"
@@ -20,6 +21,7 @@ type Stream struct {
 	OriginalURI string               `json:"-"`
 	StorePath   string               `json:"-"`
 	KeepFiles   bool                 `json:"-"`
+	Logger      *lumberjack.Logger   `json:"-"`
 }
 
 // CleanProcess makes sure that the transcoding process is killed correctly