Skip to content

Commit

Permalink
Initial 2
Browse files Browse the repository at this point in the history
Test Improvements

Initial
  • Loading branch information
Adrian Gheorghe committed Jul 13, 2019
1 parent b47c369 commit 6840246
Show file tree
Hide file tree
Showing 22 changed files with 1,106 additions and 570 deletions.
66 changes: 66 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
version: 2

jobs:
test:
docker:
- image: circleci/golang:1.11
steps:
- checkout
- run:
command: |
go test -v ./... > /dev/null 2>&1
echo "Temp remove tests"
build:
docker:
- image: circleci/golang:1.11
steps:
- checkout
- run:
command: |
mkdir -p ./artefacts
cp VERSION ./artefacts/VERSION
CGO_ENABLED=0 GOOS=linux go build -o ./artefacts/mediafaker-linux
CGO_ENABLED=0 GOOS=darwin go build -o ./artefacts/mediafaker-darwin
CGO_ENABLED=0 GOOS=windows go build -o ./artefacts/mediafaker-windows
CGO_ENABLED=0 go build -o ./artefacts/mediafaker
ls -al ./artefacts
- persist_to_workspace:
root: artefacts
paths:
- mediafaker-linux
- mediafaker-darwin
- mediafaker-windows
- mediafaker
- VERSION
release:
docker:
# specify the version
- image: cibuilds/github:0.10
steps:
- attach_workspace:
# Must be absolute path or relative path from working_directory
at: ./artefacts
- run:
command: |
ls -al ./artefacts
VERSION=$(cat ./artefacts/VERSION)
echo $VERSION
ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${VERSION} ./artefacts/
workflows:
version: 2
test_and_build_and_release:
jobs:
- test
- build:
requires:
- test
filters:
branches:
only: master
- release:
requires:
- build
filters:
branches:
only: master

17 changes: 14 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,15 @@
base:
go run $$(ls -1 *.go | grep -v _test.go)
local:
go run $$(ls -1 *.go | grep -v _test.go) local --source $(SOURCEPATH) --destination $(DESTPATH) --extcopy=".htaccess" --extcopy=".gitignore"
tree-url:
go run $$(ls -1 *.go | grep -v _test.go) url --extcopy=".htaccess" --extcopy=".gitignore" --destination "$(DESTPATH)" --source "$(URL)"
go run $$(ls -1 *.go | grep -v _test.go) local --source $(SOURCEPATH) --destination $(DESTPATH) --extcopy=".htaccess" --extcopy=".gitignore" --jsonlog=$(JSONLOG)
url:
go run $$(ls -1 *.go | grep -v _test.go) url --extcopy=".htaccess" --extcopy=".gitignore" --destination "$(DESTPATH)" --source "$(URL)" --jsonlog=$(JSONLOG)

ssh:
go run $$(ls -1 *.go | grep -v _test.go) ssh --extcopy=".htaccess" --extcopy=".gitignore" --destination $(DESTPATH) --source "$(SOURCE)" --ssh-host "$(SSHHOST)" --ssh-user "$(SSHUSER)" --ssh-key "$(SSHKEY)" --jsonlog=$(JSONLOG)

ver:
go run $$(ls -1 *.go | grep -v _test.go) --version

test:
go test -v ./...
116 changes: 76 additions & 40 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,42 @@ mediafaker is a small utility written in go that creates a fake version of a sou

The path can be local or over SSH provided correct credentials are passed to the utility.

# usage
the mediafaker binary can be executed in 3 different ways
# Manual
```bash
This utility creates a fake simplified version of a directory tree of your chosing, making it easier to work locally on legacy projects that have a large media asset folder.

Usage:
mediafaker [command]

Available Commands:
help Help about any command
local runs mediafaker on a local directory
ssh runs mediafaker on a remote tree file accessible via ssh
url runs mediafaker from a tree json stored remotely

Flags:
-d, --destination string Local Destination directory path where mediafaker should store the files
-e, --extcopy strings List of extensions that should be copied automatically
-h, --help help for mediafaker
-j, --jsonlog Change logger format to json
-m, --maxcopy int Maximum Size a file should have to be copied automatically (default 300000)
--version version for mediafaker

Use "mediafaker [command] --help" for more information about a command.
```
# Usage
mediafaker can be used in 2 ways.

## local
The simplest way is when the path you want to fake is on the same host as the destination path. Then you can run mediafaker local

```sh
mediafaker --source sourceDirPath --destination destDirPath
mediafaker local \
--source="/opt/media" \
--destination="/home/project/public/fake/destination"
```
## ssh

## from moni tree export
Using https://github.com/adrian-gheorghe/moni you can create a json tree export of the directory you want to fake.
This json file can be passed into filefaker to create a fake directory
## remote
When the source and destination paths are on different hosts mediafaker uses https://github.com/adrian-gheorghe/moni in order to generate a json representation of the directory you want to fake.

```json
[
Expand All @@ -39,24 +63,6 @@ This json file can be passed into filefaker to create a fake directory
},
"Children": null
},
{
"Path": "testdata/.DS_Store",
"Type": "file",
"Mode": "-rw-r--r--",
"Size": 8196,
"Modtime": "2019-06-15 12:15:50.423694836 +0100 BST",
"Sum": "46e1c0012e80786a970b3f8569222b51",
"MediaType": "application/octet-stream",
"Content": "",
"ImageInfo": {
"Width": 0,
"Height": 0,
"PixelInfo": null,
"BlockWidth": 0,
"BlockHeight": 0
},
"Children": null
},
{
"Path": "testdata/a",
"Type": "directory",
Expand Down Expand Up @@ -114,23 +120,53 @@ This json file can be passed into filefaker to create a fake directory
]
```

## Image Pixelation
One of the main features of mediafaker is it will read image files and create a pixelated version of the original image keeping the same dimensions and main colors. This saves time and precious disk space while also allowing you to have a drop in replacement for your images
### url
You can use moni https://github.com/adrian-gheorghe/moni or any other tool to generate a json output of the following form. mediafaker can use this to generate a fake version of the tree in your desired source.

```bash
moni url \
--source="http://example.org/path/to/moni.output.json" \
--destination="/home/project/public/fake/destination" \
```

This flow can easily be automated to give you access to a fresh abstracted version of the tree on demand.
You can get moni to run on your server periodically and create a new version of the output.json file as often as you like.

You can run mediafaker locally or on your dev/stage environments in order to have an up to date faked version of the public assets in production.

### ssh
mediafaker has a ssh client built in. When calling the ssh command, mediafaker attempts to:
- connect to your remote host using the credentials provided
- download moni from github
- run moni to generate a new output.json file
- downloads the compressed output file
- removes moni and the output tree from your remote host
- fakes the output tree locally

```bash
mediafaker ssh \
--source "/var/www/html/public" \
--destination "/home/project/public/fake/destination" \
--ssh-host "22.22.22.22" \
--ssh-user "user" \
--ssh-key "/home/.ssh/id_rsa_private_key_to_use"
```

## Image Pixelation
One of the main features of mediafaker is that it will read image files and create a pixelated version of the original image keeping the same dimensions and main colors. This saves time and precious disk space while also allowing you to have a drop in replacement for your images

filefaker will proceed to create an exact replica of the source directory, by creating mock files. The mock files created try to follow the originals as much as possible.


## TODOS
- pdf and doc files
- csv files
- load from export
- ssh deploy moni and retrieve info


Files use their initial names
Directory depth parameter
File Age ignore parameter
Documents created are valid, but empty.
## Documents
- Xlsx support added using the https://github.com/tealeg/xlsx package
- Pdf support added using the https://github.com/johnfercher/maroto package
- Docx, Pptx, CSV to be added
- Audio and Video file support to be added soon (mp3, wav, mp4, avi)
- Json and CSV file support to be added soon

## Features not yet implemented
- directory/file permissions
- fake from mediatype
- fake file and directory names
- directory depth parameter
- file age ignore parameter
54 changes: 54 additions & 0 deletions cmd/local.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package cmd

import (
"github.com/adrian-gheorghe/mediafaker/fakers"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

// SourcePath represents the local directory path that should be faked
var SourcePath string

func init() {
RootCmd.AddCommand(Local)
Local.Flags().StringVarP(&SourcePath, "source", "s", "", "Local Source Directory path which should be faked")
}

// Local represents the tree tree command
var Local = &cobra.Command{
Use: "local",
Short: "runs mediafaker on a local directory",
Long: `This subcommand runs mediafaker on a local source directory and stores the files in a destination directory of your choice.`,
Run: func(cmd *cobra.Command, args []string) {
if SourcePath == "" {
log.Fatal("Source directory has not been provided")
return
}

if DestinationPath == "" {
log.Fatal("Destination directory has not been provided")
return
}

mediaFake := fakers.MediaFake{
SourcePath: SourcePath,
MoniTreePath: "",
DestinationPath: DestinationPath,
ExtensionsToCopyAutomatically: ExtensionsToCopyAutomatically,
MaximumSizeForCopy: MaximumSizeForCopy,
TotalFaked: 0,
TotalMissed: 0,
TotalCopied: 0,
TotalSourceSize: 0,
TotalDestinationSize: 0,
}
error := mediaFake.Fake()
if error != nil {
log.Error(error)
}

log.Info("Media directory has been faked:", SourcePath)
mediaFake.CalculateTotalDestinationSize()
mediaFake.PrintInfo()
},
}
24 changes: 24 additions & 0 deletions cmd/local_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package cmd_test

import (
"github.com/adrian-gheorghe/mediafaker/cmd"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"testing"
)

func TestLocalBadParameters(t *testing.T) {
defer func() { log.StandardLogger().ExitFunc = nil }()
var fatal bool
log.StandardLogger().ExitFunc = func(int) { fatal = true }

fatal = false
cmd.RootCmd.SetArgs([]string{"local"})
cmd.RootCmd.Execute()
assert.Equal(t, true, fatal)

fatal = false
cmd.RootCmd.SetArgs([]string{"local", "--source", "--destination"})
cmd.RootCmd.Execute()
assert.Equal(t, true, fatal)
}
10 changes: 7 additions & 3 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ func Execute() {

func init() {
cobra.OnInitialize(initConfig)
RootCmd.PersistentFlags().StringVarP(&DestinationPath, "destination", "d", "", "Destination Directory path where mediafaker should store the files")
RootCmd.PersistentFlags().StringVarP(&DestinationPath, "destination", "d", "", "Local Destination directory path where mediafaker should store the files")
RootCmd.PersistentFlags().StringSliceVarP(&ExtensionsToCopyAutomatically, "extcopy", "e", []string{}, "List of extensions that should be copied automatically")
RootCmd.PersistentFlags().Int64VarP(&MaximumSizeForCopy, "maxcopy", "m", 300000, "Maximum Size a file should have to be copied")
RootCmd.PersistentFlags().Int64VarP(&MaximumSizeForCopy, "maxcopy", "m", 30000, "Maximum Size(in bytes) a file should have to be copied automatically if it cannot be faked")
RootCmd.PersistentFlags().BoolVarP(&jsonlog, "jsonlog", "j", false, "Change logger format to json")
}

Expand All @@ -51,5 +51,9 @@ func initConfig() {
myFigure.Print()
fmt.Println("")

log.SetOutput(os.Stdout)
if jsonlog {
log.SetFormatter(&log.JSONFormatter{})
} else {
log.SetOutput(os.Stdout)
}
}
22 changes: 22 additions & 0 deletions cmd/root_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package cmd_test

import (
"io/ioutil"
"testing"

"github.com/adrian-gheorghe/mediafaker/cmd"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)

func TestMain(m *testing.M) {
log.SetOutput(ioutil.Discard)
m.Run()
}

func TestVersionWorks(t *testing.T) {
cmd.RootCmd.SetArgs([]string{"--version"})
assert.NotPanics(t, func() {
assert.NoError(t, cmd.RootCmd.Execute())
})
}
Loading

0 comments on commit 6840246

Please sign in to comment.