diff --git a/.github/workflows/testapp.yml b/.github/workflows/testapp.yml index 2c78a4d0..248c4df6 100644 --- a/.github/workflows/testapp.yml +++ b/.github/workflows/testapp.yml @@ -18,6 +18,9 @@ jobs: - name: Build run: go build -v ./... - name: Test with the Go CLI - run: go test -v ./tests/ + run: | + go mod tidy + go build -o build/l2 -ldflags "-X main.version=`git tag --sort=-version:refname | head -n 1`" l2.go + go test -v ./tests/ - name: Deploy hexmos doc run: curl -X POST --fail -F token=${{ secrets.TRIGGER_TOKEN }} -F ref=main https://git.apps.hexmos.com/api/v4/projects/85/trigger/pipeline \ No newline at end of file diff --git a/codegen/codegen.go b/codegen/codegen.go index decbd402..1982034e 100644 --- a/codegen/codegen.go +++ b/codegen/codegen.go @@ -15,10 +15,8 @@ import ( "github.com/atotto/clipboard" ) -var ( - //go:embed httpsnippet.js - snippetcore string -) +//go:embed httpsnippet.js +var snippetcore string type SnippetArgs struct { Language string @@ -43,7 +41,6 @@ func PrepareHTTPSnippetGenerator(snippetArgs SnippetArgs) string { // takes in the headers in L2 format, and generates // HAR compatible func GetHARHeadersCookies(headers *gabs.Container) (*gabs.Container, *gabs.Container) { - headersData := gabs.New() headersData.Array() diff --git a/controller/controller.go b/controller/controller.go index 6131300d..3a8111a6 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -7,7 +7,6 @@ package contoller import ( "fmt" "os" - "path" "github.com/HexmosTech/gabs/v2" "github.com/HexmosTech/httpie-go" @@ -69,7 +68,6 @@ func HandleParsedFile(parsedAPI *gabs.Container, o *lama2cmd.Opts, dir string) { if o.Output != "" { outputmanager.WriteJSONOutput(resp, o.Output) } - } // Process initiates the following tasks in the given order: @@ -88,7 +86,19 @@ func Process(version string) { _, dir, _ := utils.GetFilePathComponents(o.Positional.LamaAPIFile) oldDir, _ := os.Getwd() utils.ChangeWorkingDir(dir) - preprocess.LoadElfEnv(path.Join(dir, "l2.env")) + + if o.Env { + jsonEnvs, err := preprocess.GetL2EnvVariables(dir) + if err != nil { + log.Error().Str("Type", "Preprocess").Msg(err.Error()) + return + } + // Frontend can read the stdout for this command and get the JSON of all the env's + fmt.Println(string(jsonEnvs)) + return + } + + preprocess.LoadEnvironments(dir) utils.ChangeWorkingDir(oldDir) p := parser.NewLama2Parser() parsedAPI, e := p.Parse(apiContent) diff --git a/docs/Lama2/docs/explanation/l2format.md b/docs/Lama2/docs/explanation/l2format.md index 1a7c3f9c..916d6f1c 100644 --- a/docs/Lama2/docs/explanation/l2format.md +++ b/docs/Lama2/docs/explanation/l2format.md @@ -3,12 +3,12 @@ the rules for authoring `.l2` API files. This document expects some familiarity with *Lama2*. To quickly get started with *Lama2*, head over -to [Examples](../tutorials/examples.md). +to [Examples](../tutorials/examples.md). On the other hand, if you are a developer and wish to learn more about the formal grammar underlying -*l2*, visit the [Grammar](../reference/grammar.md) +*l2*, visit the [Grammar](../reference/grammar.md) section. ### Comments start with `#` @@ -21,7 +21,6 @@ Fully supported: `GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH` ### JSON is the default submission type, but MULTIPART is supported too - #### `varjson` is a simpler syntax to specify flat JSONs `varjson` values are defined as follows: @@ -42,10 +41,10 @@ The above results in a JSON submission of the form: #### Nested JSON can simply be dumped at the end of the document -The JSON recognition engine is quite lenient. It can deal with +The JSON recognition engine is quite lenient. It can deal with minor errors in the format (such as having single quotes instead of double quotes, trailing garbage, or an extra comma after the -last element in an array,). +last element in an array,). ``` POST @@ -57,8 +56,6 @@ https://httpbin.org/post } ``` - - #### MULTIPART allows both file uploads & the usual fields Example: @@ -75,7 +72,7 @@ file@./helloworld.jpg The *file path is relative to the request file.* -### Cookies are sent as headers +### Cookies are sent as headers Cookies are specified in a `Cookie` header as follows: @@ -83,16 +80,63 @@ Cookies are specified in a `Cookie` header as follows: Cookie:'sessionid=foo;another-cookie=bar' ``` -### Environments variables/commands can be defined in `/l2.env` +### API variables can be defined in `apirequest.l2` + +L2 uses the variables declared inside the `.l2` file and makes the request -By default, *l2* looks for a `l2.env` file in the same directory as the given -request file directory. Example `l2.env`: +Example `login.l2`: + +``` +let REMOTE = "httpbin.org" +let EMAIL = "customer1@gmail.com" + +--- + +POST +${REMOTE}/login +{ + "email": "${EMAIL}", + "password": "customer1@gmail.com" +} +``` + +Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0021_varjson_variable/0021_varjson_variable.l2) + + +### API environment variables can be defined locally in `l2.env` + +`l2.env` is searched for, from the present directory and variables(local) are loaded from this file. + +Example `l2.env`: ``` export PHOTO=`base64 aadhaarlarge.jpg` export AHOST="http://localhost:8000" ``` +Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0004_env_switch_root) + +### API environment variables can be defined at root using `l2config.env` +`l2config.env` is searched for, from the present directory to all its ancestors (upto `/`) and +variables(root) are loaded from this file. +Example `l2config.env`: + +``` +export PHOTO=`base64 aadhaarsmall.jpg` +export AHOST="http://localhost:8001" +``` + +Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0019_env_switch_global_root) + +### If `l2config.env`(root) variables are redeclared in `l2.env`(local) + +The local variable's value is taken into consideration regardless of both files residing in same directory + +Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0020_override_project_root_local) + +![Override of l2config.env with l2.env variable](image.png) + + #### The environment file can load results of commands Use the backtick notation `\`command\`` to place the results of @@ -126,4 +170,4 @@ variable `result` contains the response from previous stages. For example, in the above case, `Javascript 2` can access the response from `L2 Request 1` through the `result` variable. -Learn more about request chaining in [Examples](../tutorials/examples.md#chain-requests-using-javascript). \ No newline at end of file +Learn more about request chaining in [Examples](../tutorials/examples.md#chain-requests-using-javascript). diff --git a/docs/Lama2/docs/reference/architecture.md b/docs/Lama2/docs/reference/architecture.md index a4ae5fd7..d04a042e 100644 --- a/docs/Lama2/docs/reference/architecture.md +++ b/docs/Lama2/docs/reference/architecture.md @@ -5,7 +5,7 @@ graph TD K["Controller Entry
(controller)"] A["Parse CLI
(lama2cmd)"] - B["Parser
(parser)"] + B["Parser
(parser)"] D["Request Executor
(cmdexec)"] E["Output Format Manager
(outputmanager)"] F["Error Reporting (TODO)"] @@ -48,7 +48,8 @@ From a high level, how does it work now? 2. Else if block is Requestor block 1. Replace variables with values in the following order 1. Try fetch variable from Javascript VM - 2. If (1) fails, try fetch variable from `l2.env` + 2. If (1) fails, try fetch Local env variable from `l2.env` + 3. Try fetch root env variable from `l2config.env` 2. Use the processed elements to create an httpie-go request 3. Fetch response 5. If necessary, write the last transaction to `.json` file \ No newline at end of file diff --git a/docs/Lama2/docs/reference/preprocess.md b/docs/Lama2/docs/reference/preprocess.md index 14b0557a..050880bd 100644 --- a/docs/Lama2/docs/reference/preprocess.md +++ b/docs/Lama2/docs/reference/preprocess.md @@ -10,18 +10,17 @@ Package preprocess provides facilities to expand environment variables in \`.l2\ ## Index -- [func Expand(s string, vm *goja.Runtime, mapping map[string]string) string](<#func-expand>) -- [func ExpandEnv(s string, vm *goja.Runtime) string](<#func-expandenv>) -- [func ExpandHeaders(block *gabs.Container, vm *goja.Runtime)](<#func-expandheaders>) -- [func ExpandJSON(block *gabs.Container, vm *goja.Runtime)](<#func-expandjson>) -- [func ExpandURL(block *gabs.Container, vm *goja.Runtime)](<#func-expandurl>) -- [func GetLamaFileAsString(path string) string](<#func-getlamafileasstring>) -- [func LamaFile(inputFile string) (string, string)](<#func-lamafile>) -- [func LoadElfEnv(l2path string)](<#func-loadelfenv>) -- [func ProcessVarsInBlock(block *gabs.Container, vm *goja.Runtime)](<#func-processvarsinblock>) - - -## func [Expand]() +- [func Expand(s string, vm \*goja.Runtime, mapping map[string]string) string](#func-expand) +- [func ExpandEnv(s string, vm \*goja.Runtime) string](#func-expandenv) +- [func ExpandHeaders(block *gabs.Container, vm *goja.Runtime)](#func-expandheaders) +- [func ExpandJSON(block *gabs.Container, vm *goja.Runtime)](#func-expandjson) +- [func ExpandURL(block *gabs.Container, vm *goja.Runtime)](#func-expandurl) +- [func GetLamaFileAsString(path string) string](#func-getlamafileasstring) +- [func LamaFile(inputFile string) (string, string)](#func-lamafile) +- [func LoadEnvFile(l2path string)](#func-loadenvfile) +- [func ProcessVarsInBlock(block *gabs.Container, vm *goja.Runtime)](#func-processvarsinblock) + +## func [Expand](https://github.com/HexmosTech/Lama2/blob/master/preprocess/expandvar.go#L19) ```go func Expand(s string, vm *goja.Runtime, mapping map[string]string) string @@ -29,7 +28,7 @@ func Expand(s string, vm *goja.Runtime, mapping map[string]string) string Expand replaces $\{var\} or $var in the string based on the mapping function. For example, os.ExpandEnv\(s\) is equivalent to os.Expand\(s, os.Getenv\). -## func [ExpandEnv]() +## func [ExpandEnv](https://github.com/HexmosTech/Lama2/blob/master/preprocess/expandvar.go#L74) ```go func ExpandEnv(s string, vm *goja.Runtime) string @@ -37,50 +36,50 @@ func ExpandEnv(s string, vm *goja.Runtime) string ExpandEnv replaces $\{var\} or $var in the string according to the values of the current environment variables. References to undefined variables are replaced by the empty string. -## func [ExpandHeaders]() +## func [ExpandHeaders](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L25) ```go func ExpandHeaders(block *gabs.Container, vm *goja.Runtime) ``` -## func [ExpandJSON]() +## func [ExpandJSON](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L77) ```go func ExpandJSON(block *gabs.Container, vm *goja.Runtime) ``` -## func [ExpandURL]() +## func [ExpandURL](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L45) ```go func ExpandURL(block *gabs.Container, vm *goja.Runtime) ``` -## func [GetLamaFileAsString]() +## func [GetLamaFileAsString](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L104) ```go func GetLamaFileAsString(path string) string ``` -## func [LamaFile]() +## func [LamaFile](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L118) ```go func LamaFile(inputFile string) (string, string) ``` -LamaFile takes in a path to an API file. It moves into the API file directory, reads the API contents, loads the \`l2.env\` file if available, and finally substitutes environment vars in the API contents Once done, it reverts back to the original directory, and returns the processed l2 file. +LamaFile takes in a path to an API file. It moves into the API file directory, reads the API contents, loads the \`l2config.env\` root variable file from the project directory. + +Then loads the \`l2.env\` local variables file from the present directory, Variables which are already declared in \`l2config.env\` will be overwritten with local variable of \`l2.env\`. Once done, it reverts back to the original directory, and returns the processed l2 file. -## func [LoadElfEnv]() +## func [LoadEnvFile](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L97) ```go -func LoadElfEnv(l2path string) +func LoadEnvFile(l2path string) ``` -## func [ProcessVarsInBlock]() +## func [ProcessVarsInBlock](https://github.com/HexmosTech/Lama2/blob/master/preprocess/preprocess.go#L19) ```go func ProcessVarsInBlock(block *gabs.Container, vm *goja.Runtime) ``` - - -Generated by [gomarkdoc]() +Generated by [gomarkdoc](https://github.com/princjef/gomarkdoc) diff --git a/docs/Lama2/docs/tutorials/examples.md b/docs/Lama2/docs/tutorials/examples.md index 46c2f847..a9029fde 100644 --- a/docs/Lama2/docs/tutorials/examples.md +++ b/docs/Lama2/docs/tutorials/examples.md @@ -18,6 +18,7 @@ requests from there. GET https://httpbin.org/get ``` + Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0000_sample_get.l2) ## JSON POST request @@ -33,13 +34,14 @@ https://httpbin.org/post "c": "d" } ``` + Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0002_sample_post.l2) ## JSON POST in VarJSON format Make a POST request with JSON body specified as `key=value`. *Lama2* converts the input into -a corresponding JSON value `{"a": "b", "c": "d"}`. We call the `key=value` format *VarJSON*. This example produces an effect identical to the [previous one](#basic-json-post) +a corresponding JSON value `{"a": "b", "c": "d"}`. We call the `key=value` format _VarJSON_. This example produces an effect identical to the [previous one](#basic-json-post) ``` POST @@ -48,6 +50,7 @@ https://httpbin.org/post a=b c=d ``` + Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0001_sample_post_varjson.l2) ## Comments @@ -65,12 +68,15 @@ c=d # Comments work even after the payload ``` + Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0003_comment.l2) ## Environment Variables: Switch base URL -Specify variables in `l2.env` and then load -them up in the API files. Presently, **the `l2.env` file should reside in the same directory as the `.l2` API file.** +### Case 1: `l2.env` adjacent to an API file + +For any given `.l2` file, one can place an `l2.env` file to store relevant variables. +These variables will be available to be used within the API file **l2.env** @@ -79,17 +85,22 @@ export LOCAL="http://localhost:8000" export REMOTE="http://httpbin.org" ``` -**env_example.l2** -``` -POST -${REMOTE}/post +### Case 2: Root variables -{ - "lorem": "ipsum" -} +In Lama2, you can have a large number of API files stored in a hierarchical folder configuration. +The root of such a project can be signified through `l2config.env`: + +Within such a structure, you can have an API file anywhere, which can use variables defined in the root variables: +**l2config.env** + +``` +export LOCAL="http://localhost:8000" +export REMOTE="http://httpbin.org" ``` -Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0004_env_switch_root) +### Case 3: Override Root variable with local variable + +![Override of l2config.env with l2.env variable](image.png) ## Headers @@ -102,7 +113,7 @@ Specify strings for key/value in three ways: 1. Unquoted (`hello`) ``` -POST +POST https://httpbin.org/post # HEADERS @@ -119,8 +130,9 @@ Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0005_he !!! Note The data section may appear *before* headers as well (see below) + ``` -POST +POST https://httpbin.org/post @@ -133,13 +145,13 @@ X-Parse-Application-Id:'helloworld' X-Parse-REST-API-Key:"byeworld" ``` - ## Send cookies in header -Headers represent cookies in *Lama2*. Just specify cookie key value pairs separated by +Headers represent cookies in _Lama2_. Just specify cookie key value pairs separated by `=` within the header value as shown. + ``` -POST +POST https://httpbin.org/post # HEADERS @@ -148,6 +160,7 @@ Cookie:"sessionid=foo;another-cookie=bar" # DATA hello=world ``` + Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0006_cookies.l2) ## Fill forms & attach files with MULTIPART @@ -155,7 +168,7 @@ Get [Source File](https://github.com/HexmosTech/Lama2/tree/main/examples/0006_co Use the `MULTIPART` keyword after the HTTP verb to enable forms and file attachments. -The data section may contain any number of +The data section may contain any number of form inputs using the `key=value` syntax. Following the data section, one can specify @@ -167,7 +180,7 @@ POST MULTIPART http://httpbin.org/post -'X-Parse-Application-Id':hello +'X-Parse-Application-Id':hello X-Parse-REST-API-Key:"world" # DATA @@ -176,12 +189,13 @@ first=second # FILES myfile@./image.jpeg ``` + Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0007_multipart_file) ## Image as Base64 encoded JSON field -We can embed images (or other files) as -base64 strings in JSON using *Lama2*. +We can embed images (or other files) as +base64 strings in JSON using _Lama2_. First, we define a `PHOTO` variable, loaded up with the results of the `base64` command. @@ -194,8 +208,7 @@ export PHOTO=`base64 -w 0 image.jpeg` Next, we refer to the `PHOTO` variable in the API file. Pay special attention to the -quoting mechanism `"'{PHOTO}'"`. - +quoting mechanism `"'{PHOTO}'"`. !!! warning @@ -213,6 +226,7 @@ http://httpbin.org/post "imageb64_field": "'${PHOTO}'", } ``` + Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0008_base64_image) ## Chain requests using Javascript @@ -260,5 +274,4 @@ Authorization: 'Bearer ${TOKEN}' {} ``` - -Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0009_processor_basic) \ No newline at end of file +Get [Source Files](https://github.com/HexmosTech/Lama2/tree/main/examples/0009_processor_basic) diff --git a/docs/Lama2/docs/tutorials/image.png b/docs/Lama2/docs/tutorials/image.png new file mode 100644 index 00000000..df8b877f Binary files /dev/null and b/docs/Lama2/docs/tutorials/image.png differ diff --git a/elfparser/ElfTestSuite/echo_get/l2.env b/elfparser/ElfTestSuite/echo_get/l2.env new file mode 100644 index 00000000..2a9324b2 --- /dev/null +++ b/elfparser/ElfTestSuite/echo_get/l2.env @@ -0,0 +1 @@ +export AHOST=`echo http://httpbin.org` \ No newline at end of file diff --git a/elfparser/ElfTestSuite/echo_get/y_0019_echo_get.l2 b/elfparser/ElfTestSuite/echo_get/y_0019_echo_get.l2 new file mode 100644 index 00000000..eeea2a10 --- /dev/null +++ b/elfparser/ElfTestSuite/echo_get/y_0019_echo_get.l2 @@ -0,0 +1,2 @@ +GET +${AHOST}/get \ No newline at end of file diff --git a/elfparser/ElfTestSuite/root_variable_override/api/l2.env b/elfparser/ElfTestSuite/root_variable_override/api/l2.env new file mode 100644 index 00000000..2a9324b2 --- /dev/null +++ b/elfparser/ElfTestSuite/root_variable_override/api/l2.env @@ -0,0 +1 @@ +export AHOST=`echo http://httpbin.org` \ No newline at end of file diff --git a/elfparser/ElfTestSuite/root_variable_override/api/y_0020_root_override.l2 b/elfparser/ElfTestSuite/root_variable_override/api/y_0020_root_override.l2 new file mode 100644 index 00000000..eeea2a10 --- /dev/null +++ b/elfparser/ElfTestSuite/root_variable_override/api/y_0020_root_override.l2 @@ -0,0 +1,2 @@ +GET +${AHOST}/get \ No newline at end of file diff --git a/elfparser/ElfTestSuite/root_variable_override/l2config.env b/elfparser/ElfTestSuite/root_variable_override/l2config.env new file mode 100644 index 00000000..02744d12 --- /dev/null +++ b/elfparser/ElfTestSuite/root_variable_override/l2config.env @@ -0,0 +1,2 @@ +export AHOST=`echo NO URL` +export BHOST="https://httpbin.org" \ No newline at end of file diff --git a/examples/0019_env_switch_project_root/0019_env_switch_project_root/0019_env_switch_project_root.l2 b/examples/0019_env_switch_project_root/0019_env_switch_project_root/0019_env_switch_project_root.l2 new file mode 100644 index 00000000..11264122 --- /dev/null +++ b/examples/0019_env_switch_project_root/0019_env_switch_project_root/0019_env_switch_project_root.l2 @@ -0,0 +1,6 @@ +POST +${REMOTE}/post + +{ + "lorem": "ipsum" +} \ No newline at end of file diff --git a/examples/0019_env_switch_project_root/l2config.env b/examples/0019_env_switch_project_root/l2config.env new file mode 100644 index 00000000..4e72e78f --- /dev/null +++ b/examples/0019_env_switch_project_root/l2config.env @@ -0,0 +1,2 @@ +export LOCAL="http://localhost:8000" +export REMOTE="http://httpbin.org" \ No newline at end of file diff --git a/examples/0020_override_project_root_local/0020_override_project_root_local/0020_override_project_root_local.l2 b/examples/0020_override_project_root_local/0020_override_project_root_local/0020_override_project_root_local.l2 new file mode 100644 index 00000000..11264122 --- /dev/null +++ b/examples/0020_override_project_root_local/0020_override_project_root_local/0020_override_project_root_local.l2 @@ -0,0 +1,6 @@ +POST +${REMOTE}/post + +{ + "lorem": "ipsum" +} \ No newline at end of file diff --git a/examples/0020_override_project_root_local/0020_override_project_root_local/l2.env b/examples/0020_override_project_root_local/0020_override_project_root_local/l2.env new file mode 100644 index 00000000..d75a291d --- /dev/null +++ b/examples/0020_override_project_root_local/0020_override_project_root_local/l2.env @@ -0,0 +1 @@ +export REMOTE="http://httpbin.org/dev" diff --git a/examples/0020_override_project_root_local/l2config.env b/examples/0020_override_project_root_local/l2config.env new file mode 100644 index 00000000..000e4958 --- /dev/null +++ b/examples/0020_override_project_root_local/l2config.env @@ -0,0 +1 @@ +export REMOTE="http://httpbin.org/prod" \ No newline at end of file diff --git a/examples/0021_varjson_variable/0021_varjson_variable.l2 b/examples/0021_varjson_variable/0021_varjson_variable.l2 new file mode 100644 index 00000000..4a1ab510 --- /dev/null +++ b/examples/0021_varjson_variable/0021_varjson_variable.l2 @@ -0,0 +1,24 @@ +let REMOTE = "httpbin.org" +let EMAIL = "customer1@gmail.com" + +--- + +POST +${REMOTE}/login +{ + "email": "${EMAIL}", + "password": "customer1@gmail.com" +} + +--- + +let TOKEN = result['jwt'] + +--- + +POST +${REMOTE}/logout +"Authorization": "Bearer ${TOKEN}" +{ + "email": "${EMAIL}", +} \ No newline at end of file diff --git a/lama2cmd/lama2cmd.go b/lama2cmd/lama2cmd.go index 664a1eb1..e264b302 100644 --- a/lama2cmd/lama2cmd.go +++ b/lama2cmd/lama2cmd.go @@ -26,6 +26,7 @@ type Opts struct { PostmanFile string `short:"p" long:"postmanfile" description:"JSON export from Postman (Settings -> Data -> Export Data)"` LamaDir string `short:"l" long:"lama2dir" description:"Output directory to put .l2 files after conversion from Postman format"` Help bool `short:"h" long:"help" group:"AddHelp" description:"Usage help for Lama2"` + Env bool `short:"e" long:"env" description:"Get a JSON of environment variables"` Version bool `long:"version" description:"Print Lama2 binary version"` Positional struct { @@ -101,7 +102,6 @@ func ArgParsing(o *Opts, version string) { log.Fatal().Msg("To convert Postman export to Lama2, try: l2 -p PostmanFile -l Lama2Dir") os.Exit(1) } - } // GetAndValidateCmd takes in the user's CLI input, and checks diff --git a/makefile b/makefile index fe1500e6..f9c1f3c4 100644 --- a/makefile +++ b/makefile @@ -23,8 +23,9 @@ gofumpt: gofumpt -w . test: - go test ./tests/ -count=1 - + make buildme + go test ./tests/ -count=1 -v + benchmark: go test -bench=. -count 1 -run=^# -benchmem ./tests/ diff --git a/outputManager/output_manager.go b/outputManager/output_manager.go index f6033733..d8d5bb2a 100644 --- a/outputManager/output_manager.go +++ b/outputManager/output_manager.go @@ -20,7 +20,7 @@ import ( var LogBuff bytes.Buffer func init() { - consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout} + consoleWriter := zerolog.ConsoleWriter{Out: os.Stderr} consoleWriter2 := zerolog.ConsoleWriter{Out: &LogBuff} multi := zerolog.MultiLevelWriter(consoleWriter, consoleWriter2) logger := zerolog.New(multi).With().Timestamp().Logger() diff --git a/parser/jsonparser.go b/parser/jsonparser.go index 46ee958a..d4edf9e8 100644 --- a/parser/jsonparser.go +++ b/parser/jsonparser.go @@ -25,7 +25,8 @@ func (p *Lama2Parser) PrimitiveType() (*gabs.Container, error) { // CustomPairMerge uses a gabs feature to deal with merge conflicts. // More here: https://github.com/HexmosTech/gabs/blob/master/gabs.go#L511 -func CustomPairMerge(destination, source interface{}) interface{} { +//nolint:all +func CustomPairMerge(destination, source interface{}) interface{} { return source } diff --git a/parser/lama2parser.go b/parser/lama2parser.go index 0abe82ba..665ddec2 100644 --- a/parser/lama2parser.go +++ b/parser/lama2parser.go @@ -38,7 +38,6 @@ func (p *Lama2Parser) Start() (*gabs.Container, error) { } func (p *Lama2Parser) Lama2File() (*gabs.Container, error) { - // Trying to get: // PSBlock? Requestor [SPSBlock Requestor]* diff --git a/preprocess/expandvar.go b/preprocess/expandvar.go index dd4c5cc6..9c723655 100644 --- a/preprocess/expandvar.go +++ b/preprocess/expandvar.go @@ -27,6 +27,7 @@ func Expand(s string, vm *goja.Runtime, mapping map[string]string) string { } buf = append(buf, s[i:j]...) name, w := getShellName(s[j+1:]) + //nolint:all if name == "" && w > 0 { // Encountered invalid syntax; eat the // characters. @@ -114,8 +115,6 @@ func getShellName(s string) (string, int) { } // Scan alphanumerics. var i int - for i = 0; i < len(s) && isAlphaNum(s[i]); i++ { - } return s[:i], i } diff --git a/preprocess/preprocess.go b/preprocess/preprocess.go index 89de5372..96486ea0 100644 --- a/preprocess/preprocess.go +++ b/preprocess/preprocess.go @@ -5,9 +5,12 @@ package preprocess import ( "encoding/json" + "errors" "fmt" "io/ioutil" "os" + "path" + "path/filepath" "strings" "github.com/HexmosTech/gabs/v2" @@ -70,7 +73,7 @@ func debugOp(str string) { func escapeString(input string) string { output, err := json.Marshal(input) if err != nil { - // handle error + log.Error().Str("Error marshaling JSON:","escapeString()") } return string(output) } @@ -96,11 +99,111 @@ func ExpandJSON(block *gabs.Container, vm *goja.Runtime) { log.Debug().Str("Processed JSON block", block.String()).Msg("") } -func LoadElfEnv(l2path string) { +func SearchL2ConfigEnv(dir string) (string, error) { + parentDir := dir + for parentDir != string(filepath.Separator) { + l2ConfigPath := filepath.Join(parentDir, "l2config.env") + _, err := os.Stat(l2ConfigPath) + if err == nil { + return l2ConfigPath, nil // Found the l2config.env file + } + parentDir = filepath.Dir(parentDir) + } + return "", errors.New("Didn't find l2config.env in the API directory") +} + +func LoadEnvFile(l2path string) { + envFileName := filepath.Base(l2path) err := godotenv.Load(l2path) if err != nil { - log.Info().Str("Type", "Preprocess").Msg("Didn't find l2.env in the API directory") + log.Info().Str("Type", "Preprocess").Msg("Didn't find " + envFileName + " in the API directory") + } +} + +func LoadEnvironments(dir string) { + l2ConfigPath, err := SearchL2ConfigEnv(dir) + if err != nil { + log.Info().Str("Type", "Preprocess").Msg(err.Error()) + } else { + LoadEnvFile(l2ConfigPath) // Loads global variables from l2config.env + } + LoadEnvFile(path.Join(dir, "l2.env")) // Overwrites the global variables if declared again in l2.env +} + +func readFile(filename string) (envMap map[string]string, err error) { + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + return godotenv.Parse(file) +} + +func populateEnvMap(envMap map[string]map[string]interface{}, envPath string, source string) error { + envs, err := readFile(envPath) + if err != nil { + return err + } + for key, value := range envs { + variable := make(map[string]interface{}) + variable["val"] = value + variable["src"] = source + envMap[key] = variable + } + return nil +} + +func getEnvMap(envPath string, source string) (map[string]map[string]interface{}, error) { + envs, err := readFile(envPath) + if err != nil { + return nil, err + } + envMap := make(map[string]map[string]interface{}) + for key, value := range envs { + variable := make(map[string]interface{}) + variable["val"] = value + variable["src"] = source + envMap[key] = variable + } + return envMap, nil +} + +func combineEnvMaps(envMaps ...map[string]map[string]interface{}) map[string]map[string]interface{} { + // First l2config varibales are added to the map. + // Then l2env varibales, so if a variable is redeclared in l2env then it will overwritten and taken to consideration. + finalEnvMap := make(map[string]map[string]interface{}) + for _, envMap := range envMaps { + for key, value := range envMap { + finalEnvMap[key] = value + } + } + return finalEnvMap +} + +func GetL2EnvVariables(dir string) ([]byte, error) { + l2ConfigPath, err := SearchL2ConfigEnv(dir) + if err != nil { + return nil, err } + l2ConfigEnvMap, err := getEnvMap(l2ConfigPath, "l2configenv") + if err != nil { + return nil, err + } + + l2EnvPath := path.Join(dir, "l2.env") + l2EnvMap, err := getEnvMap(l2EnvPath, "l2env") + if err != nil { + return nil, err + } + + finalEnvMap := combineEnvMaps(l2ConfigEnvMap, l2EnvMap) + + jsonEnvs, err := json.MarshalIndent(finalEnvMap, "", " ") + if err != nil { + return nil, fmt.Errorf("Failed to marshal map env's to JSON: %v", err) + } + + return jsonEnvs, nil } func GetLamaFileAsString(path string) string { @@ -123,7 +226,7 @@ func LamaFile(inputFile string) (string, string) { oldDir, _ := os.Getwd() utils.ChangeWorkingDir(dir) - LoadElfEnv("l2.env") + LoadEnvFile("l2.env") content = os.ExpandEnv(content) utils.ChangeWorkingDir(oldDir) diff --git a/tests/base64_test.go b/tests/base64_test.go index 0c1d915c..764b9297 100644 --- a/tests/base64_test.go +++ b/tests/base64_test.go @@ -14,14 +14,13 @@ import ( ) func TestBase64(t *testing.T) { - // parse file l2Path := "../examples/0008_base64_image/0008_base64_image.l2" apiContent, _ := os.ReadFile(l2Path) _, dir, _ := utils.GetFilePathComponents(l2Path) nowPwd, _ := os.Getwd() utils.ChangeWorkingDir(dir) - preprocess.LoadElfEnv(path.Join(dir, "l2.env")) + preprocess.LoadEnvFile(path.Join(dir, "l2.env")) p := parser.NewLama2Parser() parsedAPI, _ := p.Parse(string(apiContent)) fmt.Println(parsedAPI) @@ -43,5 +42,4 @@ func TestBase64(t *testing.T) { fmt.Println(block) } utils.ChangeWorkingDir(nowPwd) - } diff --git a/tests/env_command_test.go b/tests/env_command_test.go new file mode 100644 index 00000000..f48b6009 --- /dev/null +++ b/tests/env_command_test.go @@ -0,0 +1,111 @@ +package tests + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "testing" + + "github.com/rs/zerolog/log" +) + +type EnvData struct { + Src string `json:"src"` + Val string `json:"val"` +} + +func TestL2EnvCommand(t *testing.T) { + cmdArgs := []string{"-e", "../elfparser/ElfTestSuite/root_variable_override/api/y_0020_root_override.l2"} + runL2CommandAndParseJSON(t, cmdArgs...) +} + +func TestL2EnvCommandVerbose(t *testing.T) { + cmdArgs := []string{"-ev", "../elfparser/ElfTestSuite/root_variable_override/api/y_0020_root_override.l2"} + runL2CommandAndParseJSON(t, cmdArgs...) +} + +func runL2CommandAndParseJSON(t *testing.T, cmdArgs ...string) { + // Get the full path to the l2 binary + l2BinPath := "../build/l2" + + // Check if the l2 binary file exists + if err := checkL2BinaryExists(l2BinPath); err != nil { + t.Error(err) + return + } + + // Your existing code to run the l2 command and parse JSON + cmd := exec.Command(l2BinPath, cmdArgs...) + + var stdout bytes.Buffer + cmd.Stdout = &stdout + + // Execute the command + err := cmd.Run() + if err != nil { + // Handle the error if needed + t.Errorf("Error running l2 command: %v\n", err) + return + } + + // Retrieve the captured stdout + stdoutOutput := stdout.String() + + log.Debug().Str("Test env_command", stdoutOutput).Msg("output from command") + + // Convert the stdoutOutput string to []byte slice + outputBytes := []byte(stdoutOutput) + + envMap := make(map[string]EnvData) + err = json.Unmarshal(outputBytes, &envMap) + if err != nil { + t.Fatalf("Error unmarshaling JSON env: %v\nOutput:\n%s", err, stdoutOutput) + } + + // Check the "AHOST" key + checkAHost(t, envMap) + + // Check the "BHOST" key + checkBHost(t, envMap) +} + +// checkL2BinaryExists checks if the l2 binary file exists in the specified path +func checkL2BinaryExists(l2BinPath string) error { + // Check if the l2 binary file exists + if _, err := os.Stat(l2BinPath); os.IsNotExist(err) { + return fmt.Errorf("l2 binary not found in the build folder %s, please change the path", l2BinPath) + } + return nil +} + +// checkAHost checks the "AHOST" key in the JSON map +func checkAHost(t *testing.T, envMap map[string]EnvData) { + if ahost, ok := envMap["AHOST"]; !ok { + t.Error("Expected 'AHOST' key in the JSON, but it was not found") + } else { + // Example assertion: Check the "AHOST" src and val values + if ahost.Src != "l2env" { + t.Errorf(`Expected "src" value to be "l2env" for "AHOST", but got: %v`, ahost.Src) + } + if ahost.Val != "`echo http://httpbin.org`" { + t.Errorf(`Expected "val" value to be "echo http://httpbin.org" for "AHOST", but got: %v`, ahost.Val) + } + } +} + +// checkBHost checks the "BHOST" key in the JSON map +func checkBHost(t *testing.T, envMap map[string]EnvData) { + if bhost, ok := envMap["BHOST"]; !ok { + t.Error("Expected 'BHOST' key in the JSON, but it was not found") + } else { + // Example assertion: Check the "BHOST" src and val values + if bhost.Src != "l2configenv" { + t.Errorf(`Expected "src" value to be "l2configenv" for "BHOST", but got: %v`, bhost.Src) + } + if bhost.Val != "https://httpbin.org" { + t.Errorf(`Expected "val" value to be "https://httpbin.org" for "BHOST", but got: %v`, bhost.Val) + } + } +} diff --git a/tests/importer_test.go b/tests/importer_test.go index d4509a43..38f08443 100644 --- a/tests/importer_test.go +++ b/tests/importer_test.go @@ -9,7 +9,7 @@ import ( "github.com/HexmosTech/lama2/importer" ) -func TestPostman(t *testing.T) { +func TestPostman(_ *testing.T) { path, _ := os.Getwd() fmt.Println(path) pJSON := importer.ReadPostmanFile(filepath.Join(".", "data", "Backup.postman_dump.json")) diff --git a/tests/js_test.go b/tests/js_test.go index adfd61be..16ba966e 100644 --- a/tests/js_test.go +++ b/tests/js_test.go @@ -8,7 +8,7 @@ import ( "github.com/HexmosTech/lama2/cmdexec" ) -func TestRunVMCode(t *testing.T) { +func TestRunVMCode(_ *testing.T) { vm := cmdexec.GetJSVm() cmdexec.RunVMCode("let meaning=42", vm) } diff --git a/tests/lama2_test.go b/tests/lama2_test.go index c324cd49..e73f361d 100644 --- a/tests/lama2_test.go +++ b/tests/lama2_test.go @@ -63,14 +63,11 @@ func PerformParserMatch(text string) (*gabs.Container, error) { got, e := p.Parse(text) if e == nil { log.Debug().Str("Got", got.String()).Msg("") - } else { - // t.Errorf("Error not expected") - // fmt.Println(e) } return got, e } -func TestValidFiles(t *testing.T) { +func TestValidFiles(_ *testing.T) { matchFiles, _ := getDataFiles("../elfparser/ElfTestSuite", "y_*") // matchFiles, _ := getDataFiles("../elfparser/ElfTestSuite", "y_0009_varjson_basic.l2") // matchFiles, _ := getDataFiles("../elfparser/ElfTestSuite", "y_0012_varjson_multipart.l2") @@ -90,7 +87,7 @@ func TestValidFiles(t *testing.T) { } } -func TestInvalidFiles(t *testing.T) { +func TestInvalidFiles(_ *testing.T) { matchFiles, _ := getDataFiles("../elfparser/ElfTestSuite", "n_*") for _, m := range matchFiles { b, err := os.ReadFile(m) // just pass the file name diff --git a/tests/multistage_test.go b/tests/multistage_test.go index 72810497..1b8f98ef 100644 --- a/tests/multistage_test.go +++ b/tests/multistage_test.go @@ -30,9 +30,9 @@ func TestMultiStageCount(t *testing.T) { for _, block := range parsedAPIblocks { blockType := block.S("type").Data().(string) if blockType == "processor" { - procCount += 1 + procCount++ } else if blockType == "Lama2File" { - reqCount += 1 + reqCount++ } } diff --git a/tests/parser_test.go b/tests/parser_test.go index 44caf110..c679b5b8 100644 --- a/tests/parser_test.go +++ b/tests/parser_test.go @@ -9,13 +9,13 @@ import ( "github.com/rs/zerolog/log" ) -func TestLama2Parser(t *testing.T) { +func TestLama2Parser(_ *testing.T) { p := parser.NewLama2Parser() got, _ := p.Parse("GET http://google.com") log.Debug().Str("Received", got.String()).Msg("") } -func TestCharFunc(t *testing.T) { +func TestCharFunc(_ *testing.T) { p := parser.NewLama2Parser() p.SetText("GET http://google.com") @@ -127,7 +127,7 @@ func TestFailMatch(t *testing.T) { // fmt.Println("===") } -func TestLama2Start(t *testing.T) { +func TestLama2Start(_ *testing.T) { p := parser.NewLama2Parser() p.SetText("GET http://google.com") got, _ := p.Start() diff --git a/tests/preprocess_test.go b/tests/preprocess_test.go index 915a22ae..4e024acd 100644 --- a/tests/preprocess_test.go +++ b/tests/preprocess_test.go @@ -7,7 +7,7 @@ import ( "github.com/rs/zerolog/log" ) -func TestPreprocessBasic(t *testing.T) { +func TestPreprocessBasic(_ *testing.T) { op, _ := preprocess.LamaFile("../elfparser/ElfTestSuite/env1/sample.l2") log.Debug().Str("Preprocessed string", op).Msg("") }