diff --git a/dsl.go b/dsl.go index a4567d1..62f2c79 100644 --- a/dsl.go +++ b/dsl.go @@ -36,6 +36,8 @@ import ( "github.com/Knetic/govaluate" "github.com/Mzack9999/gcache" "github.com/asaskevich/govalidator" + "github.com/brianvoe/gofakeit/v7" + "github.com/gosimple/slug" "github.com/hashicorp/go-version" "github.com/iangcarroll/cookiemonster/pkg/monster" "github.com/kataras/jwt" @@ -71,25 +73,41 @@ var ( // Use With Caution: Nuclei ignores this error in extractors(ref: https://github.com/projectdiscovery/nuclei/issues/3950) ErrParsingArg = errkit.New("error parsing argument value") + errDuplicateFunc = errors.New("duplicate function") + DefaultMaxDecompressionSize = int64(10 * 1024 * 1024) // 10MB DefaultCacheSize = 6144 resultCache = gcache.New[string, interface{}](DefaultCacheSize).Build() + + // Initialize faker functions + faker = gofakeit.New(0) ) var PrintDebugCallback func(args ...interface{}) error var functions []dslFunction +var fakerFunctions []dslFunction func AddFunction(function dslFunction) error { for _, f := range functions { if function.Name == f.Name { - return errkit.New("duplicate helper function key defined") + return errkit.Wrapf(errDuplicateFunc, "duplicate helper function key: %q", f.Name) } } functions = append(functions, function) return nil } +func addFakerFunction(function dslFunction) error { + for _, f := range functions { + if function.Name == f.Name { + return fmt.Errorf("%w: %q", errDuplicateFunc, f.Name) + } + } + fakerFunctions = append(fakerFunctions, function) + return nil +} + func MustAddFunction(function dslFunction) { if err := AddFunction(function); err != nil { panic(err) @@ -1546,6 +1564,15 @@ func init() { FunctionNames = GetFunctionNames(DefaultHelperFunctions) } +// Helper function to generate function signatures for faker functions +func getFakerSignature(info gofakeit.Info) string { + var params []string + for _, p := range info.Params { + params = append(params, fmt.Sprintf("%s %s", p.Field, p.Type)) + } + return fmt.Sprintf("(%s) %s", strings.Join(params, ", "), info.Output) +} + func NewWithSingleSignature(name, signature string, cacheable bool, logic govaluate.ExpressionFunction) dslFunction { return NewWithMultipleSignatures(name, []string{signature}, cacheable, logic) } @@ -1571,6 +1598,80 @@ func NewWithPositionalArgs(name string, numberOfArgs int, cacheable bool, expr g return function } +// FakerFunctions returns the faker functions +// +// Note: It does not support backwards compatibility for function names +func FakerFunctions() map[string]govaluate.ExpressionFunction { + funcs := make(map[string]govaluate.ExpressionFunction) + slug.CustomSub = map[string]string{" ": "_"} + + for _, fInfo := range gofakeit.FuncLookups { + // NOTE(dwisiswant0): Skipping function because it not callable or the + // output is not printable. + hasSliceParam := false + for _, p := range fInfo.Params { + if strings.Contains(p.Type, "[]") { + hasSliceParam = true + break + } + } + if hasSliceParam { + continue + } + if strings.Contains(fInfo.Output, "map") { + continue + } + + funcName := "rand_" + slug.Make(fInfo.Display) + fakerFunc := func(fInfo gofakeit.Info) func(args ...any) (any, error) { + return func(args ...any) (any, error) { + // Set function and params + // Copied from: https://github.com/brianvoe/gofakeit/blob/e7c55ca0031ef39bb7673deedfc9f04fc17d8072/cmd/gofakeit/gofakeit.go#L112 + params := gofakeit.NewMapParams() + paramsLen := len(fInfo.Params) + argsLen := len(args) + if argsLen != paramsLen { + return nil, fmt.Errorf("expected %d arguments, got %d", paramsLen, argsLen) + } + + if paramsLen > 0 { + for i := 0; i < argsLen; i++ { + if i == 0 { + continue + } + + // Map argument to param field + if paramsLen >= i { + p := fInfo.Params[i-1] + arg := fmt.Sprintf("%v", args[i]) + params.Add(p.Field, arg) + } + } + } + + value, err := fInfo.Generate(faker, params, &fInfo) + if err != nil { + return "", fmt.Errorf("faker error: %w", err) + } + + return value, nil + } + } + + // Register the function with the DSL + f := fakerFunc(fInfo) + err := addFakerFunction(NewWithSingleSignature( + funcName, getFakerSignature(fInfo), false, f, + )) + if err != nil && !errors.Is(err, errDuplicateFunc) { + panic(fmt.Errorf("%w (faker)", err)) + } + funcs[funcName] = f + } + + return funcs +} + // HelperFunctions returns the dsl helper functions func HelperFunctions() map[string]govaluate.ExpressionFunction { helperFunctions := make(map[string]govaluate.ExpressionFunction) @@ -1595,23 +1696,37 @@ func GetFunctionNames(heperFunctions map[string]govaluate.ExpressionFunction) [] return maputils.GetKeys(heperFunctions) } +// GetPrintableDslFunctionSignatures returns the function signatures for the +// default DSL functions func GetPrintableDslFunctionSignatures(noColor bool) string { if noColor { - return aggregate(getDslFunctionSignatures()) + return aggregate(getDslFunctionSignatures(functions)) } - return aggregate(colorizeDslFunctionSignatures()) + return aggregate(colorizeDslFunctionSignatures(functions)) } -func getDslFunctionSignatures() []string { +// GetPrintableFakerDslFunctionSignatures returns the function signatures for +// the faker functions. +// +// Note: [FakerFunctions] must be called first to populate the functions +// map with the faker functions. +func GetPrintableFakerDslFunctionSignatures(noColor bool) string { + if noColor { + return aggregate(getDslFunctionSignatures(fakerFunctions)) + } + return aggregate(colorizeDslFunctionSignatures(fakerFunctions)) +} + +func getDslFunctionSignatures(funcs []dslFunction) []string { var result []string - for _, function := range functions { - result = append(result, function.GetSignatures()...) + for _, f := range funcs { + result = append(result, f.GetSignatures()...) } return result } -func colorizeDslFunctionSignatures() []string { - signatures := getDslFunctionSignatures() +func colorizeDslFunctionSignatures(funcs []dslFunction) []string { + signatures := getDslFunctionSignatures(funcs) colorToOrange := func(value string) string { return aurora.Index(208, value).String() diff --git a/dsl_test.go b/dsl_test.go index c42a156..a8d9f90 100644 --- a/dsl_test.go +++ b/dsl_test.go @@ -593,15 +593,14 @@ func TestDateTimeDslExpressions(t *testing.T) { func TestRandDslExpressions(t *testing.T) { randDslExpressions := map[string]string{ - `rand_base(10, "")`: `[a-zA-Z0-9]{10}`, - `rand_base(5, "abc")`: `[abc]{5}`, - `rand_base(5)`: `[a-zA-Z0-9]{5}`, - `rand_char("abc")`: `[abc]{1}`, - `rand_char("")`: `[a-zA-Z0-9]{1}`, - `rand_char()`: `[a-zA-Z0-9]{1}`, - `rand_ip("192.168.0.0/24")`: `(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`, - `rand_ip("2001:db8::/64")`: `(?:[A-Fa-f0-9]{0,4}:){0,7}[A-Fa-f0-9]{0,4}$`, - + `rand_base(10, "")`: `[a-zA-Z0-9]{10}`, + `rand_base(5, "abc")`: `[abc]{5}`, + `rand_base(5)`: `[a-zA-Z0-9]{5}`, + `rand_char("abc")`: `[abc]{1}`, + `rand_char("")`: `[a-zA-Z0-9]{1}`, + `rand_char()`: `[a-zA-Z0-9]{1}`, + `rand_ip("192.168.0.0/24")`: `(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`, + `rand_ip("2001:db8::/64")`: `(?:[A-Fa-f0-9]{0,4}:){0,7}[A-Fa-f0-9]{0,4}$`, `rand_text_alpha(10, "abc")`: `[^abc]{10}`, `rand_text_alpha(10, "")`: `[a-zA-Z]{10}`, `rand_text_alpha(10)`: `[a-zA-Z]{10}`, @@ -627,6 +626,283 @@ func TestRandDslExpressions(t *testing.T) { } } +func TestFakerDslExpressions(t *testing.T) { + zeroArgsFakerFunctions := []string{ + "rand_ach_account_number", + "rand_ach_routing_number", + "rand_action_verb", + "rand_adjective", + "rand_adverb", + "rand_adverb_degree", + "rand_adverb_frequency_definite", + "rand_adverb_frequency_indefinite", + "rand_adverb_manner", + "rand_adverb_phrase", + "rand_adverb_place", + "rand_adverb_time_definite", + "rand_adverb_time_indefinite", + "rand_animal", + "rand_animal_type", + "rand_app_author", + "rand_app_name", + "rand_app_version", + "rand_author", + "rand_beer_alcohol", + "rand_beer_blg", + "rand_beer_hop", + "rand_beer_ibu", + "rand_beer_malt", + "rand_beer_name", + "rand_beer_style", + "rand_beer_yeast", + "rand_bird", + "rand_bitcoin_address", + "rand_bitcoin_private_key", + "rand_blurb", + "rand_boolean", + "rand_breakfast", + "rand_bs", + "rand_buzzword", + "rand_car_fuel_type", + "rand_car_maker", + "rand_car_model", + "rand_car_transmission_type", + "rand_car_type", + "rand_cat", + "rand_celebrity_actor", + "rand_celebrity_business", + "rand_celebrity_sport", + "rand_chrome_user_agent", + "rand_city", + "rand_color", + "rand_comment", + "rand_company", + "rand_company_suffix", + "rand_connective", + "rand_connective_casual", + "rand_connective_comparitive", + "rand_connective_complaint", + "rand_connective_examplify", + "rand_connective_listing", + "rand_connective_time", + "rand_country", + "rand_country_abbreviation", + "rand_credit_card_cvv", + "rand_credit_card_exp", + "rand_credit_card_type", + "rand_currency_long", + "rand_currency_short", + "rand_cusip", + "rand_database_error", + "rand_day", + "rand_demonstrative_adjective", + "rand_descriptive_adjective", + "rand_dessert", + "rand_digit", + "rand_dinner", + "rand_dog", + "rand_domain_name", + "rand_domain_suffix", + "rand_drink", + "rand_email", + "rand_emoji", + "rand_emoji_alias", + "rand_emoji_category", + "rand_emoji_description", + "rand_emoji_tag", + "rand_error", + "rand_error_object_word", + "rand_farm_animal", + "rand_file_extension", + "rand_file_mime_type", + "rand_firefox_user_agent", + "rand_first_name", + "rand_flip_a_coin", + "rand_float32", + "rand_float64", + "rand_fruit", + "rand_futuredate", + "rand_gamertag", + "rand_gender", + "rand_genre", + "rand_grpc_error", + "rand_hacker_abbreviation", + "rand_hacker_adjective", + "rand_hacker_noun", + "rand_hacker_phrase", + "rand_hacker_verb", + "rand_hackering_verb", + "rand_helping_verb", + "rand_hex_color", + "rand_hipster_word", + "rand_hobby", + "rand_hour", + "rand_http_client_error", + "rand_http_error", + "rand_http_method", + "rand_http_server_error", + "rand_http_status_code", + "rand_http_status_code_simple", + "rand_http_version", + "rand_indefinite_adjective", + "rand_input_name", + "rand_int16", + "rand_int32", + "rand_int64", + "rand_int8", + "rand_interjection", + "rand_interrogative_adjective", + "rand_intransitive_verb", + "rand_ipv4_address", + "rand_ipv6_address", + "rand_isin", + "rand_job_descriptor", + "rand_job_level", + "rand_job_title", + "rand_language", + "rand_language_abbreviation", + "rand_language_bcp", + "rand_last_name", + "rand_latitude", + "rand_letter", + "rand_linking_verb", + "rand_log_level", + "rand_longitude", + "rand_lorem_ipsum_word", + "rand_lunch", + "rand_mac_address", + "rand_middle_name", + "rand_minecraft_animal", + "rand_minecraft_armor_part", + "rand_minecraft_armor_tier", + "rand_minecraft_biome", + "rand_minecraft_dye", + "rand_minecraft_food", + "rand_minecraft_mob_boss", + "rand_minecraft_mob_hostile", + "rand_minecraft_mob_neutral", + "rand_minecraft_mob_passive", + "rand_minecraft_ore", + "rand_minecraft_tool", + "rand_minecraft_villager_job", + "rand_minecraft_villager_level", + "rand_minecraft_villager_station", + "rand_minecraft_weapon", + "rand_minecraft_weather", + "rand_minecraft_wood", + "rand_minute", + "rand_month", + "rand_month_string", + "rand_movie_name", + "rand_name", + "rand_name_prefix", + "rand_name_suffix", + "rand_nanosecond", + "rand_nice_colors", + "rand_noun", + "rand_noun_abstract", + "rand_noun_collective_animal", + "rand_noun_collective_people", + "rand_noun_collective_thing", + "rand_noun_common", + "rand_noun_concrete", + "rand_noun_countable", + "rand_noun_determiner", + "rand_noun_phrase", + "rand_noun_proper", + "rand_noun_uncountable", + "rand_opera_user_agent", + "rand_pastdate", + "rand_pet_name", + "rand_phone", + "rand_phone_formatted", + "rand_phrase", + "rand_possessive_adjective", + "rand_preposition", + "rand_preposition_compound", + "rand_preposition_double", + "rand_preposition_phrase", + "rand_preposition_simple", + "rand_product_audience", + "rand_product_benefit", + "rand_product_category", + "rand_product_description", + "rand_product_dimension", + "rand_product_feature", + "rand_product_material", + "rand_product_name", + "rand_product_suffix", + "rand_product_upc", + "rand_product_use_case", + "rand_programming_language", + "rand_pronoun", + "rand_pronoun_demonstrative", + "rand_pronoun_indefinite", + "rand_pronoun_interrogative", + "rand_pronoun_object", + "rand_pronoun_personal", + "rand_pronoun_possessive", + "rand_pronoun_reflective", + "rand_pronoun_relative", + "rand_proper_adjective", + "rand_quantitative_adjective", + "rand_question", + "rand_quote", + "rand_random_markdown_document", + "rand_random_text_email_document", + "rand_rgb_color", + "rand_runtime_error", + "rand_safari_user_agent", + "rand_safe_color", + "rand_school", + "rand_second", + "rand_simple_sentence", + "rand_slogan", + "rand_snack", + "rand_song_artist", + "rand_song_name", + "rand_ssn", + "rand_state", + "rand_state_abbreviation", + "rand_street", + "rand_street_name", + "rand_street_number", + "rand_street_prefix", + "rand_street_suffix", + "rand_timezone", + "rand_timezone_abbreviation", + "rand_timezone_full", + "rand_timezone_offset", + "rand_timezone_region", + "rand_title", + "rand_transitive_verb", + "rand_uint", + "rand_uint16", + "rand_uint32", + "rand_uint64", + "rand_uint8", + "rand_url", + "rand_user_agent", + "rand_username", + "rand_uuid", + "rand_validation_error", + "rand_vegetable", + "rand_verb", + "rand_verb_phrase", + "rand_vowel", + "rand_weekday", + "rand_word", + "rand_year", + "rand_zip", + } + + for _, functionName := range zeroArgsFakerFunctions { + t.Run(functionName, func(t *testing.T) { + evaluateFakerExpression(t, functionName+"()") + }) + } +} + func TestRandIntDslExpressions(t *testing.T) { randIntDslExpressions := map[string]func(int) bool{ `rand_int(5, 9)`: func(i int) bool { @@ -692,6 +968,22 @@ func evaluateExpression(t *testing.T, dslExpression string, functions ...dslFunc return actualResult } +func evaluateFakerExpression(t *testing.T, dslExpression string, functions ...dslFunction) interface{} { + helperFunctions := maps.Clone(FakerFunctions()) + for _, function := range functions { + helperFunctions[function.Name] = function.Exec + } + compiledExpression, err := govaluate.NewEvaluableExpressionWithFunctions(dslExpression, helperFunctions) + require.NoError(t, err, "Error while compiling the %q expression", dslExpression) + + actualResult, err := compiledExpression.Evaluate(make(map[string]interface{})) + require.NoError(t, err, "Error while evaluating the compiled %q expression", dslExpression) + + require.NotContains(t, fmt.Sprintf("%v", actualResult), "faker error") + + return actualResult +} + func testDslExpressions(t *testing.T, dslExpressions map[string]interface{}) { for dslExpression, expectedResult := range dslExpressions { t.Run(dslExpression, func(t *testing.T) { @@ -706,6 +998,10 @@ func testDslExpressions(t *testing.T, dslExpressions map[string]interface{}) { } } +func Test_GetPrintableDslFunctionSignatures(t *testing.T) { + fmt.Print(GetPrintableDslFunctionSignatures(true)) +} + func Test_Zlib_decompression_bomb(t *testing.T) { compressedFile := "testdata/zlib_bomb.zlib" diff --git a/go.mod b/go.mod index cfaa2a6..2927a95 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 + github.com/brianvoe/gofakeit/v7 v7.2.1 + github.com/gosimple/slug v1.15.0 github.com/hashicorp/go-version v1.6.0 github.com/iangcarroll/cookiemonster v1.6.0 github.com/kataras/jwt v0.1.8 @@ -32,6 +34,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/gorilla/css v1.0.1 // indirect + github.com/gosimple/unidecode v1.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect diff --git a/go.sum b/go.sum index 1bb043e..8c76089 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/brianvoe/gofakeit/v7 v7.2.1 h1:AGojgaaCdgq4Adzrd2uWdbGNDyX6MWNhHdQBraNfOHI= +github.com/brianvoe/gofakeit/v7 v7.2.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= @@ -84,6 +86,10 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= +github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= +github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= +github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=