Skip to content

Commit

Permalink
var separate syntax for variable (#348)
Browse files Browse the repository at this point in the history
* Feat: support var
* support variable templating and randomGenerations
  • Loading branch information
cedric05 authored Dec 30, 2024
1 parent 8e663ae commit 225c85c
Show file tree
Hide file tree
Showing 24 changed files with 592 additions and 151 deletions.
57 changes: 26 additions & 31 deletions dotextensions/server/handlers/basic_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def get_request_result(self, command, comp: RequestCompiler):
# redirects can add cookies
comp.httpdef.headers["cookie"] = resp.request.headers["cookie"]
try:
data.update({"http": self.get_http_from_req(comp.httpdef)})
data.update({"http": self.get_http_from_req(comp.httpdef, comp.property_util)})
except Exception as e:
logger.error("ran into error regenerating http def from parsed object")
data.update(
Expand All @@ -172,9 +172,9 @@ def get_request_comp(self, config):
return RequestCompiler(config)

@staticmethod
def get_http_from_req(request: HttpDef):
def get_http_from_req(request: HttpDef, property_util: "PropertyProvider"):
http_def = MultidefHttp(import_list=[], allhttps=[request.get_http_from_req()])
return HttpFileFormatter.format(http_def)
return HttpFileFormatter.format(http_def, property_util=property_util)


CONTEXT_SEP = """
Expand All @@ -201,34 +201,29 @@ def load_model(self):
self.content = self.content + CONTEXT_SEP + CONTEXT_SEP.join(self.args.contexts)

def select_target(self):
try:
# first try to resolve target from current context
super().select_target()
except UndefinedHttpToExtend as ex:
# if it weren't able to figure out context, try to resolve from
# contexts
for context in self.args.contexts:
try:
# if model is generated, try to figure out target
model: MultidefHttp = dothttp_model.model_from_str(context)
# by including targets in to model
self.model.allhttps = self.model.allhttps + model.allhttps
if model.import_list and model.import_list.filename:
if self.model.import_list and self.model.import_list.filename:
self.model.import_list.filename += (
model.import_list.filename
)
else:
self.model.import_list = model.import_list
self.load_imports()
self.content += context + "\n\n" + context
return super(ContentBase, self).select_target()
except Exception as e:
# contexts, can not always be correct syntax
# in such scenarios, don't complain, try to resolve with
# next contexts
logger.info("ignoring exception, context is not looking good")
raise ex
for context in self.args.contexts:
try:
# if model is generated, try to figure out target
model: MultidefHttp = dothttp_model.model_from_str(context)
# by including targets in to model
self.load_properties_from_var(model, self.property_util)
self.model.allhttps = self.model.allhttps + model.allhttps
if model.import_list and model.import_list.filename:
if self.model.import_list and self.model.import_list.filename:
self.model.import_list.filename += (
model.import_list.filename
)
else:
self.model.import_list = model.import_list
self.load_imports()
self.content += context + "\n\n" + context

except Exception as e:
# contexts, can not always be correct syntax
# in such scenarios, don't complain, try to resolve with
# next contexts
logger.info("ignoring exception, context is not looking good")
return super(ContentBase, self).select_target()


class ContentRequestCompiler(ContentBase, RequestCompiler):
Expand Down
6 changes: 4 additions & 2 deletions dotextensions/server/handlers/http2postman.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
)
from dothttp.parse.request_base import RequestCompiler
from dothttp.utils.common import json_to_urlencoded_array
from dothttp.utils.property_util import get_no_replace_property_provider
from ..models import Command, Result
from ..postman2_1 import (
POSTMAN_2_1,
Expand Down Expand Up @@ -63,6 +64,7 @@ class PostManCompiler(RequestCompiler):
def __init__(self, config, model):
self.model = model
super(PostManCompiler, self).__init__(config)
self.property_util = get_no_replace_property_provider()

def load_content(self):
return
Expand Down Expand Up @@ -349,14 +351,14 @@ def get_http_to_postman_request(http: HttpDef, description="") -> RequestClass:
# json to key value pairs
json_to_urlencoded_array(
# textx object to json
json_or_array_to_json(payload.data, lambda k: k)
json_or_array_to_json(payload.data)
)
]
request.body = body
elif json_payload := payload.json:
body.mode = Mode.RAW
body.options = {"language": "json"}
body.raw = json.dumps(json_or_array_to_json(json_payload, lambda x: x))
body.raw = json.dumps(json_or_array_to_json(json_payload))
if not request.header:
request.header = []
request.header.append(
Expand Down
19 changes: 16 additions & 3 deletions dothttp/http.tx
Original file line number Diff line number Diff line change
@@ -1,5 +1,18 @@
//HTTP: ram=HTTP2
MULTISET: (import_list=IMPORT)? (allhttps=HTTP+)?;

MULTISET: (import_list=IMPORT)? (variables=VAR*)? (allhttps=HTTP+)?;


VAR:
"var" name=ID ("=" ( func=FunctionCall | inter=InterpolatedString | value=Value ))? ';'
;

InterpolatedString:
'$"' /[^"]*/ '"'
;

FunctionCall:
name=ID ('(' args=INT ')')?
;

IMPORT: ('import' filename=String ';')* ;

Expand Down Expand Up @@ -208,7 +221,7 @@ Array:
;

Value:
flt=Float | int=Int | bl=Bool | null="null" |strs += TRIPLE_OR_DOUBLE_STRING | var=VarString | object=Object | array=Array | expr=Expression
flt=Float | int=Int | bl=Bool | null="null" | strs += TRIPLE_OR_DOUBLE_STRING | var=VarString | object=Object | array=Array | expr=Expression
;

Expression:
Expand Down
6 changes: 6 additions & 0 deletions dothttp/models/parse_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
from ..exceptions import DotHttpException


@dataclass
class Variable:
name: str
value: Union[None, str]

@dataclass
class NameWrap:
name: str
Expand Down Expand Up @@ -241,6 +246,7 @@ class ImportStmt:
class MultidefHttp:
import_list: Optional[ImportStmt]
allhttps: List[Http]
variables : List[Variable] = field(default_factory=lambda: [])


# one can get list of services and regions from
Expand Down
43 changes: 40 additions & 3 deletions dothttp/parse/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
from ..utils.common import get_real_file_path, triple_or_double_tostring
from ..utils.constants import *
from ..utils.property_util import PropertyProvider
from .dsl_jsonparser import json_or_array_to_json
from .dsl_jsonparser import json_or_array_to_json, jsonmodel_to_json


def install_unix_socket_scheme():
Expand Down Expand Up @@ -236,6 +236,7 @@ def _load_imports(
model, filename
):
import_list += model.allhttps
BaseModelProcessor.load_properties_from_var(model, property_util)
property_util.add_infile_properties(content)

@staticmethod
Expand Down Expand Up @@ -352,7 +353,43 @@ def load_props_needed_for_content(self):
self._load_props_from_content(self.content, self.property_util)

def _load_props_from_content(self, content, property_util: PropertyProvider):
self.load_properties_from_var(self.model, property_util)
property_util.add_infile_properties(content)

@staticmethod
def load_properties_from_var(model:MultidefHttp, property_util: PropertyProvider):
## this has to taken care by property util
## but it will complicate the code
for variable in model.variables:
if variable.value:
var_value = jsonmodel_to_json(variable.value)
property_util.add_infile_property_from_var(variable.name, var_value)
elif variable.func:
func_name = f"${variable.func.name}"
if func_name in property_util.rand_map:
func = property_util.rand_map[func_name]
if variable.func.args:
args = variable.func.args
var_value = func(args)
else:
var_value = func()
else:
var_value = variable.func.name
property_util.add_infile_property_from_var(variable.name, var_value)
elif variable.inter:
class PropertyResolver:
# hassle of creating class is to make it work with format_map
# instead of format which can be used with dict and can cause memory leak
def __getitem__(self, key):
if key in property_util.command_line_properties:
return property_util.command_line_properties[key]
if key in property_util.env_properties:
return property_util.env_properties[key]
if key in property_util.infile_properties and property_util.infile_properties[key].value is not None:
return property_util.infile_properties[key].value
raise KeyError(key)
var_value = variable.inter[2:-1].format_map(PropertyResolver())
property_util.add_infile_property_from_var(variable.name, var_value)


class HttpDefBase(BaseModelProcessor):
Expand Down Expand Up @@ -552,7 +589,7 @@ def _load_payload(self):
request_logger.debug(f"payload for request is `{content}`")
return Payload(content, header=mimetype)
elif data_json := self.http.payload.datajson:
d = json_or_array_to_json(data_json, self.get_updated_content)
d = json_or_array_to_json(data_json, self.property_util)
if isinstance(d, list):
raise PayloadDataNotValidException(
payload=f"data should be json/str, current: {d}"
Expand All @@ -563,7 +600,7 @@ def _load_payload(self):
elif upload_filename := self.http.payload.file:
return self.load_payload_fileinput(upload_filename)
elif json_data := self.http.payload.json:
d = json_or_array_to_json(json_data, self.get_updated_content)
d = json_or_array_to_json(json_data, self.property_util)
return Payload(json=d, header=MIME_TYPE_JSON)
elif files_wrap := self.http.payload.fileswrap:
files = []
Expand Down
112 changes: 56 additions & 56 deletions dothttp/parse/dsl_jsonparser.py
Original file line number Diff line number Diff line change
@@ -1,67 +1,67 @@
import json
from typing import Dict, List, Union
from typing import Dict, List, Optional, Union

from ..utils.property_util import PropertyProvider, get_no_replace_property_provider
from ..utils.common import triple_or_double_tostring


def json_or_array_to_json(model, update_content_func) -> Union[Dict, List]:
if isinstance(model, Dict) or isinstance(model, List):
# TODO
# this is bad
# hooking here could lead to other issues
return model
if array := model.array:
return [jsonmodel_to_json(value, update_content_func) for value in array.values]
elif json_object := model.object:
return {
# TODO i'm confused about key weather it should be string or int or float (value has float, number,
# boolean, null) but key is unsupported by requests
get_key(member, update_content_func): jsonmodel_to_json(
member.value, update_content_func
)
for member in json_object.members
}
class JsonParser:
def __init__(self, property_util: PropertyProvider):
self.property_util = property_util

def json_or_array_to_json(self, model) -> Union[Dict, List]:
if isinstance(model, Dict) or isinstance(model, List):
return model
if array := model.array:
return [self.jsonmodel_to_json(value) for value in array.values]
elif json_object := model.object:
return {
self.get_key(member): self.jsonmodel_to_json(member.value)
for member in json_object.members
}

def get_key(member, update_content_func):
if member.key:
return triple_or_double_tostring(member.key, update_content_func)
elif member.var:
return update_content_func(member.var)
def get_key(self, member):
if member.key:
return triple_or_double_tostring(member.key, self.property_util.get_updated_content)
elif member.var:
return self.property_util.get_updated_obj_content(member.var)

def jsonmodel_to_json(self, model):
if str_value := model.strs:
return triple_or_double_tostring(str_value, self.property_util.get_updated_content)
elif var_value := model.var:
return self.property_util.get_updated_obj_content(var_value)
elif int_val := model.int:
return int_val.value
elif flt := model.flt:
return flt.value
elif bl := model.bl:
return bl.value
elif json_object := model.object:
return {
self.get_key(member): self.jsonmodel_to_json(member.value)
for member in json_object.members
}
elif array := model.array:
return [self.jsonmodel_to_json(value) for value in array.values]
elif model == "null":
return None
elif expr := model.expr:
return eval(expr)

def jsonmodel_to_json(model, update_content_func):
# if length if array is 0, which means, its not string
if str_value := model.strs:
return triple_or_double_tostring(str_value, update_content_func)
elif var_value := model.var:
return get_json_data(var_value, update_content_func)
elif int_val := model.int:
return int_val.value
elif flt := model.flt:
return flt.value
elif bl := model.bl:
return bl.value
elif json_object := model.object:
return {
get_key(member, update_content_func): jsonmodel_to_json(
member.value, update_content_func
)
for member in json_object.members
}
elif array := model.array:
return [jsonmodel_to_json(value, update_content_func) for value in array.values]
elif model == "null":
return None
elif expr := model.expr:
return eval(expr)

# Supporting function
def json_or_array_to_json(model, property_util: Optional[PropertyProvider]=None) -> Union[Dict, List]:
if property_util is None:
# This is a hack to ignore replacement of variables where it is not needed
property_util = get_no_replace_property_provider()
parser = JsonParser(property_util)
return parser.json_or_array_to_json(model)

def get_json_data(var_value, update_content_func):
content: str = update_content_func(var_value)
if content == var_value:
return var_value
try:
return json.loads(content)
except ValueError:
return content

def jsonmodel_to_json(model, property_util: Optional[PropertyProvider]=None) -> Union[Dict, List]:
if property_util is None:
# This is a hack to ignore replacement of variables where it is not needed
property_util = get_no_replace_property_provider()
parser = JsonParser(property_util)
return parser.jsonmodel_to_json(model)
Loading

0 comments on commit 225c85c

Please sign in to comment.