Skip to content

Commit

Permalink
feat: toggle url decoding in message operations save
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattebit
mattebit committed Mar 12, 2024
1 parent 1303c15 commit 9866cf9
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 26 deletions.
101 changes: 93 additions & 8 deletions tool/src/main/java/migt/HTTPReqRes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,20 @@
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.message.ParserCursor;
import org.apache.http.message.TokenParser;
import org.apache.http.util.Args;
import org.apache.http.util.CharArrayBuffer;

import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -114,6 +120,50 @@ public HTTPReqRes(IHttpRequestResponse message, IExtensionHelpers helpers, boole
instances++;
}

/**
* Function taken from URLEncodedUtils and edited
* Returns a list of {@link NameValuePair}s parameters.
*
* @param buf text to parse.
* @param charset Encoding to use when decoding the parameters.
* @param separators element separators.
* @return a list of {@link NameValuePair} as built from the URI's query portion.
* @since 4.4
*/
public static List<NameValuePair> parse_url_query_no_decoding(
final CharArrayBuffer buf, final Charset charset, final char... separators) {
Args.notNull(buf, "Char array buffer");
final TokenParser tokenParser = TokenParser.INSTANCE;
final BitSet delimSet = new BitSet();
for (final char separator : separators) {
delimSet.set(separator);
}
final ParserCursor cursor = new ParserCursor(0, buf.length());
final List<NameValuePair> list = new ArrayList<NameValuePair>();
while (!cursor.atEnd()) {
delimSet.set('=');
final String name = tokenParser.parseToken(buf, cursor, delimSet);
String value = null;
if (!cursor.atEnd()) {
final int delim = buf.charAt(cursor.getPos());
cursor.updatePos(cursor.getPos() + 1);
if (delim == '=') {
delimSet.clear('=');
value = tokenParser.parseToken(buf, cursor, delimSet);
if (!cursor.atEnd()) {
cursor.updatePos(cursor.getPos() + 1);
}
}
}
if (!name.isEmpty()) {
list.add(new BasicNameValuePair(
name,
value));
}
}
return list;
}

/**
* Function used to replace an IHttpRequestResponse message with the values contained in this object
*
Expand Down Expand Up @@ -360,26 +410,62 @@ public Object clone() throws CloneNotSupportedException {
}

/**
* Get the given parameter value from the url of the request messsage
* Get the Name Value Pair list of the query parameters in the url.
*
* @param param the parameter name to be searched
* @return the value of the parameter
* @return the List of Name Value pairs
*/
public String getUrlParam(String param) {
private List<NameValuePair> getNameValuePairUrl() {
if (!isRequest || request_url == null) {
throw new RuntimeException("Trying to access the url of a response message");
}

List<NameValuePair> params = new ArrayList<>();

try {
params = URLEncodedUtils.parse(
new URI(request_url), StandardCharsets.UTF_8
);
URI u = new URI(request_url);
params = URLEncodedUtils.parse(u, StandardCharsets.UTF_8);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}

return params;
}

/**
* Get the given parameter value from the url of the request messsage
*
* @param param the parameter name to be searched
* @return the value of the parameter
*/
public String getUrlParam(String param) {
List<NameValuePair> params = getNameValuePairUrl();

for (NameValuePair p : params) {
if (p.getName().equals(param)) {
return p.getValue();
}
}

return "";
}

/**
* Get the given parameter value from the url of the request messsage.
*
* @param disable_url_encode Set to true to get the value of the parameter without URL decoding it
* @param param the parameter name to be searched
* @return the value of the parameter
*/
public String getUrlParam(String param, boolean disable_url_encode) {
List<NameValuePair> params = new ArrayList<>();
if (disable_url_encode) {
final CharArrayBuffer buffer = new CharArrayBuffer(getUrl().length());
buffer.append(getUrl());
params = parse_url_query_no_decoding(buffer, StandardCharsets.UTF_8, '&', ';');
} else {
params = getNameValuePairUrl();
}

for (NameValuePair p : params) {
if (p.getName().equals(param)) {
return p.getValue();
Expand Down Expand Up @@ -620,7 +706,6 @@ public String getHeadRegex(boolean isRequest, String regex) {
return res;
}


/**
* Edits the Header of the given message
*
Expand Down
6 changes: 5 additions & 1 deletion tool/src/main/java/migt/MessageOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public class MessageOperation extends Module {
String save_as; // The name of the variable to save the parameter's value
String use;
MessageOpType type;
boolean url_decode = true; // enable or disable url decoding of url parameters

// GENERATE POC
String template;
Expand Down Expand Up @@ -97,6 +98,9 @@ public MessageOperation(JSONObject message_op_json) throws ParsingException {
case "output_path":
output_path = message_op_json.getString("output_path");
break;
case "url decode":
url_decode = message_op_json.getBoolean("url decode");
break;
default:
throw new ParsingException("Message operation key \" " + key + "\" not valid");
}
Expand Down Expand Up @@ -360,7 +364,7 @@ public void execute() {
}

String value = action == SAVE ?
((Operation_API) imported_api).message.getUrlParam(what) :
((Operation_API) imported_api).message.getUrlParam(what, !url_decode) :
((Operation_API) imported_api).message.getUrlRegex(what);

if (value.isEmpty()) {
Expand Down
1 change: 0 additions & 1 deletion tool/src/test/java/DecodeOperation_Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class DecodeOperation_Test {
String input = "{\n" +
Expand Down
43 changes: 27 additions & 16 deletions tool/src/test/java/HTTPReqRes_Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
public class HTTPReqRes_Test {

public static HTTPReqRes initMessage_ok() {
String raw = "POST /log?format=json&hasfast=true&authuser=0 HTTPS/2\r\n" +
String raw = "POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTPS/2\r\n" +
"Host: play.google.com\r\n" +
"Cookie: CONSENT=PENDING+392; SOCS=CAISHAgCEhJnd3NfMjAyMzAyMjgtMF9SQzIaAml0IAEaBgiA2pSgBg; AEC=AUEFqZdSS4hmP6dNNRrldXefJFuHK2ldiLrZLJG24hUqaFA2L0jJxZwSBA; NID=511=SPj3DZBbWBMVstxl414okznEMUOaUHRzxZehEHxoaTi0Fr_X9RQ6UmFDBvI6wWn1Iivh7lzi_q7Ktri2q8hHc9nVY3XNgQP-IQ4AHNz7lCKra72IjxzhBvEBQFdXy7lEaIVC3wK5TfPIXLX3TWhKwrZAVEg77UkqV2oHYohcSXg\r\n" +
"Content-Length: 11\r\n" +
Expand Down Expand Up @@ -50,14 +50,13 @@ public static HTTPReqRes initMessage_ok() {
message.setHeaders(true, headers);
message.isRequest = true;
message.isResponse = false;
message.setRequest_url("https://play.google.com/log?format=json&hasfast=true&authuser=0");
message.setRequest_url("https://play.google.com/log?format=json&hasfast=true&authuser=0&paramwithspace=first+last");

return message;
}

@Test
public HTTPReqRes init_message_no_body() {
String raw = "POST /log?format=json&hasfast=true&authuser=0 HTTPS/2\r\n" +
String raw = "POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTPS/2\r\n" +
"Host: play.google.com\r\n" +
"Cookie: CONSENT=PENDING+392; SOCS=CAISHAgCEhJnd3NfMjAyMzAyMjgtMF9SQzIaAml0IAEaBgiA2pSgBg; AEC=AUEFqZdSS4hmP6dNNRrldXefJFuHK2ldiLrZLJG24hUqaFA2L0jJxZwSBA; NID=511=SPj3DZBbWBMVstxl414okznEMUOaUHRzxZehEHxoaTi0Fr_X9RQ6UmFDBvI6wWn1Iivh7lzi_q7Ktri2q8hHc9nVY3XNgQP-IQ4AHNz7lCKra72IjxzhBvEBQFdXy7lEaIVC3wK5TfPIXLX3TWhKwrZAVEg77UkqV2oHYohcSXg\r\n" +
"Content-Length: 11\r\n" +
Expand Down Expand Up @@ -90,15 +89,15 @@ public HTTPReqRes init_message_no_body() {
message.setHeaders(true, headers);
message.isRequest = true;
message.isResponse = false;
message.setRequest_url("https://play.google.com/log?format=json&hasfast=true&authuser=0");
message.setRequest_url("https://play.google.com/log?format=json&hasfast=true&authuser=0&paramwithspace=first+last");

return message;
}

@Test
@DisplayName("")
public void test_build() {
String raw = "POST /log?format=json&hasfast=true&authuser=0 HTTP/2\r\n" +
String raw = "POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTP/2\r\n" +
"Host: play.google.com\r\n" +
"Cookie: CONSENT=PENDING+392; SOCS=CAISHAgCEhJnd3NfMjAyMzAyMjgtMF9SQzIaAml0IAEaBgiA2pSgBg; AEC=AUEFqZdSS4hmP6dNNRrldXefJFuHK2ldiLrZLJG24hUqaFA2L0jJxZwSBA; NID=511=SPj3DZBbWBMVstxl414okznEMUOaUHRzxZehEHxoaTi0Fr_X9RQ6UmFDBvI6wWn1Iivh7lzi_q7Ktri2q8hHc9nVY3XNgQP-IQ4AHNz7lCKra72IjxzhBvEBQFdXy7lEaIVC3wK5TfPIXLX3TWhKwrZAVEg77UkqV2oHYohcSXg\r\n" +
"Content-Length: 11\r\n" +
Expand Down Expand Up @@ -140,7 +139,7 @@ public void test_build() {
@Test
@DisplayName("")
public void test_build_no_body() {
String raw = "POST /log?format=json&hasfast=true&authuser=0 HTTP/2\r\n" +
String raw = "POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTP/2\r\n" +
"Host: play.google.com\r\n" +
"Cookie: CONSENT=PENDING+392; SOCS=CAISHAgCEhJnd3NfMjAyMzAyMjgtMF9SQzIaAml0IAEaBgiA2pSgBg; AEC=AUEFqZdSS4hmP6dNNRrldXefJFuHK2ldiLrZLJG24hUqaFA2L0jJxZwSBA; NID=511=SPj3DZBbWBMVstxl414okznEMUOaUHRzxZehEHxoaTi0Fr_X9RQ6UmFDBvI6wWn1Iivh7lzi_q7Ktri2q8hHc9nVY3XNgQP-IQ4AHNz7lCKra72IjxzhBvEBQFdXy7lEaIVC3wK5TfPIXLX3TWhKwrZAVEg77UkqV2oHYohcSXg\r\n" +
"Sec-Ch-Ua: \"Chromium\";v=\"111\", \"Not(A:Brand\";v=\"8\"\r\n" +
Expand Down Expand Up @@ -178,7 +177,7 @@ public void test_build_no_body() {
@Test
@DisplayName("")
public void test_build_no_body_with_content_len() {
String raw = "POST /log?format=json&hasfast=true&authuser=0 HTTP/2\r\n" +
String raw = "POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTP/2\r\n" +
"Host: play.google.com\r\n" +
"Cookie: CONSENT=PENDING+392; SOCS=CAISHAgCEhJnd3NfMjAyMzAyMjgtMF9SQzIaAml0IAEaBgiA2pSgBg; AEC=AUEFqZdSS4hmP6dNNRrldXefJFuHK2ldiLrZLJG24hUqaFA2L0jJxZwSBA; NID=511=SPj3DZBbWBMVstxl414okznEMUOaUHRzxZehEHxoaTi0Fr_X9RQ6UmFDBvI6wWn1Iivh7lzi_q7Ktri2q8hHc9nVY3XNgQP-IQ4AHNz7lCKra72IjxzhBvEBQFdXy7lEaIVC3wK5TfPIXLX3TWhKwrZAVEg77UkqV2oHYohcSXg\r\n" +
"Sec-Ch-Ua: \"Chromium\";v=\"111\", \"Not(A:Brand\";v=\"8\"\r\n" +
Expand Down Expand Up @@ -224,7 +223,7 @@ public void test_getUrlHeader() {
HTTPReqRes message = initMessage_ok();
String header_0 = message.getUrlHeader();

assertEquals("POST /log?format=json&hasfast=true&authuser=0 HTTPS/2", header_0);
assertEquals("POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTPS/2", header_0);
}

@Test
Expand All @@ -244,7 +243,7 @@ public void test_getUrlParam() {
public void test_editUrlParam() throws ParsingException {
HTTPReqRes message = initMessage_ok();
message.editUrlParam("format", "new");
assertEquals("https://play.google.com/log?format=new&hasfast=true&authuser=0", message.getUrl());
assertEquals("https://play.google.com/log?format=new&hasfast=true&authuser=0&paramwithspace=first+last", message.getUrl());
message.setRequest_url("https://play.google.com:8080/log?format=new&hasfast=true&authuser=0#123123123");
message.editUrlParam("format", "newnew");
assertEquals("https://play.google.com:8080/log?format=newnew&hasfast=true&authuser=0#123123123", message.getUrl());
Expand All @@ -263,29 +262,29 @@ public void test_url_update_header_0() {
public void test_editUrlHeaders() throws ParsingException {
HTTPReqRes message = initMessage_ok();
message.updateHeadersWHurl();
assertEquals("POST /log?format=json&hasfast=true&authuser=0 HTTPS/2", message.getHeaders(true).get(0));
assertEquals("POST /log?format=json&hasfast=true&authuser=0&paramwithspace=first+last HTTPS/2", message.getHeaders(true).get(0));
message.editUrlParam("format", "new");
assertEquals("POST /log?format=new&hasfast=true&authuser=0 HTTPS/2", message.getHeaders(true).get(0));
assertEquals("POST /log?format=new&hasfast=true&authuser=0&paramwithspace=first+last HTTPS/2", message.getHeaders(true).get(0));
message.removeUrlParam("hasfast");
assertEquals("POST /log?format=new&authuser=0 HTTPS/2", message.getHeaders(true).get(0));
assertEquals("POST /log?format=new&authuser=0&paramwithspace=first+last HTTPS/2", message.getHeaders(true).get(0));
message.addUrlParam("prova", "provona");
assertEquals("POST /log?format=new&authuser=0&prova=provona HTTPS/2", message.getHeaders(true).get(0));
assertEquals("POST /log?format=new&authuser=0&paramwithspace=first+last&prova=provona HTTPS/2", message.getHeaders(true).get(0));
}

@Test
@DisplayName("")
public void test_removeUrlParam() throws ParsingException {
HTTPReqRes message = initMessage_ok();
message.removeUrlParam("format");
assertEquals("https://play.google.com/log?hasfast=true&authuser=0", message.getUrl());
assertEquals("https://play.google.com/log?hasfast=true&authuser=0&paramwithspace=first+last", message.getUrl());
}

@Test
@DisplayName("")
public void test_addUrlParam() throws ParsingException {
HTTPReqRes message = initMessage_ok();
message.addUrlParam("test", "test");
assertEquals("https://play.google.com/log?format=json&hasfast=true&authuser=0&test=test", message.getUrl());
assertEquals("https://play.google.com/log?format=json&hasfast=true&authuser=0&paramwithspace=first+last&test=test", message.getUrl());
}

@Test
Expand Down Expand Up @@ -348,5 +347,17 @@ public void test_edit_body_regex() {
assertEquals("body1234nt", new String(message.getBody(true)));
}

@Test
public void test_get_url_param_no_urlencode() {
HTTPReqRes msg = initMessage_ok();

// with url encode (default)
String value = msg.getUrlParam("paramwithspace");
assertEquals("first last", value);

// without url encode
value = msg.getUrlParam("paramwithspace", true);
assertEquals("first+last", value);
}

}
20 changes: 20 additions & 0 deletions tool/src/test/java/MessageOeration_Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,26 @@ void test_save_url_param() throws ParsingException {
assertEquals("json", op_api.vars.get(0).value);
}

@Test
void test_save_url_param_no_decode() throws ParsingException {
HTTPReqRes msg = HTTPReqRes_Test.initMessage_ok();
String msg_op_txt =
"{\n" +
" \"from\": \"url\",\n" +
" \"save\": \"paramwithspace\",\n" +
" \"as\": \"var_name\",\n" +
" \"url decode\": false" +
" }";

MessageOperation mop = new MessageOperation(new JSONObject(msg_op_txt));
Operation_API op_api = new Operation_API(msg, true);
mop.loader(op_api);
mop.execute();
op_api = mop.exporter();
assertTrue(mop.getResult());
assertEquals("first+last", op_api.vars.get(0).value);
}

@Test
void test_save_url_regex() throws ParsingException {
HTTPReqRes msg = HTTPReqRes_Test.initMessage_ok();
Expand Down

0 comments on commit 9866cf9

Please sign in to comment.