Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Jason Instead of JSX #179

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+70 −20
Open

Use Jason Instead of JSX #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions lib/exvcr/iex.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ defmodule ExVCR.IEx do
unquote(test)
after
ExVCR.MockLock.release_lock()
Recorder.get(recorder)
|> JSX.encode!
|> JSX.prettify!

recorder
|> Recorder.get()
|> Jason.encode_to_iodata!(pretty: true)
|> IO.puts
end
:ok
Expand Down
12 changes: 6 additions & 6 deletions lib/exvcr/json.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ defmodule ExVCR.JSON do
Save responses into the json file.
"""
def save(file_name, recordings) do
json = recordings
|> Enum.map(&encode_binary_data/1)
|> Enum.reverse()
|> JSX.encode!()
|> JSX.prettify!()
json =
recordings
|> Enum.map(&encode_binary_data/1)
|> Enum.reverse()
|> Jason.encode_to_iodata!(pretty: true)

unless File.exists?(path = Path.dirname(file_name)), do: File.mkdir_p!(path)
File.write!(file_name, json)
Expand Down Expand Up @@ -48,7 +48,7 @@ defmodule ExVCR.JSON do
def read_json_file(file_name) do
file_name
|> File.read!()
|> JSX.decode!()
|> Jason.decode!()
|> Enum.map(&load_binary_data/1)
end

Expand Down
36 changes: 36 additions & 0 deletions lib/exvcr/records.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,53 @@ defmodule ExVCR.Record do
defstruct options: nil, responses: nil
end

defmodule ExVCR.Utils do
@moduledoc false

def keyword_to_map(kw) do
kw
|> Enum.sort(fn {k1, _}, {k2, _} -> k1 <= k2 end)
|> Enum.reduce(%{}, fn {k, v}, acc -> Map.put(acc, k, v) end)
end
end

defmodule ExVCR.Request do
defstruct url: nil, headers: [], method: nil, body: nil, options: [], request_body: ""

defimpl Jason.Encoder, for: ExVCR.Request do
def encode(value, opts) do
%{headers: headers, options: options} = Map.take(value, [:headers, :options])

map = value
|> Map.take([:url, :method, :body, :request_body])
|> Map.put(:headers, ExVCR.Utils.keyword_to_map(headers))
|> Map.put(:options, ExVCR.Utils.keyword_to_map(options))

Jason.Encode.map(map, opts)
end
end
end

defmodule ExVCR.Response do
defstruct type: "ok", status_code: nil, headers: [], body: nil, binary: false

defimpl Jason.Encoder, for: ExVCR.Response do
def encode(value, opts) do
headers = Map.get(value, :headers)

map = value
|> Map.take([:type, :status_code, :body, :binary])
|> Map.put(:headers, ExVCR.Utils.keyword_to_map(headers))

Jason.Encode.map(map, opts)
end
end
end

defmodule ExVCR.Checker.Results do
defstruct dirs: nil, files: []
end

defmodule ExVCR.Checker.Counts do
defstruct server: 0, cache: 0
end
19 changes: 12 additions & 7 deletions lib/exvcr/task/show.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ defmodule ExVCR.Task.Show do
IO.puts "\e[32mShowing #{file}\e[m"
IO.puts "\e[32m**************************************\e[m"
json = File.read!(file)
IO.puts json |> JSX.prettify! |> String.replace(~r/\\n/, "\n")
IO.puts json |> Jason.Formatter.pretty_print() |> String.replace(~r/\\n/, "\n")
display_parsed_body(json)
IO.puts "\e[32m**************************************\e[m"
else
Expand All @@ -25,17 +25,22 @@ defmodule ExVCR.Task.Show do
end

defp display_parsed_body(json) do
case extract_body(json) |> JSX.prettify do
{:ok, body_json } ->
IO.puts "\n\e[33m[Showing parsed JSON body]\e[m"
IO.puts body_json
jherdman marked this conversation as resolved.
Show resolved Hide resolved
_ -> nil
try do
output = json
|> extract_body()
|> Jason.Formatter.pretty_print()

IO.puts "\n\e[33m[Showing parsed JSON body]\e[m"
IO.puts output
rescue
_ ->
nil
end
end

defp extract_body(json) do
json
|> JSX.decode!()
|> Jason.decode!()
|> List.first()
|> Enum.into(%{})
|> get_in(["responce", "body"])
Expand Down
4 changes: 2 additions & 2 deletions mix.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ defmodule ExVCR.Mixfile do
end

def application do
[applications: [:meck, :exactor, :exjsx], mod: {ExVCR.Application, []}]
[applications: [:meck, :exactor], mod: {ExVCR.Application, []}]
end

def deps do
[
{:meck, "~> 0.8"},
{:exactor, "~> 2.2"},
{:exjsx, "~> 4.0"},
{:jason, "~> 1.0"},
{:ibrowse, "4.4.0", optional: true},
{:httpotion, "~> 3.1", optional: true},
{:httpoison, "~> 1.0", optional: true},
Expand Down
2 changes: 0 additions & 2 deletions mix.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
"ex_doc": {:hex, :ex_doc, "0.21.3", "857ec876b35a587c5d9148a2512e952e24c24345552259464b98bfbb883c7b42", [:mix], [{:earmark, "~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0db1ee8d1547ab4877c5b5dffc6604ef9454e189928d5ba8967d4a58a801f161"},
"exactor": {:hex, :exactor, "2.2.4", "5efb4ddeb2c48d9a1d7c9b465a6fffdd82300eb9618ece5d34c3334d5d7245b1", [:mix], [], "hexpm", "1222419f706e01bfa1095aec9acf6421367dcfab798a6f67c54cf784733cd6b5"},
"excoveralls": {:hex, :excoveralls, "0.14.2", "f9f5fd0004d7bbeaa28ea9606251bb643c313c3d60710bad1f5809c845b748f0", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "ca6fd358621cb4d29311b29d4732c4d47dac70e622850979bc54ed9a3e50f3e1"},
"exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "32e95820a97cffea67830e91514a2ad53b888850442d6d395f53a1ac60c82e07"},
"finch": {:hex, :finch, "0.8.0", "9fe1b7b1613f4f0f43ac4be94462a0f3eb13264e5e9a624005da5670e452a1d1", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.5", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "07650fd74da3fb51984b9f62bf4db3ef720f3c06d914d716cceeb13a2bc2542d"},
"hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"},
"http_server": {:git, "https://github.com/parroty/http_server.git", "922d10420836a51289ed04f0bb5022bf695da1ab", []},
Expand All @@ -16,7 +15,6 @@
"ibrowse": {:hex, :ibrowse, "4.4.0", "2d923325efe0d2cb09b9c6a047b2835a5eda69d8a47ed6ff8bc03628b764e991", [:rebar3], [], "hexpm", "6a8e5988872086f0506bef68311493551ac5beae7c06ba2a00d5e9f97a60f1c2"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
"jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"},
"makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"},
"makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"},
"meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"},
Expand Down
10 changes: 10 additions & 0 deletions test/recorder_hackney_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ defmodule ExVCR.RecorderHackneyTest do
ExVCR.Config.response_headers_blacklist([])
end

@tag :wip
test "hackney request with ssl options" do
use_cassette "record_hackney_with_ssl_options" do
host = @url |> URI.parse() |> Map.get(:host) |> to_charlist()
Expand All @@ -163,6 +164,15 @@ defmodule ExVCR.RecorderHackneyTest do
end
end

test "HTTPoison with ssl options" do
use_cassette "record_hackney_with_ssl_options" do
response =
HTTPoison.post!("https://example.com", {:form, []}, [], ssl: [{:versions, [:"tlsv1.2"]}])
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added this for two reasons:

  1. Jason Encoder needs to be set up for this form of options
  2. This should fail the same as L157 given that HTTPoison uses hackney under the hood

Copy link

@ruslandoga ruslandoga Jun 23, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👋 @jherdman

Sorry to bother you over an old PR. But I've been looking at it and it seems like the request here is different (according to a rexbug trace):

:hackney.request(:post, "https://example.com", [], {:form, []}, [ssl_options: [versions: [:"tlsv1.2"]]])

whereas in the failing test it's

opts = [
  customize_hostname_check: [
    match_fun: #Function<6.29392970/2 in :public_key.pkix_verify_hostname_match_fun/1>
  ],
  server_name_indication: 'localhost',
  verify: :verify_peer,
  depth: 100,
  cacerts: [
    <<48, 130, 2, 11, 48, 130, 1, 145, 160, 3, 2, 1, 2, 2, 18, 17, 210, 187,
      186, 51, 110, 212, 188, 230, 36, 104, 197, 13, 132, 29, 152, 232, 67, 48,
      10, 6, 8, 42, 134, 72, 206, 61, 4, 3, 3, 48, 70, 49, 11, 48, 9, 6, 3, 85,
      4, 6, 19, 2, 66, 69, 49, 25, 48, 23, 6, 3, 85, 4, 10, 19, 16, 71, 108,
      111, 98, 97, 108, 83, 105, 103, 110, 32, 110, 118, 45, 115, 97, 49, 28,
      48, 26, 6, 3, 85, 4, 3, 19, 19, 71, 108, 111, 98, 97, 108, 83, 105, 103,
      110, 32, 82, 111, 111, 116, 32, 69, 52, 54, 48, 30, 23, 13, 49, 57, 48,
      51, 50, 48, 48, 48, 48, 48, 48, 48, 90, 23, 13, 52, 54, 48, 51, 50, 48,
      48, 48, 48, 48, 48, 48, 90, 48, 70, 49, 11, 48, 9, 6, 3, 85, 4, 6, 19, 2,
      66, 69, 49, 25, 48, 23, 6, 3, 85, 4, 10, 19, 16, 71, 108, 111, 98, 97,
      108, 83, 105, 103, 110, 32, 110, 118, 45, 115, 97, 49, 28, 48, 26, 6, 3,
      85, 4, 3, 19, 19, 71, 108, 111, 98, 97, 108, 83, 105, 103, 110, 32, 82,
      111, 111, 116, 32, 69, 52, 54, 48, 118, 48, 16, 6, 7, 42, 134, 72, 206,
      61, 2, 1, 6, 5, 43, 129, 4, 0, 34, 3, 98, 0, 4, 156, 14, 177, 207, 183,
      232, 158, 82, 119, 117, 52, 250, 165, 70, 167, 173, 50, 25, 50, 180, 7,
      169, 39, 202, 148, 187, 12, 210, 10, 16, 199, 218, 137, 176, 151, 12, 112,
      19, 9, 1, 142, 216, 234, 71, 234, 190, 178, 128, 43, 205, 252, 40, 13,
      219, 172, 188, 164, 134, 55, 237, 112, 8, 0, 117, 234, 147, 11, 123, 46,
      82, 156, 35, 104, 35, 6, 67, 236, 146, 47, 83, 132, 219, 251, 71, 20, 7,
      232, 95, 148, 103, 93, 201, 122, 129, 60, 32, 163, 66, 48, 64, 48, 14, 6,
      3, 85, 29, 15, 1, 1, 255, 4, 4, 3, 2, 1, 134, 48, 15, 6, 3, 85, 29, 19, 1,
      1, 255, 4, 5, 48, 3, 1, 1, 255, 48, 29, 6, 3, 85, 29, 14, 4, 22, 4, 20,
      49, 10, 144, 143, 182, 198, 157, 210, 68, 75, 128, 181, 162, 230, 31, 177,
      18, 79, 27, 149, 48, 10, 6, 8, 42, 134, 72, 206, 61, 4, 3, 3, 3, 104, 0,
      48, 101, 2, 49, 0, 223, 84, 144, 237, 155, 239, 139, 148, 2, 147, 23, 130,
      153, 190, 179, 158, 44, 246, 11, 145, 140, 159, 74, 20, 177, 246, 100,
      188, 187, 104, 81, 19, 12, 3, 247, 21, 139, 132, 96, 185, 139, 255, 82,
      142, 231, 140, 188, 28, 2, 48, 60, 249, 17, 212, 140, 78, 192, 193, 97,
      194, 21, 76, 170, 171, 29, 11, 49, 95, 59, 28, 226, 0, 151, 68, 49, 230,
      254, 115, 150, 47, 218, 150, 211, 254, 8, 7, 179, 52, 137, 188, 5, 159,
      247, 30, 134, 238, 139, 112>>,
     # etc.
  ],
  partial_chain: #Function<0.89509999/1 in :hackney_ssl.partial_chain>,
  verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: 'localhost']}
]

:hackney.request(:post, "http://localhost:1234/server", [], [], opts)

and the :cacerts option is what makes it fail.

In non-strict mode jsx seems to escape invalid bytes with �:

https://github.com/talentdeficit/jsx/blob/1d3407aa9752430ec0a06111ac1b046ffafdca22/src/jsx_parser.erl#L608C7-L608C14

Here's what the encoded request for the failing test looks like. This is a data loss, so I wonder if request.options is actually used anywhere in ExVCR's logic.

Anyway, we can preprocess the data in the Jason impl for ExVCR.Request to escape all invalid bytes in options with �.

Copy link

@ruslandoga ruslandoga Jun 23, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to work:

defmodule ExVCR.Request do
  defstruct url: nil, headers: [], method: nil, body: nil, options: [], request_body: ""

  defimpl Jason.Encoder do
    def encode(request, opts) do
      request
      |> Map.update!(:headers, &Map.new/1)
      |> Map.update!(:options, fn options -> options |> clean_invalid() |> Map.new() end)
      |> Map.take([:url, :headers, :method, :body, :options, :request_body])
      |> Jason.Encode.map(opts)
    end

    defp clean_invalid([{_, _} | _] = kw) do
      kw |> Enum.map(fn {k, v} -> {k, clean_invalid(v)} end) |> Map.new()
    end

    defp clean_invalid([value | rest]) do
      [clean_invalid(value) | clean_invalid(rest)]
    end

    defp clean_invalid([] = empty), do: empty

    defp clean_invalid(map) when is_map(map) do
      map |> Map.to_list() |> clean_invalid() |> Map.new()
    end

    defp clean_invalid(tuple) when is_tuple(tuple) do
      tuple |> Tuple.to_list() |> clean_invalid() |> List.to_tuple()
    end

    defp clean_invalid(bin) when is_binary(bin) do
      clean_bin(bin)
    end

    defp clean_invalid(other), do: other

    defp clean_bin(<<b::utf8, rest::bytes>>), do: <<b::utf8>> <> clean_bin(rest)
    defp clean_bin(<<_invalid, rest::bytes>>), do: "�" <> clean_bin(rest)
    defp clean_bin(<<>> = empty), do: empty
  end
end

Resulting JSON: https://gist.github.com/ruslandoga/be54ef0cbcd69bdb66b01a5e9bd7a062

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's OK, I'll try to finish up this PR in #206


assert response.status_code == 200
end
end

for option <- [:with_body, {:with_body, true}] do
@option option

Expand Down