diff --git a/pom.xml b/pom.xml index 974957a..6a1c854 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.zavtech morpheus-yahoo - 0.9.21 + 0.10.0 jar Morpheus-Yahoo @@ -79,7 +79,7 @@ com.zavtech morpheus-viz - 0.9.16 + 0.9.21 test diff --git a/src/main/java/com/zavtech/morpheus/yahoo/YahooIndicatorsJsonParser.java b/src/main/java/com/zavtech/morpheus/yahoo/YahooIndicatorsJsonParser.java new file mode 100644 index 0000000..866a858 --- /dev/null +++ b/src/main/java/com/zavtech/morpheus/yahoo/YahooIndicatorsJsonParser.java @@ -0,0 +1,137 @@ +package com.zavtech.morpheus.yahoo; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.time.LocalDate; +import java.time.LocalDateTime; + +/** + * Parses the JSON returned by the Yahoo Finance Quote service, + * enabling reading stocks metadata and indicators such as + * open, high, low and close value (between other data). + * + * @author Manoel Campos da Silva Filho + */ +public class YahooIndicatorsJsonParser { + /** + * The name of the fields representing the indicators in the Yahoo Finance Quote service response. + */ + public enum Indicator {OPEN, HIGH, LOW, CLOSE, VOLUME, ADJCLOSE}; + + /** + * A timestamp array where each item correspond to a given date + * represented in seconds since 1970, jan 1st. + */ + private final JsonArray timestamps; + + /** + * Values for the {@link Indicator}s, except the {@link Indicator#ADJCLOSE}. + * Each indicator is a property in the JSON object. + * Each property is an array containing the double value + * for the indicator (one for each date defined in the {@link #timestamps} array). + * + * @see #adjClose + */ + private final JsonObject quotes; + + /** + * Values for the {@link Indicator#ADJCLOSE}. + * It is an array containing for the adjusted close values + * (one for each date defined in the {@link #timestamps} array). + */ + private final JsonArray adjClose; + + /** + * Instantiates the class, parsing the JSON response from a request sent to the Yahoo Finance Quote service. + * It uses a given input stream to obtain the response data. + * @param stream the input stream to read the JSON response data + */ + public YahooIndicatorsJsonParser(final InputStream stream){ + final JsonObject result = parseYahooFinanceResult(new InputStreamReader(stream)); + final JsonObject indicators = result.getAsJsonObject("indicators"); + + timestamps = result.getAsJsonArray("timestamp"); + quotes = indicators.getAsJsonArray("quote").get(0).getAsJsonObject(); + adjClose = indicators.getAsJsonArray("adjclose").get(0).getAsJsonObject().getAsJsonArray("adjclose"); + } + + /** + * Parses a JSON response got from a reader and try to return the JSON object containing + * the stocks quotes. + * + * @param reader the reader to get the JSON String from + * @return an {@link JsonObject} containing the data for the chart.result JSON field + * or an empty object if the result is empty + */ + private JsonObject parseYahooFinanceResult(final InputStreamReader reader) { + final JsonElement element = new JsonParser().parse(reader); + if(!element.isJsonObject()){ + throw new IllegalStateException("The Yahoo Finance response is not a JSON object as expected."); + } + + try { + return element + .getAsJsonObject() + .getAsJsonObject("chart") + .getAsJsonArray("result") + .get(0) + .getAsJsonObject(); + }catch(ArrayIndexOutOfBoundsException|NullPointerException e){ + return new JsonObject(); + } + } + + /** + * Gets the quote timestamp at a given position of the timestamp array + * and converts to a LocalDate value. + * @param index the desired position in the array + * @return the quote date + */ + public LocalDate getDate(final int index){ + return secondsToLocalDate(timestamps.get(index).getAsLong()); + } + + /** + * Converts a given number of seconds (timestamp) since 1970/jan/01 to LocalDate. + * This timestamp value is the date format in Yahoo Finance (at least since v8). + * @param seconds the number of seconds to convert + * @return a LocalDate representing that number of seconds + */ + public LocalDate secondsToLocalDate(final long seconds) { + return LocalDateTime.of(1970, 1, 1, 0, 0).plusSeconds(seconds).toLocalDate(); + } + + /** + * Gets the value for a specific metric of the stock in a given date, + * represented by the index of the quotes array. + * The metric values are + * @param index the desired position in the array + * @return the metric value. + */ + public double getQuote(final Indicator indicator, final int index){ + if(indicator.equals(Indicator.ADJCLOSE)) { + return getJsonDoubleValue(adjClose.get(index)); + } + + final String metricName = indicator.name().toLowerCase(); + final JsonElement element = quotes.getAsJsonArray(metricName).get(index); + return getJsonDoubleValue(element); + } + + private double getJsonDoubleValue(final JsonElement element){ + return element.isJsonNull() ? Double.NaN : element.getAsDouble(); + } + + public boolean isEmpty() { + return timestamps.size() == 0; + } + + public int rows(){ + return timestamps.size(); + } +} diff --git a/src/main/java/com/zavtech/morpheus/yahoo/YahooQuoteHistorySource.java b/src/main/java/com/zavtech/morpheus/yahoo/YahooQuoteHistorySource.java index 7266d1a..5559f6c 100644 --- a/src/main/java/com/zavtech/morpheus/yahoo/YahooQuoteHistorySource.java +++ b/src/main/java/com/zavtech/morpheus/yahoo/YahooQuoteHistorySource.java @@ -16,7 +16,6 @@ package com.zavtech.morpheus.yahoo; import java.io.IOException; -import java.io.InputStream; import java.net.URL; import java.time.Duration; import java.time.LocalDate; @@ -40,11 +39,12 @@ import com.zavtech.morpheus.range.Range; import com.zavtech.morpheus.util.Asserts; import com.zavtech.morpheus.util.IO; -import com.zavtech.morpheus.util.TextStreamReader; import com.zavtech.morpheus.util.http.HttpClient; import com.zavtech.morpheus.util.http.HttpException; import com.zavtech.morpheus.util.http.HttpHeader; +import static com.zavtech.morpheus.yahoo.YahooIndicatorsJsonParser.Indicator; + /** * A DataFrameSource implementation that loads historical quote data from Yahoo Finance using their CSV API. * @@ -58,7 +58,7 @@ public class YahooQuoteHistorySource extends DataFrameSource weekdayPredicate = date -> { if (date == null) { @@ -125,23 +125,20 @@ public DataFrame read(Consumer configurator) thro final int code = response.getStatus().getCode(); throw new HttpException(httpRequest, "Yahoo Finance responded with status code " + code, null); } else { - final InputStream stream = response.getStream(); - final TextStreamReader reader = new TextStreamReader(stream); - if (reader.hasNext()) reader.nextLine(); //Swallow the header + final YahooIndicatorsJsonParser indicators = new YahooIndicatorsJsonParser(response.getStream()); final Index rowKeys = createDateIndex(options); final Index colKeys = Index.of(fields.copy()); final DataFrame frame = DataFrame.ofDoubles(rowKeys, colKeys); final DataFrameCursor cursor = frame.cursor(); - while (reader.hasNext()) { - final String line = reader.nextLine(); - final String[] elements = line.split(","); - final LocalDate date = parseDate(elements[0]); - final double open = Double.parseDouble(elements[1]); - final double high = Double.parseDouble(elements[2]); - final double low = Double.parseDouble(elements[3]); - final double close = Double.parseDouble(elements[4]); - final double closeAdj = Double.parseDouble(elements[5]); - final double volume = Double.parseDouble(elements[6]); + for (int i = 0; i < indicators.rows(); i++) { + final LocalDate date = indicators.getDate(i); + final double open = indicators.getQuote(Indicator.OPEN, i); + final double high = indicators.getQuote(Indicator.HIGH, i); + final double low = indicators.getQuote(Indicator.LOW, i); + final double close = indicators.getQuote(Indicator.CLOSE, i); + final double closeAdj = indicators.getQuote(Indicator.ADJCLOSE, i); + + final double volume = indicators.getQuote(Indicator.VOLUME, i); final double splitRatio = Math.abs(closeAdj - close) > 0.00001d ? closeAdj / close : 1d; final double adjustment = options.dividendAdjusted ? splitRatio : 1d; if (options.paddedHolidays) { @@ -169,6 +166,7 @@ public DataFrame read(Consumer configurator) thro if (options.paddedHolidays) { frame.fill().down(2); } + calculateChanges(frame); return Optional.of(frame); } @@ -181,7 +179,6 @@ public DataFrame read(Consumer configurator) thro } } - /** * Returns the date index to initialize the row axis * @param options the options for the request @@ -233,23 +230,6 @@ private URL createURL(String symbol, LocalDate start, LocalDate end) throws Exce } } - /** - * Parses dates in the formatSqlDate YYYY-MM-DD - * @param dateString the string to parse - * @return the parsed date value - */ - private LocalDate parseDate(String dateString) { - if (dateString == null) { - return null; - } else { - final String[] elements = dateString.trim().split("-"); - final int year = Integer.parseInt(elements[0]); - final int month = Integer.parseInt(elements[1]); - final int date = Integer.parseInt(elements[2]); - return LocalDate.of(year, month, date); - } - } - /** * Returns the cookies to send with the request @@ -403,11 +383,16 @@ public Options withDividendAdjusted(boolean dividendAdjusted) { } } - - public static void main(String[] args) { final LocalDate start = LocalDate.of(2010, 1, 1); final LocalDate end = LocalDate.of(2012, 1, 1); + final String brazilianStock = "MGLU3.sa"; + System.out.printf("%n%s quotes from %s to %s%n", brazilianStock, start, end); + final YahooFinance yahoo = new YahooFinance(); + final DataFrame returns = yahoo.getDailyReturns(start, end, Array.of(brazilianStock, "BID3.sa", "ITUB4.sa")); + returns.out().print(returns.rowCount()); + System.out.println(); + final Array tickers = Array.of("AAPL", "MSFT", "ORCL", "GE", "C"); final YahooQuoteHistorySource source = new YahooQuoteHistorySource(); tickers.forEach(ticker -> { diff --git a/src/main/java/com/zavtech/morpheus/yahoo/YahooReturnSource.java b/src/main/java/com/zavtech/morpheus/yahoo/YahooReturnSource.java index 990c5a2..f4f3449 100644 --- a/src/main/java/com/zavtech/morpheus/yahoo/YahooReturnSource.java +++ b/src/main/java/com/zavtech/morpheus/yahoo/YahooReturnSource.java @@ -23,10 +23,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -76,9 +73,9 @@ public DataFrame read(Consumer configurator) throws D final Options options = initOptions(new Options(), configurator); final List>> tasks = createTasks(options); final List>> futures = executor.invokeAll(tasks); - final List> frames = futures.stream().map(Try::get).collect(Collectors.toList()); + final List> frames = futures.stream().map(this::futureGet).filter(f -> !f.isEmpty()).collect(Collectors.toList()); final DataFrame result = DataFrame.combineFirst(frames); - final DataFrame returns = result.cols().select(options.tickers).rows().sort(true).copy(); + final DataFrame returns = result.cols().select(col -> options.tickers.contains(col.key())).rows().sort(true).copy(); if (options.emaHalfLife == null) { return returns; } else { @@ -91,6 +88,13 @@ public DataFrame read(Consumer configurator) throws D } } + private DataFrame futureGet(final Future> future) { + try { + return future.get(); + } catch (Exception e) { + return DataFrame.empty(); + } + } /** * Returns the list of tasks for the request specified