Merge pull request #322 from onaio/321-mbtiles-support

Implement mbtiles support

.gitignore

@@ -11,3 +11,4 @@
 **/secrets.xml
 
 **/jacoco.exec
+**/*-journal

library/src/main/java/io/ona/kujaku/mbtiles/MBTilesHelper.java

@@ -0,0 +1,200 @@
+package io.ona.kujaku.mbtiles;
+
+import android.content.Context;
+import android.graphics.Color;
+import android.os.Environment;
+import android.support.annotation.NonNull;
+import android.text.TextUtils;
+import android.util.Pair;
+
+import com.mapbox.mapboxsdk.maps.Style;
+import com.mapbox.mapboxsdk.style.layers.FillLayer;
+import com.mapbox.mapboxsdk.style.layers.Layer;
+import com.mapbox.mapboxsdk.style.layers.LineLayer;
+import com.mapbox.mapboxsdk.style.layers.RasterLayer;
+import com.mapbox.mapboxsdk.style.sources.RasterSource;
+import com.mapbox.mapboxsdk.style.sources.Source;
+import com.mapbox.mapboxsdk.style.sources.TileSet;
+import com.mapbox.mapboxsdk.style.sources.VectorSource;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import io.ona.kujaku.plugin.switcher.BaseLayerSwitcherPlugin;
+import io.ona.kujaku.plugin.switcher.layer.MBTilesLayer;
+import timber.log.Timber;
+
+import static com.mapbox.mapboxsdk.style.layers.PropertyFactory.fillColor;
+import static com.mapbox.mapboxsdk.style.layers.PropertyFactory.fillOpacity;
+import static com.mapbox.mapboxsdk.style.layers.PropertyFactory.lineColor;
+import static com.mapbox.mapboxsdk.style.layers.PropertyFactory.lineOpacity;
+import static com.mapbox.mapboxsdk.style.layers.PropertyFactory.lineWidth;
+import static com.mapbox.mapboxsdk.style.layers.PropertyFactory.rasterOpacity;
+
+/**
+ * Created by samuelgithengi on 9/29/19.
+ */
+public class MBTilesHelper {
+
+    public static final String MB_TILES_EXTENSION = ".mbtiles";
+
+    public static final String MB_TILES_DIRECTORY = "/mbtiles";
+
+    protected TileHttpServer tileServer;
+
+    private File mbtilesDir = new File(Environment.getExternalStorageDirectory().getPath() + MB_TILES_DIRECTORY);
+
+    private void init(List<File> offlineFiles) {
+        if (offlineFiles == null || offlineFiles.isEmpty()) {
+            return;
+        } else if (tileServer == null || !tileServer.isStarted()) {
+            initializeMbTilesServer();
+        }
+    }
+
+    public void initializeMbTileslayers(@NonNull Style style, List<File> offlineFiles) {
+        init(offlineFiles);
+        for (File file : offlineFiles) {
+            String name = file.getName();
+            if (name.endsWith(MB_TILES_EXTENSION)) {
+                String id = name.substring(0, name.length() - MB_TILES_EXTENSION.length());
+                addMbtiles(style, id, file);
+            }
+        }
+    }
+
+    public Pair<Set<Source>, Set<Layer>> initializeMbTileslayers(File offlineFile) {
+        init(Collections.singletonList(offlineFile));
+        Set<Source> sources = new HashSet<>();
+        Set<Layer> layers = new HashSet<>();
+        String name = offlineFile.getName();
+        if (name.endsWith(MB_TILES_EXTENSION)) {
+            String id = name.substring(0, name.length() - MB_TILES_EXTENSION.length());
+            Pair<Source, List<Layer>> sourceAndLayers = addMbtiles(id, offlineFile);
+            if (sourceAndLayers != null) {
+                sources.add(sourceAndLayers.first);
+                layers.addAll(sourceAndLayers.second);
+            }
+            return new Pair<>(sources, layers);
+        }
+        return null;
+    }
+
+    public void setMBTileLayers(Context context, BaseLayerSwitcherPlugin baseLayerSwitcherPlugin) {
+        if (mbtilesDir.exists() && mbtilesDir.exists() && mbtilesDir.listFiles() != null) {
+            for (File mbTile : mbtilesDir.listFiles()) {
+                MBTilesLayer mbTilesLayer = new MBTilesLayer(context, mbTile, this);
+                if (!TextUtils.isEmpty(mbTilesLayer.getDisplayName())) {
+                    baseLayerSwitcherPlugin.addBaseLayer(mbTilesLayer, false);
+                }
+            }
+        }
+
+    }
+
+    private void initializeMbTilesServer() {
+        // Mapbox SDK only knows how to fetch tiles via HTTP.  If we want it to
+        // display tiles from a local file, we have to serve them locally over HTTP.
+        try {
+            tileServer = new TileHttpServer();
+            tileServer.start();
+        } catch (IOException e) {
+            Timber.e(e, "Could not start the TileHttpServer");
+        }
+
+    }
+
+    public void onDestroy() {
+        if (tileServer != null) {
+            tileServer.destroy();
+        }
+    }
+
+    private void addMbtiles(Style style, String id, File file) {
+        Pair<Source, List<Layer>> sourceAndLayers = addMbtiles(id, file);
+        if (sourceAndLayers != null) {
+            style.addSource(sourceAndLayers.first);
+            for (Layer layer : sourceAndLayers.second)
+                style.addLayer(layer);
+        }
+    }
+
+    private Pair<Source, List<Layer>> addMbtiles(String id, File file) {
+        MbtilesFile mbtiles;
+        List<Layer> mapLayers = new ArrayList<>();
+        Source source = null;
+        try {
+            mbtiles = new MbtilesFile(file);
+        } catch (MbtilesFile.UnsupportedFormatException e) {
+            Timber.w(e, "The mbtiles format is not known ");
+            return null;
+        }
+
+        TileSet tileSet = createTileSet(mbtiles, tileServer.getUrlTemplate(id));
+        tileServer.addSource(id, mbtiles);
+
+        if (mbtiles.getType() == MbtilesFile.Type.VECTOR) {
+            source = new VectorSource(id, tileSet);
+            List<MbtilesFile.VectorLayer> layers = mbtiles.getVectorLayers();
+            for (MbtilesFile.VectorLayer layer : layers) {
+                // Pick a colour that's a function of the filename and layer name.
+                int hue = (((id + "." + layer.name).hashCode()) & 0x7fffffff) % 360;
+                mapLayers.add(new FillLayer(id + "/" + layer.name + ".fill", id).withProperties(
+                        fillColor(Color.HSVToColor(new float[]{hue, 0.3f, 1})),
+                        fillOpacity(0.1f)
+                ).withSourceLayer(layer.name));
+                mapLayers.add(new LineLayer(id + "/" + layer.name + ".line", id).withProperties(
+                        lineColor(Color.HSVToColor(new float[]{hue, 0.7f, 1})),
+                        lineWidth(1f),
+                        lineOpacity(0.7f)
+                ).withSourceLayer(layer.name));
+            }
+        }
+        if (mbtiles.getType() == MbtilesFile.Type.RASTER) {
+            source = new RasterSource(id, tileSet);
+            mapLayers.add(new RasterLayer(id + ".raster", id).withProperties(
+                    rasterOpacity(0.5f)
+            ));
+        }
+        Timber.i("Added %s as a %s layer at /%s", file, mbtiles.getType(), id);
+        return new Pair<>(source, mapLayers);
+    }
+
+    private TileSet createTileSet(MbtilesFile mbtiles, String urlTemplate) {
+        TileSet tileSet = new TileSet("2.2.0", urlTemplate);
+
+        // Configure the TileSet using the metadata in the .mbtiles file.
+        tileSet.setName(mbtiles.getMetadata("name"));
+        try {
+            tileSet.setMinZoom(Integer.parseInt(mbtiles.getMetadata("minzoom")));
+            tileSet.setMaxZoom(Integer.parseInt(mbtiles.getMetadata("maxzoom")));
+        } catch (NumberFormatException e) { /* ignore */ }
+
+        String[] parts = mbtiles.getMetadata("center").split(",");
+        if (parts.length == 3) {  // latitude, longitude, zoom
+            try {
+                tileSet.setCenter(
+                        Float.parseFloat(parts[0]), Float.parseFloat(parts[1]),
+                        (float) Integer.parseInt(parts[2])
+                );
+            } catch (NumberFormatException e) { /* ignore */ }
+        }
+
+        parts = mbtiles.getMetadata("bounds").split(",");
+        if (parts.length == 4) {  // left, bottom, right, top
+            try {
+                tileSet.setBounds(
+                        Float.parseFloat(parts[0]), Float.parseFloat(parts[1]),
+                        Float.parseFloat(parts[2]), Float.parseFloat(parts[3])
+                );
+            } catch (NumberFormatException e) { /* ignore */ }
+        }
+
+        return tileSet;
+    }
+}

library/src/main/java/io/ona/kujaku/mbtiles/MbtilesFile.java

@@ -0,0 +1,153 @@
+package io.ona.kujaku.mbtiles;
+
+import android.database.Cursor;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteException;
+import android.support.annotation.NonNull;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import timber.log.Timber;
+
+/**
+ * This class provides access to the metadata and tiles in a .mbtiles file.
+ * An .mbtiles file is a SQLite database file containing specific tables and
+ * columns, including tiles that may contain raster images or vector geometry.
+ * See https://github.com/mapbox/mbtiles-spec for the detailed specification.
+ */
+public class MbtilesFile implements Closeable, TileHttpServer.TileSource {
+    public enum Type { RASTER, VECTOR }
+
+    protected File file;
+    protected SQLiteDatabase db;
+    protected String format;
+    protected Type type;
+    protected String contentType = "application/octet-stream";
+    protected String contentEncoding = "identity";
+
+    public MbtilesFile(File file) throws SQLiteException, UnsupportedFormatException {
+        this.file = file;
+        db = SQLiteDatabase.openOrCreateDatabase(file, null);
+
+        // The "format" code indicates whether the binary tiles are raster image
+        // files (JPEG, PNG) or protobuf-encoded vector geometry (PBF, MVT).
+        format = getMetadata("format").toLowerCase(Locale.US);
+
+        //Added this as a default some mbtiles lacks format in the metadata
+        if(format.isEmpty())
+            format="png";
+
+        if (format.equals("pbf") || format.equals("mvt")) {
+            contentType = "application/protobuf";
+            contentEncoding = "gzip";
+            type = Type.VECTOR;
+        } else if (format.equals("jpg") || format.equals("jpeg")) {
+            contentType = "image/jpeg";
+            type = Type.RASTER;
+        } else if (format.equals("png")) {
+            contentType = "image/png";
+            type = Type.RASTER;
+        } else {
+            db.close();
+            throw new UnsupportedFormatException(file, format);
+        }
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public void close() {
+        db.close();
+    }
+
+    /** Queries the "metadata" table, which has just "name" and "value" columns. */
+    public @NonNull String getMetadata(String key) {
+        try (Cursor results = db.query("metadata", new String[] {"value"},
+                "name = ?", new String[] {key}, null, null, null, null)) {
+            return results.moveToFirst() ? results.getString(0) : "";
+        }
+    }
+
+    /** Puts together the HTTP response for a given tile. */
+    public TileHttpServer.Response getTile(int zoom, int x, int y) {
+        // TMS coordinates are used in .mbtiles files, so Y needs to be flipped.
+        byte[] data = getTileBlob(zoom, x, (1 << zoom) - 1 - y);
+        return data == null ? null :
+                new TileHttpServer.Response(data, contentType, contentEncoding);
+    }
+
+    /** Fetches a tile out of the .mbtiles SQLite database. */
+    // PMD complains about returning null for an array return type, but we
+    // really do want to return null when there is no tile available.
+    @SuppressWarnings("PMD.ReturnEmptyArrayRatherThanNull")
+    public byte[] getTileBlob(int zoom, int column, int row) {
+        // We have to use String.format because the templating mechanism in
+        // SQLiteDatabase.query is written for a strange alternate universe
+        // in which numbers don't exist -- it only supports strings!
+        String selection = String.format(
+                Locale.US,
+                "zoom_level = %d and tile_column = %d and tile_row = %d",
+                zoom, column, row
+        );
+
+        try (Cursor results = db.query("tiles", new String[] {"tile_data"},
+                selection, null, null, null, null)) {
+            if (results.moveToFirst()) {
+                try {
+                    return results.getBlob(0);
+                } catch (IllegalStateException e) {
+                    Timber.w(e, "Could not select tile data at zoom %d, column %d, row %d", zoom, column, row);
+                    // In Android, the SQLite cursor can handle at most 2 MB in one row;
+                    // exceeding 2 MB in an .mbtiles file is rare, but it can happen.
+                    // When an attempt to fetch a large row fails, the database ends up
+                    // in an unusable state, so we need to close it and reopen it.
+                    // See https://stackoverflow.com/questions/20094421/cursor-window-window-is-full
+                    db.close();
+                    db = SQLiteDatabase.openOrCreateDatabase(file, null);
+                }
+            }
+        }
+        return null;
+    }
+
+    /** Returns information about the vector layers available in the tiles. */
+    public List<VectorLayer> getVectorLayers() {
+        List<VectorLayer> layers = new ArrayList<>();
+        JSONArray jsonLayers;
+        try {
+            JSONObject json = new JSONObject(getMetadata("json"));
+            jsonLayers = json.getJSONArray("vector_layers");
+            for (int i = 0; i < jsonLayers.length(); i++) {
+                layers.add(new VectorLayer(jsonLayers.getJSONObject(i)));
+            }
+        } catch (JSONException e) {
+            Timber.e(e);
+        }
+        return layers;
+    }
+
+    /** Vector layer metadata.  See https://github.com/mapbox/mbtiles-spec for details. */
+    public static class VectorLayer {
+        public final String name;
+
+        public VectorLayer(JSONObject json) {
+            name = json.optString("id", "");
+        }
+    }
+
+    public class UnsupportedFormatException extends IOException {
+        public UnsupportedFormatException(File file, String format) {
+            super(String.format("Unrecognized .mbtiles format \"%s\" in %s", format, file));
+        }
+    }
+}