From 5fed6f2de073fb5e7ec6805501b36f318283ccfd Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 30 Oct 2018 17:39:30 +0100 Subject: [PATCH] Initial commit --- .appveyor.yml | 8 ++ .ci/common | 1 + .clang-format | 4 + .gitignore | 37 ++++++ .gitmodules | 6 + .travis.yml | 14 +++ AUTHORS | 1 + CMakeLists.txt | 65 +++++++++++ LICENSE | 26 +++++ README.md | 5 + cmake/Modules | 1 + mkmmdb-client.cpp | 47 ++++++++ mkmmdb.cpp | 2 + mkmmdb.h | 286 ++++++++++++++++++++++++++++++++++++++++++++++ 14 files changed, 503 insertions(+) create mode 100644 .appveyor.yml create mode 160000 .ci/common create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .travis.yml create mode 100644 AUTHORS create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 README.md create mode 160000 cmake/Modules create mode 100644 mkmmdb-client.cpp create mode 100644 mkmmdb.cpp create mode 100644 mkmmdb.h diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..bdac45f --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,8 @@ +image: Visual Studio 2017 +environment: + matrix: + - CMAKE_GENERATOR: "Visual Studio 15 2017 Win64" + - CMAKE_GENERATOR: "Visual Studio 15 2017" +build_script: + - cmd: git submodule update --init --recursive + - cmd: .\.ci\common\script\appveyor.bat "%CMAKE_GENERATOR%" diff --git a/.ci/common b/.ci/common new file mode 160000 index 0000000..30a13d9 --- /dev/null +++ b/.ci/common @@ -0,0 +1 @@ +Subproject commit 30a13d983baa92775306f99708762652e51dc500 diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..37e1393 --- /dev/null +++ b/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: Google +ColumnLimit: 0 +AllowShortIfStatementsOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0593f08 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +/ALL_BUILD.vcxproj +/ALL_BUILD.vcxproj.filters +/CMakeCache.txt +/CMakeFiles/ +/CTestTestfile.cmake +/GeoLite2-ASN_20*/ +/GeoLite2-Country_20*/ +/INSTALL.vcxproj +/INSTALL.vcxproj.filters +/MK_DIST/ +/Makefile +/RUN_TESTS.vcxproj +/RUN_TESTS.vcxproj.filters +/Release/ +/Testing +/ZERO_CHECK.vcxproj +/ZERO_CHECK.vcxproj.filters +/argh.h +/asn.mmdb +/ca-bundle.pem +/catch.hpp +/cmake_install.cmake +/country.mmdb +/geolite2-asn.tar.gz +/geolite2-country.tar.gz +/install_manifest.txt +/libmkmmdb.a +/mkcurl.h +/mkdata.h +/mkmmdb-client +/mkmmdb-client.dir/ +/mkmmdb-client.vcxproj +/mkmmdb-client.vcxproj.filters +/mkmmdb.sln +/windows-curl-*.tar.gz +/windows-libmaxminddb-*.tar.gz +/x64/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fed7891 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "cmake/Modules"] + path = cmake/Modules + url = https://github.com/measurement-kit/cmake-modules +[submodule ".ci/common"] + path = .ci/common + url = https://github.com/measurement-kit/ci-common diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c943e8a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: c++ +services: + - docker +sudo: required +matrix: + include: + - env: BUILD_TYPE="asan" + - env: BUILD_TYPE="clang" + - env: BUILD_TYPE="coverage" + - env: BUILD_TYPE="lsan" + - env: BUILD_TYPE="ubsan" + - env: BUILD_TYPE="vanilla" +script: + - ./.ci/common/script/travis $BUILD_TYPE diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..59989f4 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Simone Basso diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9518e26 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,65 @@ +cmake_minimum_required(VERSION 3.1.0) +project(mkmmdb LANGUAGES CXX) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") +include(MkUtils) + +# Download dependencies +# --------------------- + +MkDownloadMMDBDatabases() + +if("${MSVC}") + MkDownloadMeasurementKitPrebuiltWindowsLibmaxminddb() + list(APPEND CMAKE_REQUIRED_INCLUDES "${MK_WINDOWS_LIBMAXMINDDB_INCLUDE_PATH}") + list(APPEND CMAKE_LIBRARY_PATH "${MK_WINDOWS_LIBMAXMINDDB_LIBRARY_PATH}") +endif() + +# Checks +# ------ + +include(CheckIncludeFiles) + +check_include_files(maxminddb.h MKMMDB_HAVE_MAXMINDDB_H) +find_library(LIBMAXMINDDB_LIBRARY maxminddb) +if (("${MKMMDB_HAVE_MAXMINDDB_H}" STREQUAL "") OR + ("${LIBMAXMINDDB_LIBRARY}" STREQUAL "LIBMAXMINDDB_LIBRARY-NOTFOUND")) + message(FATAL_ERROR "Cannot find libmaxminddb") +endif() +list(APPEND MKMMDB_LIBS "${LIBMAXMINDDB_LIBRARY}") + +find_package(CURL REQUIRED) + +# Compiler flags +# -------------- + +MkSetCompilerFlags() + +# Library and binary +# ------------------ + +set(MKMMDB_INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} + ${CURL_INCLUDE_DIRS} ${CMAKE_REQUIRED_INCLUDES}) + +add_library(mkmmdb mkmmdb.cpp) +add_executable(mkmmdb-client mkmmdb-client.cpp) +target_include_directories(mkmmdb-client PUBLIC ${MKMMDB_INCLUDES}) +if("${WIN32}" OR "${MINGW}") + list(APPEND MKMMDB_LIBS "ws2_32") + if ("${MINGW}") + list(APPEND MKMMDB_LIBS -static-libgcc -static-libstdc++) + endif() +endif() +list(APPEND MKMMDB_LIBS Threads::Threads) +target_link_libraries(mkmmdb-client "${MKMMDB_LIBS}" mkmmdb) + +# Testing +# ------- + +set(BUILD_TESTING "ON" CACHE BOOL "Whether to build tests") +if(${BUILD_TESTING}) + enable_testing() + add_test(NAME basic_test COMMAND mkmmdb-client 8.8.8.8) + add_test(NAME name_not_found_test COMMAND mkmmdb-client 127.0.0.1) + add_test(NAME invalid_ip_test COMMAND mkmmdb-client 8.8.8.x) +endif() diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4cb6d13 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +Copyright 2018 Open Observatory of Network Interference (OONI), The Tor Project + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4de100a --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Measurement Kit MMDB wrappers + +[![GitHub license](https://img.shields.io/github/license/measurement-kit/mkmmdb.svg)](https://raw.githubusercontent.com/measurement-kit/mkmmdb/master/LICENSE) [![Github Releases](https://img.shields.io/github/release/measurement-kit/mkmmdb.svg)](https://github.com/measurement-kit/mkmmdb/releases) [![Build Status](https://img.shields.io/travis/measurement-kit/mkmmdb/master.svg?label=travis)](https://travis-ci.org/measurement-kit/mkmmdb) [![codecov](https://codecov.io/gh/measurement-kit/mkmmdb/branch/master/graph/badge.svg)](https://codecov.io/gh/measurement-kit/mkmmdb) [![Build status](https://img.shields.io/appveyor/ci/bassosimone/mkmmdb/master.svg?label=appveyor)](https://ci.appveyor.com/project/bassosimone/mkmmdb/branch/master) + +Experimental library to use libmaxminddb from Measurement Kit. diff --git a/cmake/Modules b/cmake/Modules new file mode 160000 index 0000000..7041216 --- /dev/null +++ b/cmake/Modules @@ -0,0 +1 @@ +Subproject commit 70412163c491e575bb7d33b2b729d7cb15cd016a diff --git a/mkmmdb-client.cpp b/mkmmdb-client.cpp new file mode 100644 index 0000000..8679bdd --- /dev/null +++ b/mkmmdb-client.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +#include "mkmmdb.h" + +int main(int argc, char **argv) { + if (argc != 2) { + std::clog << "Usage: mkmmdb-client " << std::endl; + exit(EXIT_FAILURE); + } + std::string ip = argv[1]; + { + mkmmdb_uptr db{mkmmdb_open("country.mmdb")}; + if (!mkmmdb_good(db.get())) { + std::clog << "Cannot open country.mmdb" << std::endl; + // FALLTHROUGH + } + std::string s = mkmmdb_lookup_cc(db.get(), ip.c_str()); + std::clog << "=== BEGIN LOGS === " << std::endl; + std::clog << mkmmdb_get_last_lookup_logs(db.get()); + std::clog << "=== END LOGS === " << std::endl; + std::clog << "CC: " << s << std::endl; + } + { + mkmmdb_uptr db{mkmmdb_open("asn.mmdb")}; + if (!mkmmdb_good(db.get())) { + std::clog << "Cannot open asn.mmdb" << std::endl; + // FALLTHROUGH + } + { + std::string s = mkmmdb_lookup_org(db.get(), ip.c_str()); + std::clog << "=== BEGIN LOGS === " << std::endl; + std::clog << mkmmdb_get_last_lookup_logs(db.get()); + std::clog << "=== END LOGS === " << std::endl; + std::clog << "ORG: " << s << std::endl; + } + { + int64_t v = mkmmdb_lookup_asn(db.get(), ip.c_str()); + std::clog << "=== BEGIN LOGS === " << std::endl; + std::clog << mkmmdb_get_last_lookup_logs(db.get()); + std::clog << "=== END LOGS === " << std::endl; + std::clog << "ASN: " << v << std::endl; + } + } +} diff --git a/mkmmdb.cpp b/mkmmdb.cpp new file mode 100644 index 0000000..5e0533f --- /dev/null +++ b/mkmmdb.cpp @@ -0,0 +1,2 @@ +#define MKMMDB_INLINE_IMPL +#include "mkmmdb.h" diff --git a/mkmmdb.h b/mkmmdb.h new file mode 100644 index 0000000..b9f0262 --- /dev/null +++ b/mkmmdb.h @@ -0,0 +1,286 @@ +// Part of Measurement Kit . +// Measurement Kit is free software under the BSD license. See AUTHORS +// and LICENSE for more information on the copying conditions. +#ifndef MEASUREMENT_KIT_MKMMDB_H +#define MEASUREMENT_KIT_MKMMDB_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// mkmmdb_t is a MMDB database. +typedef struct mkmmdb mkmmdb_t; + +/// mkmmdb_open opens a database pointing to @p path. It always returns a +/// valid pointer. Call mkmmdb_good to understand whether the database was +/// successfully openned or not. This function will call std::abort when +/// the @p path argument is a null pointer. +mkmmdb_t *mkmmdb_open(const char *path); + +/// mkmmdb_good returns true if the database has been successfully open +/// and false otherwise. This function aborts if @p mmdb is null. +int64_t mkmmdb_good(mkmmdb_t *mmdb); + +/// mkmmdb_lookup_cc returns the country code of @p ip using the +/// @p mmdb database, or the empty string on failure. The returned string +/// will be valid until @p mmdb is valid _and_ you don't call other +/// functions using the same @p mmdb instance. This function calls +/// std::abort if passed null parameters. +const char *mkmmdb_lookup_cc(mkmmdb_t *mmdb, const char *ip); + +/// mkmmdb_lookup_asn is like mkmmdb_lookup_cc but returns +/// the ASN on success and zero on failure. Also this function +/// calls std::abort if passed null parameters. +int64_t mkmmdb_lookup_asn(mkmmdb_t *mmdb, const char *ip); + +/// mkmmdb_lookup_org is like mkmmdb_lookup_cc but returns +/// the organization bound to @p ip on success, the empty string on +/// failure. The returned string will be valid until @p mmdb is +/// valid _and_ you don't call other lookup APIs using the +/// same @p mmdb instance. This function calls std::abort if +/// passed a null @p mmdb or a null @p ip. +const char *mkmmdb_lookup_org(mkmmdb_t *mmdb, const char *ip); + +/// mkmmdb_get_last_lookup_logs returns the logs of the last lookup +/// or an empty string. Calls std::abort if @p mmdb is null. +const char *mkmmdb_get_last_lookup_logs(mkmmdb_t *mmdb); + +/// mkmmdb_close closes @p mmdb. Note that @p mmdb MAY be null. +void mkmmdb_close(mkmmdb_t *mmdb); + +#ifdef __cplusplus +} // extern "C" + +#include +#include + +/// mkmmdb_deleter is a deleter for mkmmdb_t. +struct mkmmdb_deleter { + void operator()(mkmmdb_t *p) { mkmmdb_close(p); } +}; + +/// mkmmdb_uptr is a unique pointer to mkmmdb_t. +using mkmmdb_uptr = std::unique_ptr; + +// By default the implementation is not included. You can force it being +// included by providing the following definition to the compiler. +// +// If you're just into understanding the API, you can stop reading here. +#ifdef MKMMDB_INLINE_IMPL + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include +#include + +#include + +// mkMMDB_s_deleter is a deleter for a MMDB_s pointer. +struct mkMMDB_s_deleter { + void operator()(MMDB_s *p) { + MMDB_close(p); + delete p; + } +}; + +// mkMMDB_s_uptr is a unique pointer to MMDB_s. +using mkMMDB_s_uptr = std::unique_ptr; + +// mkmmdb is a MMDB database. +struct mkmmdb { + // mmdbs is a unique pointer to the real database instance. + mkMMDB_s_uptr mmdbs; + // last_lookup_result is the place where we save the latest lookup result. + std::string last_lookup_result; + // last_lookup_logs contains the logs of the last lookup. + std::string last_lookup_logs; +}; + +#ifndef MKMMDB_MMDB_OPEN +// MKMMDB_MMDB_OPEN allows to mock MMDB_open +#define MKMMDB_MMDB_OPEN MMDB_open +#endif + +#ifndef MKMMDB_ABORT +// MKMMDB_ABORT allows to mock std::abort +#define MKMMDB_ABORT std::abort +#endif + +mkmmdb_t *mkmmdb_open(const char *path) { + if (path == nullptr) { + MKMMDB_ABORT(); + } + mkmmdb_uptr mmdb{new mkmmdb_t}; + mmdb->mmdbs.reset(new MMDB_s); + if (MKMMDB_MMDB_OPEN(path, MMDB_MODE_MMAP, mmdb->mmdbs.get()) != 0) { + mmdb->mmdbs = nullptr; + } + return mmdb.release(); +} + +int64_t mkmmdb_good(mkmmdb_t *mmdb) { + if (mmdb == nullptr) { + MKMMDB_ABORT(); + } + return mmdb->mmdbs != nullptr; +} + +#ifndef MKMMDB_MMDB_GET_VALUE +// MKMMDB_MMDB_GET_VALUE allows to mock MMDB_get_value +#define MKMMDB_MMDB_GET_VALUE MMDB_get_value +#endif + +#ifndef MKMMDB_MMDB_LOOKUP_STRING +// MKMMDB_MMDB_LOOKUP_STRING allows to mock MMDB_lookup_string +#define MKMMDB_MMDB_LOOKUP_STRING MMDB_lookup_string +#endif + +static void mkmmdb_lookup_mmdb( + mkmmdb_t *mmdb, const std::string &ip, + std::function fun) { + if (mmdb == nullptr || fun == nullptr) { + MKMMDB_ABORT(); + } + { + mmdb->last_lookup_logs += "Start lookup for: "; + mmdb->last_lookup_logs += ip; + mmdb->last_lookup_logs += "\n"; + } + if (mmdb->mmdbs == nullptr) { + mmdb->last_lookup_logs += "The database is not open.\n"; + return; + } + auto gai_error = 0; + auto mmdb_error = 0; + auto record = MKMMDB_MMDB_LOOKUP_STRING(mmdb->mmdbs.get(), ip.c_str(), + &gai_error, &mmdb_error); + if (gai_error != 0) { + mmdb->last_lookup_logs += "mmdb_lookup_string: "; + mmdb->last_lookup_logs += gai_strerror(gai_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_lookup_string: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!record.found_entry) { + mmdb->last_lookup_logs += "mmdb_lookup_string: entry not found.\n"; + return; + } + fun(&record.entry); +} + +const char *mkmmdb_lookup_cc(mkmmdb_t *mmdb, const char *ip) { + if (mmdb == nullptr || ip == nullptr) { + MKMMDB_ABORT(); + } + mmdb->last_lookup_result = ""; + mmdb->last_lookup_logs = ""; + mkmmdb_lookup_mmdb( + mmdb, ip, [&](MMDB_entry_s *entry) { + MMDB_entry_data_s data{}; + auto mmdb_error = MKMMDB_MMDB_GET_VALUE( + entry, &data, "registered_country", "iso_code", nullptr); + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_get_value: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!data.has_data) { + mmdb->last_lookup_logs += "mmdb_get_value: no data for entry.\n"; + return; + } + if (data.type != MMDB_DATA_TYPE_UTF8_STRING) { + mmdb->last_lookup_logs += "mmdb_get_value: unexpected data type.\n"; + return; + } + mmdb->last_lookup_result = std::string{ + data.utf8_string, data.data_size}; + }); + return mmdb->last_lookup_result.c_str(); +} + +int64_t mkmmdb_lookup_asn(mkmmdb_t *mmdb, const char *ip) { + if (mmdb == nullptr || ip == nullptr) { + MKMMDB_ABORT(); + } + mmdb->last_lookup_logs = ""; + int64_t rv = {}; + mkmmdb_lookup_mmdb( + mmdb, ip, [&](MMDB_entry_s *entry) { + MMDB_entry_data_s data{}; + auto mmdb_error = MKMMDB_MMDB_GET_VALUE( + entry, &data, "autonomous_system_number", nullptr); + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_get_value: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!data.has_data) { + mmdb->last_lookup_logs += "mmdb_get_value: no data for entry.\n"; + return; + } + if (data.type != MMDB_DATA_TYPE_UINT32) { + mmdb->last_lookup_logs += "mmdb_get_value: unexpected data type.\n"; + return; + } + rv = data.uint32; + }); + return rv; +} + +const char *mkmmdb_lookup_org(mkmmdb_t *mmdb, const char *ip) { + if (mmdb == nullptr || ip == nullptr) { + MKMMDB_ABORT(); + } + mmdb->last_lookup_logs = ""; + mmdb->last_lookup_result = ""; + mkmmdb_lookup_mmdb( + mmdb, ip, [&](MMDB_entry_s *entry) { + MMDB_entry_data_s data{}; + auto mmdb_error = MKMMDB_MMDB_GET_VALUE( + entry, &data, "autonomous_system_organization", nullptr); + if (mmdb_error != 0) { + mmdb->last_lookup_logs += "mmdb_get_value: "; + mmdb->last_lookup_logs += MMDB_strerror(mmdb_error); + mmdb->last_lookup_logs += "\n"; + return; + } + if (!data.has_data) { + mmdb->last_lookup_logs += "mmdb_get_value: no data for entry.\n"; + return; + } + if (data.type != MMDB_DATA_TYPE_UTF8_STRING) { + mmdb->last_lookup_logs += "mmdb_get_value: unexpected data type.\n"; + return; + } + mmdb->last_lookup_result = std::string{ + data.utf8_string, data.data_size}; + }); + return mmdb->last_lookup_result.c_str(); +} + +const char *mkmmdb_get_last_lookup_logs(mkmmdb_t *mmdb) { + if (mmdb == nullptr) { + MKMMDB_ABORT(); + } + return mmdb->last_lookup_logs.c_str(); +} + +void mkmmdb_close(mkmmdb_t *mmdb) { delete mmdb; } + +#endif // MKMMDB_INLINE_IMPL +#endif // __cplusplus +#endif // MEASUREMENT_KIT_MKMMDB_H