upload the project code

upload the project code

LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -178,15 +179,15 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
       file or class name and description of purpose be included on the
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,3 +200,5 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+    附加声明:经营此系统衍生分支版本请联系作者授权方可使用

README.md

@@ -0,0 +1,75 @@
+### Ragnar Fiery分布式性能跟踪系统
+
+### 项目简介
+ * Fiery跟踪系统是基于内部Ragnar分布式跟踪设计做的嵌入式精简版
+ * 此系统针对 PV 2000w以下的系统提供分布式调用跟踪及性能监控服务
+ * 提供分布式系统性能跟踪，依赖接口性能跟踪，系统故障去重合并等功能
+
+---------------------------------------
+
+### 功能组成
+ * 系统埋点库(ragnarsdk)用于植入系统埋点库产生调用日志
+ * 日志收集端(logpusher收集埋点库日志推送到服务端)
+ * 服务端（日志存储，索引，web管理界面服务）
+
+---------------------------------------
+
+### 开源协议 
+ * Apache 2.0 
+ * 附加声明:经营此系统衍生分支版本请联系作者授权方可使用
+
+---------------------------------------
+
+### 系统安装
+
+---------------------------------------
+
+#### RagnarSDK埋点库
+埋点库使用PHP制作,在ragnarsdk目录内 INSTALL.md 及 README.md 有详细介绍，简单的说就是系统埋点RagnarSDK，Nginx配置一些变量。
+
+##### RagnarSDK埋点库-环境依赖：
+ * PHP 5.5+
+ * bcmath
+
+---------------------------------------
+
+#### 服务端
+ * 此服务是主要服务，用于记录、索引、展示所有相关日志及性能信息
+
+##### 服务端-系统要求
+ * 16G 内存，闲置 8G 以上。
+ * CPU 四线程以上限制
+ * 存储空间取决于日志量大小及存储天数
+ * Java 8 Runtime http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
+
+##### 服务端-安装
+ * 安装 Java 8 Runtime 
+ * 拷贝 ragnarserver-0.5.1-SNAPSHOT.jar 包到服务器
+ * 创建系统日志目录：mkdir logs 
+ * 创建索引存储目录：mkdir index
+ * 创建日志存储目录：mkdir db
+ * 启动：nohup java -Xms3750m -Xmx3750m -XX:ReservedCodeCacheSize=240m -XX:+UseCompressedOops -jar ragnarserver-0.5.1-SNAPSHOT.jar -type server --server.port=9090 &
+ * 启动成功后浏览器访问 http://服务器IP:9090/ragnar/ 即可看到相关界面
+ * 注意：此服务重启会丢失性能统计信息，后续版本会增加统计信息落地功能
+
+---------------------------------------
+
+#### 日志收集服务
+ * 安装 Java 8 Runtime
+ * 拷贝 ragnarserver-0.5.1-SNAPSHOT.jar 包到服务器
+ * 启动：nohup java -Xms128m -Xmx750m -XX:ReservedCodeCacheSize=240m -XX:+UseCompressedOops -jar target/ragnarserver-0.5.1-SNAPSHOT.jar -type logpush -path 这里写ragnarsdk产生日志的目录地址 -host 主服务IP:主服务端口（9090） -outtime 7 &
+ * 其他方式 nohup php logcollector/tail.php ragnarsdk日志目录 0 & （仅用于测试，服务器地址等信息需要在文件内更改）
+
+---------------------------------------
+
+## 依赖第三方开源
+ * Spring Boot 1.4.5
+ * FreeMarker 1.4.5
+ * Google Gson 2.7
+ * Lucene 6.3.0
+ * Swagger 2.6.1
+ * Rocksdb Jni 4.9.0
+
+## 联系我们
+ * 微博广告 ADINF 团队 出品
+ * QQ群: 318051466

logcollector/tail.php

@@ -0,0 +1,208 @@
+<?php
+ini_set("display_error", "On");
+error_reporting(E_ALL);
+ini_set('memory_limit', '2048M');
+
+$fileInfo = array();
+$sendBizLogQueue = array();
+$sendMetaLogQueue = array();
+
+/**
+ * get the file new append content
+ * @param $file
+ * @param $modifyTime
+ */
+function fetchFileAppendContent($file, $modifyTime)
+{
+    global $fileInfo;
+    global $sendBizLogQueue;
+    global $sendMetaLogQueue;
+
+    clearstatcache();
+
+    //not found
+    if (!file_exists($file)) {
+        unset($fileInfo[$file]);
+        echo "=== File Not Found ... " . PHP_EOL;
+        return;
+    }
+
+    //get the file size
+    $fileSize = filesize($file);
+
+    //have record
+    if (isset($fileInfo[$file])) {
+        //old file
+        $offset = $fileInfo[$file]["offset"];
+        $fileInfo[$file]["lastupdate"] = $modifyTime;
+    } else {
+        //new file
+        $fileInfo[$file] = array(
+            "offset" => 0,
+            "lastupdate" => $modifyTime,
+            "size" => $fileSize,
+        );
+        $offset = 0;
+    }
+
+    //file have been move or empty
+    if ($offset > $fileSize) {
+        $offset = 0;
+    }
+
+    $processedCount = 0;
+    $fh = fopen($file, "r");
+    fseek($fh, $offset);
+
+    while ($d = fgets($fh)) {
+        $processedCount++;
+        if ($processedCount > 100) {
+            break;
+        }
+        $offset = bcadd($offset, strlen($d), 0);
+        //if it's json(must biz log)
+        if ($d[0] != "[") {
+            $sendMetaLogQueue[] = $d;
+        } else {
+            //put to the queue
+            $sendBizLogQueue[] = $d;
+        }
+    }
+
+    $fileInfo[$file]["offset"] = $offset;
+    fclose($fh);
+}
+
+/**
+ * scan the folder the new update file
+ * @param $path
+ */
+function scanRecentUpdateFile($path)
+{
+    clearstatcache();
+
+    $filelist = glob($path . "/*");
+
+    foreach ($filelist as $filepath) {
+
+        $filepath = realpath($filepath);
+
+        if (is_dir($filepath)) {
+            scanRecentUpdateFile($filepath);
+        } else {
+            $modifyTime = filemtime($filepath);
+
+            //20 second have been modify
+            if ($modifyTime > time() - 120) {
+                //echo "==== Found File:" . $filepath . PHP_EOL;
+                fetchFileAppendContent($filepath, $modifyTime);
+            }
+        }
+    }
+}
+
+function pushBizLogToServer($url)
+{
+    global $sendBizLogQueue;
+
+    $sendList = array();
+
+    $pushCount = 0;
+
+    foreach ($sendBizLogQueue as $key => $log) {
+        $sendList[] = $log;
+        unset($sendBizLogQueue[$key]);
+        $pushCount++;
+        if ($pushCount > 100) {
+            break;
+        }
+    }
+
+    if (count($sendList) > 0) {
+        $postdata = implode("", $sendList);
+
+        $ch = curl_init();
+        curl_setopt($ch, CURLOPT_URL, $url);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+        curl_setopt($ch, CURLOPT_POST, 1);
+        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Expect:'));
+        curl_setopt($ch, CURLOPT_POSTFIELDS, array("contents" => ($postdata)));
+
+        $output = curl_exec($ch);
+        curl_close($ch);
+
+        var_dump($output);
+    }
+}
+
+function pushMetaLogToServer($url)
+{
+    global $sendMetaLogQueue;
+
+    $sendList = array();
+
+    $pushCount = 0;
+
+    foreach ($sendMetaLogQueue as $key => $log) {
+        $sendList[] = $log;
+        unset($sendMetaLogQueue[$key]);
+        $pushCount++;
+        if ($pushCount > 100) {
+            break;
+        }
+    }
+
+    if (count($sendList) > 0) {
+        $postdata = implode("", $sendList);
+
+        $ch = curl_init();
+        curl_setopt($ch, CURLOPT_URL, $url);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+        curl_setopt($ch, CURLOPT_POST, 1);
+        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Expect:'));
+        curl_setopt($ch, CURLOPT_POSTFIELDS, array("contents" => urlencode($postdata)));
+
+        $output = curl_exec($ch);
+        curl_close($ch);
+
+        var_dump($output);
+    }
+}
+
+/**
+ * remove the old file on the list
+ */
+function cleanupOldFileInfoList()
+{
+    global $fileInfo;
+
+    clearstatcache();
+    foreach ($fileInfo as $filepath => $info) {
+        if (!file_exists($filepath)) {
+            unset($fileInfo[$filepath]);
+            echo "=== File Was Delete:" . $filepath . PHP_EOL;
+        }
+
+        // there is no update 60 minute
+        //if (filemtime($filepath) < time() - 3600) {
+        //unset($fileInfo[$filepath]);
+        //}
+    }
+}
+
+/**
+ * main
+ */
+
+if ($argc == 3) {
+    while (true) {
+        scanRecentUpdateFile($argv[1]);
+        pushBizLogToServer("http://127.0.0.1:8888/ragnar/log/bizlog/put");
+        pushMetaLogToServer("http://127.0.0.1:8888/ragnar/log/metalog/put");
+        cleanupOldFileInfoList();
+        usleep(200);
+    }
+} else {
+    echo "cmd: php logpusher.php path posturl";
+    exit;
+}

logpusher.sh

@@ -0,0 +1 @@
+java -Xms128m -Xmx750m -XX:ReservedCodeCacheSize=240m -XX:+UseCompressedOops -jar target/ragnarserver-0.5.1-SNAPSHOT.jar -type logpush -path ./logcollector/ -host 127.0.0.1:9090 -outtime 7