diff --git a/app/Parser.php b/app/Parser.php index b74cf7b9f7..c61cb44ecc 100644 --- a/app/Parser.php +++ b/app/Parser.php @@ -2,12 +2,48 @@ namespace App; -use Exception; - final class Parser { - public function parse(string $inputPath, string $outputPath): void - { - throw new Exception('TODO'); - } -} \ No newline at end of file + public function parse(string $inputPath, string $outputPath): void + { + gc_disable(); + $posts = []; + $chunk_size = 16 * 1024; + + $handle = fopen($inputPath, "r"); + stream_set_read_buffer($handle, 0); + $last_line = ''; + while (!feof($handle)) + { + $chunk = fread($handle, $chunk_size); + $lines = explode("\n", $chunk); + $count = count($lines); + for ($i = 0; $i < $count; $i++) + { + if ($i === $count - 1) + { + $last_line = $lines[$i]; + } + else + { + if ($i === 0) + $line = $last_line . $lines[$i]; + else + $line = $lines[$i]; + + $path = substr($line, 19, -26); + $date = substr($line, -25, 10); + $posts[$path][] = $date; + } + } + } + fclose($handle); + foreach ($posts as &$dates) + { + $dates = array_count_values($dates); + ksort($dates); + } + + file_put_contents($outputPath, json_encode($posts, JSON_PRETTY_PRINT)); + } +} diff --git a/index.php b/index.php new file mode 100644 index 0000000000..ada51d5605 --- /dev/null +++ b/index.php @@ -0,0 +1,16 @@ +parse('data/numbers.csv', 'data/data.json'); +//$p->parse('data/test-data.csv', 'data/test-data-actual.json'); +//$p->parse('/home/roey/data.csv', 'data/data.json'); +//$p->parse('data/data.csv', 'data/data.json'); +$p->parse('data/data-10M.csv', 'data/data.json'); +//$p->parse('data/data-100M.csv', 'data/data.json'); + +echo "Total time: " . (microtime(true) - $script_start) . "\n";