Skip to content
This repository has been archived by the owner on May 1, 2020. It is now read-only.

Commit

Permalink
Added collision detection hashes and ids
Browse files Browse the repository at this point in the history
  • Loading branch information
@tobybatch
tobybatch committed May 30, 2019
1 parent 0d26187 commit e9e4c6b
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 79 deletions.
77 changes: 67 additions & 10 deletions Controller/NeontribeCvsImportController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,16 @@ public function indexAction(Request $request) {
$file = $data['csvfile'];
$filename = $this->generateUniqueFileName() . '.csv';

$file->move($this->repository->getCsvUploadDir(), $filename);
$file->move($this->repository->getCsvDir(), $filename);
$token = uniqid();
$this->repository->saveToken($filename, $token);
$this->repository->clearHistory();

return $this->redirectToRoute('neontribe_cvs_import_batch', [
'filename' => $filename,
'dryrun' => $data["dryrun"] ? 1 : 0,
'checkhashes' => $data["checkhashes"] ? 1 : 0,
'checkids' => $data["checkids"] ? 1 : 0,
'token' => $this->repository->makePublicToken($token),
'offset' => 0,
'chunk' => $data['chunk']
Expand All @@ -76,12 +79,12 @@ public function indexAction(Request $request) {

/**
*
* @Route(path="/batch/{filename}/{token}/{offset}/{chunk}/{dryrun}", name="neontribe_cvs_import_batch")
* @Route(path="/batch/{filename}/{token}/{offset}/{chunk}/{dryrun}/{checkhashes}/{checkids}", name="neontribe_cvs_import_batch")
*
* @param Request $request
* @return \Symfony\Component\HttpFoundation\Response
*/
public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
public function batchAction($filename, $token, $offset, $chunk, $dryrun, $checkhashes, $checkids) {
$privToken = $this->repository->getToken($filename);

if (! $this->repository->checkPublicToken($token, $privToken)) {
Expand All @@ -90,24 +93,44 @@ public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
}

// Are we done yet?
$file = new \SplFileObject($this->repository->getCsvUploadDir() . $filename, 'r');
$file = new \SplFileObject($this->repository->getCsvDir() . $filename, 'r');
$file->seek(PHP_INT_MAX);
$linecount = $file->key() + 1;
if ($offset > $linecount) {
$this->flashSuccess('Timesheets imported.');
return $this->redirectToRoute('neontribe_cvs_import_admin');
return $this->redirectToRoute('neontribe_cvs_import_finish');
}

// Now read chunk timesheets
$counter = 0;
$errorCount = 0;
$file->rewind();
$file->seek($offset); // go to line 200
for ($i = 0; $i < $chunk and $file->valid(); $i ++, $file->next()) {
$line = $file->current();
try {
$this->importService->importLine(str_getcsv($file->current()), $dryrun);
} catch (\Throwable $error) {
// Check hash and/or id
$data = str_getcsv($line);
$id = $data[0];
$hash = md5($line);
$lineNumber = $offset + $i;
$this->flashError("Could not parse line " . $lineNumber . ": " . $file->current() . ' - ' . $error->getMessage());
if ($checkhashes && $this->repository->checkHash($hash)) {
$this->flashWarning("Duplicate line line (" . $hash . ") " . $lineNumber . ": " . $line);
$this->repository->appendHistory('Duplicate (hash): ' . $line);
continue;
}
if ($checkids && $this->repository->checkId($id)) {
$this->flashWarning("Duplicate line line (" . $id . ") " . $lineNumber . ": " . $line);
$this->repository->appendHistory('Duplicate (id): ' . $line);
continue;
}

if ($this->importService->importLine($data, $dryrun)) {
$this->repository->saveHash($id, $hash);
}
} catch (\Throwable $error) {
$this->flashError("Could not parse line " . $lineNumber . ": " . $line . ' - ' . $error->getMessage());
$this->repository->appendHistory('Error: ' . $line);
$errorCount ++;
}
$counter ++;
}
Expand All @@ -121,6 +144,8 @@ public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
$nextPhase = $this->generateUrl('neontribe_cvs_import_batch', [
'filename' => $filename,
'dryrun' => $dryrun,
'checkhashes' => $checkhashes,
'checkids' => $checkids,
'token' => $this->repository->makePublicToken($token),
'offset' => $offset + $chunk,
'chunk' => $chunk
Expand All @@ -132,7 +157,39 @@ public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
'offset' => $offset,
'count' => $counter,
'total' => $linecount,
'progress' => round($progressPercent, 2)
'progress' => round($progressPercent, 2),
'errorCount' => $errorCount
]);
}

/**
*
* @Route(path="/batch/finished", name="neontribe_cvs_import_finish")
*
* @param Request $request
* @return \Symfony\Component\HttpFoundation\Response
*/
public function batchFinished() {
$history = $this->repository->getHistory();

$errors = [];
$warnings = [];
$other = [];

foreach ($history as $line) {
if (strpos($line, 'Duplicate') === 0) {
$warnings[] = $line;
} elseif (strpos($line, 'Error') === 0) {
$errors[] = $line;
} else {
$other[] = $line;
}
}

return $this->render('@NeontribeCvsImport/finished.html.twig', [
'errors' => $errors,
'warnings' => $warnings,
'other' => $other
]);
}

Expand Down
2 changes: 1 addition & 1 deletion Form/CsvUploadForm.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public function buildForm(FormBuilderInterface $builder, array $options) {
'help' => 'If checked each line of the cvs will be hashed and checked against hashes of previously imported records. Duplicates will be ignored.'
]);

$builder->add('timesheet_ids', CheckboxType::class, [
$builder->add('checkids', CheckboxType::class, [
'label' => 'Check timesheet ids',
'required' => False,
'data' => False,
Expand Down
40 changes: 1 addition & 39 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,41 +1,3 @@
## Kimai import plugin

This creates a console command that imports timesheets into Kimai.

It reads a CSV file and creates a timesheet for each line it finds. If the Customer, Project or Activity does not exist it will create one. The deafult password for a new user is scrambled (but not fully secured). Use the password reset to set that.

The fields in the CSV are:

* id, The UID for the row. This is used to log each row imported.
* customer name, The name of the customer.
* project name, The name of the project
* activity name, The name of the activity
* start, The sate of the activity in the format YYYY-MM-DD HH:MM:SS
* duration, In minutes
* description, The text description of the timesheet entry
* user name, The username of the logger
* email, The email of the logger

Newly created projects are assigned the customer specified by the line. Activities are assigned to the Project specified by the line.

### Getting the data

To get timesheets out of an existing kimai then use a version of this SQL:

select
t.id as id,
c.name as customer,
p.name as project,
a.name as activity,
t.start_time as date,
t.duration as duration,
t.description as title,
u.username as username,
u.email as email
from
kimai2_timesheet t
inner join kimai2_users u on t.user=u.id
inner join kimai2_activities a on t.activity_id=a.id
inner join kimai2_projects p on a.project_id=p.id
inner join kimai2_customers c on p.customer_id=c.id;

See the full README bundled with the Kimai store: https://github.com/neontribe/www.kimai.org/blob/master/_store/neontribe-cvs-import-bundle.md
115 changes: 88 additions & 27 deletions Repository/NeontribeCvsImportRepository.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,78 +8,125 @@
class NeontribeCvsImportRepository {

/**
* Plugin directory supplied from the application.
*
* @var string
*/
protected $pluginDirectory = null;

/**
* Data directory supplied from the application.
*
* @var string
*/
protected $dataDirectory = null;

/**
* The app secret supplied from the application.
*
* @var string
*/
protected $appSecret;

/**
* Hashes of each imported line and the UID of the line are stored in this file.
*
* @var string
*/
protected $hashesFile;

/**
* This file lists history items to be displayed after the import id complete.
*
* @var string
*/
protected $historyFile;

/**
* The folder where this plugin will store files.
*
* @var string
*/
protected $myDataDir;

/**
* The folder where this plugin will store uploaded csv files.
*
* @var string
*/
protected $csvDir;

/**
*
* @param string $pluginDirectory
* @param string $dataDirectory
* @param string $appSecret
*/
public function __construct(string $pluginDirectory, string $dataDirectory, string $appSecret) {
$this->pluginDirectory = $pluginDirectory;
$this->dataDirectory = $dataDirectory;
$this->appSecret = $appSecret;

$this->hashesFile = $dataDirectory . '/hashes.json';
$filesystem = new Filesystem();
$myDataDir = $this->dataDirectory . '/neontribe-csv-import';
if (! $filesystem->exists($myDataDir)) {
$filesystem->mkdir($myDataDir, 0700);
}
$this->myDataDir = $myDataDir;

$csvDir = $myDataDir . '/csv/';
if (! $filesystem->exists($csvDir)) {
$filesystem->mkdir($csvDir, 0700);
}
$this->csvDir = $csvDir;

$this->hashesFile = $myDataDir . '/hashes.json';
$this->historyFile = $myDataDir . '/history.txt';
}

/**
* Get the persistent data folder for this plugin.
*
* @param array $entity
* @return bool
* @throws \Exception
* @return string
*/
public function saveHashData(array $hashes) {
if (file_exists($this->hashesFile) && ! is_writable($this->hashesFile)) {
throw new \Exception('Hashes file is not writable: ' . $this->hashesFile);
}

if (false === file_put_contents($this->hashesFile, json_encode($hashes, JSON_PRETTY_PRINT))) {
throw new \Exception('Failed saving hashes rules to file: ' . $this->hashesFile);
}

return true;
public function getDataDir() {
return $this->myDataDir;
}

/**
* Get the csv folder.
*
* @return array
* @return string
*/
public function getHashData(): array {
if (file_exists($this->hashesFile)) {
return json_decode(file_get_contents($this->hashesFile), True);
} else {
public function getCsvDir() {
return $this->csvDir;
}

public function saveHash($id, $hash) {
$data = $this->loadhashes();
$data[$id] = $hash;
$this->saveHashes($data);
}

public function checkId($id) {
$data = $this->loadhashes();
return in_array($id, array_keys($data));
}

public function checkHash($hash) {
$data = $this->loadhashes();
return in_array($hash, $data);
}

protected function loadhashes() {
if (! file_exists($this->hashesFile)) {
return [];
}
return json_decode(file_get_contents($this->hashesFile), True);
}

public function getCsvUploadDir() {
$filesystem = new Filesystem();
$csvDir = $this->dataDirectory . '/csv-import-files/';
if (! $filesystem->exists($csvDir)) {
$filesystem->mkdir($csvDir, 0700);
}
return $csvDir;
protected function saveHashes($data) {
return file_put_contents($this->hashesFile, json_encode($data, JSON_PRETTY_PRINT));
}

public function saveToken($filename, $token) {
Expand All @@ -91,7 +138,7 @@ public function getToken($filename) {
}

protected function generateTokenFileName($filename) {
$csvDir = $this->getCsvUploadDir();
$csvDir = $this->getDataDir();
$tokenfile = $csvDir . \basename($filename, 'csv') . 'txt';
return $tokenfile;
}
Expand All @@ -103,4 +150,18 @@ public function makePublicToken($token) {
public function checkPublicToken($pubtoken, $privToken) {
return $pubtoken === $this->makePublicToken($privToken);
}

public function clearHistory() {
file_put_contents($this->historyFile, '');
}

public function appendHistory($line) {
$fh = fopen($this->historyFile, 'a');
fwrite($fh, $line);
fclose($fh);
}

public function getHistory() {
return file($this->historyFile);
}
}
6 changes: 6 additions & 0 deletions Resources/views/batch.html.twig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,17 @@
{% if dryrun %}
<div class='alert alert-error fade in'>DRY RUN</div>
{% endif %}

<div style='background: grey; width: 100%;'>
<div style='background: green; height: 30px; width: {{ progress }}%;'></div>
</div>

<div>
Processed lines {{ offset }} to {{ offset + count }} of {{ total }}
({{ progress }}%)
</div>

<script>window.location = "{{ nextPhase }}"</script>
<div><a href="{{ nextPhase }}">NEXT</a></div>

{% endblock %}
Loading

0 comments on commit e9e4c6b

Please sign in to comment.