diff --git a/Controller/NeontribeCvsImportController.php b/Controller/NeontribeCvsImportController.php
index ed603ec..7f37aad 100644
--- a/Controller/NeontribeCvsImportController.php
+++ b/Controller/NeontribeCvsImportController.php
@@ -56,13 +56,16 @@ public function indexAction(Request $request) {
$file = $data['csvfile'];
$filename = $this->generateUniqueFileName() . '.csv';
- $file->move($this->repository->getCsvUploadDir(), $filename);
+ $file->move($this->repository->getCsvDir(), $filename);
$token = uniqid();
$this->repository->saveToken($filename, $token);
+ $this->repository->clearHistory();
return $this->redirectToRoute('neontribe_cvs_import_batch', [
'filename' => $filename,
'dryrun' => $data["dryrun"] ? 1 : 0,
+ 'checkhashes' => $data["checkhashes"] ? 1 : 0,
+ 'checkids' => $data["checkids"] ? 1 : 0,
'token' => $this->repository->makePublicToken($token),
'offset' => 0,
'chunk' => $data['chunk']
@@ -76,12 +79,12 @@ public function indexAction(Request $request) {
/**
*
- * @Route(path="/batch/{filename}/{token}/{offset}/{chunk}/{dryrun}", name="neontribe_cvs_import_batch")
+ * @Route(path="/batch/{filename}/{token}/{offset}/{chunk}/{dryrun}/{checkhashes}/{checkids}", name="neontribe_cvs_import_batch")
*
* @param Request $request
* @return \Symfony\Component\HttpFoundation\Response
*/
- public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
+ public function batchAction($filename, $token, $offset, $chunk, $dryrun, $checkhashes, $checkids) {
$privToken = $this->repository->getToken($filename);
if (! $this->repository->checkPublicToken($token, $privToken)) {
@@ -90,24 +93,44 @@ public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
}
// Are we done yet?
- $file = new \SplFileObject($this->repository->getCsvUploadDir() . $filename, 'r');
+ $file = new \SplFileObject($this->repository->getCsvDir() . $filename, 'r');
$file->seek(PHP_INT_MAX);
$linecount = $file->key() + 1;
if ($offset > $linecount) {
- $this->flashSuccess('Timesheets imported.');
- return $this->redirectToRoute('neontribe_cvs_import_admin');
+ return $this->redirectToRoute('neontribe_cvs_import_finish');
}
// Now read chunk timesheets
$counter = 0;
+ $errorCount = 0;
$file->rewind();
$file->seek($offset); // go to line 200
for ($i = 0; $i < $chunk and $file->valid(); $i ++, $file->next()) {
+ $line = $file->current();
try {
- $this->importService->importLine(str_getcsv($file->current()), $dryrun);
- } catch (\Throwable $error) {
+ // Check hash and/or id
+ $data = str_getcsv($line);
+ $id = $data[0];
+ $hash = md5($line);
$lineNumber = $offset + $i;
- $this->flashError("Could not parse line " . $lineNumber . ": " . $file->current() . ' - ' . $error->getMessage());
+ if ($checkhashes && $this->repository->checkHash($hash)) {
+ $this->flashWarning("Duplicate line line (" . $hash . ") " . $lineNumber . ": " . $line);
+ $this->repository->appendHistory('Duplicate (hash): ' . $line);
+ continue;
+ }
+ if ($checkids && $this->repository->checkId($id)) {
+ $this->flashWarning("Duplicate line line (" . $id . ") " . $lineNumber . ": " . $line);
+ $this->repository->appendHistory('Duplicate (id): ' . $line);
+ continue;
+ }
+
+ if ($this->importService->importLine($data, $dryrun)) {
+ $this->repository->saveHash($id, $hash);
+ }
+ } catch (\Throwable $error) {
+ $this->flashError("Could not parse line " . $lineNumber . ": " . $line . ' - ' . $error->getMessage());
+ $this->repository->appendHistory('Error: ' . $line);
+ $errorCount ++;
}
$counter ++;
}
@@ -121,6 +144,8 @@ public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
$nextPhase = $this->generateUrl('neontribe_cvs_import_batch', [
'filename' => $filename,
'dryrun' => $dryrun,
+ 'checkhashes' => $checkhashes,
+ 'checkids' => $checkids,
'token' => $this->repository->makePublicToken($token),
'offset' => $offset + $chunk,
'chunk' => $chunk
@@ -132,7 +157,39 @@ public function batchAction($filename, $token, $offset, $chunk, $dryrun) {
'offset' => $offset,
'count' => $counter,
'total' => $linecount,
- 'progress' => round($progressPercent, 2)
+ 'progress' => round($progressPercent, 2),
+ 'errorCount' => $errorCount
+ ]);
+ }
+
+ /**
+ *
+ * @Route(path="/batch/finished", name="neontribe_cvs_import_finish")
+ *
+ * @param Request $request
+ * @return \Symfony\Component\HttpFoundation\Response
+ */
+ public function batchFinished() {
+ $history = $this->repository->getHistory();
+
+ $errors = [];
+ $warnings = [];
+ $other = [];
+
+ foreach ($history as $line) {
+ if (strpos($line, 'Duplicate') === 0) {
+ $warnings[] = $line;
+ } elseif (strpos($line, 'Error') === 0) {
+ $errors[] = $line;
+ } else {
+ $other[] = $line;
+ }
+ }
+
+ return $this->render('@NeontribeCvsImport/finished.html.twig', [
+ 'errors' => $errors,
+ 'warnings' => $warnings,
+ 'other' => $other
]);
}
diff --git a/Form/CsvUploadForm.php b/Form/CsvUploadForm.php
index 3d6004a..d4bc6e5 100644
--- a/Form/CsvUploadForm.php
+++ b/Form/CsvUploadForm.php
@@ -32,7 +32,7 @@ public function buildForm(FormBuilderInterface $builder, array $options) {
'help' => 'If checked each line of the cvs will be hashed and checked against hashes of previously imported records. Duplicates will be ignored.'
]);
- $builder->add('timesheet_ids', CheckboxType::class, [
+ $builder->add('checkids', CheckboxType::class, [
'label' => 'Check timesheet ids',
'required' => False,
'data' => False,
diff --git a/README.md b/README.md
index e2a44fa..3f2d110 100644
--- a/README.md
+++ b/README.md
@@ -1,41 +1,3 @@
## Kimai import plugin
-This creates a console command that imports timesheets into Kimai.
-
-It reads a CSV file and creates a timesheet for each line it finds. If the Customer, Project or Activity does not exist it will create one. The deafult password for a new user is scrambled (but not fully secured). Use the password reset to set that.
-
-The fields in the CSV are:
-
- * id, The UID for the row. This is used to log each row imported.
- * customer name, The name of the customer.
- * project name, The name of the project
- * activity name, The name of the activity
- * start, The sate of the activity in the format YYYY-MM-DD HH:MM:SS
- * duration, In minutes
- * description, The text description of the timesheet entry
- * user name, The username of the logger
- * email, The email of the logger
-
-Newly created projects are assigned the customer specified by the line. Activities are assigned to the Project specified by the line.
-
-### Getting the data
-
-To get timesheets out of an existing kimai then use a version of this SQL:
-
- select
- t.id as id,
- c.name as customer,
- p.name as project,
- a.name as activity,
- t.start_time as date,
- t.duration as duration,
- t.description as title,
- u.username as username,
- u.email as email
- from
- kimai2_timesheet t
- inner join kimai2_users u on t.user=u.id
- inner join kimai2_activities a on t.activity_id=a.id
- inner join kimai2_projects p on a.project_id=p.id
- inner join kimai2_customers c on p.customer_id=c.id;
-
+See the full README bundled with the Kimai store: https://github.com/neontribe/www.kimai.org/blob/master/_store/neontribe-cvs-import-bundle.md
diff --git a/Repository/NeontribeCvsImportRepository.php b/Repository/NeontribeCvsImportRepository.php
index 15dfcd5..7b5fac0 100644
--- a/Repository/NeontribeCvsImportRepository.php
+++ b/Repository/NeontribeCvsImportRepository.php
@@ -8,78 +8,125 @@
class NeontribeCvsImportRepository {
/**
+ * Plugin directory supplied from the application.
*
* @var string
*/
protected $pluginDirectory = null;
/**
+ * Data directory supplied from the application.
*
* @var string
*/
protected $dataDirectory = null;
/**
+ * The app secret supplied from the application.
*
* @var string
*/
protected $appSecret;
/**
+ * Hashes of each imported line and the UID of the line are stored in this file.
*
* @var string
*/
protected $hashesFile;
+ /**
+ * This file lists history items to be displayed after the import id complete.
+ *
+ * @var string
+ */
+ protected $historyFile;
+
+ /**
+ * The folder where this plugin will store files.
+ *
+ * @var string
+ */
+ protected $myDataDir;
+
+ /**
+ * The folder where this plugin will store uploaded csv files.
+ *
+ * @var string
+ */
+ protected $csvDir;
+
/**
*
* @param string $pluginDirectory
+ * @param string $dataDirectory
+ * @param string $appSecret
*/
public function __construct(string $pluginDirectory, string $dataDirectory, string $appSecret) {
$this->pluginDirectory = $pluginDirectory;
$this->dataDirectory = $dataDirectory;
$this->appSecret = $appSecret;
- $this->hashesFile = $dataDirectory . '/hashes.json';
+ $filesystem = new Filesystem();
+ $myDataDir = $this->dataDirectory . '/neontribe-csv-import';
+ if (! $filesystem->exists($myDataDir)) {
+ $filesystem->mkdir($myDataDir, 0700);
+ }
+ $this->myDataDir = $myDataDir;
+
+ $csvDir = $myDataDir . '/csv/';
+ if (! $filesystem->exists($csvDir)) {
+ $filesystem->mkdir($csvDir, 0700);
+ }
+ $this->csvDir = $csvDir;
+
+ $this->hashesFile = $myDataDir . '/hashes.json';
+ $this->historyFile = $myDataDir . '/history.txt';
}
/**
+ * Get the persistent data folder for this plugin.
*
- * @param array $entity
- * @return bool
- * @throws \Exception
+ * @return string
*/
- public function saveHashData(array $hashes) {
- if (file_exists($this->hashesFile) && ! is_writable($this->hashesFile)) {
- throw new \Exception('Hashes file is not writable: ' . $this->hashesFile);
- }
-
- if (false === file_put_contents($this->hashesFile, json_encode($hashes, JSON_PRETTY_PRINT))) {
- throw new \Exception('Failed saving hashes rules to file: ' . $this->hashesFile);
- }
-
- return true;
+ public function getDataDir() {
+ return $this->myDataDir;
}
/**
+ * Get the csv folder.
*
- * @return array
+ * @return string
*/
- public function getHashData(): array {
- if (file_exists($this->hashesFile)) {
- return json_decode(file_get_contents($this->hashesFile), True);
- } else {
+ public function getCsvDir() {
+ return $this->csvDir;
+ }
+
+ public function saveHash($id, $hash) {
+ $data = $this->loadhashes();
+ $data[$id] = $hash;
+ $this->saveHashes($data);
+ }
+
+ public function checkId($id) {
+ $data = $this->loadhashes();
+ return in_array($id, array_keys($data));
+ }
+
+ public function checkHash($hash) {
+ $data = $this->loadhashes();
+ return in_array($hash, $data);
+ }
+
+ protected function loadhashes() {
+ if (! file_exists($this->hashesFile)) {
return [];
}
+ return json_decode(file_get_contents($this->hashesFile), True);
}
- public function getCsvUploadDir() {
- $filesystem = new Filesystem();
- $csvDir = $this->dataDirectory . '/csv-import-files/';
- if (! $filesystem->exists($csvDir)) {
- $filesystem->mkdir($csvDir, 0700);
- }
- return $csvDir;
+ protected function saveHashes($data) {
+ return file_put_contents($this->hashesFile, json_encode($data, JSON_PRETTY_PRINT));
}
public function saveToken($filename, $token) {
@@ -91,7 +138,7 @@ public function getToken($filename) {
}
protected function generateTokenFileName($filename) {
- $csvDir = $this->getCsvUploadDir();
+ $csvDir = $this->getDataDir();
$tokenfile = $csvDir . \basename($filename, 'csv') . 'txt';
return $tokenfile;
}
@@ -103,4 +150,18 @@ public function makePublicToken($token) {
public function checkPublicToken($pubtoken, $privToken) {
return $pubtoken === $this->makePublicToken($privToken);
}
+
+ public function clearHistory() {
+ file_put_contents($this->historyFile, '');
+ }
+
+ public function appendHistory($line) {
+ $fh = fopen($this->historyFile, 'a');
+ fwrite($fh, $line);
+ fclose($fh);
+ }
+
+ public function getHistory() {
+ return file($this->historyFile);
+ }
}
diff --git a/Resources/views/batch.html.twig b/Resources/views/batch.html.twig
index 14dfed2..0d6429b 100644
--- a/Resources/views/batch.html.twig
+++ b/Resources/views/batch.html.twig
@@ -11,11 +11,17 @@
{% if dryrun %}
DRY RUN
{% endif %}
+
+
+
Processed lines {{ offset }} to {{ offset + count }} of {{ total }}
({{ progress }}%)
+
{% endblock %}
diff --git a/Resources/views/finished.html.twig b/Resources/views/finished.html.twig
new file mode 100644
index 0000000..88dd71b
--- /dev/null
+++ b/Resources/views/finished.html.twig
@@ -0,0 +1,57 @@
+{% extends 'base.html.twig' %}
+{% import "macros/widgets.html.twig" as widgets %}
+{% import "macros/datatables.html.twig" as tables %}
+
+{% block page_title %}
+CSV Import - Done
+{% endblock %}
+
+
+{% block main %}
+
+
+
+
+
+{% if warnings | length %}
+
+
+
+ {% for line in warnings %}
+ - {{ line }}
+ {% endfor %}
+
+
+
+
+ {% for line in errors %}
+ - {{ line }}
+ {% endfor %}
+
+
+
+
+ {% for line in other %}
+ - {{ line }}
+ {% endfor %}
+
+
- {{ form_row(form.timesheet_ids) }}
- {{ form_help(form.timesheet_ids) }}
+ {{ form_row(form.checkids) }}
+ {{ form_help(form.checkids) }}
{{ form_row(form.csvfile) }}
diff --git a/Service/NeontribeCvsImportService.php b/Service/NeontribeCvsImportService.php
index 5794c54..409cde3 100644
--- a/Service/NeontribeCvsImportService.php
+++ b/Service/NeontribeCvsImportService.php
@@ -87,7 +87,10 @@ public function importLine(array $elements, $dryrun = True) {
if (! $dryrun) {
$this->doctrine->getManager()->persist($timesheet);
+ return True;
}
+
+ return False;
}
public function getCompany($customerName): Customer {