Fix comma sensitivity (#5368)

* update debug.csv

change separator to ';' and add to entries with a comma as decimal to detect problems like jasp-stats/jasp-issues#2499

* Fix comma import from csv/ods/etc

Partial fix for jasp-stats/jasp-issues#2499

* add some 123.456.789,000 dots combined with commas to the debug dataset

* the event can just be aware of when it ought to be readonly

Author: JorisGoosen

CommonData/column.cpp

@@ -1622,18 +1622,18 @@ bool Column::convertValueToIntForImport(const std::string &strValue, int &intVal
 	return ColumnUtils::getIntValue(strValue, intValue);
 }
 
+/// There is a ColumnUtils::convertValueToDoubleForImport that is used during import from readstat and which only looks at workspace empty values
 bool Column::convertValueToDoubleForImport(const std::string & strValue, double & doubleValue) const
 {
-	std::string v = strValue;
-	ColumnUtils::deEuropeaniseForImport(v);
+	std::string v = ColumnUtils::deEuropeaniseForImport(strValue);
 
 	if(isEmptyValue(v))
 	{
 			doubleValue = NAN;
 			return true;
 	}
 
-	return ColumnUtils::getDoubleValue(strValue, doubleValue);
+	return ColumnUtils::getDoubleValue(v, doubleValue);
 }
 
 

CommonData/columnutils.cpp

@@ -88,7 +88,7 @@ bool ColumnUtils::isDoubleValue(const string &value)
 }
 
 
-void ColumnUtils::deEuropeaniseForImport(std::string & value)
+std::string ColumnUtils::deEuropeaniseForImport(std::string value)
 {
 	int dots	= 0,
 		commas	= 0;
@@ -102,7 +102,7 @@ void ColumnUtils::deEuropeaniseForImport(std::string & value)
 			if(k == ',')
 			{
 				k = '.';
-				return;
+				return value;
 			}
 
 	if (commas > 0)
@@ -133,6 +133,8 @@ void ColumnUtils::deEuropeaniseForImport(std::string & value)
 
 		value = uneurope;
 	}
+	
+	return value;
 }
 
 bool ColumnUtils::convertValueToIntForImport(const std::string &strValue, int &intValue)
@@ -152,8 +154,7 @@ bool ColumnUtils::convertValueToIntForImport(const std::string &strValue, int &i
 
 bool ColumnUtils::convertValueToDoubleForImport(const std::string & strValue, double & doubleValue)
 {
-	std::string v = strValue;
-	deEuropeaniseForImport(v);
+	std::string v = deEuropeaniseForImport(strValue);
 
 	if(isEmptyValue(v, EmptyValues::singleton()->workspaceEmptyValues()))
 		doubleValue = NAN;

CommonData/columnutils.h

@@ -14,20 +14,19 @@ class ColumnUtils
 
 	static       std::string emptyValue;
 
-	static bool getIntValue(	const std::string	& value, int	& intValue);
-	static bool getIntValue(	const double		& value, int	& intValue);
-	static bool getDoubleValue(	const std::string	& value, double	& doubleValue);
-
-	static bool isIntValue(		const std::string	& value);
-	static bool isDoubleValue(	const std::string	& value);
-
-	static bool	convertValueToIntForImport(		const	std::string & strValue, int		& intValue);
-	static bool	convertValueToDoubleForImport(	const	std::string & strValue, double	& doubleValue);
-	static bool	isEmptyValue(					const	std::string	& val,		const stringset & emptyValues);
-	static bool	isEmptyValue(					const	double		  val,		const doubleset & doubleEmptyValues);
-	static void	convertEscapedUnicodeToUTF8(			std::string & inputStr);
-	static void	deEuropeaniseForImport(					std::string & value);
-
+	static bool			getIntValue(	const std::string	& value, int	& intValue);
+	static bool			getIntValue(	const double		& value, int	& intValue);
+	static bool			getDoubleValue(	const std::string	& value, double	& doubleValue);
+
+	static bool			isIntValue(		const std::string	& value);
+	static bool			isDoubleValue(	const std::string	& value);
+
+	static bool			convertValueToIntForImport(		const	std::string &	strValue, int		& intValue);
+	static bool			convertValueToDoubleForImport(	const	std::string &	strValue, double	& doubleValue);
+	static bool			isEmptyValue(					const	std::string	&	val,		const stringset & emptyValues);
+	static bool			isEmptyValue(					const	double			val,		const doubleset & doubleEmptyValues);
+	static void			convertEscapedUnicodeToUTF8(			std::string &	inputStr);
+	static std::string	deEuropeaniseForImport(					std::string		value);
 	static std::string	doubleToString(			double dbl, int precision = 10);
 
 private:

Desktop/analysis/analysis.h

@@ -178,7 +178,7 @@ class Analysis : public AnalysisBase
 	void					requestComputedColumnDestruction(	const std::string & columnName);
 
 	void					refreshTableViewModels();
-	Q_INVOKABLE void		expandAnalysis();
+	void					expandAnalysis();
 	void					emptyQMLCache();
 
 	void					createFormWhenYouHaveAMoment(QQuickItem* parent = nullptr);

Desktop/data/fileevent.cpp

@@ -22,8 +22,7 @@
 #include "exporters/jaspexporter.h"
 
 #include <QTimer>
-#include "utilities/qutils.h"
-#include "log.h"
+#include "utilities/appdirs.h"
 
 FileEvent::FileEvent(QObject *parent, FileEvent::FileMode fileMode)
 	: QObject(parent), _operation(fileMode)
@@ -51,8 +50,6 @@ void FileEvent::setDataFilePath(const QString & path)
 
 void FileEvent::setDatabase(const Json::Value & dbInfo)
 {
-	setReadOnly();
-	
 	_database = dbInfo;
 }
 
@@ -110,6 +107,11 @@ void FileEvent::chain(FileEvent *event)
 	connect(event, &FileEvent::completed, this, &FileEvent::chainedComplete);
 }
 
+bool FileEvent::isExample() const
+{
+	return path().startsWith(AppDirs::examples());
+}
+
 const std::string FileEvent::databaseStr() const 
 { 
 	return _database.toStyledString();

Desktop/data/fileevent.h

@@ -47,11 +47,10 @@ class FileEvent : public QObject
 	void				setComplete(bool success = true, const QString &message = "");
 	void				chain(FileEvent *event);
 
-	void				setReadOnly()		  { _readOnly = true;		}
-
 	bool				isDatabase()	const { return _database != Json::nullValue;	}
 	bool				isOnlineNode()	const { return _path.startsWith("http");		}
-	bool				isReadOnly()	const { return _readOnly;						}
+	bool				isExample()		const;
+	bool				isReadOnly()	const { return isExample() || isDatabase();		}
 	bool				isCompleted()	const { return _completed;						}
 	bool				isSuccessful()	const { return _success;						}
 
@@ -83,8 +82,7 @@ private slots:
 						_dataFilePath,
 						_last_error		= "Unknown error",
 						_message;
-	bool				_readOnly		= false,
-						_completed		= false,
+	bool				_completed		= false,
 						_success		= false;
 	FileEvent		*	_chainedTo		= nullptr;
 	Exporter		*	_exporter		= nullptr;

Desktop/widgets/filemenu/datalibrarylistmodel.cpp

@@ -52,7 +52,6 @@ void DataLibraryListModel::openFile(const QString &path)
 {
 	FileEvent *event = new FileEvent(this->parent(), FileEvent::FileOpen);
 	event->setPath(path);
-	event->setReadOnly(); // A file from the Data Library should be read only, also csv file so that its path is not added in the recent list
 
 	emit openFileEvent(event);
 

Desktop/widgets/filemenu/filemenu.cpp

@@ -260,7 +260,7 @@ void FileMenu::dataSetIOCompleted(FileEvent *event)
 			if (!event->isDatabase())
 			{
 				_recentFiles->pushRecentFilePath(event->path());
-				if (!event->path().startsWith(AppDirs::examples()))
+				if (!event->isExample())
 					_computer->addRecentFolder(event->path());
 			}