From 16058e590ee696012a76a627f0bf7940b7283ed6 Mon Sep 17 00:00:00 2001 From: Stephen Morgan Date: Thu, 10 Mar 2022 11:29:43 +1100 Subject: [PATCH] fix: utf-8: Use with-utf8 to ensure all files are read and written with utf8 encoding. (#1619) May also fix #1154, #1033, #708, #536, #73: testing is needed. --- bin/hledger-balance-as-budget.hs | 2 +- bin/hledger-check-fancyassertions.hs | 2 +- bin/hledger-combine-balances.hs | 2 +- bin/hledger-swap-dates.hs | 2 +- .../Hledger/Data/PeriodicTransaction.hs | 6 ++-- .../Hledger/Data/TransactionModifier.hs | 4 +-- hledger-lib/Hledger/Read.hs | 6 ++-- hledger-lib/Hledger/Read/CsvReader.hs | 28 ++++++++++--------- hledger-lib/Hledger/Utils.hs | 17 ++++------- hledger-lib/hledger-lib.cabal | 5 +++- hledger-lib/package.yaml | 5 ++-- hledger/Hledger/Cli/Commands/Accounts.hs | 4 +-- hledger/Hledger/Cli/Commands/Add.hs | 10 +++---- hledger/Hledger/Cli/Commands/Checkdates.hs | 4 +-- hledger/Hledger/Cli/Commands/Close.hs | 6 ++-- hledger/Hledger/Cli/Commands/Codes.hs | 4 +-- hledger/Hledger/Cli/Commands/Commodities.hs | 4 +-- hledger/Hledger/Cli/Commands/Descriptions.hs | 4 +-- hledger/Hledger/Cli/Commands/Diff.hs | 6 ++-- hledger/Hledger/Cli/Commands/Import.hs | 4 +-- hledger/Hledger/Cli/Commands/Notes.hs | 4 +-- hledger/Hledger/Cli/Commands/Payees.hs | 4 +-- hledger/Hledger/Cli/Commands/Prices.hs | 4 +-- hledger/Hledger/Cli/Commands/Print.hs | 6 ++-- hledger/Hledger/Cli/Commands/Registermatch.hs | 6 ++-- hledger/Hledger/Cli/Commands/Rewrite.hs | 4 +-- hledger/Hledger/Cli/Commands/Roi.hs | 10 +++---- hledger/Hledger/Cli/Commands/Tags.hs | 4 +-- hledger/Hledger/Cli/Main.hs | 19 +++++++------ hledger/Hledger/Cli/Utils.hs | 24 ++++++++-------- hledger/hledger.cabal | 6 +++- hledger/package.yaml | 1 + stack8.6.yaml | 7 +++-- 33 files changed, 117 insertions(+), 107 deletions(-) diff --git a/bin/hledger-balance-as-budget.hs b/bin/hledger-balance-as-budget.hs index 8f4aa3ca3830..d0be20a17fba 100755 --- a/bin/hledger-balance-as-budget.hs +++ b/bin/hledger-balance-as-budget.hs @@ -8,7 +8,7 @@ {-| Construct two balance reports for two different time periods and use one of the as "budget" for the other, thus comparing them -} -import Data.Text.Lazy.IO as TL +import Data.Text.Lazy.IO as TL (putStrLn) -- Only putStr and friends are safe import System.Environment (getArgs) import Hledger.Cli diff --git a/bin/hledger-check-fancyassertions.hs b/bin/hledger-check-fancyassertions.hs index ef7ff1feb48e..bd96f3fbc0be 100755 --- a/bin/hledger-check-fancyassertions.hs +++ b/bin/hledger-check-fancyassertions.hs @@ -111,7 +111,7 @@ import Data.Time.Calendar (toGregorian) import Data.Time.Calendar.OrdinalDate (mondayStartWeek, sundayStartWeek, toOrdinalDate) import Data.Text (Text, isPrefixOf, pack, unpack) import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as T (putStrLn) -- Only putStr and friends are safe import qualified Hledger.Data as H import qualified Hledger.Query as H import qualified Hledger.Read as H diff --git a/bin/hledger-combine-balances.hs b/bin/hledger-combine-balances.hs index 124f9af01213..f5a89fcc4689 100755 --- a/bin/hledger-combine-balances.hs +++ b/bin/hledger-combine-balances.hs @@ -11,7 +11,7 @@ import System.Environment (getArgs) import Hledger.Cli import qualified Data.Map as M import Data.Map.Merge.Strict -import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.Lazy.IO as TL (putStrLn) -- Only putStr and friends are safe appendReports :: MultiBalanceReport -> MultiBalanceReport -> MultiBalanceReport appendReports r1 r2 = diff --git a/bin/hledger-swap-dates.hs b/bin/hledger-swap-dates.hs index 669a742c242b..e4ddf9f196a2 100755 --- a/bin/hledger-swap-dates.hs +++ b/bin/hledger-swap-dates.hs @@ -9,7 +9,7 @@ {-# LANGUAGE RecordWildCards #-} import Data.String.QQ (s) -import qualified Data.Text.IO as T +import qualified Data.Text.IO as T (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli diff --git a/hledger-lib/Hledger/Data/PeriodicTransaction.hs b/hledger-lib/Hledger/Data/PeriodicTransaction.hs index 340f19776561..6181da857c8d 100644 --- a/hledger-lib/Hledger/Data/PeriodicTransaction.hs +++ b/hledger-lib/Hledger/Data/PeriodicTransaction.hs @@ -12,7 +12,7 @@ module Hledger.Data.PeriodicTransaction ( where import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStr) -- Only putStr and friends are safe import Text.Printf import Hledger.Data.Types @@ -36,7 +36,7 @@ _ptgen str = do case checkPeriodicTransactionStartDate i s t of Just e -> error' e -- PARTIAL: Nothing -> - mapM_ (T.putStr . showTransaction) $ + mapM_ (TIO.putStr . showTransaction) $ runPeriodicTransaction nullperiodictransaction{ ptperiodexpr=t , ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1] } nulldatespan @@ -48,7 +48,7 @@ _ptgenspan str span = do case checkPeriodicTransactionStartDate i s t of Just e -> error' e -- PARTIAL: Nothing -> - mapM_ (T.putStr . showTransaction) $ + mapM_ (TIO.putStr . showTransaction) $ runPeriodicTransaction nullperiodictransaction{ ptperiodexpr=t , ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1] } span diff --git a/hledger-lib/Hledger/Data/TransactionModifier.hs b/hledger-lib/Hledger/Data/TransactionModifier.hs index 1624f078110f..3d42e2ae487c 100644 --- a/hledger-lib/Hledger/Data/TransactionModifier.hs +++ b/hledger-lib/Hledger/Data/TransactionModifier.hs @@ -63,10 +63,10 @@ modifyTransactions atypes atags styles d tmods ts = do -- Currently the only kind of modification possible is adding automated -- postings when certain other postings are present. -- --- >>> import qualified Data.Text.IO as T +-- >>> import qualified Data.Text.IO as TIO (putStr) -- Only putStr and friends are safe -- >>> t = nulltransaction{tpostings=["ping" `post` usd 1]} -- >>> tmpost acc amt = TMPostingRule (acc `post` amt) False --- >>> test = either putStr (T.putStr.showTransaction) . fmap ($ t) . transactionModifierToFunction (const Nothing) (const []) mempty nulldate +-- >>> test = either putStr (TIO.putStr.showTransaction) . fmap ($ t) . transactionModifierToFunction (const Nothing) (const []) mempty nulldate -- >>> test $ TransactionModifier "" ["pong" `tmpost` usd 2] -- 0000-01-01 -- ping $1.00 diff --git a/hledger-lib/Hledger/Read.hs b/hledger-lib/Hledger/Read.hs index c7d0b46fd492..22b2359ed224 100644 --- a/hledger-lib/Hledger/Read.hs +++ b/hledger-lib/Hledger/Read.hs @@ -58,7 +58,7 @@ import Data.Ord (comparing) import Data.Semigroup (sconcat) import Data.Text (Text) import qualified Data.Text as T -import qualified Data.Text.IO as T +import Data.Text.IO.Utf8 (writeFile) import Data.Time (Day) import Safe (headDef) import System.Directory (doesFileExist, getHomeDirectory) @@ -212,7 +212,7 @@ ensureJournalFileExists f = do hPutStr stderr $ "Creating hledger journal file " <> show f <> ".\n" -- note Hledger.Utils.UTF8.* do no line ending conversion on windows, -- we currently require unix line endings on all platforms. - newJournalContent >>= T.writeFile f + newJournalContent >>= writeFile f -- | Does any part of this path contain non-. characters and end with a . ? -- Such paths are not safe to use on Windows (cf #1056). @@ -239,7 +239,7 @@ latestDates = headDef [] . take 1 . group . reverse . sort -- | Remember that these transaction dates were the latest seen when -- reading this journal file. saveLatestDates :: LatestDates -> FilePath -> IO () -saveLatestDates dates f = T.writeFile (latestDatesFileFor f) $ T.unlines $ map showDate dates +saveLatestDates dates f = writeFile (latestDatesFileFor f) $ T.unlines $ map showDate dates -- | What were the latest transaction dates seen the last time this -- journal file was read ? If there were multiple transactions on the diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 90c6d2b397d2..a9b272009fe1 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -37,6 +37,7 @@ module Hledger.Read.CsvReader ( where --- ** imports +import Prelude hiding (getContents, writeFile) import Control.Applicative (liftA2) import Control.Exception (IOException, handle, throw) import Control.Monad (unless, when) @@ -45,8 +46,13 @@ import qualified Control.Monad.Fail as Fail import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.State.Strict (StateT, get, modify', evalStateT) import Control.Monad.Trans.Class (lift) -import Data.Char (toLower, isDigit, isSpace, isAlphaNum, ord) import Data.Bifunctor (first) +import qualified Data.ByteString as B +import qualified Data.ByteString.Lazy as BL +import Data.Char (toLower, isDigit, isSpace, isAlphaNum, ord) +import qualified Data.Csv as Cassava +import qualified Data.Csv.Parser.Megaparsec as CassavaMP +import Data.Foldable (asum, toList) import Data.List (elemIndex, foldl', intersperse, mapAccumL, nub, sortBy) import Data.Maybe (catMaybes, fromMaybe, isJust) import Data.MemoUgly (memo) @@ -54,8 +60,9 @@ import Data.Ord (comparing) import qualified Data.Set as S import Data.Text (Text) import qualified Data.Text as T -import qualified Data.Text.Encoding as T -import qualified Data.Text.IO as T +import Data.Text.Encoding (decodeUtf8, encodeUtf8) +import Data.Text.IO (getContents) -- Only putStr and friends are safe +import Data.Text.IO.Utf8 (writeFile) import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import Data.Time.Calendar (Day) @@ -63,11 +70,6 @@ import Data.Time.Format (parseTimeM, defaultTimeLocale) import Safe (atMay, headMay, lastMay, readDef, readMay) import System.Directory (doesFileExist) import System.FilePath ((), takeDirectory, takeExtension, takeFileName) -import qualified Data.Csv as Cassava -import qualified Data.Csv.Parser.Megaparsec as CassavaMP -import qualified Data.ByteString as B -import qualified Data.ByteString.Lazy as BL -import Data.Foldable (asum, toList) import Text.Megaparsec hiding (match, parse) import Text.Megaparsec.Char (char, newline, string) import Text.Megaparsec.Custom (customErrorBundlePretty, parseErrorAt) @@ -200,7 +202,7 @@ expandIncludes dir content = mapM (expandLine dir) (T.lines content) >>= return where expandLine dir line = case line of - (T.stripPrefix "include " -> Just f) -> expandIncludes dir' =<< T.readFile f' + (T.stripPrefix "include " -> Just f) -> expandIncludes dir' =<< readFilePortably f' where f' = dir T.unpack (T.dropWhile isSpace f) dir' = takeDirectory f' @@ -770,7 +772,7 @@ readJournalFromCsv mrulesfile csvfile csvdata = when (not rulesfileexists) $ do dbg1IO "creating conversion rules file" rulesfile - T.writeFile rulesfile rulestext + writeFile rulesfile rulestext return $ Right nulljournal{jtxns=txns''} @@ -785,14 +787,14 @@ parseSeparator = specials . T.toLower parseCsv :: Char -> FilePath -> Text -> IO (Either String CSV) parseCsv separator filePath csvdata = case filePath of - "-" -> parseCassava separator "(stdin)" <$> T.getContents + "-" -> parseCassava separator "(stdin)" <$> getContents _ -> return $ if T.null csvdata then Right mempty else parseCassava separator filePath csvdata parseCassava :: Char -> FilePath -> Text -> Either String CSV parseCassava separator path content = either (Left . errorBundlePretty) (Right . parseResultToCsv) <$> CassavaMP.decodeWith (decodeOptions separator) Cassava.NoHeader path $ - BL.fromStrict $ T.encodeUtf8 content + BL.fromStrict $ encodeUtf8 content decodeOptions :: Char -> Cassava.DecodeOptions decodeOptions separator = Cassava.defaultDecodeOptions { @@ -803,7 +805,7 @@ parseResultToCsv :: (Foldable t, Functor t) => t (t B.ByteString) -> CSV parseResultToCsv = toListList . unpackFields where toListList = toList . fmap toList - unpackFields = (fmap . fmap) T.decodeUtf8 + unpackFields = (fmap . fmap) decodeUtf8 printCSV :: CSV -> TL.Text printCSV = TB.toLazyText . unlinesB . map printRecord diff --git a/hledger-lib/Hledger/Utils.hs b/hledger-lib/Hledger/Utils.hs index a80abc3ee91a..79879b16b919 100644 --- a/hledger-lib/Hledger/Utils.hs +++ b/hledger-lib/Hledger/Utils.hs @@ -30,13 +30,12 @@ module Hledger.Utils (---- provide these frequently used modules - or not, for c ) where -import Control.Monad (when) import Data.Char (toLower) import Data.FileEmbed (makeRelativeToProject, embedStringFile) import Data.List.Extra (foldl', foldl1', uncons, unsnoc) import qualified Data.Set as Set import Data.Text (Text) -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (hGetContents) -- Only putStr and friends are safe import qualified Data.Text.Lazy.Builder as TB import Data.Time.Clock (getCurrentTime) import Data.Time.LocalTime (LocalTime, ZonedTime, getCurrentTimeZone, @@ -48,10 +47,9 @@ import Lens.Micro ((&), (.~)) import Lens.Micro.TH (DefName(TopName), lensClass, lensField, makeLensesWith, classyRules) import System.Console.ANSI (Color,ColorIntensity,ConsoleLayer(..), SGR(..), setSGRCode) import System.Directory (getHomeDirectory) -import System.FilePath (isRelative, ()) -import System.IO - (Handle, IOMode (..), hGetEncoding, hSetEncoding, hSetNewlineMode, - openFile, stdin, universalNewlineMode, utf8_bom) +import System.FilePath ((), isRelative) +import System.IO (Handle, IOMode (..), hSetNewlineMode, stdin, universalNewlineMode) +import System.IO.Utf8 (openFile) import Hledger.Utils.Debug import Hledger.Utils.Parse @@ -186,12 +184,7 @@ readFileOrStdinPortably f = openFileOrStdin f ReadMode >>= readHandlePortably openFileOrStdin f m = openFile f m readHandlePortably :: Handle -> IO Text -readHandlePortably h = do - hSetNewlineMode h universalNewlineMode - menc <- hGetEncoding h - when (fmap show menc == Just "UTF-8") $ -- XXX no Eq instance, rely on Show - hSetEncoding h utf8_bom - T.hGetContents h +readHandlePortably h = hSetNewlineMode h universalNewlineMode *> TIO.hGetContents h -- | Total version of maximum, for integral types, giving 0 for an empty list. maximum' :: Integral a => [a] -> a diff --git a/hledger-lib/hledger-lib.cabal b/hledger-lib/hledger-lib.cabal index 2a6d61a334f1..7e16e56debbe 100644 --- a/hledger-lib/hledger-lib.cabal +++ b/hledger-lib/hledger-lib.cabal @@ -1,6 +1,6 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.34.4. +-- This file has been generated from package.yaml by hpack version 0.34.6. -- -- see: https://github.com/sol/hpack @@ -133,6 +133,7 @@ library , uglymemo , unordered-containers >=0.2 , utf8-string >=0.3.5 + , with-utf8 >=1.0.0 default-language: Haskell2010 test-suite doctest @@ -184,6 +185,7 @@ test-suite doctest , uglymemo , unordered-containers >=0.2 , utf8-string >=0.3.5 + , with-utf8 >=1.0.0 if impl(ghc >= 9.0) buildable: False default-language: Haskell2010 @@ -237,5 +239,6 @@ test-suite unittest , uglymemo , unordered-containers >=0.2 , utf8-string >=0.3.5 + , with-utf8 >=1.0.0 buildable: True default-language: Haskell2010 diff --git a/hledger-lib/package.yaml b/hledger-lib/package.yaml index ee723bb7e484..a01c2addcea9 100644 --- a/hledger-lib/package.yaml +++ b/hledger-lib/package.yaml @@ -48,8 +48,10 @@ dependencies: - Decimal >=0.5.1 - directory - doclayout >=0.3 && <0.4 +- extra >=1.6.3 - file-embed >=0.0.10 - filepath +- Glob >= 0.9 - hashtables >=1.2.3.1 - megaparsec >=7.0.0 && <9.3 - microlens >=0.4 @@ -70,8 +72,7 @@ dependencies: - unordered-containers >=0.2 - uglymemo - utf8-string >=0.3.5 -- extra >=1.6.3 -- Glob >= 0.9 +- with-utf8 >=1.0.0 # for ledger-parse: #- parsers >=0.5 #- system-filepath diff --git a/hledger/Hledger/Cli/Commands/Accounts.hs b/hledger/Hledger/Cli/Commands/Accounts.hs index c61b3e87ae7a..404c2d3cbf93 100644 --- a/hledger/Hledger/Cli/Commands/Accounts.hs +++ b/hledger/Hledger/Cli/Commands/Accounts.hs @@ -22,7 +22,7 @@ module Hledger.Cli.Commands.Accounts ( import Data.List import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import System.Console.CmdArgs.Explicit as C import Hledger @@ -96,4 +96,4 @@ accounts CliOpts{rawopts_=rawopts, reportspec_=ReportSpec{_rsQuery=query,_rsRepo where spacer = T.replicate (maxwidth - T.length (showName a)) " " maxwidth = maximum $ map (T.length . showName) clippedaccts - forM_ clippedaccts $ \a -> T.putStrLn $ showName a <> showType a + forM_ clippedaccts $ \a -> TIO.putStrLn $ showName a <> showType a diff --git a/hledger/Hledger/Cli/Commands/Add.hs b/hledger/Hledger/Cli/Commands/Add.hs index a422b46c3238..61a810ff62b0 100644 --- a/hledger/Hledger/Cli/Commands/Add.hs +++ b/hledger/Hledger/Cli/Commands/Add.hs @@ -26,9 +26,9 @@ import Data.List (isPrefixOf) import Data.Maybe (fromJust, fromMaybe, isJust) import Data.Text (Text) import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (hPutStr, putStr) -- Only putStr and friends are safe import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.Lazy.IO as TLIO (putStrLn) -- Only putStr and friends are safe import Data.Time.Calendar (Day) import Data.Time.Format (formatTime, defaultTimeLocale, iso8601DateFormat) import Lens.Micro ((^.)) @@ -184,7 +184,7 @@ confirmedTransactionWizard prevInput es@EntryState{..} stack@(currentStage : _) prevInput' = prevInput{prevDescAndCmnt=Just descAndCommentString} when (isJust mbaset) . liftIO $ do hPutStrLn stderr "Using this similar transaction for defaults:" - T.hPutStr stderr $ showTransaction (fromJust mbaset) + TIO.hPutStr stderr $ showTransaction (fromJust mbaset) confirmedTransactionWizard prevInput' es' ((EnterNewPosting TxnParams{txnDate=date, txnCode=code, txnDesc=desc, txnCmnt=comment} Nothing) : stack) Nothing -> confirmedTransactionWizard prevInput es (drop 1 stack) @@ -435,7 +435,7 @@ journalAddTransaction j@Journal{jtxns=ts} opts t = do -- unelided shows all amounts explicitly, in case there's a price, cf #283 when (debug_ opts > 0) $ do putStrLn $ printf "\nAdded transaction to %s:" f - TL.putStrLn =<< registerFromString (showTransaction t) + TLIO.putStrLn =<< registerFromString (showTransaction t) return j{jtxns=ts++[t]} -- | Append a string, typically one or more transactions, to a journal @@ -448,7 +448,7 @@ journalAddTransaction j@Journal{jtxns=ts} opts t = do -- appendToJournalFileOrStdout :: FilePath -> Text -> IO () appendToJournalFileOrStdout f s - | f == "-" = T.putStr s' + | f == "-" = TIO.putStr s' | otherwise = appendFile f $ T.unpack s' where s' = "\n" <> ensureOneNewlineTerminated s diff --git a/hledger/Hledger/Cli/Commands/Checkdates.hs b/hledger/Hledger/Cli/Commands/Checkdates.hs index a2686bd07f4c..088d6a5551e6 100755 --- a/hledger/Hledger/Cli/Commands/Checkdates.hs +++ b/hledger/Hledger/Cli/Commands/Checkdates.hs @@ -7,7 +7,7 @@ module Hledger.Cli.Commands.Checkdates ( ) where import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions import System.Console.CmdArgs.Explicit @@ -43,7 +43,7 @@ checkdates CliOpts{rawopts_=rawopts,reportspec_=rspec} j = do positionstr = T.pack . showGenericSourcePos $ tsourcepos error txn1str = linesPrepend (T.pack " ") $ showTransaction previous txn2str = linesPrepend2 (T.pack "> ") (T.pack " ") $ showTransaction error - T.putStrLn $ + TIO.putStrLn $ T.pack "Error: transaction date is out of order" <> uniquestr <> T.pack "\nat " <> positionstr <> T.pack ":\n\n" <> txn1str <> txn2str diff --git a/hledger/Hledger/Cli/Commands/Close.hs b/hledger/Hledger/Cli/Commands/Close.hs index 41350287abf8..9b2cd4463b8c 100755 --- a/hledger/Hledger/Cli/Commands/Close.hs +++ b/hledger/Hledger/Cli/Commands/Close.hs @@ -12,7 +12,7 @@ import Data.Function (on) import Data.List (groupBy) import Data.Maybe (fromMaybe) import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStr) -- Only putStr and friends are safe import Data.Time.Calendar (addDays) import System.Console.CmdArgs.Explicit as C @@ -169,5 +169,5 @@ close CliOpts{rawopts_=rawopts, reportspec_=rspec'} j = do ++ [posting{paccount=openingacct, pamount=if explicit then mixedAmountSetFullPrecision (maNegate totalamt) else missingmixedamt} | not interleaved] -- print them - when closing . T.putStr $ showTransaction closingtxn - when opening . T.putStr $ showTransaction openingtxn + when closing . TIO.putStr $ showTransaction closingtxn + when opening . TIO.putStr $ showTransaction openingtxn diff --git a/hledger/Hledger/Cli/Commands/Codes.hs b/hledger/Hledger/Cli/Commands/Codes.hs index 93fd915a1fdb..2719d0605555 100644 --- a/hledger/Hledger/Cli/Commands/Codes.hs +++ b/hledger/Hledger/Cli/Commands/Codes.hs @@ -16,7 +16,7 @@ module Hledger.Cli.Commands.Codes ( ) where import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions @@ -36,4 +36,4 @@ codes CliOpts{reportspec_=rspec} j = do let ts = entriesReport rspec j codes = (if empty_ (_rsReportOpts rspec) then id else filter (not . T.null)) $ map tcode ts - mapM_ T.putStrLn codes + mapM_ TIO.putStrLn codes diff --git a/hledger/Hledger/Cli/Commands/Commodities.hs b/hledger/Hledger/Cli/Commands/Commodities.hs index 1525731466c8..94075ced3247 100644 --- a/hledger/Hledger/Cli/Commands/Commodities.hs +++ b/hledger/Hledger/Cli/Commands/Commodities.hs @@ -13,7 +13,7 @@ module Hledger.Cli.Commands.Commodities ( ) where import qualified Data.Set as S -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions @@ -30,4 +30,4 @@ commoditiesmode = hledgerCommandMode commodities :: CliOpts -> Journal -> IO () commodities _copts = -- TODO support --declared/--used like accounts, payees - mapM_ T.putStrLn . S.filter (/= "AUTO") . journalCommodities + mapM_ TIO.putStrLn . S.filter (/= "AUTO") . journalCommodities diff --git a/hledger/Hledger/Cli/Commands/Descriptions.hs b/hledger/Hledger/Cli/Commands/Descriptions.hs index 448f3d246e3c..37dd4e5abb13 100644 --- a/hledger/Hledger/Cli/Commands/Descriptions.hs +++ b/hledger/Hledger/Cli/Commands/Descriptions.hs @@ -15,7 +15,7 @@ module Hledger.Cli.Commands.Descriptions ( ) where import Data.List.Extra (nubSort) -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions @@ -35,4 +35,4 @@ descriptions CliOpts{reportspec_=rspec} j = do let ts = entriesReport rspec j descriptions = nubSort $ map tdescription ts - mapM_ T.putStrLn descriptions + mapM_ TIO.putStrLn descriptions diff --git a/hledger/Hledger/Cli/Commands/Diff.hs b/hledger/Hledger/Cli/Commands/Diff.hs index a968519a19c1..067f8fff8bfd 100644 --- a/hledger/Hledger/Cli/Commands/Diff.hs +++ b/hledger/Hledger/Cli/Commands/Diff.hs @@ -18,7 +18,7 @@ import Data.Ord (comparing) import Data.Maybe (fromJust) import Data.Time (diffDays) import Data.Either (partitionEithers) -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStr) -- Only putStr and friends are safe import System.Exit (exitFailure) import Hledger @@ -111,10 +111,10 @@ diff CliOpts{file_=[f1, f2], reportspec_=ReportSpec{_rsQuery=Acct acctRe}} _ = d let unmatchedtxn2 = unmatchedtxns R pp2 m putStrLn "These transactions are in the first file only:\n" - mapM_ (T.putStr . showTransaction) unmatchedtxn1 + mapM_ (TIO.putStr . showTransaction) unmatchedtxn1 putStrLn "These transactions are in the second file only:\n" - mapM_ (T.putStr . showTransaction) unmatchedtxn2 + mapM_ (TIO.putStr . showTransaction) unmatchedtxn2 diff _ _ = do putStrLn "Please specify two input files. Usage: hledger diff -f FILE1 -f FILE2 FULLACCOUNTNAME" diff --git a/hledger/Hledger/Cli/Commands/Import.hs b/hledger/Hledger/Cli/Commands/Import.hs index 87efec2138dc..73cfcb44bbe4 100755 --- a/hledger/Hledger/Cli/Commands/Import.hs +++ b/hledger/Hledger/Cli/Commands/Import.hs @@ -9,7 +9,7 @@ where import Control.Monad import Data.List -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStr) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions import Hledger.Cli.Commands.Add (journalAddTransaction) @@ -60,7 +60,7 @@ importcmd opts@CliOpts{rawopts_=rawopts,inputopts_=iopts} j = do printf "; would import %d new transactions from %s:\n\n" (length newts) inputstr -- TODO how to force output here ? -- length (jtxns newj) `seq` print' opts{rawopts_=("explicit",""):rawopts} newj - mapM_ (T.putStr . showTransaction) newts + mapM_ (TIO.putStr . showTransaction) newts newts | catchup -> do printf "marked %s as caught up, skipping %d unimported transactions\n\n" inputstr (length newts) newts -> do diff --git a/hledger/Hledger/Cli/Commands/Notes.hs b/hledger/Hledger/Cli/Commands/Notes.hs index 26493cbdc03a..c1b82d99e3d9 100644 --- a/hledger/Hledger/Cli/Commands/Notes.hs +++ b/hledger/Hledger/Cli/Commands/Notes.hs @@ -16,7 +16,7 @@ module Hledger.Cli.Commands.Notes ( ) where import Data.List.Extra (nubSort) -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions @@ -35,4 +35,4 @@ notes :: CliOpts -> Journal -> IO () notes CliOpts{reportspec_=rspec} j = do let ts = entriesReport rspec j notes = nubSort $ map transactionNote ts - mapM_ T.putStrLn notes + mapM_ TIO.putStrLn notes diff --git a/hledger/Hledger/Cli/Commands/Payees.hs b/hledger/Hledger/Cli/Commands/Payees.hs index d72537794dad..6c701c5d6212 100644 --- a/hledger/Hledger/Cli/Commands/Payees.hs +++ b/hledger/Hledger/Cli/Commands/Payees.hs @@ -15,7 +15,7 @@ module Hledger.Cli.Commands.Payees ( ) where import qualified Data.Set as S -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import System.Console.CmdArgs.Explicit as C import Hledger @@ -45,4 +45,4 @@ payees CliOpts{rawopts_=rawopts, reportspec_=ReportSpec{_rsQuery=query}} j = do if | declared && not used -> matcheddeclaredpayees | not declared && used -> matchedusedpayees | otherwise -> matcheddeclaredpayees <> matchedusedpayees - mapM_ T.putStrLn payees + mapM_ TIO.putStrLn payees diff --git a/hledger/Hledger/Cli/Commands/Prices.hs b/hledger/Hledger/Cli/Commands/Prices.hs index 132a4446ff77..9601468b670a 100755 --- a/hledger/Hledger/Cli/Commands/Prices.hs +++ b/hledger/Hledger/Cli/Commands/Prices.hs @@ -10,7 +10,7 @@ where import qualified Data.Map as M import Data.List import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions import System.Console.CmdArgs.Explicit @@ -45,7 +45,7 @@ prices opts j = do ++ ifBoolOpt "infer-market-prices" cprices ++ ifBoolOpt "infer-reverse-prices" rcprices -- TODO: shouldn't this show reversed P prices also ? valuation will use them - mapM_ (T.putStrLn . showPriceDirective) $ + mapM_ (TIO.putStrLn . showPriceDirective) $ sortOn pddate $ filter (matchesPriceDirective q) $ allprices diff --git a/hledger/Hledger/Cli/Commands/Print.hs b/hledger/Hledger/Cli/Commands/Print.hs index d32a3afb8dcf..e4d066bf48b1 100644 --- a/hledger/Hledger/Cli/Commands/Print.hs +++ b/hledger/Hledger/Cli/Commands/Print.hs @@ -18,7 +18,7 @@ where import Data.Text (Text) import Data.List (intersperse) import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStr, putStrLn) -- Only putStr and friends are safe import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB import Lens.Micro ((^.), _Just, has) @@ -206,5 +206,5 @@ postingToCSV p = printMatch :: CliOpts -> Journal -> Text -> IO () printMatch opts j desc = do case journalSimilarTransaction opts j desc of - Nothing -> putStrLn "no matches found." - Just t -> T.putStr $ showTransaction t + Nothing -> TIO.putStrLn "no matches found." + Just t -> TIO.putStr $ showTransaction t diff --git a/hledger/Hledger/Cli/Commands/Registermatch.hs b/hledger/Hledger/Cli/Commands/Registermatch.hs index 0465653ed6b5..54ce80d5e8bb 100755 --- a/hledger/Hledger/Cli/Commands/Registermatch.hs +++ b/hledger/Hledger/Cli/Commands/Registermatch.hs @@ -10,7 +10,7 @@ where import Data.Char (toUpper) import Data.List import qualified Data.Text as T -import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.Lazy.IO as TLIO (putStr, putStrLn) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions import Hledger.Cli.Commands.Register @@ -28,8 +28,8 @@ registermatch opts@CliOpts{rawopts_=rawopts,reportspec_=rspec} j = [desc] -> do let ps = [p | (_,_,_,p,_) <- postingsReport rspec j] case similarPosting ps desc of - Nothing -> putStrLn "no matches found." - Just p -> TL.putStr $ postingsReportAsText opts [pri] + Nothing -> TLIO.putStrLn "no matches found." + Just p -> TLIO.putStr $ postingsReportAsText opts [pri] where pri = (Just (postingDate p) ,Nothing ,tdescription <$> ptransaction p diff --git a/hledger/Hledger/Cli/Commands/Rewrite.hs b/hledger/Hledger/Cli/Commands/Rewrite.hs index 45a1e5617777..51d951a76c66 100755 --- a/hledger/Hledger/Cli/Commands/Rewrite.hs +++ b/hledger/Hledger/Cli/Commands/Rewrite.hs @@ -14,7 +14,7 @@ import Data.Functor.Identity import Data.List (sortOn, foldl') import Data.Text (Text) import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStr) -- Only putStr and friends are safe import Hledger import Hledger.Cli.CliOptions import Hledger.Cli.Commands.Print @@ -66,7 +66,7 @@ printOrDiff opts diffOutput :: Journal -> Journal -> IO () diffOutput j j' = do let changed = [(originalTransaction t, originalTransaction t') | (t, t') <- zip (jtxns j) (jtxns j'), t /= t'] - T.putStr $ renderPatch $ map (uncurry $ diffTxn j) changed + TIO.putStr $ renderPatch $ map (uncurry $ diffTxn j) changed type Chunk = (SourcePos, [DiffLine Text]) diff --git a/hledger/Hledger/Cli/Commands/Roi.hs b/hledger/Hledger/Cli/Commands/Roi.hs index 6a9c62115745..5b4104953bd8 100644 --- a/hledger/Hledger/Cli/Commands/Roi.hs +++ b/hledger/Hledger/Cli/Commands/Roi.hs @@ -25,7 +25,7 @@ import Data.List import Numeric.RootFinding import Data.Decimal import qualified Data.Text as T -import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.Lazy.IO as TLIO (putStr, putStrLn) -- Only putStr and friends are safe import System.Console.CmdArgs.Explicit as CmdArgs import Text.Tabular.AsciiWide as Tab @@ -85,7 +85,7 @@ roi CliOpts{rawopts_=rawopts, reportspec_=rspec@ReportSpec{_rsReportOpts=ReportO trans = dbg3 "investments" $ jtxns filteredj when (null trans) $ do - putStrLn "No relevant transactions found. Check your investments query" + TLIO.putStrLn "No relevant transactions found. Check your investments query" exitFailure let spans = snd $ reportSpan filteredj rspec @@ -146,7 +146,7 @@ roi CliOpts{rawopts_=rawopts, reportspec_=rspec@ReportSpec{_rsReportOpts=ReportO , Tab.Group Tab.SingleLine [Header "IRR", Header "TWR"]]) tableBody - TL.putStrLn $ Tab.render prettyTables id id id table + TLIO.putStrLn $ Tab.render prettyTables id id id table timeWeightedReturn showCashFlow prettyTables investmentsQuery trans mixedAmountValue (OneSpan spanBegin spanEnd valueBeforeAmt valueAfter cashFlow pnl) = do let valueBefore = unMix valueBeforeAmt @@ -229,7 +229,7 @@ timeWeightedReturn showCashFlow prettyTables investmentsQuery trans mixedAmountV unitPrices = add initialUnitPrice unitPrices' unitBalances = add initialUnits unitBalances' - TL.putStr $ Tab.render prettyTables id id T.pack + TLIO.putStr $ Tab.render prettyTables id id T.pack (Table (Tab.Group NoLine (map (Header . showDate) dates)) (Tab.Group DoubleLine [ Tab.Group Tab.SingleLine [Tab.Header "Portfolio value", Tab.Header "Unit balance"] @@ -259,7 +259,7 @@ internalRateOfReturn showCashFlow prettyTables (OneSpan spanBegin spanEnd valueB when showCashFlow $ do printf "\nIRR cash flow for %s - %s\n" (showDate spanBegin) (showDate (addDays (-1) spanEnd)) let (dates, amounts) = unzip totalCF - TL.putStrLn $ Tab.render prettyTables id id id + TLIO.putStrLn $ Tab.render prettyTables id id id (Table (Tab.Group Tab.NoLine (map (Header . showDate) dates)) (Tab.Group Tab.SingleLine [Header "Amount"]) diff --git a/hledger/Hledger/Cli/Commands/Tags.hs b/hledger/Hledger/Cli/Commands/Tags.hs index b61f72a5f634..e95204c3d5f9 100755 --- a/hledger/Hledger/Cli/Commands/Tags.hs +++ b/hledger/Hledger/Cli/Commands/Tags.hs @@ -10,7 +10,7 @@ where import qualified Control.Monad.Fail as Fail import Data.List.Extra (nubSort) import qualified Data.Text as T -import qualified Data.Text.IO as T +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe import Safe import System.Console.CmdArgs.Explicit as C import Hledger @@ -48,4 +48,4 @@ tags CliOpts{rawopts_=rawopts,reportspec_=rspec} j = do , let r = if values then v else t , not (values && T.null v && not empty) ] - mapM_ T.putStrLn tagsorvalues + mapM_ TIO.putStrLn tagsorvalues diff --git a/hledger/Hledger/Cli/Main.hs b/hledger/Hledger/Cli/Main.hs index b8e42d71ecb9..99d42883f6ba 100644 --- a/hledger/Hledger/Cli/Main.hs +++ b/hledger/Hledger/Cli/Main.hs @@ -41,17 +41,18 @@ etc. module Hledger.Cli.Main where import Data.Char (isDigit) -import Data.List -import Safe +import Data.List (isPrefixOf) +import Data.Time.Clock.POSIX (getPOSIXTime) +import Main.Utf8 (withUtf8) +import Safe (headDef, headMay) import qualified System.Console.CmdArgs.Explicit as C -import System.Environment -import System.Exit -import System.FilePath -import System.Process -import Text.Printf +import System.Environment (getArgs) +import System.Exit (exitFailure, exitWith) +import System.FilePath (dropExtension) +import System.Process (system) +import Text.Printf (printf) import Hledger.Cli -import Data.Time.Clock.POSIX (getPOSIXTime) -- | The overall cmdargs mode describing hledger's command-line options and subcommands. @@ -96,7 +97,7 @@ mainmode addons = defMode { -- | Let's go! main :: IO () -main = do +main = withUtf8 $ do progstarttime <- getPOSIXTime -- Choose and run the appropriate internal or external command based diff --git a/hledger/Hledger/Cli/Utils.hs b/hledger/Hledger/Cli/Utils.hs index 60835ebf4506..9854d636685f 100644 --- a/hledger/Hledger/Cli/Utils.hs +++ b/hledger/Hledger/Cli/Utils.hs @@ -29,15 +29,19 @@ module Hledger.Cli.Utils tests_Cli_Utils, ) where -import Control.Exception as C +import Prelude hiding (putStr, putStrLn, writeFile) + +import Control.Exception as C import Data.List import Data.Maybe import qualified Data.Text as T -import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.Builder as TB -import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.IO as TIO (putStrLn) -- Only putStr and friends are safe +import qualified Data.Text.IO.Utf8 as TIO +import qualified Data.Text.Lazy.IO as TLIO (putStr) -- Only putStr and friends are safe +import qualified Data.Text.Lazy.IO.Utf8 as TLIO import Data.Time (Day) import Data.Time.Clock.POSIX (POSIXTime, utcTimeToPOSIXSeconds) import Lens.Micro ((^.)) @@ -110,9 +114,7 @@ anonymiseByOpts opts = -- | Write some output to stdout or to a file selected by --output-file. -- If the file exists it will be overwritten. writeOutput :: CliOpts -> String -> IO () -writeOutput opts s = do - f <- outputFileFromOpts opts - (maybe putStr writeFile f) s +writeOutput opts = writeOutputLazyText opts . TL.pack -- | Write some output to stdout or to a file selected by --output-file. -- If the file exists it will be overwritten. This function operates on Lazy @@ -120,7 +122,7 @@ writeOutput opts s = do writeOutputLazyText :: CliOpts -> TL.Text -> IO () writeOutputLazyText opts s = do f <- outputFileFromOpts opts - (maybe TL.putStr TL.writeFile f) s + (maybe TLIO.putStr TLIO.writeFile f) s -- -- | Get a journal from the given string and options, or throw an error. -- readJournal :: CliOpts -> String -> IO Journal @@ -189,8 +191,8 @@ openBrowserOn u = trybrowsers browsers u ExitSuccess -> return ExitSuccess ExitFailure _ -> trybrowsers bs u trybrowsers [] u = do - putStrLn $ printf "Could not start a web browser (tried: %s)" $ intercalate ", " browsers - putStrLn $ printf "Please open your browser and visit %s" u + TIO.putStrLn . T.pack $ "Could not start a web browser (tried: " <> intercalate ", " browsers <> ")" + TIO.putStrLn . T.pack $ "Please open your browser and visit " <> u return $ ExitFailure 127 browsers | os=="darwin" = ["open"] | os=="mingw32" = ["c:/Program Files/Mozilla Firefox/firefox.exe"] @@ -217,12 +219,12 @@ writeFileWithBackupIfChanged :: FilePath -> T.Text -> IO Bool writeFileWithBackupIfChanged f t = do s <- readFilePortably f if t == s then return False - else backUpFile f >> T.writeFile f t >> return True + else backUpFile f >> TIO.writeFile f t >> return True -- | Back up this file with a (incrementing) numbered suffix, then -- overwrite it with this new text, or give an error. writeFileWithBackup :: FilePath -> String -> IO () -writeFileWithBackup f t = backUpFile f >> writeFile f t +writeFileWithBackup f t = backUpFile f >> TIO.writeFile f (T.pack t) readFileStrictly :: FilePath -> IO T.Text readFileStrictly f = readFilePortably f >>= \s -> C.evaluate (T.length s) >> return s diff --git a/hledger/hledger.cabal b/hledger/hledger.cabal index 763a3b9e6bf6..4e7ef866f551 100644 --- a/hledger/hledger.cabal +++ b/hledger/hledger.cabal @@ -1,6 +1,6 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.34.4. +-- This file has been generated from package.yaml by hpack version 0.34.6. -- -- see: https://github.com/sol/hpack @@ -174,6 +174,7 @@ library , unordered-containers , utf8-string >=0.3.5 , utility-ht >=0.0.13 + , with-utf8 >=1.0.0 , wizards >=1.0 if (!(os(windows))) && (flag(terminfo)) build-depends: @@ -223,6 +224,7 @@ executable hledger , unordered-containers , utf8-string >=0.3.5 , utility-ht >=0.0.13 + , with-utf8 >=1.0.0 , wizards >=1.0 if (!(os(windows))) && (flag(terminfo)) build-depends: @@ -273,6 +275,7 @@ test-suite unittest , unordered-containers , utf8-string >=0.3.5 , utility-ht >=0.0.13 + , with-utf8 >=1.0.0 , wizards >=1.0 if (!(os(windows))) && (flag(terminfo)) build-depends: @@ -322,6 +325,7 @@ benchmark bench , unordered-containers , utf8-string >=0.3.5 , utility-ht >=0.0.13 + , with-utf8 >=1.0.0 , wizards >=1.0 buildable: False if (!(os(windows))) && (flag(terminfo)) diff --git a/hledger/package.yaml b/hledger/package.yaml index 5a791d8b9ce1..ce673b0f500b 100644 --- a/hledger/package.yaml +++ b/hledger/package.yaml @@ -129,6 +129,7 @@ dependencies: - unordered-containers - utf8-string >=0.3.5 - utility-ht >=0.0.13 +- with-utf8 >=1.0.0 - wizards >=1.0 when: diff --git a/stack8.6.yaml b/stack8.6.yaml index 1285e761eaa0..d2119f718dc9 100644 --- a/stack8.6.yaml +++ b/stack8.6.yaml @@ -15,8 +15,6 @@ packages: extra-deps: # for Shake.hs (regex doesn't support base-compat-0.11): - regex-1.0.2.0@rev:1 -- doclayout-0.3.1.1 -- emojis-0.1.2 # for testing base-compat 0.11 compatibility (mutually exclusive with the above): # - aeson-1.4.6.0 # - aeson-compat-0.3.9 @@ -29,6 +27,11 @@ extra-deps: - prettyprinter-1.7.0 - prettyprinter-ansi-terminal-1.1.2 - doctest-0.18.1 +- doclayout-0.3.1.1 +- emojis-0.1.2 +- with-utf8-1.0.2.3 +- th-compat-0.1.3 +- th-env-0.1.0.3 # for hledger: - githash-0.1.4.0 # for hledger-ui: