From 5cc923fb6bbf7715bc847fdcde87c1dffca20fe7 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sun, 1 Sep 2024 17:29:15 +0200 Subject: [PATCH 01/12] Skip already indexed documents --- go.mod | 14 ++++- go.sum | 24 +++++++++ internal/index/bleve_write.go | 24 +++++++-- .../webserver/controller/document/download.go | 54 +++++++++++++++---- .../webserver/controller/document/search.go | 2 + .../webserver/embedded/views/document.html | 9 ++++ .../embedded/views/partials/actions.html | 9 ++++ 7 files changed, 122 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 88e040d5..83621b6a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/svera/coreander/v4 -go 1.21 +go 1.21.0 + +toolchain go1.21.1 require ( github.com/blevesearch/bleve/v2 v2.4.0 @@ -27,9 +29,11 @@ require ( require ( github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/beevik/etree v1.4.1 // indirect github.com/blevesearch/go-faiss v1.0.13 // indirect github.com/blevesearch/zapx/v16 v16.0.12 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/geek1011/kepubify v2.3.2+incompatible // indirect github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/gofiber/template v1.8.3 // indirect github.com/gofiber/utils v1.1.0 // indirect @@ -37,14 +41,21 @@ require ( github.com/hhrutter/lzw v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/kr/smartypants v0.1.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-zglob v0.0.5 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/pgaskin/kepubify v2.3.2+incompatible // indirect + github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523 // indirect + github.com/pgaskin/kepubify/_/html v0.0.0-20211223234002-6ee2cc632cdc // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/tinylib/msgp v1.1.9 // indirect + golang.org/x/sync v0.7.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect modernc.org/libc v1.49.3 // indirect @@ -90,6 +101,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/pgaskin/kepubify/v4 v4.0.4 github.com/pirmd/epub v0.3.0 github.com/rivo/uniseg v0.4.7 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect diff --git a/go.sum b/go.sum index 2be4c03b..113c2ae5 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,10 @@ github.com/antzucaro/matchr v0.0.0-20210222213004-b04723ef80f0 h1:R/qAiUxFT3mNgQ github.com/antzucaro/matchr v0.0.0-20210222213004-b04723ef80f0/go.mod h1:v3ZDlfVAL1OrkKHbGSFFK60k0/7hruHPDq2XMs9Gu6U= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b/go.mod h1:obBQGGIFbbv9KWg92Qu9UHeD94JXmHD1jovY/z6I3O8= +github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= +github.com/beevik/etree v1.4.1 h1:PmQJDDYahBGNKDcpdX8uPy1xRCwoCGVUiW669MEirVI= +github.com/beevik/etree v1.4.1/go.mod h1:gPNJNaBGVZ9AwsidazFZyygnd+0pAU38N4D+WemwKNs= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= @@ -67,6 +71,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/flotzilla/pdf_parser v0.1.96 h1:SlgvO7NZqFzhBO+o6X1u7rUYjhv+81V3dYQF+LTfGOE= github.com/flotzilla/pdf_parser v0.1.96/go.mod h1:/CPB1OWEeFqRbtnFWXgArmOnA3u7smVHxr5dFy4U6Nk= +github.com/geek1011/kepubify v2.3.2+incompatible h1:G1dAwpTpSHN79/bOqQ64SjkllYPhpp25kUkJ95WNFnA= +github.com/geek1011/kepubify v2.3.2+incompatible/go.mod h1:xMWLgn5FQSh6oNcq//MwSwPf2WgyQq6T+fY9nSn9+Cs= github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= @@ -104,6 +110,7 @@ github.com/gosimple/slug v1.14.0 h1:RtTL/71mJNDfpUbCOmnf/XFkzKRtD6wL6Uy+3akm4Es= github.com/gosimple/slug v1.14.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0= github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo= github.com/hhrutter/tiff v1.0.1 h1:MIus8caHU5U6823gx7C6jrfoEvfSTGtEFRiM8/LOzC0= @@ -122,6 +129,8 @@ github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/smartypants v0.1.0 h1:Sn8hn5XrY+uXrxSWUdcr621Gfpk11mOGGVs4XX06kEw= +github.com/kr/smartypants v0.1.0/go.mod h1:EcTX9ge+SWNaGwbQvHwNICsMGavh98FLUqyOWFr+j9c= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -138,6 +147,9 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-zglob v0.0.5 h1:LKgpZXHg94zoBPDbb6aeiCs4SiQSS8otal4JnzkIvMc= +github.com/mattn/go-zglob v0.0.5/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -153,6 +165,15 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pdfcpu/pdfcpu v0.7.0 h1:cd7/z7hAyyDuzdciKfNZyQ3TYreJza2DsuPdIHYURcA= github.com/pdfcpu/pdfcpu v0.7.0/go.mod h1:kmpD0rk8YnZj0l3qSeGBlAB+XszHUgNv//ORH/E7EYo= +github.com/pgaskin/kepubify v2.3.2+incompatible h1:0wWWxop5T5O+p6tlZ59saC9hZYtkrXaKWLiTXmUNHHg= +github.com/pgaskin/kepubify v2.3.2+incompatible/go.mod h1:vQR3SJUwNyKStXpUPsVcCjBZtjZ1TgYtgKb8jHMyEDg= +github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523 h1:pYGj3rKTy+TDs5Z707kT+ztjoIDCy76lc2UPkZocAFM= +github.com/pgaskin/kepubify/_/go116-zip.go117 v0.0.0-20210611152744-2d89b3182523/go.mod h1:FNMbV/TSSnhqyzjq8jsS+VD0o/gwpuCH0dh8G1uQ/fw= +github.com/pgaskin/kepubify/_/html v0.0.0-20211223234002-6ee2cc632cdc h1:mJk4TIXTO+JmxgHJ5iyil42PLQJWkyaKB/qNcjJU6h4= +github.com/pgaskin/kepubify/_/html v0.0.0-20211223234002-6ee2cc632cdc/go.mod h1:fxzoIpMFAReNKunZ+ttVbf3hNVrJGtrSZMI4olZizbs= +github.com/pgaskin/kepubify/v4 v4.0.4 h1:8ePyepo4eRNSmeDs5MdJLJtit+zxmK36wILmGcvpccU= +github.com/pgaskin/kepubify/v4 v4.0.4/go.mod h1:wzUdFNYW2uZh2xfHDuzNRRUO4WqV+y99UBxVd3rBTus= +github.com/pgaskin/koboutils/v2 v2.1.2-0.20220306004009-a07e72ebae42/go.mod h1:wTzkDIlsxmUyfwfspGcm0Ap+HOxSUYV0S8kMYrf+0gM= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= @@ -181,6 +202,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -235,6 +258,7 @@ golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index 1cee28eb..748f2b91 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -52,6 +52,9 @@ func (b *BleveIndexer) AddLibrary(batchSize int) error { languages := []string{} b.indexStartTime = float64(time.Now().UnixNano()) e := afero.Walk(b.fs, b.libraryPath, func(fullPath string, f os.FileInfo, err error) error { + if b.isAlreadyIndexed(fullPath) { + return nil + } ext := strings.ToLower(filepath.Ext(fullPath)) if _, ok := b.reader[ext]; !ok { return nil @@ -80,13 +83,26 @@ func (b *BleveIndexer) AddLibrary(batchSize int) error { } return nil }) + if b.indexedDocuments > 0 { + batch.SetInternal(internalLanguages, []byte(strings.Join(languages, ","))) + b.idx.Batch(batch) + } b.indexStartTime = 0 b.indexedDocuments = 0 - batch.SetInternal(internalLanguages, []byte(strings.Join(languages, ","))) - b.idx.Batch(batch) return e } +func (b *BleveIndexer) isAlreadyIndexed(fullPath string) bool { + doc, err := b.idx.Document(b.id(fullPath)) + if err != nil { + log.Fatalln(err) + } + if doc == nil { + return false + } + return true +} + func addLanguage(lang string, languages []string) []string { if !slices.Contains(languages, defaultAnalyzer) && lang == "" { return append(languages, defaultAnalyzer) @@ -117,7 +133,7 @@ func (b *BleveIndexer) createDocument(meta metadata.Metadata, fullPath string, b SubjectsEq: make([]string, len(meta.Subjects)), } - document.ID = b.ID(document, fullPath) + document.ID = b.id(fullPath) document.Slug = b.Slug(document, batchSlugs) copy(document.AuthorsEq, meta.Authors) for i := range document.AuthorsEq { @@ -161,7 +177,7 @@ func (b *BleveIndexer) Slug(document DocumentWrite, batchSlugs map[string]struct } } -func (b *BleveIndexer) ID(meta DocumentWrite, file string) string { +func (b *BleveIndexer) id(file string) string { ID := strings.ReplaceAll(file, b.libraryPath, "") return strings.TrimPrefix(ID, string(filepath.Separator)) } diff --git a/internal/webserver/controller/document/download.go b/internal/webserver/controller/document/download.go index 7c997900..ea6e139e 100644 --- a/internal/webserver/controller/document/download.go +++ b/internal/webserver/controller/document/download.go @@ -1,6 +1,9 @@ package document import ( + "archive/zip" + "bytes" + "context" "fmt" "io" "os" @@ -8,9 +11,16 @@ import ( "strings" "github.com/gofiber/fiber/v2" + "github.com/pgaskin/kepubify/v4/kepub" ) func (d *Controller) Download(c *fiber.Ctx) error { + var ( + output []byte + err error + fileName string + ) + document, err := d.idx.Document(c.Params("slug")) if err != nil { return fiber.ErrBadRequest @@ -22,13 +32,23 @@ func (d *Controller) Download(c *fiber.Ctx) error { return fiber.ErrNotFound } - file, err := os.Open(fullPath) - if err != nil { - return fiber.ErrInternalServerError - } - contents, err := io.ReadAll(file) - if err != nil { - return fiber.ErrInternalServerError + if c.Query("format") == "kepub" { + output, err = kepubify(fullPath) + if err != nil { + return fiber.ErrInternalServerError + } + fileName = strings.TrimSuffix(filepath.Base(fullPath), filepath.Ext(fullPath)) + fileName = fileName + ".kepub.epub" + } else { + file, err := os.Open(fullPath) + if err != nil { + return fiber.ErrInternalServerError + } + output, err = io.ReadAll(file) + if err != nil { + return fiber.ErrInternalServerError + } + fileName = filepath.Base(document.ID) } ext := strings.ToLower(filepath.Ext(document.ID)) @@ -39,7 +59,23 @@ func (d *Controller) Download(c *fiber.Ctx) error { c.Response().Header.Set(fiber.HeaderContentType, "application/pdf") } - c.Response().Header.Set(fiber.HeaderContentDisposition, fmt.Sprintf("inline; filename=\"%s\"", filepath.Base(document.ID))) - c.Response().BodyWriter().Write(contents) + c.Response().Header.Set(fiber.HeaderContentDisposition, fmt.Sprintf("inline; filename=\"%s\"", fileName)) + c.Response().BodyWriter().Write(output) return nil } + +func kepubify(fullPath string) ([]byte, error) { + output := bytes.NewBuffer(nil) + r, err := zip.OpenReader(fullPath) + if err != nil { + return nil, fiber.ErrInternalServerError + } + defer r.Close() + + err = kepub.NewConverter().Convert(context.Background(), output, r) + if err != nil { + return nil, fiber.ErrInternalServerError + } + + return output.Bytes(), nil +} diff --git a/internal/webserver/controller/document/search.go b/internal/webserver/controller/document/search.go index dc91b140..a21b8f41 100644 --- a/internal/webserver/controller/document/search.go +++ b/internal/webserver/controller/document/search.go @@ -1,6 +1,7 @@ package document import ( + "log" "strconv" "github.com/gofiber/fiber/v2" @@ -35,6 +36,7 @@ func (d *Controller) Search(c *fiber.Ctx) error { if keywords := c.Query("search"); keywords != "" { if searchResults, err = d.idx.Search(keywords, page, model.ResultsPerPage); err != nil { + log.Println(err) return fiber.ErrInternalServerError } diff --git a/internal/webserver/embedded/views/document.html b/internal/webserver/embedded/views/document.html index 2f17f35f..aa163e1f 100644 --- a/internal/webserver/embedded/views/document.html +++ b/internal/webserver/embedded/views/document.html @@ -40,6 +40,15 @@

{{.Document.Type}} + + + + + +   {{t .Lang "Download"}} + KEPUB + + {{if and (.Session) (ne .Session.Name "")}} diff --git a/internal/webserver/embedded/views/partials/actions.html b/internal/webserver/embedded/views/partials/actions.html index c5f24fe5..e6096135 100644 --- a/internal/webserver/embedded/views/partials/actions.html +++ b/internal/webserver/embedded/views/partials/actions.html @@ -16,6 +16,15 @@ {{.Document.Type}} +
  • + + + + +   {{t .Lang "Download"}} + KEPUB + +
    • {{if and (.Session) (ne .Session.Name "")}}
  • From cc38f3afa25ba721de833497251047a1f3924c56 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 2 Sep 2024 15:11:10 +0200 Subject: [PATCH 02/12] Fixed language filters not being added in already indexed docs --- internal/index/bleve_write.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index 748f2b91..b92d701b 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -10,6 +10,7 @@ import ( "strings" "time" + index "github.com/blevesearch/bleve_index_api" "github.com/gosimple/slug" "github.com/spf13/afero" "github.com/svera/coreander/v4/internal/metadata" @@ -45,14 +46,17 @@ func (b *BleveIndexer) RemoveFile(file string) error { return nil } -// AddLibrary scans for documents and adds them to the index in batches of +// AddLibrary scans for documents and adds them to the index in batches of if they +// haven't been previously indexed func (b *BleveIndexer) AddLibrary(batchSize int) error { batch := b.idx.NewBatch() batchSlugs := make(map[string]struct{}, batchSize) languages := []string{} b.indexStartTime = float64(time.Now().UnixNano()) e := afero.Walk(b.fs, b.libraryPath, func(fullPath string, f os.FileInfo, err error) error { - if b.isAlreadyIndexed(fullPath) { + if indexed, lang := b.isAlreadyIndexed(fullPath); indexed { + b.indexedDocuments += 1 + languages = addLanguage(lang, languages) return nil } ext := strings.ToLower(filepath.Ext(fullPath)) @@ -83,24 +87,31 @@ func (b *BleveIndexer) AddLibrary(batchSize int) error { } return nil }) - if b.indexedDocuments > 0 { + if len(languages) > 0 { batch.SetInternal(internalLanguages, []byte(strings.Join(languages, ","))) - b.idx.Batch(batch) } + b.idx.Batch(batch) b.indexStartTime = 0 b.indexedDocuments = 0 return e } -func (b *BleveIndexer) isAlreadyIndexed(fullPath string) bool { +func (b *BleveIndexer) isAlreadyIndexed(fullPath string) (bool, string) { doc, err := b.idx.Document(b.id(fullPath)) if err != nil { log.Fatalln(err) } if doc == nil { - return false + return false, "" } - return true + lang := "" + doc.VisitFields(func(f index.Field) { + if f.Name() == "Language" { + lang = string(f.Value()) + return + } + }) + return true, lang } func addLanguage(lang string, languages []string) []string { From c443408ea27b232483673eb50ffaa554bc1c0f89 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Sep 2024 19:44:58 +0200 Subject: [PATCH 03/12] Added forceIndexing parameter --- README.md | 3 ++- config.go | 4 ++-- internal/index/bleve_test.go | 2 +- internal/index/bleve_write.go | 4 ++-- internal/webserver/webserver_test.go | 2 +- main.go | 12 ++++-------- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 312403d9..a517b2b7 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A personal documents server, Coreander indexes the documents (EPUBs and PDFs wit * Read indexed epubs and PDFs from Coreander's interface thanks to [foliate-js](https://github.com/johnfactotum/foliate-js). * Restrictable access only to registered users. * Upload documents through the web interface. +* Download as kepub (epub for Kobo devices) converted on the fly thanks to [Kepubify](https://github.com/pgaskin/kepubify). ## Installation @@ -97,7 +98,7 @@ On first run, Coreander creates an admin user with the following credentials: * `PORT`: Port number in which the webserver listens for requests. Defaults to 3000. * `BATCH_SIZE`: Number of documents persisted by the indexer in one write operation. Defaults to 100. * `COVER_MAX_WIDTH`: Maximum horizontal size for documents cover thumbnails in pixels. Defaults to 600. -* `SKIP_INDEXING`: Whether to bypass the indexing process or not. +* `FORCE_INDEXING`: Whether to force indexing already indexed documents or not. Defaults to false. * `SMTP_SERVER`: Address of the send mail server. * `SMTP_PORT`: Port number of the send mail server. Defaults to 587. * `SMTP_USER`: User to authenticate against the SMTP server. diff --git a/config.go b/config.go index 83ba03f0..b27a385f 100644 --- a/config.go +++ b/config.go @@ -12,8 +12,8 @@ type Config struct { BatchSize int `env:"BATCH_SIZE" env-default:"100"` // CoverMaxWidth sets the maximum horizontal size for documents cover thumbnails in pixels CoverMaxWidth int `env:"COVER_MAX_WIDTH" env-default:"600"` - // SkipIndexing signals whether to bypass the indexing process or not - SkipIndexing bool `env:"SKIP_INDEXING" env-default:"false"` + // ForceIndexing signals whether to force indexing already indexed documents or not + ForceIndexing bool `env:"FORCE_INDEXING" env-default:"false"` // SmtpServer points to the address of the send mail server SmtpServer string `env:"SMTP_SERVER"` // SmtpPort defines the port in which the mail server listens for requests diff --git a/internal/index/bleve_test.go b/internal/index/bleve_test.go index 26b71843..456fb909 100644 --- a/internal/index/bleve_test.go +++ b/internal/index/bleve_test.go @@ -35,7 +35,7 @@ func TestIndexAndSearch(t *testing.T) { appFS.MkdirAll("lib", 0755) afero.WriteFile(appFS, tcase.filename, []byte(""), 0644) - err = idx.AddLibrary(1) + err = idx.AddLibrary(1, true) if err != nil { t.Errorf("Error indexing: %s", err.Error()) } diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index b92d701b..a5b3d181 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -48,13 +48,13 @@ func (b *BleveIndexer) RemoveFile(file string) error { // AddLibrary scans for documents and adds them to the index in batches of if they // haven't been previously indexed -func (b *BleveIndexer) AddLibrary(batchSize int) error { +func (b *BleveIndexer) AddLibrary(batchSize int, forceIndexing bool) error { batch := b.idx.NewBatch() batchSlugs := make(map[string]struct{}, batchSize) languages := []string{} b.indexStartTime = float64(time.Now().UnixNano()) e := afero.Walk(b.fs, b.libraryPath, func(fullPath string, f os.FileInfo, err error) error { - if indexed, lang := b.isAlreadyIndexed(fullPath); indexed { + if indexed, lang := b.isAlreadyIndexed(fullPath); indexed && !forceIndexing { b.indexedDocuments += 1 languages = addLanguage(lang, languages) return nil diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 91c441ea..e5cf53e5 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -77,7 +77,7 @@ func bootstrapApp(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserve idx = index.NewBleve(indexFile, appFs, webserverConfig.LibraryPath, metadataReaders) } - err = idx.AddLibrary(100) + err = idx.AddLibrary(100, true) if err != nil { log.Fatal(err) } diff --git a/main.go b/main.go index c1dade71..16599f90 100644 --- a/main.go +++ b/main.go @@ -81,11 +81,7 @@ func migrateDir() { func main() { defer idx.Close() - if !cfg.SkipIndexing { - go startIndex(idx, appFs, cfg.BatchSize, cfg.LibPath) - } else { - go fileWatcher(idx, cfg.LibPath) - } + go startIndex(idx, appFs, cfg.BatchSize, cfg.LibPath) sender = &infrastructure.NoEmail{} if cfg.SmtpServer != "" && cfg.SmtpUser != "" && cfg.SmtpPassword != "" { @@ -133,7 +129,7 @@ func main() { func startIndex(idx *index.BleveIndexer, appFs afero.Fs, batchSize int, libPath string) { start := time.Now().Unix() log.Printf("Indexing documents at %s, this can take a while depending on the size of your library.", libPath) - err := idx.AddLibrary(batchSize) + err := idx.AddLibrary(batchSize, cfg.ForceIndexing) if err != nil { log.Fatal(err) } @@ -147,7 +143,7 @@ func getIndexFile(fs afero.Fs) bleve.Index { indexFile, err := bleve.Open(homeDir + indexPath) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No index found, creating a new one.") - cfg.SkipIndexing = false + cfg.ForceIndexing = false indexFile = index.Create(homeDir + indexPath) } version, err := indexFile.GetInternal([]byte("version")) @@ -159,7 +155,7 @@ func getIndexFile(fs afero.Fs) bleve.Index { if err = fs.RemoveAll(homeDir + indexPath); err != nil { log.Fatal(err) } - cfg.SkipIndexing = false + cfg.ForceIndexing = false indexFile = index.Create(homeDir + indexPath) } return indexFile From b581857b07b6bcc9b7da4378667a3f5a486a073b Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sun, 8 Sep 2024 16:55:40 +0200 Subject: [PATCH 04/12] Updated routes --- .../webserver/controller/document/send.go | 9 ++-- internal/webserver/embedded/js/send-email.js | 5 +- .../webserver/embedded/views/document.html | 10 ++-- .../embedded/views/partials/actions.html | 7 ++- .../embedded/views/partials/delete-modal.html | 2 +- .../embedded/views/partials/docs-list.html | 4 +- .../embedded/views/partials/related.html | 2 +- internal/webserver/embedded/views/reader.html | 2 +- .../webserver/embedded/views/users/index.html | 2 +- internal/webserver/highlights_test.go | 6 +-- internal/webserver/remove_document_test.go | 2 +- internal/webserver/routes.go | 48 ++++++++++--------- internal/webserver/send_document_test.go | 4 +- internal/webserver/user_management_test.go | 10 ++-- 14 files changed, 57 insertions(+), 56 deletions(-) diff --git a/internal/webserver/controller/document/send.go b/internal/webserver/controller/document/send.go index f8898882..ba6b6c14 100644 --- a/internal/webserver/controller/document/send.go +++ b/internal/webserver/controller/document/send.go @@ -1,6 +1,7 @@ package document import ( + "log" "net/mail" "os" "path/filepath" @@ -10,7 +11,8 @@ import ( ) func (d *Controller) Send(c *fiber.Ctx) error { - if strings.Trim(c.FormValue("slug"), " ") == "" { + slug := "" + if slug = strings.Trim(c.Params("slug"), " "); slug == "" { return fiber.ErrBadRequest } @@ -18,13 +20,14 @@ func (d *Controller) Send(c *fiber.Ctx) error { return fiber.ErrBadRequest } - document, err := d.idx.Document(c.FormValue("slug")) + document, err := d.idx.Document(slug) if err != nil { return fiber.ErrBadRequest } if _, err := os.Stat(filepath.Join(d.config.LibraryPath, document.ID)); err != nil { - return fiber.ErrBadRequest + log.Println(err) + return fiber.ErrInternalServerError } return d.sender.SendDocument(c.FormValue("email"), d.config.LibraryPath, document.ID) diff --git a/internal/webserver/embedded/js/send-email.js b/internal/webserver/embedded/js/send-email.js index 73dd487e..60860bb0 100644 --- a/internal/webserver/embedded/js/send-email.js +++ b/internal/webserver/embedded/js/send-email.js @@ -13,14 +13,13 @@ Array.from(forms).forEach(form => { submit.setAttribute("disabled", true); spinner.classList.remove("visually-hidden"); sendIcon.classList.add("visually-hidden"); - fetch('/send', { + fetch(form.getAttribute("action"), { method: "POST", headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ - 'email': form.elements[0].value, - 'slug': form.elements[1].value, + 'email': form.elements[0].value }) }) .then((response) => { diff --git a/internal/webserver/embedded/views/document.html b/internal/webserver/embedded/views/document.html index aa163e1f..9c5978cf 100644 --- a/internal/webserver/embedded/views/document.html +++ b/internal/webserver/embedded/views/document.html @@ -7,7 +7,7 @@
    - {{t $lang "\"%s\" cover" .Document.Title}} @@ -23,7 +23,7 @@

    - + @@ -31,7 +31,7 @@

      {{t .Lang "Read"}} - + @@ -40,7 +40,7 @@

    {{.Document.Type}} - + @@ -69,7 +69,7 @@

    {{t .Lang "Send to email unavailable, no email service configured"}}

    {{else}}

    • -
  • +
  • @@ -16,7 +16,7 @@ {{.Document.Type}}
    • -
  • +
  • @@ -49,7 +49,7 @@
  • {{t .Lang "Send to email unavailable, no email service configured"}}
    • {{else}}
  • - +
    - diff --git a/internal/webserver/embedded/views/partials/docs-list.html b/internal/webserver/embedded/views/partials/docs-list.html index 51860bd3..ee93a525 100644 --- a/internal/webserver/embedded/views/partials/docs-list.html +++ b/internal/webserver/embedded/views/partials/docs-list.html @@ -9,7 +9,7 @@
    - {{t $lang "\"%s\" cover" $document.Title}} + {{t $lang "\"%s\" cover" $document.Title}}
    {{$document.Title}}

    @@ -91,7 +91,7 @@

    {{t $lang "Unknown author"}}
    {{end}} {{end}}
    -{{template "partials/delete-modal" dict "Lang" $lang "Action" "/documents" "ModalHeader" "Delete document" "ModalBody" "Are you sure you want to delete this document?" "ModalErrorMessage" "There was an error deleting the document"}} +{{template "partials/delete-modal" dict "Lang" $lang "Action" "documents" "ModalHeader" "Delete document" "ModalBody" "Are you sure you want to delete this document?" "ModalErrorMessage" "There was an error deleting the document"}} diff --git a/internal/webserver/embedded/views/partials/related.html b/internal/webserver/embedded/views/partials/related.html index 93c895f5..f3a952ab 100644 --- a/internal/webserver/embedded/views/partials/related.html +++ b/internal/webserver/embedded/views/partials/related.html @@ -1,5 +1,5 @@
    - {{t .Lang "\"%s\" cover" .Document.Title}} + {{t .Lang "\"%s\" cover" .Document.Title}}
    {{.Document.Title}}
    {{if .Document.Authors}} diff --git a/internal/webserver/embedded/views/reader.html b/internal/webserver/embedded/views/reader.html index 67d5daf0..25f8cf63 100644 --- a/internal/webserver/embedded/views/reader.html +++ b/internal/webserver/embedded/views/reader.html @@ -13,7 +13,7 @@ - +
    diff --git a/internal/webserver/embedded/views/users/index.html b/internal/webserver/embedded/views/users/index.html index c463bbeb..8022a889 100644 --- a/internal/webserver/embedded/views/users/index.html +++ b/internal/webserver/embedded/views/users/index.html @@ -40,6 +40,6 @@

    {{t $lang "Users"}}

    {{template "partials/pagination" .}} {{end}} -{{template "partials/delete-modal" dict "Lang" $lang "Action" "/users" "ModalHeader" "Delete user" "ModalBody" "Are you sure you want to delete this user?" "ModalErrorMessage" "There was an error deleting the user, try again later"}} +{{template "partials/delete-modal" dict "Lang" $lang "Action" "users" "ModalHeader" "Delete user" "ModalBody" "Are you sure you want to delete this user?" "ModalErrorMessage" "There was an error deleting the user, try again later"}} diff --git a/internal/webserver/highlights_test.go b/internal/webserver/highlights_test.go index 140ca844..0ec178fe 100644 --- a/internal/webserver/highlights_test.go +++ b/internal/webserver/highlights_test.go @@ -112,7 +112,7 @@ func TestHighlights(t *testing.T) { t.Fatalf("Unexpected error: %v", err.Error()) } - _, err = deleteRequest(url.Values{}, adminCookie, app, "/documents/john-doe-test-epub", t) + _, err = deleteRequest(url.Values{}, adminCookie, app, "/en/documents/john-doe-test-epub", t) if err != nil { t.Fatalf("Unexpected error: %v", err.Error()) } @@ -150,7 +150,7 @@ func TestHighlights(t *testing.T) { t.Fatalf("Unexpected error: %v", err.Error()) } - _, err = deleteRequest(url.Values{}, adminCookie, app, fmt.Sprintf("/users/%s", regularUser.Username), t) + _, err = deleteRequest(url.Values{}, adminCookie, app, fmt.Sprintf("/en/users/%s", regularUser.Username), t) if err != nil { t.Fatalf("Unexpected error: %v", err.Error()) } @@ -165,7 +165,7 @@ func TestHighlights(t *testing.T) { func highlight(cookie *http.Cookie, app *fiber.App, slug string, method string, t *testing.T) (*http.Response, error) { t.Helper() - req, err := http.NewRequest(method, fmt.Sprintf("/highlights/%s", slug), nil) + req, err := http.NewRequest(method, fmt.Sprintf("/en/highlights/%s", slug), nil) if err != nil { return nil, err } diff --git a/internal/webserver/remove_document_test.go b/internal/webserver/remove_document_test.go index 7e29b8e8..3983f849 100644 --- a/internal/webserver/remove_document_test.go +++ b/internal/webserver/remove_document_test.go @@ -60,7 +60,7 @@ func TestRemoveDocument(t *testing.T) { t.Fatalf("Unexpected error: %v", err.Error()) } - response, err = deleteRequest(url.Values{}, cookie, app, fmt.Sprintf("/documents/%s", tcase.slug), t) + response, err = deleteRequest(url.Values{}, cookie, app, fmt.Sprintf("/en/documents/%s", tcase.slug), t) if err != nil { t.Fatalf("Unexpected error: %v", err.Error()) } diff --git a/internal/webserver/routes.go b/internal/webserver/routes.go index ac80a582..fcd0a7ad 100644 --- a/internal/webserver/routes.go +++ b/internal/webserver/routes.go @@ -11,9 +11,12 @@ import ( ) func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Sender, requireAuth bool) { - var allowIfNotLoggedIn = AllowIfNotLoggedIn(jwtSecret) - var alwaysRequireAuthentication = AlwaysRequireAuthentication(jwtSecret, sender) - var configurableAuthentication = ConfigurableAuthentication(jwtSecret, sender, requireAuth) + // Middlewares + var ( + allowIfNotLoggedIn = AllowIfNotLoggedIn(jwtSecret) + alwaysRequireAuthentication = AlwaysRequireAuthentication(jwtSecret, sender) + configurableAuthentication = ConfigurableAuthentication(jwtSecret, sender, requireAuth) + ) app.Use("/css", filesystem.New(filesystem.Config{ Root: http.FS(cssFS), @@ -54,35 +57,34 @@ func routes(app *fiber.App, controllers Controllers, jwtSecret []byte, sender Se usersGroup := langGroup.Group("/users", alwaysRequireAuthentication) - usersGroup.Get("/", alwaysRequireAuthentication, RequireAdmin, controllers.Users.List) - usersGroup.Get("/new", alwaysRequireAuthentication, RequireAdmin, controllers.Users.New) - usersGroup.Post("/", alwaysRequireAuthentication, RequireAdmin, controllers.Users.Create) - usersGroup.Get("/:username", alwaysRequireAuthentication, controllers.Users.Edit) - usersGroup.Put("/:username", alwaysRequireAuthentication, controllers.Users.Update) - app.Delete("/users/:username", alwaysRequireAuthentication, RequireAdmin, controllers.Users.Delete) + usersGroup.Get("/", RequireAdmin, controllers.Users.List) + usersGroup.Get("/new", RequireAdmin, controllers.Users.New) + usersGroup.Post("/", RequireAdmin, controllers.Users.Create) + usersGroup.Get("/:username", controllers.Users.Edit) + usersGroup.Put("/:username", controllers.Users.Update) + usersGroup.Delete("/:username", RequireAdmin, controllers.Users.Delete) - langGroup.Get("/highlights", alwaysRequireAuthentication, controllers.Highlights.List) - app.Post("/highlights/:slug", alwaysRequireAuthentication, controllers.Highlights.Create) - app.Delete("/highlights/:slug", alwaysRequireAuthentication, controllers.Highlights.Delete) - - app.Delete("/documents/:slug", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Delete) + highlightsGroup := langGroup.Group("/highlights", alwaysRequireAuthentication) + highlightsGroup.Get("/", controllers.Highlights.List) + highlightsGroup.Post("/:slug", controllers.Highlights.Create) + highlightsGroup.Delete("/:slug", controllers.Highlights.Delete) + docsGroup := langGroup.Group("/documents") langGroup.Get("/upload", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.UploadForm) - langGroup.Post("/documents", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Upload) + docsGroup.Post("/", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Upload) + docsGroup.Delete("/:slug", alwaysRequireAuthentication, RequireAdmin, controllers.Documents.Delete) // Authentication requirement is configurable for all routes below this middleware langGroup.Use(configurableAuthentication) app.Use(configurableAuthentication) - app.Get("/documents/:slug/cover", controllers.Documents.Cover) - langGroup.Get("/documents/:slug/read", controllers.Documents.Reader) - app.Get("/documents/:slug/download", controllers.Documents.Download) - - langGroup.Get("/documents/:slug", controllers.Documents.Detail) - - app.Post("/send", controllers.Documents.Send) + docsGroup.Get("/:slug/cover", controllers.Documents.Cover) + docsGroup.Get("/:slug/read", controllers.Documents.Reader) + docsGroup.Get("/:slug/download", controllers.Documents.Download) + docsGroup.Post("/:slug/send", controllers.Documents.Send) + docsGroup.Get("/:slug", controllers.Documents.Detail) + docsGroup.Get("/", controllers.Documents.Search) - langGroup.Get("/documents", controllers.Documents.Search) langGroup.Get("/", controllers.Documents.Search) app.Get("/", func(c *fiber.Ctx) error { diff --git a/internal/webserver/send_document_test.go b/internal/webserver/send_document_test.go index e222aae6..a1abd02a 100644 --- a/internal/webserver/send_document_test.go +++ b/internal/webserver/send_document_test.go @@ -22,7 +22,6 @@ func TestSendDocument(t *testing.T) { slug string expectedHTTPStatus int }{ - {"Send no document slug", "admin@example.com", "", http.StatusBadRequest}, {"Send no email address", "", "empty", http.StatusBadRequest}, {"Send non existing document slug", "admin@example.com", "wrong", http.StatusBadRequest}, {"Send document slug and email address", "admin@example.com", "john-doe-test-epub", http.StatusOK}, @@ -37,10 +36,9 @@ func TestSendDocument(t *testing.T) { data := url.Values{ "email": {tcase.email}, - "slug": {tcase.slug}, } - req, err := http.NewRequest(http.MethodPost, "/send", strings.NewReader(data.Encode())) + req, err := http.NewRequest(http.MethodPost, "/en/documents/"+tcase.slug+"/send", strings.NewReader(data.Encode())) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") if err != nil { t.Fatalf("Unexpected error: %v", err.Error()) diff --git a/internal/webserver/user_management_test.go b/internal/webserver/user_management_test.go index d8144057..fb76539c 100644 --- a/internal/webserver/user_management_test.go +++ b/internal/webserver/user_management_test.go @@ -304,7 +304,7 @@ func TestUserManagement(t *testing.T) { t.Run("Try to delete a user without an active session", func(t *testing.T) { reset() - response, err := deleteRequest(url.Values{}, &http.Cookie{}, app, fmt.Sprintf("/users/%s", regularUser.Username), t) + response, err := deleteRequest(url.Values{}, &http.Cookie{}, app, fmt.Sprintf("/en/users/%s", regularUser.Username), t) if response == nil { t.Fatalf("Unexpected error: %v", err.Error()) } @@ -315,7 +315,7 @@ func TestUserManagement(t *testing.T) { t.Run("Try to delete a user with a regular user's session", func(t *testing.T) { reset() - response, err := deleteRequest(url.Values{}, regularUserCookie, app, fmt.Sprintf("/users/%s", regularUser.Username), t) + response, err := deleteRequest(url.Values{}, regularUserCookie, app, fmt.Sprintf("/en/users/%s", regularUser.Username), t) if response == nil { t.Fatalf("Unexpected error: %v", err.Error()) } @@ -326,7 +326,7 @@ func TestUserManagement(t *testing.T) { t.Run("Try to delete a user with an admin session", func(t *testing.T) { reset() - response, err := deleteRequest(url.Values{}, adminCookie, app, fmt.Sprintf("/users/%s", regularUser.Username), t) + response, err := deleteRequest(url.Values{}, adminCookie, app, fmt.Sprintf("/en/users/%s", regularUser.Username), t) if response == nil { t.Fatalf("Unexpected error: %v", err.Error()) } @@ -341,7 +341,7 @@ func TestUserManagement(t *testing.T) { t.Run("Try to delete the only existing admin user", func(t *testing.T) { reset() - response, err := deleteRequest(url.Values{}, adminCookie, app, fmt.Sprintf("/users/%s", adminUser.Username), t) + response, err := deleteRequest(url.Values{}, adminCookie, app, fmt.Sprintf("/en/users/%s", adminUser.Username), t) if response == nil { t.Fatalf("Unexpected error: %v", err.Error()) } @@ -352,7 +352,7 @@ func TestUserManagement(t *testing.T) { t.Run("Try to delete a non existing user with an admin session", func(t *testing.T) { reset() - response, err := deleteRequest(url.Values{}, adminCookie, app, "/users/wrong", t) + response, err := deleteRequest(url.Values{}, adminCookie, app, "/en/users/wrong", t) if response == nil { t.Fatalf("Unexpected error: %v", err.Error()) } From 487b2dc9e7d26ff3ff315ad03a02fbc774bbeb9a Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Sun, 8 Sep 2024 20:26:53 +0200 Subject: [PATCH 05/12] Corrected comment --- internal/index/bleve_write.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index a5b3d181..030ff15a 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -46,8 +46,8 @@ func (b *BleveIndexer) RemoveFile(file string) error { return nil } -// AddLibrary scans for documents and adds them to the index in batches of if they -// haven't been previously indexed +// AddLibrary scans for documents and adds them to the index in batches of if they +// haven't been previously indexed or if is true func (b *BleveIndexer) AddLibrary(batchSize int, forceIndexing bool) error { batch := b.idx.NewBatch() batchSlugs := make(map[string]struct{}, batchSize) From a7c8114280e1838cb1896d1986cb9af7e6231b3b Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:38:09 +0200 Subject: [PATCH 06/12] Update internal/webserver/controller/document/download.go --- internal/webserver/controller/document/download.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/webserver/controller/document/download.go b/internal/webserver/controller/document/download.go index ea6e139e..c2cdd667 100644 --- a/internal/webserver/controller/document/download.go +++ b/internal/webserver/controller/document/download.go @@ -35,6 +35,7 @@ func (d *Controller) Download(c *fiber.Ctx) error { if c.Query("format") == "kepub" { output, err = kepubify(fullPath) if err != nil { + log.Println(err) return fiber.ErrInternalServerError } fileName = strings.TrimSuffix(filepath.Base(fullPath), filepath.Ext(fullPath)) From ff152f27cfb9ffab8e7e5b6f01630fab4ef84286 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:38:32 +0200 Subject: [PATCH 07/12] Update internal/webserver/controller/document/download.go --- internal/webserver/controller/document/download.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/webserver/controller/document/download.go b/internal/webserver/controller/document/download.go index c2cdd667..1abc9035 100644 --- a/internal/webserver/controller/document/download.go +++ b/internal/webserver/controller/document/download.go @@ -43,6 +43,7 @@ func (d *Controller) Download(c *fiber.Ctx) error { } else { file, err := os.Open(fullPath) if err != nil { + log.Println(err) return fiber.ErrInternalServerError } output, err = io.ReadAll(file) From e58c1414e3868a3949883c2dd717f5afab9e8f8c Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:38:43 +0200 Subject: [PATCH 08/12] Update internal/webserver/controller/document/download.go --- internal/webserver/controller/document/download.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/webserver/controller/document/download.go b/internal/webserver/controller/document/download.go index 1abc9035..3c5ba723 100644 --- a/internal/webserver/controller/document/download.go +++ b/internal/webserver/controller/document/download.go @@ -46,8 +46,8 @@ func (d *Controller) Download(c *fiber.Ctx) error { log.Println(err) return fiber.ErrInternalServerError } - output, err = io.ReadAll(file) - if err != nil { + if output, err = io.ReadAll(file); err != nil { + log.Println(err) return fiber.ErrInternalServerError } fileName = filepath.Base(document.ID) From b057617f24fecf9ab7c585dfef5229dc2f5fa4b8 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:38:59 +0200 Subject: [PATCH 09/12] Update internal/webserver/controller/document/download.go --- internal/webserver/controller/document/download.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/webserver/controller/document/download.go b/internal/webserver/controller/document/download.go index 3c5ba723..af9d57b6 100644 --- a/internal/webserver/controller/document/download.go +++ b/internal/webserver/controller/document/download.go @@ -74,8 +74,7 @@ func kepubify(fullPath string) ([]byte, error) { } defer r.Close() - err = kepub.NewConverter().Convert(context.Background(), output, r) - if err != nil { + if err = kepub.NewConverter().Convert(context.Background(), output, r); err != nil { return nil, fiber.ErrInternalServerError } From a87c5b88e78ef50ea4c889a9c6c473b78b270b4b Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:39:11 +0200 Subject: [PATCH 10/12] Update main.go --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 16599f90..9f797baa 100644 --- a/main.go +++ b/main.go @@ -143,7 +143,6 @@ func getIndexFile(fs afero.Fs) bleve.Index { indexFile, err := bleve.Open(homeDir + indexPath) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No index found, creating a new one.") - cfg.ForceIndexing = false indexFile = index.Create(homeDir + indexPath) } version, err := indexFile.GetInternal([]byte("version")) From 65bf8906bf04d8337da6e89b6e616a111c4b4c6b Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:39:30 +0200 Subject: [PATCH 11/12] Update main.go --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 9f797baa..18abdc10 100644 --- a/main.go +++ b/main.go @@ -154,7 +154,6 @@ func getIndexFile(fs afero.Fs) bleve.Index { if err = fs.RemoveAll(homeDir + indexPath); err != nil { log.Fatal(err) } - cfg.ForceIndexing = false indexFile = index.Create(homeDir + indexPath) } return indexFile From e7672c64732300543816b447f8d937556e81571f Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Mon, 9 Sep 2024 09:43:39 +0200 Subject: [PATCH 12/12] Added missing log package --- internal/webserver/controller/document/download.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/webserver/controller/document/download.go b/internal/webserver/controller/document/download.go index af9d57b6..abc4b305 100644 --- a/internal/webserver/controller/document/download.go +++ b/internal/webserver/controller/document/download.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "io" + "log" "os" "path/filepath" "strings" @@ -35,7 +36,7 @@ func (d *Controller) Download(c *fiber.Ctx) error { if c.Query("format") == "kepub" { output, err = kepubify(fullPath) if err != nil { - log.Println(err) + log.Println(err) return fiber.ErrInternalServerError } fileName = strings.TrimSuffix(filepath.Base(fullPath), filepath.Ext(fullPath)) @@ -43,11 +44,11 @@ func (d *Controller) Download(c *fiber.Ctx) error { } else { file, err := os.Open(fullPath) if err != nil { - log.Println(err) + log.Println(err) return fiber.ErrInternalServerError } if output, err = io.ReadAll(file); err != nil { - log.Println(err) + log.Println(err) return fiber.ErrInternalServerError } fileName = filepath.Base(document.ID)