From c443408ea27b232483673eb50ffaa554bc1c0f89 Mon Sep 17 00:00:00 2001 From: Sergio Vera Date: Wed, 4 Sep 2024 19:44:58 +0200 Subject: [PATCH] Added forceIndexing parameter --- README.md | 3 ++- config.go | 4 ++-- internal/index/bleve_test.go | 2 +- internal/index/bleve_write.go | 4 ++-- internal/webserver/webserver_test.go | 2 +- main.go | 12 ++++-------- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 312403d..a517b2b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A personal documents server, Coreander indexes the documents (EPUBs and PDFs wit * Read indexed epubs and PDFs from Coreander's interface thanks to [foliate-js](https://github.com/johnfactotum/foliate-js). * Restrictable access only to registered users. * Upload documents through the web interface. +* Download as kepub (epub for Kobo devices) converted on the fly thanks to [Kepubify](https://github.com/pgaskin/kepubify). ## Installation @@ -97,7 +98,7 @@ On first run, Coreander creates an admin user with the following credentials: * `PORT`: Port number in which the webserver listens for requests. Defaults to 3000. * `BATCH_SIZE`: Number of documents persisted by the indexer in one write operation. Defaults to 100. * `COVER_MAX_WIDTH`: Maximum horizontal size for documents cover thumbnails in pixels. Defaults to 600. -* `SKIP_INDEXING`: Whether to bypass the indexing process or not. +* `FORCE_INDEXING`: Whether to force indexing already indexed documents or not. Defaults to false. * `SMTP_SERVER`: Address of the send mail server. * `SMTP_PORT`: Port number of the send mail server. Defaults to 587. * `SMTP_USER`: User to authenticate against the SMTP server. diff --git a/config.go b/config.go index 83ba03f..b27a385 100644 --- a/config.go +++ b/config.go @@ -12,8 +12,8 @@ type Config struct { BatchSize int `env:"BATCH_SIZE" env-default:"100"` // CoverMaxWidth sets the maximum horizontal size for documents cover thumbnails in pixels CoverMaxWidth int `env:"COVER_MAX_WIDTH" env-default:"600"` - // SkipIndexing signals whether to bypass the indexing process or not - SkipIndexing bool `env:"SKIP_INDEXING" env-default:"false"` + // ForceIndexing signals whether to force indexing already indexed documents or not + ForceIndexing bool `env:"FORCE_INDEXING" env-default:"false"` // SmtpServer points to the address of the send mail server SmtpServer string `env:"SMTP_SERVER"` // SmtpPort defines the port in which the mail server listens for requests diff --git a/internal/index/bleve_test.go b/internal/index/bleve_test.go index 26b7184..456fb90 100644 --- a/internal/index/bleve_test.go +++ b/internal/index/bleve_test.go @@ -35,7 +35,7 @@ func TestIndexAndSearch(t *testing.T) { appFS.MkdirAll("lib", 0755) afero.WriteFile(appFS, tcase.filename, []byte(""), 0644) - err = idx.AddLibrary(1) + err = idx.AddLibrary(1, true) if err != nil { t.Errorf("Error indexing: %s", err.Error()) } diff --git a/internal/index/bleve_write.go b/internal/index/bleve_write.go index b92d701..a5b3d18 100644 --- a/internal/index/bleve_write.go +++ b/internal/index/bleve_write.go @@ -48,13 +48,13 @@ func (b *BleveIndexer) RemoveFile(file string) error { // AddLibrary scans for documents and adds them to the index in batches of if they // haven't been previously indexed -func (b *BleveIndexer) AddLibrary(batchSize int) error { +func (b *BleveIndexer) AddLibrary(batchSize int, forceIndexing bool) error { batch := b.idx.NewBatch() batchSlugs := make(map[string]struct{}, batchSize) languages := []string{} b.indexStartTime = float64(time.Now().UnixNano()) e := afero.Walk(b.fs, b.libraryPath, func(fullPath string, f os.FileInfo, err error) error { - if indexed, lang := b.isAlreadyIndexed(fullPath); indexed { + if indexed, lang := b.isAlreadyIndexed(fullPath); indexed && !forceIndexing { b.indexedDocuments += 1 languages = addLanguage(lang, languages) return nil diff --git a/internal/webserver/webserver_test.go b/internal/webserver/webserver_test.go index 91c441e..e5cf53e 100644 --- a/internal/webserver/webserver_test.go +++ b/internal/webserver/webserver_test.go @@ -77,7 +77,7 @@ func bootstrapApp(db *gorm.DB, sender webserver.Sender, appFs afero.Fs, webserve idx = index.NewBleve(indexFile, appFs, webserverConfig.LibraryPath, metadataReaders) } - err = idx.AddLibrary(100) + err = idx.AddLibrary(100, true) if err != nil { log.Fatal(err) } diff --git a/main.go b/main.go index c1dade7..16599f9 100644 --- a/main.go +++ b/main.go @@ -81,11 +81,7 @@ func migrateDir() { func main() { defer idx.Close() - if !cfg.SkipIndexing { - go startIndex(idx, appFs, cfg.BatchSize, cfg.LibPath) - } else { - go fileWatcher(idx, cfg.LibPath) - } + go startIndex(idx, appFs, cfg.BatchSize, cfg.LibPath) sender = &infrastructure.NoEmail{} if cfg.SmtpServer != "" && cfg.SmtpUser != "" && cfg.SmtpPassword != "" { @@ -133,7 +129,7 @@ func main() { func startIndex(idx *index.BleveIndexer, appFs afero.Fs, batchSize int, libPath string) { start := time.Now().Unix() log.Printf("Indexing documents at %s, this can take a while depending on the size of your library.", libPath) - err := idx.AddLibrary(batchSize) + err := idx.AddLibrary(batchSize, cfg.ForceIndexing) if err != nil { log.Fatal(err) } @@ -147,7 +143,7 @@ func getIndexFile(fs afero.Fs) bleve.Index { indexFile, err := bleve.Open(homeDir + indexPath) if err == bleve.ErrorIndexPathDoesNotExist { log.Println("No index found, creating a new one.") - cfg.SkipIndexing = false + cfg.ForceIndexing = false indexFile = index.Create(homeDir + indexPath) } version, err := indexFile.GetInternal([]byte("version")) @@ -159,7 +155,7 @@ func getIndexFile(fs afero.Fs) bleve.Index { if err = fs.RemoveAll(homeDir + indexPath); err != nil { log.Fatal(err) } - cfg.SkipIndexing = false + cfg.ForceIndexing = false indexFile = index.Create(homeDir + indexPath) } return indexFile