From b1c05d658a6b3be693bf8cb67d0cb0f0b4da4d50 Mon Sep 17 00:00:00 2001 From: sama-004 <70210929+Sama-004@users.noreply.github.com> Date: Sat, 12 Oct 2024 23:50:41 +0530 Subject: [PATCH 01/13] add: parse all the links that are inside the href tag after searching for "Monthly portfolio for the month end" --- services/parse_service.go | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 services/parse_service.go diff --git a/services/parse_service.go b/services/parse_service.go new file mode 100644 index 0000000..1b6fb7a --- /dev/null +++ b/services/parse_service.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "regexp" +) + +func main() { + req, err := http.NewRequest("GET", "https://mf.nipponindiaim.com/investor-service/downloads/factsheet-portfolio-and-other-disclosures", nil) + if err != nil { + fmt.Println("Error sending request:", err) + return + } + + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + req.Header.Set("Accept-Language", "en-GB,en-US;q=0.9,en;q=0.8") + req.Header.Set("Cache-Control", "max-age=0") + req.Header.Set("Priority", "u=0, i") + req.Header.Set("Sec-Ch-Ua", "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"") + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", "\"macOS\"") + req.Header.Set("Sec-Fetch-Dest", "document") + req.Header.Set("Sec-Fetch-Mode", "navigate") + req.Header.Set("Sec-Fetch-Site", "none") + req.Header.Set("Sec-Fetch-User", "?1") + req.Header.Set("Upgrade-Insecure-Requests", "1") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + fmt.Println("Error making request:", err) + return + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return + } + + // Use regex to find all href links for "Monthly portfolio for the month end" + re := regexp.MustCompile(`Monthly portfolio for the month end.*?]+href="([^"]+)"`) + matches := re.FindAllStringSubmatch(string(body), -1) + + var links []string + for _, match := range matches { + if len(match) > 1 { + links = append(links, match[1]) + } + } + + if len(links) == 0 { + fmt.Println("No links found") + return + } + + for _, link := range links { + fmt.Println(link) + } +} From 9144281b935cfeeedc9092eecf6a58c19abffca9 Mon Sep 17 00:00:00 2001 From: cu8code Date: Sun, 13 Oct 2024 09:23:11 +0530 Subject: [PATCH 02/13] added support for upload to cloudinary --- services/parse_service.go | 66 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index 1b6fb7a..9a740b1 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -1,10 +1,19 @@ package main import ( + "context" "fmt" "io/ioutil" + "log" "net/http" + "os" + "path" "regexp" + "strings" + + "github.com/cloudinary/cloudinary-go/v2" + "github.com/cloudinary/cloudinary-go/v2/api/admin" + "github.com/cloudinary/cloudinary-go/v2/api/uploader" ) func main() { @@ -13,7 +22,6 @@ func main() { fmt.Println("Error sending request:", err) return } - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req.Header.Set("Accept-Language", "en-GB,en-US;q=0.9,en;q=0.8") req.Header.Set("Cache-Control", "max-age=0") @@ -51,12 +59,60 @@ func main() { } } - if len(links) == 0 { - fmt.Println("No links found") + // upload the downloaded file to cloudinary + for _, link := range links { + uploadFileToCloudinary("https://mf.nipponindiaim.com/" + link) + } +} + +func uploadFileToCloudinary(fileURL string) { + // Initialize Cloudinary + cloudinaryURL := os.Getenv("CLOUDINARY_URL") + cld, err := cloudinary.NewFromURL(cloudinaryURL) + if err != nil { + fmt.Println("Error creating Cloudinary instance:", err) return } - for _, link := range links { - fmt.Println(link) + // Extract the file name from the file URL + publicID := getFileNameFromURL(fileURL) + + // Check if the file already exists + exists, err := checkIfFileExists(cld, publicID) + if err != nil { + fmt.Println("Error checking file existence:", err) + return } + + if exists { + fmt.Printf("File already exists on Cloudinary: %s\n", publicID) + return + } + + // Upload the file if it doesn't exist + resp, err := cld.Upload.Upload(context.Background(), fileURL, uploader.UploadParams{ + PublicID: publicID, + }) + if err != nil { + fmt.Println("Error uploading to Cloudinary:", err) + return + } + fmt.Printf("File uploaded successfully: %s\n", resp.SecureURL) +} + +func checkIfFileExists(cld *cloudinary.Cloudinary, publicID string) (bool, error) { + e, _ := cld.Admin.Asset(context.Background(), admin.AssetParams{ + PublicID: publicID, + }) + log.Println(e.Error.Message) + if strings.Contains(e.Error.Message, "not found") { + return false, nil + } + return true, nil +} + +func getFileNameFromURL(fileURL string) string { + fileName := path.Base(fileURL) + fileName = strings.TrimSuffix(fileName, path.Ext(fileName)) + return fileName } From 786b87aee9b346172c5a477015cff9473f675fe8 Mon Sep 17 00:00:00 2001 From: cu8code Date: Sun, 13 Oct 2024 09:55:36 +0530 Subject: [PATCH 03/13] added support for scheduling --- go.mod | 1 + go.sum | 2 + services/parse_service.go | 137 +++++++++++++++++++++++++------------- 3 files changed, 93 insertions(+), 47 deletions(-) diff --git a/go.mod b/go.mod index fa8e41a..1cd8bd0 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index c47a509..d36742e 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7 github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/services/parse_service.go b/services/parse_service.go index 9a740b1..8eedd65 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "io/ioutil" "log" "net/http" @@ -10,18 +9,87 @@ import ( "path" "regexp" "strings" + "time" "github.com/cloudinary/cloudinary-go/v2" "github.com/cloudinary/cloudinary-go/v2/api/admin" "github.com/cloudinary/cloudinary-go/v2/api/uploader" + "github.com/robfig/cron/v3" ) func main() { + scheduler := cron.New() + + // cron dose not support the L flag, for last day of the month! + // For April, June, September, November + _, err := scheduler.AddFunc("0 0 30 4,6,9,11 *", executeMonthlyTask) + // For January, March, May, July, August, October, December + _, err2 := scheduler.AddFunc("0 0 31 1,3,5,7,8,10,12 *", executeMonthlyTask) + // For February + _, err3 := scheduler.AddFunc("0 0 28 2 *", executeMonthlyTask) + if err != nil { + log.Fatal("Error scheduling task:", err) + } + if err2 != nil { + log.Fatal("Error scheduling task:", err) + } + if err3 != nil { + log.Fatal("Error scheduling task:", err) + } + + scheduler.Start() + + log.Println("Scheduler started. Waiting for month-end...") + + select {} +} + +func executeMonthlyTask() { + now := time.Now() + tomorrow := now.Add(24 * time.Hour) + + if now.Month() != tomorrow.Month() { + log.Println("Month-end detected. Executing upload task...") + performUploadTask() + } else { + log.Println("Not month-end. Skipping upload task.") + } +} + +func performUploadTask() { + log.Println("Starting monthly upload task...") + req, err := http.NewRequest("GET", "https://mf.nipponindiaim.com/investor-service/downloads/factsheet-portfolio-and-other-disclosures", nil) if err != nil { - fmt.Println("Error sending request:", err) + log.Println("Error creating request:", err) + return + } + + setRequestHeaders(req) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Println("Error making request:", err) return } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Println("Error reading response body:", err) + return + } + + portfolioLinks := extractPortfolioLinks(string(body)) + + for _, link := range portfolioLinks { + uploadToCloudinary("https://mf.nipponindiaim.com/" + link) + } + + log.Println("Monthly upload task completed.") +} + +func setRequestHeaders(req *http.Request) { req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req.Header.Set("Accept-Language", "en-GB,en-US;q=0.9,en;q=0.8") req.Header.Set("Cache-Control", "max-age=0") @@ -34,23 +102,11 @@ func main() { req.Header.Set("Sec-Fetch-Site", "none") req.Header.Set("Sec-Fetch-User", "?1") req.Header.Set("Upgrade-Insecure-Requests", "1") +} - resp, err := http.DefaultClient.Do(req) - if err != nil { - fmt.Println("Error making request:", err) - return - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - fmt.Println("Error reading response body:", err) - return - } - - // Use regex to find all href links for "Monthly portfolio for the month end" - re := regexp.MustCompile(`Monthly portfolio for the month end.*?]+href="([^"]+)"`) - matches := re.FindAllStringSubmatch(string(body), -1) +func extractPortfolioLinks(htmlContent string) []string { + re := regexp.MustCompile(`Monthly portfolio for the month end.*?]+href="([^"]+)"`) + matches := re.FindAllStringSubmatch(htmlContent, -1) var links []string for _, match := range matches { @@ -58,61 +114,48 @@ func main() { links = append(links, match[1]) } } - - // upload the downloaded file to cloudinary - for _, link := range links { - uploadFileToCloudinary("https://mf.nipponindiaim.com/" + link) - } + return links } -func uploadFileToCloudinary(fileURL string) { - // Initialize Cloudinary - cloudinaryURL := os.Getenv("CLOUDINARY_URL") - cld, err := cloudinary.NewFromURL(cloudinaryURL) +func uploadToCloudinary(fileURL string) { + cld, err := cloudinary.NewFromURL(os.Getenv("CLOUDINARY_URL")) if err != nil { - fmt.Println("Error creating Cloudinary instance:", err) + log.Println("Error creating Cloudinary instance:", err) return } - // Extract the file name from the file URL - publicID := getFileNameFromURL(fileURL) + publicID := extractFileName(fileURL) - // Check if the file already exists - exists, err := checkIfFileExists(cld, publicID) + exists, err := checkFileExistence(cld, publicID) if err != nil { - fmt.Println("Error checking file existence:", err) + log.Println("Error checking file existence:", err) return } if exists { - fmt.Printf("File already exists on Cloudinary: %s\n", publicID) + log.Printf("File already exists on Cloudinary: %s\n", publicID) return } - // Upload the file if it doesn't exist resp, err := cld.Upload.Upload(context.Background(), fileURL, uploader.UploadParams{ PublicID: publicID, }) if err != nil { - fmt.Println("Error uploading to Cloudinary:", err) + log.Println("Error uploading to Cloudinary:", err) return } - fmt.Printf("File uploaded successfully: %s\n", resp.SecureURL) + + log.Printf("File uploaded successfully: %s\n", resp.SecureURL) } -func checkIfFileExists(cld *cloudinary.Cloudinary, publicID string) (bool, error) { - e, _ := cld.Admin.Asset(context.Background(), admin.AssetParams{ +func checkFileExistence(cld *cloudinary.Cloudinary, publicID string) (bool, error) { + _, err := cld.Admin.Asset(context.Background(), admin.AssetParams{ PublicID: publicID, }) - log.Println(e.Error.Message) - if strings.Contains(e.Error.Message, "not found") { - return false, nil - } - return true, nil + return !strings.Contains(err.Error(), "not found"), nil } -func getFileNameFromURL(fileURL string) string { +func extractFileName(fileURL string) string { fileName := path.Base(fileURL) - fileName = strings.TrimSuffix(fileName, path.Ext(fileName)) - return fileName + return strings.TrimSuffix(fileName, path.Ext(fileName)) } From d77b6601188eb02a22736a5718799e89e1d28de0 Mon Sep 17 00:00:00 2001 From: cu8code Date: Thu, 17 Oct 2024 19:37:20 +0530 Subject: [PATCH 04/13] added checks & added a setup step also removed checkFileExistence as I could not get it working --- services/parse_service.go | 112 +++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 43 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index 8eedd65..f905db3 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "io/ioutil" "log" "net/http" @@ -9,7 +10,6 @@ import ( "path" "regexp" "strings" - "time" "github.com/cloudinary/cloudinary-go/v2" "github.com/cloudinary/cloudinary-go/v2/api/admin" @@ -17,43 +17,68 @@ import ( "github.com/robfig/cron/v3" ) -func main() { - scheduler := cron.New() - - // cron dose not support the L flag, for last day of the month! - // For April, June, September, November - _, err := scheduler.AddFunc("0 0 30 4,6,9,11 *", executeMonthlyTask) - // For January, March, May, July, August, October, December - _, err2 := scheduler.AddFunc("0 0 31 1,3,5,7,8,10,12 *", executeMonthlyTask) - // For February - _, err3 := scheduler.AddFunc("0 0 28 2 *", executeMonthlyTask) - if err != nil { - log.Fatal("Error scheduling task:", err) - } - if err2 != nil { - log.Fatal("Error scheduling task:", err) +func assert(b bool, mess string){ + red := "\033[31m" + green := "\033[32m" + reset := "\033[0m" + if b { + panic(red + "Assert FAILED: " + mess + reset) } - if err3 != nil { - log.Fatal("Error scheduling task:", err) + if os.Getenv("DEBUG") == "true" { + fmt.Println(green + "Assert PASSED: ", mess + reset) } - - scheduler.Start() - - log.Println("Scheduler started. Waiting for month-end...") - - select {} } -func executeMonthlyTask() { - now := time.Now() - tomorrow := now.Add(24 * time.Hour) +func setupCheck() { + if (len(os.Getenv("CLOUDINARY_URL")) < 5 ) { + panic("Please provied a CLOUDINARY_URL. Run `export CLOUDINARY_URL=your@url` before in your shell for linux and MacOS") + } + if os.Getenv("DEBUG") != "true" { + log.Printf("IGNORE THIS ONLY FOR DEV: To run the script in debug mode use `export DEBUG=\"true\"`") + } +} - if now.Month() != tomorrow.Month() { - log.Println("Month-end detected. Executing upload task...") - performUploadTask() +func main() { + setupCheck() + scheduler := cron.New() + debug := os.Getenv("DEBUG") == "true" + + + if !debug { + // cron dose not support the L flag, for last day of the month! + // For April, June, September, November + _, err := scheduler.AddFunc("0 0 30 4,6,9,11 *", performUploadTask) + // For January, March, May, July, August, October, December + _, err2 := scheduler.AddFunc("0 0 31 1,3,5,7,8,10,12 *", performUploadTask) + // For February + _, err3 := scheduler.AddFunc("0 0 28 2 *", performUploadTask) + if err != nil { + log.Fatal("Error scheduling task:", err) + } + if err2 != nil { + log.Fatal("Error scheduling task:", err) + } + if err3 != nil { + log.Fatal("Error scheduling task:", err) + } + scheduler.Start() } else { - log.Println("Not month-end. Skipping upload task.") + fmt.Println("Skipping the regular scheduler as debug mode is enabled.") + fmt.Println("Creating a scheduler that will run every 1 minute.") + jobID, err := scheduler.AddFunc("* * * * *", performUploadTask) + + // Need this here for proper Next time calculation + scheduler.Start() + if err != nil { + fmt.Println("An error occurred: the scheduler could not be added.") + } else { + fmt.Println("Next run time for Debug Scheduler:", scheduler.Entry(jobID).Next) + } } + + log.Println("Scheduler started") + + select {} } func performUploadTask() { @@ -105,19 +130,27 @@ func setRequestHeaders(req *http.Request) { } func extractPortfolioLinks(htmlContent string) []string { - re := regexp.MustCompile(`Monthly portfolio for the month end.*?]+href="([^"]+)"`) + assert(len(htmlContent) == 0, "extractPortfolioLinks len(htmlContent) == 0") + + re := regexp.MustCompile(`Monthly portfolio for the month end.*?]+href="([^"]+)"`) matches := re.FindAllStringSubmatch(htmlContent, -1) + assert(len(matches) == 0, "extractPortfolioLinks len(matches) == 0") + var links []string for _, match := range matches { if len(match) > 1 { links = append(links, match[1]) } } + + assert(len(links) == 0, "extractPortfolioLinks len(links) == 0") return links } func uploadToCloudinary(fileURL string) { + assert(len(fileURL) == 0, "uploadToCloudinary len(fileURL) == 0") + cld, err := cloudinary.NewFromURL(os.Getenv("CLOUDINARY_URL")) if err != nil { log.Println("Error creating Cloudinary instance:", err) @@ -126,17 +159,6 @@ func uploadToCloudinary(fileURL string) { publicID := extractFileName(fileURL) - exists, err := checkFileExistence(cld, publicID) - if err != nil { - log.Println("Error checking file existence:", err) - return - } - - if exists { - log.Printf("File already exists on Cloudinary: %s\n", publicID) - return - } - resp, err := cld.Upload.Upload(context.Background(), fileURL, uploader.UploadParams{ PublicID: publicID, }) @@ -149,6 +171,8 @@ func uploadToCloudinary(fileURL string) { } func checkFileExistence(cld *cloudinary.Cloudinary, publicID string) (bool, error) { + assert(cld == nil, "checkFileExistence cld == null") + assert(publicID == "", "checkFileExistence publicId == \"\"") _, err := cld.Admin.Asset(context.Background(), admin.AssetParams{ PublicID: publicID, }) @@ -156,6 +180,8 @@ func checkFileExistence(cld *cloudinary.Cloudinary, publicID string) (bool, erro } func extractFileName(fileURL string) string { + assert(fileURL == "", "extractFileName fileURL == \"\"") + fileName := path.Base(fileURL) return strings.TrimSuffix(fileName, path.Ext(fileName)) } From 2c377e8ce7d7af7470c124b2ab1c5a06b86f1e64 Mon Sep 17 00:00:00 2001 From: sama-004 <70210929+Sama-004@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:33:57 +0530 Subject: [PATCH 05/13] add: parse dates from uploaded file names - Doesn't work for filenames with weird date format like IN_MF_RLMF_MONTHLY_PORTFOLIO_REPORT-%25282%2529.xls RVSD-IN_MF_RLMF_MONTHLY_PORTFOLIO_REPORT-revised.xls RelianceMonthlyPortfolios31122013.xls RelianceMonthlyPortfolios31102013.xls RelianceMonthlyPortfolios31032014.xls Reliance-Monthly-Portfolios-30062015.xls Reliance-Monthly-Portfolios-31052015.xls Reliance-Monthly-Portfolios-30042015.xls Reliance%2520Monthly%2520Portfolios-31.03.2015-1.xlsx Reliance-Monthly-Portfolios-28022015.xls Reliance-Monthly-Portfolios-31012015.xls Reliance-Monthly-Portfolios-31122014.xls Reliance-Monthly-Portfolios-30092015.xls Reliance-Monthly-Portfolios-31082015.xls Reliance-Monthly-Portfolios-31072015.xls Reliance%2520Monthly%2520Portfolios-30.11.2014.xls Reliance%2520Monthly%2520Portfolios-31.10.2014.xls RTFOLIO-MAR-23.xls MONTHLY-PORTFOLIO-FEB-23.xls NIMF-Portfolio-with-Rikometer-March-21.xlsx Portfolio-June-With-riskometer.xls need to add dates for these files manually --- services/parse_service.go | 93 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index f905db3..069f515 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -10,6 +10,7 @@ import ( "path" "regexp" "strings" + "time" "github.com/cloudinary/cloudinary-go/v2" "github.com/cloudinary/cloudinary-go/v2/api/admin" @@ -17,20 +18,20 @@ import ( "github.com/robfig/cron/v3" ) -func assert(b bool, mess string){ - red := "\033[31m" +func assert(b bool, mess string) { + red := "\033[31m" green := "\033[32m" reset := "\033[0m" if b { panic(red + "Assert FAILED: " + mess + reset) } if os.Getenv("DEBUG") == "true" { - fmt.Println(green + "Assert PASSED: ", mess + reset) + fmt.Println(green+"Assert PASSED: ", mess+reset) } } func setupCheck() { - if (len(os.Getenv("CLOUDINARY_URL")) < 5 ) { + if len(os.Getenv("CLOUDINARY_URL")) < 5 { panic("Please provied a CLOUDINARY_URL. Run `export CLOUDINARY_URL=your@url` before in your shell for linux and MacOS") } if os.Getenv("DEBUG") != "true" { @@ -40,9 +41,10 @@ func setupCheck() { func main() { setupCheck() + fmt.Println("Env", os.Getenv("MONGO_URI")) scheduler := cron.New() - debug := os.Getenv("DEBUG") == "true" + debug := os.Getenv("DEBUG") == "true" if !debug { // cron dose not support the L flag, for last day of the month! @@ -70,7 +72,7 @@ func main() { // Need this here for proper Next time calculation scheduler.Start() if err != nil { - fmt.Println("An error occurred: the scheduler could not be added.") + fmt.Println("An error occurred: the scheduler could not be added.") } else { fmt.Println("Next run time for Debug Scheduler:", scheduler.Entry(jobID).Next) } @@ -168,6 +170,85 @@ func uploadToCloudinary(fileURL string) { } log.Printf("File uploaded successfully: %s\n", resp.SecureURL) + + // for db + //resp.SecureURL is the Cloudinary Link + // (resp.SecureURL) + //publicId is the completeName + // fmt.Println(publicID) + // fmt.Println(uuid) + + month := extractMonth(publicID) + if month == "" { + fmt.Println("Not found month") + } else { + fmt.Println(month) + } + //fund house???(ask) +} + +func extractMonth(fileName string) string { + patterns := []*regexp.Regexp{ + regexp.MustCompile(`\d{2}\.\d{2}\.\d{4}`), // dd.mm.yyyy + regexp.MustCompile(`\d{2}-\d{2}-\d{4}`), // dd-mm-yyyy + regexp.MustCompile(`\d{2}-\d{2}-\d{2}`), // dd-mm-yy + regexp.MustCompile(`\d{2}-\d{4}`), // mm-yyyy + regexp.MustCompile(`(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}`), // Abbreviated month-year (e.g., Mar-23) + regexp.MustCompile(`(January|February|March|April|May|June|July|August|September|October|November|December)-\d{4}`), // Full month-year (e.g., March-2021) + } + for _, pattern := range patterns { + match := pattern.FindString(fileName) + if match != "" { + parsedDate := parseDate(match) + if parsedDate != "" { + return parsedDate + } + } + } + return "" +} + +func parseDate(dateStr string) string { + if strings.Contains(dateStr, ".") { + t, err := time.Parse("02.01.2006", dateStr) + if err == nil { + return t.Format("2006-01-02") + } + } + layouts := []string{ + "02-01-2006", // dd-mm-yyyy + "02-01-06", // dd-mm-yy + "01-2006", // mm-yyyy + "Jan-06", // Abbreviated month-year + "January-2006", // Full month-year + } + + for _, layout := range layouts { + t, err := time.Parse(layout, dateStr) + if err == nil { + // Format the date in YYYY-MM-DD format + return t.Format("2006-01-02") + } + } + + // Handle month-year patterns (e.g., Mar-23, January-2021) + if len(dateStr) == 7 || len(dateStr) == 10 { + monthAbbrevToFull := map[string]string{ + "Jan": "January", "Feb": "February", "Mar": "March", "Apr": "April", + "May": "May", "Jun": "June", "Jul": "July", "Aug": "August", + "Sep": "September", "Oct": "October", "Nov": "November", "Dec": "December", + } + for abbr, full := range monthAbbrevToFull { + if strings.Contains(dateStr, abbr) { + dateStr = strings.Replace(dateStr, abbr, full, 1) + t, err := time.Parse("January-06", dateStr) + if err == nil { + return t.Format("2006-01") + } + } + } + } + return "" } func checkFileExistence(cld *cloudinary.Cloudinary, publicID string) (bool, error) { From d0d1f188e8548a83661fc5d0a97bcbf88cd58d94 Mon Sep 17 00:00:00 2001 From: sama-004 <70210929+Sama-004@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:17:58 +0530 Subject: [PATCH 06/13] add: enter details in db after upload --- services/parse_service.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index 069f515..fa477d3 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -9,13 +9,16 @@ import ( "os" "path" "regexp" + mongo_client "stockbackend/clients/mongo" "strings" "time" "github.com/cloudinary/cloudinary-go/v2" "github.com/cloudinary/cloudinary-go/v2/api/admin" "github.com/cloudinary/cloudinary-go/v2/api/uploader" + "github.com/google/uuid" "github.com/robfig/cron/v3" + "gopkg.in/mgo.v2/bson" ) func assert(b bool, mess string) { @@ -41,7 +44,6 @@ func setupCheck() { func main() { setupCheck() - fmt.Println("Env", os.Getenv("MONGO_URI")) scheduler := cron.New() debug := os.Getenv("DEBUG") == "true" @@ -171,20 +173,24 @@ func uploadToCloudinary(fileURL string) { log.Printf("File uploaded successfully: %s\n", resp.SecureURL) - // for db - //resp.SecureURL is the Cloudinary Link - // (resp.SecureURL) - //publicId is the completeName - // fmt.Println(publicID) - // fmt.Println(uuid) - month := extractMonth(publicID) - if month == "" { - fmt.Println("Not found month") - } else { - fmt.Println(month) + fileUUID := uuid.New().String() + document := bson.M{ + "month": month, + "completeName": publicID, + "cloudinaryLink": resp.SecureURL, + "id": fileUUID, + } + + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("COLLECTION")) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err = collection.InsertOne(ctx, document) + if err != nil { + log.Println("Error inserting document into MongoDB:", err) + return } - //fund house???(ask) + log.Printf("Document inserted successfully into MongoDB. UUID: %s\n", fileUUID) } func extractMonth(fileName string) string { From 9003f93a3d1c15046e59b8aa16edc34b110a6bec Mon Sep 17 00:00:00 2001 From: sama-004 <70210929+Sama-004@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:45:35 +0530 Subject: [PATCH 07/13] fix: id and add fund house --- services/parse_service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index fa477d3..3e03746 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -176,13 +176,14 @@ func uploadToCloudinary(fileURL string) { month := extractMonth(publicID) fileUUID := uuid.New().String() document := bson.M{ + "_id": fileUUID, "month": month, "completeName": publicID, "cloudinaryLink": resp.SecureURL, - "id": fileUUID, + "fund_house": "nippon", } - collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("COLLECTION")) + collection := mongo_client.Client.Database(os.Getenv("DATABASE_NAME")).Collection(os.Getenv("COLLECTION_NAME")) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _, err = collection.InsertOne(ctx, document) From 704a6fbddc502fee59da4c78aa4c203535af56a4 Mon Sep 17 00:00:00 2001 From: shivamsouravjha <2019145@iiitdmj.ac.in> Date: Tue, 5 Nov 2024 00:24:06 +0530 Subject: [PATCH 08/13] fix: package names Signed-off-by: shivamsouravjha <2019145@iiitdmj.ac.in> --- main.go | 1 + services/parse_service.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index ffd2ba8..43fddbf 100644 --- a/main.go +++ b/main.go @@ -116,6 +116,7 @@ func main() { router.Use(sentrygin.New(sentrygin.Options{})) router.Use(CORSMiddleware()) + services.UpdateFunds() ticker := startTicker() rankUpdater := startRankUpdater() routes.Routes(router) diff --git a/services/parse_service.go b/services/parse_service.go index 3e03746..c84db3b 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -1,9 +1,9 @@ -package main +package services import ( "context" "fmt" - "io/ioutil" + "io" "log" "net/http" "os" @@ -42,7 +42,7 @@ func setupCheck() { } } -func main() { +func UpdateFunds() { setupCheck() scheduler := cron.New() @@ -103,7 +103,7 @@ func performUploadTask() { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { log.Println("Error reading response body:", err) return @@ -136,7 +136,7 @@ func setRequestHeaders(req *http.Request) { func extractPortfolioLinks(htmlContent string) []string { assert(len(htmlContent) == 0, "extractPortfolioLinks len(htmlContent) == 0") - re := regexp.MustCompile(`Monthly portfolio for the month end.*?]+href="([^"]+)"`) + re := regexp.MustCompile(`Monthly portfolio for the month *?]+href="([^"]+)"`) matches := re.FindAllStringSubmatch(htmlContent, -1) assert(len(matches) == 0, "extractPortfolioLinks len(matches) == 0") From 1550c33a06dd4592ca7306a4fe48cb23b47afa51 Mon Sep 17 00:00:00 2001 From: shivamsouravjha <2019145@iiitdmj.ac.in> Date: Tue, 5 Nov 2024 02:15:13 +0530 Subject: [PATCH 09/13] fix: update logic for parsing data Signed-off-by: shivamsouravjha <2019145@iiitdmj.ac.in> --- services/parse_service.go | 133 +++++++++++++++++++++++++++----------- 1 file changed, 97 insertions(+), 36 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index c84db3b..fb49045 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -12,6 +12,7 @@ import ( mongo_client "stockbackend/clients/mongo" "strings" "time" + "unicode" "github.com/cloudinary/cloudinary-go/v2" "github.com/cloudinary/cloudinary-go/v2/api/admin" @@ -22,12 +23,8 @@ import ( ) func assert(b bool, mess string) { - red := "\033[31m" green := "\033[32m" reset := "\033[0m" - if b { - panic(red + "Assert FAILED: " + mess + reset) - } if os.Getenv("DEBUG") == "true" { fmt.Println(green+"Assert PASSED: ", mess+reset) } @@ -109,10 +106,10 @@ func performUploadTask() { return } - portfolioLinks := extractPortfolioLinks(string(body)) + mfDatas := extractPortfolioLinks(string(body)) - for _, link := range portfolioLinks { - uploadToCloudinary("https://mf.nipponindiaim.com/" + link) + for _, mfData := range mfDatas { + uploadToCloudinary("https://mf.nipponindiaim.com/", mfData) } log.Println("Monthly upload task completed.") @@ -132,58 +129,124 @@ func setRequestHeaders(req *http.Request) { req.Header.Set("Sec-Fetch-User", "?1") req.Header.Set("Upgrade-Insecure-Requests", "1") } +func normalizeWhitespace(s string) string { + var b strings.Builder + prevIsSpace := false + for _, r := range s { + if unicode.IsSpace(r) { + if !prevIsSpace { + b.WriteRune(' ') + prevIsSpace = true + } + } else { + b.WriteRune(r) + prevIsSpace = false + } + } + return b.String() +} -func extractPortfolioLinks(htmlContent string) []string { - assert(len(htmlContent) == 0, "extractPortfolioLinks len(htmlContent) == 0") +func removeZeroWidthChars(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case '\u200B', '\u200C', '\u200D', '\uFEFF': + // Exclude zero-width characters + return -1 + default: + // Include other characters + return r + } + }, s) +} - re := regexp.MustCompile(`Monthly portfolio for the month *?]+href="([^"]+)"`) - matches := re.FindAllStringSubmatch(htmlContent, -1) +func cleanHTMLContent(s string) string { + s = removeZeroWidthChars(s) + s = normalizeWhitespace(s) + return s +} + +type MFCOLLECTION struct { + month string + year string + link string +} - assert(len(matches) == 0, "extractPortfolioLinks len(matches) == 0") +func extractPortfolioLinks(htmlContent string) []MFCOLLECTION { + // Updated regex pattern to handle various formats + re := regexp.MustCompile(`(?i)Monthly[\s\p{Zs}]+portfolio[\s\p{Zs}]+for[\s\p{Zs}]+the[\s\p{Zs}]+month(?:[\s\p{Zs}]+(?:of|end))?[\s\p{Zs}]*(?:(\d{1,2})(?:st|nd|rd|th)?[\s\p{Zs}]+)?(\w+)[\s\p{Zs}]*(\d{4})?.*?]+href="([^"]+)"`) + htmlContent = cleanHTMLContent(htmlContent) - var links []string + matches := re.FindAllStringSubmatch(htmlContent, -1) + fmt.Println("Total Matches Found:", len(matches)) // Debugging: Show total matches found + + var mfDetails []MFCOLLECTION for _, match := range matches { - if len(match) > 1 { - links = append(links, match[1]) + if len(match) > 4 { + // entireText := match[0] // Entire matched text + + // Extract day, month, year, and link + month := match[2] // Month + year := match[3] // Optional year + link := match[4] // Extracted link + + // If year is missing in match[3], try to extract it from the following content + if year == "" { + // Attempt to find a 4-digit year after the month + yearRe := regexp.MustCompile(`\b(\d{4})\b`) + yearMatch := yearRe.FindStringSubmatch(htmlContent) + if len(yearMatch) > 1 { + year = yearMatch[1] + } + } + + // Append the link + mfDetails = append(mfDetails, MFCOLLECTION{ + month: month, + year: year, + link: link, + }) + // fmt.Println("Entire matched text:", entireText) + // fmt.Println("Month:", month) // Print extracted month + // fmt.Println("Year:", year) // Print extracted year + // fmt.Println("Link:", link) // Print the link } } - - assert(len(links) == 0, "extractPortfolioLinks len(links) == 0") - return links + return mfDetails } -func uploadToCloudinary(fileURL string) { - assert(len(fileURL) == 0, "uploadToCloudinary len(fileURL) == 0") - +func uploadToCloudinary(fileURL string, mfData MFCOLLECTION) { cld, err := cloudinary.NewFromURL(os.Getenv("CLOUDINARY_URL")) if err != nil { log.Println("Error creating Cloudinary instance:", err) return } - - publicID := extractFileName(fileURL) - - resp, err := cld.Upload.Upload(context.Background(), fileURL, uploader.UploadParams{ - PublicID: publicID, - }) - if err != nil { - log.Println("Error uploading to Cloudinary:", err) + publicID := extractFileName(fileURL + mfData.link) + asset, err := cld.Admin.Asset(context.Background(), admin.AssetParams{PublicID: publicID}) + + secureUrl := asset.SecureURL + if err == nil && asset.PublicID == "" { + resp, err := cld.Upload.Upload(context.Background(), fileURL+mfData.link, uploader.UploadParams{ + PublicID: publicID, + }) + if err != nil { + log.Println("Error uploading to Cloudinary:", err) + return + } + secureUrl = resp.SecureURL + } else if err != nil { return } - log.Printf("File uploaded successfully: %s\n", resp.SecureURL) - month := extractMonth(publicID) fileUUID := uuid.New().String() document := bson.M{ "_id": fileUUID, "month": month, "completeName": publicID, - "cloudinaryLink": resp.SecureURL, + "cloudinaryLink": secureUrl, "fund_house": "nippon", } - - collection := mongo_client.Client.Database(os.Getenv("DATABASE_NAME")).Collection(os.Getenv("COLLECTION_NAME")) + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("MFCOLLECTION")) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _, err = collection.InsertOne(ctx, document) @@ -268,8 +331,6 @@ func checkFileExistence(cld *cloudinary.Cloudinary, publicID string) (bool, erro } func extractFileName(fileURL string) string { - assert(fileURL == "", "extractFileName fileURL == \"\"") - fileName := path.Base(fileURL) return strings.TrimSuffix(fileName, path.Ext(fileName)) } From 7d1bec204e82497d555c79ec47a57325dc4e9e46 Mon Sep 17 00:00:00 2001 From: shivamsouravjha <2019145@iiitdmj.ac.in> Date: Fri, 8 Nov 2024 11:36:59 +0530 Subject: [PATCH 10/13] chore: storing data in mongo databases Signed-off-by: shivamsouravjha <2019145@iiitdmj.ac.in> --- services/parse_service.go | 514 ++++++++++++++++++++++++++++++++- services/parse_service_test.go | 60 ++++ utils/helpers/helpers_test.go | 290 ++++++++++++++++++- 3 files changed, 841 insertions(+), 23 deletions(-) create mode 100644 services/parse_service_test.go diff --git a/services/parse_service.go b/services/parse_service.go index fb49045..f3688b9 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -2,6 +2,8 @@ package services import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" "log" @@ -10,6 +12,8 @@ import ( "path" "regexp" mongo_client "stockbackend/clients/mongo" + "stockbackend/utils/helpers" + "strconv" "strings" "time" "unicode" @@ -17,8 +21,14 @@ import ( "github.com/cloudinary/cloudinary-go/v2" "github.com/cloudinary/cloudinary-go/v2/api/admin" "github.com/cloudinary/cloudinary-go/v2/api/uploader" + "github.com/extrame/xls" + "github.com/getsentry/sentry-go" "github.com/google/uuid" "github.com/robfig/cron/v3" + "github.com/wailsapp/mimetype" + "github.com/xuri/excelize/v2" + "go.mongodb.org/mongo-driver/mongo" + "go.uber.org/zap" "gopkg.in/mgo.v2/bson" ) @@ -64,17 +74,17 @@ func UpdateFunds() { } scheduler.Start() } else { - fmt.Println("Skipping the regular scheduler as debug mode is enabled.") - fmt.Println("Creating a scheduler that will run every 1 minute.") - jobID, err := scheduler.AddFunc("* * * * *", performUploadTask) - + // fmt.Println("Skipping the regular scheduler as debug mode is enabled.") + // fmt.Println("Creating a scheduler that will run every 1 minute.") + // jobID, err := scheduler.AddFunc("* * * * *", performUploadTask) + performUploadTask() // Need this here for proper Next time calculation - scheduler.Start() - if err != nil { - fmt.Println("An error occurred: the scheduler could not be added.") - } else { - fmt.Println("Next run time for Debug Scheduler:", scheduler.Entry(jobID).Next) - } + // scheduler.Start() + // if err != nil { + // // fmt.Println("An error occurred: the scheduler could not be added.") + // } else { + // // fmt.Println("Next run time for Debug Scheduler:", scheduler.Entry(jobID).Next) + // } } log.Println("Scheduler started") @@ -177,7 +187,7 @@ func extractPortfolioLinks(htmlContent string) []MFCOLLECTION { htmlContent = cleanHTMLContent(htmlContent) matches := re.FindAllStringSubmatch(htmlContent, -1) - fmt.Println("Total Matches Found:", len(matches)) // Debugging: Show total matches found + // fmt.Println("Total Matches Found:", len(matches)) // Debugging: Show total matches found var mfDetails []MFCOLLECTION for _, match := range matches { @@ -205,10 +215,10 @@ func extractPortfolioLinks(htmlContent string) []MFCOLLECTION { year: year, link: link, }) - // fmt.Println("Entire matched text:", entireText) - // fmt.Println("Month:", month) // Print extracted month - // fmt.Println("Year:", year) // Print extracted year - // fmt.Println("Link:", link) // Print the link + // // fmt.Println("Entire matched text:", entireText) + // // fmt.Println("Month:", month) // Print extracted month + // // fmt.Println("Year:", year) // Print extracted year + // // fmt.Println("Link:", link) // Print the link } } return mfDetails @@ -237,7 +247,63 @@ func uploadToCloudinary(fileURL string, mfData MFCOLLECTION) { return } + response, err := http.Get(secureUrl) + if err != nil { + log.Println("Error downloading xlsx file:", err) + return + } + bodyBytes, err := io.ReadAll(response.Body) + if err != nil { + log.Println("Error reading response body:", err) + return + } + + defer response.Body.Close() + + // Save the downloaded file to a temporary location + m := mimetype.Detect(bodyBytes) + // log.Println("Detected MIME type:", m.String()) + + // Assign file extension based on MIME type + var fileExt string + if m.Is("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") { + fileExt = ".xlsx" + } else if m.Is("application/vnd.ms-excel") { + fileExt = ".xls" + } else { + // log.Println("Downloaded file is not a supported Excel format (.xlsx or .xls).") + return + } + tempInputFile := fmt.Sprintf("%s%s", uuid.New().String(), fileExt) + err = os.WriteFile(tempInputFile, bodyBytes, 0644) + if err != nil { + log.Println("Error saving Excel file:", err) + return + } + defer os.Remove(tempInputFile) month := extractMonth(publicID) + + if fileExt == ".xlsx" { + // Process .xlsx file + err = processXLSXFile(tempInputFile, month) + if err != nil { + // fmt.Println("tempInputFile", tempInputFile) + log.Println("Error processing .xlsx file:", err) + return + } + } else if fileExt == ".xls" { + // Process .xls file + // fmt.Println("tempInputFile", tempInputFile) + err = processXLSFile(tempInputFile, month) + if err != nil { + log.Println("Error processing .xls file:", err) + return + } + } else { + log.Println("Unsupported file format:", fileExt) + return + } + fileUUID := uuid.New().String() document := bson.M{ "_id": fileUUID, @@ -257,6 +323,424 @@ func uploadToCloudinary(fileURL string, mfData MFCOLLECTION) { log.Printf("Document inserted successfully into MongoDB. UUID: %s\n", fileUUID) } +func processXLSXFile(tempInputFile, month string) error { + // Open the xlsx file using excelize + xlsxFile, err := excelize.OpenFile(tempInputFile) + if err != nil { + // fmt.Println("Error opening xlsx file:", err) + return fmt.Errorf("error opening xlsx file: %v", err) + } + defer xlsxFile.Close() + + // Get all sheet names + sheetList := xlsxFile.GetSheetList() + // stock := make([]map[string]interface{}, 0) + + for _, sheet := range sheetList { + rows, err := xlsxFile.GetRows(sheet) + if err != nil { + sentry.CaptureException(err) + zap.L().Error("Error reading rows from sheet", zap.String("sheet", sheet), zap.Error(err)) + continue + } + headerFound := false + headerMap := make(map[string]int) + stopExtracting := false + stockDetail := make([]map[string]interface{}, 0) + processingStarted := false + mutualFundName := "" + if len(rows) > 0 && len(rows[0]) > 1 { + mutualFundName = rows[0][1] + // log.Printf("Mutual Fund Name for sheet %s: %s", sheet, mutualFundName) + } + for _, row := range rows { + stock := make(map[string]interface{}, 0) + + if len(row) == 0 { + continue + } + if !headerFound { + for _, cell := range row { + if helpers.MatchHeader(cell, []string{`name\s*of\s*(the)?\s*instrument`}) { + headerFound = true + // Build the header map + for i, headerCell := range row { + normalizedHeader := helpers.NormalizeString(headerCell) + // Map possible variations to standard keys + switch { + case helpers.MatchHeader(normalizedHeader, []string{`name\s*of\s*(the)?\s*instrument`}): + headerMap["Name of the Instrument"] = i + case helpers.MatchHeader(normalizedHeader, []string{`isin`}): + headerMap["ISIN"] = i + case helpers.MatchHeader(normalizedHeader, []string{`rating\s*/\s*industry`, `industry\s*/\s*rating`}): + headerMap["Industry/Rating"] = i + case helpers.MatchHeader(normalizedHeader, []string{`quantity`}): + headerMap["Quantity"] = i + case helpers.MatchHeader(normalizedHeader, []string{`market\s*/\s*fair\s*value.*`, `market\s*value.*`}): + headerMap["Market/Fair Value"] = i + case helpers.MatchHeader(normalizedHeader, []string{`%.*nav`, `%.*net\s*assets`}): + headerMap["Percentage of AUM"] = i + } + } + // zap.L().Info("Header found", zap.Any("headerMap", headerMap)) + break + } + } + continue + } + + joinedRow := strings.Join(row, "") + if strings.Contains(strings.ToLower(joinedRow), "subtotal") || strings.Contains(strings.ToLower(joinedRow), "total") { + stopExtracting = true + break + } + if !processingStarted { + nameOfInstrument := "" + if idx, exists := headerMap["Name of the Instrument"]; exists && idx < len(row) { + nameOfInstrument = row[idx] + } + if strings.Contains(nameOfInstrument, "Equity & Equity related") { + processingStarted = true + continue // Skip the header description row and move to the next row + } + } + if processingStarted && !stopExtracting { + for key, idx := range headerMap { + if idx < len(row) { + // println(stockDetail["Name of the Instrument"].(string), "Name of the Instrument in stockDetail") + stock[key] = row[idx] + } else { + stock[key] = "" + } + } + _, ok := stock["Name of the Instrument"].(string) + if ok { + stockDetail = append(stockDetail, stock) + // println("Stock Detail: isin", stock["ISIN"].(string), "Name of the Instrument", stock["Name of the Instrument"].(string)) + } + } + } + // println(mutualFundName, "Mutual Fund Name") + hash := sha256.New() + hashName := sha256.New() + hashName.Write([]byte(mutualFundName)) + + hash.Write([]byte(mutualFundName + month)) + hashedId := hex.EncodeToString(hash.Sum(nil)) + hashedName := hex.EncodeToString(hashName.Sum(nil)) + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("MFHOLDING")) + + validStockDetails := []map[string]interface{}{} + + for _, stockDetail := range stockDetail { + if _, ok := stockDetail["Name of the Instrument"]; !ok { + // fmt.Println("Skipping entry: Missing Name of the Instrument") + continue + } + if _, ok := stockDetail["ISIN"]; !ok || stockDetail["ISIN"] == "" { + // fmt.Println("Skipping entry: Missing ISIN") + continue + } + if _, ok := stockDetail["Quantity"]; !ok { + // // fmt.Println("Skipping entry: Missing Quantity") + continue + } + if _, ok := stockDetail["Market/Fair Value"]; !ok { + // // fmt.Println("Skipping entry: Missing Market/Fair Value") + continue + } + if _, ok := stockDetail["Percentage of AUM"]; !ok { + // // fmt.Println("Skipping entry: Missing Percentage of AUM") + continue + } + validStockDetails = append(validStockDetails, stockDetail) + } + stockDetail = validStockDetails + + document := bson.M{ + "_id": hashedId, + "month": month, + "mutual_fund_name": mutualFundName, + "stock_details": stockDetail, + "hash": hashedName, + "created_at": time.Now(), + } + if mutualFundName == "" || stockDetail == nil || len(stockDetail) == 0 { + // fmt.Println("Skipping empty document") + continue + } + var existingDocument bson.M + err = collection.FindOne(context.TODO(), bson.M{"_id": hashedId}).Decode(&existingDocument) + if err == mongo.ErrNoDocuments { + // Document does not exist, so insert it + _, err := collection.InsertOne(context.TODO(), document) + if err != nil { + log.Fatal(err) + } + // fmt.Println("Document inserted with ID:", insertResult.InsertedID) + } else if err != nil { + log.Fatal(err) + } else { + // Document already exists + // // fmt.Println("Document already exists, skipping insertion.") + } + + } + return nil +} + +func safeGetRow(sheet *xls.WorkSheet, rowIndex int) (*xls.Row, bool) { + defer func() { + if r := recover(); r != nil { + // log.Printf("Recovered from panic when accessing row %d: %v", rowIndex, r) + } + }() + + // Attempt to access the row + row := sheet.Row(rowIndex) + if row == nil { + return nil, false + } + + return row, true +} + +// { +// "mutual_fund_name": "text", +// "stock_details.ISIN": "text", +// "stock_details.Name of the Instrument": "text" +// } + +func processXLSFile(tempInputFile, month string) error { + xlsFile, err := xls.Open(tempInputFile, "utf-8") + if err != nil { + log.Fatalf("Failed to open file: %v", err) + } + + for sheetIndex := 0; sheetIndex < xlsFile.NumSheets(); sheetIndex++ { + sheet := xlsFile.GetSheet(sheetIndex) + if sheet == nil { + // log.Printf("Sheet at index %d is nil. Skipping.", sheetIndex) + continue + } + + // log.Printf("Processing sheet: %s with MaxRow: %d", sheet.Name, sheet.MaxRow) + + headerFound := false + headerMap := make(map[string]int) + stopExtracting := false + processingStarted := false + stockDetail := make([]map[string]interface{}, 0) + + // Extract mutual fund name from the first row, second column + var mutualFundName string + if sheet.MaxRow > 0 { + firstRow, ok := safeGetRow(sheet, 0) + if !ok { + continue + } + if firstRow != nil && firstRow.LastCol() > 1 { + mutualFundName = firstRow.Col(1) + // log.Printf("Mutual Fund Name for sheet %s: %s", sheet.Name, mutualFundName) + } + } + + for rowIndex := 0; rowIndex < int(sheet.MaxRow); rowIndex++ { + if rowIndex >= int(sheet.MaxRow) { + // log.Printf("Row %d in sheet %s is nil or out of range. Skipping.", rowIndex, sheet.Name) + continue + } + // println("Row Index: ", rowIndex, int(sheet.MaxRow)) + row, ok := safeGetRow(sheet, rowIndex) + if !ok || row.LastCol() == 0 { + continue + } + + // Detect headers if not already found + if !headerFound { + for colIndex := 0; colIndex < min(row.LastCol(), 10); colIndex++ { // Limit columns for header detection + cellValue := strings.TrimSpace(strings.ToLower(row.Col(colIndex))) + if strings.Contains(cellValue, "name of the instrument") { + headerFound = true + // Build the header map + for i := 0; i < min(row.LastCol(), 10); i++ { // Limit column count to avoid extra empty columns + header := strings.TrimSpace(strings.ToLower(row.Col(i))) + switch { + case strings.Contains(header, "name of the instrument"): + headerMap["Name of the Instrument"] = i + case strings.Contains(header, "isin"): + headerMap["ISIN"] = i + case strings.Contains(header, "rating/industry") || strings.Contains(header, "industry/rating"): + headerMap["Industry/Rating"] = i + case strings.Contains(header, "quantity"): + headerMap["Quantity"] = i + case strings.Contains(header, "market/fair value") || strings.Contains(header, "market value"): + headerMap["Market/Fair Value"] = i + case strings.Contains(header, "% nav") || strings.Contains(header, "% to nav") || strings.Contains(header, "% net assets"): + headerMap["Percentage of AUM"] = i + } + } + // log.Printf("Header found: %v", headerMap) + break + } + } + continue + } + + // Stop extraction if "Subtotal" or "Total" is encountered + var joinedRow string + for colIndex := 0; colIndex < row.LastCol(); colIndex++ { + joinedRow += row.Col(colIndex) + } + + if strings.Contains(strings.ToLower(joinedRow), "subtotal") || strings.Contains(strings.ToLower(joinedRow), "total") { + stopExtracting = true + break + } + + // Start processing only after "Equity & Equity related" is encountered + if !processingStarted { + nameOfInstrument := "" + if idx, exists := headerMap["Name of the Instrument"]; exists && idx < row.LastCol() { + nameOfInstrument = row.Col(idx) + } + if strings.Contains(nameOfInstrument, "Equity & Equity related") { + processingStarted = true + continue // Skip the header description row and move to the next row + } + } + + // Check if we need to adjust column indices for this row + adjustColumns := false + if headerMap["ISIN"] < row.LastCol() { + isinValue := row.Col(headerMap["ISIN"]) + if !strings.HasPrefix(isinValue, "INE") { // Assuming valid ISINs start with "INE" + // log.Printf("Adjusting row %d columns by 1 due to misalignment", rowIndex) + adjustColumns = true + } + } + + // Data extraction based on header map with potential adjustment + if processingStarted && !stopExtracting { + stock := make(map[string]interface{}) + // log.Printf("Header map: %v", headerMap) + for key, colIndex := range headerMap { + adjustedColIndex := colIndex + if adjustColumns { + adjustedColIndex = colIndex + 1 + } + + if adjustedColIndex < row.LastCol() { + cellValue := row.Col(adjustedColIndex) + // log.Printf("Row %d, Key: %s, Expected Column: %d, Value: %s", rowIndex, key, adjustedColIndex, cellValue) + cleanedValue := strings.ReplaceAll(cellValue, ",", "") // Remove commas + + // Handle percentage values + if strings.HasSuffix(cleanedValue, "%") { + stock[key] = cleanedValue + } else if key == "Market/Fair Value" || key == "Quantity" { + // Force parsing Market/Fair Value and Quantity as floats + if parsedValue, err := strconv.ParseFloat(cleanedValue, 64); err == nil { + stock[key] = parsedValue + // log.Printf("Forced float parsing for key: %s, value: %v", key, parsedValue) + } else { + stock[key] = cellValue // Fallback to original if parsing fails + // log.Printf("Failed to parse as float for key: %s, value: %s", key, cellValue) + } + } else { + stock[key] = cellValue + } + } else { + stock[key] = "" + } + } + // println("*********************") + // Skip rows without meaningful data in "Name of the Instrument" + if stock["Name of the Instrument"] == "" { + continue + } + + // Append the stock detail to the list + stockDetail = append(stockDetail, stock) + } + } + // println(mutualFundName, "Mutual Fund Name") + hash := sha256.New() + hashName := sha256.New() + hashName.Write([]byte(mutualFundName)) + + hash.Write([]byte(mutualFundName + month)) + hashedId := hex.EncodeToString(hash.Sum(nil)) + hashedName := hex.EncodeToString(hashName.Sum(nil)) + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("MFHOLDING")) + + validStockDetails := []map[string]interface{}{} + + for _, stockDetail := range stockDetail { + if _, ok := stockDetail["Name of the Instrument"]; !ok { + // fmt.Println("Skipping entry: Missing Name of the Instrument") + continue + } + if _, ok := stockDetail["ISIN"]; !ok || stockDetail["ISIN"] == "" { + // fmt.Println("Skipping entry: Missing ISIN") + continue + } + if _, ok := stockDetail["Quantity"]; !ok { + // // fmt.Println("Skipping entry: Missing Quantity") + continue + } + if _, ok := stockDetail["Market/Fair Value"]; !ok { + // // fmt.Println("Skipping entry: Missing Market/Fair Value") + continue + } + if _, ok := stockDetail["Percentage of AUM"]; !ok { + // // fmt.Println("Skipping entry: Missing Percentage of AUM") + continue + } + validStockDetails = append(validStockDetails, stockDetail) + } + stockDetail = validStockDetails + + document := bson.M{ + "_id": hashedId, + "month": month, + "mutual_fund_name": mutualFundName, + "stock_details": stockDetail, + "hash": hashedName, + "created_at": time.Now(), + } + if mutualFundName == "" || stockDetail == nil || len(stockDetail) == 0 { + // fmt.Println("Skipping empty document") + continue + } + var existingDocument bson.M + err = collection.FindOne(context.TODO(), bson.M{"_id": hashedId}).Decode(&existingDocument) + if err == mongo.ErrNoDocuments { + // Document does not exist, so insert it + _, err := collection.InsertOne(context.TODO(), document) + if err != nil { + log.Fatal(err) + } + // // fmt.Println("Document inserted with ID:", insertResult.InsertedID) + } else if err != nil { + log.Fatal(err) + } else { + // Document already exists + // // fmt.Println("Document already exists, skipping insertion.") + } + + } + return nil +} + +// Helper function to limit column count to avoid misalignment issues +func min(a, b int) int { + if a < b { + return a + } + return b +} + func extractMonth(fileName string) string { patterns := []*regexp.Regexp{ regexp.MustCompile(`\d{2}\.\d{2}\.\d{4}`), // dd.mm.yyyy diff --git a/services/parse_service_test.go b/services/parse_service_test.go new file mode 100644 index 0000000..8c50a70 --- /dev/null +++ b/services/parse_service_test.go @@ -0,0 +1,60 @@ +package services + +import ( + "testing" +) + + +// Test generated using Keploy +func TestNormalizeWhitespace(t *testing.T) { + input := "This is a test" + expected := "This is a test" + result := normalizeWhitespace(input) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } +} + +// Test generated using Keploy +func TestRemoveZeroWidthChars(t *testing.T) { + input := "Hello\u200BWorld" + expected := "HelloWorld" + result := removeZeroWidthChars(input) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } +} + + +// Test generated using Keploy +func TestExtractMonth(t *testing.T) { + fileName := "report-03-2023.xlsx" + expected := "2023-03-01" + result := extractMonth(fileName) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } +} + + +// Test generated using Keploy +func TestExtractFileName(t *testing.T) { + url := "https://example.com/path/to/file.xlsx" + expected := "file" + result := extractFileName(url) + if result != expected { + t.Errorf("Expected %v but got %v", expected, result) + } +} + + +// Test generated using Keploy +func TestCleanHTMLContent(t *testing.T) { + input := "Hello\u200B World !" + expected := "Hello World !" + result := cleanHTMLContent(input) + if result != expected { + t.Errorf("Expected %v but got %v", expected, result) + } +} + diff --git a/utils/helpers/helpers_test.go b/utils/helpers/helpers_test.go index 7961d54..2794764 100644 --- a/utils/helpers/helpers_test.go +++ b/utils/helpers/helpers_test.go @@ -1,14 +1,19 @@ package helpers import ( - "reflect" - "stockbackend/types" - "strings" - "testing" - - "github.com/PuerkitoBio/goquery" - "go.mongodb.org/mongo-driver/bson/primitive" - "gopkg.in/mgo.v2/bson" + "reflect" + "stockbackend/types" + "strings" + "testing" + + "fmt" + "net/http" + "net/http/httptest" + "os" + + "github.com/PuerkitoBio/goquery" + "go.mongodb.org/mongo-driver/bson/primitive" + "gopkg.in/mgo.v2/bson" ) func TestMatchHeader_NonMatchingPattern(t *testing.T) { @@ -1274,3 +1279,272 @@ func TestCalculateOperatingEfficiencyScore_TotalAssetsMissing(t *testing.T) { t.Errorf("Expected %v got %v", expected, result) } } + +// Test generated using Keploy +func TestIncreaseInRoa_NotEnoughData(t *testing.T) { + netProfit := primitive.A{"1000"} + totalAssets := primitive.A{"5000"} + result := increaseInRoa(netProfit, totalAssets) + if result { + t.Errorf("Expected false, got %v", result) + } +} + +// Test generated using Keploy +func TestSafeToFloat_EmptyString(t *testing.T) { + input := "" + _, err := safeToFloat(input) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +// Test generated using Keploy +func TestFetchPeerData_Success(t *testing.T) { + // Create a mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a valid HTML response + fmt.Fprintln(w, ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Peer Company100015.050002.0%1005%20010%15%
Total1100015.050002.0%1005%20010%15%
+ `) + })) + defer server.Close() + + // Override the COMPANY_URL environment variable + os.Setenv("COMPANY_URL", server.URL) + + dataWarehouseID := "validID" + + peersData, err := FetchPeerData(dataWarehouseID) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(peersData) != 2 { + t.Errorf("Expected 2 peers data, got %d", len(peersData)) + } +} + +// Test generated using Keploy +func TestGetMarketCapCategory_NaNInput(t *testing.T) { + input := "NaN" + expected := "Unknown Category" + result := GetMarketCapCategory(input) + if result != expected { + t.Errorf("Expected %v, got %v", expected, result) + } +} + +// Test generated using Keploy +func TestGetMarketCapCategory_NonNumericInput(t *testing.T) { + input := "invalid" + expected := "Small Cap" + result := GetMarketCapCategory(input) + if result != expected { + t.Errorf("Expected %v, got %v", expected, result) + } +} + +// Test generated using Keploy +func TestCompareWithPeers_MedianParseError(t *testing.T) { + stock := types.Stock{ + Name: "Test Stock", + PE: 15.5, + } + peers := primitive.A{ + bson.M{ + "pe": "10.0", + }, + "invalid median data", + } + result := compareWithPeers(stock, peers) + if result == 0.0 { + t.Errorf("Expected a non-zero score, got %v", result) + } +} + +// Test generated using Keploy +func TestFetchPeerData_Non200Response(t *testing.T) { + // Create a mock server that returns a 500 Internal Server Error + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + // Override the COMPANY_URL environment variable + os.Setenv("COMPANY_URL", server.URL) + + dataWarehouseID := "validID" + + _, err := FetchPeerData(dataWarehouseID) + if err == nil { + t.Errorf("Expected error due to non-200 response, got nil") + } +} + +// Test generated using Keploy +func TestFetchCompanyData_ValidURL(t *testing.T) { + // Create a mock server that returns valid HTML + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, `
`) + })) + defer server.Close() + + // Fetch company data using the mock server URL + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if data == nil { + t.Errorf("Expected non-nil data, got nil") + } +} + +// Test generated using Keploy +func TestFetchCompanyData_MissingSections(t *testing.T) { + html := ` + + +
+
  • + Market Cap + 100000 +
    • + + ` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprintln(w, html) + })) + defer server.Close() + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if _, ok := data["profitLoss"]; ok { + t.Errorf("Did not expect profitLoss data") + } + if _, ok := data["balanceSheet"]; ok { + t.Errorf("Did not expect balanceSheet data") + } +} + + +// Test generated using Keploy +func TestFetchCompanyData_ProsAndCons(t *testing.T) { + html := ` + + +
    +
      +
    • Strong financials
      • +
    • Growing revenue
      • +
    +
    +
    +
      +
    • High debt
      • +
    +
    + + ` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, html) + })) + defer server.Close() + + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + expectedPros := []string{"Strong financials", "Growing revenue"} + expectedCons := []string{"High debt"} + + if !reflect.DeepEqual(data["pros"], expectedPros) { + t.Errorf("Expected pros %v, got %v", expectedPros, data["pros"]) + } + + if !reflect.DeepEqual(data["cons"], expectedCons) { + t.Errorf("Expected cons %v, got %v", expectedCons, data["cons"]) + } +} + + + +// Test generated using Keploy +func TestCompareWithPeers_ValidData(t *testing.T) { + stock := types.Stock{ + Name: "Test Stock", + PE: 15.5, + MarketCap: 10000, + DividendYield: 2.5, + ROCE: 20.0, + QuarterlySales: 7000.0, + QuarterlyProfit: 2000.0, + } + peers := primitive.A{ + bson.M{"pe": "10.0", "market_cap": "8000", "div_yield": "2.0%", "roce": "18.0", "sales_qtr": "500", "np_qtr": "50"}, + bson.M{"pe": "12.0", "market_cap": "9000", "div_yield": "2.2%", "roce": "19.0", "sales_qtr": "600", "np_qtr": "60"}, + bson.M{"pe": "11.0", "market_cap": "8500", "div_yield": "2.1%", "roce": "18.5", "sales_qtr": "550", "np_qtr": "55"}, + } + result := compareWithPeers(stock, peers) + if result == 0.0 { + t.Errorf("Expected non-zero peer comparison score, got %v", result) + } +} + +// Test generated using Keploy +func TestFetchPeerData_MalformedURL(t *testing.T) { + // Override the COMPANY_URL environment variable with a malformed URL + os.Setenv("COMPANY_URL", "http://%41:8080/") // %41 is 'A', but this is a malformed URL + + dataWarehouseID := "validID" + _, err := FetchPeerData(dataWarehouseID) + if err == nil { + t.Errorf("Expected error due to malformed URL, got nil") + } +} + +// Test generated using Keploy +func TestIncreaseInRoa_InvalidElements(t *testing.T) { + netProfit := primitive.A{1000, 1500, 2000} // Integers instead of strings + totalAssets := primitive.A{5000, 5500, "invalid"} // Last element is not a valid string + result := increaseInRoa(netProfit, totalAssets) + if result != false { + t.Errorf("Expected false, got %v", result) + } +} + + From 25548fe77f3a80d0781959286a7153f5a601c99e Mon Sep 17 00:00:00 2001 From: shivamsouravjha <2019145@iiitdmj.ac.in> Date: Fri, 8 Nov 2024 11:40:32 +0530 Subject: [PATCH 11/13] chore: update mod files Signed-off-by: shivamsouravjha <2019145@iiitdmj.ac.in> --- go.mod | 23 +++++++++++++---------- go.sum | 49 +++++++++++++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 1cd8bd0..7a8f53e 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,13 @@ go 1.23.0 require ( github.com/PuerkitoBio/goquery v1.10.0 github.com/cloudinary/cloudinary-go/v2 v2.9.0 + github.com/extrame/xls v0.0.1 github.com/getsentry/sentry-go v0.29.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 - github.com/xuri/excelize/v2 v2.8.1 + github.com/robfig/cron/v3 v3.0.1 + github.com/wailsapp/mimetype v1.4.1 + github.com/xuri/excelize/v2 v2.9.0 go.mongodb.org/mongo-driver v1.17.1 go.uber.org/zap v1.27.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 @@ -23,7 +26,8 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/creasty/defaults v1.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -42,22 +46,21 @@ require ( github.com/montanaflynn/stats v0.7.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect - github.com/richardlehane/msoleps v1.0.3 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect - github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect + github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect + github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d36742e..c67720d 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,12 @@ github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbD github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c= +github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw= +github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k= +github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA= github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -78,8 +82,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= -github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -98,18 +102,20 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/wailsapp/mimetype v1.4.1 h1:pQN9ycO7uo4vsUUuPeHEYoUkLVkaRntMnHJxVwYhwHs= +github.com/wailsapp/mimetype v1.4.1/go.mod h1:9aV5k31bBOv5z6u+QP8TltzvNGJPmNJD4XlAL3U+j3o= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0= -github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ= -github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE= -github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4= -github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d h1:llb0neMWDQe87IzJLS4Ci7psK/lVsjIS2otl+1WyRyY= +github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= +github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -126,19 +132,20 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= -golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -146,26 +153,28 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From c60d221db42b728a3429760961290dc0895e035f Mon Sep 17 00:00:00 2001 From: shivamsouravjha <2019145@iiitdmj.ac.in> Date: Fri, 8 Nov 2024 11:54:06 +0530 Subject: [PATCH 12/13] fix build errors Signed-off-by: shivamsouravjha <2019145@iiitdmj.ac.in> --- services/parse_service.go | 31 ++++--------------------------- utils/helpers/helpers.go | 4 ++-- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/services/parse_service.go b/services/parse_service.go index f3688b9..adee3cd 100644 --- a/services/parse_service.go +++ b/services/parse_service.go @@ -32,14 +32,6 @@ import ( "gopkg.in/mgo.v2/bson" ) -func assert(b bool, mess string) { - green := "\033[32m" - reset := "\033[0m" - if os.Getenv("DEBUG") == "true" { - fmt.Println(green+"Assert PASSED: ", mess+reset) - } -} - func setupCheck() { if len(os.Getenv("CLOUDINARY_URL")) < 5 { panic("Please provied a CLOUDINARY_URL. Run `export CLOUDINARY_URL=your@url` before in your shell for linux and MacOS") @@ -475,14 +467,11 @@ func processXLSXFile(tempInputFile, month string) error { // Document does not exist, so insert it _, err := collection.InsertOne(context.TODO(), document) if err != nil { - log.Fatal(err) + println(err.Error()) } // fmt.Println("Document inserted with ID:", insertResult.InsertedID) } else if err != nil { - log.Fatal(err) - } else { - // Document already exists - // // fmt.Println("Document already exists, skipping insertion.") + println(err.Error()) } } @@ -719,14 +708,11 @@ func processXLSFile(tempInputFile, month string) error { // Document does not exist, so insert it _, err := collection.InsertOne(context.TODO(), document) if err != nil { - log.Fatal(err) + println(err.Error()) } // // fmt.Println("Document inserted with ID:", insertResult.InsertedID) } else if err != nil { - log.Fatal(err) - } else { - // Document already exists - // // fmt.Println("Document already exists, skipping insertion.") + println(err.Error()) } } @@ -805,15 +791,6 @@ func parseDate(dateStr string) string { return "" } -func checkFileExistence(cld *cloudinary.Cloudinary, publicID string) (bool, error) { - assert(cld == nil, "checkFileExistence cld == null") - assert(publicID == "", "checkFileExistence publicId == \"\"") - _, err := cld.Admin.Asset(context.Background(), admin.AssetParams{ - PublicID: publicID, - }) - return !strings.Contains(err.Error(), "not found"), nil -} - func extractFileName(fileURL string) string { fileName := path.Base(fileURL) return strings.TrimSuffix(fileName, path.Ext(fileName)) diff --git a/utils/helpers/helpers.go b/utils/helpers/helpers.go index 13e7629..81c01f1 100644 --- a/utils/helpers/helpers.go +++ b/utils/helpers/helpers.go @@ -3,7 +3,7 @@ package helpers import ( "errors" "fmt" - "io/ioutil" + "io" "math" "net/http" "os" @@ -379,7 +379,7 @@ func FetchPeerData(dataWarehouseID string) ([]map[string]string, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - bodyBytes, _ := ioutil.ReadAll(resp.Body) + bodyBytes, _ := io.ReadAll(resp.Body) bodyString := string(bodyBytes) zap.L().Error("Received non-200 response code", zap.Int("status_code", resp.StatusCode), zap.String("body", bodyString)) return nil, fmt.Errorf("received non-200 response code from peers API: %d", resp.StatusCode) From c5ed20de77cc1fadba493ff57b11475bab62af6d Mon Sep 17 00:00:00 2001 From: shivamsouravjha <2019145@iiitdmj.ac.in> Date: Fri, 8 Nov 2024 11:55:34 +0530 Subject: [PATCH 13/13] fix build errors Signed-off-by: shivamsouravjha <2019145@iiitdmj.ac.in> --- services/parse_service_test.go | 67 +++++++------- utils/helpers/helpers_test.go | 159 ++++++++++++++++----------------- 2 files changed, 108 insertions(+), 118 deletions(-) diff --git a/services/parse_service_test.go b/services/parse_service_test.go index 8c50a70..e63a7e4 100644 --- a/services/parse_service_test.go +++ b/services/parse_service_test.go @@ -1,60 +1,55 @@ package services import ( - "testing" + "testing" ) - // Test generated using Keploy func TestNormalizeWhitespace(t *testing.T) { - input := "This is a test" - expected := "This is a test" - result := normalizeWhitespace(input) - if result != expected { - t.Errorf("Expected %s but got %s", expected, result) - } + input := "This is a test" + expected := "This is a test" + result := normalizeWhitespace(input) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } } // Test generated using Keploy func TestRemoveZeroWidthChars(t *testing.T) { - input := "Hello\u200BWorld" - expected := "HelloWorld" - result := removeZeroWidthChars(input) - if result != expected { - t.Errorf("Expected %s but got %s", expected, result) - } + input := "Hello\u200BWorld" + expected := "HelloWorld" + result := removeZeroWidthChars(input) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } } - // Test generated using Keploy func TestExtractMonth(t *testing.T) { - fileName := "report-03-2023.xlsx" - expected := "2023-03-01" - result := extractMonth(fileName) - if result != expected { - t.Errorf("Expected %s but got %s", expected, result) - } + fileName := "report-03-2023.xlsx" + expected := "2023-03-01" + result := extractMonth(fileName) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } } - // Test generated using Keploy func TestExtractFileName(t *testing.T) { - url := "https://example.com/path/to/file.xlsx" - expected := "file" - result := extractFileName(url) - if result != expected { - t.Errorf("Expected %v but got %v", expected, result) - } + url := "https://example.com/path/to/file.xlsx" + expected := "file" + result := extractFileName(url) + if result != expected { + t.Errorf("Expected %v but got %v", expected, result) + } } - // Test generated using Keploy func TestCleanHTMLContent(t *testing.T) { - input := "Hello\u200B World !" - expected := "Hello World !" - result := cleanHTMLContent(input) - if result != expected { - t.Errorf("Expected %v but got %v", expected, result) - } + input := "Hello\u200B World !" + expected := "Hello World !" + result := cleanHTMLContent(input) + if result != expected { + t.Errorf("Expected %v but got %v", expected, result) + } } - diff --git a/utils/helpers/helpers_test.go b/utils/helpers/helpers_test.go index 2794764..1e88703 100644 --- a/utils/helpers/helpers_test.go +++ b/utils/helpers/helpers_test.go @@ -1,19 +1,19 @@ package helpers import ( - "reflect" - "stockbackend/types" - "strings" - "testing" - - "fmt" - "net/http" - "net/http/httptest" - "os" - - "github.com/PuerkitoBio/goquery" - "go.mongodb.org/mongo-driver/bson/primitive" - "gopkg.in/mgo.v2/bson" + "reflect" + "stockbackend/types" + "strings" + "testing" + + "fmt" + "net/http" + "net/http/httptest" + "os" + + "github.com/PuerkitoBio/goquery" + "go.mongodb.org/mongo-driver/bson/primitive" + "gopkg.in/mgo.v2/bson" ) func TestMatchHeader_NonMatchingPattern(t *testing.T) { @@ -1433,7 +1433,7 @@ func TestFetchCompanyData_ValidURL(t *testing.T) { // Test generated using Keploy func TestFetchCompanyData_MissingSections(t *testing.T) { - html := ` + html := `
    @@ -1443,27 +1443,26 @@ func TestFetchCompanyData_MissingSections(t *testing.T) { ` - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/html") - fmt.Fprintln(w, html) - })) - defer server.Close() - data, err := FetchCompanyData(server.URL) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if _, ok := data["profitLoss"]; ok { - t.Errorf("Did not expect profitLoss data") - } - if _, ok := data["balanceSheet"]; ok { - t.Errorf("Did not expect balanceSheet data") - } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprintln(w, html) + })) + defer server.Close() + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if _, ok := data["profitLoss"]; ok { + t.Errorf("Did not expect profitLoss data") + } + if _, ok := data["balanceSheet"]; ok { + t.Errorf("Did not expect balanceSheet data") + } } - // Test generated using Keploy func TestFetchCompanyData_ProsAndCons(t *testing.T) { - html := ` + html := `
    @@ -1479,72 +1478,68 @@ func TestFetchCompanyData_ProsAndCons(t *testing.T) {
    ` - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, html) - })) - defer server.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, html) + })) + defer server.Close() - data, err := FetchCompanyData(server.URL) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } - expectedPros := []string{"Strong financials", "Growing revenue"} - expectedCons := []string{"High debt"} + expectedPros := []string{"Strong financials", "Growing revenue"} + expectedCons := []string{"High debt"} - if !reflect.DeepEqual(data["pros"], expectedPros) { - t.Errorf("Expected pros %v, got %v", expectedPros, data["pros"]) - } + if !reflect.DeepEqual(data["pros"], expectedPros) { + t.Errorf("Expected pros %v, got %v", expectedPros, data["pros"]) + } - if !reflect.DeepEqual(data["cons"], expectedCons) { - t.Errorf("Expected cons %v, got %v", expectedCons, data["cons"]) - } + if !reflect.DeepEqual(data["cons"], expectedCons) { + t.Errorf("Expected cons %v, got %v", expectedCons, data["cons"]) + } } - - // Test generated using Keploy func TestCompareWithPeers_ValidData(t *testing.T) { - stock := types.Stock{ - Name: "Test Stock", - PE: 15.5, - MarketCap: 10000, - DividendYield: 2.5, - ROCE: 20.0, - QuarterlySales: 7000.0, - QuarterlyProfit: 2000.0, - } - peers := primitive.A{ - bson.M{"pe": "10.0", "market_cap": "8000", "div_yield": "2.0%", "roce": "18.0", "sales_qtr": "500", "np_qtr": "50"}, - bson.M{"pe": "12.0", "market_cap": "9000", "div_yield": "2.2%", "roce": "19.0", "sales_qtr": "600", "np_qtr": "60"}, - bson.M{"pe": "11.0", "market_cap": "8500", "div_yield": "2.1%", "roce": "18.5", "sales_qtr": "550", "np_qtr": "55"}, - } - result := compareWithPeers(stock, peers) - if result == 0.0 { - t.Errorf("Expected non-zero peer comparison score, got %v", result) - } + stock := types.Stock{ + Name: "Test Stock", + PE: 15.5, + MarketCap: 10000, + DividendYield: 2.5, + ROCE: 20.0, + QuarterlySales: 7000.0, + QuarterlyProfit: 2000.0, + } + peers := primitive.A{ + bson.M{"pe": "10.0", "market_cap": "8000", "div_yield": "2.0%", "roce": "18.0", "sales_qtr": "500", "np_qtr": "50"}, + bson.M{"pe": "12.0", "market_cap": "9000", "div_yield": "2.2%", "roce": "19.0", "sales_qtr": "600", "np_qtr": "60"}, + bson.M{"pe": "11.0", "market_cap": "8500", "div_yield": "2.1%", "roce": "18.5", "sales_qtr": "550", "np_qtr": "55"}, + } + result := compareWithPeers(stock, peers) + if result == 0.0 { + t.Errorf("Expected non-zero peer comparison score, got %v", result) + } } // Test generated using Keploy func TestFetchPeerData_MalformedURL(t *testing.T) { - // Override the COMPANY_URL environment variable with a malformed URL - os.Setenv("COMPANY_URL", "http://%41:8080/") // %41 is 'A', but this is a malformed URL + // Override the COMPANY_URL environment variable with a malformed URL + os.Setenv("COMPANY_URL", "http://%41:8080/") // %41 is 'A', but this is a malformed URL - dataWarehouseID := "validID" - _, err := FetchPeerData(dataWarehouseID) - if err == nil { - t.Errorf("Expected error due to malformed URL, got nil") - } + dataWarehouseID := "validID" + _, err := FetchPeerData(dataWarehouseID) + if err == nil { + t.Errorf("Expected error due to malformed URL, got nil") + } } // Test generated using Keploy func TestIncreaseInRoa_InvalidElements(t *testing.T) { - netProfit := primitive.A{1000, 1500, 2000} // Integers instead of strings - totalAssets := primitive.A{5000, 5500, "invalid"} // Last element is not a valid string - result := increaseInRoa(netProfit, totalAssets) - if result != false { - t.Errorf("Expected false, got %v", result) - } + netProfit := primitive.A{1000, 1500, 2000} // Integers instead of strings + totalAssets := primitive.A{5000, 5500, "invalid"} // Last element is not a valid string + result := increaseInRoa(netProfit, totalAssets) + if result != false { + t.Errorf("Expected false, got %v", result) + } } - -