From 53ebdca758d12219ab985c1c5f3b260dd401f5c7 Mon Sep 17 00:00:00 2001 From: Atharva Date: Thu, 3 Oct 2024 21:53:50 +0100 Subject: [PATCH] restructing the codebase --- .gitignore | 1 + clients/mongo/mongo_client.go | 36 ++ controllers/file_controller.go | 45 ++ controllers/health_controller.go | 15 + main.go | 1040 +----------------------------- routes/routes.go | 17 + services/file_service.go | 274 ++++++++ types/types.go | 20 + utils/constants/constants.go | 11 + utils/helpers/helpers.go | 677 +++++++++++++++++++ 10 files changed, 1116 insertions(+), 1020 deletions(-) create mode 100644 .gitignore create mode 100644 clients/mongo/mongo_client.go create mode 100644 controllers/file_controller.go create mode 100644 controllers/health_controller.go create mode 100644 routes/routes.go create mode 100644 services/file_service.go create mode 100644 types/types.go create mode 100644 utils/constants/constants.go create mode 100644 utils/helpers/helpers.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..54eb2cb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +stockbackend \ No newline at end of file diff --git a/clients/mongo/mongo_client.go b/clients/mongo/mongo_client.go new file mode 100644 index 0000000..3090c93 --- /dev/null +++ b/clients/mongo/mongo_client.go @@ -0,0 +1,36 @@ +package mongo_client + +import ( + "context" + "os" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" + "gopkg.in/mgo.v2/bson" +) + +var ( + Client *mongo.Client +) + +func init() { + serverAPI := options.ServerAPI(options.ServerAPIVersion1) + mongoURI := os.Getenv("MONGO_URI") + // zap.L().Info("Mongo URI", zap.String("uri", mongoURI)) + opts := options.Client().ApplyURI(mongoURI).SetServerAPIOptions(serverAPI) + // Create a new client and connect to the server + var err error + Client, err = mongo.Connect(context.TODO(), opts) + if err != nil { + panic(err) + } + + // Send a ping to confirm a successful connection + pingCmd := bson.M{"ping": 1} + if err := Client.Database("admin").RunCommand(context.TODO(), pingCmd).Err(); err != nil { + panic(err) + } + + zap.L().Info("Connected to MongoDB") +} diff --git a/controllers/file_controller.go b/controllers/file_controller.go new file mode 100644 index 0000000..32db37a --- /dev/null +++ b/controllers/file_controller.go @@ -0,0 +1,45 @@ +package controllers + +import ( + "stockbackend/services" + + "github.com/gin-gonic/gin" +) + +type FileControllerI interface { + ParseXLSXFile(ctx *gin.Context) +} + +type fileController struct{} + +var FileController FileControllerI = &fileController{} + +func (f *fileController) ParseXLSXFile(ctx *gin.Context) { + // Parse the form and retrieve the uploaded files + form, err := ctx.MultipartForm() + if err != nil { + ctx.JSON(400, gin.H{"error": "Error parsing form data"}) + return + } + + // Retrieve the files from the form + files := form.File["files"] + if len(files) == 0 { + ctx.JSON(400, gin.H{"error": "No files found"}) + return + } + + // Set headers for chunked transfer (if needed) + ctx.Writer.Header().Set("Content-Type", "text/plain") + ctx.Writer.Header().Set("Cache-Control", "no-cache") + ctx.Writer.Header().Set("Connection", "keep-alive") + + err = services.FileService.ParseXLSXFile(ctx, files) + if err != nil { + ctx.JSON(500, gin.H{"error": err.Error()}) + return + } + + ctx.Writer.Write([]byte("\nStream complete.\n")) + ctx.Writer.Flush() // Ensure the final response is sent +} diff --git a/controllers/health_controller.go b/controllers/health_controller.go new file mode 100644 index 0000000..86a2d0e --- /dev/null +++ b/controllers/health_controller.go @@ -0,0 +1,15 @@ +package controllers + +import "github.com/gin-gonic/gin" + +type HealthControllerI interface { + IsRunning(ctx *gin.Context) +} + +type healthController struct{} + +var HealthController HealthControllerI = &healthController{} + +func (h *healthController) IsRunning(ctx *gin.Context) { + ctx.JSON(200, gin.H{"message": "Server is running"}) +} diff --git a/main.go b/main.go index 84a64fe..edbec2a 100644 --- a/main.go +++ b/main.go @@ -2,50 +2,20 @@ package main import ( "context" - "encoding/json" - "fmt" - "io/ioutil" "log" - "math" "net/http" - "net/url" "os" "os/exec" "os/signal" - "regexp" - "strconv" - "strings" - "sync" + "stockbackend/routes" "syscall" "time" - "github.com/PuerkitoBio/goquery" - "github.com/cloudinary/cloudinary-go/v2" - "github.com/cloudinary/cloudinary-go/v2/api/uploader" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/joho/godotenv" - "github.com/xuri/excelize/v2" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "go.uber.org/zap" - "gopkg.in/mgo.v2/bson" ) -// Stock represents the data of a stock -type Stock struct { - Name string - PE float64 - MarketCap float64 - DividendYield float64 - ROCE float64 - QuarterlySales float64 - QuarterlyProfit float64 - Cons []string - Pros []string -} - // Peer represents the data of a peer stock type Peer struct { Name string @@ -65,271 +35,6 @@ type QuarterlyData struct { ROCE float64 } -// compareWithPeers calculates a peer comparison score -func compareWithPeers(stock Stock, peers interface{}) float64 { - peerScore := 0.0 - var medianScore float64 - - if arr, ok := peers.(primitive.A); ok { - // Ensure there are enough peers to compare - if len(arr) < 2 { - zap.L().Warn("Not enough peers to compare") - return 0.0 - } - - for _, peerRaw := range arr[:len(arr)-1] { - peer := peerRaw.(bson.M) - - // Parse peer values to float64 - peerPE := parseFloat(peer["pe"]) - peerMarketCap := parseFloat(peer["market_cap"]) - peerDividendYield := parseFloat(peer["div_yield"]) - peerROCE := parseFloat(peer["roce"]) - peerQuarterlySales := parseFloat(peer["sales_qtr"]) - peerQuarterlyProfit := parseFloat(peer["np_qtr"]) - - // Example scoring logic - if stock.PE < peerPE { - peerScore += 10 - } else { - peerScore += math.Max(0, 10-(stock.PE-peerPE)) - } - - if stock.MarketCap > peerMarketCap { - peerScore += 5 - } - - if stock.DividendYield > peerDividendYield { - peerScore += 5 - } - - if stock.ROCE > peerROCE { - peerScore += 10 - } - - if stock.QuarterlySales > peerQuarterlySales { - peerScore += 5 - } - - if stock.QuarterlyProfit > peerQuarterlyProfit { - peerScore += 10 - } - } - medianRaw := arr[len(arr)-1] - median := medianRaw.(bson.M) - - // Parse median values to float64 - medianPE := parseFloat(median["pe"]) - medianMarketCap := parseFloat(median["market_cap"]) - medianDividendYield := parseFloat(median["div_yield"]) - medianROCE := parseFloat(median["roce"]) - medianQuarterlySales := parseFloat(median["sales_qtr"]) - medianQuarterlyProfit := parseFloat(median["np_qtr"]) - - // Adjust score based on median comparison - if stock.PE < medianPE { - peerScore += 5 - } else { - peerScore += math.Max(0, 5-(stock.PE-medianPE)) - } - - if stock.MarketCap > medianMarketCap { - peerScore += 3 - } - - if stock.DividendYield > medianDividendYield { - peerScore += 3 - } - - if stock.ROCE > medianROCE { - peerScore += 5 - } - - if stock.QuarterlySales > medianQuarterlySales { - peerScore += 2 - } - - if stock.QuarterlyProfit > medianQuarterlyProfit { - peerScore += 5 - } - - // Normalize by the number of peers (excluding the median) - peerCount := len(arr) - 1 - if peerCount > 0 { - return peerScore / float64(peerCount) - } - - // Normalize by the number of peers excluding the last element - } - - // Combine peerScore with medianScore (example: giving 10% weight to the median) - finalScore := (peerScore * 0.9) + (medianScore * 0.1) - - return finalScore -} - -// Helper function to convert values from map to float64 -func parseFloat(value interface{}) float64 { - switch v := value.(type) { - case string: - f, err := strconv.ParseFloat(v, 64) - if err != nil { - return 0.0 - } - return f - case float64: - return v - case int: - return float64(v) - default: - return 0.0 - } -} -func analyzeTrend(stock Stock, pastData interface{}) float64 { - trendScore := 0.0 - comparisons := 0 // Keep track of the number of comparisons - - // Ensure pastData is in bson.M format - if data, ok := pastData.(bson.M); ok { - for _, quarterData := range data { - // zap.L().Info("Processing quarter", zap.String("quarter", key)) - - // Process the quarter data if it's a primitive.A (array of quarter maps) - if quarterArray, ok := quarterData.(primitive.A); ok { - var prevElem bson.M - for i, elem := range quarterArray { - if elemMap, ok := elem.(bson.M); ok { - // zap.L().Info("Processing quarter element", zap.Any("element", elemMap)) - - // Only perform comparisons starting from the second element - if i > 0 && prevElem != nil { - // zap.L().Info("Comparing with previous element", zap.Any("previous", prevElem), zap.Any("current", elemMap)) - - // Iterate over the keys in the current quarter and compare with previous quarter - for key, v := range elemMap { - if prevVal, ok := prevElem[key]; ok { - // Compare consecutive values for the same key - if toFloat(v) > toFloat(prevVal) { - trendScore += 5 - } else if toFloat(v) < toFloat(prevVal) { - trendScore -= 5 - } - // Increment comparisons for each valid comparison - comparisons++ - } - } - } - // Update previous element for next iteration - prevElem = elemMap - } - } - } - } - } - - // Normalize the score by dividing it by the number of comparisons - if comparisons > 0 { - return trendScore / float64(comparisons) - } - return 0.0 // Return 0 if no comparisons were made -} - -// prosConsAdjustment calculates score adjustments based on pros and cons -func prosConsAdjustment(stock Stock) float64 { - adjustment := 0.0 - - // Adjust score based on pros - // for _, pro := range stock.Pros { - // zap.L().Info("Pro", zap.String("pro", pro)) // This line is optional, just showing how we could use 'pro' - adjustment += toFloat(1.0 * len(stock.Pros)) - // } - - // Adjust score based on cons - // for _, con := range stock.Cons { - // zap.L().Info("Con", zap.String("con", con)) // This line is optional, just showing how we could use 'con' - adjustment -= toFloat(1.0 * len(stock.Cons)) - // }/ - - return adjustment -} - -// rateStock calculates the final stock rating -func rateStock(stock map[string]interface{}) float64 { - // zap.L().Info("Stock data", zap.Any("stock", stock)) - stockData := Stock{ - Name: stock["name"].(string), - PE: toFloat(stock["stockPE"]), - MarketCap: toFloat(stock["marketCap"]), - DividendYield: toFloat(stock["dividendYield"]), - ROCE: toFloat(stock["roce"]), - Cons: toStringArray(stock["cons"]), - Pros: toStringArray(stock["pros"]), - } - // zap.L().Info("Stock data", zap.Any("stock", stockData)) - // zap.L().Info("Stock data", zap.Any("stock", stockData)) - peerComparisonScore := compareWithPeers(stockData, stock["peers"]) * 0.5 - trendScore := analyzeTrend(stockData, stock["quarterlyResults"]) * 0.4 - // prosConsScore := prosConsAdjustment(stock) * 0.1 - // zap.L().Info("Peer comparison score", zap.Float64("peerComparisonScore", peerComparisonScore)) - - finalScore := peerComparisonScore + trendScore - finalScore = math.Round(finalScore*100) / 100 - return finalScore -} - -// Helper function to normalize strings -func normalizeString(s string) string { - return strings.ToLower(strings.TrimSpace(s)) -} - -// Helper function to match header titles -func matchHeader(cellValue string, patterns []string) bool { - normalizedValue := normalizeString(cellValue) - for _, pattern := range patterns { - matched, _ := regexp.MatchString(pattern, normalizedValue) - if matched { - return true - } - } - return false -} - -var ( - client *mongo.Client - once sync.Once -) - -func init() { - err := godotenv.Load() - if err != nil { - log.Println("Error loading .env file") - } - once.Do(func() { - serverAPI := options.ServerAPI(options.ServerAPIVersion1) - mongoURI := os.Getenv("MONGO_URI") - // zap.L().Info("Mongo URI", zap.String("uri", mongoURI)) - opts := options.Client().ApplyURI(mongoURI).SetServerAPIOptions(serverAPI) - // Create a new client and connect to the server - var err error - client, err = mongo.Connect(context.TODO(), opts) - if err != nil { - panic(err) - } - - // Send a ping to confirm a successful connection - pingCmd := bson.M{"ping": 1} - if err := client.Database("admin").RunCommand(context.TODO(), pingCmd).Err(); err != nil { - panic(err) - } - - zap.L().Info("Connected to MongoDB") - - }) - - return - -} - func CORSMiddleware() gin.HandlerFunc { return func(c *gin.Context) { @@ -373,390 +78,22 @@ func GracefulShutdown(server *http.Server, ticker *time.Ticker) { }() } -func checkInstrumentName(input string) bool { - // Regular expression to match "Name of the Instrument" or "Name of Instrument" - pattern := `Name of (the )?Instrument` - - // Compile the regex - re := regexp.MustCompile(pattern) - - // Check if the pattern matches the input string - return re.MatchString(input) -} - -var ( - // map few values to some in constand string - mapValues = map[string]string{ - "Sun Pharmaceutical Industries Limited": "Sun Pharma.Inds.", - "KEC International Limited": "K E C Intl.", - "Sandhar Technologies Limited": "Sandhar Tech", - "Samvardhana Motherson International Limited": "Samvardh. Mothe.", - "Coromandel International Limited": "Coromandel Inter", - } -) - -func parseXlsxFile(c *gin.Context) { - // Parse the form and retrieve the uploaded files - form, err := c.MultipartForm() - if err != nil { - c.JSON(400, gin.H{"error": "Error parsing form data"}) - return - } - - // Retrieve the files from the form - files := form.File["files"] - if len(files) == 0 { - c.JSON(400, gin.H{"error": "No files found"}) - return - } - - // Initialize Cloudinary - cld, err := cloudinary.NewFromURL(os.Getenv("CLOUDINARY_URL")) - if err != nil { - c.JSON(500, gin.H{"error": "Error initializing Cloudinary"}) - return - } - - // Set headers for chunked transfer (if needed) - c.Writer.Header().Set("Content-Type", "text/plain") - c.Writer.Header().Set("Cache-Control", "no-cache") - c.Writer.Header().Set("Connection", "keep-alive") - - // Iterate over the uploaded XLSX files - for _, fileHeader := range files { - // Open each file for processing - file, err := fileHeader.Open() - if err != nil { - zap.L().Error("Error opening file", zap.String("filename", fileHeader.Filename), zap.Error(err)) - continue - } - defer file.Close() - - // Generate a UUID for the filename - uuid := uuid.New().String() - cloudinaryFilename := uuid + ".xlsx" - - // Upload file to Cloudinary - uploadResult, err := cld.Upload.Upload(c, file, uploader.UploadParams{ - PublicID: cloudinaryFilename, - Folder: "xlsx_uploads", - }) - if err != nil { - zap.L().Error("Error uploading file to Cloudinary", zap.String("filename", fileHeader.Filename), zap.Error(err)) - continue - } - - zap.L().Info("File uploaded to Cloudinary", zap.String("filename", fileHeader.Filename), zap.String("url", uploadResult.SecureURL)) - - // Create a new reader from the uploaded file - file.Seek(0, 0) // Reset file pointer to the beginning - f, err := excelize.OpenReader(file) - if err != nil { - zap.L().Error("Error parsing XLSX file", zap.String("filename", fileHeader.Filename), zap.Error(err)) - continue - } - defer f.Close() - - // Get all the sheet names - sheetList := f.GetSheetList() - // Loop through the sheets and extract relevant information - for _, sheet := range sheetList { - zap.L().Info("Processing file", zap.String("filename", fileHeader.Filename), zap.String("sheet", sheet)) - - // Get all the rows in the sheet - rows, err := f.GetRows(sheet) - if err != nil { - zap.L().Error("Error reading rows from sheet", zap.String("sheet", sheet), zap.Error(err)) - continue - } - - headerFound := false - headerMap := make(map[string]int) - stopExtracting := false - - // Loop through the rows in the sheet - for _, row := range rows { - if len(row) == 0 { - continue - } - - if !headerFound { - for _, cell := range row { - if matchHeader(cell, []string{`name\s*of\s*(the)?\s*instrument`}) { - headerFound = true - // Build the header map - for i, headerCell := range row { - normalizedHeader := normalizeString(headerCell) - // Map possible variations to standard keys - switch { - case matchHeader(normalizedHeader, []string{`name\s*of\s*(the)?\s*instrument`}): - headerMap["Name of the Instrument"] = i - case matchHeader(normalizedHeader, []string{`isin`}): - headerMap["ISIN"] = i - case matchHeader(normalizedHeader, []string{`rating\s*/\s*industry`, `industry\s*/\s*rating`}): - headerMap["Industry/Rating"] = i - case matchHeader(normalizedHeader, []string{`quantity`}): - headerMap["Quantity"] = i - case matchHeader(normalizedHeader, []string{`market\s*/\s*fair\s*value.*`, `market\s*value.*`}): - headerMap["Market/Fair Value"] = i - case matchHeader(normalizedHeader, []string{`%.*nav`, `%.*net\s*assets`}): - headerMap["Percentage of AUM"] = i - } - } - // zap.L().Info("Header found", zap.Any("headerMap", headerMap)) - break - } - } - continue - } - - // Check for the end marker "Subtotal" or "Total" - joinedRow := strings.Join(row, "") - if strings.Contains(strings.ToLower(joinedRow), "subtotal") || strings.Contains(strings.ToLower(joinedRow), "total") { - stopExtracting = true - break - } - - if !stopExtracting { - stockDetail := make(map[string]interface{}) - - // Extract data using the header map - for key, idx := range headerMap { - if idx < len(row) { - stockDetail[key] = row[idx] - } else { - stockDetail[key] = "" - } - } - - // Check if the stockDetail has meaningful data - if stockDetail["Name of the Instrument"] == nil || stockDetail["Name of the Instrument"] == "" { - continue - } - - // Additional processing - instrumentName, ok := stockDetail["Name of the Instrument"].(string) - if !ok { - continue - } - - // Apply mapping if exists - if mappedName, exists := mapValues[instrumentName]; exists { - stockDetail["Name of the Instrument"] = mappedName - instrumentName = mappedName - } - - // Clean up the query string - queryString := instrumentName - queryString = strings.ReplaceAll(queryString, " Corporation ", " Corpn ") - queryString = strings.ReplaceAll(queryString, " corporation ", " Corpn ") - queryString = strings.ReplaceAll(queryString, " Limited", " Ltd ") - queryString = strings.ReplaceAll(queryString, " limited", " Ltd ") - queryString = strings.ReplaceAll(queryString, " and ", " & ") - queryString = strings.ReplaceAll(queryString, " And ", " & ") - - // Prepare the text search filter - textSearchFilter := bson.M{ - "$text": bson.M{ - "$search": queryString, - }, - } - - // MongoDB collection - collection := client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("COLLECTION")) - - // Set find options - findOptions := options.FindOne() - findOptions.SetProjection(bson.M{ - "score": bson.M{"$meta": "textScore"}, - }) - findOptions.SetSort(bson.M{ - "score": bson.M{"$meta": "textScore"}, - }) - - // Perform the search - var result bson.M - err = collection.FindOne(context.TODO(), textSearchFilter, findOptions).Decode(&result) - if err != nil { - zap.L().Error("Error finding document", zap.Error(err)) - continue - } - - // Process based on the score - if score, ok := result["score"].(float64); ok { - if score >= 1 { - // zap.L().Info("marketCap", zap.Any("marketCap", result["marketCap"]), zap.Any("name", stockDetail["Name of the Instrument"])) - stockDetail["marketCapValue"] = result["marketCap"] - stockDetail["url"] = result["url"] - stockDetail["marketCap"] = getMarketCapCategory(fmt.Sprintf("%v", result["marketCap"])) - stockDetail["stockRate"] = rateStock(result) - } else { - // zap.L().Info("score less than 1", zap.Float64("score", score)) - results, err := searchCompany(instrumentName) - if err != nil || len(results) == 0 { - zap.L().Error("No company found", zap.Error(err)) - continue - } - data, err := fetchCompanyData(results[0].URL) - if err != nil { - zap.L().Error("Error fetching company data", zap.Error(err)) - continue - } - // Update MongoDB with fetched data - update := bson.M{ - "$set": bson.M{ - "marketCap": data["Market Cap"], - "currentPrice": data["Current Price"], - "highLow": data["High / Low"], - "stockPE": data["Stock P/E"], - "bookValue": data["Book Value"], - "dividendYield": data["Dividend Yield"], - "roce": data["ROCE"], - "roe": data["ROE"], - "faceValue": data["Face Value"], - "pros": data["pros"], - "cons": data["cons"], - "quarterlyResults": data["quarterlyResults"], - "profitLoss": data["profitLoss"], - "balanceSheet": data["balanceSheet"], - "cashFlows": data["cashFlows"], - "ratios": data["ratios"], - "shareholdingPattern": data["shareholdingPattern"], - "peersTable": data["peersTable"], - "peers": data["peers"], - }, - } - updateOptions := options.Update().SetUpsert(true) - filter := bson.M{"name": results[0].Name} - _, err = collection.UpdateOne(context.TODO(), filter, update, updateOptions) - if err != nil { - zap.L().Error("Failed to update document", zap.Error(err)) - } else { - zap.L().Info("Successfully updated document", zap.String("company", results[0].Name)) - } - } - } else { - zap.L().Error("No score available for", zap.String("company", instrumentName)) - } - - // Marshal and write the stockDetail - stockDataMarshal, err := json.Marshal(stockDetail) - if err != nil { - zap.L().Error("Error marshalling data", zap.Error(err)) - continue - } - - _, err = c.Writer.Write(append(stockDataMarshal, '\n')) // Send each stockDetail as JSON with a newline separator - - if err != nil { - zap.L().Error("Error writing data", zap.Error(err)) - break - } - c.Writer.Flush() // Flush each chunk immediately - } - } - } - } - c.Writer.Write([]byte("\nStream complete.\n")) - c.Writer.Flush() // Ensure the final response is sent -} - -func runningServer(c *gin.Context) { - c.JSON(200, gin.H{"message": "Server is running"}) -} -func toFloat(value interface{}) float64 { - if str, ok := value.(string); ok { - // Remove commas from the string - cleanStr := strings.ReplaceAll(str, ",", "") - - // Check if the string contains a percentage symbol - if strings.Contains(cleanStr, "%") { - // Remove the percentage symbol - cleanStr = strings.ReplaceAll(cleanStr, "%", "") - // Convert to float and divide by 100 to get the decimal equivalent - f, err := strconv.ParseFloat(cleanStr, 64) - if err != nil { - zap.L().Error("Error converting to float64", zap.Error(err)) - return 0.0 - } - return f / 100.0 - } - - // Parse the cleaned string to float - f, err := strconv.ParseFloat(cleanStr, 64) - if err != nil { - zap.L().Error("Error converting to float64", zap.Error(err)) - return 0.0 - } - return f - } - return 0.0 -} - -func toStringArray(value interface{}) []string { - if arr, ok := value.(primitive.A); ok { - var strArr []string - for _, v := range arr { - if str, ok := v.(string); ok { - strArr = append(strArr, str) - } - } - return strArr - } - return []string{} -} - -func getMarketCapCategory(marketCapValue string) string { - - cleanMarketCapValue := strings.ReplaceAll(marketCapValue, ",", "") +func main() { - marketCap, err := strconv.ParseFloat(cleanMarketCapValue, 64) // 64-bit float + err := godotenv.Load() if err != nil { - log.Println("Failed to convert market cap to integer: %v", err) - } - // Define market cap categories in crore (or billions as per comment) - if marketCap >= 20000 { - return "Large Cap" - } else if marketCap >= 5000 && marketCap < 20000 { - return "Mid Cap" - } else if marketCap < 5000 { - return "Small Cap" + log.Println("Error loading .env file") } - return "Unknown Category" -} - -func main() { log.Println("MONGO_URI:", os.Getenv("MONGO_URI")) log.Println("CLOUDINARY_URL:", os.Getenv("CLOUDINARY_URL")) - ticker := time.NewTicker(48 * time.Second) - - go func() { - for t := range ticker.C { - log.Println("Tick at", t) - cmd := exec.Command("curl", "https://stock-backend-hz83.onrender.com/api/keepServerRunning") - output, err := cmd.CombinedOutput() - if err != nil { - log.Println("Error running curl:", err) - return - } - - // Print the output of the curl command - log.Println("Curl output:", string(output)) - - } - }() - router := gin.New() router.Use(CORSMiddleware()) - v1 := router.Group("/api") + ticker := startTicker() - { - v1.POST("/uploadXlsx", parseXlsxFile) - v1.GET("/keepServerRunning", runningServer) - } + routes.Routes(router) port := os.Getenv("PORT") if port == "" { @@ -779,361 +116,24 @@ func main() { } -func fetchCompanyData(url string) (map[string]interface{}, error) { - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("failed to fetch the URL: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("failed to retrieve the content, status code: %d", resp.StatusCode) - } - - // Parse the HTML content of the company page - doc, err := goquery.NewDocumentFromReader(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to parse the HTML content: %v", err) - } - // Extract data-warehouse-id - companyData := make(map[string]interface{}) - - dataWarehouseID, exists := doc.Find("div[data-warehouse-id]").Attr("data-warehouse-id") - if exists { - peerData, err := fetchPeerData(dataWarehouseID) - if err == nil { - companyData["peers"] = peerData - } - } - - // Extract the data we need - // Extract data as specified - doc.Find("li.flex.flex-space-between[data-source='default']").Each(func(index int, item *goquery.Selection) { - key := strings.TrimSpace(item.Find("span.name").Text()) - - // Extract value text and clean it up - value := strings.TrimSpace(item.Find("span.nowrap.value").Text()) - value = strings.ReplaceAll(value, "\n", "") // Remove newlines - value = strings.ReplaceAll(value, " ", "") // Remove extra spaces - - // Extract the numeric value if it exists inside the nested span and clean it up - number := item.Find("span.number").Text() - if number != "" { - number = strings.TrimSpace(number) - value = strings.ReplaceAll(value, number, number) // Ensure no extra spaces around numbers - } - - // Remove currency symbols and units from value - value = strings.ReplaceAll(value, "₹", "") - value = strings.ReplaceAll(value, "Cr.", "") - value = strings.ReplaceAll(value, "%", "") - - // Add to company data - companyData[key] = value - - // Print cleaned key-value pairs - zap.L().Info("Company Data", zap.String("key", key), zap.String("value", value)) - log.Printf("%s: %s\n", key, value) - }) - // Extract pros - var pros []string - doc.Find("div.pros ul li").Each(func(index int, item *goquery.Selection) { - pro := strings.TrimSpace(item.Text()) - pros = append(pros, pro) - }) - companyData["pros"] = pros - - // Extract cons - var cons []string - doc.Find("div.cons ul li").Each(func(index int, item *goquery.Selection) { - con := strings.TrimSpace(item.Text()) - cons = append(cons, con) - }) - companyData["cons"] = cons - // Extract Quarterly Results - quarterlyResults := make(map[string][]map[string]string) - // Get the months (headers) from the table - var months []string - doc.Find("table.data-table thead tr th").Each(func(index int, item *goquery.Selection) { - month := strings.TrimSpace(item.Text()) - if month != "" && month != "-" { // Skip empty or irrelevant headers - months = append(months, month) - } - }) - - // Iterate over each row in the tbody - doc.Find("table.data-table tbody tr").Each(func(index int, row *goquery.Selection) { - fieldName := strings.TrimSpace(row.Find("td.text").Text()) - var fieldData []map[string]string - - // Iterate over each column in the row - row.Find("td").Each(func(colIndex int, col *goquery.Selection) { - if colIndex > 0 && colIndex <= len(months) { // Ensure we are within the bounds of the months array - value := strings.TrimSpace(col.Text()) - month := months[colIndex] - fieldData = append(fieldData, map[string]string{ - month: value, - }) - } - }) - - if len(fieldData) > 0 { - quarterlyResults[fieldName] = fieldData - } - }) - - companyData["quarterlyResults"] = quarterlyResults - profitLossSection := doc.Find("section#profit-loss") - if profitLossSection.Length() > 0 { - companyData["profitLoss"] = parseTableData(profitLossSection, "div[data-result-table]") - } - balanceSheetSection := doc.Find("section#balance-sheet") - if balanceSheetSection.Length() > 0 { - companyData["balanceSheet"] = parseTableData(balanceSheetSection, "div[data-result-table]") - } - shareHoldingPattern := doc.Find("section#shareholding") - if shareHoldingPattern.Length() > 0 { - companyData["shareholdingPattern"] = parseShareholdingPattern(shareHoldingPattern) - } - - ratiosSection := doc.Find("section#ratios") - if ratiosSection.Length() > 0 { - companyData["ratios"] = parseTableData(ratiosSection, "div[data-result-table]") - } - cashFlowsSection := doc.Find("section#cash-flow") - if cashFlowsSection.Length() > 0 { - companyData["cashFlows"] = parseTableData(cashFlowsSection, "div[data-result-table]") - } - return companyData, nil -} - -func parsePeersTable(doc *goquery.Document, selector string) []map[string]string { - var peers []map[string]string - headers := []string{} - - // Extract table headers - doc.Find(fmt.Sprintf("%s table thead tr th", selector)).Each(func(i int, s *goquery.Selection) { - headers = append(headers, strings.TrimSpace(s.Text())) - }) - - // Parse each row of the peers table - doc.Find(fmt.Sprintf("%s table tbody tr", selector)).Each(func(i int, row *goquery.Selection) { - peerData := map[string]string{} - row.Find("td").Each(func(j int, cell *goquery.Selection) { - if j < len(headers) { - peerData[headers[j]] = strings.TrimSpace(cell.Text()) - } - }) - peers = append(peers, peerData) - }) - - return peers -} - -func fetchPeerData(dataWarehouseID string) ([]map[string]string, error) { - time.Sleep(1 * time.Second) - peerURL := fmt.Sprintf(os.Getenv("COMPANY_URL")+"/api/company/%s/peers/", dataWarehouseID) - - // Create a new HTTP request - req, err := http.NewRequest("GET", peerURL, nil) - if err != nil { - return nil, fmt.Errorf("error creating request to peers API: %w", err) - } - - // Add any required headers or cookies here - client := &http.Client{ - Timeout: 10 * time.Second, - } - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("error fetching peers data from API: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - bodyBytes, _ := ioutil.ReadAll(resp.Body) - bodyString := string(bodyBytes) - zap.L().Error("Received non-200 response code", zap.Int("status_code", resp.StatusCode), zap.String("body", bodyString)) - return nil, fmt.Errorf("received non-200 response code from peers API: %d", resp.StatusCode) - } - - // Parse the HTML response - doc, err := goquery.NewDocumentFromReader(resp.Body) - if err != nil { - return nil, fmt.Errorf("error parsing HTML response: %w", err) - } - - var peersData []map[string]string - var medianData map[string]string - - // Parse peers data from the table rows - doc.Find("tr[data-row-company-id]").Each(func(index int, item *goquery.Selection) { - peer := make(map[string]string) - - peer["name"] = item.Find("td.text a").Text() - peer["current_price"] = strings.TrimSpace(item.Find("td").Eq(2).Text()) - peer["pe"] = strings.TrimSpace(item.Find("td").Eq(3).Text()) - peer["market_cap"] = strings.TrimSpace(item.Find("td").Eq(4).Text()) - peer["div_yield"] = strings.TrimSpace(item.Find("td").Eq(5).Text()) - peer["np_qtr"] = strings.TrimSpace(item.Find("td").Eq(6).Text()) - peer["qtr_profit_var"] = strings.TrimSpace(item.Find("td").Eq(7).Text()) - peer["sales_qtr"] = strings.TrimSpace(item.Find("td").Eq(8).Text()) - peer["qtr_sales_var"] = strings.TrimSpace(item.Find("td").Eq(9).Text()) - peer["roce"] = strings.TrimSpace(item.Find("td").Eq(10).Text()) - - peersData = append(peersData, peer) - }) - - // Parse median data from the footer of the table - doc.Find("tfoot tr").Each(func(index int, item *goquery.Selection) { - medianData = make(map[string]string) - medianData["company_count"] = strings.TrimSpace(item.Find("td").Eq(1).Text()) - medianData["current_price"] = strings.TrimSpace(item.Find("td").Eq(2).Text()) - medianData["pe"] = strings.TrimSpace(item.Find("td").Eq(3).Text()) - medianData["market_cap"] = strings.TrimSpace(item.Find("td").Eq(4).Text()) - medianData["div_yield"] = strings.TrimSpace(item.Find("td").Eq(5).Text()) - medianData["np_qtr"] = strings.TrimSpace(item.Find("td").Eq(6).Text()) - medianData["qtr_profit_var"] = strings.TrimSpace(item.Find("td").Eq(7).Text()) - medianData["sales_qtr"] = strings.TrimSpace(item.Find("td").Eq(8).Text()) - medianData["qtr_sales_var"] = strings.TrimSpace(item.Find("td").Eq(9).Text()) - medianData["roce"] = strings.TrimSpace(item.Find("td").Eq(10).Text()) - }) - - peersData = append(peersData, medianData) - return peersData, nil -} - -type Company struct { - ID int `json:"id"` - Name string `json:"name"` - URL string `json:"url"` -} - -func searchCompany(queryString string) ([]Company, error) { - // Replace "corporation" with "Corpn" and "limited" with "Ltd" - queryString = strings.ReplaceAll(queryString, " Corporation ", " Corpn ") - queryString = strings.ReplaceAll(queryString, " corporation ", " Corpn ") - queryString = strings.ReplaceAll(queryString, " Limited", " Ltd ") - queryString = strings.ReplaceAll(queryString, " limited", " Ltd ") - queryString = strings.ReplaceAll(queryString, " and ", " & ") - queryString = strings.ReplaceAll(queryString, " And ", " & ") - // Base URL for the Screener API - baseURL := os.Getenv("COMPANY_URL") + "/api/company/search/" - - // Create the URL with query parameters - params := url.Values{} - params.Add("q", queryString) - params.Add("v", "3") - params.Add("fts", "1") - - // Create the request - req, err := http.NewRequest("GET", baseURL+"?"+params.Encode(), nil) - if err != nil { - return nil, err - } - - // Create the client and send the request - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // Read the response - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var searchResponse []Company - err = json.Unmarshal(body, &searchResponse) - if err != nil { - zap.L().Error("Failed to unmarshal search response", zap.Error(err)) - return nil, err - } - - // Return the list of results - return searchResponse, nil -} - -func parseTableData(section *goquery.Selection, tableSelector string) map[string]interface{} { - table := section.Find(tableSelector) - if table.Length() == 0 { - return nil - } - - // Extract months/years from table headers - headers := []string{} - table.Find("thead th").Each(func(i int, th *goquery.Selection) { - headers = append(headers, strings.TrimSpace(th.Text())) - }) +func startTicker() *time.Ticker { + ticker := time.NewTicker(48 * time.Second) - // Extract table rows and values - data := make(map[string]interface{}) - table.Find("tbody tr").Each(func(i int, tr *goquery.Selection) { - rowKey := strings.TrimSpace(tr.Find("td.text").Text()) - rowValues := []string{} - tr.Find("td").Each(func(i int, td *goquery.Selection) { - if i > 0 { // Skip the first column which is the row key - rowValues = append(rowValues, strings.TrimSpace(td.Text())) + go func() { + for t := range ticker.C { + log.Println("Tick at", t) + cmd := exec.Command("curl", "https://stock-backend-hz83.onrender.com/api/keepServerRunning") + output, err := cmd.CombinedOutput() + if err != nil { + log.Println("Error running curl:", err) + return } - }) - data[rowKey] = rowValues - }) - - return data -} - -func parseShareholdingPattern(section *goquery.Selection) map[string]interface{} { - shareholdingData := make(map[string]interface{}) - - // Extract quarterly data - quarterlyData := parseTable(section.Find("div#quarterly-shp")) - if len(quarterlyData) > 0 { - shareholdingData["quarterly"] = quarterlyData - } - - // Extract yearly data - yearlyData := parseTable(section.Find("div#yearly-shp")) - if len(yearlyData) > 0 { - shareholdingData["yearly"] = yearlyData - } - - return shareholdingData -} -func parseTable(tableDiv *goquery.Selection) []map[string]interface{} { - var tableData []map[string]interface{} + // Print the output of the curl command + log.Println("Curl output:", string(output)) - // Get the headers (dates) from the table - var headers []string - tableDiv.Find("table thead th").Each(func(index int, header *goquery.Selection) { - if index > 0 { // Skip the first column header (e.g., "Promoters", "FIIs", etc.) - headers = append(headers, strings.TrimSpace(header.Text())) } - }) - - // Iterate over each row in the table body - tableDiv.Find("table tbody tr").Each(func(index int, row *goquery.Selection) { - rowData := make(map[string]interface{}) - - // Extract the row label (e.g., "Promoters", "FIIs", etc.) - label := strings.TrimSpace(row.Find("td.text").Text()) - rowData["category"] = label - - // Extract values for each date (column) - values := make(map[string]string) - row.Find("td").Each(func(i int, cell *goquery.Selection) { - if i > 0 && i <= len(headers) { // Ensure we are within the bounds of the headers array - date := headers[i-1] // Corresponding date (column header) - values[date] = strings.TrimSpace(cell.Text()) - } - }) - - rowData["values"] = values - tableData = append(tableData, rowData) - }) + }() - return tableData + return ticker } diff --git a/routes/routes.go b/routes/routes.go new file mode 100644 index 0000000..84db122 --- /dev/null +++ b/routes/routes.go @@ -0,0 +1,17 @@ +package routes + +import ( + "stockbackend/controllers" + + "github.com/gin-gonic/gin" +) + +func Routes(r *gin.Engine) { + + v1 := r.Group("/api") + + { + v1.POST("/uploadXlsx", controllers.FileController.ParseXLSXFile) + v1.GET("/keepServerRunning", controllers.HealthController.IsRunning) + } +} diff --git a/services/file_service.go b/services/file_service.go new file mode 100644 index 0000000..095db38 --- /dev/null +++ b/services/file_service.go @@ -0,0 +1,274 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "mime/multipart" + "os" + mongo_client "stockbackend/clients/mongo" + "stockbackend/utils/constants" + "stockbackend/utils/helpers" + "strings" + + "github.com/cloudinary/cloudinary-go/v2" + "github.com/cloudinary/cloudinary-go/v2/api/uploader" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/xuri/excelize/v2" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" + "gopkg.in/mgo.v2/bson" +) + +type FileServiceI interface { + ParseXLSXFile(ctx *gin.Context, files []*multipart.FileHeader) error +} + +type fileService struct{} + +var FileService FileServiceI = &fileService{} + +func (fs *fileService) ParseXLSXFile(ctx *gin.Context, files []*multipart.FileHeader) error { + cld, err := cloudinary.NewFromURL(os.Getenv("CLOUDINARY_URL")) + if err != nil { + return fmt.Errorf("error initializing Cloudinary: %w", err) + } + // Iterate over the uploaded XLSX files + for _, fileHeader := range files { + // Open each file for processing + file, err := fileHeader.Open() + if err != nil { + zap.L().Error("Error opening file", zap.String("filename", fileHeader.Filename), zap.Error(err)) + continue + } + defer file.Close() + + // Generate a UUID for the filename + uuid := uuid.New().String() + cloudinaryFilename := uuid + ".xlsx" + + // Upload file to Cloudinary + uploadResult, err := cld.Upload.Upload(ctx, file, uploader.UploadParams{ + PublicID: cloudinaryFilename, + Folder: "xlsx_uploads", + }) + if err != nil { + zap.L().Error("Error uploading file to Cloudinary", zap.String("filename", fileHeader.Filename), zap.Error(err)) + continue + } + + zap.L().Info("File uploaded to Cloudinary", zap.String("filename", fileHeader.Filename), zap.String("url", uploadResult.SecureURL)) + + // Create a new reader from the uploaded file + file.Seek(0, 0) // Reset file pointer to the beginning + f, err := excelize.OpenReader(file) + if err != nil { + zap.L().Error("Error parsing XLSX file", zap.String("filename", fileHeader.Filename), zap.Error(err)) + continue + } + defer f.Close() + + // Get all the sheet names + sheetList := f.GetSheetList() + // Loop through the sheets and extract relevant information + for _, sheet := range sheetList { + zap.L().Info("Processing file", zap.String("filename", fileHeader.Filename), zap.String("sheet", sheet)) + + // Get all the rows in the sheet + rows, err := f.GetRows(sheet) + if err != nil { + zap.L().Error("Error reading rows from sheet", zap.String("sheet", sheet), zap.Error(err)) + continue + } + + headerFound := false + headerMap := make(map[string]int) + stopExtracting := false + + // Loop through the rows in the sheet + for _, row := range rows { + if len(row) == 0 { + continue + } + + if !headerFound { + for _, cell := range row { + if helpers.MatchHeader(cell, []string{`name\s*of\s*(the)?\s*instrument`}) { + headerFound = true + // Build the header map + for i, headerCell := range row { + normalizedHeader := helpers.NormalizeString(headerCell) + // Map possible variations to standard keys + switch { + case helpers.MatchHeader(normalizedHeader, []string{`name\s*of\s*(the)?\s*instrument`}): + headerMap["Name of the Instrument"] = i + case helpers.MatchHeader(normalizedHeader, []string{`isin`}): + headerMap["ISIN"] = i + case helpers.MatchHeader(normalizedHeader, []string{`rating\s*/\s*industry`, `industry\s*/\s*rating`}): + headerMap["Industry/Rating"] = i + case helpers.MatchHeader(normalizedHeader, []string{`quantity`}): + headerMap["Quantity"] = i + case helpers.MatchHeader(normalizedHeader, []string{`market\s*/\s*fair\s*value.*`, `market\s*value.*`}): + headerMap["Market/Fair Value"] = i + case helpers.MatchHeader(normalizedHeader, []string{`%.*nav`, `%.*net\s*assets`}): + headerMap["Percentage of AUM"] = i + } + } + // zap.L().Info("Header found", zap.Any("headerMap", headerMap)) + break + } + } + continue + } + + // Check for the end marker "Subtotal" or "Total" + joinedRow := strings.Join(row, "") + if strings.Contains(strings.ToLower(joinedRow), "subtotal") || strings.Contains(strings.ToLower(joinedRow), "total") { + stopExtracting = true + break + } + + if !stopExtracting { + stockDetail := make(map[string]interface{}) + + // Extract data using the header map + for key, idx := range headerMap { + if idx < len(row) { + stockDetail[key] = row[idx] + } else { + stockDetail[key] = "" + } + } + + // Check if the stockDetail has meaningful data + if stockDetail["Name of the Instrument"] == nil || stockDetail["Name of the Instrument"] == "" { + continue + } + + // Additional processing + instrumentName, ok := stockDetail["Name of the Instrument"].(string) + if !ok { + continue + } + + // Apply mapping if exists + if mappedName, exists := constants.MapValues[instrumentName]; exists { + stockDetail["Name of the Instrument"] = mappedName + instrumentName = mappedName + } + + // Clean up the query string + queryString := instrumentName + queryString = strings.ReplaceAll(queryString, " Corporation ", " Corpn ") + queryString = strings.ReplaceAll(queryString, " corporation ", " Corpn ") + queryString = strings.ReplaceAll(queryString, " Limited", " Ltd ") + queryString = strings.ReplaceAll(queryString, " limited", " Ltd ") + queryString = strings.ReplaceAll(queryString, " and ", " & ") + queryString = strings.ReplaceAll(queryString, " And ", " & ") + + // Prepare the text search filter + textSearchFilter := bson.M{ + "$text": bson.M{ + "$search": queryString, + }, + } + + // MongoDB collection + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("COLLECTION")) + + // Set find options + findOptions := options.FindOne() + findOptions.SetProjection(bson.M{ + "score": bson.M{"$meta": "textScore"}, + }) + findOptions.SetSort(bson.M{ + "score": bson.M{"$meta": "textScore"}, + }) + + // Perform the search + var result bson.M + err = collection.FindOne(context.TODO(), textSearchFilter, findOptions).Decode(&result) + if err != nil { + zap.L().Error("Error finding document", zap.Error(err)) + continue + } + + // Process based on the score + if score, ok := result["score"].(float64); ok { + if score >= 1 { + // zap.L().Info("marketCap", zap.Any("marketCap", result["marketCap"]), zap.Any("name", stockDetail["Name of the Instrument"])) + stockDetail["marketCapValue"] = result["marketCap"] + stockDetail["url"] = result["url"] + stockDetail["marketCap"] = helpers.GetMarketCapCategory(fmt.Sprintf("%v", result["marketCap"])) + stockDetail["stockRate"] = helpers.RateStock(result) + } else { + // zap.L().Info("score less than 1", zap.Float64("score", score)) + results, err := helpers.SearchCompany(instrumentName) + if err != nil || len(results) == 0 { + zap.L().Error("No company found", zap.Error(err)) + continue + } + data, err := helpers.FetchCompanyData(results[0].URL) + if err != nil { + zap.L().Error("Error fetching company data", zap.Error(err)) + continue + } + // Update MongoDB with fetched data + update := bson.M{ + "$set": bson.M{ + "marketCap": data["Market Cap"], + "currentPrice": data["Current Price"], + "highLow": data["High / Low"], + "stockPE": data["Stock P/E"], + "bookValue": data["Book Value"], + "dividendYield": data["Dividend Yield"], + "roce": data["ROCE"], + "roe": data["ROE"], + "faceValue": data["Face Value"], + "pros": data["pros"], + "cons": data["cons"], + "quarterlyResults": data["quarterlyResults"], + "profitLoss": data["profitLoss"], + "balanceSheet": data["balanceSheet"], + "cashFlows": data["cashFlows"], + "ratios": data["ratios"], + "shareholdingPattern": data["shareholdingPattern"], + "peersTable": data["peersTable"], + "peers": data["peers"], + }, + } + updateOptions := options.Update().SetUpsert(true) + filter := bson.M{"name": results[0].Name} + _, err = collection.UpdateOne(context.TODO(), filter, update, updateOptions) + if err != nil { + zap.L().Error("Failed to update document", zap.Error(err)) + } else { + zap.L().Info("Successfully updated document", zap.String("company", results[0].Name)) + } + } + } else { + zap.L().Error("No score available for", zap.String("company", instrumentName)) + } + + // Marshal and write the stockDetail + stockDataMarshal, err := json.Marshal(stockDetail) + if err != nil { + zap.L().Error("Error marshalling data", zap.Error(err)) + continue + } + + _, err = ctx.Writer.Write(append(stockDataMarshal, '\n')) // Send each stockDetail as JSON with a newline separator + + if err != nil { + zap.L().Error("Error writing data", zap.Error(err)) + break + } + ctx.Writer.Flush() // Flush each chunk immediately + } + } + } + } + + return nil +} diff --git a/types/types.go b/types/types.go new file mode 100644 index 0000000..b4314e6 --- /dev/null +++ b/types/types.go @@ -0,0 +1,20 @@ +package types + +// Stock represents the data of a stock +type Stock struct { + Name string + PE float64 + MarketCap float64 + DividendYield float64 + ROCE float64 + QuarterlySales float64 + QuarterlyProfit float64 + Cons []string + Pros []string +} + +type Company struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` +} diff --git a/utils/constants/constants.go b/utils/constants/constants.go new file mode 100644 index 0000000..6940112 --- /dev/null +++ b/utils/constants/constants.go @@ -0,0 +1,11 @@ +package constants + +var ( + MapValues = map[string]string{ + "Sun Pharmaceutical Industries Limited": "Sun Pharma.Inds.", + "KEC International Limited": "K E C Intl.", + "Sandhar Technologies Limited": "Sandhar Tech", + "Samvardhana Motherson International Limited": "Samvardh. Mothe.", + "Coromandel International Limited": "Coromandel Inter", + } +) diff --git a/utils/helpers/helpers.go b/utils/helpers/helpers.go new file mode 100644 index 0000000..f2981e2 --- /dev/null +++ b/utils/helpers/helpers.go @@ -0,0 +1,677 @@ +package helpers + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "math" + "net/http" + "net/url" + "os" + "regexp" + "stockbackend/types" + "strconv" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.uber.org/zap" + "gopkg.in/mgo.v2/bson" +) + +// Helper function to match header titles +func MatchHeader(cellValue string, patterns []string) bool { + normalizedValue := NormalizeString(cellValue) + for _, pattern := range patterns { + matched, _ := regexp.MatchString(pattern, normalizedValue) + if matched { + return true + } + } + return false +} + +// Helper function to normalize strings +func NormalizeString(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + +func CheckInstrumentName(input string) bool { + // Regular expression to match "Name of the Instrument" or "Name of Instrument" + pattern := `Name of (the )?Instrument` + + // Compile the regex + re := regexp.MustCompile(pattern) + + // Check if the pattern matches the input string + return re.MatchString(input) +} + +func ToFloat(value interface{}) float64 { + if str, ok := value.(string); ok { + // Remove commas from the string + cleanStr := strings.ReplaceAll(str, ",", "") + + // Check if the string contains a percentage symbol + if strings.Contains(cleanStr, "%") { + // Remove the percentage symbol + cleanStr = strings.ReplaceAll(cleanStr, "%", "") + // Convert to float and divide by 100 to get the decimal equivalent + f, err := strconv.ParseFloat(cleanStr, 64) + if err != nil { + zap.L().Error("Error converting to float64", zap.Error(err)) + return 0.0 + } + return f / 100.0 + } + + // Parse the cleaned string to float + f, err := strconv.ParseFloat(cleanStr, 64) + if err != nil { + zap.L().Error("Error converting to float64", zap.Error(err)) + return 0.0 + } + return f + } + return 0.0 +} + +func ToStringArray(value interface{}) []string { + if arr, ok := value.(primitive.A); ok { + var strArr []string + for _, v := range arr { + if str, ok := v.(string); ok { + strArr = append(strArr, str) + } + } + return strArr + } + return []string{} +} + +func GetMarketCapCategory(marketCapValue string) string { + + cleanMarketCapValue := strings.ReplaceAll(marketCapValue, ",", "") + + marketCap, err := strconv.ParseFloat(cleanMarketCapValue, 64) // 64-bit float + if err != nil { + log.Println("Failed to convert market cap to integer: %v", err) + } + // Define market cap categories in crore (or billions as per comment) + if marketCap >= 20000 { + return "Large Cap" + } else if marketCap >= 5000 && marketCap < 20000 { + return "Mid Cap" + } else if marketCap < 5000 { + return "Small Cap" + } + return "Unknown Category" +} + +// rateStock calculates the final stock rating + +func RateStock(stock map[string]interface{}) float64 { + // zap.L().Info("Stock data", zap.Any("stock", stock)) + stockData := types.Stock{ + Name: stock["name"].(string), + PE: ToFloat(stock["stockPE"]), + MarketCap: ToFloat(stock["marketCap"]), + DividendYield: ToFloat(stock["dividendYield"]), + ROCE: ToFloat(stock["roce"]), + Cons: ToStringArray(stock["cons"]), + Pros: ToStringArray(stock["pros"]), + } + // zap.L().Info("Stock data", zap.Any("stock", stockData)) + // zap.L().Info("Stock data", zap.Any("stock", stockData)) + peerComparisonScore := compareWithPeers(stockData, stock["peers"]) * 0.5 + trendScore := AnalyzeTrend(stockData, stock["quarterlyResults"]) * 0.4 + // prosConsScore := prosConsAdjustment(stock) * 0.1 + // zap.L().Info("Peer comparison score", zap.Float64("peerComparisonScore", peerComparisonScore)) + + finalScore := peerComparisonScore + trendScore + finalScore = math.Round(finalScore*100) / 100 + return finalScore +} + +// compareWithPeers calculates a peer comparison score +func compareWithPeers(stock types.Stock, peers interface{}) float64 { + peerScore := 0.0 + var medianScore float64 + + if arr, ok := peers.(primitive.A); ok { + // Ensure there are enough peers to compare + if len(arr) < 2 { + zap.L().Warn("Not enough peers to compare") + return 0.0 + } + + for _, peerRaw := range arr[:len(arr)-1] { + peer := peerRaw.(bson.M) + + // Parse peer values to float64 + peerPE := ParseFloat(peer["pe"]) + peerMarketCap := ParseFloat(peer["market_cap"]) + peerDividendYield := ParseFloat(peer["div_yield"]) + peerROCE := ParseFloat(peer["roce"]) + peerQuarterlySales := ParseFloat(peer["sales_qtr"]) + peerQuarterlyProfit := ParseFloat(peer["np_qtr"]) + + // Example scoring logic + if stock.PE < peerPE { + peerScore += 10 + } else { + peerScore += math.Max(0, 10-(stock.PE-peerPE)) + } + + if stock.MarketCap > peerMarketCap { + peerScore += 5 + } + + if stock.DividendYield > peerDividendYield { + peerScore += 5 + } + + if stock.ROCE > peerROCE { + peerScore += 10 + } + + if stock.QuarterlySales > peerQuarterlySales { + peerScore += 5 + } + + if stock.QuarterlyProfit > peerQuarterlyProfit { + peerScore += 10 + } + } + medianRaw := arr[len(arr)-1] + median := medianRaw.(bson.M) + + // Parse median values to float64 + medianPE := ParseFloat(median["pe"]) + medianMarketCap := ParseFloat(median["market_cap"]) + medianDividendYield := ParseFloat(median["div_yield"]) + medianROCE := ParseFloat(median["roce"]) + medianQuarterlySales := ParseFloat(median["sales_qtr"]) + medianQuarterlyProfit := ParseFloat(median["np_qtr"]) + + // Adjust score based on median comparison + if stock.PE < medianPE { + peerScore += 5 + } else { + peerScore += math.Max(0, 5-(stock.PE-medianPE)) + } + + if stock.MarketCap > medianMarketCap { + peerScore += 3 + } + + if stock.DividendYield > medianDividendYield { + peerScore += 3 + } + + if stock.ROCE > medianROCE { + peerScore += 5 + } + + if stock.QuarterlySales > medianQuarterlySales { + peerScore += 2 + } + + if stock.QuarterlyProfit > medianQuarterlyProfit { + peerScore += 5 + } + + // Normalize by the number of peers (excluding the median) + peerCount := len(arr) - 1 + if peerCount > 0 { + return peerScore / float64(peerCount) + } + + // Normalize by the number of peers excluding the last element + } + + // Combine peerScore with medianScore (example: giving 10% weight to the median) + finalScore := (peerScore * 0.9) + (medianScore * 0.1) + + return finalScore +} + +// Helper function to convert values from map to float64 +func ParseFloat(value interface{}) float64 { + switch v := value.(type) { + case string: + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0.0 + } + return f + case float64: + return v + case int: + return float64(v) + default: + return 0.0 + } +} +func AnalyzeTrend(stock types.Stock, pastData interface{}) float64 { + trendScore := 0.0 + comparisons := 0 // Keep track of the number of comparisons + + // Ensure pastData is in bson.M format + if data, ok := pastData.(bson.M); ok { + for _, quarterData := range data { + // zap.L().Info("Processing quarter", zap.String("quarter", key)) + + // Process the quarter data if it's a primitive.A (array of quarter maps) + if quarterArray, ok := quarterData.(primitive.A); ok { + var prevElem bson.M + for i, elem := range quarterArray { + if elemMap, ok := elem.(bson.M); ok { + // zap.L().Info("Processing quarter element", zap.Any("element", elemMap)) + + // Only perform comparisons starting from the second element + if i > 0 && prevElem != nil { + // zap.L().Info("Comparing with previous element", zap.Any("previous", prevElem), zap.Any("current", elemMap)) + + // Iterate over the keys in the current quarter and compare with previous quarter + for key, v := range elemMap { + if prevVal, ok := prevElem[key]; ok { + // Compare consecutive values for the same key + if ToFloat(v) > ToFloat(prevVal) { + trendScore += 5 + } else if ToFloat(v) < ToFloat(prevVal) { + trendScore -= 5 + } + // Increment comparisons for each valid comparison + comparisons++ + } + } + } + // Update previous element for next iteration + prevElem = elemMap + } + } + } + } + } + + // Normalize the score by dividing it by the number of comparisons + if comparisons > 0 { + return trendScore / float64(comparisons) + } + return 0.0 // Return 0 if no comparisons were made +} + +// prosConsAdjustment calculates score adjustments based on pros and cons +func ProsConsAdjustment(stock types.Stock) float64 { + adjustment := 0.0 + + // Adjust score based on pros + // for _, pro := range stock.Pros { + // zap.L().Info("Pro", zap.String("pro", pro)) // This line is optional, just showing how we could use 'pro' + adjustment += ToFloat(1.0 * len(stock.Pros)) + // } + + // Adjust score based on cons + // for _, con := range stock.Cons { + // zap.L().Info("Con", zap.String("con", con)) // This line is optional, just showing how we could use 'con' + adjustment -= ToFloat(1.0 * len(stock.Cons)) + // }/ + + return adjustment +} + +func ParsePeersTable(doc *goquery.Document, selector string) []map[string]string { + var peers []map[string]string + headers := []string{} + + // Extract table headers + doc.Find(fmt.Sprintf("%s table thead tr th", selector)).Each(func(i int, s *goquery.Selection) { + headers = append(headers, strings.TrimSpace(s.Text())) + }) + + // Parse each row of the peers table + doc.Find(fmt.Sprintf("%s table tbody tr", selector)).Each(func(i int, row *goquery.Selection) { + peerData := map[string]string{} + row.Find("td").Each(func(j int, cell *goquery.Selection) { + if j < len(headers) { + peerData[headers[j]] = strings.TrimSpace(cell.Text()) + } + }) + peers = append(peers, peerData) + }) + + return peers +} + +func FetchPeerData(dataWarehouseID string) ([]map[string]string, error) { + time.Sleep(1 * time.Second) + peerURL := fmt.Sprintf(os.Getenv("COMPANY_URL")+"/api/company/%s/peers/", dataWarehouseID) + + // Create a new HTTP request + req, err := http.NewRequest("GET", peerURL, nil) + if err != nil { + return nil, fmt.Errorf("error creating request to peers API: %w", err) + } + + // Add any required headers or cookies here + client := &http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("error fetching peers data from API: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := ioutil.ReadAll(resp.Body) + bodyString := string(bodyBytes) + zap.L().Error("Received non-200 response code", zap.Int("status_code", resp.StatusCode), zap.String("body", bodyString)) + return nil, fmt.Errorf("received non-200 response code from peers API: %d", resp.StatusCode) + } + + // Parse the HTML response + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("error parsing HTML response: %w", err) + } + + var peersData []map[string]string + var medianData map[string]string + + // Parse peers data from the table rows + doc.Find("tr[data-row-company-id]").Each(func(index int, item *goquery.Selection) { + peer := make(map[string]string) + + peer["name"] = item.Find("td.text a").Text() + peer["current_price"] = strings.TrimSpace(item.Find("td").Eq(2).Text()) + peer["pe"] = strings.TrimSpace(item.Find("td").Eq(3).Text()) + peer["market_cap"] = strings.TrimSpace(item.Find("td").Eq(4).Text()) + peer["div_yield"] = strings.TrimSpace(item.Find("td").Eq(5).Text()) + peer["np_qtr"] = strings.TrimSpace(item.Find("td").Eq(6).Text()) + peer["qtr_profit_var"] = strings.TrimSpace(item.Find("td").Eq(7).Text()) + peer["sales_qtr"] = strings.TrimSpace(item.Find("td").Eq(8).Text()) + peer["qtr_sales_var"] = strings.TrimSpace(item.Find("td").Eq(9).Text()) + peer["roce"] = strings.TrimSpace(item.Find("td").Eq(10).Text()) + + peersData = append(peersData, peer) + }) + + // Parse median data from the footer of the table + doc.Find("tfoot tr").Each(func(index int, item *goquery.Selection) { + medianData = make(map[string]string) + medianData["company_count"] = strings.TrimSpace(item.Find("td").Eq(1).Text()) + medianData["current_price"] = strings.TrimSpace(item.Find("td").Eq(2).Text()) + medianData["pe"] = strings.TrimSpace(item.Find("td").Eq(3).Text()) + medianData["market_cap"] = strings.TrimSpace(item.Find("td").Eq(4).Text()) + medianData["div_yield"] = strings.TrimSpace(item.Find("td").Eq(5).Text()) + medianData["np_qtr"] = strings.TrimSpace(item.Find("td").Eq(6).Text()) + medianData["qtr_profit_var"] = strings.TrimSpace(item.Find("td").Eq(7).Text()) + medianData["sales_qtr"] = strings.TrimSpace(item.Find("td").Eq(8).Text()) + medianData["qtr_sales_var"] = strings.TrimSpace(item.Find("td").Eq(9).Text()) + medianData["roce"] = strings.TrimSpace(item.Find("td").Eq(10).Text()) + }) + + peersData = append(peersData, medianData) + return peersData, nil +} + +func SearchCompany(queryString string) ([]types.Company, error) { + // Replace "corporation" with "Corpn" and "limited" with "Ltd" + queryString = strings.ReplaceAll(queryString, " Corporation ", " Corpn ") + queryString = strings.ReplaceAll(queryString, " corporation ", " Corpn ") + queryString = strings.ReplaceAll(queryString, " Limited", " Ltd ") + queryString = strings.ReplaceAll(queryString, " limited", " Ltd ") + queryString = strings.ReplaceAll(queryString, " and ", " & ") + queryString = strings.ReplaceAll(queryString, " And ", " & ") + // Base URL for the Screener API + baseURL := os.Getenv("COMPANY_URL") + "/api/company/search/" + + // Create the URL with query parameters + params := url.Values{} + params.Add("q", queryString) + params.Add("v", "3") + params.Add("fts", "1") + + // Create the request + req, err := http.NewRequest("GET", baseURL+"?"+params.Encode(), nil) + if err != nil { + return nil, err + } + + // Create the client and send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Read the response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var searchResponse []types.Company + err = json.Unmarshal(body, &searchResponse) + if err != nil { + zap.L().Error("Failed to unmarshal search response", zap.Error(err)) + return nil, err + } + + // Return the list of results + return searchResponse, nil +} + +func ParseTableData(section *goquery.Selection, tableSelector string) map[string]interface{} { + table := section.Find(tableSelector) + if table.Length() == 0 { + return nil + } + + // Extract months/years from table headers + headers := []string{} + table.Find("thead th").Each(func(i int, th *goquery.Selection) { + headers = append(headers, strings.TrimSpace(th.Text())) + }) + + // Extract table rows and values + data := make(map[string]interface{}) + table.Find("tbody tr").Each(func(i int, tr *goquery.Selection) { + rowKey := strings.TrimSpace(tr.Find("td.text").Text()) + rowValues := []string{} + tr.Find("td").Each(func(i int, td *goquery.Selection) { + if i > 0 { // Skip the first column which is the row key + rowValues = append(rowValues, strings.TrimSpace(td.Text())) + } + }) + data[rowKey] = rowValues + }) + + return data +} + +func ParseShareholdingPattern(section *goquery.Selection) map[string]interface{} { + shareholdingData := make(map[string]interface{}) + + // Extract quarterly data + quarterlyData := ParseTable(section.Find("div#quarterly-shp")) + if len(quarterlyData) > 0 { + shareholdingData["quarterly"] = quarterlyData + } + + // Extract yearly data + yearlyData := ParseTable(section.Find("div#yearly-shp")) + if len(yearlyData) > 0 { + shareholdingData["yearly"] = yearlyData + } + + return shareholdingData +} + +func ParseTable(tableDiv *goquery.Selection) []map[string]interface{} { + var tableData []map[string]interface{} + + // Get the headers (dates) from the table + var headers []string + tableDiv.Find("table thead th").Each(func(index int, header *goquery.Selection) { + if index > 0 { // Skip the first column header (e.g., "Promoters", "FIIs", etc.) + headers = append(headers, strings.TrimSpace(header.Text())) + } + }) + + // Iterate over each row in the table body + tableDiv.Find("table tbody tr").Each(func(index int, row *goquery.Selection) { + rowData := make(map[string]interface{}) + + // Extract the row label (e.g., "Promoters", "FIIs", etc.) + label := strings.TrimSpace(row.Find("td.text").Text()) + rowData["category"] = label + + // Extract values for each date (column) + values := make(map[string]string) + row.Find("td").Each(func(i int, cell *goquery.Selection) { + if i > 0 && i <= len(headers) { // Ensure we are within the bounds of the headers array + date := headers[i-1] // Corresponding date (column header) + values[date] = strings.TrimSpace(cell.Text()) + } + }) + + rowData["values"] = values + tableData = append(tableData, rowData) + }) + + return tableData +} + +func FetchCompanyData(url string) (map[string]interface{}, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch the URL: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("failed to retrieve the content, status code: %d", resp.StatusCode) + } + + // Parse the HTML content of the company page + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to parse the HTML content: %v", err) + } + // Extract data-warehouse-id + companyData := make(map[string]interface{}) + + dataWarehouseID, exists := doc.Find("div[data-warehouse-id]").Attr("data-warehouse-id") + if exists { + peerData, err := FetchPeerData(dataWarehouseID) + if err == nil { + companyData["peers"] = peerData + } + } + + // Extract the data we need + // Extract data as specified + doc.Find("li.flex.flex-space-between[data-source='default']").Each(func(index int, item *goquery.Selection) { + key := strings.TrimSpace(item.Find("span.name").Text()) + + // Extract value text and clean it up + value := strings.TrimSpace(item.Find("span.nowrap.value").Text()) + value = strings.ReplaceAll(value, "\n", "") // Remove newlines + value = strings.ReplaceAll(value, " ", "") // Remove extra spaces + + // Extract the numeric value if it exists inside the nested span and clean it up + number := item.Find("span.number").Text() + if number != "" { + number = strings.TrimSpace(number) + value = strings.ReplaceAll(value, number, number) // Ensure no extra spaces around numbers + } + + // Remove currency symbols and units from value + value = strings.ReplaceAll(value, "₹", "") + value = strings.ReplaceAll(value, "Cr.", "") + value = strings.ReplaceAll(value, "%", "") + + // Add to company data + companyData[key] = value + + // Print cleaned key-value pairs + zap.L().Info("Company Data", zap.String("key", key), zap.String("value", value)) + log.Printf("%s: %s\n", key, value) + }) + // Extract pros + var pros []string + doc.Find("div.pros ul li").Each(func(index int, item *goquery.Selection) { + pro := strings.TrimSpace(item.Text()) + pros = append(pros, pro) + }) + companyData["pros"] = pros + + // Extract cons + var cons []string + doc.Find("div.cons ul li").Each(func(index int, item *goquery.Selection) { + con := strings.TrimSpace(item.Text()) + cons = append(cons, con) + }) + companyData["cons"] = cons + // Extract Quarterly Results + quarterlyResults := make(map[string][]map[string]string) + // Get the months (headers) from the table + var months []string + doc.Find("table.data-table thead tr th").Each(func(index int, item *goquery.Selection) { + month := strings.TrimSpace(item.Text()) + if month != "" && month != "-" { // Skip empty or irrelevant headers + months = append(months, month) + } + }) + + // Iterate over each row in the tbody + doc.Find("table.data-table tbody tr").Each(func(index int, row *goquery.Selection) { + fieldName := strings.TrimSpace(row.Find("td.text").Text()) + var fieldData []map[string]string + + // Iterate over each column in the row + row.Find("td").Each(func(colIndex int, col *goquery.Selection) { + if colIndex > 0 && colIndex <= len(months) { // Ensure we are within the bounds of the months array + value := strings.TrimSpace(col.Text()) + month := months[colIndex] + fieldData = append(fieldData, map[string]string{ + month: value, + }) + } + }) + + if len(fieldData) > 0 { + quarterlyResults[fieldName] = fieldData + } + }) + + companyData["quarterlyResults"] = quarterlyResults + profitLossSection := doc.Find("section#profit-loss") + if profitLossSection.Length() > 0 { + companyData["profitLoss"] = ParseTableData(profitLossSection, "div[data-result-table]") + } + balanceSheetSection := doc.Find("section#balance-sheet") + if balanceSheetSection.Length() > 0 { + companyData["balanceSheet"] = ParseTableData(balanceSheetSection, "div[data-result-table]") + } + shareHoldingPattern := doc.Find("section#shareholding") + if shareHoldingPattern.Length() > 0 { + companyData["shareholdingPattern"] = ParseShareholdingPattern(shareHoldingPattern) + } + + ratiosSection := doc.Find("section#ratios") + if ratiosSection.Length() > 0 { + companyData["ratios"] = ParseTableData(ratiosSection, "div[data-result-table]") + } + cashFlowsSection := doc.Find("section#cash-flow") + if cashFlowsSection.Length() > 0 { + companyData["cashFlows"] = ParseTableData(cashFlowsSection, "div[data-result-table]") + } + return companyData, nil +}