diff --git a/go.mod b/go.mod index fa8e41a..7a8f53e 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,13 @@ go 1.23.0 require ( github.com/PuerkitoBio/goquery v1.10.0 github.com/cloudinary/cloudinary-go/v2 v2.9.0 + github.com/extrame/xls v0.0.1 github.com/getsentry/sentry-go v0.29.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 - github.com/xuri/excelize/v2 v2.8.1 + github.com/robfig/cron/v3 v3.0.1 + github.com/wailsapp/mimetype v1.4.1 + github.com/xuri/excelize/v2 v2.9.0 go.mongodb.org/mongo-driver v1.17.1 go.uber.org/zap v1.27.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 @@ -23,7 +26,8 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/creasty/defaults v1.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -42,21 +46,21 @@ require ( github.com/montanaflynn/stats v0.7.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect - github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect - github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect + github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect + github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c47a509..c67720d 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,12 @@ github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbD github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c= +github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw= +github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k= +github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA= github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -78,8 +82,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= -github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -96,18 +102,20 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/wailsapp/mimetype v1.4.1 h1:pQN9ycO7uo4vsUUuPeHEYoUkLVkaRntMnHJxVwYhwHs= +github.com/wailsapp/mimetype v1.4.1/go.mod h1:9aV5k31bBOv5z6u+QP8TltzvNGJPmNJD4XlAL3U+j3o= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0= -github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ= -github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE= -github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4= -github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d h1:llb0neMWDQe87IzJLS4Ci7psK/lVsjIS2otl+1WyRyY= +github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= +github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= +github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -124,19 +132,20 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= -golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -144,26 +153,28 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/main.go b/main.go index ffd2ba8..43fddbf 100644 --- a/main.go +++ b/main.go @@ -116,6 +116,7 @@ func main() { router.Use(sentrygin.New(sentrygin.Options{})) router.Use(CORSMiddleware()) + services.UpdateFunds() ticker := startTicker() rankUpdater := startRankUpdater() routes.Routes(router) diff --git a/services/parse_service.go b/services/parse_service.go new file mode 100644 index 0000000..adee3cd --- /dev/null +++ b/services/parse_service.go @@ -0,0 +1,797 @@ +package services + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "os" + "path" + "regexp" + mongo_client "stockbackend/clients/mongo" + "stockbackend/utils/helpers" + "strconv" + "strings" + "time" + "unicode" + + "github.com/cloudinary/cloudinary-go/v2" + "github.com/cloudinary/cloudinary-go/v2/api/admin" + "github.com/cloudinary/cloudinary-go/v2/api/uploader" + "github.com/extrame/xls" + "github.com/getsentry/sentry-go" + "github.com/google/uuid" + "github.com/robfig/cron/v3" + "github.com/wailsapp/mimetype" + "github.com/xuri/excelize/v2" + "go.mongodb.org/mongo-driver/mongo" + "go.uber.org/zap" + "gopkg.in/mgo.v2/bson" +) + +func setupCheck() { + if len(os.Getenv("CLOUDINARY_URL")) < 5 { + panic("Please provied a CLOUDINARY_URL. Run `export CLOUDINARY_URL=your@url` before in your shell for linux and MacOS") + } + if os.Getenv("DEBUG") != "true" { + log.Printf("IGNORE THIS ONLY FOR DEV: To run the script in debug mode use `export DEBUG=\"true\"`") + } +} + +func UpdateFunds() { + setupCheck() + scheduler := cron.New() + + debug := os.Getenv("DEBUG") == "true" + + if !debug { + // cron dose not support the L flag, for last day of the month! + // For April, June, September, November + _, err := scheduler.AddFunc("0 0 30 4,6,9,11 *", performUploadTask) + // For January, March, May, July, August, October, December + _, err2 := scheduler.AddFunc("0 0 31 1,3,5,7,8,10,12 *", performUploadTask) + // For February + _, err3 := scheduler.AddFunc("0 0 28 2 *", performUploadTask) + if err != nil { + log.Fatal("Error scheduling task:", err) + } + if err2 != nil { + log.Fatal("Error scheduling task:", err) + } + if err3 != nil { + log.Fatal("Error scheduling task:", err) + } + scheduler.Start() + } else { + // fmt.Println("Skipping the regular scheduler as debug mode is enabled.") + // fmt.Println("Creating a scheduler that will run every 1 minute.") + // jobID, err := scheduler.AddFunc("* * * * *", performUploadTask) + performUploadTask() + // Need this here for proper Next time calculation + // scheduler.Start() + // if err != nil { + // // fmt.Println("An error occurred: the scheduler could not be added.") + // } else { + // // fmt.Println("Next run time for Debug Scheduler:", scheduler.Entry(jobID).Next) + // } + } + + log.Println("Scheduler started") + + select {} +} + +func performUploadTask() { + log.Println("Starting monthly upload task...") + + req, err := http.NewRequest("GET", "https://mf.nipponindiaim.com/investor-service/downloads/factsheet-portfolio-and-other-disclosures", nil) + if err != nil { + log.Println("Error creating request:", err) + return + } + + setRequestHeaders(req) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Println("Error making request:", err) + return + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Println("Error reading response body:", err) + return + } + + mfDatas := extractPortfolioLinks(string(body)) + + for _, mfData := range mfDatas { + uploadToCloudinary("https://mf.nipponindiaim.com/", mfData) + } + + log.Println("Monthly upload task completed.") +} + +func setRequestHeaders(req *http.Request) { + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + req.Header.Set("Accept-Language", "en-GB,en-US;q=0.9,en;q=0.8") + req.Header.Set("Cache-Control", "max-age=0") + req.Header.Set("Priority", "u=0, i") + req.Header.Set("Sec-Ch-Ua", "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"") + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", "\"macOS\"") + req.Header.Set("Sec-Fetch-Dest", "document") + req.Header.Set("Sec-Fetch-Mode", "navigate") + req.Header.Set("Sec-Fetch-Site", "none") + req.Header.Set("Sec-Fetch-User", "?1") + req.Header.Set("Upgrade-Insecure-Requests", "1") +} +func normalizeWhitespace(s string) string { + var b strings.Builder + prevIsSpace := false + for _, r := range s { + if unicode.IsSpace(r) { + if !prevIsSpace { + b.WriteRune(' ') + prevIsSpace = true + } + } else { + b.WriteRune(r) + prevIsSpace = false + } + } + return b.String() +} + +func removeZeroWidthChars(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case '\u200B', '\u200C', '\u200D', '\uFEFF': + // Exclude zero-width characters + return -1 + default: + // Include other characters + return r + } + }, s) +} + +func cleanHTMLContent(s string) string { + s = removeZeroWidthChars(s) + s = normalizeWhitespace(s) + return s +} + +type MFCOLLECTION struct { + month string + year string + link string +} + +func extractPortfolioLinks(htmlContent string) []MFCOLLECTION { + // Updated regex pattern to handle various formats + re := regexp.MustCompile(`(?i)Monthly[\s\p{Zs}]+portfolio[\s\p{Zs}]+for[\s\p{Zs}]+the[\s\p{Zs}]+month(?:[\s\p{Zs}]+(?:of|end))?[\s\p{Zs}]*(?:(\d{1,2})(?:st|nd|rd|th)?[\s\p{Zs}]+)?(\w+)[\s\p{Zs}]*(\d{4})?.*?]+href="([^"]+)"`) + htmlContent = cleanHTMLContent(htmlContent) + + matches := re.FindAllStringSubmatch(htmlContent, -1) + // fmt.Println("Total Matches Found:", len(matches)) // Debugging: Show total matches found + + var mfDetails []MFCOLLECTION + for _, match := range matches { + if len(match) > 4 { + // entireText := match[0] // Entire matched text + + // Extract day, month, year, and link + month := match[2] // Month + year := match[3] // Optional year + link := match[4] // Extracted link + + // If year is missing in match[3], try to extract it from the following content + if year == "" { + // Attempt to find a 4-digit year after the month + yearRe := regexp.MustCompile(`\b(\d{4})\b`) + yearMatch := yearRe.FindStringSubmatch(htmlContent) + if len(yearMatch) > 1 { + year = yearMatch[1] + } + } + + // Append the link + mfDetails = append(mfDetails, MFCOLLECTION{ + month: month, + year: year, + link: link, + }) + // // fmt.Println("Entire matched text:", entireText) + // // fmt.Println("Month:", month) // Print extracted month + // // fmt.Println("Year:", year) // Print extracted year + // // fmt.Println("Link:", link) // Print the link + } + } + return mfDetails +} + +func uploadToCloudinary(fileURL string, mfData MFCOLLECTION) { + cld, err := cloudinary.NewFromURL(os.Getenv("CLOUDINARY_URL")) + if err != nil { + log.Println("Error creating Cloudinary instance:", err) + return + } + publicID := extractFileName(fileURL + mfData.link) + asset, err := cld.Admin.Asset(context.Background(), admin.AssetParams{PublicID: publicID}) + + secureUrl := asset.SecureURL + if err == nil && asset.PublicID == "" { + resp, err := cld.Upload.Upload(context.Background(), fileURL+mfData.link, uploader.UploadParams{ + PublicID: publicID, + }) + if err != nil { + log.Println("Error uploading to Cloudinary:", err) + return + } + secureUrl = resp.SecureURL + } else if err != nil { + return + } + + response, err := http.Get(secureUrl) + if err != nil { + log.Println("Error downloading xlsx file:", err) + return + } + bodyBytes, err := io.ReadAll(response.Body) + if err != nil { + log.Println("Error reading response body:", err) + return + } + + defer response.Body.Close() + + // Save the downloaded file to a temporary location + m := mimetype.Detect(bodyBytes) + // log.Println("Detected MIME type:", m.String()) + + // Assign file extension based on MIME type + var fileExt string + if m.Is("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") { + fileExt = ".xlsx" + } else if m.Is("application/vnd.ms-excel") { + fileExt = ".xls" + } else { + // log.Println("Downloaded file is not a supported Excel format (.xlsx or .xls).") + return + } + tempInputFile := fmt.Sprintf("%s%s", uuid.New().String(), fileExt) + err = os.WriteFile(tempInputFile, bodyBytes, 0644) + if err != nil { + log.Println("Error saving Excel file:", err) + return + } + defer os.Remove(tempInputFile) + month := extractMonth(publicID) + + if fileExt == ".xlsx" { + // Process .xlsx file + err = processXLSXFile(tempInputFile, month) + if err != nil { + // fmt.Println("tempInputFile", tempInputFile) + log.Println("Error processing .xlsx file:", err) + return + } + } else if fileExt == ".xls" { + // Process .xls file + // fmt.Println("tempInputFile", tempInputFile) + err = processXLSFile(tempInputFile, month) + if err != nil { + log.Println("Error processing .xls file:", err) + return + } + } else { + log.Println("Unsupported file format:", fileExt) + return + } + + fileUUID := uuid.New().String() + document := bson.M{ + "_id": fileUUID, + "month": month, + "completeName": publicID, + "cloudinaryLink": secureUrl, + "fund_house": "nippon", + } + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("MFCOLLECTION")) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err = collection.InsertOne(ctx, document) + if err != nil { + log.Println("Error inserting document into MongoDB:", err) + return + } + log.Printf("Document inserted successfully into MongoDB. UUID: %s\n", fileUUID) +} + +func processXLSXFile(tempInputFile, month string) error { + // Open the xlsx file using excelize + xlsxFile, err := excelize.OpenFile(tempInputFile) + if err != nil { + // fmt.Println("Error opening xlsx file:", err) + return fmt.Errorf("error opening xlsx file: %v", err) + } + defer xlsxFile.Close() + + // Get all sheet names + sheetList := xlsxFile.GetSheetList() + // stock := make([]map[string]interface{}, 0) + + for _, sheet := range sheetList { + rows, err := xlsxFile.GetRows(sheet) + if err != nil { + sentry.CaptureException(err) + zap.L().Error("Error reading rows from sheet", zap.String("sheet", sheet), zap.Error(err)) + continue + } + headerFound := false + headerMap := make(map[string]int) + stopExtracting := false + stockDetail := make([]map[string]interface{}, 0) + processingStarted := false + mutualFundName := "" + if len(rows) > 0 && len(rows[0]) > 1 { + mutualFundName = rows[0][1] + // log.Printf("Mutual Fund Name for sheet %s: %s", sheet, mutualFundName) + } + for _, row := range rows { + stock := make(map[string]interface{}, 0) + + if len(row) == 0 { + continue + } + if !headerFound { + for _, cell := range row { + if helpers.MatchHeader(cell, []string{`name\s*of\s*(the)?\s*instrument`}) { + headerFound = true + // Build the header map + for i, headerCell := range row { + normalizedHeader := helpers.NormalizeString(headerCell) + // Map possible variations to standard keys + switch { + case helpers.MatchHeader(normalizedHeader, []string{`name\s*of\s*(the)?\s*instrument`}): + headerMap["Name of the Instrument"] = i + case helpers.MatchHeader(normalizedHeader, []string{`isin`}): + headerMap["ISIN"] = i + case helpers.MatchHeader(normalizedHeader, []string{`rating\s*/\s*industry`, `industry\s*/\s*rating`}): + headerMap["Industry/Rating"] = i + case helpers.MatchHeader(normalizedHeader, []string{`quantity`}): + headerMap["Quantity"] = i + case helpers.MatchHeader(normalizedHeader, []string{`market\s*/\s*fair\s*value.*`, `market\s*value.*`}): + headerMap["Market/Fair Value"] = i + case helpers.MatchHeader(normalizedHeader, []string{`%.*nav`, `%.*net\s*assets`}): + headerMap["Percentage of AUM"] = i + } + } + // zap.L().Info("Header found", zap.Any("headerMap", headerMap)) + break + } + } + continue + } + + joinedRow := strings.Join(row, "") + if strings.Contains(strings.ToLower(joinedRow), "subtotal") || strings.Contains(strings.ToLower(joinedRow), "total") { + stopExtracting = true + break + } + if !processingStarted { + nameOfInstrument := "" + if idx, exists := headerMap["Name of the Instrument"]; exists && idx < len(row) { + nameOfInstrument = row[idx] + } + if strings.Contains(nameOfInstrument, "Equity & Equity related") { + processingStarted = true + continue // Skip the header description row and move to the next row + } + } + if processingStarted && !stopExtracting { + for key, idx := range headerMap { + if idx < len(row) { + // println(stockDetail["Name of the Instrument"].(string), "Name of the Instrument in stockDetail") + stock[key] = row[idx] + } else { + stock[key] = "" + } + } + _, ok := stock["Name of the Instrument"].(string) + if ok { + stockDetail = append(stockDetail, stock) + // println("Stock Detail: isin", stock["ISIN"].(string), "Name of the Instrument", stock["Name of the Instrument"].(string)) + } + } + } + // println(mutualFundName, "Mutual Fund Name") + hash := sha256.New() + hashName := sha256.New() + hashName.Write([]byte(mutualFundName)) + + hash.Write([]byte(mutualFundName + month)) + hashedId := hex.EncodeToString(hash.Sum(nil)) + hashedName := hex.EncodeToString(hashName.Sum(nil)) + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("MFHOLDING")) + + validStockDetails := []map[string]interface{}{} + + for _, stockDetail := range stockDetail { + if _, ok := stockDetail["Name of the Instrument"]; !ok { + // fmt.Println("Skipping entry: Missing Name of the Instrument") + continue + } + if _, ok := stockDetail["ISIN"]; !ok || stockDetail["ISIN"] == "" { + // fmt.Println("Skipping entry: Missing ISIN") + continue + } + if _, ok := stockDetail["Quantity"]; !ok { + // // fmt.Println("Skipping entry: Missing Quantity") + continue + } + if _, ok := stockDetail["Market/Fair Value"]; !ok { + // // fmt.Println("Skipping entry: Missing Market/Fair Value") + continue + } + if _, ok := stockDetail["Percentage of AUM"]; !ok { + // // fmt.Println("Skipping entry: Missing Percentage of AUM") + continue + } + validStockDetails = append(validStockDetails, stockDetail) + } + stockDetail = validStockDetails + + document := bson.M{ + "_id": hashedId, + "month": month, + "mutual_fund_name": mutualFundName, + "stock_details": stockDetail, + "hash": hashedName, + "created_at": time.Now(), + } + if mutualFundName == "" || stockDetail == nil || len(stockDetail) == 0 { + // fmt.Println("Skipping empty document") + continue + } + var existingDocument bson.M + err = collection.FindOne(context.TODO(), bson.M{"_id": hashedId}).Decode(&existingDocument) + if err == mongo.ErrNoDocuments { + // Document does not exist, so insert it + _, err := collection.InsertOne(context.TODO(), document) + if err != nil { + println(err.Error()) + } + // fmt.Println("Document inserted with ID:", insertResult.InsertedID) + } else if err != nil { + println(err.Error()) + } + + } + return nil +} + +func safeGetRow(sheet *xls.WorkSheet, rowIndex int) (*xls.Row, bool) { + defer func() { + if r := recover(); r != nil { + // log.Printf("Recovered from panic when accessing row %d: %v", rowIndex, r) + } + }() + + // Attempt to access the row + row := sheet.Row(rowIndex) + if row == nil { + return nil, false + } + + return row, true +} + +// { +// "mutual_fund_name": "text", +// "stock_details.ISIN": "text", +// "stock_details.Name of the Instrument": "text" +// } + +func processXLSFile(tempInputFile, month string) error { + xlsFile, err := xls.Open(tempInputFile, "utf-8") + if err != nil { + log.Fatalf("Failed to open file: %v", err) + } + + for sheetIndex := 0; sheetIndex < xlsFile.NumSheets(); sheetIndex++ { + sheet := xlsFile.GetSheet(sheetIndex) + if sheet == nil { + // log.Printf("Sheet at index %d is nil. Skipping.", sheetIndex) + continue + } + + // log.Printf("Processing sheet: %s with MaxRow: %d", sheet.Name, sheet.MaxRow) + + headerFound := false + headerMap := make(map[string]int) + stopExtracting := false + processingStarted := false + stockDetail := make([]map[string]interface{}, 0) + + // Extract mutual fund name from the first row, second column + var mutualFundName string + if sheet.MaxRow > 0 { + firstRow, ok := safeGetRow(sheet, 0) + if !ok { + continue + } + if firstRow != nil && firstRow.LastCol() > 1 { + mutualFundName = firstRow.Col(1) + // log.Printf("Mutual Fund Name for sheet %s: %s", sheet.Name, mutualFundName) + } + } + + for rowIndex := 0; rowIndex < int(sheet.MaxRow); rowIndex++ { + if rowIndex >= int(sheet.MaxRow) { + // log.Printf("Row %d in sheet %s is nil or out of range. Skipping.", rowIndex, sheet.Name) + continue + } + // println("Row Index: ", rowIndex, int(sheet.MaxRow)) + row, ok := safeGetRow(sheet, rowIndex) + if !ok || row.LastCol() == 0 { + continue + } + + // Detect headers if not already found + if !headerFound { + for colIndex := 0; colIndex < min(row.LastCol(), 10); colIndex++ { // Limit columns for header detection + cellValue := strings.TrimSpace(strings.ToLower(row.Col(colIndex))) + if strings.Contains(cellValue, "name of the instrument") { + headerFound = true + // Build the header map + for i := 0; i < min(row.LastCol(), 10); i++ { // Limit column count to avoid extra empty columns + header := strings.TrimSpace(strings.ToLower(row.Col(i))) + switch { + case strings.Contains(header, "name of the instrument"): + headerMap["Name of the Instrument"] = i + case strings.Contains(header, "isin"): + headerMap["ISIN"] = i + case strings.Contains(header, "rating/industry") || strings.Contains(header, "industry/rating"): + headerMap["Industry/Rating"] = i + case strings.Contains(header, "quantity"): + headerMap["Quantity"] = i + case strings.Contains(header, "market/fair value") || strings.Contains(header, "market value"): + headerMap["Market/Fair Value"] = i + case strings.Contains(header, "% nav") || strings.Contains(header, "% to nav") || strings.Contains(header, "% net assets"): + headerMap["Percentage of AUM"] = i + } + } + // log.Printf("Header found: %v", headerMap) + break + } + } + continue + } + + // Stop extraction if "Subtotal" or "Total" is encountered + var joinedRow string + for colIndex := 0; colIndex < row.LastCol(); colIndex++ { + joinedRow += row.Col(colIndex) + } + + if strings.Contains(strings.ToLower(joinedRow), "subtotal") || strings.Contains(strings.ToLower(joinedRow), "total") { + stopExtracting = true + break + } + + // Start processing only after "Equity & Equity related" is encountered + if !processingStarted { + nameOfInstrument := "" + if idx, exists := headerMap["Name of the Instrument"]; exists && idx < row.LastCol() { + nameOfInstrument = row.Col(idx) + } + if strings.Contains(nameOfInstrument, "Equity & Equity related") { + processingStarted = true + continue // Skip the header description row and move to the next row + } + } + + // Check if we need to adjust column indices for this row + adjustColumns := false + if headerMap["ISIN"] < row.LastCol() { + isinValue := row.Col(headerMap["ISIN"]) + if !strings.HasPrefix(isinValue, "INE") { // Assuming valid ISINs start with "INE" + // log.Printf("Adjusting row %d columns by 1 due to misalignment", rowIndex) + adjustColumns = true + } + } + + // Data extraction based on header map with potential adjustment + if processingStarted && !stopExtracting { + stock := make(map[string]interface{}) + // log.Printf("Header map: %v", headerMap) + for key, colIndex := range headerMap { + adjustedColIndex := colIndex + if adjustColumns { + adjustedColIndex = colIndex + 1 + } + + if adjustedColIndex < row.LastCol() { + cellValue := row.Col(adjustedColIndex) + // log.Printf("Row %d, Key: %s, Expected Column: %d, Value: %s", rowIndex, key, adjustedColIndex, cellValue) + cleanedValue := strings.ReplaceAll(cellValue, ",", "") // Remove commas + + // Handle percentage values + if strings.HasSuffix(cleanedValue, "%") { + stock[key] = cleanedValue + } else if key == "Market/Fair Value" || key == "Quantity" { + // Force parsing Market/Fair Value and Quantity as floats + if parsedValue, err := strconv.ParseFloat(cleanedValue, 64); err == nil { + stock[key] = parsedValue + // log.Printf("Forced float parsing for key: %s, value: %v", key, parsedValue) + } else { + stock[key] = cellValue // Fallback to original if parsing fails + // log.Printf("Failed to parse as float for key: %s, value: %s", key, cellValue) + } + } else { + stock[key] = cellValue + } + } else { + stock[key] = "" + } + } + // println("*********************") + // Skip rows without meaningful data in "Name of the Instrument" + if stock["Name of the Instrument"] == "" { + continue + } + + // Append the stock detail to the list + stockDetail = append(stockDetail, stock) + } + } + // println(mutualFundName, "Mutual Fund Name") + hash := sha256.New() + hashName := sha256.New() + hashName.Write([]byte(mutualFundName)) + + hash.Write([]byte(mutualFundName + month)) + hashedId := hex.EncodeToString(hash.Sum(nil)) + hashedName := hex.EncodeToString(hashName.Sum(nil)) + collection := mongo_client.Client.Database(os.Getenv("DATABASE")).Collection(os.Getenv("MFHOLDING")) + + validStockDetails := []map[string]interface{}{} + + for _, stockDetail := range stockDetail { + if _, ok := stockDetail["Name of the Instrument"]; !ok { + // fmt.Println("Skipping entry: Missing Name of the Instrument") + continue + } + if _, ok := stockDetail["ISIN"]; !ok || stockDetail["ISIN"] == "" { + // fmt.Println("Skipping entry: Missing ISIN") + continue + } + if _, ok := stockDetail["Quantity"]; !ok { + // // fmt.Println("Skipping entry: Missing Quantity") + continue + } + if _, ok := stockDetail["Market/Fair Value"]; !ok { + // // fmt.Println("Skipping entry: Missing Market/Fair Value") + continue + } + if _, ok := stockDetail["Percentage of AUM"]; !ok { + // // fmt.Println("Skipping entry: Missing Percentage of AUM") + continue + } + validStockDetails = append(validStockDetails, stockDetail) + } + stockDetail = validStockDetails + + document := bson.M{ + "_id": hashedId, + "month": month, + "mutual_fund_name": mutualFundName, + "stock_details": stockDetail, + "hash": hashedName, + "created_at": time.Now(), + } + if mutualFundName == "" || stockDetail == nil || len(stockDetail) == 0 { + // fmt.Println("Skipping empty document") + continue + } + var existingDocument bson.M + err = collection.FindOne(context.TODO(), bson.M{"_id": hashedId}).Decode(&existingDocument) + if err == mongo.ErrNoDocuments { + // Document does not exist, so insert it + _, err := collection.InsertOne(context.TODO(), document) + if err != nil { + println(err.Error()) + } + // // fmt.Println("Document inserted with ID:", insertResult.InsertedID) + } else if err != nil { + println(err.Error()) + } + + } + return nil +} + +// Helper function to limit column count to avoid misalignment issues +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func extractMonth(fileName string) string { + patterns := []*regexp.Regexp{ + regexp.MustCompile(`\d{2}\.\d{2}\.\d{4}`), // dd.mm.yyyy + regexp.MustCompile(`\d{2}-\d{2}-\d{4}`), // dd-mm-yyyy + regexp.MustCompile(`\d{2}-\d{2}-\d{2}`), // dd-mm-yy + regexp.MustCompile(`\d{2}-\d{4}`), // mm-yyyy + regexp.MustCompile(`(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d{2}`), // Abbreviated month-year (e.g., Mar-23) + regexp.MustCompile(`(January|February|March|April|May|June|July|August|September|October|November|December)-\d{4}`), // Full month-year (e.g., March-2021) + } + for _, pattern := range patterns { + match := pattern.FindString(fileName) + if match != "" { + parsedDate := parseDate(match) + if parsedDate != "" { + return parsedDate + } + } + } + return "" +} + +func parseDate(dateStr string) string { + if strings.Contains(dateStr, ".") { + t, err := time.Parse("02.01.2006", dateStr) + if err == nil { + return t.Format("2006-01-02") + } + } + layouts := []string{ + "02-01-2006", // dd-mm-yyyy + "02-01-06", // dd-mm-yy + "01-2006", // mm-yyyy + "Jan-06", // Abbreviated month-year + "January-2006", // Full month-year + } + + for _, layout := range layouts { + t, err := time.Parse(layout, dateStr) + if err == nil { + // Format the date in YYYY-MM-DD format + return t.Format("2006-01-02") + } + } + + // Handle month-year patterns (e.g., Mar-23, January-2021) + if len(dateStr) == 7 || len(dateStr) == 10 { + monthAbbrevToFull := map[string]string{ + "Jan": "January", "Feb": "February", "Mar": "March", "Apr": "April", + "May": "May", "Jun": "June", "Jul": "July", "Aug": "August", + "Sep": "September", "Oct": "October", "Nov": "November", "Dec": "December", + } + for abbr, full := range monthAbbrevToFull { + if strings.Contains(dateStr, abbr) { + dateStr = strings.Replace(dateStr, abbr, full, 1) + t, err := time.Parse("January-06", dateStr) + if err == nil { + return t.Format("2006-01") + } + } + } + } + return "" +} + +func extractFileName(fileURL string) string { + fileName := path.Base(fileURL) + return strings.TrimSuffix(fileName, path.Ext(fileName)) +} diff --git a/services/parse_service_test.go b/services/parse_service_test.go new file mode 100644 index 0000000..e63a7e4 --- /dev/null +++ b/services/parse_service_test.go @@ -0,0 +1,55 @@ +package services + +import ( + "testing" +) + +// Test generated using Keploy +func TestNormalizeWhitespace(t *testing.T) { + input := "This is a test" + expected := "This is a test" + result := normalizeWhitespace(input) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } +} + +// Test generated using Keploy +func TestRemoveZeroWidthChars(t *testing.T) { + input := "Hello\u200BWorld" + expected := "HelloWorld" + result := removeZeroWidthChars(input) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } +} + +// Test generated using Keploy +func TestExtractMonth(t *testing.T) { + fileName := "report-03-2023.xlsx" + expected := "2023-03-01" + result := extractMonth(fileName) + if result != expected { + t.Errorf("Expected %s but got %s", expected, result) + } +} + +// Test generated using Keploy +func TestExtractFileName(t *testing.T) { + url := "https://example.com/path/to/file.xlsx" + expected := "file" + result := extractFileName(url) + if result != expected { + t.Errorf("Expected %v but got %v", expected, result) + } +} + +// Test generated using Keploy +func TestCleanHTMLContent(t *testing.T) { + input := "Hello\u200B World !" + expected := "Hello World !" + result := cleanHTMLContent(input) + if result != expected { + t.Errorf("Expected %v but got %v", expected, result) + } +} diff --git a/utils/helpers/helpers.go b/utils/helpers/helpers.go index 13e7629..81c01f1 100644 --- a/utils/helpers/helpers.go +++ b/utils/helpers/helpers.go @@ -3,7 +3,7 @@ package helpers import ( "errors" "fmt" - "io/ioutil" + "io" "math" "net/http" "os" @@ -379,7 +379,7 @@ func FetchPeerData(dataWarehouseID string) ([]map[string]string, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - bodyBytes, _ := ioutil.ReadAll(resp.Body) + bodyBytes, _ := io.ReadAll(resp.Body) bodyString := string(bodyBytes) zap.L().Error("Received non-200 response code", zap.Int("status_code", resp.StatusCode), zap.String("body", bodyString)) return nil, fmt.Errorf("received non-200 response code from peers API: %d", resp.StatusCode) diff --git a/utils/helpers/helpers_test.go b/utils/helpers/helpers_test.go index 7961d54..1e88703 100644 --- a/utils/helpers/helpers_test.go +++ b/utils/helpers/helpers_test.go @@ -6,6 +6,11 @@ import ( "strings" "testing" + "fmt" + "net/http" + "net/http/httptest" + "os" + "github.com/PuerkitoBio/goquery" "go.mongodb.org/mongo-driver/bson/primitive" "gopkg.in/mgo.v2/bson" @@ -1274,3 +1279,267 @@ func TestCalculateOperatingEfficiencyScore_TotalAssetsMissing(t *testing.T) { t.Errorf("Expected %v got %v", expected, result) } } + +// Test generated using Keploy +func TestIncreaseInRoa_NotEnoughData(t *testing.T) { + netProfit := primitive.A{"1000"} + totalAssets := primitive.A{"5000"} + result := increaseInRoa(netProfit, totalAssets) + if result { + t.Errorf("Expected false, got %v", result) + } +} + +// Test generated using Keploy +func TestSafeToFloat_EmptyString(t *testing.T) { + input := "" + _, err := safeToFloat(input) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +// Test generated using Keploy +func TestFetchPeerData_Success(t *testing.T) { + // Create a mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a valid HTML response + fmt.Fprintln(w, ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Peer Company100015.050002.0%1005%20010%15%
Total1100015.050002.0%1005%20010%15%
+ `) + })) + defer server.Close() + + // Override the COMPANY_URL environment variable + os.Setenv("COMPANY_URL", server.URL) + + dataWarehouseID := "validID" + + peersData, err := FetchPeerData(dataWarehouseID) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(peersData) != 2 { + t.Errorf("Expected 2 peers data, got %d", len(peersData)) + } +} + +// Test generated using Keploy +func TestGetMarketCapCategory_NaNInput(t *testing.T) { + input := "NaN" + expected := "Unknown Category" + result := GetMarketCapCategory(input) + if result != expected { + t.Errorf("Expected %v, got %v", expected, result) + } +} + +// Test generated using Keploy +func TestGetMarketCapCategory_NonNumericInput(t *testing.T) { + input := "invalid" + expected := "Small Cap" + result := GetMarketCapCategory(input) + if result != expected { + t.Errorf("Expected %v, got %v", expected, result) + } +} + +// Test generated using Keploy +func TestCompareWithPeers_MedianParseError(t *testing.T) { + stock := types.Stock{ + Name: "Test Stock", + PE: 15.5, + } + peers := primitive.A{ + bson.M{ + "pe": "10.0", + }, + "invalid median data", + } + result := compareWithPeers(stock, peers) + if result == 0.0 { + t.Errorf("Expected a non-zero score, got %v", result) + } +} + +// Test generated using Keploy +func TestFetchPeerData_Non200Response(t *testing.T) { + // Create a mock server that returns a 500 Internal Server Error + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + // Override the COMPANY_URL environment variable + os.Setenv("COMPANY_URL", server.URL) + + dataWarehouseID := "validID" + + _, err := FetchPeerData(dataWarehouseID) + if err == nil { + t.Errorf("Expected error due to non-200 response, got nil") + } +} + +// Test generated using Keploy +func TestFetchCompanyData_ValidURL(t *testing.T) { + // Create a mock server that returns valid HTML + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, `
`) + })) + defer server.Close() + + // Fetch company data using the mock server URL + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if data == nil { + t.Errorf("Expected non-nil data, got nil") + } +} + +// Test generated using Keploy +func TestFetchCompanyData_MissingSections(t *testing.T) { + html := ` + + +
+
  • + Market Cap + 100000 +
    • + + ` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprintln(w, html) + })) + defer server.Close() + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if _, ok := data["profitLoss"]; ok { + t.Errorf("Did not expect profitLoss data") + } + if _, ok := data["balanceSheet"]; ok { + t.Errorf("Did not expect balanceSheet data") + } +} + +// Test generated using Keploy +func TestFetchCompanyData_ProsAndCons(t *testing.T) { + html := ` + + +
    + +
    +
    + +
    + + ` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, html) + })) + defer server.Close() + + data, err := FetchCompanyData(server.URL) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + expectedPros := []string{"Strong financials", "Growing revenue"} + expectedCons := []string{"High debt"} + + if !reflect.DeepEqual(data["pros"], expectedPros) { + t.Errorf("Expected pros %v, got %v", expectedPros, data["pros"]) + } + + if !reflect.DeepEqual(data["cons"], expectedCons) { + t.Errorf("Expected cons %v, got %v", expectedCons, data["cons"]) + } +} + +// Test generated using Keploy +func TestCompareWithPeers_ValidData(t *testing.T) { + stock := types.Stock{ + Name: "Test Stock", + PE: 15.5, + MarketCap: 10000, + DividendYield: 2.5, + ROCE: 20.0, + QuarterlySales: 7000.0, + QuarterlyProfit: 2000.0, + } + peers := primitive.A{ + bson.M{"pe": "10.0", "market_cap": "8000", "div_yield": "2.0%", "roce": "18.0", "sales_qtr": "500", "np_qtr": "50"}, + bson.M{"pe": "12.0", "market_cap": "9000", "div_yield": "2.2%", "roce": "19.0", "sales_qtr": "600", "np_qtr": "60"}, + bson.M{"pe": "11.0", "market_cap": "8500", "div_yield": "2.1%", "roce": "18.5", "sales_qtr": "550", "np_qtr": "55"}, + } + result := compareWithPeers(stock, peers) + if result == 0.0 { + t.Errorf("Expected non-zero peer comparison score, got %v", result) + } +} + +// Test generated using Keploy +func TestFetchPeerData_MalformedURL(t *testing.T) { + // Override the COMPANY_URL environment variable with a malformed URL + os.Setenv("COMPANY_URL", "http://%41:8080/") // %41 is 'A', but this is a malformed URL + + dataWarehouseID := "validID" + _, err := FetchPeerData(dataWarehouseID) + if err == nil { + t.Errorf("Expected error due to malformed URL, got nil") + } +} + +// Test generated using Keploy +func TestIncreaseInRoa_InvalidElements(t *testing.T) { + netProfit := primitive.A{1000, 1500, 2000} // Integers instead of strings + totalAssets := primitive.A{5000, 5500, "invalid"} // Last element is not a valid string + result := increaseInRoa(netProfit, totalAssets) + if result != false { + t.Errorf("Expected false, got %v", result) + } +}