File tree 4 files changed +14
-10
lines changed
4 files changed +14
-10
lines changed Original file line number Diff line number Diff line change @@ -20,8 +20,8 @@ require (
20
20
)
21
21
22
22
require (
23
+ github.com/PuerkitoBio/goquery v1.8.0
23
24
github.com/gocolly/colly/v2 v2.1.0
24
25
github.com/golang/protobuf v1.5.2
25
- github.com/mengzhuo/cookiestxt v1.0.3
26
- github.com/PuerkitoBio/goquery v1.8.0
26
+ github.com/mengzhuo/cookiestxt v1.0.4
27
27
)
Original file line number Diff line number Diff line change @@ -57,8 +57,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
57
57
github.com/jawher/mow.cli v1.1.0 /go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg =
58
58
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o =
59
59
github.com/kennygrant/sanitize v1.2.4 /go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak =
60
- github.com/mengzhuo/cookiestxt v1.0.3 h1:fdVqvdnxNAtZq5VJp5KuMv1xImmgs/m9bG5zNCy53M4 =
61
- github.com/mengzhuo/cookiestxt v1.0.3 /go.mod h1:hK5Q6nTJi1tZ0x1Sj3kuxPYpdDPVxF0m+1ebSgBheSs =
60
+ github.com/mengzhuo/cookiestxt v1.0.4 h1:86h9CiU88KEM3bB2syZ8cJHABfrkmP4i5g0ytPUHvIs =
61
+ github.com/mengzhuo/cookiestxt v1.0.4 /go.mod h1:TVR3++zJTvADjIgChwqpw5giKn3BHOATkLSSLGKHJP8 =
62
62
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM =
63
63
github.com/pmezard/go-difflib v1.0.0 /go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4 =
64
64
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 /go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA =
Original file line number Diff line number Diff line change @@ -2,14 +2,15 @@ package util
2
2
3
3
import (
4
4
"flag"
5
- "github.com/gocolly/colly/v2"
6
- "github.com/gocolly/colly/v2/queue"
7
- "github.com/its-my-data/doubak/proto"
8
5
"log"
9
6
"net"
10
7
"net/http"
11
8
"strings"
12
9
"time"
10
+
11
+ "github.com/gocolly/colly/v2"
12
+ "github.com/gocolly/colly/v2/queue"
13
+ "github.com/its-my-data/doubak/proto"
13
14
)
14
15
15
16
const RequestTimeout = 5 * time .Minute
@@ -39,10 +40,11 @@ func NewColly() *colly.Collector {
39
40
}
40
41
cookies = c
41
42
}
43
+ log .Println ("Cookies: " , cookies )
42
44
43
45
c := colly .NewCollector (
44
46
colly .MaxDepth (1 ),
45
- colly .UserAgent ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109 .0.0.0 Safari/537.36 Edg/109.0.1518.61 " ),
47
+ colly .UserAgent ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127 .0.0.0 Safari/537.36" ),
46
48
)
47
49
48
50
c .OnError (func (r * colly.Response , err error ) {
Original file line number Diff line number Diff line change @@ -3,15 +3,16 @@ package util
3
3
import (
4
4
"flag"
5
5
"fmt"
6
- "github.com/its-my-data/doubak/proto"
7
- "github.com/mengzhuo/cookiestxt"
8
6
"html"
9
7
"io/fs"
10
8
"log"
11
9
"net/http"
12
10
"os"
13
11
"path/filepath"
14
12
"strings"
13
+
14
+ "github.com/its-my-data/doubak/proto"
15
+ "github.com/mengzhuo/cookiestxt"
15
16
)
16
17
17
18
const CollectorPathPrefix = "collector/"
@@ -88,6 +89,7 @@ func LoadCookiesFile(filePath string) ([]*http.Cookie, error) {
88
89
func LoadCookiesFileToString (filePath string ) (string , error ) {
89
90
cookies , err := LoadCookiesFile (filePath )
90
91
if err != nil {
92
+ log .Fatal (err )
91
93
return "" , nil
92
94
}
93
95
You can’t perform that action at this time.
0 commit comments