Skip to content

Commit 0b5de71

Browse files
committed
Upgrade the cookiestxt lib to 1.0.4 which fixed a fatal error
1 parent fa087c0 commit 0b5de71

File tree

4 files changed

+14
-10
lines changed

4 files changed

+14
-10
lines changed

go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ require (
2020
)
2121

2222
require (
23+
github.com/PuerkitoBio/goquery v1.8.0
2324
github.com/gocolly/colly/v2 v2.1.0
2425
github.com/golang/protobuf v1.5.2
25-
github.com/mengzhuo/cookiestxt v1.0.3
26-
github.com/PuerkitoBio/goquery v1.8.0
26+
github.com/mengzhuo/cookiestxt v1.0.4
2727
)

go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
5757
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
5858
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
5959
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
60-
github.com/mengzhuo/cookiestxt v1.0.3 h1:fdVqvdnxNAtZq5VJp5KuMv1xImmgs/m9bG5zNCy53M4=
61-
github.com/mengzhuo/cookiestxt v1.0.3/go.mod h1:hK5Q6nTJi1tZ0x1Sj3kuxPYpdDPVxF0m+1ebSgBheSs=
60+
github.com/mengzhuo/cookiestxt v1.0.4 h1:86h9CiU88KEM3bB2syZ8cJHABfrkmP4i5g0ytPUHvIs=
61+
github.com/mengzhuo/cookiestxt v1.0.4/go.mod h1:TVR3++zJTvADjIgChwqpw5giKn3BHOATkLSSLGKHJP8=
6262
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
6363
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
6464
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

util/crawlers.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@ package util
22

33
import (
44
"flag"
5-
"github.com/gocolly/colly/v2"
6-
"github.com/gocolly/colly/v2/queue"
7-
"github.com/its-my-data/doubak/proto"
85
"log"
96
"net"
107
"net/http"
118
"strings"
129
"time"
10+
11+
"github.com/gocolly/colly/v2"
12+
"github.com/gocolly/colly/v2/queue"
13+
"github.com/its-my-data/doubak/proto"
1314
)
1415

1516
const RequestTimeout = 5 * time.Minute
@@ -39,10 +40,11 @@ func NewColly() *colly.Collector {
3940
}
4041
cookies = c
4142
}
43+
log.Println("Cookies: ", cookies)
4244

4345
c := colly.NewCollector(
4446
colly.MaxDepth(1),
45-
colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.61"),
47+
colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"),
4648
)
4749

4850
c.OnError(func(r *colly.Response, err error) {

util/files.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@ package util
33
import (
44
"flag"
55
"fmt"
6-
"github.com/its-my-data/doubak/proto"
7-
"github.com/mengzhuo/cookiestxt"
86
"html"
97
"io/fs"
108
"log"
119
"net/http"
1210
"os"
1311
"path/filepath"
1412
"strings"
13+
14+
"github.com/its-my-data/doubak/proto"
15+
"github.com/mengzhuo/cookiestxt"
1516
)
1617

1718
const CollectorPathPrefix = "collector/"
@@ -88,6 +89,7 @@ func LoadCookiesFile(filePath string) ([]*http.Cookie, error) {
8889
func LoadCookiesFileToString(filePath string) (string, error) {
8990
cookies, err := LoadCookiesFile(filePath)
9091
if err != nil {
92+
log.Fatal(err)
9193
return "", nil
9294
}
9395

0 commit comments

Comments
 (0)