Skip to content

Commit

Permalink
Fixed bug with item.url
Browse files Browse the repository at this point in the history
  • Loading branch information
witjem committed Jan 6, 2023
1 parent ee969bc commit ed1b3ed
Show file tree
Hide file tree
Showing 5 changed files with 234 additions and 6 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-pkgz/expirable-cache v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
Expand Down
19 changes: 19 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ github.com/go-pkgz/rest v1.16.0 h1:3An51XJXUq1XtICHvfAEp048ZCbUHBEga3g0WEu3zK0=
github.com/go-pkgz/rest v1.16.0/go.mod h1:HHlLOt02NJc2sgffXBF6hYVMcRo4Gz3vjg43zTzN7VM=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
Expand All @@ -53,21 +55,38 @@ github.com/urfave/cli/v2 v2.17.1 h1:UzjDEw2dJQUE3iRaiNQ1VrVFbyAtKGH3VdkMoHA58V0=
github.com/urfave/cli/v2 v2.17.1/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b h1:6e93nYa3hNqAvLr0pD4PN1fFS+gKzp2zAXqrnTCstqU=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
83 changes: 83 additions & 0 deletions internal/pkg/feed/httpclient_mock_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 10 additions & 6 deletions internal/pkg/feed/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,9 @@ func (m Matcher) FindItem(doc *query.Document) (Item, error) {
return res, nil
}

func (c Config) Validate() error {
// todo
return nil
}

// HTTPClient interface for get content by URL.
//
//go:generate moq -out httpclient_mock_test.go -pkg feed_test . HTTPClient
type HTTPClient interface {
Get(ctx context.Context, url string) (io.ReadCloser, error)
}
Expand Down Expand Up @@ -156,7 +153,14 @@ func (r *Repository) getItem(ctx context.Context, matcher Matcher, itemURL strin
return Item{}, fmt.Errorf("get item content: %w", err)
}

return matcher.FindItem(doc)
res, err := matcher.FindItem(doc)
if err != nil {
return Item{}, err
}

res.Link = itemURL

return res, nil
}

func (r *Repository) IDs() []string {
Expand Down
121 changes: 121 additions & 0 deletions internal/pkg/feed/repository_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package feed_test

import (
"context"
"io"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"

"github.com/witjem/feedpls/internal/pkg/feed"
"github.com/witjem/feedpls/internal/pkg/query"
)

var feedHTML = `
<!doctype html>
<html lang="uk" class="no-js" dir="ltr">
<head>
<meta name="twitter:description" content="The world and galactic good news"/>
<meta name="twitter:title" content="Good news"/>
<title>The good news</title>
</head>
<body>
<article class="Post">
<ul>
<li><a href="/posts/what_is_the_best_way_to_learn_go">What is the best way to learn Go?</a></li>
<li><a href="/posts/special_way_for_us">Special way for us</a></li>
</ul>
</article>
</body>
`

var firstItemHTML = `
<!doctype html>
<html lang="uk" class="no-js" dir="ltr">
<head>
<meta name="twitter:description" content="Go is a great language. It is simple, fast, and has a large ecosystem"/>
<meta name="twitter:title" content="What is the best way to learn Go?"/>
<meta name="parsely-pub-date" content="2021-12-22T12:30:00Z"/>
<title>The good news</title>
</head>
`

var secondItemHTML = `
<!doctype html>
<html lang="uk" class="no-js" dir="ltr">
<head>
<meta name="twitter:description" content="It is a special way for us"/>
<meta name="twitter:title" content="The special way for us"/>
<meta name="parsely-pub-date" content="2021-12-23T14:48:00Z"/>
<title>The good news</title>
</head>
`

func TestGetFeed(t *testing.T) {
configs := []feed.Config{
{
FeedID: "news",
Title: "World news",
Description: "The world and galactic good news",
URL: "https://goodnews.com",
Matcher: feed.Matcher{
Engine: query.XPath,
ItemURL: query.Selector{
Expr: "//ul//a/@href",
},
Title: query.Selector{
Expr: "//meta[@name='twitter:title']/@content",
},
Description: query.Selector{
Expr: "//meta[@name='twitter:description']/@content",
},
Published: query.SelectorTime{
Selector: query.Selector{
Expr: "//meta[@name='parsely-pub-date']/@content",
},
Layout: "2006-01-02T15:04:05Z",
},
},
},
}

pages := map[string]string{
"https://goodnews.com": feedHTML,
"https://goodnews.com/posts/what_is_the_best_way_to_learn_go": firstItemHTML,
"https://goodnews.com/posts/special_way_for_us": secondItemHTML,
}

mockedHTTPClient := &HTTPClientMock{
GetFunc: func(ctx context.Context, url string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader(pages[url])), nil
},
}

repo := feed.NewRepository(configs, mockedHTTPClient)

res, err := repo.Get(context.Background(), "news")
assert.NoError(t, err)
assert.Equal(t, feed.Feed{
ID: "news",
Title: "World news",
Link: "https://goodnews.com",
Description: "The world and galactic good news",
Items: []feed.Item{
{
Title: "What is the best way to learn Go?",
Link: "https://goodnews.com/posts/what_is_the_best_way_to_learn_go",
Description: "Go is a great language. It is simple, fast, and has a large ecosystem",
Published: time.Date(2021, 12, 22, 12, 30, 0, 0, time.UTC),
},
{
Title: "The special way for us",
Link: "https://goodnews.com/posts/special_way_for_us",
Description: "It is a special way for us",
Published: time.Date(2021, 12, 23, 14, 48, 0, 0, time.UTC),
},
},
}, res)

}

0 comments on commit ed1b3ed

Please sign in to comment.