Merge pull request #21 from otiai10/develop

v2

Author: otiai10

README.md

@@ -1,5 +1,7 @@
 # Open Graph Parser for Golang
 
+Go implementation of https://ogp.me/
+
 [![Go](https://github.com/otiai10/opengraph/workflows/Go/badge.svg)](https://github.com/otiai10/opengraph/actions)
 [![codecov](https://codecov.io/gh/otiai10/opengraph/branch/master/graph/badge.svg)](https://codecov.io/gh/otiai10/opengraph)
 [![GoDoc](https://godoc.org/github.com/otiai10/opengraph?status.svg)](https://pkg.go.dev/github.com/otiai10/opengraph)
@@ -11,27 +13,57 @@ package main
 
 import (
 	"fmt"
-	"github.com/otiai10/opengraph"
+	"github.com/otiai10/opengraph/v2"
 )
 
 func main() {
-	og, err := opengraph.Fetch("https://www.youtube.com/watch?v=5blm22DeeHY")
-	fmt.Printf("OpenGraph: %+v\nError: %v\n", og, err)
+	ogp, err := opengraph.Fetch("https://github.com/")
+	fmt.Println(ogp, err)
 }
 ```
 
+# Advanced usage
+
+Set an option for fetching:
+```go
+intent := opengraph.Intent{
+	Context:     ctx,
+	HTTPClient:  client,
+	Strict:      true,
+	TrustedTags: []string{"meta", "title"},
+}
+ogp, err := opengraph.Fetch("https://ogp.me", intent)
+```
+
+Use any `io.Reader` as a data source:
+```go
+f, _ := os.Open("my_test.html")
+defer f.Close()
+ogp := &opengraph.OpenGraph{}
+err := ogp.Parse(f)
+```
+
+of if you already have parsed `*html.Node`:
+
+```go
+err := ogp.Walk(node)
+```
+
+Do you wanna make absolute URLs?:
+```go
+ogp.Image[0].URL // /logo.png
+ogp.ToAbs()
+ogp.Image[0].URL // https://ogp.me/logo.png
+```
+
 # CLI as a working example
 
 ```sh
 % go get github.com/otiai10/opengraph/ogp
 % ogp --help
+% ogp -A otiai10.com
 ```
 
-For more details, see [ogp/main.go](https://github.com/otiai10/opengraph/blob/master/ogp/main.go).
-
-# Advanced
+# Issues
 
-- [`og.Parse(body *io.Reader)`](https://godoc.org/github.com/otiai10/opengraph#OpenGraph.Parse) to re-use `*http.Response`
-- [`og.HTTPClient`](https://godoc.org/github.com/otiai10/opengraph#OpenGraph) to customize `*http.Client` for fetching
-- [`og.ToAbsURL()`](https://godoc.org/github.com/otiai10/opengraph#OpenGraph.ToAbsURL) to restore relative URL, e.g. `og.Favicon`
-- ~~[`og.Fulfill()`](https://godoc.org/github.com/otiai10/opengraph#OpenGraph.Fulfill) to fill empty fileds.~~ You ain't gonna need it
+- https://github.com/otiai10/opengraph/issues

all_test.go

@@ -2,157 +2,180 @@ package opengraph
 
 import (
 	"context"
-	"fmt"
-	"html/template"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
-	"github.com/otiai10/marmoset"
+	mmst "github.com/otiai10/marmoset"
 	. "github.com/otiai10/mint"
 	"golang.org/x/net/html"
 )
 
+var testserver *httptest.Server
+
+func TestMain(m *testing.M) {
+	testserver = createTestServer()
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestNew(t *testing.T) {
-	og := New(dummyServer(1).URL)
+	og := New("https://github.com")
 	Expect(t, og).TypeOf("*opengraph.OpenGraph")
-	Expect(t, og.Error).ToBe(nil)
-
-	When(t, "invalid url is given", func(t *testing.T) {
-		og := New(":invalid_url")
-		Expect(t, og.Error).Not().ToBe(nil)
-	})
 }
 
 func TestFetch(t *testing.T) {
+	og, err := Fetch(testserver.URL + "/case/01_hello")
+	Expect(t, err).ToBe(nil)
+	Expect(t, og).TypeOf("*opengraph.OpenGraph")
+	Expect(t, og.Description).ToBe("This description should be preferred")
+
+	When(t, "invalid URL provided", func(t *testing.T) {
+		_, err := Fetch(":SOME_INVALID_URL")
+		Expect(t, err).Not().ToBe(nil)
+	})
 
-	When(t, "invalid scheme is given", func(t *testing.T) {
-		_, err := Fetch(":invalid_url")
+	When(t, "og:image provided", func(t *testing.T) {
+		og, err := Fetch(testserver.URL + "/case/03_image")
+		Expect(t, err).ToBe(nil)
+		Expect(t, og).TypeOf("*opengraph.OpenGraph")
+	})
+	When(t, "structured image properties provided", func(t *testing.T) {
+		og, err := Fetch(testserver.URL + "/case/04_image_props")
+		Expect(t, err).ToBe(nil)
+		Expect(t, og).TypeOf("*opengraph.OpenGraph")
+	})
+	When(t, "content-type is NOT text/html", func(t *testing.T) {
+		_, err := Fetch(testserver.URL + "/case/00_data.json")
 		Expect(t, err).Not().ToBe(nil)
+		Expect(t, err.Error()).ToBe("Content type must be text/html")
+	})
+	When(t, "strict flag is on", func(t *testing.T) {
+		og, err := Fetch(testserver.URL+"/case/03_image", Intent{Strict: true})
+		Expect(t, err).ToBe(nil)
+		Expect(t, og.Title).ToBe("")
+		og, err = Fetch(testserver.URL+"/case/03_image", Intent{Strict: false})
+		Expect(t, err).ToBe(nil)
+		Expect(t, og.Title).ToBe("TEST: 03_image (title tag)")
 	})
-	When(t, "invalid url is given", func(t *testing.T) {
-		_, err := Fetch("htt://xxx/yyy")
+	When(t, "context is specified", func(t *testing.T) {
+		ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+		intent := Intent{
+			Context: ctx,
+		}
+		ogp, err := Fetch(testserver.URL+"/slow/100", intent)
+		cancel()
+		Expect(t, err).ToBe(nil)
+		Expect(t, ogp.Title).ToBe("Hello! Open Graph!!")
+		ctx, cancel = context.WithTimeout(context.Background(), 200*time.Millisecond)
+		_, err = Fetch(testserver.URL+"/slow/300", intent)
+		cancel()
 		Expect(t, err).Not().ToBe(nil)
 	})
-
-	s := dummyServer(1)
-	og, err := Fetch(s.URL)
-
-	Expect(t, err).ToBe(nil)
-	Expect(t, og.Title).ToBe("Hello! Open Graph!!")
-	Expect(t, og.Type).ToBe("website")
-	Expect(t, og.URL.Source).ToBe(s.URL)
-	Expect(t, len(og.Image)).ToBe(1)
-
-	Expect(t, og.Image[0].URL).ToBe("/images/01.png")
-	Expect(t, og.Favicon).ToBe("/images/01.favicon.png")
-	og.ToAbsURL()
-	Expect(t, og.Image[0].URL).ToBe(s.URL + "/images/01.png")
-	Expect(t, og.Favicon).ToBe(s.URL + "/images/01.favicon.png")
 }
 
-func TestFetch_02(t *testing.T) {
-	s := dummyServer(2)
-	og, err := Fetch(s.URL)
-
-	Expect(t, err).ToBe(nil)
-	Expect(t, og.Title).ToBe("はいさいナイト")
-	Expect(t, og.Description).ToBe("All Genre Music Party")
-	Expect(t, og.URL.Value).ToBe("https://haisai.party/")
-
-	// b := bytes.NewBuffer(nil)
-	// json.NewEncoder(b).Encode(og)
-	// Expect(t, strings.Trim(b.String(), "\n")).ToBe(fmt.Sprintf(
-	// 	`{"Policy":{"TrustedTags":["meta","link","title"]},"Title":"はいさいナイト","Type":"website","URL":{"Source":"%s","Scheme":"http","Opaque":"","User":null,"Host":"%s","Path":"","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":"","Value":"%s"},"SiteName":"","Image":[],"Video":[],"Audio":[],"Description":"All Genre Music Party","Determiner":"","Locale":"","LocaleAlt":[],"Favicon":"/favicon.ico"}`,
-	// 	s.URL,
-	// 	strings.Replace(s.URL, "http://", "", -1),
-	// 	og.URL.Value,
-	// ))
-}
+func TestOpenGraph_Fetch(t *testing.T) {
+	ogp := &OpenGraph{}
+	err := ogp.Fetch()
+	Expect(t, err.Error()).ToBe("no URL given yet")
 
-func TestFetch_03(t *testing.T) {
-	s := dummyServer(3)
-	og, err := Fetch(s.URL)
+	ogp.Intent.URL = testserver.URL + "/case/01_hello"
+	err = ogp.Fetch()
 	Expect(t, err).ToBe(nil)
-	err = og.ToAbsURL().Fulfill()
-	Expect(t, err).ToBe(nil)
-	Expect(t, og.Image[0].URL).ToBe("http://www-cdn.jtvnw.net/images/twitch_logo3.jpg")
-}
 
-func TestFetch_04(t *testing.T) {
-	s := dummyServer(4)
-	og, err := Fetch(s.URL)
-	Expect(t, err).ToBe(nil)
-	Expect(t, len(og.Image)).ToBe(1)
-	Expect(t, og.Image[0].URL).ToBe("/images/01.png")
+	When(t, "ogp already has an error", func(t *testing.T) {
+		ogp := New(":INVALID_URL")
+		err := ogp.Fetch()
+		Expect(t, err).Not().ToBe(nil)
+	})
 }
 
-func TestFetch_05_TrustedTags(t *testing.T) {
-	s := dummyServer(2)
-	res, err := http.Get(s.URL)
+func TestFetchVideo(t *testing.T) {
+	og, err := Fetch(testserver.URL + "/case/05_video")
 	Expect(t, err).ToBe(nil)
-	defer res.Body.Close()
-	og := New(s.URL)
-	og.Policy.TrustedTags = []string{HTMLMetaTag}
-	err = og.Parse(res.Body)
-	Expect(t, err).ToBe(nil)
-	Expect(t, og.Title).Not().ToBe("はいさいナイト")
-	Expect(t, og.Title).ToBe("")
+	Expect(t, og.Video).ToBe([]Video{
+		{
+			URL:       "https://www.youtube.com/embed/1MxA0i2rxQo",
+			SecureURL: "https://www.youtube.com/embed/1MxA0i2rxQo",
+			Type:      "text/html",
+			Width:     1280,
+			Height:    720,
+		},
+	})
 }
 
-func TestFetchWithContext(t *testing.T) {
-	s := dummySlowServer(time.Millisecond * 300)
-	defer s.Close()
-
-	ctx500ms, cancel500ms := context.WithTimeout(context.Background(), time.Millisecond*500)
-	_, err := FetchWithContext(ctx500ms, s.URL)
+func TestOpenGraph_Parse(t *testing.T) {
+	ogp := New("")
+	r := strings.NewReader(`<html><meta property="og:title" content="test_test"></html>`)
+	err := ogp.Parse(r)
 	Expect(t, err).ToBe(nil)
-	cancel500ms()
-
-	ctx100ms, cancel100ms := context.WithTimeout(context.Background(), time.Millisecond*100)
-	_, err = FetchWithContext(ctx100ms, s.URL)
-	Expect(t, err).Match(context.DeadlineExceeded.Error())
-	cancel100ms()
+	Expect(t, ogp.Title).ToBe("test_test")
 }
 
 func TestOpenGraph_Walk(t *testing.T) {
-	s := dummyServer(2)
-	res, err := http.Get(s.URL)
+	res, err := http.Get(testserver.URL + "/case/01_hello")
 	Expect(t, err).ToBe(nil)
 
 	node, err := html.Parse(res.Body)
 	Expect(t, err).ToBe(nil)
 
-	og := New(s.URL)
+	og := New(testserver.URL + "/case/01_hello")
 	err = og.Walk(node)
 	Expect(t, err).ToBe(nil)
-	Expect(t, og.Title).ToBe("はいさいナイト")
+	Expect(t, og.Title).ToBe("Hello! Open Graph!!")
 	res.Body.Close()
 
-	again := New(s.URL)
+	again := New(testserver.URL + "/case/01_hello")
 	err = again.Walk(node)
 	Expect(t, err).ToBe(nil)
-	Expect(t, again.Title).ToBe("はいさいナイト")
+	Expect(t, again.Title).ToBe("Hello! Open Graph!!")
 }
 
-func dummyServer(id int) *httptest.Server {
-	marmoset.LoadViews("./test/html")
-	r := marmoset.NewRouter()
-	r.GET("/", func(w http.ResponseWriter, r *http.Request) {
-		marmoset.Render(w).HTML(fmt.Sprintf("%02d", id), nil)
-	})
-	r.GET("/case/01", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusNotFound)
-	})
-	return httptest.NewServer(r)
+func TestOpenGraph_ToAbs(t *testing.T) {
+	ogp := New(testserver.URL + "/case/01_hello")
+	err := ogp.Fetch()
+	Expect(t, err).ToBe(nil)
+	u, err := url.Parse(ogp.Image[0].URL)
+	Expect(t, err).ToBe(nil)
+	Expect(t, u.IsAbs()).ToBe(false)
+	err = ogp.ToAbs()
+	Expect(t, err).ToBe(nil)
+	u, err = url.Parse(ogp.Image[0].URL)
+	Expect(t, err).ToBe(nil)
+	Expect(t, u.IsAbs()).ToBe(true)
+	u, err = url.Parse(ogp.Audio[0].URL)
+	Expect(t, err).ToBe(nil)
+	Expect(t, u.IsAbs()).ToBe(true)
 }
 
-func dummySlowServer(d time.Duration) *httptest.Server {
-	var h = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(d)
-		t, _ := template.ParseFiles("./test/html/02.html")
-		t.Execute(w, nil)
+// This server is ONLY for testing.
+func createTestServer() *httptest.Server {
+	mmst.LoadViews("./test/html")
+	r := mmst.NewRouter()
+	r.GET("/case/(?P<name>[_a-zA-Z0-9]+)", func(w http.ResponseWriter, r *http.Request) {
+		name := r.FormValue("name")
+		switch {
+		case strings.HasSuffix(name, ".json"):
+			mmst.Render(w).JSON(200, mmst.P{})
+		case name == "notfound":
+			w.WriteHeader(http.StatusNotFound)
+		default:
+			mmst.Render(w).HTML(name, nil)
+		}
 	})
-	return httptest.NewServer(h)
+	r.GET("/slow/(?P<msec>[0-9]+)", func(w http.ResponseWriter, r *http.Request) {
+		msec, err := strconv.Atoi(r.FormValue("msec"))
+		if err != nil {
+			w.WriteHeader(404)
+			return
+		}
+		time.Sleep(time.Duration(msec) * time.Millisecond)
+		mmst.Render(w).HTML("01_hello", nil)
+	})
+	return httptest.NewServer(r)
 }

example_test.go

@@ -1,28 +1,14 @@
 package opengraph
 
-import (
-	"fmt"
-	"log"
-	"net/http"
-)
+import "fmt"
 
 func ExampleFetch() {
-	ogp, _ := Fetch("https://github.com/otiai10/gosseract")
-	fmt.Println(ogp.Title)
-	// Output: otiai10/gosseract
-}
-
-func ExampleOpenGraph_Parse() {
-
-	client := http.DefaultClient
-	res, err := client.Get("https://github.com/otiai10/amesh")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer res.Body.Close()
-
-	ogp := new(OpenGraph)
-	ogp.Parse(res.Body)
-	fmt.Println(ogp.Title)
-	// Output: otiai10/amesh
+	ogp, err := Fetch("https://ogp.me/")
+	fmt.Println("title:", ogp.Title)
+	fmt.Println("type:", ogp.Type)
+	fmt.Println("error:", err)
+	// Output:
+	// title: Open Graph protocol
+	// type: website
+	// error: <nil>
 }