goquery/example_test.go at master · PuerkitoBio/goquery · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
package goquery_test

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// This example scrapes the reviews shown on the home page of metalsucks.net.
func Example() {
	// Request the HTML page.
	res, err := http.Get("http://metalsucks.net")
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
	}

	// Load the HTML document
	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	// Find the review items
	doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) {
		// For each item found, get the band and title
		band := s.Find("a").Text()
		title := s.Find("i").Text()
		fmt.Printf("Review %d: %s - %s\n", i, band, title)
	})
	// To see the output of the Example while running the test suite (go test), simply
	// remove the leading "x" before Output on the next line. This will cause the
	// example to fail (all the "real" tests should pass).

	// xOutput: voluntarily fail the Example output.
}

// This example shows how to use NewDocumentFromReader from a file.
func ExampleNewDocumentFromReader_file() {
	// create from a file
	f, err := os.Open("some/file.html")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()
	doc, err := goquery.NewDocumentFromReader(f)
	if err != nil {
		log.Fatal(err)
	}
	// use the goquery document...
	_ = doc.Find("h1")
}

// This example shows how to use NewDocumentFromReader from a string.
func ExampleNewDocumentFromReader_string() {
	// create from a string
	data := `
<html>
	<head>
		<title>My document</title>
	</head>
	<body>
		<h1>Header</h1>
	</body>
</html>`

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
	if err != nil {
		log.Fatal(err)
	}
	header := doc.Find("h1").Text()
	fmt.Println(header)

	// Output: Header
}

func ExampleSingle() {
	html := `
<html>
  <body>
    <div>1</div>
    <div>2</div>
    <div>3</div>
  </body>
</html>
`
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
	if err != nil {
		log.Fatal(err)
	}

	// By default, the selector string selects all matching nodes
	multiSel := doc.Find("div")
	fmt.Println(multiSel.Text())

	// Using goquery.Single, only the first match is selected
	singleSel := doc.FindMatcher(goquery.Single("div"))
	fmt.Println(singleSel.Text())

	// Output:
	// 123
	// 1
}

// This example shows how to use the goquery.Text function to extract clean,
// human-readable text from a selection, similar to BeautifulSoup's get_text.
func ExampleText() {
	page := `
<html>
  <body>
    <div id="content">
      <h1>  Hello  </h1>
      <p>world</p>
      <script>var ignored = 1;</script>
    </div>
  </body>
</html>
`
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(page))
	if err != nil {
		log.Fatal(err)
	}

	// Trim each text node, join the remaining ones with a space, and skip the
	// text of <script> and <style> elements.
	text := goquery.Text(doc.Find("#content"), &goquery.TextOptions{
		Separator: " ",
		Trim:      true,
		Keep: func(n *html.Node) bool {
			if p := n.Parent; p != nil && p.Type == html.ElementNode {
				return p.Data != "script" && p.Data != "style"
			}
			return true
		},
	})
	fmt.Println(text)

	// Output:
	// Hello world
}