代码拉取完成,页面将自动刷新
// Copyright 2018 Adam Tauber
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package colly
import (
"bytes"
"testing"
"github.com/PuerkitoBio/goquery"
)
var basicTestData = []byte(`<ul><li class="x">list <span>item</span> 1</li><li>list item 2</li><li>3</li></ul>`)
var nestedTestData = []byte(`<div><p>a</p><div><p>b</p><div><p>c</p></div></div></div>`)
var pointerSliceTestData = []byte(`<ul class="object"><li class="info">Information: <span>Info 1</span></li><li class="info">Information: <span>Info 2</span></li></ul>`)
func TestBasicUnmarshal(t *testing.T) {
doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(basicTestData))
e := &HTMLElement{
DOM: doc.First(),
}
s := struct {
String string `selector:"li:first-child" attr:"class"`
Items []string `selector:"li"`
Struct struct {
String string `selector:"li:last-child"`
}
}{}
if err := e.Unmarshal(&s); err != nil {
t.Error("Cannot unmarshal struct: " + err.Error())
}
if s.String != "x" {
t.Errorf(`Invalid data for String: %q, expected "x"`, s.String)
}
if s.Struct.String != "3" {
t.Errorf(`Invalid data for Struct.String: %q, expected "3"`, s.Struct.String)
}
}
func TestNestedUnmarshalMap(t *testing.T) {
doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(nestedTestData))
e := &HTMLElement{
DOM: doc.First(),
}
doc2, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(basicTestData))
e2 := &HTMLElement{
DOM: doc2.First(),
}
type nested struct {
String string
}
mapSelector := make(map[string]string)
mapSelector["String"] = "div > p"
mapSelector2 := make(map[string]string)
mapSelector2["String"] = "span"
s := nested{}
s2 := nested{}
if err := e.UnmarshalWithMap(&s, mapSelector); err != nil {
t.Error("Cannot unmarshal struct: " + err.Error())
}
if err := e2.UnmarshalWithMap(&s2, mapSelector2); err != nil {
t.Error("Cannot unmarshal struct: " + err.Error())
}
if s.String != "a" {
t.Errorf(`Invalid data for String: %q, expected "a"`, s.String)
}
if s2.String != "item" {
t.Errorf(`Invalid data for String: %q, expected "a"`, s.String)
}
}
func TestNestedUnmarshal(t *testing.T) {
doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(nestedTestData))
e := &HTMLElement{
DOM: doc.First(),
}
type nested struct {
String string `selector:"div > p"`
Struct *nested `selector:"div > div"`
}
s := nested{}
if err := e.Unmarshal(&s); err != nil {
t.Error("Cannot unmarshal struct: " + err.Error())
}
if s.String != "a" {
t.Errorf(`Invalid data for String: %q, expected "a"`, s.String)
}
if s.Struct.String != "b" {
t.Errorf(`Invalid data for Struct.String: %q, expected "b"`, s.Struct.String)
}
if s.Struct.Struct.String != "c" {
t.Errorf(`Invalid data for Struct.Struct.String: %q, expected "c"`, s.Struct.Struct.String)
}
}
func TestPointerSliceUnmarshall(t *testing.T) {
type info struct {
Text string `selector:"span"`
}
type object struct {
Info []*info `selector:"li.info"`
}
doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(pointerSliceTestData))
e := HTMLElement{DOM: doc.First()}
o := object{}
err := e.Unmarshal(&o)
if err != nil {
t.Fatalf("Failed to unmarshal page: %s\n", err.Error())
}
if len(o.Info) != 2 {
t.Errorf("Invalid length for Info: %d, expected 2", len(o.Info))
}
if o.Info[0].Text != "Info 1" {
t.Errorf("Invalid data for Info.[0].Text: %s, expected Info 1", o.Info[0].Text)
}
if o.Info[1].Text != "Info 2" {
t.Errorf("Invalid data for Info.[1].Text: %s, expected Info 2", o.Info[1].Text)
}
}
func TestStructSliceUnmarshall(t *testing.T) {
type info struct {
Text string `selector:"span"`
}
type object struct {
Info []info `selector:"li.info"`
}
doc, _ := goquery.NewDocumentFromReader(bytes.NewBuffer(pointerSliceTestData))
e := HTMLElement{DOM: doc.First()}
o := object{}
err := e.Unmarshal(&o)
if err != nil {
t.Fatalf("Failed to unmarshal page: %s\n", err.Error())
}
if len(o.Info) != 2 {
t.Errorf("Invalid length for Info: %d, expected 2", len(o.Info))
}
if o.Info[0].Text != "Info 1" {
t.Errorf("Invalid data for Info.[0].Text: %s, expected Info 1", o.Info[0].Text)
}
if o.Info[1].Text != "Info 2" {
t.Errorf("Invalid data for Info.[1].Text: %s, expected Info 2", o.Info[1].Text)
}
}
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。