sitemapper/httpfetch_test.go at master · mone/sitemapper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
package main_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/mone/sitemapper"
	"net/url"
	"net/http/httptest"
	"bytes"
	"net/http"
	"errors"
)

type HttpClientMock struct {
	response map[url.URL]string
}

func (client *HttpClientMock) Get (address url.URL) (resp *http.Response, err error) {
	recorder := httptest.NewRecorder()
	recorder.Body = bytes.NewBufferString(client.response[address])
	return recorder.Result(), nil
}

type BrokenHttpClientMock struct {}

func (client *BrokenHttpClientMock) Get (address url.URL) (resp *http.Response, err error) {
	return nil, errors.New("error")
}

type ComposedHttpClientMock struct {
	clients map[url.URL]HttpClient
}

func (client *ComposedHttpClientMock) Get (address url.URL) (resp *http.Response, err error) {
	return client.clients[address].Get(address)
}


var _ = Describe("StartHttpFetchers", func() {

	var (
		url1 *url.URL
		url2 *url.URL
		page1 string
		page2 string
	)

	BeforeEach(func() {
		url1, _ = url.Parse("http://www.example.com/")
		url2, _ = url.Parse("https://www.monzo.com/")

		page1 = "<html>page1</html>"
		page2 = "<html>page2</html>"
	})

	It("should fetch a couple of addresses", func(done Done) {
		client := HttpClientMock {
			map[url.URL]string{
				*url1: page1,
				*url2: page2,
			},
		}

		// add a buffer so we can fill it at once and wait for the responses
		inChan := make(chan url.URL, 2)

		outChan := StartHttpFetchers(inChan, &client)

		inChan <- *url1
		inChan <- *url2

		res1 := <-outChan
		res2 := <-outChan
		Consistently(outChan).ShouldNot(Receive())

		// order of output is not guaranteed, so we put everything together
		res := []HtmlPage{res1, res2}

		Expect(res).To(ContainElement(HtmlPage{
			*url1, []byte(page1),
		}))
		Expect(res).To(ContainElement(HtmlPage{
			*url2, []byte(page2),
		}))

		close(inChan)
		Eventually(outChan).Should(BeClosed())

		// tell ginko we're done
		close(done)

	})

	It("keep going in case of failures", func(done Done) {
		okClient := HttpClientMock {
			map[url.URL]string{
				*url2: page2,
			},
		}

		client := ComposedHttpClientMock{
			map[url.URL]HttpClient{
				*url1: &BrokenHttpClientMock{},
				*url2: &okClient,
			},
		}

		// add a buffer so we can fill it at once and wait for the responses
		inChan := make(chan url.URL, 2)

		outChan := StartHttpFetchers(inChan, &client)

		inChan <- *url1
		inChan <- *url2

		// currently there is no retry mechanism, so the failed request will produce an empty page
		res1 := <-outChan
		res2 := <-outChan
		Consistently(outChan).ShouldNot(Receive())

		res := []HtmlPage{res1, res2}

		Expect(res).To(ContainElement(HtmlPage{
			*url1, make([]byte, 0),
		}))
		Expect(res).To(ContainElement(HtmlPage{
			*url2, []byte(page2),
		}))

		close(inChan)
		Eventually(outChan).Should(BeClosed())

		// tell ginko we're done
		close(done)

	})

})