Content-Length: 4234 | pFad | http://github.com/nfx/slrp/commit/a66d12e37c944d2fc3841a12b14e6704c3bcee47.patch
2C From a66d12e37c944d2fc3841a12b14e6704c3bcee47 Mon Sep 17 00:00:00 2001 From: Serge Smertin <259697+nfx@users.noreply.github.com> Date: Sun, 6 Nov 2022 23:40:16 +0100 Subject: [PATCH] Added `geocode` source --- sources/geonode.go | 91 +++++++++++++++++++++++++++++++++++++++++ sources/geonode_test.go | 65 +++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 sources/geonode.go create mode 100644 sources/geonode_test.go diff --git a/sources/geonode.go b/sources/geonode.go new file mode 100644 index 0000000..7418695 --- /dev/null +++ b/sources/geonode.go @@ -0,0 +1,91 @@ +package sources + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/nfx/slrp/pmux" + "github.com/rs/zerolog/log" +) + +var geoNodeURL = "https://proxylist.geonode.com/api/proxy-list" + +type geoNodeResult struct { + IP string `json:"ip"` + Port string `json:"port"` + Protocols []string `json:"protocols"` + AnonymityLevel string `json:"anonymityLevel"` + ResponseTime int `json:"responseTime"` + Country string `json:"country"` +} + +func (r *geoNodeResult) IsGood() bool { + return r.AnonymityLevel == "elite" || r.AnonymityLevel == "anonymous" +} + +func (r *geoNodeResult) Proxies() (proxies []pmux.Proxy) { + addr := fmt.Sprintf("%s:%s", r.IP, r.Port) + for _, v := range r.Protocols { + proxies = append(proxies, pmux.NewProxy(addr, v)) + } + return +} + +type geoNodeResultPage struct { + Data []geoNodeResult `json:"data"` + Total int `json:"total"` + Page int `json:"page"` + Limit int `json:"limit"` +} + +func init() { + Sources = append(Sources, Source{ + ID: 23, + Homepage: "https://geonode.com/free-proxy-list/", + Frequency: 3 * time.Hour, + Seed: true, + Feed: simpleGen(geoNode), + }) +} + +func geoNode(ctx context.Context, h *http.Client) (found []pmux.Proxy, err error) { + qs := url.Values{} + qs.Set("sort_by", "lastChecked") + qs.Set("sort_type", "desc") + qs.Set("limit", "500") + page := 1 + var results geoNodeResultPage + for { + log.Info().Int("page", page).Msg("Loading geocode database") + qs.Set("page", fmt.Sprint(page)) + r, err := h.Get(geoNodeURL + "?" + qs.Encode()) + if err != nil { + return nil, err + } + raw, err := io.ReadAll(r.Body) + _ = r.Body.Close() + if err != nil { + return nil, err + } + err = json.Unmarshal(raw, &results) + if err != nil { + return nil, err + } + if len(results.Data) == 0 { + break + } + page = results.Page + 1 + for _, d := range results.Data { + if !d.IsGood() { + continue + } + found = append(found, d.Proxies()...) + } + } + return +} diff --git a/sources/geonode_test.go b/sources/geonode_test.go new file mode 100644 index 0000000..4fbe9c2 --- /dev/null +++ b/sources/geonode_test.go @@ -0,0 +1,65 @@ +package sources + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func TestGeoNodeFixtures(t *testing.T) { + pages := map[int][]geoNodeResult{ + 1: { + { + IP: "127.0.0.1", + Port: "12345", + AnonymityLevel: "transparent", + Protocols: []string{"socks4"}, + }, + { + IP: "127.0.0.1", + Port: "12346", + AnonymityLevel: "elite", + Protocols: []string{"socks5"}, + }, + { + IP: "127.0.0.1", + Port: "12347", + AnonymityLevel: "anonymous", + Protocols: []string{"http"}, + }, + }, + 2: { + { + IP: "127.0.0.1", + Port: "12348", + AnonymityLevel: "anonymous", + Protocols: []string{"http", "https"}, + }, + { + IP: "127.0.0.1", + Port: "12350", + AnonymityLevel: "anonymous", + Protocols: []string{"socks4"}, + }, + }, + 3: {}, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var page int + _, _ = fmt.Sscan(r.FormValue("page"), &page) + w.WriteHeader(200) + raw, _ := json.Marshal(geoNodeResultPage{ + Data: pages[page], + Page: page, + }) + w.Write(raw) + })) + defer server.Close() + geoNodeURL = server.URL + testSource(t, func(ctx context.Context) Src { + return ByName("geonode.com").Feed(ctx, http.DefaultClient) + }, 5) +}Fetched URL: http://github.com/nfx/slrp/commit/a66d12e37c944d2fc3841a12b14e6704c3bcee47.patch
Alternative Proxies: