Added geocode source

nfx · Nov 6, 2022 · a66d12e · a66d12e
1 parent d13e2bd
commit a66d12e
Show file tree

Hide file tree

Showing 2 changed files with 156 additions and 0 deletions.
diff --git a/sources/geonode.go b/sources/geonode.go
@@ -0,0 +1,91 @@
+package sources
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/nfx/slrp/pmux"
+	"github.com/rs/zerolog/log"
+)
+
+var geoNodeURL = "https://proxylist.geonode.com/api/proxy-list"
+
+type geoNodeResult struct {
+	IP             string   `json:"ip"`
+	Port           string   `json:"port"`
+	Protocols      []string `json:"protocols"`
+	AnonymityLevel string   `json:"anonymityLevel"`
+	ResponseTime   int      `json:"responseTime"`
+	Country        string   `json:"country"`
+}
+
+func (r *geoNodeResult) IsGood() bool {
+	return r.AnonymityLevel == "elite" || r.AnonymityLevel == "anonymous"
+}
+
+func (r *geoNodeResult) Proxies() (proxies []pmux.Proxy) {
+	addr := fmt.Sprintf("%s:%s", r.IP, r.Port)
+	for _, v := range r.Protocols {
+		proxies = append(proxies, pmux.NewProxy(addr, v))
+	}
+	return
+}
+
+type geoNodeResultPage struct {
+	Data  []geoNodeResult `json:"data"`
+	Total int             `json:"total"`
+	Page  int             `json:"page"`
+	Limit int             `json:"limit"`
+}
+
+func init() {
+	Sources = append(Sources, Source{
+		ID:        23,
+		Homepage:  "https://geonode.com/free-proxy-list/",
+		Frequency: 3 * time.Hour,
+		Seed:      true,
+		Feed:      simpleGen(geoNode),
+	})
+}
+
+func geoNode(ctx context.Context, h *http.Client) (found []pmux.Proxy, err error) {
+	qs := url.Values{}
+	qs.Set("sort_by", "lastChecked")
+	qs.Set("sort_type", "desc")
+	qs.Set("limit", "500")
+	page := 1
+	var results geoNodeResultPage
+	for {
+		log.Info().Int("page", page).Msg("Loading geocode database")
+		qs.Set("page", fmt.Sprint(page))
+		r, err := h.Get(geoNodeURL + "?" + qs.Encode())
+		if err != nil {
+			return nil, err
+		}
+		raw, err := io.ReadAll(r.Body)
+		_ = r.Body.Close()
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(raw, &results)
+		if err != nil {
+			return nil, err
+		}
+		if len(results.Data) == 0 {
+			break
+		}
+		page = results.Page + 1
+		for _, d := range results.Data {
+			if !d.IsGood() {
+				continue
+			}
+			found = append(found, d.Proxies()...)
+		}
+	}
+	return
+}
diff --git a/sources/geonode_test.go b/sources/geonode_test.go
@@ -0,0 +1,65 @@
+package sources
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGeoNodeFixtures(t *testing.T) {
+	pages := map[int][]geoNodeResult{
+		1: {
+			{
+				IP: "127.0.0.1",
+				Port: "12345",
+				AnonymityLevel: "transparent",
+				Protocols: []string{"socks4"},
+			},
+			{
+				IP: "127.0.0.1",
+				Port: "12346",
+				AnonymityLevel: "elite",
+				Protocols: []string{"socks5"},
+			},
+			{
+				IP: "127.0.0.1",
+				Port: "12347",
+				AnonymityLevel: "anonymous",
+				Protocols: []string{"http"},
+			},
+		},
+		2: {
+			{
+				IP: "127.0.0.1",
+				Port: "12348",
+				AnonymityLevel: "anonymous",
+				Protocols: []string{"http", "https"},
+			},
+			{
+				IP: "127.0.0.1",
+				Port: "12350",
+				AnonymityLevel: "anonymous",
+				Protocols: []string{"socks4"},
+			},
+		},
+		3: {},
+	}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var page int
+		_, _ = fmt.Sscan(r.FormValue("page"), &page)
+		w.WriteHeader(200)
+		raw, _ := json.Marshal(geoNodeResultPage{
+			Data: pages[page],
+			Page: page,
+		})
+		w.Write(raw)
+	}))
+	defer server.Close()
+	geoNodeURL = server.URL
+	testSource(t, func(ctx context.Context) Src {
+		return ByName("geonode.com").Feed(ctx, http.DefaultClient)
+	}, 5)
+}