mirror of
https://github.com/isboyjc/GoProxy.git
synced 2026-05-25 18:29:56 +08:00
feat: ✨ init
This commit is contained in:
isboyjc
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"proxy-pool/storage"
|
||||
"goproxy/storage"
|
||||
)
|
||||
|
||||
// 代理来源定义
|
||||
@@ -18,27 +18,171 @@ type Source struct {
|
||||
Protocol string // http 或 socks5
|
||||
}
|
||||
|
||||
// 内置多个免费代理来源
|
||||
var defaultSources = []Source{
|
||||
// 快速更新源（5-30分钟更新）- 用于紧急和补充模式
|
||||
var fastUpdateSources = []Source{
|
||||
// proxifly - 每5分钟更新
|
||||
{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/http/data.txt", "http"},
|
||||
{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/socks4/data.txt", "socks5"},
|
||||
{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/socks5/data.txt", "socks5"},
|
||||
// ProxyScraper - 每30分钟更新
|
||||
{"https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/http.txt", "http"},
|
||||
{"https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/socks4.txt", "socks5"},
|
||||
{"https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/socks5.txt", "socks5"},
|
||||
// monosans - 每小时更新
|
||||
{"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt", "http"},
|
||||
}
|
||||
|
||||
// 慢速更新源（每天更新）- 用于优化轮换模式
|
||||
var slowUpdateSources = []Source{
|
||||
// TheSpeedX - 每天更新
|
||||
{"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt", "http"},
|
||||
{"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt", "socks5"},
|
||||
{"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt", "socks5"},
|
||||
// monosans SOCKS
|
||||
{"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt", "socks5"},
|
||||
{"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt", "socks5"},
|
||||
// databay-labs - 备用源
|
||||
{"https://cdn.jsdelivr.net/gh/databay-labs/free-proxy-list/http.txt", "http"},
|
||||
{"https://cdn.jsdelivr.net/gh/databay-labs/free-proxy-list/socks5.txt", "socks5"},
|
||||
{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.txt", ""},
|
||||
}
|
||||
|
||||
// 所有源
|
||||
var allSources = append(fastUpdateSources, slowUpdateSources...)
|
||||
|
||||
type Fetcher struct {
|
||||
sources []Source
|
||||
client *http.Client
|
||||
sources []Source
|
||||
client *http.Client
|
||||
sourceManager *SourceManager
|
||||
}
|
||||
|
||||
func New(httpURL, socks5URL string) *Fetcher {
|
||||
func New(httpURL, socks5URL string, sourceManager *SourceManager) *Fetcher {
|
||||
return &Fetcher{
|
||||
sources: defaultSources,
|
||||
sources: allSources,
|
||||
sourceManager: sourceManager,
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FetchSmart 智能抓取：根据模式和协议需求选择源
|
||||
func (f *Fetcher) FetchSmart(mode string, preferredProtocol string) ([]storage.Proxy, error) {
|
||||
var sources []Source
|
||||
|
||||
switch mode {
|
||||
case "emergency":
|
||||
// 紧急模式：使用所有可用源
|
||||
sources = f.filterAvailableSources(allSources, preferredProtocol)
|
||||
log.Printf("[fetch] 🚨 紧急模式: 使用 %d 个源", len(sources))
|
||||
|
||||
case "refill":
|
||||
// 补充模式：使用快更新源
|
||||
sources = f.filterAvailableSources(fastUpdateSources, preferredProtocol)
|
||||
log.Printf("[fetch] 🔄 补充模式: 使用 %d 个快更新源", len(sources))
|
||||
|
||||
case "optimize":
|
||||
// 优化模式：随机选择2-3个慢更新源
|
||||
sources = f.selectRandomSources(slowUpdateSources, 3, preferredProtocol)
|
||||
log.Printf("[fetch] ⚡ 优化模式: 使用 %d 个源", len(sources))
|
||||
|
||||
default:
|
||||
sources = f.filterAvailableSources(fastUpdateSources, preferredProtocol)
|
||||
}
|
||||
|
||||
if len(sources) == 0 {
|
||||
return nil, fmt.Errorf("no available sources")
|
||||
}
|
||||
|
||||
return f.fetchFromSources(sources)
|
||||
}
|
||||
|
||||
// filterAvailableSources 过滤可用的源（通过断路器）
|
||||
func (f *Fetcher) filterAvailableSources(sources []Source, preferredProtocol string) []Source {
|
||||
var available []Source
|
||||
for _, src := range sources {
|
||||
// 检查断路器
|
||||
if f.sourceManager != nil && !f.sourceManager.CanUseSource(src.URL) {
|
||||
continue
|
||||
}
|
||||
// 如果指定了协议偏好，优先该协议的源
|
||||
if preferredProtocol != "" && src.Protocol != "" && src.Protocol != preferredProtocol {
|
||||
continue
|
||||
}
|
||||
available = append(available, src)
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
// selectRandomSources 随机选择N个源
|
||||
func (f *Fetcher) selectRandomSources(sources []Source, count int, preferredProtocol string) []Source {
|
||||
available := f.filterAvailableSources(sources, preferredProtocol)
|
||||
if len(available) <= count {
|
||||
return available
|
||||
}
|
||||
|
||||
// 随机打乱
|
||||
shuffled := make([]Source, len(available))
|
||||
copy(shuffled, available)
|
||||
for i := range shuffled {
|
||||
j := i + int(time.Now().UnixNano())%(len(shuffled)-i)
|
||||
shuffled[i], shuffled[j] = shuffled[j], shuffled[i]
|
||||
}
|
||||
|
||||
return shuffled[:count]
|
||||
}
|
||||
|
||||
// fetchFromSources 从指定源列表抓取
|
||||
func (f *Fetcher) fetchFromSources(sources []Source) ([]storage.Proxy, error) {
|
||||
type result struct {
|
||||
proxies []storage.Proxy
|
||||
source Source
|
||||
err error
|
||||
}
|
||||
|
||||
ch := make(chan result, len(sources))
|
||||
for _, src := range sources {
|
||||
go func(s Source) {
|
||||
proxies, err := f.fetchFromURL(s.URL, s.Protocol)
|
||||
ch <- result{proxies: proxies, source: s, err: err}
|
||||
}(src)
|
||||
}
|
||||
|
||||
var all []storage.Proxy
|
||||
seen := make(map[string]bool)
|
||||
for range sources {
|
||||
r := <-ch
|
||||
if r.err != nil {
|
||||
log.Printf("[fetch] ❌ %s error: %v", r.source.URL, r.err)
|
||||
if f.sourceManager != nil {
|
||||
f.sourceManager.RecordFail(r.source.URL, 3, 5, 30)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// 记录成功
|
||||
if f.sourceManager != nil {
|
||||
f.sourceManager.RecordSuccess(r.source.URL)
|
||||
}
|
||||
|
||||
// 去重
|
||||
var deduped []storage.Proxy
|
||||
for _, p := range r.proxies {
|
||||
if !seen[p.Address] {
|
||||
seen[p.Address] = true
|
||||
deduped = append(deduped, p)
|
||||
}
|
||||
}
|
||||
log.Printf("[fetch] ✅ %d 个 %s 代理 from %s", len(deduped), r.source.Protocol, r.source.URL)
|
||||
all = append(all, deduped...)
|
||||
}
|
||||
|
||||
if len(all) == 0 {
|
||||
return nil, fmt.Errorf("no proxies fetched")
|
||||
}
|
||||
log.Printf("[fetch] 总共抓取: %d 个代理（去重后）", len(all))
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// Fetch 从所有来源并发抓取代理
|
||||
func (f *Fetcher) Fetch() ([]storage.Proxy, error) {
|
||||
type result struct {
|
||||
|
||||
152
fetcher/ip_query.go
Normal file
152
fetcher/ip_query.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// IPQueryLimiter 全局IP查询限流器
|
||||
var IPQueryLimiter *rate.Limiter
|
||||
|
||||
// InitIPQueryLimiter 初始化限流器
|
||||
func InitIPQueryLimiter(rps int) {
|
||||
IPQueryLimiter = rate.NewLimiter(rate.Limit(rps), rps*2)
|
||||
}
|
||||
|
||||
// GetExitIPInfo 通过代理获取出口 IP 和地理位置（多源降级）
|
||||
func GetExitIPInfo(client *http.Client) (string, string) {
|
||||
// 等待限流令牌
|
||||
if IPQueryLimiter != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := IPQueryLimiter.Wait(ctx); err != nil {
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
// 优先级1：ip-api.com
|
||||
if ip, loc := tryIPAPI(client); ip != "" {
|
||||
return ip, loc
|
||||
}
|
||||
|
||||
// 优先级2：ipapi.co
|
||||
if ip, loc := tryIPAPICo(client); ip != "" {
|
||||
return ip, loc
|
||||
}
|
||||
|
||||
// 优先级3：ipinfo.io
|
||||
if ip, loc := tryIPInfo(client); ip != "" {
|
||||
return ip, loc
|
||||
}
|
||||
|
||||
// 优先级4：仅获取IP
|
||||
if ip := tryHTTPBinIP(client); ip != "" {
|
||||
return ip, "UNKNOWN"
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// tryIPAPI 尝试 ip-api.com
|
||||
func tryIPAPI(client *http.Client) (string, string) {
|
||||
resp, err := client.Get("http://ip-api.com/json/?fields=status,country,countryCode,city,query")
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Status string `json:"status"`
|
||||
Query string `json:"query"`
|
||||
Country string `json:"country"`
|
||||
CountryCode string `json:"countryCode"`
|
||||
City string `json:"city"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil || result.Status != "success" {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
location := result.CountryCode
|
||||
if result.City != "" {
|
||||
location = fmt.Sprintf("%s %s", result.CountryCode, result.City)
|
||||
}
|
||||
|
||||
return result.Query, location
|
||||
}
|
||||
|
||||
// tryIPAPICo 尝试 ipapi.co
|
||||
func tryIPAPICo(client *http.Client) (string, string) {
|
||||
resp, err := client.Get("https://ipapi.co/json/")
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
IP string `json:"ip"`
|
||||
City string `json:"city"`
|
||||
CountryCode string `json:"country_code"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
location := result.CountryCode
|
||||
if result.City != "" {
|
||||
location = fmt.Sprintf("%s %s", result.CountryCode, result.City)
|
||||
}
|
||||
|
||||
return result.IP, location
|
||||
}
|
||||
|
||||
// tryIPInfo 尝试 ipinfo.io
|
||||
func tryIPInfo(client *http.Client) (string, string) {
|
||||
resp, err := client.Get("https://ipinfo.io/json")
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
IP string `json:"ip"`
|
||||
City string `json:"city"`
|
||||
Country string `json:"country"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
location := result.Country
|
||||
if result.City != "" {
|
||||
location = fmt.Sprintf("%s %s", result.Country, result.City)
|
||||
}
|
||||
|
||||
return result.IP, location
|
||||
}
|
||||
|
||||
// tryHTTPBinIP 尝试 httpbin（仅获取IP）
|
||||
func tryHTTPBinIP(client *http.Client) string {
|
||||
resp, err := client.Get("https://httpbin.org/ip")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result struct {
|
||||
Origin string `json:"origin"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return result.Origin
|
||||
}
|
||||
132
fetcher/source_manager.go
Normal file
132
fetcher/source_manager.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SourceManager 代理源管理器（断路器）
|
||||
type SourceManager struct {
|
||||
db *sql.DB
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func NewSourceManager(db *sql.DB) *SourceManager {
|
||||
return &SourceManager{db: db}
|
||||
}
|
||||
|
||||
// CanUseSource 判断源是否可用
|
||||
func (sm *SourceManager) CanUseSource(url string) bool {
|
||||
sm.mu.RLock()
|
||||
defer sm.mu.RUnlock()
|
||||
|
||||
var status string
|
||||
var disabledUntil sql.NullTime
|
||||
err := sm.db.QueryRow(
|
||||
`SELECT status, disabled_until FROM source_status WHERE url = ?`,
|
||||
url,
|
||||
).Scan(&status, &disabledUntil)
|
||||
|
||||
// 源不存在，默认可用
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// 检查是否被禁用且还在冷却期
|
||||
if status == "disabled" && disabledUntil.Valid {
|
||||
if time.Now().Before(disabledUntil.Time) {
|
||||
return false
|
||||
}
|
||||
// 冷却期结束，重置状态
|
||||
sm.db.Exec(`UPDATE source_status SET status = 'active', consecutive_fails = 0 WHERE url = ?`, url)
|
||||
return true
|
||||
}
|
||||
|
||||
return status != "disabled"
|
||||
}
|
||||
|
||||
// RecordSuccess 记录源抓取成功
|
||||
func (sm *SourceManager) RecordSuccess(url string) {
|
||||
sm.mu.Lock()
|
||||
defer sm.mu.Unlock()
|
||||
|
||||
sm.db.Exec(`
|
||||
INSERT INTO source_status (url, success_count, consecutive_fails, last_success, status)
|
||||
VALUES (?, 1, 0, CURRENT_TIMESTAMP, 'active')
|
||||
ON CONFLICT(url) DO UPDATE SET
|
||||
success_count = success_count + 1,
|
||||
consecutive_fails = 0,
|
||||
last_success = CURRENT_TIMESTAMP,
|
||||
status = 'active'
|
||||
`, url)
|
||||
}
|
||||
|
||||
// RecordFail 记录源抓取失败
|
||||
func (sm *SourceManager) RecordFail(url string, failThreshold, disableThreshold, cooldownMinutes int) {
|
||||
sm.mu.Lock()
|
||||
defer sm.mu.Unlock()
|
||||
|
||||
// 增加失败计数
|
||||
sm.db.Exec(`
|
||||
INSERT INTO source_status (url, fail_count, consecutive_fails, last_fail)
|
||||
VALUES (?, 1, 1, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(url) DO UPDATE SET
|
||||
fail_count = fail_count + 1,
|
||||
consecutive_fails = consecutive_fails + 1,
|
||||
last_fail = CURRENT_TIMESTAMP
|
||||
`, url)
|
||||
|
||||
// 检查是否需要降级或禁用
|
||||
var consecutiveFails int
|
||||
sm.db.QueryRow(`SELECT consecutive_fails FROM source_status WHERE url = ?`, url).Scan(&consecutiveFails)
|
||||
|
||||
if consecutiveFails >= disableThreshold {
|
||||
// 禁用源
|
||||
disabledUntil := time.Now().Add(time.Duration(cooldownMinutes) * time.Minute)
|
||||
sm.db.Exec(
|
||||
`UPDATE source_status SET status = 'disabled', disabled_until = ? WHERE url = ?`,
|
||||
disabledUntil, url,
|
||||
)
|
||||
log.Printf("[source] ⛔ 禁用源（连续失败%d次）: %s (冷却%d分钟)", consecutiveFails, url, cooldownMinutes)
|
||||
} else if consecutiveFails >= failThreshold {
|
||||
// 降级源
|
||||
sm.db.Exec(`UPDATE source_status SET status = 'degraded' WHERE url = ?`, url)
|
||||
log.Printf("[source] ⚠️ 降级源（连续失败%d次）: %s", consecutiveFails, url)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSourceStats 获取所有源的统计信息
|
||||
func (sm *SourceManager) GetSourceStats() ([]map[string]interface{}, error) {
|
||||
rows, err := sm.db.Query(`
|
||||
SELECT url, success_count, fail_count, consecutive_fails,
|
||||
last_success, last_fail, status
|
||||
FROM source_status
|
||||
ORDER BY success_count DESC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var stats []map[string]interface{}
|
||||
for rows.Next() {
|
||||
var url, status string
|
||||
var successCount, failCount, consecutiveFails int
|
||||
var lastSuccess, lastFail sql.NullTime
|
||||
|
||||
rows.Scan(&url, &successCount, &failCount, &consecutiveFails, &lastSuccess, &lastFail, &status)
|
||||
|
||||
stats = append(stats, map[string]interface{}{
|
||||
"url": url,
|
||||
"success_count": successCount,
|
||||
"fail_count": failCount,
|
||||
"consecutive_fails": consecutiveFails,
|
||||
"last_success": lastSuccess,
|
||||
"last_fail": lastFail,
|
||||
"status": status,
|
||||
})
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
Reference in New Issue
Block a user