feat: ✨ init

fetcher/fetcher.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"proxy-pool/storage"
+	"goproxy/storage"
 )
 
 // 代理来源定义
@@ -18,27 +18,171 @@ type Source struct {
 	Protocol string // http 或 socks5
 }
 
-// 内置多个免费代理来源
-var defaultSources = []Source{
+// 快速更新源（5-30分钟更新）- 用于紧急和补充模式
+var fastUpdateSources = []Source{
+	// proxifly - 每5分钟更新
+	{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/http/data.txt", "http"},
+	{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/socks4/data.txt", "socks5"},
+	{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/socks5/data.txt", "socks5"},
+	// ProxyScraper - 每30分钟更新
+	{"https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/http.txt", "http"},
+	{"https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/socks4.txt", "socks5"},
+	{"https://raw.githubusercontent.com/ProxyScraper/ProxyScraper/main/socks5.txt", "socks5"},
+	// monosans - 每小时更新
+	{"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt", "http"},
+}
+
+// 慢速更新源（每天更新）- 用于优化轮换模式
+var slowUpdateSources = []Source{
+	// TheSpeedX - 每天更新
+	{"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt", "http"},
+	{"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt", "socks5"},
+	{"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt", "socks5"},
+	// monosans SOCKS
+	{"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt", "socks5"},
+	{"https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt", "socks5"},
+	// databay-labs - 备用源
 	{"https://cdn.jsdelivr.net/gh/databay-labs/free-proxy-list/http.txt", "http"},
 	{"https://cdn.jsdelivr.net/gh/databay-labs/free-proxy-list/socks5.txt", "socks5"},
-	{"https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.txt", ""},
 }
 
+// 所有源
+var allSources = append(fastUpdateSources, slowUpdateSources...)
+
 type Fetcher struct {
-	sources []Source
-	client  *http.Client
+	sources       []Source
+	client        *http.Client
+	sourceManager *SourceManager
 }
 
-func New(httpURL, socks5URL string) *Fetcher {
+func New(httpURL, socks5URL string, sourceManager *SourceManager) *Fetcher {
 	return &Fetcher{
-		sources: defaultSources,
+		sources:       allSources,
+		sourceManager: sourceManager,
 		client: &http.Client{
 			Timeout: 30 * time.Second,
 		},
 	}
 }
 
+// FetchSmart 智能抓取：根据模式和协议需求选择源
+func (f *Fetcher) FetchSmart(mode string, preferredProtocol string) ([]storage.Proxy, error) {
+	var sources []Source
+
+	switch mode {
+	case "emergency":
+		// 紧急模式：使用所有可用源
+		sources = f.filterAvailableSources(allSources, preferredProtocol)
+		log.Printf("[fetch] 🚨 紧急模式: 使用 %d 个源", len(sources))
+
+	case "refill":
+		// 补充模式：使用快更新源
+		sources = f.filterAvailableSources(fastUpdateSources, preferredProtocol)
+		log.Printf("[fetch] 🔄 补充模式: 使用 %d 个快更新源", len(sources))
+
+	case "optimize":
+		// 优化模式：随机选择2-3个慢更新源
+		sources = f.selectRandomSources(slowUpdateSources, 3, preferredProtocol)
+		log.Printf("[fetch] ⚡ 优化模式: 使用 %d 个源", len(sources))
+
+	default:
+		sources = f.filterAvailableSources(fastUpdateSources, preferredProtocol)
+	}
+
+	if len(sources) == 0 {
+		return nil, fmt.Errorf("no available sources")
+	}
+
+	return f.fetchFromSources(sources)
+}
+
+// filterAvailableSources 过滤可用的源（通过断路器）
+func (f *Fetcher) filterAvailableSources(sources []Source, preferredProtocol string) []Source {
+	var available []Source
+	for _, src := range sources {
+		// 检查断路器
+		if f.sourceManager != nil && !f.sourceManager.CanUseSource(src.URL) {
+			continue
+		}
+		// 如果指定了协议偏好，优先该协议的源
+		if preferredProtocol != "" && src.Protocol != "" && src.Protocol != preferredProtocol {
+			continue
+		}
+		available = append(available, src)
+	}
+	return available
+}
+
+// selectRandomSources 随机选择N个源
+func (f *Fetcher) selectRandomSources(sources []Source, count int, preferredProtocol string) []Source {
+	available := f.filterAvailableSources(sources, preferredProtocol)
+	if len(available) <= count {
+		return available
+	}
+
+	// 随机打乱
+	shuffled := make([]Source, len(available))
+	copy(shuffled, available)
+	for i := range shuffled {
+		j := i + int(time.Now().UnixNano())%(len(shuffled)-i)
+		shuffled[i], shuffled[j] = shuffled[j], shuffled[i]
+	}
+
+	return shuffled[:count]
+}
+
+// fetchFromSources 从指定源列表抓取
+func (f *Fetcher) fetchFromSources(sources []Source) ([]storage.Proxy, error) {
+	type result struct {
+		proxies []storage.Proxy
+		source  Source
+		err     error
+	}
+
+	ch := make(chan result, len(sources))
+	for _, src := range sources {
+		go func(s Source) {
+			proxies, err := f.fetchFromURL(s.URL, s.Protocol)
+			ch <- result{proxies: proxies, source: s, err: err}
+		}(src)
+	}
+
+	var all []storage.Proxy
+	seen := make(map[string]bool)
+	for range sources {
+		r := <-ch
+		if r.err != nil {
+			log.Printf("[fetch] ❌ %s error: %v", r.source.URL, r.err)
+			if f.sourceManager != nil {
+				f.sourceManager.RecordFail(r.source.URL, 3, 5, 30)
+			}
+			continue
+		}
+
+		// 记录成功
+		if f.sourceManager != nil {
+			f.sourceManager.RecordSuccess(r.source.URL)
+		}
+
+		// 去重
+		var deduped []storage.Proxy
+		for _, p := range r.proxies {
+			if !seen[p.Address] {
+				seen[p.Address] = true
+				deduped = append(deduped, p)
+			}
+		}
+		log.Printf("[fetch] ✅ %d 个 %s 代理 from %s", len(deduped), r.source.Protocol, r.source.URL)
+		all = append(all, deduped...)
+	}
+
+	if len(all) == 0 {
+		return nil, fmt.Errorf("no proxies fetched")
+	}
+	log.Printf("[fetch] 总共抓取: %d 个代理（去重后）", len(all))
+	return all, nil
+}
+
 // Fetch 从所有来源并发抓取代理
 func (f *Fetcher) Fetch() ([]storage.Proxy, error) {
 	type result struct {