diff --git a/parsers/js_api.go b/parsers/js/api.go similarity index 70% rename from parsers/js_api.go rename to parsers/js/api.go index bc92fcf..c99f70e 100644 --- a/parsers/js_api.go +++ b/parsers/js/api.go @@ -1,19 +1,17 @@ -package parsers +package js import ( "encoding/json" "errors" "fmt" "io" - "log/slog" "net/http" - "sync" "github.com/blang/semver" "github.com/charmbracelet/log" "github.com/dop251/goja" "github.com/krau/SaveAny-Bot/common/utils/netutil" - "github.com/playwright-community/playwright-go" + "github.com/krau/SaveAny-Bot/parsers/parsers" ) func jsRegisterParser(vm *goja.Runtime) func(call goja.FunctionCall) goja.Value { @@ -57,7 +55,7 @@ func jsRegisterParser(vm *goja.Runtime) func(call goja.FunctionCall) goja.Value if parseFn == nil || goja.IsUndefined(parseFn) { return vm.NewGoError(errors.New("parser must provide a parse function")) } - AddParser(newJSParser(vm, handleFn, parseFn, metadata)) + parsers.Add(newJSParser(vm, handleFn, parseFn, metadata)) return goja.Undefined() } } @@ -173,74 +171,3 @@ var jsGhttp = func(vm *goja.Runtime) *goja.Object { }) return ghttp } - -var jsPlaywright = func(vm *goja.Runtime, logger *log.Logger) *goja.Object { - pwObj := vm.NewObject() - var installOnce sync.Once - slogger := slog.New(logger) - pwObj.Set("get", func(call goja.FunctionCall) goja.Value { - url := call.Argument(0).String() - var installErr error - installOnce.Do(func() { - installErr = playwright.Install(&playwright.RunOptions{ - Browsers: []string{"chromium"}, - DriverDirectory: "./playwright", - Logger: slogger, - }) - }) - if installErr != nil { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("failed to install playwright: %v", installErr), - }) - } - - pw, err := playwright.Run(&playwright.RunOptions{ - DriverDirectory: "./playwright", - Logger: slogger, - }) - if err != nil { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("failed to start playwright: %v", err), - }) - } - defer pw.Stop() - - browser, err := pw.Chromium.Launch() - if err != nil { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("failed to launch browser: %v", err), - }) - } - defer browser.Close() - - page, err := browser.NewPage() - if err != nil { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("failed to create page: %v", err), - }) - } - - resp, err := page.Goto(url, playwright.PageGotoOptions{ - WaitUntil: playwright.WaitUntilStateNetworkidle, - Timeout: playwright.Float(60000), - }) - if err != nil { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("failed to navigate: %v", err), - }) - } - if resp != nil && resp.Status() >= 400 { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("bad status code: %d", resp.Status()), - }) - } - content, err := page.Content() - if err != nil { - return vm.ToValue(map[string]any{ - "error": fmt.Sprintf("failed to get page content: %v", err), - }) - } - return vm.ToValue(content) - }) - return pwObj -} diff --git a/parsers/js/api_playwright.go b/parsers/js/api_playwright.go new file mode 100644 index 0000000..fbd04ff --- /dev/null +++ b/parsers/js/api_playwright.go @@ -0,0 +1,82 @@ +package js + +import ( + "fmt" + "log/slog" + "sync" + + "github.com/charmbracelet/log" + "github.com/dop251/goja" + "github.com/playwright-community/playwright-go" +) + +var jsPlaywright = func(vm *goja.Runtime, logger *log.Logger) *goja.Object { + pwObj := vm.NewObject() + var installOnce sync.Once + slogger := slog.New(logger) + pwObj.Set("get", func(call goja.FunctionCall) goja.Value { + url := call.Argument(0).String() + var installErr error + installOnce.Do(func() { + installErr = playwright.Install(&playwright.RunOptions{ + Browsers: []string{"chromium"}, + DriverDirectory: "./playwright", + Logger: slogger, + }) + }) + if installErr != nil { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("failed to install playwright: %v", installErr), + }) + } + + pw, err := playwright.Run(&playwright.RunOptions{ + DriverDirectory: "./playwright", + Logger: slogger, + }) + if err != nil { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("failed to start playwright: %v", err), + }) + } + defer pw.Stop() + + browser, err := pw.Chromium.Launch() + if err != nil { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("failed to launch browser: %v", err), + }) + } + defer browser.Close() + + page, err := browser.NewPage() + if err != nil { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("failed to create page: %v", err), + }) + } + + resp, err := page.Goto(url, playwright.PageGotoOptions{ + WaitUntil: playwright.WaitUntilStateNetworkidle, + Timeout: playwright.Float(60000), + }) + if err != nil { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("failed to navigate: %v", err), + }) + } + if resp != nil && resp.Status() >= 400 { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("bad status code: %d", resp.Status()), + }) + } + content, err := page.Content() + if err != nil { + return vm.ToValue(map[string]any{ + "error": fmt.Sprintf("failed to get page content: %v", err), + }) + } + return vm.ToValue(content) + }) + return pwObj +} diff --git a/parsers/js/api_playwright_stub.go b/parsers/js/api_playwright_stub.go new file mode 100644 index 0000000..2a2b7c0 --- /dev/null +++ b/parsers/js/api_playwright_stub.go @@ -0,0 +1,19 @@ +//go:build no_playwright + +package js + +import ( + "github.com/charmbracelet/log" + "github.com/dop251/goja" +) + +var jsPlaywright = func(vm *goja.Runtime, _ *log.Logger) *goja.Object { + pwObj := vm.NewObject() + unsupported := vm.ToValue(map[string]any{ + "error": "playwright is not supported in this build", + }) + pwObj.Set("get", func(call goja.FunctionCall) goja.Value { + return unsupported + }) + return pwObj +} diff --git a/parsers/js.go b/parsers/js/js.go similarity index 99% rename from parsers/js.go rename to parsers/js/js.go index 9533465..0bc1570 100644 --- a/parsers/js.go +++ b/parsers/js/js.go @@ -1,4 +1,4 @@ -package parsers +package js import ( "context" diff --git a/parsers/plugin.go b/parsers/js/plugin.go similarity index 96% rename from parsers/plugin.go rename to parsers/js/plugin.go index a2b94d6..404429d 100644 --- a/parsers/plugin.go +++ b/parsers/js/plugin.go @@ -1,4 +1,4 @@ -package parsers +package js import "github.com/blang/semver" diff --git a/parsers/kemono/download.go b/parsers/native/kemono/download.go similarity index 100% rename from parsers/kemono/download.go rename to parsers/native/kemono/download.go diff --git a/parsers/kemono/kemono.go b/parsers/native/kemono/kemono.go similarity index 100% rename from parsers/kemono/kemono.go rename to parsers/native/kemono/kemono.go diff --git a/parsers/kemono/post_info.go b/parsers/native/kemono/post_info.go similarity index 100% rename from parsers/kemono/post_info.go rename to parsers/native/kemono/post_info.go diff --git a/parsers/kemono/post_legacy.go b/parsers/native/kemono/post_legacy.go similarity index 100% rename from parsers/kemono/post_legacy.go rename to parsers/native/kemono/post_legacy.go diff --git a/parsers/kemono/user_profile.go b/parsers/native/kemono/user_profile.go similarity index 100% rename from parsers/kemono/user_profile.go rename to parsers/native/kemono/user_profile.go diff --git a/parsers/twitter/parser.go b/parsers/native/twitter/parser.go similarity index 100% rename from parsers/twitter/parser.go rename to parsers/native/twitter/parser.go diff --git a/parsers/twitter/types.go b/parsers/native/twitter/types.go similarity index 100% rename from parsers/twitter/types.go rename to parsers/native/twitter/types.go diff --git a/parsers/parser.go b/parsers/parser.go index 5c4df6d..2f13f52 100644 --- a/parsers/parser.go +++ b/parsers/parser.go @@ -3,41 +3,16 @@ package parsers import ( "context" "fmt" - "sync" - "github.com/krau/SaveAny-Bot/config" - "github.com/krau/SaveAny-Bot/parsers/kemono" - "github.com/krau/SaveAny-Bot/parsers/twitter" + "github.com/krau/SaveAny-Bot/parsers/js" + "github.com/krau/SaveAny-Bot/parsers/native/kemono" + "github.com/krau/SaveAny-Bot/parsers/native/twitter" + "github.com/krau/SaveAny-Bot/parsers/parsers" "github.com/krau/SaveAny-Bot/pkg/parser" ) -var ( - parsers []parser.Parser - parsersMu sync.Mutex - doConfig sync.Once - configParsers = func() { - if len(parsers) == 0 { - return - } - for _, pser := range parsers { - if configurable, ok := pser.(parser.ConfigurableParser); ok { - cfg := config.C().GetParserConfigByName(configurable.Name()) - if err := configurable.Configure(cfg); err != nil { - fmt.Printf("Error configuring parser %s: %v\n", configurable.Name(), err) - } - } - } - } -) - -func AddParser(p ...parser.Parser) { - parsersMu.Lock() - defer parsersMu.Unlock() - parsers = append(parsers, p...) -} - func init() { - AddParser(new(twitter.TwitterParser), new(kemono.KemonoParser)) + parsers.Add(new(twitter.TwitterParser), new(kemono.KemonoParser)) } var ( @@ -45,12 +20,11 @@ var ( ) func ParseWithContext(ctx context.Context, url string) (*parser.Item, error) { - doConfig.Do(configParsers) ch := make(chan *parser.Item, 1) errCh := make(chan error, 1) go func() { - for _, pser := range parsers { + for _, pser := range parsers.Get() { if !pser.CanHandle(url) { continue } @@ -76,11 +50,18 @@ func ParseWithContext(ctx context.Context, url string) (*parser.Item, error) { } func CanHandle(url string) (bool, parser.Parser) { - doConfig.Do(configParsers) - for _, pser := range parsers { + for _, pser := range parsers.Get() { if pser.CanHandle(url) { return true, pser } } return false, nil } + +func LoadPlugins(ctx context.Context, dir string) error { + return js.LoadPlugins(ctx, dir) +} + +func AddPlugin(ctx context.Context, code string, name string) error { + return js.AddPlugin(ctx, code, name) +} diff --git a/parsers/parsers/parsers.go b/parsers/parsers/parsers.go new file mode 100644 index 0000000..de5036b --- /dev/null +++ b/parsers/parsers/parsers.go @@ -0,0 +1,44 @@ +package parsers + +import ( + "fmt" + "sync" + + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/parser" +) + +var ( + parsers []parser.Parser + mu sync.Mutex + configOnce sync.Once + configParsers = func() { + mu.Lock() + defer mu.Unlock() + if len(parsers) == 0 { + return + } + for _, pser := range parsers { + if configurable, ok := pser.(parser.ConfigurableParser); ok { + cfg := config.C().GetParserConfigByName(configurable.Name()) + if err := configurable.Configure(cfg); err != nil { + fmt.Printf("Error configuring parser %s: %v\n", configurable.Name(), err) + } + } + } + } +) + +func Add(p ...parser.Parser) { + configOnce.Do(configParsers) + mu.Lock() + defer mu.Unlock() + parsers = append(parsers, p...) +} + +func Get() []parser.Parser { + configOnce.Do(configParsers) + mu.Lock() + defer mu.Unlock() + return parsers +}