Simulate real web app cookies & WebSocket fields

9 files changed, 130 insertions, 57 deletions

diff --git a/bilibili/client.go b/bilibili/client.go
index 4abad60..f4b64fd 100644
--- a/bilibili/client.go
+++ b/bilibili/client.go
@@ -10,6 +10,7 @@ import (
 	"github.com/keuin/slbr/types"
 	"net"
 	"net/http"
+	"net/http/cookiejar"
 )
 
 const (
@@ -39,10 +40,15 @@ func NewBilibiliWithContext(ctx context.Context, netTypes []types.IpNetType, log
 	transport.DialTLSContext = nil
 	transport.DialContext, _ = np.NextNetworkType(dialer)
 
+	cookieJar, _ := cookiejar.New(nil)
+	httpClient := &http.Client{
+		Jar: cookieJar,
+	}
+
 	return &Bilibili{
 		logger:    logger,
 		userAgent: userAgent,
-		http:      http.DefaultClient,
+		http:      httpClient,
 		ctx:       ctx,
 		netTypes:  nets,
 	}
diff --git a/bilibili/connect.go b/bilibili/connect.go
new file mode 100644
index 0000000..472c979
--- /dev/null
+++ b/bilibili/connect.go
@@ -0,0 +1,13 @@
+package bilibili
+
+import (
+	"context"
+	"nhooyr.io/websocket"
+)
+
+func (b Bilibili) DialWebSocket(ctx context.Context, url string) (*websocket.Conn, error) {
+	ws, _, err := websocket.Dial(ctx, url, &websocket.DialOptions{
+		HTTPClient: b.http,
+	})
+	return ws, err
+}
diff --git a/bilibili/danmaku_server_info.go b/bilibili/danmaku_server_info.go
index 4d26873..8c79d51 100644
--- a/bilibili/danmaku_server_info.go
+++ b/bilibili/danmaku_server_info.go
@@ -1,11 +1,50 @@
 package bilibili
 
 import (
+	"errors"
 	"fmt"
 	"github.com/keuin/slbr/types"
+	"net/url"
 )
 
+type WebBannerResponse = BaseResponse[interface{}]
+
+const apiUrlPrefix = "https://api.live.bilibili.com"
+
 func (b *Bilibili) GetDanmakuServerInfo(roomId types.RoomId) (resp types.DanmakuServerInfoResponse, err error) {
-	url := fmt.Sprintf("https://api.live.bilibili.com/xlive/web-room/v1/index/getDanmuInfo?id=%d&type=0", roomId)
-	return callGet[types.DanmakuServerInfoResponse](b, url)
+	u := fmt.Sprintf("https://api.live.bilibili.com/xlive/web-room/v1/index/getDanmuInfo?id=%d&type=0", roomId)
+	return callGet[types.DanmakuServerInfoResponse](b, u)
+}
+
+// GetBUVID initializes cookie `buvid3`. If success, returns its value.
+func (b *Bilibili) GetBUVID() (string, error) {
+	const u = "https://data.bilibili.com/v/web/web_page_view"
+	_, _, err := callGetRaw(b, u)
+	if err != nil {
+		return "", err
+	}
+	uu, _ := url.Parse(apiUrlPrefix)
+	cookies := b.http.Jar.Cookies(uu)
+	var buvid3 *string
+	for _, c := range cookies {
+		if c.Name == "buvid3" {
+			buvid3 = &c.Value
+		}
+	}
+	if buvid3 == nil {
+		return "", errors.New("failed to get buvid3")
+	}
+	return *buvid3, nil
+}
+
+// GetLiveBUVID initializes cookie `LIVE_BUVID`. This should be called before GetDanmakuServerInfo.
+func (b *Bilibili) GetLiveBUVID(roomId types.RoomId) (resp WebBannerResponse, err error) {
+	u := fmt.Sprintf("https://api.live.bilibili.com/activity/v1/Common/webBanner?"+
+		"platform=web&position=6&roomid=%d&area_v2_parent_id=0&area_v2_id=0&from=", roomId)
+	resp, err = callGet[WebBannerResponse](b, u)
+	if err == nil {
+		uu, _ := url.Parse(apiUrlPrefix)
+		b.logger.Info("Cookie info: %v", b.http.Jar.Cookies(uu))
+	}
+	return resp, err
 }
diff --git a/bilibili/request.go b/bilibili/request.go
index 11ff995..613530c 100644
--- a/bilibili/request.go
+++ b/bilibili/request.go
@@ -29,8 +29,8 @@ func (b *Bilibili) newGet(url string) (req *http.Request, err error) {
 	return b.newRequest("GET", url, strings.NewReader(""))
 }
 
-// callGet make a GET request and parse response as a JSON document with given model.
-func callGet[T types.BaseResponse[V], V any](b *Bilibili, url string) (resp T, err error) {
+// callGetRaw make a GET request and returns the raw response body.
+func callGetRaw(b *Bilibili, url string) (resp *http.Response, respBody []byte, err error) {
 	req, err := b.newGet(url)
 	if err != nil {
 		b.logger.Error("Cannot create HTTP request instance on API %v: %v", url, err)
@@ -56,6 +56,16 @@ func callGet[T types.BaseResponse[V], V any](b *Bilibili, url string) (resp T, e
 		return
 	}
 
+	return r, data, err
+}
+
+// callGet make a GET request and parse response as a JSON document with given model.
+func callGet[T BaseResponse[V], V any](b Bilibili, url string) (resp T, err error) {
+	r, data, err := callGetRaw(b, url)
+	if err != nil {
+		return
+	}
+
 	err = json.Unmarshal(data, &resp)
 	if err != nil {
 		b.logger.Error("Invalid JSON body of HTTP response on API %v: %v. Text: \"%v\"",
diff --git a/danmaku/client.go b/danmaku/client.go
index a94afbf..d7826d0 100644
--- a/danmaku/client.go
+++ b/danmaku/client.go
@@ -23,6 +23,16 @@ type DanmakuClient struct {
 	wsio wsDatagramIO
 }
 
+func NewClient(ctx context.Context, ws *websocket.Conn) DanmakuClient {
+	return DanmakuClient{
+		ws: ws,
+		wsio: wsDatagramIO{
+			ws:  ws,
+			ctx: ctx,
+		},
+	}
+}
+
 type DanmakuMessageType int
 
 // wsDatagramIO wraps websocket into a datagram I/O,
@@ -48,34 +58,6 @@ func (w *wsDatagramIO) Get() (data []byte, err error) {
 	return
 }
 
-func NewDanmakuClient() DanmakuClient {
-	return DanmakuClient{
-		ws: nil,
-	}
-}
-
-func (d *DanmakuClient) Connect(ctx context.Context, url string) error {
-	// thread unsafe
-
-	// dial
-	if d.ws != nil {
-		return fmt.Errorf("already connected")
-	}
-	ws, _, err := websocket.Dial(ctx, url, nil)
-	if err != nil {
-		return fmt.Errorf("failed to establish WebSocket connection: %w", err)
-	}
-	d.ws = ws
-
-	// init wsio
-	d.wsio = wsDatagramIO{
-		ws:  ws,
-		ctx: ctx,
-	}
-
-	return nil
-}
-
 func (d *DanmakuClient) Disconnect() error {
 	// thread unsafe
 	ws := d.ws
@@ -87,8 +69,8 @@ func (d *DanmakuClient) Disconnect() error {
 	return ws.Close(websocket.StatusInternalError, "disconnected")
 }
 
-func (d *DanmakuClient) Authenticate(roomId types.RoomId, authKey string) error {
-	pkg := dmpkg.NewAuth(dmpkg.ProtoBrotli, roomId, authKey)
+func (d *DanmakuClient) Authenticate(roomId types.RoomId, authKey, buvid3 string) error {
+	pkg := dmpkg.NewAuth(dmpkg.ProtoBrotli, roomId, authKey, buvid3)
 	data, err := pkg.Marshal()
 	if err != nil {
 		return fmt.Errorf("exchange marshal failed: %w", err)
diff --git a/danmaku/dmpkg/auth.go b/danmaku/dmpkg/auth.go
index b6a90d3..81c365e 100644
--- a/danmaku/dmpkg/auth.go
+++ b/danmaku/dmpkg/auth.go
@@ -15,17 +15,19 @@ type authInfo struct {
 	UID      uint64       `json:"uid"`
 	RoomId   types.RoomId `json:"roomid"`
 	ProtoVer int          `json:"protover"`
+	BUVID3   string       `json:"buvid"`
 	Platform string       `json:"platform"`
 	Type     int          `json:"type"`
 	Key      string       `json:"key"`
 }
 
 // NewAuth creates a new authentication exchange.
-func NewAuth(protocol ProtocolVer, roomId types.RoomId, authKey string) (exc DanmakuExchange) {
+func NewAuth(protocol ProtocolVer, roomId types.RoomId, authKey, buvid3 string) (exc DanmakuExchange) {
 	exc, _ = NewPlainExchange(OpConnect, authInfo{
 		UID:      UidGuest,
 		RoomId:   roomId,
 		ProtoVer: int(protocol),
+		BUVID3:   buvid3,
 		Platform: PlatformWeb,
 		Type:     AuthTypeDefault,
 		Key:      authKey,
diff --git a/danmaku/dmpkg/raw.go b/danmaku/dmpkg/raw.go
index 34cd2e6..6de10d7 100644
--- a/danmaku/dmpkg/raw.go
+++ b/danmaku/dmpkg/raw.go
@@ -37,7 +37,7 @@ func NewPlainExchange(operation Operation, body interface{}) (exc DanmakuExchang
 		DanmakuExchangeHeader: DanmakuExchangeHeader{
 			Length:       uint32(length),
 			HeaderLength: HeaderLength,
-			ProtocolVer:  ProtoPlainJson,
+			ProtocolVer:  ProtoMinimal,
 			Operation:    operation,
 			SequenceId:   SequenceId,
 		},
diff --git a/recording/runner.go b/recording/runner.go
index 9ef2cfd..9e5f83c 100644
--- a/recording/runner.go
+++ b/recording/runner.go
@@ -76,15 +76,9 @@ func tryRunTask(t *RunningTask) error {
 
 	t.logger.Info("Getting notification server info...")
 
-	type dmServerInfo struct {
-		AuthKey string
-		DmUrl   string
-	}
-
 	dmInfo, err := AutoRetryWithTask(
-		t, func() (info dmServerInfo, err error) {
-			info.AuthKey, info.DmUrl, err = getDanmakuServer(&t.TaskConfig, bi)
-			return
+		t, func() (*danmakuServerInfo, error) {
+			return getDanmakuServer(&t.TaskConfig, bi)
 		},
 	)
 	if err != nil {
@@ -124,13 +118,17 @@ func tryRunTask(t *RunningTask) error {
 		}
 	loop:
 		for run {
+			t.logger.Info("Start watching, ws url: %v, auth key: %v, buvid3: %v",
+				dmInfo.DanmakuWebsocketUrl, dmInfo.AuthKey, dmInfo.BUVID3)
 			err = watch(
 				ctxWatcher,
 				t.TaskConfig,
-				dmInfo.DmUrl,
+				dmInfo.DanmakuWebsocketUrl,
 				dmInfo.AuthKey,
+				dmInfo.BUVID3,
 				liveStatusChecker,
 				t.logger,
+				&bi,
 			)
 			// the context is cancelled
 			if errors.Is(err, context.Canceled) {
@@ -361,23 +359,45 @@ func record(
 	return errs.NewError(errs.StreamCopy, err)
 }
 
+type danmakuServerInfo struct {
+	DanmakuWebsocketUrl string
+	AuthKey             string
+	BUVID3              string
+}
+
 func getDanmakuServer(
 	task *TaskConfig,
 	bi *bilibili.Bilibili,
-) (string, string, error) {
+) (*danmakuServerInfo, error) {
+	buvid3, err := bi.GetBUVID()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get buvid: %w", err)
+	}
+
+	resp, err := bi.GetLiveBUVID(task.RoomId)
+	if err != nil || resp.Code != 0 {
+		if err != nil {
+			return nil, fmt.Errorf("failed to get LIVE_BUVID with api `webBanner`: %w", err)
+		}
+		return nil, fmt.Errorf("failed to get LIVE_BUVID with api `webBanner`: invalid response: %v", resp)
+	}
 	dmInfo, err := bi.GetDanmakuServerInfo(task.RoomId)
 	if err != nil {
-		return "", "", fmt.Errorf("failed to read stream server info: %w", err)
+		return nil, fmt.Errorf("failed to read stream server info: %w", err)
 	}
 	if len(dmInfo.Data.HostList) == 0 {
-		return "", "", fmt.Errorf("no available stream server")
+		return nil, fmt.Errorf("no available stream server")
 	}
 
 	// get authkey and ws url
 	authKey := dmInfo.Data.Token
 	host := dmInfo.Data.HostList[0]
 	url := fmt.Sprintf("wss://%s:%d/sub", host.Host, host.WssPort)
-	return authKey, url, nil
+	return &danmakuServerInfo{
+		DanmakuWebsocketUrl: url,
+		AuthKey:             authKey,
+		BUVID3:              buvid3,
+	}, nil
 }
 
 func GenerateFileName(roomName string, t time.Time) string {
diff --git a/recording/watcher.go b/recording/watcher.go
index 515def1..ff9bde1 100644
--- a/recording/watcher.go
+++ b/recording/watcher.go
@@ -4,7 +4,8 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
-	errs "github.com/keuin/slbr/bilibili/errors"
+	errs "github.com/keuin/slbr/bilibili"
+	"github.com/keuin/slbr/bilibili/errors"
 	"github.com/keuin/slbr/danmaku"
 	"github.com/keuin/slbr/danmaku/dmmsg"
 	"github.com/keuin/slbr/danmaku/dmpkg"
@@ -41,19 +42,19 @@ func watch(
 	ctx context.Context,
 	t TaskConfig,
 	url string,
-	authKey string,
+	authKey, buvid3 string,
 	liveStatusChecker func() (bool, error),
 	logger logging.Logger,
+	bi *bilibili.Bilibili,
 ) error {
 	var err error
 
-	dm := danmaku.NewDanmakuClient()
-
-	// connect to danmaku server for live online/offline notifications
-	err = dm.Connect(ctx, url)
+	ws, err := bi.DialWebSocket(ctx, url)
 	if err != nil {
 		return errs.NewError(errs.DanmakuServerConnection, err)
 	}
+
+	dm := danmaku.NewClient(ctx, ws)
 	defer func() {
 		// this operation may be time-consuming, so run in another goroutine
 		go func() {
@@ -63,7 +64,7 @@ func watch(
 
 	// the danmaku server requires an auth token and room id when connected
 	logger.Info("ws connected. Authenticating...")
-	err = dm.Authenticate(t.RoomId, authKey)
+	err = dm.Authenticate(t.RoomId, authKey, buvid3)
 	if err != nil {
 		return errs.NewError(errs.InvalidAuthProtocol, err)
 	}