add cache
Marco Andronaco andronacomarco@gmail.com
Tue, 19 Dec 2023 01:32:32 +0100
7 files changed,
194 insertions(+),
14 deletions(-)
M
fixyoutube.go
→
fixyoutube.go
@@ -18,6 +18,7 @@
var templatesDirectory = "templates/" var indexTemplate = template.Must(template.ParseFiles(templatesDirectory + "index.html")) var videoTemplate = template.Must(template.ParseFiles(templatesDirectory + "video.html")) +var blacklist = []string{"favicon.ico", "robots.txt"} func indexHandler(w http.ResponseWriter, r *http.Request) { buf := &bytes.Buffer{}@@ -30,7 +31,13 @@
buf.WriteTo(w) } -func watchHandler(newsapi *invidious.Client) http.HandlerFunc { +func clearHandler(w http.ResponseWriter, r *http.Request) { + //TODO: check with some secret API key before clearing cache. + invidious.ClearDB() + //TODO: return a message +} + +func watchHandler(invidiousClient *invidious.Client) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { u, err := url.Parse(r.URL.String()) if err != nil {@@ -41,7 +48,7 @@
params := u.Query() videoId := params.Get("v") - video, err := newsapi.FetchEverything(videoId) + video, err := invidiousClient.GetVideo(videoId) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return@@ -58,9 +65,7 @@ buf.WriteTo(w)
} } -var blacklist = []string{"favicon.ico", "robots.txt"} - -func shortHandler(newsapi *invidious.Client) http.HandlerFunc { +func shortHandler(invidiousClient *invidious.Client) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { videoId := mux.Vars(r)["videoId"]@@ -69,7 +74,7 @@ http.Error(w, "Not a valid ID.", http.StatusBadRequest)
return } - video, err := newsapi.FetchEverything(videoId) + video, err := invidiousClient.GetVideo(videoId) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return@@ -98,13 +103,14 @@ port = "3000"
} myClient := &http.Client{Timeout: 10 * time.Second} - videoapi := invidious.NewClient(myClient, "y.birabittoh.duckdns.org") + videoapi := invidious.NewClient(myClient) r := mux.NewRouter() r.HandleFunc("/", indexHandler) + r.HandleFunc("/clear", clearHandler) r.HandleFunc("/watch", watchHandler(videoapi)) r.HandleFunc("/{videoId}", shortHandler(videoapi)) - //r.HandleFunc("/proxy/{videoId}", proxyHandler) + //TODO: r.HandleFunc("/proxy/{videoId}", proxyHandler) /* // native go implementation (useless until february 2024)
M
go.sum
→
go.sum
@@ -2,3 +2,5 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
A
invidious/cache.go
@@ -0,0 +1,98 @@
+package invidious + +import ( + "database/sql" + "log" + + _ "github.com/mattn/go-sqlite3" +) + +var dbConnectionString = "file:cache.sqlite?cache=shared&mode=" +var createQuery = ` +CREATE TABLE IF NOT EXISTS videos ( + videoId TEXT PRIMARY KEY, + title TEXT NOT NULL, + description TEXT NOT NULL, + uploader TEXT NOT NULL, + duration int NOT NULL, + height TEXT NOT NULL, + width TEXT NOT NULL, + url TEXT, + timestamp DATETIME DEFAULT (datetime('now')) +);` +var getVideoQuery = "SELECT * FROM videos WHERE videoId = (?);" +var cacheVideoQuery = "INSERT OR REPLACE INTO videos (videoId, title, description, uploader, duration, height, width, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);" +var clearQuery = "DELETE FROM videos;" + +func getDb(mode string) *sql.DB { + db, err := sql.Open("sqlite3", dbConnectionString+mode) + if err != nil { + log.Fatal("Error opening database") + } + db.SetMaxOpenConns(1) + return db +} + +func InitDB() { + db := getDb("rwc") + defer db.Close() + + _, err := db.Exec(createQuery) + if err != nil { + log.Printf("%q: %s\n", err, createQuery) + return + } +} + +func CacheVideoDB(v Video) { + db := getDb("rw") + defer db.Close() + + stmt, err := db.Prepare(cacheVideoQuery) + if err != nil { + log.Fatal(err) + } + defer stmt.Close() + + _, err = stmt.Exec(v.VideoId, v.Title, v.Description, v.Uploader, v.Duration, v.Height, v.Width, v.Url) + if err != nil { + log.Printf("%q: %s\n", err, cacheVideoQuery) + return + } +} + +func GetVideoDB(videoId string) (*Video, error) { + db := getDb("ro") + defer db.Close() + + stmt, err := db.Prepare(getVideoQuery) + if err != nil { + log.Fatal(err) + } + defer stmt.Close() + + t := &Video{} + err = stmt.QueryRow(videoId).Scan(&t.VideoId, &t.Title, &t.Description, &t.Uploader, &t.Duration, &t.Height, &t.Width, &t.Url, &t.Timestamp) + if err != nil { + //log.Printf("%q: %s\n", err, getVideoQuery) + return &Video{}, err + } + return t, nil +} + +func ClearDB() { + db := getDb("rw") + defer db.Close() + + stmt, err := db.Prepare(clearQuery) + if err != nil { + log.Fatal(err) + } + defer stmt.Close() + + _, err = stmt.Exec() + if err != nil { + log.Printf("%q: %s\n", err, cacheVideoQuery) + return + } +}
M
invidious/invidious.go
→
invidious/invidious.go
@@ -4,12 +4,19 @@ import (
"encoding/json" "fmt" "io" + "log" "net/http" "net/url" + "os" "strconv" "strings" + "time" ) +var instancesEndpoint = "https://api.invidious.io/instances.json?sort_by=api,type" +var videosEndpoint = "https://%s/api/v1/videos/%s?fields=videoId,title,description,author,lengthSeconds,size,formatStreams" +var timeToLive, _ = time.ParseDuration("6h") + type Client struct { http *http.Client Instance string@@ -31,6 +38,7 @@ FormatStreams []Format `json:"formatStreams"`
Url string Height int Width int + Timestamp time.Time } func filter[T any](ss []T, test func(T) bool) (ret []T) {@@ -50,13 +58,19 @@ }
return res } -func (c *Client) FetchEverything(videoId string) (*Video, error) { - endpoint := fmt.Sprintf("https://%s/api/v1/videos/%s?fields=videoId,title,description,author,lengthSeconds,size,formatStreams", c.Instance, url.QueryEscape(videoId)) +func (c *Client) FetchVideo(videoId string) (*Video, error) { + if c.Instance == "" { + err := c.NewInstance() + if err != nil { + log.Fatal(err, "Could not get a new instance.") + os.Exit(1) + } + } + endpoint := fmt.Sprintf(videosEndpoint, c.Instance, url.QueryEscape(videoId)) resp, err := c.http.Get(endpoint) if err != nil { return nil, err } - defer resp.Body.Close() body, err := io.ReadAll(resp.Body)@@ -86,6 +100,63 @@
return res, err } -func NewClient(httpClient *http.Client, instance string) *Client { - return &Client{httpClient, instance} +func (c *Client) GetVideo(videoId string) (*Video, error) { + log.Println("Video", videoId, "was requested.") + + video, err := GetVideoDB(videoId) + if err == nil { + now := time.Now() + delta := now.Sub(video.Timestamp) + if delta < timeToLive { + log.Println("Found a valid cache entry from", delta, "ago.") + return video, nil + } + } + + video, err = c.FetchVideo(videoId) + if err != nil { + log.Println(err) + err = c.NewInstance() + if err != nil { + log.Fatal("Could not get a new instance: ", err) + time.Sleep(10) + } + return c.GetVideo(videoId) + } + log.Println("Retrieved by API.") + + CacheVideoDB(*video) + return video, nil +} + +func (c *Client) NewInstance() error { + resp, err := c.http.Get(instancesEndpoint) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf(string(body)) + } + + var jsonArray [][]interface{} + err = json.Unmarshal(body, &jsonArray) + if err != nil { + return err + } + + c.Instance = jsonArray[0][0].(string) + log.Println("Using new instance:", c.Instance) + return nil +} + +func NewClient(httpClient *http.Client) *Client { + InitDB() + return &Client{httpClient, ""} }
M
templates/video.html
→
templates/video.html
@@ -15,7 +15,7 @@ <meta property="twitter:card" content="player" />
<meta property="twitter:site" content="{{ .Uploader }}" /> <meta property="twitter:creator" content="{{ .Uploader }}" /> <meta property="twitter:title" content="{{ .Title }}" /> - <!--<meta http-equiv="refresh" content="0;url=https://www.youtube.com/watch?v={{ .VideoId }}" />--> + <meta http-equiv="refresh" content="0;url=https://www.youtube.com/watch?v={{ .VideoId }}" /> <meta property="og:url" content="https://www.youtube.com/watch?v={{ .VideoId }}" /> <meta property="og:title" content="{{ .Title }}" /> <meta property="og:description" content="{{ .Description }}" />