Merge branch 'main' of https://github.com/Preloading/MastodonTwitterAPI

Author: Preloading

bluesky/blueskyapi.go

@@ -80,7 +80,7 @@ type PostReason struct {
 type Embed struct {
 	Type   string  `json:"$type"`
 	Images []Image `json:"images,omitempty"`
-	//Video  `json:",omitempty"`
+	Video  `json:",omitempty"`
 }
 
 type Image struct {
@@ -92,7 +92,7 @@ type Image struct {
 type Video struct {
 	Alt         string      `json:"alt"`
 	AspectRatio AspectRatio `json:"aspectRatio"`
-	Video       Blob        `json:"video"`
+	Video       *Blob       `json:"video"`
 }
 
 type AspectRatio struct {

bridge/bridge.go

@@ -194,6 +194,7 @@ type URLXMLFormat struct {
 	End         int      `xml:"end,attr"`
 	URL         string   `xml:"url"`
 	ExpandedURL string   `xml:"expanded_url"`
+	DisplayURL  string   `xml:"display_url"`
 }
 
 type Hashtag struct {

config.sample.yaml

@@ -18,10 +18,14 @@ CDN_URL: 'http://127.0.0.1:3000'
 # {fullblob} is the full bluesky blob
 # {user_did} is the image owner's DID 
 
-IMG_DISPLAY_TEXT: 'pic.twitter.com/{shortblob}'
+IMG_DISPLAY_TEXT: 'pic.twitter.com/{shortcode}'
+
+# Now this link is where it actually will go to (sometimes), although this is not displayed to the user*
+# IMG_URL_TEXT: 'http://127.0.0.1:3000/cdn/img/bsky/{user_did}/{fullblob}.jpg' # if you don't want to do any redirects, it will eventually arrive here.
+IMG_URL_TEXT: 'http://127.0.0.1:3000/img/{shortcode}' # if you want to do a redirect, and have it be short, it should arrive here.
 
 # Same as the above, but for videos
-VID_DISPLAY_TEXT: 'pic.twitter.com/{shortblob}'
+VID_DISPLAY_TEXT: 'pic.twitter.com/{shortcode}'
 
 # SERVER_PORT is the port the server will listen on.
 SERVER_PORT: 3000

config/config.go

@@ -2,6 +2,8 @@
 // This is not the config file. The config file can be found in the root directory as config.yaml
 // (you may need to create it, copy config.sample.yml )
 
+// This file is where the config file is parsed & loaded. Modifying this is highly discouraged (unless your a developer).
+
 package config
 
 import (
@@ -27,7 +29,9 @@ type Config struct {
     UseXForwardedFor bool `mapstructure:"USE_X_FORWARDED_FOR"`
 
     ImgDisplayText string `mapstructure:"IMG_DISPLAY_TEXT"`
+    ImgURLText string `mapstructure:"IMG_URL_TEXT"`
     VidDisplayText string `mapstructure:"VID_DISPLAY_TEXT"`
+    VidURLText string `mapstructure:"VID_URL_TEXT"`
 }
 
 // Loads our config files.

db_controller/db_controller.go

@@ -87,8 +87,17 @@ type AnalyticData struct {
 	Timestamp            time.Time `gorm:"type:timestamp"`
 }
 
-var db *gorm.DB
-var cfg config.Config
+// ShortLink represents the schema for the short_links table
+type ShortLink struct {
+	ShortCode   string `gorm:"type:string;primaryKey;not null"`
+	OriginalURL string `gorm:"type:string;uniqueIndex;not null"` // Add unique index
+}
+
+var (
+	db          *gorm.DB
+	cfg         config.Config
+	base62Chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+)
 
 func InitDB(_cfg config.Config) {
 	cfg = _cfg
@@ -122,6 +131,7 @@ func InitDB(_cfg config.Config) {
 	db.AutoMigrate(&MessageContext{})
 	db.AutoMigrate(&TwitterIDs{})
 	db.AutoMigrate(&AnalyticData{})
+	db.AutoMigrate(&ShortLink{}) // Add this line
 }
 
 // StoreToken stores an encrypted access token and refresh token in the database.
@@ -412,3 +422,65 @@ func StoreAnalyticData(data AnalyticData) {
 		fmt.Println("Failed to store analytic data:", result.Error)
 	}
 }
+
+// StoreShortLink stores a short link in the database with optimized collision handling
+func StoreShortLink(shortCode string, originalURL string) error {
+	shortLink := ShortLink{
+		ShortCode:   shortCode,
+		OriginalURL: originalURL,
+	}
+
+	// Try direct insert first with DO NOTHING on conflict
+	result := db.Clauses(clause.OnConflict{
+		DoNothing: true,
+	}).Create(&shortLink)
+
+	if result.Error != nil {
+		return result.Error
+	}
+
+	// If no rows were affected, either there was a shortCode collision or the URL exists
+	if result.RowsAffected == 0 {
+		// Check if URL already exists
+		var existing ShortLink
+		if err := db.Where("original_url = ?", originalURL).First(&existing).Error; err == nil {
+			return nil // URL exists, reuse existing shortcode
+		}
+
+		// Handle collision with simple numeric suffixes
+		for i := 0; i < 10; i++ {
+			newCode := fmt.Sprintf("%s%d", shortCode[:7], i)
+			shortLink.ShortCode = newCode
+			result = db.Clauses(clause.OnConflict{
+				DoNothing: true,
+			}).Create(&shortLink)
+
+			if result.Error == nil && result.RowsAffected > 0 {
+				return nil
+			}
+		}
+
+		// If simple attempts failed, try timestamp-based suffix
+		newCode := fmt.Sprintf("%s%x", shortCode[:7], time.Now().UnixNano()%16)
+		shortLink.ShortCode = newCode
+		result = db.Clauses(clause.OnConflict{
+			DoNothing: true,
+		}).Create(&shortLink)
+
+		if result.Error == nil && result.RowsAffected > 0 {
+			return nil
+		}
+	}
+
+	return nil
+}
+
+// GetOriginalURL retrieves the original URL from the database using the short code
+func GetOriginalURL(shortCode string) (string, error) {
+	var shortLink ShortLink
+	if err := db.Where("short_code = ?", shortCode).First(&shortLink).Error; err != nil {
+		return "", err
+	}
+
+	return shortLink.OriginalURL, nil
+}

twitterv1/cdnproxy.go

@@ -192,3 +192,20 @@ func UserProfileImage(c *fiber.Ctx) error {
 	//c.Redirect("https://cdn.bsky.app/img/" + screen_name + ":profile_bigger")
 	return c.Redirect(userinfo.ProfileImageURL)
 }
+
+// This is here because it doesn't just want a direct link to the m3u8 file.
+// So we make an extremely basic site that just includes the video, and maybe the alt text if i care enough
+func CDNVideoProxy(c *fiber.Ctx) error {
+	video_url := "https://video.bsky.app/watch/" + c.Params("did") + "/" + c.Params("link") + "/720p/video.m3u8" // 720p on an iphone 2g oh god
+	thumbnail_url := "https://video.cdn.bsky.app/hls/" + c.Params("did") + "/" + c.Params("link") + "/thumbnail.jpg"
+
+	c.Context().SetContentType("text/html")
+
+	// Is the below minified? yup!
+	// tbh, you could pretty easily just run it thru a prettier and it would make sense, but i'll explain it here:
+	// It's a basic html page that includes the hls.js library, a video element, and a script that checks if the browser supports hls, and if it doesn't, it uses hls.js to play the video
+	// why u hef to be mad golang warning thingy?
+	return c.SendString(fmt.Sprintf(`
+	<meta content="width=device-width,initial-scale=1"name=viewport><title>Bluesky Video</title><style>*{margin:0;padding:0;width:100%%;height:100%%}</style><body><script src=https://cdn.jsdelivr.net/npm/hls.js@1></script><video autoplay="autoplay" controls id=v poster=%s src=%s></video><script>var v=document.getElementById("v");if(v.canPlayType("application/vnd.apple.mpegurl"));else if(Hls.isSupported()){var h=new Hls;h.loadSource(v.src),h.attachMedia(v)}</script>
+	`, thumbnail_url, video_url))
+}

twitterv1/imagelink.go

@@ -0,0 +1,97 @@
+package twitterv1
+
+import (
+	"crypto/sha256"
+	"math/big"
+	"sync"
+
+	"github.com/Preloading/TwitterAPIBridge/db_controller"
+	"github.com/gofiber/fiber/v2"
+)
+
+// Simple cache for frequently accessed URLs
+type urlCache struct {
+	cache map[string]string
+	mutex sync.RWMutex
+}
+
+var (
+	base62Chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	cache       = &urlCache{
+		cache: make(map[string]string),
+	}
+)
+
+// Get returns the cached shortcode for a URL
+func (c *urlCache) Get(url string) (string, bool) {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+	code, ok := c.cache[url]
+	return code, ok
+}
+
+// Set caches a URL and its shortcode
+func (c *urlCache) Set(url, code string) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.cache[url] = code
+}
+
+func toBase62(num *big.Int) string {
+	base := big.NewInt(62)
+	zero := big.NewInt(0)
+	result := make([]byte, 7) // Changed to 7 to leave room for collision handling
+	idx := 6
+
+	temp := new(big.Int).Set(num)
+	for idx >= 0 {
+		mod := new(big.Int)
+		temp.DivMod(temp, base, mod)
+		result[idx] = base62Chars[mod.Int64()]
+		idx--
+		if temp.Cmp(zero) == 0 {
+			break
+		}
+	}
+	// Pad with '0' if necessary
+	for i := idx; i >= 0; i-- {
+		result[i] = base62Chars[0]
+	}
+	return string(result)
+}
+
+func CreateShortLink(originalPath string) (string, error) {
+	// Check cache first
+	if code, ok := cache.Get(originalPath); ok {
+		return code, nil
+	}
+
+	// Generate SHA-256 hash of the URL
+	hash := sha256.New()
+	hash.Write([]byte(originalPath))
+	hashBytes := hash.Sum(nil)
+
+	// Convert first 8 bytes of hash to a big.Int
+	num := new(big.Int).SetBytes(hashBytes[:8])
+	shortCode := toBase62(num)
+
+	err := db_controller.StoreShortLink(shortCode, originalPath)
+	if err != nil {
+		return "", err
+	}
+
+	// Cache the result
+	cache.Set(originalPath, shortCode)
+	return shortCode, nil
+}
+
+func RedirectToLink(c *fiber.Ctx) error {
+
+	shortCode := c.Params("ref")
+	originalURL, err := db_controller.GetOriginalURL(shortCode)
+	if err != nil {
+		return c.Status(fiber.StatusNotFound).SendString("Could not find image!")
+	}
+
+	return c.Redirect(configData.CdnURL+originalURL, fiber.StatusMovedPermanently)
+}