mediaServer.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342
  1. package main
  2. import (
  3. "crypto/md5"
  4. "encoding/hex"
  5. "errors"
  6. "io"
  7. "net/http"
  8. "net/url"
  9. "os"
  10. "path/filepath"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "imuslab.com/arozos/mod/compatibility"
  15. "imuslab.com/arozos/mod/filesystem"
  16. fs "imuslab.com/arozos/mod/filesystem"
  17. "imuslab.com/arozos/mod/network/gzipmiddleware"
  18. "imuslab.com/arozos/mod/utils"
  19. )
  20. /*
  21. Media Server
  22. This function serve large file objects like video and audio file via asynchronize go routine :)
  23. Example usage:
  24. /media/?file=user:/Desktop/test/02.Orchestra- エミール (Addendum version).mp3
  25. /media/?file=user:/Desktop/test/02.Orchestra- エミール (Addendum version).mp3&download=true
  26. This will serve / download the file located at files/users/{username}/Desktop/test/02.Orchestra- エミール (Addendum version).mp3
  27. PLEASE ALWAYS USE URLENCODE IN THE LINK PASSED INTO THE /media ENDPOINT
  28. */
  29. func mediaServer_init() {
  30. if *enable_gzip {
  31. http.HandleFunc("/media/", gzipmiddleware.CompressFunc(serverMedia))
  32. http.HandleFunc("/media/getMime/", gzipmiddleware.CompressFunc(serveMediaMime))
  33. } else {
  34. http.HandleFunc("/media/", serverMedia)
  35. http.HandleFunc("/media/getMime/", serveMediaMime)
  36. }
  37. //Download API always bypass gzip no matter if gzip mode is enabled
  38. http.HandleFunc("/media/download/", serverMedia)
  39. }
  40. //This function validate the incoming media request and return fsh, vpath, rpath and err if any
  41. func media_server_validateSourceFile(w http.ResponseWriter, r *http.Request) (*filesystem.FileSystemHandler, string, string, error) {
  42. username, err := authAgent.GetUserName(w, r)
  43. if err != nil {
  44. return nil, "", "", errors.New("User not logged in")
  45. }
  46. userinfo, _ := userHandler.GetUserInfoFromUsername(username)
  47. //Validate url valid
  48. if strings.Count(r.URL.String(), "?") > 1 {
  49. return nil, "", "", errors.New("Invalid paramters. Multiple ? found")
  50. }
  51. targetfile, _ := utils.Mv(r, "file", false)
  52. targetfile, err = url.QueryUnescape(targetfile)
  53. if err != nil {
  54. return nil, "", "", err
  55. }
  56. if targetfile == "" {
  57. return nil, "", "", errors.New("Missing paramter 'file'")
  58. }
  59. //Translate the virtual directory to realpath
  60. fsh, subpath, err := GetFSHandlerSubpathFromVpath(targetfile)
  61. if err != nil {
  62. return nil, "", "", errors.New("Unable to load from target file system")
  63. }
  64. fshAbs := fsh.FileSystemAbstraction
  65. realFilepath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  66. if fshAbs.FileExists(realFilepath) && fshAbs.IsDir(realFilepath) {
  67. return nil, "", "", errors.New("Given path is not a file")
  68. }
  69. if err != nil {
  70. return nil, "", "", errors.New("Unable to translate the given filepath")
  71. }
  72. if !fshAbs.FileExists(realFilepath) {
  73. //Sometime if url is not URL encoded, this error might be shown as well
  74. //Try to use manual segmentation
  75. originalURL := r.URL.String()
  76. //Must be pre-processed with system special URI Decode function to handle edge cases
  77. originalURL = fs.DecodeURI(originalURL)
  78. if strings.Contains(originalURL, "&download=true") {
  79. originalURL = strings.ReplaceAll(originalURL, "&download=true", "")
  80. } else if strings.Contains(originalURL, "download=true") {
  81. originalURL = strings.ReplaceAll(originalURL, "download=true", "")
  82. }
  83. if strings.Contains(originalURL, "&file=") {
  84. originalURL = strings.ReplaceAll(originalURL, "&file=", "file=")
  85. }
  86. urlInfo := strings.Split(originalURL, "file=")
  87. possibleVirtualFilePath := urlInfo[len(urlInfo)-1]
  88. possibleRealpath, err := fshAbs.VirtualPathToRealPath(possibleVirtualFilePath, userinfo.Username)
  89. if err != nil {
  90. systemWideLogger.PrintAndLog("Media Server", "Error when trying to serve file in compatibility mode", err)
  91. return nil, "", "", errors.New("Error when trying to serve file in compatibility mode")
  92. }
  93. if fshAbs.FileExists(possibleRealpath) {
  94. realFilepath = possibleRealpath
  95. systemWideLogger.PrintAndLog("Media Server", "Serving file "+filepath.Base(possibleRealpath)+" in compatibility mode. Do not to use '&' or '+' sign in filename! ", nil)
  96. return fsh, targetfile, realFilepath, nil
  97. } else {
  98. return nil, "", "", errors.New("File not exists")
  99. }
  100. }
  101. return fsh, targetfile, realFilepath, nil
  102. }
  103. func serveMediaMime(w http.ResponseWriter, r *http.Request) {
  104. targetFsh, _, realFilepath, err := media_server_validateSourceFile(w, r)
  105. if err != nil {
  106. utils.SendErrorResponse(w, err.Error())
  107. return
  108. }
  109. targetFshAbs := targetFsh.FileSystemAbstraction
  110. if targetFsh.RequireBuffer {
  111. //File is not on local. Guess its mime by extension
  112. utils.SendTextResponse(w, "application/"+filepath.Ext(realFilepath)[1:])
  113. return
  114. }
  115. mime := "text/directory"
  116. if !targetFshAbs.IsDir(realFilepath) {
  117. m, _, err := fs.GetMime(realFilepath)
  118. if err != nil {
  119. mime = ""
  120. }
  121. mime = m
  122. }
  123. utils.SendTextResponse(w, mime)
  124. }
  125. func serverMedia(w http.ResponseWriter, r *http.Request) {
  126. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  127. //Serve normal media files
  128. targetFsh, vpath, realFilepath, err := media_server_validateSourceFile(w, r)
  129. if err != nil {
  130. utils.SendErrorResponse(w, err.Error())
  131. return
  132. }
  133. targetFshAbs := targetFsh.FileSystemAbstraction
  134. //Check if downloadMode
  135. downloadMode := false
  136. dw, _ := utils.Mv(r, "download", false)
  137. if dw == "true" {
  138. downloadMode = true
  139. }
  140. //New download implementations, allow /download to be used instead of &download=true
  141. if strings.Contains(r.RequestURI, "media/download/?file=") {
  142. downloadMode = true
  143. }
  144. //Serve the file
  145. if downloadMode {
  146. escapedRealFilepath, err := url.PathUnescape(realFilepath)
  147. if err != nil {
  148. utils.SendErrorResponse(w, err.Error())
  149. return
  150. }
  151. filename := filepath.Base(escapedRealFilepath)
  152. /*
  153. //12 Jul 2022 Update: Deprecated the browser detection logic
  154. userAgent := r.Header.Get("User-Agent")
  155. if strings.Contains(userAgent, "Safari/")) {
  156. //This is Safari. Use speial header
  157. w.Header().Set("Content-Disposition", "attachment; filename="+filepath.Base(realFilepath))
  158. } else {
  159. //Fixing the header issue on Golang url encode lib problems
  160. w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+filename)
  161. }
  162. */
  163. w.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"")
  164. w.Header().Set("Content-Type", compatibility.BrowserCompatibilityOverrideContentType(r.UserAgent(), filename, r.Header.Get("Content-Type")))
  165. if targetFsh.RequireBuffer || !filesystem.FileExists(realFilepath) {
  166. //Stream it directly from remote
  167. w.Header().Set("Content-Length", strconv.Itoa(int(targetFshAbs.GetFileSize(realFilepath))))
  168. remoteStream, err := targetFshAbs.ReadStream(realFilepath)
  169. if err != nil {
  170. utils.SendErrorResponse(w, err.Error())
  171. return
  172. }
  173. io.Copy(w, remoteStream)
  174. remoteStream.Close()
  175. } else {
  176. http.ServeFile(w, r, escapedRealFilepath)
  177. }
  178. } else {
  179. if targetFsh.RequireBuffer {
  180. w.Header().Set("Content-Length", strconv.Itoa(int(targetFshAbs.GetFileSize(realFilepath))))
  181. //Check buffer exists
  182. ps, _ := targetFsh.GetUniquePathHash(vpath, userinfo.Username)
  183. buffpool := filepath.Join(*tmp_directory, "fsbuffpool")
  184. buffFile := filepath.Join(buffpool, ps)
  185. if fs.FileExists(buffFile) {
  186. //Stream the buff file if hash matches
  187. remoteFileHash, err := getHashFromRemoteFile(targetFsh.FileSystemAbstraction, realFilepath)
  188. if err == nil {
  189. localFileHash, err := os.ReadFile(buffFile + ".hash")
  190. if err == nil {
  191. if string(localFileHash) == remoteFileHash {
  192. //Hash matches. Serve local buffered file
  193. http.ServeFile(w, r, buffFile)
  194. return
  195. }
  196. }
  197. }
  198. }
  199. remoteStream, err := targetFshAbs.ReadStream(realFilepath)
  200. if err != nil {
  201. utils.SendErrorResponse(w, err.Error())
  202. return
  203. }
  204. defer remoteStream.Close()
  205. io.Copy(w, remoteStream)
  206. if *enable_buffering {
  207. os.MkdirAll(buffpool, 0775)
  208. go func() {
  209. BufferRemoteFileToTmp(buffFile, targetFsh, realFilepath)
  210. }()
  211. }
  212. } else if !filesystem.FileExists(realFilepath) {
  213. //Streaming from remote file system that support fseek
  214. f, err := targetFsh.FileSystemAbstraction.Open(realFilepath)
  215. if err != nil {
  216. w.WriteHeader(http.StatusInternalServerError)
  217. w.Write([]byte("500 - Internal Server Error"))
  218. return
  219. }
  220. fstat, _ := f.Stat()
  221. defer f.Close()
  222. http.ServeContent(w, r, filepath.Base(realFilepath), fstat.ModTime(), f)
  223. } else {
  224. http.ServeFile(w, r, realFilepath)
  225. }
  226. }
  227. }
  228. func BufferRemoteFileToTmp(buffFile string, fsh *filesystem.FileSystemHandler, rpath string) error {
  229. if fs.FileExists(buffFile + ".download") {
  230. return errors.New("another buffer process running")
  231. }
  232. //Generate a stat file for the buffer
  233. hash, err := getHashFromRemoteFile(fsh.FileSystemAbstraction, rpath)
  234. if err != nil {
  235. //Do not buffer
  236. return err
  237. }
  238. os.WriteFile(buffFile+".hash", []byte(hash), 0775)
  239. //Buffer the file from remote to local
  240. f, err := fsh.FileSystemAbstraction.ReadStream(rpath)
  241. if err != nil {
  242. os.Remove(buffFile + ".hash")
  243. return err
  244. }
  245. defer f.Close()
  246. dest, err := os.OpenFile(buffFile+".download", os.O_CREATE|os.O_WRONLY, 0775)
  247. if err != nil {
  248. os.Remove(buffFile + ".hash")
  249. return err
  250. }
  251. defer dest.Close()
  252. io.Copy(dest, f)
  253. f.Close()
  254. dest.Close()
  255. os.Rename(buffFile+".download", buffFile)
  256. //Clean the oldest buffpool item if size too large
  257. dirsize, _ := fs.GetDirctorySize(filepath.Dir(buffFile), false)
  258. oldestModtime := time.Now().Unix()
  259. oldestFile := ""
  260. for int(dirsize) > *bufferPoolSize<<20 {
  261. //fmt.Println("CLEARNING BUFF", dirsize)
  262. files, _ := filepath.Glob(filepath.ToSlash(filepath.Dir(buffFile)) + "/*")
  263. for _, file := range files {
  264. if filepath.Ext(file) == ".hash" {
  265. continue
  266. }
  267. thisModTime, _ := fs.GetModTime(file)
  268. if thisModTime < oldestModtime {
  269. oldestModtime = thisModTime
  270. oldestFile = file
  271. }
  272. }
  273. os.Remove(oldestFile)
  274. os.Remove(oldestFile + ".hash")
  275. dirsize, _ = fs.GetDirctorySize(filepath.Dir(buffFile), false)
  276. oldestModtime = time.Now().Unix()
  277. }
  278. return nil
  279. }
  280. func getHashFromRemoteFile(fshAbs filesystem.FileSystemAbstraction, rpath string) (string, error) {
  281. filestat, err := fshAbs.Stat(rpath)
  282. if err != nil {
  283. //Always pull from remote
  284. return "", err
  285. }
  286. if filestat.Size() >= int64(*bufferPoolSize<<20) {
  287. return "", errors.New("Unable to buffer: file larger than buffpool size")
  288. }
  289. if filestat.Size() >= int64(*bufferFileMaxSize<<20) {
  290. return "", errors.New("File larger than max buffer file size")
  291. }
  292. statHash := strconv.Itoa(int(filestat.ModTime().Unix() + filestat.Size()))
  293. hash := md5.Sum([]byte(statHash))
  294. return hex.EncodeToString(hash[:]), nil
  295. }