|
|
@@ -1,14 +1,18 @@
|
|
|
package main
|
|
|
|
|
|
import (
|
|
|
+ "crypto/md5"
|
|
|
+ "encoding/hex"
|
|
|
"errors"
|
|
|
"io"
|
|
|
"log"
|
|
|
"net/http"
|
|
|
"net/url"
|
|
|
+ "os"
|
|
|
"path/filepath"
|
|
|
"strconv"
|
|
|
"strings"
|
|
|
+ "time"
|
|
|
|
|
|
"imuslab.com/arozos/mod/common"
|
|
|
"imuslab.com/arozos/mod/filesystem"
|
|
|
@@ -42,41 +46,41 @@ func mediaServer_init() {
|
|
|
http.HandleFunc("/media/download/", serverMedia)
|
|
|
}
|
|
|
|
|
|
-//This function validate the incoming media request and return the real path for the targed file
|
|
|
-func media_server_validateSourceFile(w http.ResponseWriter, r *http.Request) (*filesystem.FileSystemHandler, string, error) {
|
|
|
+//This function validate the incoming media request and return fsh, vpath, rpath and err if any
|
|
|
+func media_server_validateSourceFile(w http.ResponseWriter, r *http.Request) (*filesystem.FileSystemHandler, string, string, error) {
|
|
|
username, err := authAgent.GetUserName(w, r)
|
|
|
if err != nil {
|
|
|
- return nil, "", errors.New("User not logged in")
|
|
|
+ return nil, "", "", errors.New("User not logged in")
|
|
|
}
|
|
|
|
|
|
userinfo, _ := userHandler.GetUserInfoFromUsername(username)
|
|
|
|
|
|
//Validate url valid
|
|
|
if strings.Count(r.URL.String(), "?") > 1 {
|
|
|
- return nil, "", errors.New("Invalid paramters. Multiple ? found")
|
|
|
+ return nil, "", "", errors.New("Invalid paramters. Multiple ? found")
|
|
|
}
|
|
|
|
|
|
targetfile, _ := common.Mv(r, "file", false)
|
|
|
targetfile, err = url.QueryUnescape(targetfile)
|
|
|
if err != nil {
|
|
|
- return nil, "", err
|
|
|
+ return nil, "", "", err
|
|
|
}
|
|
|
if targetfile == "" {
|
|
|
- return nil, "", errors.New("Missing paramter 'file'")
|
|
|
+ return nil, "", "", errors.New("Missing paramter 'file'")
|
|
|
}
|
|
|
|
|
|
//Translate the virtual directory to realpath
|
|
|
fsh, subpath, err := GetFSHandlerSubpathFromVpath(targetfile)
|
|
|
if err != nil {
|
|
|
- return nil, "", errors.New("Unable to load from target file system")
|
|
|
+ return nil, "", "", errors.New("Unable to load from target file system")
|
|
|
}
|
|
|
fshAbs := fsh.FileSystemAbstraction
|
|
|
realFilepath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
|
|
|
if fshAbs.FileExists(realFilepath) && fshAbs.IsDir(realFilepath) {
|
|
|
- return nil, "", errors.New("Given path is not a file")
|
|
|
+ return nil, "", "", errors.New("Given path is not a file")
|
|
|
}
|
|
|
if err != nil {
|
|
|
- return nil, "", errors.New("Unable to translate the given filepath")
|
|
|
+ return nil, "", "", errors.New("Unable to translate the given filepath")
|
|
|
}
|
|
|
|
|
|
if !fshAbs.FileExists(realFilepath) {
|
|
|
@@ -100,22 +104,22 @@ func media_server_validateSourceFile(w http.ResponseWriter, r *http.Request) (*f
|
|
|
possibleRealpath, err := fshAbs.VirtualPathToRealPath(possibleVirtualFilePath, userinfo.Username)
|
|
|
if err != nil {
|
|
|
log.Println("Error when trying to serve file in compatibility mode", err.Error())
|
|
|
- return nil, "", errors.New("Error when trying to serve file in compatibility mode")
|
|
|
+ return nil, "", "", errors.New("Error when trying to serve file in compatibility mode")
|
|
|
}
|
|
|
if fshAbs.FileExists(possibleRealpath) {
|
|
|
realFilepath = possibleRealpath
|
|
|
log.Println("[Media Server] Serving file " + filepath.Base(possibleRealpath) + " in compatibility mode. Do not to use '&' or '+' sign in filename! ")
|
|
|
- return fsh, realFilepath, nil
|
|
|
+ return fsh, targetfile, realFilepath, nil
|
|
|
} else {
|
|
|
- return nil, "", errors.New("File not exists")
|
|
|
+ return nil, "", "", errors.New("File not exists")
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- return fsh, realFilepath, nil
|
|
|
+ return fsh, targetfile, realFilepath, nil
|
|
|
}
|
|
|
|
|
|
func serveMediaMime(w http.ResponseWriter, r *http.Request) {
|
|
|
- targetFsh, realFilepath, err := media_server_validateSourceFile(w, r)
|
|
|
+ targetFsh, _, realFilepath, err := media_server_validateSourceFile(w, r)
|
|
|
if err != nil {
|
|
|
common.SendErrorResponse(w, err.Error())
|
|
|
return
|
|
|
@@ -140,8 +144,9 @@ func serveMediaMime(w http.ResponseWriter, r *http.Request) {
|
|
|
}
|
|
|
|
|
|
func serverMedia(w http.ResponseWriter, r *http.Request) {
|
|
|
+ userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
|
|
|
//Serve normal media files
|
|
|
- targetFsh, realFilepath, err := media_server_validateSourceFile(w, r)
|
|
|
+ targetFsh, vpath, realFilepath, err := media_server_validateSourceFile(w, r)
|
|
|
if err != nil {
|
|
|
common.SendErrorResponse(w, err.Error())
|
|
|
return
|
|
|
@@ -201,6 +206,26 @@ func serverMedia(w http.ResponseWriter, r *http.Request) {
|
|
|
} else {
|
|
|
if targetFsh.RequireBuffer || !filesystem.FileExists(realFilepath) {
|
|
|
w.Header().Set("Content-Length", strconv.Itoa(int(targetFshAbs.GetFileSize(realFilepath))))
|
|
|
+ //Check buffer exists
|
|
|
+ ps, _ := targetFsh.GetUniquePathHash(vpath, userinfo.Username)
|
|
|
+ buffpool := filepath.Join(*tmp_directory, "fsbuffpool")
|
|
|
+ buffFile := filepath.Join(buffpool, ps)
|
|
|
+ if fs.FileExists(buffFile) {
|
|
|
+ //Stream the buff file if hash matches
|
|
|
+ remoteFileHash, err := getHashFromRemoteFile(targetFsh.FileSystemAbstraction, realFilepath)
|
|
|
+ if err == nil {
|
|
|
+ localFileHash, err := os.ReadFile(buffFile + ".hash")
|
|
|
+ if err == nil {
|
|
|
+ if string(localFileHash) == remoteFileHash {
|
|
|
+ //Hash matches. Serve local buffered file
|
|
|
+ http.ServeFile(w, r, buffFile)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
remoteStream, err := targetFshAbs.ReadStream(realFilepath)
|
|
|
if err != nil {
|
|
|
common.SendErrorResponse(w, err.Error())
|
|
|
@@ -208,6 +233,13 @@ func serverMedia(w http.ResponseWriter, r *http.Request) {
|
|
|
}
|
|
|
io.Copy(w, remoteStream)
|
|
|
remoteStream.Close()
|
|
|
+
|
|
|
+ if *enable_buffering {
|
|
|
+ os.MkdirAll(buffpool, 0775)
|
|
|
+ go func() {
|
|
|
+ BufferRemoteFileToTmp(buffFile, targetFsh, realFilepath)
|
|
|
+ }()
|
|
|
+ }
|
|
|
} else {
|
|
|
http.ServeFile(w, r, realFilepath)
|
|
|
}
|
|
|
@@ -215,3 +247,84 @@ func serverMedia(w http.ResponseWriter, r *http.Request) {
|
|
|
}
|
|
|
|
|
|
}
|
|
|
+
|
|
|
+func BufferRemoteFileToTmp(buffFile string, fsh *filesystem.FileSystemHandler, rpath string) error {
|
|
|
+ if fs.FileExists(buffFile + ".download") {
|
|
|
+ return errors.New("another buffer process running")
|
|
|
+ }
|
|
|
+
|
|
|
+ //Generate a stat file for the buffer
|
|
|
+ hash, err := getHashFromRemoteFile(fsh.FileSystemAbstraction, rpath)
|
|
|
+ if err != nil {
|
|
|
+ //Do not buffer
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ os.WriteFile(buffFile+".hash", []byte(hash), 0775)
|
|
|
+
|
|
|
+ //Buffer the file from remote to local
|
|
|
+ f, err := fsh.FileSystemAbstraction.ReadStream(rpath)
|
|
|
+ if err != nil {
|
|
|
+ os.Remove(buffFile + ".hash")
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ defer f.Close()
|
|
|
+
|
|
|
+ dest, err := os.OpenFile(buffFile+".download", os.O_CREATE|os.O_WRONLY, 0775)
|
|
|
+ if err != nil {
|
|
|
+ os.Remove(buffFile + ".hash")
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ defer dest.Close()
|
|
|
+
|
|
|
+ io.Copy(dest, f)
|
|
|
+ f.Close()
|
|
|
+ dest.Close()
|
|
|
+
|
|
|
+ os.Rename(buffFile+".download", buffFile)
|
|
|
+
|
|
|
+ //Clean the oldest buffpool item if size too large
|
|
|
+ dirsize, _ := fs.GetDirctorySize(filepath.Dir(buffFile), false)
|
|
|
+ oldestModtime := time.Now().Unix()
|
|
|
+ oldestFile := ""
|
|
|
+ for int(dirsize) > *bufferPoolSize<<20 {
|
|
|
+ //fmt.Println("CLEARNING BUFF", dirsize)
|
|
|
+ files, _ := filepath.Glob(filepath.ToSlash(filepath.Dir(buffFile)) + "/*")
|
|
|
+ for _, file := range files {
|
|
|
+ if filepath.Ext(file) == ".hash" {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ thisModTime, _ := fs.GetModTime(file)
|
|
|
+ if thisModTime < oldestModtime {
|
|
|
+ oldestModtime = thisModTime
|
|
|
+ oldestFile = file
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ os.Remove(oldestFile)
|
|
|
+ os.Remove(oldestFile + ".hash")
|
|
|
+
|
|
|
+ dirsize, _ = fs.GetDirctorySize(filepath.Dir(buffFile), false)
|
|
|
+ oldestModtime = time.Now().Unix()
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+}
|
|
|
+
|
|
|
+func getHashFromRemoteFile(fshAbs filesystem.FileSystemAbstraction, rpath string) (string, error) {
|
|
|
+ filestat, err := fshAbs.Stat(rpath)
|
|
|
+ if err != nil {
|
|
|
+ //Always pull from remote
|
|
|
+ return "", err
|
|
|
+ }
|
|
|
+
|
|
|
+ if filestat.Size() >= int64(*bufferPoolSize<<20) {
|
|
|
+ return "", errors.New("Unable to buffer: file larger than buffpool size")
|
|
|
+ }
|
|
|
+
|
|
|
+ if filestat.Size() >= int64(*bufferFileMaxSize<<20) {
|
|
|
+ return "", errors.New("File larger than max buffer file size")
|
|
|
+ }
|
|
|
+
|
|
|
+ statHash := strconv.Itoa(int(filestat.ModTime().Unix() + filestat.Size()))
|
|
|
+ hash := md5.Sum([]byte(statHash))
|
|
|
+ return hex.EncodeToString(hash[:]), nil
|
|
|
+}
|
Toby Chui