arozos/mediaServer.go

package main

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"imuslab.com/arozos/mod/common"
	"imuslab.com/arozos/mod/filesystem"
	fs "imuslab.com/arozos/mod/filesystem"
	"imuslab.com/arozos/mod/network/gzipmiddleware"
)

/*
Media Server
This function serve large file objects like video and audio file via asynchronize go routine :)

Example usage:
/media/?file=user:/Desktop/test/02.Orchestra- エミール (Addendum version).mp3
/media/?file=user:/Desktop/test/02.Orchestra- エミール (Addendum version).mp3&download=true

This will serve / download the file located at files/users/{username}/Desktop/test/02.Orchestra- エミール (Addendum version).mp3

PLEASE ALWAYS USE URLENCODE IN THE LINK PASSED INTO THE /media ENDPOINT
*/

func mediaServer_init() {
	if *enable_gzip {
		http.HandleFunc("/media/", gzipmiddleware.CompressFunc(serverMedia))
		http.HandleFunc("/media/getMime/", gzipmiddleware.CompressFunc(serveMediaMime))
	} else {
		http.HandleFunc("/media/", serverMedia)
		http.HandleFunc("/media/getMime/", serveMediaMime)
	}

	//Download API always bypass gzip no matter if gzip mode is enabled
	http.HandleFunc("/media/download/", serverMedia)
}

//This function validate the incoming media request and return fsh, vpath, rpath and err if any
func media_server_validateSourceFile(w http.ResponseWriter, r *http.Request) (*filesystem.FileSystemHandler, string, string, error) {
	username, err := authAgent.GetUserName(w, r)
	if err != nil {
		return nil, "", "", errors.New("User not logged in")
	}

	userinfo, _ := userHandler.GetUserInfoFromUsername(username)

	//Validate url valid
	if strings.Count(r.URL.String(), "?") > 1 {
		return nil, "", "", errors.New("Invalid paramters. Multiple ? found")
	}

	targetfile, _ := common.Mv(r, "file", false)
	targetfile, err = url.QueryUnescape(targetfile)
	if err != nil {
		return nil, "", "", err
	}
	if targetfile == "" {
		return nil, "", "", errors.New("Missing paramter 'file'")
	}

	//Translate the virtual directory to realpath
	fsh, subpath, err := GetFSHandlerSubpathFromVpath(targetfile)
	if err != nil {
		return nil, "", "", errors.New("Unable to load from target file system")
	}
	fshAbs := fsh.FileSystemAbstraction
	realFilepath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if fshAbs.FileExists(realFilepath) && fshAbs.IsDir(realFilepath) {
		return nil, "", "", errors.New("Given path is not a file")
	}
	if err != nil {
		return nil, "", "", errors.New("Unable to translate the given filepath")
	}

	if !fshAbs.FileExists(realFilepath) {
		//Sometime if url is not URL encoded, this error might be shown as well

		//Try to use manual segmentation

		originalURL := r.URL.String()
		//Must be pre-processed with system special URI Decode function to handle edge cases
		originalURL = fs.DecodeURI(originalURL)
		if strings.Contains(originalURL, "&download=true") {
			originalURL = strings.ReplaceAll(originalURL, "&download=true", "")
		} else if strings.Contains(originalURL, "download=true") {
			originalURL = strings.ReplaceAll(originalURL, "download=true", "")
		}
		if strings.Contains(originalURL, "&file=") {
			originalURL = strings.ReplaceAll(originalURL, "&file=", "file=")
		}
		urlInfo := strings.Split(originalURL, "file=")
		possibleVirtualFilePath := urlInfo[len(urlInfo)-1]
		possibleRealpath, err := fshAbs.VirtualPathToRealPath(possibleVirtualFilePath, userinfo.Username)
		if err != nil {
			systemWideLogger.PrintAndLog("Media Server", "Error when trying to serve file in compatibility mode", err)
			return nil, "", "", errors.New("Error when trying to serve file in compatibility mode")
		}
		if fshAbs.FileExists(possibleRealpath) {
			realFilepath = possibleRealpath
			systemWideLogger.PrintAndLog("Media Server", "Serving file "+filepath.Base(possibleRealpath)+" in compatibility mode. Do not to use '&' or '+' sign in filename! ", nil)
			return fsh, targetfile, realFilepath, nil
		} else {
			return nil, "", "", errors.New("File not exists")
		}
	}

	return fsh, targetfile, realFilepath, nil
}

func serveMediaMime(w http.ResponseWriter, r *http.Request) {
	targetFsh, _, realFilepath, err := media_server_validateSourceFile(w, r)
	if err != nil {
		common.SendErrorResponse(w, err.Error())
		return
	}
	targetFshAbs := targetFsh.FileSystemAbstraction
	if targetFsh.RequireBuffer {
		//File is not on local. Guess its mime by extension
		common.SendTextResponse(w, "application/"+filepath.Ext(realFilepath)[1:])
		return
	}

	mime := "text/directory"
	if !targetFshAbs.IsDir(realFilepath) {
		m, _, err := fs.GetMime(realFilepath)
		if err != nil {
			mime = ""
		}
		mime = m
	}

	common.SendTextResponse(w, mime)
}

func serverMedia(w http.ResponseWriter, r *http.Request) {
	userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
	//Serve normal media files
	targetFsh, vpath, realFilepath, err := media_server_validateSourceFile(w, r)
	if err != nil {
		common.SendErrorResponse(w, err.Error())
		return
	}

	targetFshAbs := targetFsh.FileSystemAbstraction

	//Check if downloadMode
	downloadMode := false
	dw, _ := common.Mv(r, "download", false)
	if dw == "true" {
		downloadMode = true
	}

	//New download implementations, allow /download to be used instead of &download=true
	if strings.Contains(r.RequestURI, "media/download/?file=") {
		downloadMode = true
	}

	//Serve the file
	if downloadMode {
		escapedRealFilepath, err := url.PathUnescape(realFilepath)
		if err != nil {
			common.SendErrorResponse(w, err.Error())
			return
		}
		filename := filepath.Base(escapedRealFilepath)

		/*
			//12 Jul 2022 Update: Deprecated the browser detection logic
			userAgent := r.Header.Get("User-Agent")
			if strings.Contains(userAgent, "Safari/")) {
				//This is Safari. Use speial header
				w.Header().Set("Content-Disposition", "attachment; filename="+filepath.Base(realFilepath))
			} else {
				//Fixing the header issue on Golang url encode lib problems
				w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+filename)
			}
		*/

		w.Header().Set("Content-Disposition", "attachment; filename=\""+filename+"\"")
		w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
		if targetFsh.RequireBuffer || !filesystem.FileExists(realFilepath) {
			//Stream it directly from remote
			w.Header().Set("Content-Length", strconv.Itoa(int(targetFshAbs.GetFileSize(realFilepath))))
			remoteStream, err := targetFshAbs.ReadStream(realFilepath)
			if err != nil {
				common.SendErrorResponse(w, err.Error())
				return
			}
			io.Copy(w, remoteStream)
			remoteStream.Close()
		} else {
			http.ServeFile(w, r, escapedRealFilepath)
		}

	} else {
		if targetFsh.RequireBuffer {
			w.Header().Set("Content-Length", strconv.Itoa(int(targetFshAbs.GetFileSize(realFilepath))))
			//Check buffer exists
			ps, _ := targetFsh.GetUniquePathHash(vpath, userinfo.Username)
			buffpool := filepath.Join(*tmp_directory, "fsbuffpool")
			buffFile := filepath.Join(buffpool, ps)
			if fs.FileExists(buffFile) {
				//Stream the buff file if hash matches
				remoteFileHash, err := getHashFromRemoteFile(targetFsh.FileSystemAbstraction, realFilepath)
				if err == nil {
					localFileHash, err := os.ReadFile(buffFile + ".hash")
					if err == nil {
						if string(localFileHash) == remoteFileHash {
							//Hash matches. Serve local buffered file
							http.ServeFile(w, r, buffFile)
							return
						}
					}
				}

			}

			remoteStream, err := targetFshAbs.ReadStream(realFilepath)
			if err != nil {
				common.SendErrorResponse(w, err.Error())
				return
			}
			defer remoteStream.Close()
			io.Copy(w, remoteStream)

			if *enable_buffering {
				os.MkdirAll(buffpool, 0775)
				go func() {
					BufferRemoteFileToTmp(buffFile, targetFsh, realFilepath)
				}()
			}

		} else if !filesystem.FileExists(realFilepath) {
			//Streaming from remote file system that support fseek
			f, err := targetFsh.FileSystemAbstraction.Open(realFilepath)
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("500 - Internal Server Error"))
				return
			}
			fstat, _ := f.Stat()
			defer f.Close()
			http.ServeContent(w, r, filepath.Base(realFilepath), fstat.ModTime(), f)
		} else {
			http.ServeFile(w, r, realFilepath)
		}

	}

}

func BufferRemoteFileToTmp(buffFile string, fsh *filesystem.FileSystemHandler, rpath string) error {
	if fs.FileExists(buffFile + ".download") {
		return errors.New("another buffer process running")
	}

	//Generate a stat file for the buffer
	hash, err := getHashFromRemoteFile(fsh.FileSystemAbstraction, rpath)
	if err != nil {
		//Do not buffer
		return err
	}
	os.WriteFile(buffFile+".hash", []byte(hash), 0775)

	//Buffer the file from remote to local
	f, err := fsh.FileSystemAbstraction.ReadStream(rpath)
	if err != nil {
		os.Remove(buffFile + ".hash")
		return err
	}
	defer f.Close()

	dest, err := os.OpenFile(buffFile+".download", os.O_CREATE|os.O_WRONLY, 0775)
	if err != nil {
		os.Remove(buffFile + ".hash")
		return err
	}
	defer dest.Close()

	io.Copy(dest, f)
	f.Close()
	dest.Close()

	os.Rename(buffFile+".download", buffFile)

	//Clean the oldest buffpool item if size too large
	dirsize, _ := fs.GetDirctorySize(filepath.Dir(buffFile), false)
	oldestModtime := time.Now().Unix()
	oldestFile := ""
	for int(dirsize) > *bufferPoolSize<<20 {
		//fmt.Println("CLEARNING BUFF", dirsize)
		files, _ := filepath.Glob(filepath.ToSlash(filepath.Dir(buffFile)) + "/*")
		for _, file := range files {
			if filepath.Ext(file) == ".hash" {
				continue
			}
			thisModTime, _ := fs.GetModTime(file)
			if thisModTime < oldestModtime {
				oldestModtime = thisModTime
				oldestFile = file
			}
		}

		os.Remove(oldestFile)
		os.Remove(oldestFile + ".hash")

		dirsize, _ = fs.GetDirctorySize(filepath.Dir(buffFile), false)
		oldestModtime = time.Now().Unix()
	}
	return nil
}

func getHashFromRemoteFile(fshAbs filesystem.FileSystemAbstraction, rpath string) (string, error) {
	filestat, err := fshAbs.Stat(rpath)
	if err != nil {
		//Always pull from remote
		return "", err
	}

	if filestat.Size() >= int64(*bufferPoolSize<<20) {
		return "", errors.New("Unable to buffer: file larger than buffpool size")
	}

	if filestat.Size() >= int64(*bufferFileMaxSize<<20) {
		return "", errors.New("File larger than max buffer file size")
	}

	statHash := strconv.Itoa(int(filestat.ModTime().Unix() + filestat.Size()))
	hash := md5.Sum([]byte(statHash))
	return hex.EncodeToString(hash[:]), nil
}