arozos/mod/filesystem/metadata/metadata.go

package metadata

import (
	"bufio"
	"encoding/base64"
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/websocket"

	"imuslab.com/arozos/mod/filesystem/fssort"
	hidden "imuslab.com/arozos/mod/filesystem/hidden"
)

/*
	This package is used to extract meta data from files like mp3 and mp4
	Also support image caching

*/

type RenderHandler struct {
	renderingFiles  sync.Map
	renderingFolder sync.Map
}

//Create a new RenderHandler
func NewRenderHandler() *RenderHandler {
	return &RenderHandler{
		renderingFiles:  sync.Map{},
		renderingFolder: sync.Map{},
	}
}

//Build cache for all files (non recursive) for the given filepath
func (rh *RenderHandler) BuildCacheForFolder(path string) error {
	//Get a list of all files inside this path
	files, err := filepath.Glob(filepath.ToSlash(filepath.Clean(path)) + "/*")
	if err != nil {
		return err
	}
	for _, file := range files {
		//Load Cache in generate mode
		rh.LoadCache(file, true)
	}

	//Check if the cache folder has file. If not, remove it
	cachedFiles, _ := filepath.Glob(filepath.ToSlash(filepath.Join(filepath.Clean(path), "/.metadata/.cache/*")))
	if len(cachedFiles) == 0 {
		os.RemoveAll(filepath.ToSlash(filepath.Join(filepath.Clean(path), "/.metadata/.cache/")) + "/")
	}
	return nil
}

func (rh *RenderHandler) LoadCacheAsBytes(file string, generateOnly bool) ([]byte, error) {
	b64, err := rh.LoadCache(file, generateOnly)
	if err != nil {
		return []byte{}, err
	}

	resultingBytes, _ := base64.StdEncoding.DecodeString(b64)
	return resultingBytes, nil
}

//Try to load a cache from file. If not exists, generate it now
func (rh *RenderHandler) LoadCache(file string, generateOnly bool) (string, error) {
	//Create a cache folder
	cacheFolder := filepath.ToSlash(filepath.Join(filepath.Clean(filepath.Dir(file)), "/.metadata/.cache/") + "/")
	os.MkdirAll(cacheFolder, 0755)
	hidden.HideFile(filepath.Dir(filepath.Clean(cacheFolder)))
	hidden.HideFile(cacheFolder)

	//Check if cache already exists. If yes, return the image from the cache folder
	if CacheExists(file) {
		if generateOnly {
			//Only generate, do not return image
			return "", nil
		}

		//Allow thumbnail to be either jpg or png file
		ext := ".jpg"
		if !fileExists(cacheFolder + filepath.Base(file) + ".jpg") {
			ext = ".png"
		}

		//Updates 02/10/2021: Check if the source file is newer than the cache. Update the cache if true
		if mtime(file) > mtime(cacheFolder+filepath.Base(file)+ext) {
			//File is newer than cache. Delete the cache
			os.Remove(cacheFolder + filepath.Base(file) + ext)
		} else {
			//Check if the file is being writting by another process. If yes, wait for it
			counter := 0
			for rh.fileIsBusy(file) && counter < 15 {
				counter += 1
				time.Sleep(1 * time.Second)
			}

			//Time out and the file is still busy
			if rh.fileIsBusy(file) {
				log.Println("Process racing for cache file. Skipping", file)
				return "", errors.New("Process racing for cache file. Skipping")
			}

			//Read and return the image
			ctx, err := getImageAsBase64(cacheFolder + filepath.Base(file) + ext)
			return ctx, err
		}

	} else {
		//This file not exists yet. Check if it is being hold by another process already
		if rh.fileIsBusy(file) {
			log.Println("Process racing for cache file. Skipping", file)
			return "", errors.New("Process racing for cache file. Skipping")
		}
	}

	//Cache image not exists. Set this file to busy
	rh.renderingFiles.Store(file, "busy")

	//That object not exists. Generate cache image
	//Audio formats that might contains id4 thumbnail
	id4Formats := []string{".mp3", ".ogg", ".flac"}
	if inArray(id4Formats, strings.ToLower(filepath.Ext(file))) {
		img, err := generateThumbnailForAudio(cacheFolder, file, generateOnly)
		rh.renderingFiles.Delete(file)
		return img, err
	}

	//Generate resized image for images
	imageFormats := []string{".png", ".jpeg", ".jpg"}
	if inArray(imageFormats, strings.ToLower(filepath.Ext(file))) {
		img, err := generateThumbnailForImage(cacheFolder, file, generateOnly)
		rh.renderingFiles.Delete(file)
		return img, err
	}

	//Video formats, extract from the 5 sec mark
	vidFormats := []string{".mkv", ".mp4", ".webm", ".ogv", ".avi", ".rmvb"}
	if inArray(vidFormats, strings.ToLower(filepath.Ext(file))) {
		img, err := generateThumbnailForVideo(cacheFolder, file, generateOnly)
		rh.renderingFiles.Delete(file)
		return img, err
	}

	//3D Model Formats
	modelFormats := []string{".stl", ".obj"}
	if inArray(modelFormats, strings.ToLower(filepath.Ext(file))) {
		img, err := generateThumbnailForModel(cacheFolder, file, generateOnly)
		rh.renderingFiles.Delete(file)
		return img, err
	}

	//Photoshop file
	if strings.ToLower(filepath.Ext(file)) == ".psd" {
		img, err := generateThumbnailForPSD(cacheFolder, file, generateOnly)
		rh.renderingFiles.Delete(file)
		return img, err
	}

	//Folder preview renderer
	if isDir(file) && len(filepath.Base(file)) > 0 && filepath.Base(file)[:1] != "." {
		img, err := generateThumbnailForFolder(cacheFolder, file, generateOnly)
		rh.renderingFiles.Delete(file)
		return img, err
	}

	//Other filters
	rh.renderingFiles.Delete(file)
	return "", errors.New("No supported format")
}

func (rh *RenderHandler) fileIsBusy(path string) bool {
	if rh == nil {
		log.Println("RenderHandler is null!")
		return true
	}
	_, ok := rh.renderingFiles.Load(path)
	if !ok {
		//File path is not being process by another process
		return false
	} else {
		return true
	}
}

func getImageAsBase64(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}

	//Added Seek to 0,0 function to prevent failed to decode: Unexpected EOF error
	f.Seek(0, 0)

	reader := bufio.NewReader(f)
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}
	encoded := base64.StdEncoding.EncodeToString(content)
	f.Close()
	return string(encoded), nil
}

//Load a list of folder cache from websocket, pass in "" (empty string) for default sorting method
func (rh *RenderHandler) HandleLoadCache(w http.ResponseWriter, r *http.Request, rpath string, sortmode string) {
	//Get a list of files pending to be cached and sent
	targetPath := filepath.ToSlash(filepath.Clean(rpath))

	//Check if this path already exists another websocket ongoing connection.
	//If yes, disconnect the oldone
	oldc, ok := rh.renderingFolder.Load(targetPath)
	if ok {
		//Close and remove the old connection
		oldc.(*websocket.Conn).Close()
	}

	files, err := specialGlob(targetPath + "/*")
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - Internal Server Error"))
		return
	}

	//Upgrade the connection to websocket
	var upgrader = websocket.Upgrader{}
	upgrader.CheckOrigin = func(r *http.Request) bool { return true }
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print("upgrade:", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - Internal Server Error"))
		return
	}

	//Set this realpath as websocket connected
	rh.renderingFolder.Store(targetPath, c)

	//For each file, serve a cached image preview
	errorExists := false
	filesWithoutCache := []string{}

	//Updates implementation 02/10/2021: Load thumbnail of files first before folder and apply user preference sort mode
	if sortmode == "" {
		sortmode = "default"
	}

	pendingFiles := []string{}
	pendingFolders := []string{}
	for _, file := range files {
		if isDir(file) {
			pendingFiles = append(pendingFiles, file)
		} else {
			pendingFolders = append(pendingFolders, file)
		}
	}
	pendingFiles = append(pendingFiles, pendingFolders...)

	files = fssort.SortFileList(pendingFiles, sortmode)

	//Updated implementation 24/12/2020: Load image with cache first before rendering those without
	for _, file := range files {
		if !CacheExists(file) {
			//Cache not exists. Render this later
			filesWithoutCache = append(filesWithoutCache, file)
		} else {
			//Cache exists. Send it out first
			cachedImage, err := rh.LoadCache(file, false)
			if err != nil {

			} else {
				jsonString, _ := json.Marshal([]string{filepath.Base(file), cachedImage})
				err := c.WriteMessage(1, jsonString)
				if err != nil {
					//Connection closed
					errorExists = true
					break
				}
			}
		}
	}

	retryList := []string{}

	//Render the remaining cache files
	for _, file := range filesWithoutCache {
		//Load the image cache
		cachedImage, err := rh.LoadCache(file, false)
		if err != nil {
			//Unable to load this file's cache. Push it to retry list
			retryList = append(retryList, file)
		} else {
			jsonString, _ := json.Marshal([]string{filepath.Base(file), cachedImage})
			err := c.WriteMessage(1, jsonString)
			if err != nil {
				//Connection closed
				errorExists = true
				break
			}
		}
	}

	//Process the retry list after some wait time
	if len(retryList) > 0 {
		time.Sleep(1000 * time.Millisecond)
		for _, file := range retryList {
			//Load the image cache
			cachedImage, err := rh.LoadCache(file, false)
			if err != nil {

			} else {
				jsonString, _ := json.Marshal([]string{filepath.Base(file), cachedImage})
				err := c.WriteMessage(1, jsonString)
				if err != nil {
					//Connection closed
					errorExists = true
					break
				}
			}
		}
	}

	//Clear record from syncmap
	if !errorExists {
		//This ended normally. Delete the targetPath
		rh.renderingFolder.Delete(targetPath)
	}
	c.Close()

}

//Check if the cache for a file exists
func CacheExists(file string) bool {
	cacheFolder := filepath.ToSlash(filepath.Join(filepath.Clean(filepath.Dir(file)), "/.metadata/.cache/") + "/")
	return fileExists(cacheFolder+filepath.Base(file)+".jpg") || fileExists(cacheFolder+filepath.Base(file)+".png")
}

//Get cache path for this file, given realpath
func GetCacheFilePath(file string) (string, error) {
	if CacheExists(file) {
		cacheFolder := filepath.ToSlash(filepath.Join(filepath.Clean(filepath.Dir(file)), "/.metadata/.cache/") + "/")
		if fileExists(cacheFolder + filepath.Base(file) + ".jpg") {
			return cacheFolder + filepath.Base(file) + ".jpg", nil
		} else if fileExists(cacheFolder + filepath.Base(file) + ".png") {
			return cacheFolder + filepath.Base(file) + ".png", nil
		} else {
			return "", errors.New("Unable to resolve thumbnail cache location")
		}
	} else {
		return "", errors.New("No thumbnail cached for this file")
	}
}

//Remove cache if exists, given realpath
func RemoveCache(file string) error {
	if CacheExists(file) {
		cachePath, err := GetCacheFilePath(file)
		//log.Println("Removing ", cachePath, err)
		if err != nil {
			return err
		}

		//Remove the thumbnail cache
		os.Remove(cachePath)
		return nil
	} else {
		//log.Println("Cache not exists: ", file)
		return errors.New("Thumbnail cache not exists for this file")
	}
}

func specialGlob(path string) ([]string, error) {
	files, err := filepath.Glob(path)
	if err != nil {
		return []string{}, err
	}

	if strings.Contains(path, "[") == true || strings.Contains(path, "]") == true {
		if len(files) == 0 {
			//Handle reverse check. Replace all [ and ] with *
			newSearchPath := strings.ReplaceAll(path, "[", "?")
			newSearchPath = strings.ReplaceAll(newSearchPath, "]", "?")
			//Scan with all the similar structure except [ and ]
			tmpFilelist, _ := filepath.Glob(newSearchPath)
			for _, file := range tmpFilelist {
				file = filepath.ToSlash(file)
				if strings.Contains(file, filepath.ToSlash(filepath.Dir(path))) {
					files = append(files, file)
				}
			}
		}
	}
	//Convert all filepaths to slash
	for i := 0; i < len(files); i++ {
		files[i] = filepath.ToSlash(files[i])
	}
	return files, nil
}