Websocket upload works but I don't know why

file_system.go

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"io"
+	"io/fs"
 	"log"
 	"math"
 	"mime"
@@ -25,7 +26,6 @@ import (
 	"imuslab.com/arozos/mod/common"
 	"imuslab.com/arozos/mod/compatibility"
 	"imuslab.com/arozos/mod/filesystem"
-	fs "imuslab.com/arozos/mod/filesystem"
 	"imuslab.com/arozos/mod/filesystem/fsdef"
 	fsp "imuslab.com/arozos/mod/filesystem/fspermission"
 	"imuslab.com/arozos/mod/filesystem/fuzzy"
@@ -250,7 +250,7 @@ func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
 	//Check if case sensitive is enabled
 	casesensitve, _ := common.Mv(r, "casesensitive", true)
 
-	vrootID, _, err := fs.GetIDFromVirtualPath(vpath)
+	vrootID, _, err := filesystem.GetIDFromVirtualPath(vpath)
 	var targetFSH *filesystem.FileSystemHandler = nil
 	if err != nil {
 		common.SendErrorResponse(w, "Invalid path given")
@@ -284,14 +284,14 @@ func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
 		}
 
 		//Prepare result struct
-		results := []fs.FileData{}
+		results := []filesystem.FileData{}
 
 		escaped := false
 		for _, matchedFile := range matchingFiles {
 			thisVpath, _ := targetFSH.FileSystemAbstraction.RealPathToVirtualPath(matchedFile, userinfo.Username)
 			isHidden, _ := hidden.IsHidden(thisVpath, true)
 			if !isHidden {
-				results = append(results, fs.GetFileDataFromPath(targetFSH, thisVpath, matchedFile, 2))
+				results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, matchedFile, 2))
 			}
 
 		}
@@ -309,7 +309,7 @@ func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
 		matcher := fuzzy.NewFuzzyMatcher(keyword, casesensitve == "true")
 
 		//Recursive keyword
-		results := []fs.FileData{}
+		results := []filesystem.FileData{}
 		var err error = nil
 
 		fshAbs := targetFSH.FileSystemAbstraction
@@ -319,11 +319,11 @@ func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
 				thisFilename = strings.ToLower(thisFilename)
 			}
 
-			if !fs.IsInsideHiddenFolder(path) {
+			if !filesystem.IsInsideHiddenFolder(path) {
 				if matcher.Match(thisFilename) {
 					//This is a matching file
 					thisVpath, _ := fshAbs.RealPathToVirtualPath(path, userinfo.Username)
-					results = append(results, fs.GetFileDataFromPath(targetFSH, thisVpath, path, 2))
+					results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, path, 2))
 				}
 			}
 
@@ -346,6 +346,13 @@ func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
 
 	This function is specailly designed to work with low memory devices
 	(e.g. ZeroPi / Orange Pi Zero with 512MB RAM)
+
+	Two cases
+	1. Not Buffer FS + Huge File
+	=> Write chunks to fsa + merge to fsa
+
+	2. Else
+	=> write chunks to tmp (via os package) + merge to fsa
 */
 func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 	//Get user info
@@ -392,7 +399,6 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte("500 - Path translation failed"))
 		return
 	}
-
 	fshAbs := fsh.FileSystemAbstraction
 
 	//Translate the upload target directory
@@ -406,7 +412,7 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 	//Check if it is huge file upload mode
 	isHugeFile := false
 	hugefile, _ := common.Mv(r, "hugefile", false)
-	if hugefile == "true" && filesystem.FileExists(realUploadPath) {
+	if hugefile == "true" && !fsh.RequireBuffer {
 		//Huge file mode is only compatible with local file systems
 		//For remote file system, use buffer to tmp then upload method
 		isHugeFile = true
@@ -424,8 +430,10 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 	if isHugeFile {
 		//Change to upload directly to target disk
 		uploadFolder = filepath.Join(realUploadPath, ".metadata/.upload", uploadUUID)
+		fshAbs.MkdirAll(uploadFolder, 0700)
+	} else {
+		os.MkdirAll(uploadFolder, 0700)
 	}
-	os.MkdirAll(uploadFolder, 0700)
 
 	//Start websocket connection
 	var upgrader = websocket.Upgrader{}
@@ -473,7 +481,12 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 			log.Println("Upload terminated by client. Cleaning tmp folder.")
 			//Clear the tmp folder
 			time.Sleep(1 * time.Second)
-			os.RemoveAll(uploadFolder)
+			if isHugeFile {
+				fshAbs.RemoveAll(uploadFolder)
+			} else {
+				os.RemoveAll(uploadFolder)
+			}
+
 			return
 		}
 		//The mt should be 2 = binary for file upload and 1 for control syntax
@@ -490,7 +503,12 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 			//File block. Save it to tmp folder
 			chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(blockCounter))
 			chunkName = append(chunkName, chunkFilepath)
-			writeErr := os.WriteFile(chunkFilepath, message, 0700)
+			var writeErr error
+			if isHugeFile {
+				writeErr = fshAbs.WriteFile(chunkFilepath, message, 0700)
+			} else {
+				writeErr = os.WriteFile(chunkFilepath, message, 0700)
+			}
 
 			if writeErr != nil {
 				//Unable to write block. Is the tmp folder fulled?
@@ -503,7 +521,11 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 				c.Close()
 
 				//Clear the tmp files
-				os.RemoveAll(uploadFolder)
+				if isHugeFile {
+					fshAbs.RemoveAll(uploadFolder)
+				} else {
+					os.RemoveAll(uploadFolder)
+				}
 				return
 			}
 
@@ -511,7 +533,7 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 			lastChunkArrivalTime = time.Now().Unix()
 
 			//Check if the file size is too big
-			totalFileSize += fs.GetFileSize(chunkFilepath)
+			totalFileSize += int64(len(message))
 
 			if totalFileSize > max_upload_size {
 				//File too big
@@ -523,7 +545,11 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 				c.Close()
 
 				//Clear the tmp files
-				os.RemoveAll(uploadFolder)
+				if isHugeFile {
+					fshAbs.RemoveAll(uploadFolder)
+				} else {
+					os.RemoveAll(uploadFolder)
+				}
 				return
 			} else if !userinfo.StorageQuota.HaveSpace(totalFileSize) {
 				//Quota exceeded
@@ -535,7 +561,11 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 				c.Close()
 
 				//Clear the tmp files
-				os.RemoveAll(uploadFolder)
+				if isHugeFile {
+					fshAbs.RemoveAll(uploadFolder)
+				} else {
+					os.RemoveAll(uploadFolder)
+				}
 			}
 			blockCounter++
 
@@ -557,10 +587,16 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 
 	//Merge the file. Merge file location must be on local machine
 	mergeFileLocation := decodedUploadLocation
+	var out *os.File
 	if fsh.RequireBuffer {
+		//The merge file location must be local buffer
 		mergeFileLocation = getFsBufferFilepath(decodedUploadLocation, false)
+		out, err = os.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
+	} else {
+		//The merge file location can be local or remote that support OpenFile.
+		out, err = fshAbs.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
 	}
-	out, err := os.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
+
 	if err != nil {
 		log.Println("Failed to open file:", err)
 		c.WriteMessage(1, []byte(`{\"error\":\"Failed to open destination file\"}`))
@@ -571,7 +607,13 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, filesrc := range chunkName {
-		srcChunkReader, err := os.Open(filesrc)
+		var srcChunkReader *os.File
+		if isHugeFile {
+			srcChunkReader, err = fshAbs.Open(filesrc)
+		} else {
+			srcChunkReader, err = os.Open(filesrc)
+		}
+
 		if err != nil {
 			log.Println("Failed to open Source Chunk", filesrc, " with error ", err.Error())
 			c.WriteMessage(1, []byte(`{\"error\":\"Failed to open Source Chunk\"}`))
@@ -581,13 +623,24 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 		srcChunkReader.Close()
 
 		//Delete file immediately to save space
-		os.Remove(filesrc)
+		if isHugeFile {
+			fshAbs.Remove(filesrc)
+		} else {
+			os.Remove(filesrc)
+		}
+
 	}
 
 	out.Close()
 
 	//Check if the size fit in user quota
-	fi, err := os.Stat(mergeFileLocation)
+	var fi fs.FileInfo
+	if fsh.RequireBuffer {
+		fi, err = os.Stat(mergeFileLocation)
+	} else {
+		fi, err = fshAbs.Stat(mergeFileLocation)
+	}
+
 	if err != nil {
 		// Could not obtain stat, handle error
 		log.Println("Failed to validate uploaded file: ", mergeFileLocation, ". Error Message: ", err.Error())
@@ -596,13 +649,17 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 	}
 	if !userinfo.StorageQuota.HaveSpace(fi.Size()) {
 		c.WriteMessage(1, []byte(`{\"error\":\"User Storage Quota Exceeded\"}`))
-		os.RemoveAll(mergeFileLocation)
+		if fsh.RequireBuffer {
+			os.RemoveAll(mergeFileLocation)
+		} else {
+			fshAbs.RemoveAll(mergeFileLocation)
+		}
 		return
 	}
 
 	//Upload it to remote side if it fits the user quota && is buffer file
 	if fsh.RequireBuffer {
-		//This is buffer file. Upload to dest fsh
+		//This is local buffer file. Upload to dest fsh
 		f, err := os.Open(mergeFileLocation)
 		if err != nil {
 			log.Println("Failed to open buffered file at ", mergeFileLocation, " with error ", err.Error())
@@ -638,7 +695,11 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 
 	//Clear the tmp folder
 	time.Sleep(300 * time.Millisecond)
-	os.RemoveAll(uploadFolder)
+	if isHugeFile {
+		fshAbs.RemoveAll(uploadFolder)
+	} else {
+		os.RemoveAll(uploadFolder)
+	}
 
 	//Close WebSocket connection after finished
 	c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
@@ -1133,7 +1194,7 @@ func system_fs_handleNewObjects(w http.ResponseWriter, r *http.Request) {
 
 	if fileType == "" && filename == "" {
 		//List all the supported new filetype
-		if !fs.FileExists("system/newitem/") {
+		if !filesystem.FileExists("system/newitem/") {
 			os.MkdirAll("system/newitem/", 0755)
 		}
 
@@ -1388,7 +1449,7 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 		}
 
 		//Create the zip file
-		fs.ArozZipFileWithProgress(realSourceFiles, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) {
+		filesystem.ArozZipFileWithProgress(realSourceFiles, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) {
 			currentStatus := ProgressUpdate{
 				LatestFile: currentFilename,
 				Progress:   int(math.Ceil(progress)),
@@ -1479,7 +1540,7 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 		}
 
 		//Unzip the files
-		fs.ArozUnzipFileWithProgress(realSourceFiles, unzipDest, func(currentFile string, filecount int, totalfile int, progress float64) {
+		filesystem.ArozUnzipFileWithProgress(realSourceFiles, unzipDest, func(currentFile string, filecount int, totalfile int, progress float64) {
 			//Generate the status update struct
 			currentStatus := ProgressUpdate{
 				LatestFile: filepath.Base(currentFile),
@@ -1544,7 +1605,7 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 			}
 
 			if operation == "move" {
-				err := fs.FileMove(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, func(progress int, currentFile string) {
+				err := filesystem.FileMove(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, func(progress int, currentFile string) {
 					//Multply child progress to parent progress
 					blockRatio := float64(100) / float64(len(sourceFiles))
 					overallRatio := blockRatio*float64(i) + blockRatio*(float64(progress)/float64(100))
@@ -1577,7 +1638,7 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 				metadata.RemoveCache(thisSrcFsh, rsrcFile)
 
 			} else if operation == "copy" {
-				err := fs.FileCopy(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, func(progress int, currentFile string) {
+				err := filesystem.FileCopy(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, func(progress int, currentFile string) {
 					//Multply child progress to parent progress
 					blockRatio := float64(100) / float64(len(sourceFiles))
 					overallRatio := blockRatio*float64(i) + blockRatio*(float64(progress)/float64(100))
@@ -1718,7 +1779,7 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 		}
 
 		//Create a zip file at target location
-		err = fs.ArozZipFile(rsrcFiles, zipFileTargetLocation, false)
+		err = filesystem.ArozZipFile(rsrcFiles, zipFileTargetLocation, false)
 		if err != nil {
 			os.Remove(zipFileTargetLocation)
 			common.SendErrorResponse(w, err.Error())
@@ -1851,12 +1912,12 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 
 				//srcAbs, _ := filepath.Abs(rsrcFile)
 				//destAbs, _ := filepath.Abs(rdestFile)
-				//underSameRoot, _ := fs.UnderTheSameRoot(srcAbs, destAbs)
+				//underSameRoot, _ := filesystem.UnderTheSameRoot(srcAbs, destAbs)
 
 				//Updates 19-10-2020: Added ownership management to file move and copy
 				userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)
 
-				err = fs.FileMove(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, nil)
+				err = filesystem.FileMove(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, nil)
 				if err != nil {
 					common.SendErrorResponse(w, err.Error())
 					//Restore the ownership if remove failed
@@ -1897,12 +1958,12 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 				existsOpr, _ := common.Mv(r, "existsresp", true)
 
 				//Check if the user have space for the extra file
-				if !userinfo.StorageQuota.HaveSpace(fs.GetFileSize(rdestFile)) {
+				if !userinfo.StorageQuota.HaveSpace(filesystem.GetFileSize(rdestFile)) {
 					common.SendErrorResponse(w, "Storage Quota Full")
 					return
 				}
 
-				err = fs.FileCopy(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, nil)
+				err = filesystem.FileCopy(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, nil)
 				if err != nil {
 					common.SendErrorResponse(w, err.Error())
 					return
@@ -2018,7 +2079,7 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 				}
 
 				//OK! Unzip to destination
-				err := fs.Unzip(unzipSource, unzipDest)
+				err := filesystem.Unzip(unzipSource, unzipDest)
 				if err != nil {
 					common.SendErrorResponse(w, err.Error())
 					return
@@ -2331,7 +2392,7 @@ func system_fs_getFileProperties(w http.ResponseWriter, r *http.Request) {
 
 	fileMime := "text/directory"
 	if !fileStat.IsDir() {
-		m, _, err := fs.GetMime(rpath)
+		m, _, err := filesystem.GetMime(rpath)
 		if err != nil {
 			fileMime = mime.TypeByExtension(filepath.Ext(rpath))
 		} else {
@@ -2426,7 +2487,7 @@ func system_fs_handleList(w http.ResponseWriter, r *http.Request) {
 
 	fshAbs := fsh.FileSystemAbstraction
 
-	var parsedFilelist []fs.FileData
+	var parsedFilelist []filesystem.FileData
 
 	//Normal file systems
 	realpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
@@ -2489,13 +2550,13 @@ func system_fs_handleList(w http.ResponseWriter, r *http.Request) {
 		rawsize := fshAbs.GetFileSize(v)
 		modtime, _ := fshAbs.GetModTime(v)
 		thisvPath, _ := fshAbs.RealPathToVirtualPath(v, userinfo.Username)
-		thisFile := fs.FileData{
+		thisFile := filesystem.FileData{
 			Filename:    filepath.Base(v),
 			Filepath:    currentDir + filepath.Base(v),
 			Realpath:    v,
 			IsDir:       fsh.FileSystemAbstraction.IsDir(v),
 			Filesize:    rawsize,
-			Displaysize: fs.GetFileDisplaySize(rawsize, 2),
+			Displaysize: filesystem.GetFileDisplaySize(rawsize, 2),
 			ModTime:     modtime,
 			IsShared:    shareManager.FileIsShared(userinfo, thisvPath),
 			Shortcut:    shortCutInfo,
@@ -2680,7 +2741,7 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 		}
 
 		//OK. Create the zip at the desired location
-		err := fs.ArozZipFile(realSourcePaths, zipOutput, false)
+		err := filesystem.ArozZipFile(realSourcePaths, zipOutput, false)
 		if err != nil {
 			common.SendErrorResponse(w, err.Error())
 			return
@@ -2689,8 +2750,7 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 		common.SendOK(w)
 	} else if opr == "tmpzip" {
 		//Zip to tmp folder
-		log.Println(realSourcePaths, zipOutput)
-		err := fs.ArozZipFile(realSourcePaths, zipOutput, false)
+		err := filesystem.ArozZipFile(realSourcePaths, zipOutput, false)
 		if err != nil {
 			common.SendErrorResponse(w, err.Error())
 			return
@@ -2997,7 +3057,7 @@ func system_fs_handleFilePermission(w http.ResponseWriter, r *http.Request) {
 	} else {
 		//Set the file mode
 		//Check if the file exists
-		if !fs.FileExists(rpath) {
+		if !filesystem.FileExists(rpath) {
 			common.SendErrorResponse(w, "File not exists!")
 			return
 		}
@@ -3054,7 +3114,7 @@ func system_fs_clearOldTmpFiles() {
 				return nil
 			}
 			//Get its modification time
-			modTime, err := fs.GetModTime(path)
+			modTime, err := filesystem.GetModTime(path)
 			if err != nil {
 				return nil
 			}

web/SystemAO/file_system/file_explorer.html

@@ -1531,6 +1531,7 @@
                 }
                
                 //Get sort mode from server side
+                let loadStartPath = currentPath;
                 $.ajax({
                     url: "../../system/file_system/sortMode",
                     data: {opr: "get", folder: currentPath},
@@ -1557,6 +1558,11 @@
                                     return;
                                 }
 
+                                if (currentPath != loadStartPath){
+                                    //Use switch to another path before the load finish. Do not render
+                                    return;
+                                }
+
                                 if (data.error !== undefined){
                                     //Parse path error. Try to refresh the page
                                     if (data.error.length >= "Redirect:".length && data.error.substr(0,9) == "Redirect:"){

web/SystemAO/storage/poolList.html

@@ -113,13 +113,17 @@
                         if (storagePool.Storages !== null && storagePool.Storages.length > 0){
                             storagePool.Storages.forEach(fsh => {
                                 var diskIcon = `grey disk icon`;
-                                if (fsh.Hierarchy == "backup"){
-                                    diskIcon = `green refresh icon`
-                                }else if (fsh.Hierarchy == "user"){
+                                if (fsh.Hierarchy == "user"){
                                     diskIcon = `blue disk icon`;
-                                }else if (fsh.Hierarchy == "share"){
-                                    diskIcon = `violet share alternate icon`
+                                }else if (fsh.Hierarchy == "public"){
+                                    diskIcon = `grey disk icon`
                                 }
+
+                                var canwriteIcon = `<i class="ui red remove icon" style="margin-left: 0.1em; font-size: 0.9em;"></i>`;
+                                if (!fsh.ReadOnly){
+                                    canwriteIcon = `<i class="ui green checkmark icon" style="margin-left: 0.1em; font-size: 0.9em;"></i>`;
+                                }
+
                                 spHTML += (`<div class="item">
                                 <i class="${diskIcon}"></i>
                                 <div class="content">
@@ -130,7 +134,7 @@
                                         <span class="divider">|</span>
                                         Hierarchy: ${fsh.Hierarchy}
                                         <span class="divider">|</span>
-                                        ReadOnly: ${fsh.ReadOnly}
+                                        ReadWrite: ${canwriteIcon}
                                     </div>
                                     </div>
                                 </div>