Merge remote-tracking branch 'TC/arozos/master' into systemLog-20220823

file_system.go

@@ -1400,6 +1400,7 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 
 		//Translate source Files into real paths
 		realSourceFiles := []string{}
+		sourceFileFsh := []*filesystem.FileSystemHandler{}
 		for _, vsrcs := range sourceFiles {
 			thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
 			if err != nil {
@@ -1426,34 +1427,19 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 
-			if thisSrcFsh.RequireBuffer {
-				localBufferFilepath, err := bufferRemoteFileToLocal(thisSrcFsh, rsrc, true)
-				if err != nil {
-					stopStatus := ProgressUpdate{
-						LatestFile: filepath.Base(rsrc),
-						Progress:   -1,
-						Error:      "Failed to buffer file to local disk",
-					}
-					js, _ := json.Marshal(stopStatus)
-					c.WriteMessage(1, js)
-					c.Close()
-					return
-				}
-
-				realSourceFiles = append(realSourceFiles, localBufferFilepath)
-			} else {
-				realSourceFiles = append(realSourceFiles, rsrc)
-			}
-
+			realSourceFiles = append(realSourceFiles, rsrc)
+			sourceFileFsh = append(sourceFileFsh, thisSrcFsh)
 		}
 
 		zipDestPath := outputFilename
+		zipDestFsh := destFsh
 		if destFsh.RequireBuffer {
 			zipDestPath = getFsBufferFilepath(outputFilename, false)
+			zipDestFsh = nil
 		}
 
 		//Create the zip file
-		filesystem.ArozZipFileWithProgress(realSourceFiles, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) {
+		err = filesystem.ArozZipFileWithProgress(sourceFileFsh, realSourceFiles, zipDestFsh, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) {
 			currentStatus := ProgressUpdate{
 				LatestFile: currentFilename,
 				Progress:   int(math.Ceil(progress)),
@@ -1464,6 +1450,10 @@ func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
 			c.WriteMessage(1, js)
 		})
 
+		if err != nil {
+			systemWideLogger.PrintAndLog("File System", "Zipping websocket request failed: "+err.Error(), err)
+		}
+
 		if destFsh.RequireBuffer {
 			//Move the buffer result to remote
 			f, _ := os.Open(zipDestPath)
@@ -1738,6 +1728,7 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 	if operation == "zip" {
 		//Zip operation. Parse the real filepath list
 		rsrcFiles := []string{}
+		srcFshs := []*filesystem.FileSystemHandler{}
 		destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
 		if err != nil {
 			common.SendErrorResponse(w, "Unable to resolve zip destination path")
@@ -1752,17 +1743,9 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 			}
 			rsrcFile, _ := vsrcFsh.FileSystemAbstraction.VirtualPathToRealPath(vsrcSubpath, userinfo.Username)
 			if vsrcFsh.FileSystemAbstraction.FileExists(rsrcFile) {
-				if vsrcFsh.RequireBuffer {
-					localBuffFile, err := bufferRemoteFileToLocal(vsrcFsh, rsrcFile, true)
-					if err != nil {
-						continue
-					}
-					rsrcFiles = append(rsrcFiles, localBuffFile)
-				} else {
-					//Push directly its local path to list
-					rsrcFiles = append(rsrcFiles, rsrcFile)
-				}
-
+				//Push directly its local path to list
+				rsrcFiles = append(rsrcFiles, rsrcFile)
+				srcFshs = append(srcFshs, vsrcFsh)
 			}
 		}
 
@@ -1778,12 +1761,14 @@ func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
 
 		//Create a buffer if destination fsh request buffer
 		zipFileTargetLocation := zipFilename
+		zipDestFsh := destFsh
 		if destFsh.RequireBuffer {
 			zipFileTargetLocation = getFsBufferFilepath(zipFilename, false)
+			zipDestFsh = nil
 		}
 
 		//Create a zip file at target location
-		err = filesystem.ArozZipFile(rsrcFiles, zipFileTargetLocation, false)
+		err = filesystem.ArozZipFile(srcFshs, rsrcFiles, zipDestFsh, zipFileTargetLocation, false)
 		if err != nil {
 			os.Remove(zipFileTargetLocation)
 			common.SendErrorResponse(w, err.Error())
@@ -2679,6 +2664,7 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 
 	//Check each of the path
 	realSourcePaths := []string{}
+	sourceFshs := []*filesystem.FileSystemHandler{}
 	for _, vpath := range virtualSourcePaths {
 		thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
 		if err != nil {
@@ -2692,17 +2678,8 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		if thisSrcFsh.RequireBuffer {
-			thisLocalBuffer, err := bufferRemoteFileToLocal(thisSrcFsh, thisrpath, true)
-			if err != nil {
-				common.SendErrorResponse(w, "Unable to buffer from remote file system: "+err.Error())
-				return
-			}
-			realSourcePaths = append(realSourcePaths, thisLocalBuffer)
-		} else {
-			realSourcePaths = append(realSourcePaths, thisrpath)
-		}
-
+		realSourcePaths = append(realSourcePaths, thisrpath)
+		sourceFshs = append(sourceFshs, thisSrcFsh)
 	}
 
 	///Convert dest to real if given
@@ -2728,8 +2705,10 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 	rdest, _ = destFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
 	destFshAbs := destFsh.FileSystemAbstraction
 	zipOutput := rdest
+	zipDestFsh := destFsh
 	if destFsh.RequireBuffer {
 		zipOutput = getFsBufferFilepath(rdest, false)
+		zipDestFsh = nil
 	}
 
 	if opr == "zip" {
@@ -2740,7 +2719,7 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 		}
 
 		//OK. Create the zip at the desired location
-		err := filesystem.ArozZipFile(realSourcePaths, zipOutput, false)
+		err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
 		if err != nil {
 			common.SendErrorResponse(w, err.Error())
 			return
@@ -2749,7 +2728,7 @@ func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
 		common.SendOK(w)
 	} else if opr == "tmpzip" {
 		//Zip to tmp folder
-		err := filesystem.ArozZipFile(realSourcePaths, zipOutput, false)
+		err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
 		if err != nil {
 			common.SendErrorResponse(w, err.Error())
 			return

mod/filesystem/abstractions/smbfs/smbfs.go

@@ -134,6 +134,7 @@ func (a ServerMessageBlockFileSystemAbstraction) Remove(filename string) error {
 func (a ServerMessageBlockFileSystemAbstraction) RemoveAll(filename string) error {
 	filename = filterFilepath(filename)
 	filename = toWinPath(filename)
+	fmt.Println("REMOVING", filename)
 	return a.share.RemoveAll(filename)
 }
 func (a ServerMessageBlockFileSystemAbstraction) Rename(oldname, newname string) error {
@@ -270,7 +271,7 @@ func (a ServerMessageBlockFileSystemAbstraction) Walk(root string, walkFn filepa
 		if err != nil {
 			return err
 		}
-		walkFn(path, statInfo, err)
+		walkFn(filepath.ToSlash(filepath.Join(root, path)), statInfo, err)
 		return nil
 	})
 	return err
@@ -306,10 +307,13 @@ func toWinPath(filename string) string {
 func filterFilepath(rawpath string) string {
 	rawpath = filepath.ToSlash(filepath.Clean(rawpath))
 	rawpath = strings.TrimSpace(rawpath)
+
 	if strings.HasPrefix(rawpath, "./") {
 		return rawpath[1:]
 	} else if rawpath == "." || rawpath == "" {
 		return "/"
+	} else if rawpath[0:1] != "/" {
+		return "/" + rawpath
 	}
 	return rawpath
 }

mod/filesystem/fileOpr.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"imuslab.com/arozos/mod/filesystem/arozfs"
 	"imuslab.com/arozos/mod/filesystem/hidden"
 
 	archiver "github.com/mholt/archiver/v3"
@@ -161,13 +162,20 @@ func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHan
 	return nil
 }
 
-//Aroz Zip File with progress update function (current filename / current file count / total file count / progress in percentage)
-func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64)) error {
+/*
+	Aroz Zip File with progress update function
+	Returns the following progress: (current filename / current file count / total file count / progress in percentage)
+	if output is local path that is out of the scope of any fsh, leave outputFsh as nil
+*/
+func ArozZipFileWithProgress(targetFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64)) error {
+	fmt.Println("WEBSOCKET ZIPPING", targetFshs, filelist)
 	//Get the file count from the filelist
 	totalFileCount := 0
-	for _, srcpath := range filelist {
-		if IsDir(srcpath) {
-			filepath.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
+	for i, srcpath := range filelist {
+		thisFsh := targetFshs[i]
+		fshAbs := thisFsh.FileSystemAbstraction
+		if thisFsh.FileSystemAbstraction.IsDir(srcpath) {
+			fshAbs.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
 				if !info.IsDir() {
 					totalFileCount++
 				}
@@ -180,9 +188,16 @@ func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLev
 	}
 
 	//Create the target zip file
-	file, err := os.Create(outputfile)
+	var file arozfs.File
+	var err error
+	if outputFsh != nil {
+		file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
+	} else {
+		//Force local fs
+		file, err = os.Create(outputfile)
+	}
 	if err != nil {
-		panic(err)
+		return err
 	}
 	defer file.Close()
 
@@ -190,11 +205,14 @@ func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLev
 	defer writer.Close()
 
 	currentFileCount := 0
-	for _, srcpath := range filelist {
-		if IsDir(srcpath) {
+	for i, srcpath := range filelist {
+		thisFsh := targetFshs[i]
+		fshAbs := thisFsh.FileSystemAbstraction
+		//Local File System
+		if fshAbs.IsDir(srcpath) {
 			//This is a directory
 			topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
-			err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
+			err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
 					return err
 				}
@@ -202,11 +220,12 @@ func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLev
 					return nil
 				}
 
-				if insideHiddenFolder(path) == true {
+				if insideHiddenFolder(path) {
 					//This is hidden file / folder. Skip this
 					return nil
 				}
-				file, err := os.Open(path)
+
+				file, err := fshAbs.ReadStream(path)
 				if err != nil {
 					return err
 				}
@@ -241,7 +260,7 @@ func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLev
 		} else {
 			//This is a file
 			topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
-			file, err := os.Open(srcpath)
+			file, err := fshAbs.ReadStream(srcpath)
 			if err != nil {
 				return err
 			}
@@ -250,6 +269,7 @@ func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLev
 			if includeTopLevelFolder {
 				relativePath = topLevelFolderName + "/" + relativePath
 			}
+
 			f, err := writer.Create(relativePath)
 			if err != nil {
 				return err
@@ -270,10 +290,22 @@ func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLev
 	return nil
 }
 
-//ArOZ Zip FIle, but with no progress display
-func ArozZipFile(filelist []string, outputfile string, includeTopLevelFolder bool) error {
+/*
+	ArozZipFile
+	Zip file without progress update, support local file system or buffer space
+	To use it with local file system, pass in nil in fsh for each item in filelist, e.g.
+	filesystem.ArozZipFile([]*filesystem.FileSystemHandler{nil}, []string{zippingSource}, nil, targetZipFilename, false)
+*/
+func ArozZipFile(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool) error {
 	//Create the target zip file
-	file, err := os.Create(outputfile)
+	var file arozfs.File
+	var err error
+	if outputFsh != nil {
+		file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
+	} else {
+		//Force local fs
+		file, err = os.Create(outputfile)
+	}
 	if err != nil {
 		return err
 	}
@@ -282,35 +314,140 @@ func ArozZipFile(filelist []string, outputfile string, includeTopLevelFolder boo
 	writer := zip.NewWriter(file)
 	defer writer.Close()
 
-	for _, srcpath := range filelist {
-		if IsDir(srcpath) {
-			//This is a directory
-			topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
-			err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
+	for i, srcpath := range filelist {
+		thisFsh := sourceFshs[i]
+		var fshAbs FileSystemAbstraction
+		if thisFsh == nil {
+			//Use local fs functions
+			if IsDir(srcpath) {
+				//This is a directory
+				topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
+				err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						return err
+					}
+					if info.IsDir() {
+						return nil
+					}
+
+					if insideHiddenFolder(path) {
+						//This is hidden file / folder. Skip this
+						return nil
+					}
+					file, err := os.Open(path)
+					if err != nil {
+						return err
+					}
+					defer file.Close()
+
+					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
+					if includeTopLevelFolder {
+						relativePath = topLevelFolderName + "/" + relativePath
+					} else {
+						relativePath = filepath.Base(srcpath) + "/" + relativePath
+					}
+
+					f, err := writer.Create(relativePath)
+					if err != nil {
+						return err
+					}
+
+					_, err = io.Copy(f, file)
+					if err != nil {
+						return err
+					}
+
+					return nil
+				})
+
 				if err != nil {
 					return err
 				}
-				if info.IsDir() {
-					return nil
+			} else {
+				//This is a file
+				topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
+				file, err := os.Open(srcpath)
+				if err != nil {
+					return err
+				}
+				defer file.Close()
+				relativePath := filepath.Base(srcpath)
+				if includeTopLevelFolder {
+					relativePath = topLevelFolderName + "/" + relativePath
+				}
+				f, err := writer.Create(relativePath)
+				if err != nil {
+					return err
 				}
 
-				if insideHiddenFolder(path) == true {
-					//This is hidden file / folder. Skip this
+				_, err = io.Copy(f, file)
+				if err != nil {
+					return err
+				}
+
+			}
+		} else {
+			//Use file system abstraction
+			fshAbs = thisFsh.FileSystemAbstraction
+			if fshAbs.IsDir(srcpath) {
+				//This is a directory
+				topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
+				err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						return err
+					}
+
+					if info.IsDir() {
+						return nil
+					}
+
+					if insideHiddenFolder(path) {
+						//This is hidden file / folder. Skip this
+						return nil
+					}
+
+					file, err := fshAbs.ReadStream(path)
+					if err != nil {
+						fmt.Println(err)
+						return err
+					}
+					defer file.Close()
+
+					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
+					if includeTopLevelFolder {
+						relativePath = topLevelFolderName + "/" + relativePath
+					} else {
+						relativePath = filepath.Base(srcpath) + "/" + relativePath
+					}
+
+					f, err := writer.Create(relativePath)
+					if err != nil {
+						return err
+					}
+
+					_, err = io.Copy(f, file)
+					if err != nil {
+						return err
+					}
+
 					return nil
+				})
+
+				if err != nil {
+					return err
 				}
-				file, err := os.Open(path)
+			} else {
+				//This is a file
+				topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
+				file, err := fshAbs.ReadStream(srcpath)
 				if err != nil {
 					return err
 				}
 				defer file.Close()
-
-				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
+				relativePath := filepath.Base(srcpath)
 				if includeTopLevelFolder {
 					relativePath = topLevelFolderName + "/" + relativePath
-				} else {
-					relativePath = filepath.Base(srcpath) + "/" + relativePath
 				}
-
 				f, err := writer.Create(relativePath)
 				if err != nil {
 					return err
@@ -321,34 +458,7 @@ func ArozZipFile(filelist []string, outputfile string, includeTopLevelFolder boo
 					return err
 				}
 
-				return nil
-			})
-
-			if err != nil {
-				return err
-			}
-		} else {
-			//This is a file
-			topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
-			file, err := os.Open(srcpath)
-			if err != nil {
-				return err
-			}
-			defer file.Close()
-			relativePath := filepath.Base(srcpath)
-			if includeTopLevelFolder {
-				relativePath = topLevelFolderName + "/" + relativePath
-			}
-			f, err := writer.Create(relativePath)
-			if err != nil {
-				return err
-			}
-
-			_, err = io.Copy(f, file)
-			if err != nil {
-				return err
 			}
-
 		}
 	}
 
@@ -682,6 +792,8 @@ func BasicFileCopy(src string, dst string) error {
 }
 
 //Use for copying large file using buffering method. Allowing copying large file with little RAM
+//Deprecated Since ArozOS v2.000
+/*
 func BufferedLargeFileCopy(src string, dst string, BUFFERSIZE int64) error {
 	sourceFileStat, err := os.Stat(src)
 	if err != nil {
@@ -724,9 +836,11 @@ func BufferedLargeFileCopy(src string, dst string, BUFFERSIZE int64) error {
 	}
 	return nil
 }
+*/
 
+//Check if a local path is dir, do not use with file system abstraction realpath
 func IsDir(path string) bool {
-	if FileExists(path) == false {
+	if !FileExists(path) {
 		return false
 	}
 	fi, err := os.Stat(path)
@@ -743,7 +857,7 @@ func IsDir(path string) bool {
 	return false
 }
 
-//Unzip tar.gz file
+//Unzip tar.gz file, use for unpacking web.tar.gz for lazy people
 func ExtractTarGzipFile(filename string, outfile string) error {
 	f, err := os.Open(filename)
 	if err != nil {

mod/share/share.go

@@ -539,6 +539,7 @@ func (s *Manager) HandleShareAccess(w http.ResponseWriter, r *http.Request) {
 					//Check if the target fs require buffer
 					zippingSource := shareOption.FileRealPath
 					localBuff := ""
+					zippingSourceFsh := targetFsh
 					if targetFsh.RequireBuffer {
 						//Buffer all the required files for zipping
 						localBuff = filepath.Join(tmpFolder, uuid.NewV4().String(), filepath.Base(fileRuntimeAbsPath))
@@ -571,10 +572,11 @@ func (s *Manager) HandleShareAccess(w http.ResponseWriter, r *http.Request) {
 						})
 
 						zippingSource = localBuff
+						zippingSourceFsh = nil
 					}
 
 					//Build a filelist
-					err := filesystem.ArozZipFile([]string{zippingSource}, targetZipFilename, false)
+					err := filesystem.ArozZipFile([]*filesystem.FileSystemHandler{zippingSourceFsh}, []string{zippingSource}, nil, targetZipFilename, false)
 					if err != nil {
 						//Failed to create zip file
 						w.WriteHeader(http.StatusInternalServerError)
@@ -623,19 +625,17 @@ func (s *Manager) HandleShareAccess(w http.ResponseWriter, r *http.Request) {
 							fileSize, _ = targetFsh.GetDirctorySizeFromRealPath(file, false)
 						}
 
-						relPath, err := filepath.Rel(fileRuntimeAbsPath, file)
-						if err != nil {
-							relPath = ""
-						}
-
-						relPath = filepath.ToSlash(filepath.Clean(relPath))
-						relDir := filepath.ToSlash(filepath.Dir(relPath))
-
-						if relPath == "." {
+						relPath := strings.TrimPrefix(filepath.ToSlash(file), filepath.ToSlash(fileRuntimeAbsPath))
+						relDir := strings.TrimPrefix(filepath.ToSlash(filepath.Dir(file)), filepath.ToSlash(fileRuntimeAbsPath))
+						if relPath == "." || relPath == "" {
 							//The root file object. Skip this
 							return nil
 						}
 
+						if relDir == "" {
+							relDir = "."
+						}
+
 						treeList[relDir] = append(treeList[relDir], File{
 							Filename: filepath.Base(file),
 							RelPath:  filepath.ToSlash(relPath),

web/SystemAO/storage/fshedit.html

@@ -166,7 +166,6 @@
             <div class="networkfs" style="display:none;">
                 <div class="ui divider"></div>
                 <p>Security and Authentication</p>
-                <small>Leave Username / Password field empty for using the old config</small>
                 <div class="field">
                     <label>Username</label>
                     <input type="text" name="username" placeholder="">
@@ -175,7 +174,8 @@
                     <label>Password</label>
                     <input type="password" name="password" placeholder="">
                 </div>
-                <br>
+                <small>Leave Username / Password field empty for using the old config</small>
+                <br><br>
             </div>
             <button class="ui right floated button" onclick='handleCancel();'>Cancel</button>
             <button class="ui green right floated button" type="submit">Confirm</button>
@@ -272,7 +272,7 @@
         }
 
         function handleFileSystemTypeChange(fstype){
-            if (fstype == "webdav" || fstype == "ftp"){
+            if (fstype == "webdav" || fstype == "ftp" || fstype == "smb"){
                 $(".localfs").hide();
                 $(".networkfs").show();
             }else{
@@ -293,6 +293,7 @@
                 $(".networkfs").show();
             }
             $("#fstype").dropdown("set selected",option.filesystem);
+            handleFileSystemTypeChange(option.filesystem);
             $("input[name=mountdev]").val(option.mountdev);
             $("input[name=mountpt]").val(option.mountpt);
             if (option.automount == true){