arozos/mod/filesystem/fileOpr.go

package filesystem

/*
	File Operation Wrapper
	author: tobychui

	This is a module seperated from the aroz online file system script
	that allows cleaner code in the main logic handler of the aroz online system.

	WARNING! ALL FILE OPERATION USING THIS WRAPPER SHOULD PASS IN REALPATH
	DO NOT USE VIRTUAL PATH FOR ANY OPERATIONS WITH THIS WRAPPER
*/

import (
	"archive/tar"
	"archive/zip"
	"compress/flate"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"imuslab.com/arozos/mod/filesystem/hidden"

	archiver "github.com/mholt/archiver/v3"
)

//A basic file zipping function
func ZipFile(filelist []string, outputfile string, includeTopLevelFolder bool) error {
	z := archiver.Zip{
		CompressionLevel:       flate.DefaultCompression,
		MkdirAll:               true,
		SelectiveCompression:   true,
		OverwriteExisting:      false,
		ImplicitTopLevelFolder: includeTopLevelFolder,
	}

	err := z.Archive(filelist, outputfile)
	return err
}

//A basic file unzip function
func Unzip(source, destination string) error {
	archive, err := zip.OpenReader(source)
	if err != nil {
		return err
	}
	defer archive.Close()
	for _, file := range archive.Reader.File {
		reader, err := file.Open()
		if err != nil {
			return err
		}
		defer reader.Close()
		path := filepath.Join(destination, file.Name)

		err = os.MkdirAll(path, os.ModePerm)
		if err != nil {
			return err
		}

		if file.FileInfo().IsDir() {
			continue
		}

		err = os.Remove(path)
		if err != nil {
			return err
		}

		writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer writer.Close()
		_, err = io.Copy(writer, reader)
		if err != nil {
			return err
		}
	}
	return nil
}

//Aroz Unzip File with progress update function  (current filename / current file count / total file count / progress in percentage)
func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHandler func(string, int, int, float64)) error {
	//Gether the total number of files in all zip files
	totalFileCounts := 0
	unzippedFileCount := 0
	for _, srcFile := range filelist {
		archive, err := zip.OpenReader(srcFile)
		if err != nil {
			return err
		}

		totalFileCounts += len(archive.Reader.File)
		archive.Close()
	}

	//Start extracting
	for _, srcFile := range filelist {
		archive, err := zip.OpenReader(srcFile)
		if err != nil {
			return err
		}
		defer archive.Close()
		for _, file := range archive.Reader.File {
			reader, err := file.Open()
			if err != nil {
				return err
			}
			defer reader.Close()
			path := filepath.Join(outputfile, file.Name)

			parentFolder := path
			if !file.FileInfo().IsDir() {
				//Is file. Change target to its parent dir
				parentFolder = filepath.Dir(path)
			}

			err = os.MkdirAll(parentFolder, 0775)
			if err != nil {
				return err
			}

			if file.FileInfo().IsDir() {
				//Folder is created already be the steps above.
				//Update the progress
				unzippedFileCount++
				progressHandler(file.Name, unzippedFileCount, totalFileCounts, float64(unzippedFileCount)/float64(totalFileCounts)*100.0)
				continue
			}

			//Extrat and write to the target file
			writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
			if err != nil {
				return err
			}
			_, err = io.Copy(writer, reader)
			if err != nil {
				//Extraction failed. Remove this file if exists
				writer.Close()
				if FileExists(path) {
					os.Remove(path)
				}
				return err
			}
			writer.Close()

			//Update the progress
			unzippedFileCount++
			progressHandler(file.Name, unzippedFileCount, totalFileCounts, float64(unzippedFileCount)/float64(totalFileCounts)*100.0)
		}
	}

	return nil
}

//Aroz Zip File with progress update function (current filename / current file count / total file count / progress in percentage)
func ArozZipFileWithProgress(filelist []string, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64)) error {
	//Get the file count from the filelist
	totalFileCount := 0
	for _, srcpath := range filelist {
		if IsDir(srcpath) {
			filepath.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
				if !info.IsDir() {
					totalFileCount++
				}
				return nil
			})
		} else {
			totalFileCount++
		}

	}

	//Create the target zip file
	file, err := os.Create(outputfile)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	writer := zip.NewWriter(file)
	defer writer.Close()

	currentFileCount := 0
	for _, srcpath := range filelist {
		if IsDir(srcpath) {
			//This is a directory
			topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
			err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}

				if insideHiddenFolder(path) == true {
					//This is hidden file / folder. Skip this
					return nil
				}
				file, err := os.Open(path)
				if err != nil {
					return err
				}
				defer file.Close()

				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
				if includeTopLevelFolder {
					relativePath = topLevelFolderName + "/" + relativePath
				} else {
					relativePath = filepath.Base(srcpath) + "/" + relativePath
				}

				f, err := writer.Create(relativePath)
				if err != nil {
					return err
				}

				_, err = io.Copy(f, file)
				if err != nil {
					return err
				}

				//Update the zip progress
				currentFileCount++
				progressHandler(filepath.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
				return nil
			})

			if err != nil {
				return err
			}
		} else {
			//This is a file
			topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
			file, err := os.Open(srcpath)
			if err != nil {
				return err
			}
			defer file.Close()
			relativePath := filepath.Base(srcpath)
			if includeTopLevelFolder {
				relativePath = topLevelFolderName + "/" + relativePath
			}
			f, err := writer.Create(relativePath)
			if err != nil {
				return err
			}

			_, err = io.Copy(f, file)
			if err != nil {
				return err
			}

			//Update the zip progress
			currentFileCount++
			progressHandler(filepath.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))

		}
	}

	return nil
}

//ArOZ Zip FIle, but with no progress display
func ArozZipFile(filelist []string, outputfile string, includeTopLevelFolder bool) error {
	//Create the target zip file
	file, err := os.Create(outputfile)
	if err != nil {
		return err
	}
	defer file.Close()

	writer := zip.NewWriter(file)
	defer writer.Close()

	for _, srcpath := range filelist {
		if IsDir(srcpath) {
			//This is a directory
			topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
			err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.IsDir() {
					return nil
				}

				if insideHiddenFolder(path) == true {
					//This is hidden file / folder. Skip this
					return nil
				}
				file, err := os.Open(path)
				if err != nil {
					return err
				}
				defer file.Close()

				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
				if includeTopLevelFolder {
					relativePath = topLevelFolderName + "/" + relativePath
				} else {
					relativePath = filepath.Base(srcpath) + "/" + relativePath
				}

				f, err := writer.Create(relativePath)
				if err != nil {
					return err
				}

				_, err = io.Copy(f, file)
				if err != nil {
					return err
				}

				return nil
			})

			if err != nil {
				return err
			}
		} else {
			//This is a file
			topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
			file, err := os.Open(srcpath)
			if err != nil {
				return err
			}
			defer file.Close()
			relativePath := filepath.Base(srcpath)
			if includeTopLevelFolder {
				relativePath = topLevelFolderName + "/" + relativePath
			}
			f, err := writer.Create(relativePath)
			if err != nil {
				return err
			}

			_, err = io.Copy(f, file)
			if err != nil {
				return err
			}

		}
	}

	return nil
}

func insideHiddenFolder(path string) bool {
	FileIsHidden, err := hidden.IsHidden(path, true)
	if err != nil {
		//Read error. Maybe permission issue, assuem is hidden
		return true
	}
	return FileIsHidden
}

func ViewZipFile(filepath string) ([]string, error) {
	z := archiver.Zip{}
	filelist := []string{}
	err := z.Walk(filepath, func(f archiver.File) error {
		filelist = append(filelist, f.Name())
		return nil
	})

	return filelist, err
}

func FileCopy(src string, dest string, mode string, progressUpdate func(int, string)) error {
	srcRealpath, _ := filepath.Abs(src)
	destRealpath, _ := filepath.Abs(dest)
	if IsDir(src) && strings.Contains(filepath.ToSlash(destRealpath)+"/", filepath.ToSlash(srcRealpath)+"/") {
		//Recursive operation. Reject
		return errors.New("Recursive copy operation.")
	}

	//Check if the copy destination file already have an identical file
	copiedFilename := filepath.Base(src)

	if fileExists(filepath.Join(dest, filepath.Base(src))) {
		if mode == "" {
			//Do not specific file exists principle
			return errors.New("Destination file already exists.")

		} else if mode == "skip" {
			//Skip this file
			return nil
		} else if mode == "overwrite" {
			//Continue with the following code
			//Check if the copy and paste dest are identical
			if filepath.ToSlash(filepath.Clean(src)) == filepath.ToSlash(filepath.Clean(filepath.Join(dest, filepath.Base(src)))) {
				//Source and target identical. Cannot overwrite.
				return errors.New("Source and destination paths are identical.")

			}

		} else if mode == "keep" {
			//Keep the file but saved with 'Copy' suffix
			newFilename := strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy" + filepath.Ext(src)
			//Check if the newFilename already exists. If yes, continue adding suffix
			duplicateCounter := 0
			for fileExists(filepath.Join(dest, newFilename)) {
				duplicateCounter++
				newFilename = strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy(" + strconv.Itoa(duplicateCounter) + ")" + filepath.Ext(src)
				if duplicateCounter > 1024 {
					//Maxmium loop encountered. For thread safty, terminate here
					return errors.New("Too many copies of identical files.")

				}
			}
			copiedFilename = newFilename
		} else {
			//This exists opr not supported.
			return errors.New("Unknown file exists rules given.")

		}

	}

	//Fix the lacking / at the end if true
	if dest[len(dest)-1:] != "/" {
		dest = dest + "/"
	}

	//Ready to move. Check if both folder are located in the same root devices. If not, use copy and delete method.
	if IsDir(src) {
		//Source file is directory. CopyFolder

		realDest := filepath.Join(dest, copiedFilename)

		//err := dircpy.Copy(src, realDest)

		err := dirCopy(src, realDest, progressUpdate)
		if err != nil {
			return err

		}

	} else {
		//Source is file only. Copy file.
		realDest := filepath.Join(dest, copiedFilename)
		source, err := os.Open(src)
		if err != nil {
			return err

		}

		destination, err := os.Create(realDest)
		if err != nil {
			return err
		}

		if progressUpdate != nil {
			//Set progress to 100, leave it to upper level abstraction to handle
			progressUpdate(100, filepath.Base(realDest))
		}

		_, err = io.Copy(destination, source)
		if err != nil {
			return err
		}
		source.Close()
		destination.Close()

	}
	return nil
}

func FileMove(src string, dest string, mode string, fastMove bool, progressUpdate func(int, string)) error {
	srcRealpath, _ := filepath.Abs(src)
	destRealpath, _ := filepath.Abs(dest)
	if IsDir(src) && strings.Contains(filepath.ToSlash(destRealpath)+"/", filepath.ToSlash(srcRealpath)+"/") {
		//Recursive operation. Reject
		return errors.New("Recursive move operation.")
	}

	if !fileExists(dest) {
		if fileExists(filepath.Dir(dest)) {
			//User pass in the whole path for the folder. Report error usecase.
			return errors.New("Dest location should be an existing folder instead of the full path of the moved file.")
		}
		return errors.New("Dest folder not found")
	}
	//Fix the lacking / at the end if true
	if dest[len(dest)-1:] != "/" {
		dest = dest + "/"
	}

	//Check if the target file already exists.
	movedFilename := filepath.Base(src)

	if fileExists(filepath.Join(dest, filepath.Base(src))) {
		//Handle cases where file already exists
		if mode == "" {
			//Do not specific file exists principle
			return errors.New("Destination file already exists.")
		} else if mode == "skip" {
			//Skip this file
			return nil
		} else if mode == "overwrite" {
			//Continue with the following code
			//Check if the copy and paste dest are identical
			if filepath.ToSlash(filepath.Clean(src)) == filepath.ToSlash(filepath.Clean(filepath.Join(dest, filepath.Base(src)))) {
				//Source and target identical. Cannot overwrite.
				return errors.New("Source and destination paths are identical.")
			}

		} else if mode == "keep" {
			//Keep the file but saved with 'Copy' suffix
			newFilename := strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy" + filepath.Ext(src)
			//Check if the newFilename already exists. If yes, continue adding suffix
			duplicateCounter := 0
			for fileExists(filepath.Join(dest, newFilename)) {
				duplicateCounter++
				newFilename = strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy(" + strconv.Itoa(duplicateCounter) + ")" + filepath.Ext(src)
				if duplicateCounter > 1024 {
					//Maxmium loop encountered. For thread safty, terminate here
					return errors.New("Too many copies of identical files.")
				}
			}
			movedFilename = newFilename
		} else {
			//This exists opr not supported.
			return errors.New("Unknown file exists rules given.")
		}
	}

	if fastMove {
		//Ready to move with the quick rename method
		realDest := dest + movedFilename
		err := os.Rename(src, realDest)
		if err == nil {
			//Fast move success
			return nil
		}

		//Fast move failed. Back to the original copy and move method
	}

	//Ready to move. Check if both folder are located in the same root devices. If not, use copy and delete method.
	if IsDir(src) {
		//Source file is directory. CopyFolder
		realDest := dest + movedFilename
		//err := dircpy.Copy(src, realDest)

		err := dirCopy(src, realDest, progressUpdate)
		if err != nil {
			return err
		} else {
			//Move completed. Remove source file.
			os.RemoveAll(src)
			return nil
		}

	} else {
		//Source is file only. Copy file.
		realDest := dest + movedFilename
		/*
			Updates 20-10-2020, replaced io.Copy to BufferedLargeFileCopy
			Legacy code removed.
		*/

		//Update the progress
		if progressUpdate != nil {
			progressUpdate(100, filepath.Base(src))
		}

		err := BufferedLargeFileCopy(src, realDest, 8192)
		if err != nil {
			log.Println("BLFC error: ", err.Error())
			return err
		}

		//Delete the source file after copy
		err = os.Remove(src)
		counter := 0
		for err != nil {
			//Sometime Windows need this to prevent windows caching bring problems to file remove
			time.Sleep(1 * time.Second)
			os.Remove(src)
			counter++
			log.Println("Retrying to remove file: " + src)
			if counter > 10 {
				return errors.New("Source file remove failed.")
			}
		}

	}

	return nil
}

//Copy a given directory, with no progress udpate
func CopyDir(src string, dest string) error {
	return dirCopy(src, dest, func(progress int, name string) {})
}

//Replacment of the legacy dirCopy plugin with filepath.Walk function. Allowing real time progress update to front end
func dirCopy(src string, realDest string, progressUpdate func(int, string)) error {

	//Get the total file counts
	totalFileCounts := int64(0)
	filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			//Updates 22 April 2021, chnaged from file count to file size for progress update
			//totalFileCounts++
			totalFileCounts += info.Size()
		}
		return nil
	})

	//Make the destinaton directory
	if !fileExists(realDest) {
		os.Mkdir(realDest, 0755)
	}

	//Start moving
	fileCounter := int64(0)

	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		srcAbs, _ := filepath.Abs(src)
		pathAbs, _ := filepath.Abs(path)

		var folderRootRelative string = strings.Replace(pathAbs, srcAbs, "", 1)
		if folderRootRelative == "" {
			return nil
		}

		if info.IsDir() {
			//Mkdir base on relative path
			return os.MkdirAll(filepath.Join(realDest, folderRootRelative), 0755)
		} else {
			//fileCounter++
			fileCounter += info.Size()
			//Move file base on relative path
			fileSrc := filepath.ToSlash(filepath.Join(filepath.Clean(src), folderRootRelative))
			fileDest := filepath.ToSlash(filepath.Join(filepath.Clean(realDest), folderRootRelative))

			//Update move progress
			if progressUpdate != nil {
				progressUpdate(int(float64(fileCounter)/float64(totalFileCounts)*100), filepath.Base(fileSrc))
			}

			//Move the file using BLFC
			err := BufferedLargeFileCopy(fileSrc, fileDest, 8192)
			if err != nil {
				//Ignore and continue
				log.Println("BLFC Error:", err.Error())
				return nil
			}
			/*
				//Move fiel using IO Copy
				err := BasicFileCopy(fileSrc, fileDest)
				if err != nil {
					log.Println("Basic Copy Error: ", err.Error())
					return nil
				}

			*/
		}
		return nil
	})

	return err
}

func BasicFileCopy(src string, dst string) error {
	sourceFileStat, err := os.Stat(src)
	if err != nil {
		return err
	}

	if !sourceFileStat.Mode().IsRegular() {
		return fmt.Errorf("%s is not a regular file", src)
	}

	source, err := os.Open(src)
	if err != nil {
		return err
	}
	defer source.Close()

	destination, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer destination.Close()
	_, err = io.Copy(destination, source)
	return err
}

//Use for copying large file using buffering method. Allowing copying large file with little RAM
func BufferedLargeFileCopy(src string, dst string, BUFFERSIZE int64) error {
	sourceFileStat, err := os.Stat(src)
	if err != nil {
		return err
	}

	if !sourceFileStat.Mode().IsRegular() {
		return errors.New("Invalid file source")
	}

	source, err := os.Open(src)
	if err != nil {
		return err
	}

	destination, err := os.Create(dst)
	if err != nil {
		return err
	}

	buf := make([]byte, BUFFERSIZE)
	for {
		n, err := source.Read(buf)
		if err != nil && err != io.EOF {
			source.Close()
			destination.Close()
			return err
		}
		if n == 0 {
			source.Close()
			destination.Close()
			break
		}

		if _, err := destination.Write(buf[:n]); err != nil {
			source.Close()
			destination.Close()
			return err
		}
	}
	return nil
}

func IsDir(path string) bool {
	if fileExists(path) == false {
		return false
	}
	fi, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
		return false
	}
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return true
	case mode.IsRegular():
		return false
	}
	return false
}

//Unzip tar.gz file
func ExtractTarGzipFile(filename string, outfile string) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}

	err = ExtractTarGzipByStream(filepath.Clean(outfile), f, true)
	if err != nil {
		return err
	}

	return f.Close()
}
func ExtractTarGzipByStream(basedir string, gzipStream io.Reader, onErrorResumeNext bool) error {
	uncompressedStream, err := gzip.NewReader(gzipStream)
	if err != nil {
		return err
	}

	tarReader := tar.NewReader(uncompressedStream)

	for {
		header, err := tarReader.Next()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		switch header.Typeflag {
		case tar.TypeDir:
			err := os.Mkdir(header.Name, 0755)
			if err != nil {
				if !onErrorResumeNext {
					return err
				}

			}
		case tar.TypeReg:
			outFile, err := os.Create(filepath.Join(basedir, header.Name))
			if err != nil {
				if !onErrorResumeNext {
					return err
				}
			}
			_, err = io.Copy(outFile, tarReader)
			if err != nil {
				if !onErrorResumeNext {
					return err
				}
			}
			outFile.Close()

		default:
			//Unknown filetype, continue

		}

	}
	return nil
}