package filesystem /* ArOZ Online File System Handler Wrappers author: tobychui This is a module design to do the followings 1. Mount / Create a fs when open 2. Provide the basic function and operations of a file system type 3. THIS MODULE **SHOULD NOT CONTAIN** CROSS FILE SYSTEM TYPE OPERATIONS */ import ( "crypto/md5" "encoding/hex" "errors" "fmt" "io" "io/fs" "log" "os" "path/filepath" "strconv" "strings" "time" uuid "github.com/satori/go.uuid" "imuslab.com/arozos/mod/filesystem/abstractions/ftpfs" "imuslab.com/arozos/mod/filesystem/abstractions/localfs" sftpfs "imuslab.com/arozos/mod/filesystem/abstractions/sftpfs" "imuslab.com/arozos/mod/filesystem/abstractions/smbfs" "imuslab.com/arozos/mod/filesystem/abstractions/webdavfs" "imuslab.com/arozos/mod/filesystem/arozfs" "imuslab.com/arozos/mod/utils" ) //Options for creating new file system handler /* type FileSystemOpeningOptions struct{ Name string `json:"name"` //Display name of this device Uuid string `json:"uuid"` //UUID of this device, e.g. S1 Path string `json:"path"` //Path for the storage root Access string `json:"access,omitempty"` //Access right, allow {readonly, readwrite} Hierarchy string `json:"hierarchy"` //Folder hierarchy, allow {public, user} Automount bool `json:"automount"` //Automount this device if exists Filesystem string `json:"filesystem,omitempty"` //Support {"ext4","ext2", "ext3", "fat", "vfat", "ntfs"} Mountdev string `json:"mountdev,omitempty"` //Device file (e.g. /dev/sda1) Mountpt string `json:"mountpt,omitempty"` //Device mount point (e.g. /media/storage1) } */ /* An interface for storing data related to a specific hierarchy settings. Example like the account information of network drive, backup mode of backup drive etc */ type HierarchySpecificConfig interface{} type FileSystemAbstraction interface { //Fundamental Functions Chmod(string, os.FileMode) error Chown(string, int, int) error Chtimes(string, time.Time, time.Time) error Create(string) (arozfs.File, error) Mkdir(string, os.FileMode) error MkdirAll(string, os.FileMode) error Name() string Open(string) (arozfs.File, error) OpenFile(string, int, os.FileMode) (arozfs.File, error) Remove(string) error RemoveAll(string) error Rename(string, string) error Stat(string) (os.FileInfo, error) Close() error //Utils Functions VirtualPathToRealPath(string, string) (string, error) RealPathToVirtualPath(string, string) (string, error) FileExists(string) bool IsDir(string) bool Glob(string) ([]string, error) GetFileSize(string) int64 GetModTime(string) (int64, error) WriteFile(string, []byte, os.FileMode) error ReadFile(string) ([]byte, error) ReadDir(string) ([]fs.DirEntry, error) WriteStream(string, io.Reader, os.FileMode) error ReadStream(string) (io.ReadCloser, error) Walk(string, filepath.WalkFunc) error Heartbeat() error } // Runtime persistence config, use to pass through startup paramters related to file system handlers type RuntimePersistenceConfig struct { LocalBufferPath string } // System Handler for returing type FileSystemHandler struct { Name string UUID string Path string Hierarchy string HierarchyConfig HierarchySpecificConfig ReadOnly bool RequireBuffer bool //Set this to true if the fsh do not provide file header functions like Open() or Create(), require WriteStream() and ReadStream() Parentuid string InitiationTime int64 FileSystemAbstraction FileSystemAbstraction Filesystem string StartOptions FileSystemOption RuntimePersistenceConfig RuntimePersistenceConfig Closed bool } // Create a list of file system handler from the given json content func NewFileSystemHandlersFromJSON(jsonContent []byte, runtimePersistenceConfig RuntimePersistenceConfig) ([]*FileSystemHandler, error) { //Generate a list of handler option from json file options, err := loadConfigFromJSON(jsonContent) if err != nil { return []*FileSystemHandler{}, err } resultingHandlers := []*FileSystemHandler{} for _, option := range options { thisHandler, err := NewFileSystemHandler(option, runtimePersistenceConfig) if err != nil { log.Println("[File System] Failed to create system handler for " + option.Name) //log.Println(err.Error()) continue } resultingHandlers = append(resultingHandlers, thisHandler) } return resultingHandlers, nil } // Create a new file system handler with the given config func NewFileSystemHandler(option FileSystemOption, RuntimePersistenceConfig RuntimePersistenceConfig) (*FileSystemHandler, error) { fstype := strings.ToLower(option.Filesystem) if inSlice([]string{"ext4", "ext2", "ext3", "fat", "vfat", "ntfs"}, fstype) || fstype == "" { //Check if the target fs require mounting if option.Automount == true { err := MountDevice(option.Mountpt, option.Mountdev, option.Filesystem) if err != nil { return &FileSystemHandler{}, err } } //Check if the path exists if !FileExists(option.Path) { return &FileSystemHandler{}, errors.New("Mount point not exists!") } //Handle Hierarchy branching if option.Hierarchy == "user" { //Create user hierarchy for this virtual device os.MkdirAll(filepath.ToSlash(filepath.Clean(option.Path))+"/users", 0755) } rootpath := filepath.ToSlash(filepath.Clean(option.Path)) + "/" return &FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: filepath.ToSlash(filepath.Clean(option.Path)) + "/", ReadOnly: option.Access == arozfs.FsReadOnly, RequireBuffer: false, Hierarchy: option.Hierarchy, HierarchyConfig: DefaultEmptyHierarchySpecificConfig, InitiationTime: time.Now().Unix(), FileSystemAbstraction: localfs.NewLocalFileSystemAbstraction(option.Uuid, rootpath, option.Hierarchy, option.Access == arozfs.FsReadOnly), Filesystem: fstype, StartOptions: option, RuntimePersistenceConfig: RuntimePersistenceConfig, Closed: false, }, nil } else if fstype == "webdav" { //WebDAV. Create an object and mount it root := option.Path user := option.Username password := option.Password webdavfs, err := webdavfs.NewWebDAVMount(option.Uuid, option.Hierarchy, root, user, password) if err != nil { return nil, err } return &FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: option.Path, ReadOnly: option.Access == arozfs.FsReadOnly, RequireBuffer: true, Hierarchy: option.Hierarchy, HierarchyConfig: nil, InitiationTime: time.Now().Unix(), FileSystemAbstraction: webdavfs, Filesystem: fstype, StartOptions: option, Closed: false, }, nil } else if fstype == "smb" { //SMB. Create an object and mount it pathChunks := strings.Split(strings.ReplaceAll(option.Path, "\\", "/"), "/") if len(pathChunks) < 2 { log.Println("[File System] Invalid configured smb filepath: Path format not matching [ip_addr]:[port]/[root_share path]") return nil, errors.New("Invalid configured smb filepath: Path format not matching [ip_addr]:[port]/[root_share path]") } ipAddr := pathChunks[0] rootShare := strings.Join(pathChunks[1:], "/") user := option.Username password := option.Password smbfs, err := smbfs.NewServerMessageBlockFileSystemAbstraction( option.Uuid, option.Hierarchy, ipAddr, rootShare, user, password, ) if err != nil { return nil, err } thisFsh := FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: option.Path, ReadOnly: option.Access == arozfs.FsReadOnly, RequireBuffer: false, Hierarchy: option.Hierarchy, HierarchyConfig: nil, InitiationTime: time.Now().Unix(), FileSystemAbstraction: smbfs, Filesystem: fstype, StartOptions: option, Closed: false, } return &thisFsh, nil } else if fstype == "sftp" { //SFTP pathChunks := strings.Split(strings.ReplaceAll(option.Path, "\\", "/"), "/") ipAddr := pathChunks[0] port := 22 if strings.Contains(ipAddr, ":") { //Custom port defined ipChunks := strings.Split(ipAddr, ":") ipAddr = ipChunks[0] p, err := strconv.Atoi(ipChunks[1]) if err == nil { port = p } } rootShare := pathChunks[1:] user := option.Username password := option.Password sftpfs, err := sftpfs.NewSFTPFileSystemAbstraction( option.Uuid, option.Hierarchy, ipAddr, port, "/"+strings.Join(rootShare, "/"), user, password, ) if err != nil { fmt.Println(err.Error()) return nil, err } thisFsh := FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: option.Path, ReadOnly: option.Access == arozfs.FsReadOnly, RequireBuffer: false, Hierarchy: option.Hierarchy, HierarchyConfig: nil, InitiationTime: time.Now().Unix(), FileSystemAbstraction: sftpfs, Filesystem: fstype, StartOptions: option, Closed: false, } return &thisFsh, nil } else if fstype == "ftp" { ftpfs, err := ftpfs.NewFTPFSAbstraction(option.Uuid, option.Hierarchy, option.Path, option.Username, option.Password) if err != nil { return nil, err } return &FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: option.Path, ReadOnly: option.Access == arozfs.FsReadOnly, RequireBuffer: true, Hierarchy: option.Hierarchy, HierarchyConfig: nil, InitiationTime: time.Now().Unix(), FileSystemAbstraction: ftpfs, Filesystem: fstype, StartOptions: option, Closed: false, }, nil } else if option.Filesystem == "virtual" { //Virtual filesystem, deprecated log.Println("[File System] Deprecated file system type: Virtual") } return nil, errors.New("Not supported file system: " + fstype) } // Check if a fsh is a network drive func (fsh *FileSystemHandler) IsNetworkDrive() bool { return arozfs.IsNetworkDrive(fsh.Filesystem) } // Check if a fsh is a local disk drive func (fsh *FileSystemHandler) IsLocalDrive() bool { //Check if network drive if arozfs.IsNetworkDrive(fsh.Filesystem) { return false } //Check if mounted locally if _, err := os.Stat(fsh.Path); os.IsNotExist(err) { return false } return true } // Buffer a file to local tmp folder and return the tmp location for further processing func (fsh *FileSystemHandler) BufferRemoteToLocal(rpath string) (string, error) { //Check if the remote file exists fsa := fsh.FileSystemAbstraction if !fsa.FileExists(rpath) { //Target file not exists return "", errors.New("target file not exists on remote") } //Check if the remote is a file (not support dir) if fsa.IsDir(rpath) { return "", errors.New("directory cannot be buffered to local") } //Get the tmp folder directory tmpdir := fsh.RuntimePersistenceConfig.LocalBufferPath if !utils.FileExists(tmpdir) { //Create the tmp dir if not exists os.MkdirAll(tmpdir, 0775) } //Generate a filename for the buffer file tmpFilename := uuid.NewV4().String() tmpFilepath := arozfs.ToSlash(filepath.Join(tmpdir, tmpFilename+filepath.Ext(rpath))) //Copy the file from remote location to local src, err := fsh.FileSystemAbstraction.ReadStream(rpath) if err != nil { return "", err } defer src.Close() dest, err := os.OpenFile(tmpFilepath, os.O_CREATE|os.O_WRONLY, 0777) if err != nil { return "", errors.New("unable to write to buffer location: " + err.Error()) } defer dest.Close() _, err = io.Copy(dest, src) if err != nil { return "", errors.New("file buffer failed: " + err.Error()) } //Return the buffered filepath on local disk return tmpFilepath, nil } func (fsh *FileSystemHandler) IsRootOf(vpath string) bool { return strings.HasPrefix(vpath, fsh.UUID+":") } func (fsh *FileSystemHandler) GetUniquePathHash(vpath string, username string) (string, error) { fshAbs := fsh.FileSystemAbstraction rpath := "" if strings.Contains(vpath, ":/") { r, err := fshAbs.VirtualPathToRealPath(vpath, username) if err != nil { return "", err } rpath = filepath.ToSlash(r) } else { //Passed in realpath as vpath. rpath = vpath } hash := md5.Sum([]byte(fsh.UUID + "_" + rpath)) return hex.EncodeToString(hash[:]), nil } func (fsh *FileSystemHandler) GetDirctorySizeFromRealPath(rpath string, includeHidden bool) (int64, int) { var size int64 = 0 var fileCount int = 0 err := fsh.FileSystemAbstraction.Walk(rpath, func(thisFilename string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { if includeHidden { //append all into the file count and size size += info.Size() fileCount++ } else { //Check if this is hidden if !IsInsideHiddenFolder(thisFilename) { size += info.Size() fileCount++ } } } return nil }) if err != nil { return 0, fileCount } return size, fileCount } func (fsh *FileSystemHandler) GetDirctorySizeFromVpath(vpath string, username string, includeHidden bool) (int64, int) { realpath, _ := fsh.FileSystemAbstraction.VirtualPathToRealPath(vpath, username) return fsh.GetDirctorySizeFromRealPath(realpath, includeHidden) } // Reload the target file system abstraction func (fsh *FileSystemHandler) ReloadFileSystelAbstraction() error { log.Println("[File System] Reloading File System Abstraction for " + fsh.Name) //Load the start option for this fsh originalStartOption := fsh.StartOptions runtimePersistenceConfig := fsh.RuntimePersistenceConfig //Close the file system handler fsh.Close() //Give it a few ms to do physical disk stuffs time.Sleep(800 * time.Millisecond) //Generate a new fsh from original start option reloadedFsh, err := NewFileSystemHandler(originalStartOption, runtimePersistenceConfig) if err != nil { return err } //Overwrite the pointers to target fsa fsh.FileSystemAbstraction = reloadedFsh.FileSystemAbstraction fsh.Closed = false return nil } // Close an openeded File System func (fsh *FileSystemHandler) Close() { //Set the close flag to true so others function wont access it fsh.Closed = true //Close the file system object err := fsh.FileSystemAbstraction.Close() if err != nil { log.Println("[File System] Unable to close File System Abstraction for Handler: " + fsh.UUID + ". Skipping.") } } // Helper function func inSlice(slice []string, val string) bool { for _, item := range slice { if item == val { return true } } return false } func FileExists(filename string) bool { _, err := os.Stat(filename) if os.IsNotExist(err) { return false } return true }