package filesystem /* ArOZ Online File System Handler Wrappers author: tobychui This is a module design to do the followings 1. Mount / Create a fs when open 2. Provide the basic function and operations of a file system type 3. THIS MODULE **SHOULD NOT CONTAIN** CROSS FILE SYSTEM TYPE OPERATIONS */ import ( "crypto/md5" "encoding/hex" "errors" "io" "log" "os" "path/filepath" "strings" "time" db "imuslab.com/arozos/mod/database" "imuslab.com/arozos/mod/filesystem/abstractions/localfs" "imuslab.com/arozos/mod/filesystem/abstractions/smbfs" "imuslab.com/arozos/mod/filesystem/abstractions/webdavfs" "imuslab.com/arozos/mod/filesystem/fsdef" ) //Options for creating new file system handler /* type FileSystemOpeningOptions struct{ Name string `json:"name"` //Display name of this device Uuid string `json:"uuid"` //UUID of this device, e.g. S1 Path string `json:"path"` //Path for the storage root Access string `json:"access,omitempty"` //Access right, allow {readonly, readwrite} Hierarchy string `json:"hierarchy"` //Folder hierarchy, allow {public, user} Automount bool `json:"automount"` //Automount this device if exists Filesystem string `json:"filesystem,omitempty"` //Support {"ext4","ext2", "ext3", "fat", "vfat", "ntfs"} Mountdev string `json:"mountdev,omitempty"` //Device file (e.g. /dev/sda1) Mountpt string `json:"mountpt,omitempty"` //Device mount point (e.g. /media/storage1) } */ /* An interface for storing data related to a specific hierarchy settings. Example like the account information of network drive, backup mode of backup drive etc */ type HierarchySpecificConfig interface{} type FileSystemAbstraction interface { //Fundemental Functions Chmod(string, os.FileMode) error Chown(string, int, int) error Chtimes(string, time.Time, time.Time) error Create(string) (*os.File, error) Mkdir(string, os.FileMode) error MkdirAll(string, os.FileMode) error Name() string Open(string) (*os.File, error) OpenFile(string, int, os.FileMode) (*os.File, error) Remove(string) error RemoveAll(string) error Rename(string, string) error Stat(string) (os.FileInfo, error) Close() error //Utils Functions VirtualPathToRealPath(string, string) (string, error) RealPathToVirtualPath(string, string) (string, error) FileExists(string) bool IsDir(string) bool Glob(string) ([]string, error) GetFileSize(string) int64 GetModTime(string) (int64, error) WriteFile(string, []byte, os.FileMode) error ReadFile(string) ([]byte, error) WriteStream(string, io.Reader, os.FileMode) error ReadStream(string) (io.ReadCloser, error) Walk(string, filepath.WalkFunc) error } //System Handler for returing type FileSystemHandler struct { Name string UUID string Path string Hierarchy string HierarchyConfig HierarchySpecificConfig ReadOnly bool RequireBuffer bool //Set this to true if the fsh do not provide file header functions like Open() or Create(), require WriteStream() and ReadStream() Parentuid string InitiationTime int64 FilesystemDatabase *db.Database FileSystemAbstraction FileSystemAbstraction Filesystem string Closed bool } //Create a list of file system handler from the given json content func NewFileSystemHandlersFromJSON(jsonContent []byte) ([]*FileSystemHandler, error) { //Generate a list of handler option from json file options, err := loadConfigFromJSON(jsonContent) if err != nil { return []*FileSystemHandler{}, err } resultingHandlers := []*FileSystemHandler{} for _, option := range options { thisHandler, err := NewFileSystemHandler(option) if err != nil { log.Println("[File System] Failed to create system handler for " + option.Name) //log.Println(err.Error()) continue } resultingHandlers = append(resultingHandlers, thisHandler) } return resultingHandlers, nil } //Create a new file system handler with the given config func NewFileSystemHandler(option FileSystemOption) (*FileSystemHandler, error) { fstype := strings.ToLower(option.Filesystem) if inSlice([]string{"ext4", "ext2", "ext3", "fat", "vfat", "ntfs"}, fstype) || fstype == "" { //Check if the target fs require mounting if option.Automount == true { err := MountDevice(option.Mountpt, option.Mountdev, option.Filesystem) if err != nil { return &FileSystemHandler{}, err } } //Check if the path exists if !FileExists(option.Path) { return &FileSystemHandler{}, errors.New("Mount point not exists!") } //Handle Hierarchy branching if option.Hierarchy == "user" { //Create user hierarchy for this virtual device os.MkdirAll(filepath.ToSlash(filepath.Clean(option.Path))+"/users", 0755) } //Create the fsdb for this handler var fsdb *db.Database = nil dbp, err := db.NewDatabase(filepath.ToSlash(filepath.Join(filepath.Clean(option.Path), "aofs.db")), false) if err != nil { if option.Access != fsdef.FsReadOnly { log.Println("[File System] Invalid config: Trying to mount a read only path as read-write mount point. Changing " + option.Name + " mount point to READONLY.") option.Access = fsdef.FsReadOnly } } else { fsdb = dbp } rootpath := filepath.ToSlash(filepath.Clean(option.Path)) + "/" return &FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: filepath.ToSlash(filepath.Clean(option.Path)) + "/", ReadOnly: option.Access == fsdef.FsReadOnly, RequireBuffer: false, Hierarchy: option.Hierarchy, HierarchyConfig: DefaultEmptyHierarchySpecificConfig, InitiationTime: time.Now().Unix(), FilesystemDatabase: fsdb, FileSystemAbstraction: localfs.NewLocalFileSystemAbstraction(option.Uuid, rootpath, option.Hierarchy, option.Access == fsdef.FsReadOnly), Filesystem: fstype, Closed: false, }, nil } else if fstype == "webdav" { //WebDAV. Create an object and mount it root := option.Path user := option.Username password := option.Password webdavfs, err := webdavfs.NewWebDAVMount(option.Uuid, option.Hierarchy, root, user, password) if err != nil { return nil, err } return &FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: option.Path, ReadOnly: option.Access == fsdef.FsReadOnly, RequireBuffer: true, Hierarchy: option.Hierarchy, HierarchyConfig: nil, InitiationTime: time.Now().Unix(), FilesystemDatabase: nil, FileSystemAbstraction: webdavfs, Filesystem: fstype, Closed: false, }, nil } else if fstype == "smb" { //WebDAV. Create an object and mount it pathChunks := strings.Split(strings.ReplaceAll(option.Path, "\\", "/"), "/") if len(pathChunks) != 2 { return nil, errors.New("Invalid configured smb filepath: Path format not matching [ip_addr]:[port]/[root_share]") } ipAddr := pathChunks[0] rootShare := pathChunks[1] user := option.Username password := option.Password smbfs, err := smbfs.NewServerMessageBlockFileSystemAbstraction(option.Uuid, option.Hierarchy, ipAddr, rootShare, user, password) if err != nil { return nil, err } return &FileSystemHandler{ Name: option.Name, UUID: option.Uuid, Path: option.Path, ReadOnly: option.Access == fsdef.FsReadOnly, RequireBuffer: true, Hierarchy: option.Hierarchy, HierarchyConfig: nil, InitiationTime: time.Now().Unix(), FilesystemDatabase: nil, FileSystemAbstraction: smbfs, Filesystem: fstype, Closed: false, }, nil } else if option.Filesystem == "virtual" { //Virtual filesystem, deprecated log.Println("[File System] Deprecated file system type: Virtual") } return nil, errors.New("Not supported file system: " + fstype) } //Check if a fsh is virtual (e.g. Network or fs Abstractions that cannot be listed with normal fs API) /* func (fsh *FileSystemHandler) IsVirtual() bool { if fsh.Hierarchy == "virtual" || fsh.Filesystem == "webdav" { //Check if the config return placeholder c, ok := fsh.HierarchyConfig.(EmptyHierarchySpecificConfig) if ok && c.HierarchyType == "placeholder" { //Real file system. return false } //Do more checking here if needed return true } return false } */ func (fsh *FileSystemHandler) IsRootOf(vpath string) bool { return strings.HasPrefix(vpath, fsh.UUID+":") } func (fsh *FileSystemHandler) GetUniquePathHash(vpath string, username string) (string, error) { fshAbs := fsh.FileSystemAbstraction rpath := "" if strings.Contains(vpath, ":/") { r, err := fshAbs.VirtualPathToRealPath(vpath, username) if err != nil { return "", err } rpath = filepath.ToSlash(r) } else { //Passed in realpath as vpath. rpath = vpath } hash := md5.Sum([]byte(fsh.UUID + "_" + rpath)) return hex.EncodeToString(hash[:]), nil } func (fsh *FileSystemHandler) GetDirctorySizeFromRealPath(rpath string, includeHidden bool) (int64, int) { var size int64 = 0 var fileCount int = 0 err := fsh.FileSystemAbstraction.Walk(rpath, func(thisFilename string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { if includeHidden { //append all into the file count and size size += info.Size() fileCount++ } else { //Check if this is hidden if !IsInsideHiddenFolder(thisFilename) { size += info.Size() fileCount++ } } } return nil }) if err != nil { return 0, fileCount } return size, fileCount } func (fsh *FileSystemHandler) GetDirctorySizeFromVpath(vpath string, username string, includeHidden bool) (int64, int) { realpath, _ := fsh.FileSystemAbstraction.VirtualPathToRealPath(vpath, username) return fsh.GetDirctorySizeFromRealPath(realpath, includeHidden) } /* File Record Related Functions fsh database that keep track of which files is owned by whom */ //Create a file ownership record func (fsh *FileSystemHandler) CreateFileRecord(rpath string, owner string) error { if fsh.FilesystemDatabase == nil { //Not supported file system type return errors.New("Not supported filesystem type") } fsh.FilesystemDatabase.NewTable("owner") fsh.FilesystemDatabase.Write("owner", "owner/"+rpath, owner) return nil } //Read the owner of a file func (fsh *FileSystemHandler) GetFileRecord(rpath string) (string, error) { if fsh.FilesystemDatabase == nil { //Not supported file system type return "", errors.New("Not supported filesystem type") } fsh.FilesystemDatabase.NewTable("owner") if fsh.FilesystemDatabase.KeyExists("owner", "owner/"+rpath) { owner := "" fsh.FilesystemDatabase.Read("owner", "owner/"+rpath, &owner) return owner, nil } else { return "", errors.New("Owner not exists") } } //Delete a file ownership record func (fsh *FileSystemHandler) DeleteFileRecord(rpath string) error { if fsh.FilesystemDatabase == nil { //Not supported file system type return errors.New("Not supported filesystem type") } fsh.FilesystemDatabase.NewTable("owner") if fsh.FilesystemDatabase.KeyExists("owner", "owner/"+rpath) { fsh.FilesystemDatabase.Delete("owner", "owner/"+rpath) } return nil } //Close an openeded File System func (fsh *FileSystemHandler) Close() { //Close the fsh database if fsh.FilesystemDatabase != nil { fsh.FilesystemDatabase.Close() } //Close the file system object err := fsh.FileSystemAbstraction.Close() if err != nil { log.Println("[File System] Unable to close File System Abstraction for Handler: " + fsh.UUID + ". Skipping.") } } //Helper function func inSlice(slice []string, val string) bool { for _, item := range slice { if item == val { return true } } return false } func FileExists(filename string) bool { _, err := os.Stat(filename) if os.IsNotExist(err) { return false } return true }