arozos/mod/filesystem/filesystem.go

package filesystem

/*
	ArOZ Online File System Handler Wrappers
	author: tobychui

	This is a module design to do the followings
	1. Mount / Create a fs when open
	2. Provide the basic function and operations of a file system type
	3. THIS MODULE **SHOULD NOT CONTAIN** CROSS FILE SYSTEM TYPE OPERATIONS
*/

import (
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	db "imuslab.com/arozos/mod/database"
	"imuslab.com/arozos/mod/disk/hybridBackup"
	"imuslab.com/arozos/mod/filesystem/abstractions/localfs"
	"imuslab.com/arozos/mod/filesystem/abstractions/sharefs"
	"imuslab.com/arozos/mod/filesystem/abstractions/webdavfs"
)

//Options for creating new file system handler
/*
type FileSystemOpeningOptions struct{
	Name      string `json:"name"`						//Display name of this device
	Uuid      string `json:"uuid"`						//UUID of this device, e.g. S1
	Path      string `json:"path"`						//Path for the storage root
	Access    string `json:"access,omitempty"`			//Access right, allow {readonly, readwrite}
	Hierarchy string `json:"hierarchy"`					//Folder hierarchy, allow {public, user}
	Automount bool   `json:"automount"`					//Automount this device if exists
	Filesystem string `json:"filesystem,omitempty"`		//Support {"ext4","ext2", "ext3", "fat", "vfat", "ntfs"}
	Mountdev  string `json:"mountdev,omitempty"`		//Device file (e.g. /dev/sda1)
	Mountpt  string `json:"mountpt,omitempty"`			//Device mount point (e.g. /media/storage1)
}
*/

/*
	An interface for storing data related to a specific hierarchy settings.
	Example like the account information of network drive,
	backup mode of backup drive etc
*/
type HierarchySpecificConfig interface{}

type FileSystemAbstraction interface {
	//Fundemental Functions
	Chmod(string, os.FileMode) error
	Chown(string, int, int) error
	Chtimes(string, time.Time, time.Time) error
	Create(string) (*os.File, error)
	Mkdir(string, os.FileMode) error
	MkdirAll(string, os.FileMode) error
	Name() string
	Open(string) (*os.File, error)
	OpenFile(string, int, os.FileMode) (*os.File, error)
	Remove(string) error
	RemoveAll(string) error
	Rename(string, string) error
	Stat(string) (os.FileInfo, error)

	//Utils Functions
	VirtualPathToRealPath(string, string) (string, error)
	RealPathToVirtualPath(string, string) (string, error)
	FileExists(string) bool
	IsDir(string) bool
	Glob(string) ([]string, error)
	GetFileSize(string) int64
	GetModTime(string) (int64, error)
	WriteFile(string, []byte, os.FileMode) error
	ReadFile(string) ([]byte, error)
	WriteStream(string, io.Reader, os.FileMode) error
	ReadStream(string) (io.ReadCloser, error)
	Walk(string, filepath.WalkFunc) error
}

//System Handler for returing
type FileSystemHandler struct {
	Name                  string
	UUID                  string
	Path                  string
	Hierarchy             string
	HierarchyConfig       HierarchySpecificConfig
	ReadOnly              bool
	RequireBuffer         bool //Set this to true if the fsh do not provide file header functions like Open() or Create(), require WriteStream() and ReadStream()
	Parentuid             string
	InitiationTime        int64
	FilesystemDatabase    *db.Database
	FileSystemAbstraction FileSystemAbstraction
	Filesystem            string
	Closed                bool
}

//Create a list of file system handler from the given json content
func NewFileSystemHandlersFromJSON(jsonContent []byte) ([]*FileSystemHandler, error) {
	//Generate a list of handler option from json file
	options, err := loadConfigFromJSON(jsonContent)
	if err != nil {
		return []*FileSystemHandler{}, err
	}

	resultingHandlers := []*FileSystemHandler{}
	for _, option := range options {
		thisHandler, err := NewFileSystemHandler(option)
		if err != nil {
			log.Println("Failed to create system handler for " + option.Name)
			log.Println(err.Error())
			continue
		}
		resultingHandlers = append(resultingHandlers, thisHandler)
	}

	return resultingHandlers, nil
}

//Create a new file system handler with the given config
func NewFileSystemHandler(option FileSystemOption) (*FileSystemHandler, error) {
	fstype := strings.ToLower(option.Filesystem)
	if inSlice([]string{"ext4", "ext2", "ext3", "fat", "vfat", "ntfs"}, fstype) || fstype == "" {
		//Check if the target fs require mounting
		if option.Automount == true {
			err := MountDevice(option.Mountpt, option.Mountdev, option.Filesystem)
			if err != nil {
				return &FileSystemHandler{}, err
			}
		}

		//Check if the path exists
		if !fileExists(option.Path) {
			return &FileSystemHandler{}, errors.New("Mount point not exists!")
		}

		//Handle Hierarchy branching
		if option.Hierarchy == "user" {
			//Create user hierarchy for this virtual device
			os.MkdirAll(filepath.ToSlash(filepath.Clean(option.Path))+"/users", 0755)
		}

		//Create the fsdb for this handler
		var fsdb *db.Database = nil
		dbp, err := db.NewDatabase(filepath.ToSlash(filepath.Join(filepath.Clean(option.Path), "aofs.db")), false)
		if err != nil {
			if option.Access != "readonly" {
				log.Println("[Filesystem] Invalid config: Trying to mount a read only path as read-write mount point. Changing " + option.Name + " mount point to READONLY.")
				option.Access = "readonly"
			}
		} else {
			fsdb = dbp
		}
		rootpath := filepath.ToSlash(filepath.Clean(option.Path)) + "/"
		if option.Hierarchy == "backup" {
			//Backup disk. Create an Hierarchy Config for this drive
			hsConfig := hybridBackup.BackupTask{
				CycleCounter:      0,
				LastCycleTime:     0,
				DiskUID:           option.Uuid,
				DiskPath:          option.Path,
				ParentUID:         option.Parentuid,
				Mode:              option.BackupMode,
				DeleteFileMarkers: map[string]int64{},
				PanicStopped:      false,
			}
			return &FileSystemHandler{
				Name:                  option.Name,
				UUID:                  option.Uuid,
				Path:                  rootpath,
				ReadOnly:              option.Access == "readonly",
				RequireBuffer:         false,
				Parentuid:             option.Parentuid,
				Hierarchy:             option.Hierarchy,
				HierarchyConfig:       hsConfig,
				InitiationTime:        time.Now().Unix(),
				FilesystemDatabase:    fsdb,
				FileSystemAbstraction: localfs.NewLocalFileSystemAbstraction(option.Uuid, rootpath, option.Hierarchy, option.Access == "readonly"),
				Filesystem:            fstype,
				Closed:                false,
			}, nil
		} else {
			return &FileSystemHandler{
				Name:                  option.Name,
				UUID:                  option.Uuid,
				Path:                  filepath.ToSlash(filepath.Clean(option.Path)) + "/",
				ReadOnly:              option.Access == "readonly",
				RequireBuffer:         false,
				Parentuid:             option.Parentuid,
				Hierarchy:             option.Hierarchy,
				HierarchyConfig:       DefaultEmptyHierarchySpecificConfig,
				InitiationTime:        time.Now().Unix(),
				FilesystemDatabase:    fsdb,
				FileSystemAbstraction: localfs.NewLocalFileSystemAbstraction(option.Uuid, rootpath, option.Hierarchy, option.Access == "readonly"),
				Filesystem:            fstype,
				Closed:                false,
			}, nil
		}

	} else if fstype == "webdav" {
		//WebDAV. Create an object and mount it
		root := option.Path
		user := option.Username
		password := option.Password

		webdavfs, err := webdavfs.NewWebDAVMount(option.Uuid, option.Hierarchy, root, user, password, "./tmp/webdavBuff")
		if err != nil {
			return nil, err
		}
		return &FileSystemHandler{
			Name:                  option.Name,
			UUID:                  option.Uuid,
			Path:                  "",
			ReadOnly:              option.Access == "readonly",
			RequireBuffer:         true,
			Parentuid:             option.Parentuid,
			Hierarchy:             option.Hierarchy,
			HierarchyConfig:       nil,
			InitiationTime:        time.Now().Unix(),
			FilesystemDatabase:    nil,
			FileSystemAbstraction: webdavfs,
			Filesystem:            fstype,
			Closed:                false,
		}, nil
	} else if option.Filesystem == "share" {
		shareFsAbstraction := sharefs.NewShareFileSystemAbstraction(
			"share",
			func(s string) (string, error) {
				log.Println(s)
				return "user:/", nil
			},
		)
		return &FileSystemHandler{
			Name:                  option.Name,
			UUID:                  option.Uuid,
			Path:                  "",
			ReadOnly:              option.Access == "readonly",
			RequireBuffer:         false,
			Parentuid:             option.Parentuid,
			Hierarchy:             option.Hierarchy,
			HierarchyConfig:       nil,
			InitiationTime:        time.Now().Unix(),
			FilesystemDatabase:    nil,
			FileSystemAbstraction: shareFsAbstraction,
			Filesystem:            fstype,
			Closed:                false,
		}, nil
	} else if option.Filesystem == "virtual" {
		//Virtual filesystem, deprecated
		log.Println("Deprecated file system type: Virtual")
	}

	return nil, errors.New("Not supported file system: " + fstype)
}

//Check if a fsh is virtual (e.g. Network or fs Abstractions that cannot be listed with normal fs API)
func (fsh *FileSystemHandler) IsVirtual() bool {
	if fsh.Hierarchy == "virtual" || fsh.Filesystem == "webdav" {
		//Check if the config return placeholder
		c, ok := fsh.HierarchyConfig.(EmptyHierarchySpecificConfig)
		if ok && c.HierarchyType == "placeholder" {
			//Real file system.
			return false
		}

		//Do more checking here if needed
		return true
	}
	return false
}

func (fsh *FileSystemHandler) IsRootOf(vpath string) bool {
	return strings.HasPrefix(vpath, fsh.UUID+":")
}

func (fsh *FileSystemHandler) GetDirctorySizeFromRealPath(rpath string, includeHidden bool) (int64, int) {
	var size int64 = 0
	var fileCount int = 0
	err := fsh.FileSystemAbstraction.Walk(rpath, func(thisFilename string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			if includeHidden {
				//append all into the file count and size
				size += info.Size()
				fileCount++
			} else {
				//Check if this is hidden
				if !IsInsideHiddenFolder(thisFilename) {
					size += info.Size()
					fileCount++
				}

			}

		}
		return nil
	})
	if err != nil {
		return 0, fileCount
	}
	return size, fileCount
}

func (fsh *FileSystemHandler) GetDirctorySizeFromVpath(vpath string, username string, includeHidden bool) (int64, int) {
	realpath, _ := fsh.FileSystemAbstraction.VirtualPathToRealPath(vpath, username)
	return fsh.GetDirctorySizeFromRealPath(realpath, includeHidden)
}

/*
	File Record Related Functions
	fsh database that keep track of which files is owned by whom
*/

//Create a file ownership record
func (fsh *FileSystemHandler) CreateFileRecord(realpath string, owner string) error {
	if fsh.FilesystemDatabase == nil {
		//Not supported file system type
		return errors.New("Not supported filesystem type")
	}
	rpabs, _ := filepath.Abs(realpath)
	fsrabs, _ := filepath.Abs(fsh.Path)
	reldir, err := filepath.Rel(fsrabs, rpabs)
	if err != nil {
		return err

	}
	fsh.FilesystemDatabase.NewTable("owner")
	fsh.FilesystemDatabase.Write("owner", "owner/"+reldir, owner)
	return nil
}

//Read the owner of a file
func (fsh *FileSystemHandler) GetFileRecord(realpath string) (string, error) {
	if fsh.FilesystemDatabase == nil {
		//Not supported file system type
		return "", errors.New("Not supported filesystem type")
	}

	rpabs, _ := filepath.Abs(realpath)
	fsrabs, _ := filepath.Abs(fsh.Path)
	reldir, err := filepath.Rel(fsrabs, rpabs)
	if err != nil {
		return "", err
	}
	fsh.FilesystemDatabase.NewTable("owner")
	if fsh.FilesystemDatabase.KeyExists("owner", "owner/"+reldir) {
		owner := ""
		fsh.FilesystemDatabase.Read("owner", "owner/"+reldir, &owner)
		return owner, nil
	} else {
		return "", errors.New("Owner not exists")
	}
}

//Delete a file ownership record
func (fsh *FileSystemHandler) DeleteFileRecord(realpath string) error {
	if fsh.FilesystemDatabase == nil {
		//Not supported file system type
		return errors.New("Not supported filesystem type")
	}

	rpabs, _ := filepath.Abs(realpath)
	fsrabs, _ := filepath.Abs(fsh.Path)
	reldir, err := filepath.Rel(fsrabs, rpabs)
	if err != nil {
		return err
	}
	fsh.FilesystemDatabase.NewTable("owner")
	if fsh.FilesystemDatabase.KeyExists("owner", "owner/"+reldir) {
		fsh.FilesystemDatabase.Delete("owner", "owner/"+reldir)
	}

	return nil
}

//Close an openeded File System
func (fsh *FileSystemHandler) Close() {
	//Close the fsh database
	if fsh.FilesystemDatabase != nil {
		fsh.FilesystemDatabase.Close()
	}

}

//Helper function
func inSlice(slice []string, val string) bool {
	for _, item := range slice {
		if item == val {
			return true
		}
	}
	return false
}

func FileExists(filename string) bool {
	return fileExists(filename)
}

//Check if file exists
func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	if os.IsNotExist(err) {
		return false
	}
	return true
}