Added concurrent FTP support

file_system.go

@@ -2511,6 +2511,7 @@ func system_fs_handleList(w http.ResponseWriter, r *http.Request) {
 
 	files, err := fshAbs.ReadDir(realpath)
 	if err != nil {
+		utils.SendErrorResponse(w, "Readdir Failed: "+err.Error())
 		systemWideLogger.PrintAndLog("File System", "Unable to read dir: "+err.Error(), err)
 		return
 	}

mod/filesystem/abstractions/ftpfs/ftpFileWrapper.go

@@ -1,146 +1,75 @@
-package ftpfs
-
-import (
-	"io/fs"
-	"time"
-
-	"github.com/jlaffaye/ftp"
-)
-
-type File struct {
-	entry ftp.Entry
-}
-
-/*
-func NewFTPFsFile(wrappingFile *smb2.File) *File {
-	return &smbfsFile{
-		file: wrappingFile,
-	}
-}
-
-func (f *File) Chdir() error {
-	return arozfs.ErrOperationNotSupported
-}
-func (f *File) Chmod(mode os.FileMode) error {
-	return arozfs.ErrOperationNotSupported
-}
-func (f *File) Chown(uid, gid int) error {
-	return arozfs.ErrOperationNotSupported
-}
-func (f *File) Close() error {
-	return f.file.Close()
-}
-func (f *File) Name() string {
-	return f.file.Name()
-}
-func (f *File) Read(b []byte) (n int, err error) {
-	return f.file.Read(b)
-}
-func (f *File) ReadAt(b []byte, off int64) (n int, err error) {
-	return f.file.ReadAt(b, off)
-}
-func (f *File) ReadDir(n int) ([]fs.DirEntry, error) {
-	return []fs.DirEntry{}, arozfs.ErrOperationNotSupported
-}
-func (f *File) Readdirnames(n int) (names []string, err error) {
-	fi, err := f.file.Readdir(n)
-	if err != nil {
-		return []string{}, err
-	}
-	names = []string{}
-	for _, i := range fi {
-		names = append(names, i.Name())
-	}
-	return names, nil
-}
-func (f *File) ReadFrom(r io.Reader) (n int64, err error) {
-	return f.file.ReadFrom(r)
-}
-func (f *File) Readdir(n int) ([]fs.FileInfo, error) {
-	return f.file.Readdir(n)
-}
-func (f *File) Seek(offset int64, whence int) (ret int64, err error) {
-	return f.file.Seek(offset, whence)
-}
-func (f *File) Stat() (fs.FileInfo, error) {
-	return f.file.Stat()
-}
-func (f *File) Sync() error {
-	return f.file.Sync()
-}
-func (f *File) Truncate(size int64) error {
-	return f.file.Truncate(size)
-}
-func (f *File) Write(b []byte) (n int, err error) {
-	return f.file.Write(b)
-}
-func (f *File) WriteAt(b []byte, off int64) (n int, err error) {
-	return f.file.WriteAt(b, off)
-}
-func (f *File) WriteString(s string) (n int, err error) {
-	return f.file.WriteString(s)
-}
-*/
-
-type DirEntry struct {
-	finfo *ftp.Entry
-	conn  *ftp.ServerConn
-	path  string
-}
-
-func newDirEntryFromFTPEntry(targetEntry *ftp.Entry, conn *ftp.ServerConn, originalPath string) *DirEntry {
-	return &DirEntry{
-		finfo: targetEntry,
-		conn:  conn,
-	}
-}
-
-func (de DirEntry) Name() string {
-	return de.finfo.Name
-}
-
-func (de DirEntry) IsDir() bool {
-	return de.finfo.Type == ftp.EntryTypeFolder
-}
-
-func (de DirEntry) Type() fs.FileMode {
-	return 0777
-}
-
-func (de DirEntry) Info() (fs.FileInfo, error) {
-	e := NewFileInfoFromEntry(de.finfo, de.conn, de.path)
-	return e, nil
-}
-
-type FileInfo struct {
-	entry *ftp.Entry
-	conn  *ftp.ServerConn
-	path  string
-}
-
-func NewFileInfoFromEntry(e *ftp.Entry, c *ftp.ServerConn, originalPath string) FileInfo {
-	return FileInfo{
-		entry: e,
-		conn:  c,
-		path:  originalPath,
-	}
-}
-
-func (fi FileInfo) Name() string {
-	return fi.entry.Name
-}
-func (fi FileInfo) Size() int64 {
-	return int64(fi.entry.Size)
-}
-func (fi FileInfo) Mode() fs.FileMode {
-	return 664
-}
-func (fi FileInfo) ModTime() time.Time {
-	return fi.entry.Time
-}
-func (fi FileInfo) IsDir() bool {
-	return fi.entry.Type == ftp.EntryTypeFolder
-}
-func (fi FileInfo) Sys() interface{} {
-	return nil
-}
+package ftpfs
+
+import (
+	"io/fs"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+)
+
+type File struct {
+	entry ftp.Entry
+}
+
+type DirEntry struct {
+	finfo *ftp.Entry
+	conn  *ftp.ServerConn
+	path  string
+}
+
+func newDirEntryFromFTPEntry(targetEntry *ftp.Entry, conn *ftp.ServerConn, originalPath string) *DirEntry {
+	return &DirEntry{
+		finfo: targetEntry,
+		conn:  conn,
+	}
+}
+
+func (de DirEntry) Name() string {
+	return de.finfo.Name
+}
+
+func (de DirEntry) IsDir() bool {
+	return de.finfo.Type == ftp.EntryTypeFolder
+}
+
+func (de DirEntry) Type() fs.FileMode {
+	return 0777
+}
+
+func (de DirEntry) Info() (fs.FileInfo, error) {
+	e := NewFileInfoFromEntry(de.finfo, de.conn, de.path)
+	return e, nil
+}
+
+type FileInfo struct {
+	entry *ftp.Entry
+	conn  *ftp.ServerConn
+	path  string
+}
+
+func NewFileInfoFromEntry(e *ftp.Entry, c *ftp.ServerConn, originalPath string) FileInfo {
+	return FileInfo{
+		entry: e,
+		conn:  c,
+		path:  originalPath,
+	}
+}
+
+func (fi FileInfo) Name() string {
+	return fi.entry.Name
+}
+func (fi FileInfo) Size() int64 {
+	return int64(fi.entry.Size)
+}
+func (fi FileInfo) Mode() fs.FileMode {
+	return 664
+}
+func (fi FileInfo) ModTime() time.Time {
+	return fi.entry.Time
+}
+func (fi FileInfo) IsDir() bool {
+	return fi.entry.Type == ftp.EntryTypeFolder
+}
+func (fi FileInfo) Sys() interface{} {
+	return nil
+}

mod/filesystem/abstractions/ftpfs/ftpfs.go

@@ -1,230 +1,361 @@
-package ftpfs
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/fs"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/jlaffaye/ftp"
-	"imuslab.com/arozos/mod/filesystem/arozfs"
-)
-
-/*
-	FTPFS.go
-
-	FTP Server as File System Abstraction
-
-*/
-
-type FTPFSAbstraction struct {
-	uuid      string
-	hierarchy string
-	conn      *ftp.ServerConn
-	closer    chan bool
-}
-
-func NewFTPFSAbstraction(uuid string, hierarchy string, hostname string, username string, password string) (FTPFSAbstraction, error) {
-	c, err := ftp.Dial(hostname, ftp.DialWithTimeout(5*time.Second))
-	if err != nil {
-		log.Println("[FTPFS] Unable to dial TCP: " + err.Error())
-		return FTPFSAbstraction{}, err
-	}
-
-	if username == "" && password == "" {
-		username = "anonymouss"
-		password = "anonymous"
-	}
-
-	//Login to the FTP account
-	err = c.Login(username, password)
-	if err != nil {
-		return FTPFSAbstraction{}, err
-	}
-
-	//Create a ticker to prevent connection close
-	ticker := time.NewTicker(180 * time.Second)
-	done := make(chan bool)
-
-	go func() {
-		for {
-			select {
-			case <-done:
-				return
-			case <-ticker.C:
-				c.NoOp()
-			}
-		}
-	}()
-
-	return FTPFSAbstraction{
-		uuid:      uuid,
-		hierarchy: hierarchy,
-		conn:      c,
-		closer:    done,
-	}, nil
-}
-func (l FTPFSAbstraction) Chmod(filename string, mode os.FileMode) error {
-	return arozfs.ErrOperationNotSupported
-}
-func (l FTPFSAbstraction) Chown(filename string, uid int, gid int) error {
-	return arozfs.ErrOperationNotSupported
-}
-func (l FTPFSAbstraction) Chtimes(filename string, atime time.Time, mtime time.Time) error {
-	return arozfs.ErrOperationNotSupported
-}
-func (l FTPFSAbstraction) Create(filename string) (arozfs.File, error) {
-	return nil, arozfs.ErrOperationNotSupported
-}
-func (l FTPFSAbstraction) Mkdir(filename string, mode os.FileMode) error {
-	return l.conn.MakeDir(filename)
-}
-func (l FTPFSAbstraction) MkdirAll(filename string, mode os.FileMode) error {
-	return l.Mkdir(filename, mode)
-}
-func (l FTPFSAbstraction) Name() string {
-	return ""
-}
-func (l FTPFSAbstraction) Open(filename string) (arozfs.File, error) {
-	return nil, arozfs.ErrOperationNotSupported
-}
-func (l FTPFSAbstraction) OpenFile(filename string, flag int, perm os.FileMode) (arozfs.File, error) {
-	return nil, arozfs.ErrOperationNotSupported
-}
-func (l FTPFSAbstraction) Remove(filename string) error {
-	filename = filterFilepath(filename)
-	return l.conn.Delete(filename)
-}
-func (l FTPFSAbstraction) RemoveAll(path string) error {
-	path = filterFilepath(path)
-	return l.conn.Delete(path)
-}
-func (l FTPFSAbstraction) Rename(oldname, newname string) error {
-	oldname = filterFilepath(oldname)
-	newname = filterFilepath(newname)
-	return l.conn.Rename(oldname, newname)
-}
-func (l FTPFSAbstraction) Stat(filename string) (os.FileInfo, error) {
-	return nil, arozfs.ErrNullOperation
-}
-func (l FTPFSAbstraction) Close() error {
-	return l.conn.Quit()
-}
-
-/*
-	Abstraction Utilities
-*/
-
-func (l FTPFSAbstraction) VirtualPathToRealPath(subpath string, username string) (string, error) {
-	return arozfs.GenericVirtualPathToRealPathTranslator(l.uuid, l.hierarchy, subpath, username)
-}
-
-func (l FTPFSAbstraction) RealPathToVirtualPath(fullpath string, username string) (string, error) {
-	return arozfs.GenericRealPathToVirtualPathTranslator(l.uuid, l.hierarchy, fullpath, username)
-}
-
-func (l FTPFSAbstraction) FileExists(realpath string) bool {
-	realpath = filterFilepath(realpath)
-	_, err := l.conn.GetEntry(realpath)
-	return err == nil
-}
-
-func (l FTPFSAbstraction) IsDir(realpath string) bool {
-	realpath = filterFilepath(realpath)
-	entry, err := l.conn.GetEntry(realpath)
-	if err != nil {
-		return false
-	}
-
-	return entry.Type == ftp.EntryTypeFolder
-}
-
-func (l FTPFSAbstraction) Glob(realpathWildcard string) ([]string, error) {
-	return []string{}, arozfs.ErrNullOperation
-}
-
-func (l FTPFSAbstraction) GetFileSize(realpath string) int64 {
-	realpath = filterFilepath(realpath)
-	entry, err := l.conn.GetEntry(realpath)
-	if err != nil {
-		return 0
-	}
-	return int64(entry.Size)
-}
-
-func (l FTPFSAbstraction) GetModTime(realpath string) (int64, error) {
-	realpath = filterFilepath(realpath)
-	entry, err := l.conn.GetEntry(realpath)
-	if err != nil {
-		return 0, err
-	}
-
-	return entry.Time.Unix(), nil
-}
-
-func (l FTPFSAbstraction) WriteFile(filename string, content []byte, mode os.FileMode) error {
-	filename = filterFilepath(filename)
-	reader := bytes.NewReader(content)
-	return l.conn.Stor(filename, reader)
-}
-
-func (l FTPFSAbstraction) ReadFile(filename string) ([]byte, error) {
-	filename = filterFilepath(filename)
-	r, err := l.conn.Retr(filename)
-	if err != nil {
-		panic(err)
-	}
-	defer r.Close()
-
-	return ioutil.ReadAll(r)
-}
-func (l FTPFSAbstraction) ReadDir(filename string) ([]fs.DirEntry, error) {
-	results := []fs.DirEntry{}
-	filename = filterFilepath(filename)
-	entries, err := l.conn.List(filename)
-	if err != nil {
-
-		return results, err
-	}
-
-	for _, entry := range entries {
-		entryFilename := arozfs.ToSlash(filepath.Join(filename, entry.Name))
-		fmt.Println(entryFilename)
-		thisDirEntry := newDirEntryFromFTPEntry(entry, l.conn, entryFilename)
-		results = append(results, thisDirEntry)
-	}
-	return results, nil
-}
-func (l FTPFSAbstraction) WriteStream(filename string, stream io.Reader, mode os.FileMode) error {
-	filename = filterFilepath(filename)
-	return l.conn.Stor(filename, stream)
-}
-func (l FTPFSAbstraction) ReadStream(filename string) (io.ReadCloser, error) {
-	filename = filterFilepath(filename)
-	return l.conn.Retr(filename)
-}
-
-func (l FTPFSAbstraction) Walk(root string, walkFn filepath.WalkFunc) error {
-	return arozfs.ErrOperationNotSupported
-}
-
-func (l FTPFSAbstraction) Heartbeat() error {
-	return nil
-}
-
-//Utilities
-func filterFilepath(rawpath string) string {
-	rawpath = arozfs.ToSlash(filepath.Clean(strings.TrimSpace(rawpath)))
-	if strings.HasPrefix(rawpath, "./") {
-		return rawpath[1:]
-	} else if rawpath == "." || rawpath == "" {
-		return "/"
-	}
-	return rawpath
-}
+package ftpfs
+
+import (
+	"bytes"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/jlaffaye/ftp"
+	"imuslab.com/arozos/mod/filesystem/arozfs"
+)
+
+/*
+	FTPFS.go
+
+	FTP Server as File System Abstraction
+
+*/
+
+type FTPFSAbstraction struct {
+	uuid      string
+	hierarchy string
+	hostname  string
+	username  string
+	password  string
+	//conn      *ftp.ServerConn
+	//closer    chan bool
+}
+
+func NewFTPFSAbstraction(uuid string, hierarchy string, hostname string, username string, password string) (FTPFSAbstraction, error) {
+
+	//Create a ticker to prevent connection close
+	/*
+		ticker := time.NewTicker(180 * time.Second)
+		done := make(chan bool)
+
+		go func() {
+			for {
+				select {
+				case <-done:
+					return
+				case <-ticker.C:
+					c.NoOp()
+				}
+			}
+		}()
+	*/
+	log.Println("[FTP FS] " + hostname + " mounted via FTP-FS")
+	return FTPFSAbstraction{
+		uuid:      uuid,
+		hierarchy: hierarchy,
+		hostname:  hostname,
+		username:  username,
+		password:  password,
+	}, nil
+}
+
+func (l FTPFSAbstraction) makeConn() (*ftp.ServerConn, error) {
+	username := l.username
+	password := l.password
+	c, err := ftp.Dial(l.hostname, ftp.DialWithTimeout(3*time.Second))
+	if err != nil {
+		log.Println("[FTPFS] Unable to dial TCP: " + err.Error())
+		return nil, err
+	}
+
+	if username == "" && password == "" {
+		username = "anonymouss"
+		password = "anonymous"
+	}
+
+	//Login to the FTP account
+	err = c.Login(username, password)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (l FTPFSAbstraction) Chmod(filename string, mode os.FileMode) error {
+	return arozfs.ErrOperationNotSupported
+}
+func (l FTPFSAbstraction) Chown(filename string, uid int, gid int) error {
+	return arozfs.ErrOperationNotSupported
+}
+func (l FTPFSAbstraction) Chtimes(filename string, atime time.Time, mtime time.Time) error {
+	return arozfs.ErrOperationNotSupported
+}
+func (l FTPFSAbstraction) Create(filename string) (arozfs.File, error) {
+	return nil, arozfs.ErrOperationNotSupported
+}
+func (l FTPFSAbstraction) Mkdir(filename string, mode os.FileMode) error {
+	c, err := l.makeConn()
+	if err != nil {
+		return err
+	}
+
+	defer c.Quit()
+
+	err = c.MakeDir(filename)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (l FTPFSAbstraction) MkdirAll(filename string, mode os.FileMode) error {
+	return l.Mkdir(filename, mode)
+}
+func (l FTPFSAbstraction) Name() string {
+	return ""
+}
+func (l FTPFSAbstraction) Open(filename string) (arozfs.File, error) {
+	return nil, arozfs.ErrOperationNotSupported
+}
+func (l FTPFSAbstraction) OpenFile(filename string, flag int, perm os.FileMode) (arozfs.File, error) {
+	return nil, arozfs.ErrOperationNotSupported
+}
+func (l FTPFSAbstraction) Remove(filename string) error {
+	filename = filterFilepath(filename)
+	c, err := l.makeConn()
+	if err != nil {
+		return err
+	}
+	defer c.Quit()
+
+	err = c.Delete(filename)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (l FTPFSAbstraction) RemoveAll(path string) error {
+	path = filterFilepath(path)
+	return l.Remove(path)
+}
+func (l FTPFSAbstraction) Rename(oldname, newname string) error {
+	oldname = filterFilepath(oldname)
+	newname = filterFilepath(newname)
+	c, err := l.makeConn()
+	if err != nil {
+		return err
+	}
+	defer c.Quit()
+	err = c.Rename(oldname, newname)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+func (l FTPFSAbstraction) Stat(filename string) (os.FileInfo, error) {
+	return nil, arozfs.ErrNullOperation
+}
+func (l FTPFSAbstraction) Close() error {
+	return nil
+}
+
+/*
+	Abstraction Utilities
+*/
+
+func (l FTPFSAbstraction) VirtualPathToRealPath(subpath string, username string) (string, error) {
+	return arozfs.GenericVirtualPathToRealPathTranslator(l.uuid, l.hierarchy, subpath, username)
+}
+
+func (l FTPFSAbstraction) RealPathToVirtualPath(fullpath string, username string) (string, error) {
+	return arozfs.GenericRealPathToVirtualPathTranslator(l.uuid, l.hierarchy, fullpath, username)
+}
+
+func (l FTPFSAbstraction) FileExists(realpath string) bool {
+	realpath = filterFilepath(realpath)
+	c, err := l.makeConn()
+	if err != nil {
+		return false
+	}
+	_, err = c.GetEntry(realpath)
+	c.Quit()
+	return err == nil
+}
+
+func (l FTPFSAbstraction) IsDir(realpath string) bool {
+	realpath = filterFilepath(realpath)
+	c, err := l.makeConn()
+	if err != nil {
+		return false
+	}
+	defer c.Quit()
+	entry, err := c.GetEntry(realpath)
+	if err != nil {
+		return false
+	}
+
+	return entry.Type == ftp.EntryTypeFolder
+}
+
+func (l FTPFSAbstraction) Glob(realpathWildcard string) ([]string, error) {
+	return []string{}, arozfs.ErrNullOperation
+}
+
+func (l FTPFSAbstraction) GetFileSize(realpath string) int64 {
+	realpath = filterFilepath(realpath)
+	c, err := l.makeConn()
+	if err != nil {
+		return 0
+	}
+	entry, err := c.GetEntry(realpath)
+	if err != nil {
+		return 0
+	}
+	return int64(entry.Size)
+}
+
+func (l FTPFSAbstraction) GetModTime(realpath string) (int64, error) {
+	realpath = filterFilepath(realpath)
+	c, err := l.makeConn()
+	if err != nil {
+		return 0, err
+	}
+	defer c.Quit()
+	entry, err := c.GetEntry(realpath)
+	if err != nil {
+		return 0, err
+	}
+
+	return entry.Time.Unix(), nil
+}
+
+func (l FTPFSAbstraction) WriteFile(filename string, content []byte, mode os.FileMode) error {
+	filename = filterFilepath(filename)
+	c, err := l.makeConn()
+	if err != nil {
+		return err
+	}
+	defer c.Quit()
+	reader := bytes.NewReader(content)
+	return c.Stor(filename, reader)
+}
+
+func (l FTPFSAbstraction) ReadFile(filename string) ([]byte, error) {
+	filename = filterFilepath(filename)
+	c, err := l.makeConn()
+	if err != nil {
+		return []byte{}, err
+	}
+	defer c.Quit()
+	r, err := c.Retr(filename)
+	if err != nil {
+		return []byte{}, err
+	}
+	defer r.Close()
+
+	return ioutil.ReadAll(r)
+}
+func (l FTPFSAbstraction) ReadDir(filename string) ([]fs.DirEntry, error) {
+	results := []fs.DirEntry{}
+	filename = filterFilepath(filename)
+	c, err := l.makeConn()
+	if err != nil {
+		return []fs.DirEntry{}, err
+	}
+	defer c.Quit()
+	entries, err := c.List(filename)
+	if err != nil {
+
+		return results, err
+	}
+
+	for _, entry := range entries {
+		entryFilename := arozfs.ToSlash(filepath.Join(filename, entry.Name))
+		//fmt.Println(entryFilename)
+		thisDirEntry := newDirEntryFromFTPEntry(entry, c, entryFilename)
+		results = append(results, thisDirEntry)
+	}
+	return results, nil
+}
+func (l FTPFSAbstraction) WriteStream(filename string, stream io.Reader, mode os.FileMode) error {
+	filename = filterFilepath(filename)
+	c, err := l.makeConn()
+	if err != nil {
+		return err
+	}
+	defer c.Quit()
+	return c.Stor(filename, stream)
+}
+func (l FTPFSAbstraction) ReadStream(filename string) (io.ReadCloser, error) {
+	filename = filterFilepath(filename)
+	c, err := l.makeConn()
+	if err != nil {
+		return nil, err
+	}
+	defer c.Quit()
+	return c.Retr(filename)
+}
+
+func (l FTPFSAbstraction) Walk(root string, walkFn filepath.WalkFunc) error {
+	root = filterFilepath(root)
+	log.Println("[FTP FS] Walking a root on FTP is extremely slow. Please consider rewritting this function. Scanning: " + root)
+	c, err := l.makeConn()
+	if err != nil {
+		return err
+	}
+	defer c.Quit()
+	rootStat, err := c.GetEntry(root)
+	rootStatInfo := NewFileInfoFromEntry(rootStat, c, root)
+	err = walkFn(root, rootStatInfo, err)
+	if err != nil {
+		return err
+	}
+	return l.walk(root, walkFn)
+}
+
+func (l FTPFSAbstraction) Heartbeat() error {
+	return nil
+}
+
+//Utilities
+func filterFilepath(rawpath string) string {
+	rawpath = arozfs.ToSlash(filepath.Clean(strings.TrimSpace(rawpath)))
+	if strings.HasPrefix(rawpath, "./") {
+		return rawpath[1:]
+	} else if rawpath == "." || rawpath == "" {
+		return "/"
+	}
+	return rawpath
+}
+
+func (l FTPFSAbstraction) walk(thisPath string, walkFun filepath.WalkFunc) error {
+	files, err := l.ReadDir(thisPath)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		thisFileFullPath := filepath.ToSlash(filepath.Join(thisPath, file.Name()))
+		finfo, _ := file.Info()
+		if file.IsDir() {
+			err = walkFun(thisFileFullPath, finfo, nil)
+			if err != nil {
+				return err
+			}
+			err = l.walk(thisFileFullPath, walkFun)
+			if err != nil {
+				return err
+			}
+		} else {
+			err = walkFun(thisFileFullPath, finfo, nil)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}