Optimzied snapshot logic in hubridbackup

mod/disk/hybridBackup/hybridBackup.go

@@ -226,7 +226,7 @@ func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
 		//This parent filesystem is mounted
 
 	} else {
-		//File system not mounted even after 3 backup cycle. Terminate backup scheduler
+		//Parent File system not mounted.Terminate backup scheduler
 		log.Println("[HybridBackup] Skipping backup cycle for " + backupConfig.ParentUID + ":/")
 		return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil
 	}
@@ -257,8 +257,8 @@ func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
 	} else if backupConfig.Mode == "nightly" {
 		if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 {
 			//24 hours from last backup. Execute deep backup now
-			executeBackup(backupConfig, true)
 			backupConfig.LastCycleTime = time.Now().Unix()
+			executeBackup(backupConfig, true)
 			log.Println("[HybridBackup] Executing nightly backup: " + backupConfig.ParentUID + ":/ -> " + backupConfig.DiskUID + ":/")
 
 			//Add one to the cycle counter
@@ -267,10 +267,10 @@ func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
 
 	} else if backupConfig.Mode == "version" {
 		//Do a versioning backup every 6 hours
-		if time.Now().Unix()-backupConfig.LastCycleTime >= 21600 || backupConfig.CycleCounter == 0 {
+		if time.Now().Unix()-backupConfig.LastCycleTime >= 21600 {
 			//Scheduled backup or initial backup
-			executeVersionBackup(backupConfig)
 			backupConfig.LastCycleTime = time.Now().Unix()
+			executeVersionBackup(backupConfig)
 			log.Println("[HybridBackup] Executing backup schedule: " + backupConfig.ParentUID + ":/ -> " + backupConfig.DiskUID + ":/")
 
 			//Add one to the cycle counter

mod/disk/hybridBackup/versionBackup.go

@@ -71,17 +71,22 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 	deletedFileList := map[string]string{}
 
 	//First pass: Check if there are any updated file from source and backup it to backup drive
+	log.Println("[HybridBackup] Snapshot Stage 1 - Started " + backupConfig.JobName)
+	rootAbs, _ := filepath.Abs(backupConfig.ParentPath)
+	rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
 	err = fastWalk(parentRootAbs, func(filename string) error {
 		if filepath.Ext(filename) == ".db" || filepath.Ext(filename) == ".lock" {
 			//Reserved filename, skipping
 			return nil
 		}
 
+		if filepath.Ext(filename) == ".datalink" {
+			//Reserved filename, skipping
+			return nil
+		}
+
 		//Get the target paste location
-		rootAbs, _ := filepath.Abs(backupConfig.ParentPath)
 		fileAbs, _ := filepath.Abs(filename)
-
-		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
 		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
 
 		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
@@ -203,6 +208,7 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 
 	//2nd pass: Check if there are anything exists in the previous backup but no longer exists in the source now
 	//For case where the file is backed up in previous snapshot but now the file has been removed
+	log.Println("[HybridBackup] Snapshot Stage 2 - Started " + backupConfig.JobName)
 	if previousSnapshotExists {
 		fastWalk(previousSnapshotLocation, func(filename string) error {
 			if filepath.Ext(filename) == ".datalink" {
@@ -238,8 +244,9 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 		}
 	}
 
-	//3rd pass: Check if there are anything (except file with .deleted) in today backup drive that didn't exists in the source drive
+	//3rd pass: Check if there are anything in today backup drive that didn't exists in the source drive
 	//For cases where the backup is applied to overwrite an eariler backup of the same day
+	log.Println("[HybridBackup] Snapshot Stage 3 - Started " + backupConfig.JobName)
 	fastWalk(snapshotLocation, func(filename string) error {
 		if filepath.Ext(filename) == ".db" || filepath.Ext(filename) == ".lock" {
 			//Reserved filename, skipping
@@ -269,12 +276,14 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 	})
 
 	//Generate linkfile for this snapshot
+	log.Println("[HybridBackup] Snapshot - Generating Linker File")
 	generateLinkFile(snapshotLocation, LinkFileMap{
 		UnchangedFile: linkedFileList,
 		DeletedFiles:  deletedFileList,
 	})
 
 	if err != nil {
+		log.Println("[HybridBackup] Error! ", err.Error())
 		return "", err
 	}
 

mod/filesystem/config.go

@@ -43,6 +43,12 @@ func ValidateOption(options *FileSystemOption) error {
 	if options.Uuid == "" {
 		return errors.New("File System Handler uuid cannot be empty")
 	}
+
+	//Check if uuid is reserved by system
+	if inSlice([]string{"user", "tmp", "share", "network"}, options.Uuid) {
+		return errors.New("This File System Handler UUID is reserved by the system")
+	}
+
 	if !fileExists(options.Path) {
 		return errors.New("Path not exists, given: " + options.Path)
 	}

mod/filesystem/fsextend/fsextend.go

@@ -0,0 +1,13 @@
+package fsextend
+
+/*
+	fsextend.go
+
+	This module extend the file system handler function to virtualized / emulated
+	interfaces
+*/
+
+type VirtualizedFileSystemPathResolver interface {
+	VirtualPathToRealPath(string) (string, error)
+	RealPathToVirtualPath(string) (string, error)
+}

mod/user/directoryHandler.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
-	"strings"
 
 	fs "imuslab.com/arozos/mod/filesystem"
 )
@@ -214,47 +213,3 @@ func (u *User) GetFileSystemHandlerFromRealPath(rpath string) (*fs.FileSystemHan
 
 	return u.GetFileSystemHandlerFromVirtualPath(vpath)
 }
-
-/*
-
-	PRIVATE FUNCTIONS HANDLERS
-
-*/
-//Get a fs handler from a virtual path, quick function for getIDFromHandler + GetHandlerFromID
-func getHandlerFromVirtualPath(storages []*fs.FileSystemHandler, vpath string) (*fs.FileSystemHandler, error) {
-	vid, _, err := getIDFromVirtualPath(vpath)
-	if err != nil {
-		return &fs.FileSystemHandler{}, err
-	}
-
-	return getHandlerFromID(storages, vid)
-}
-
-//Get a fs handler from the given virtial device id
-func getHandlerFromID(storages []*fs.FileSystemHandler, vid string) (*fs.FileSystemHandler, error) {
-	for _, storage := range storages {
-		if storage.UUID == vid {
-			//This storage is the one we are looking at
-			return storage, nil
-		}
-	}
-
-	return &fs.FileSystemHandler{}, errors.New("Handler Not Found")
-}
-
-//Get the ID part of a virtual path, return ID, subpath and error
-func getIDFromVirtualPath(vpath string) (string, string, error) {
-	if strings.Contains(vpath, ":") == false {
-		return "", "", errors.New("Path missing Virtual Device ID. Given: " + vpath)
-	}
-
-	//Clean up the virutal path
-	vpath = filepath.ToSlash(filepath.Clean(vpath))
-
-	tmp := strings.Split(vpath, ":")
-	vdID := tmp[0]
-	pathSlice := tmp[1:]
-	path := strings.Join(pathSlice, ":")
-
-	return vdID, path, nil
-}

mod/user/internal.go

@@ -0,0 +1,52 @@
+package user
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+
+	fs "imuslab.com/arozos/mod/filesystem"
+)
+
+/*
+	Private functions
+*/
+
+//Get a fs handler from a virtual path, quick function for getIDFromHandler + GetHandlerFromID
+func getHandlerFromVirtualPath(storages []*fs.FileSystemHandler, vpath string) (*fs.FileSystemHandler, error) {
+	vid, _, err := getIDFromVirtualPath(vpath)
+	if err != nil {
+		return &fs.FileSystemHandler{}, err
+	}
+
+	return getHandlerFromID(storages, vid)
+}
+
+//Get a fs handler from the given virtial device id
+func getHandlerFromID(storages []*fs.FileSystemHandler, vid string) (*fs.FileSystemHandler, error) {
+	for _, storage := range storages {
+		if storage.UUID == vid {
+			//This storage is the one we are looking at
+			return storage, nil
+		}
+	}
+
+	return &fs.FileSystemHandler{}, errors.New("Handler Not Found")
+}
+
+//Get the ID part of a virtual path, return ID, subpath and error
+func getIDFromVirtualPath(vpath string) (string, string, error) {
+	if strings.Contains(vpath, ":") == false {
+		return "", "", errors.New("Path missing Virtual Device ID. Given: " + vpath)
+	}
+
+	//Clean up the virutal path
+	vpath = filepath.ToSlash(filepath.Clean(vpath))
+
+	tmp := strings.Split(vpath, ":")
+	vdID := tmp[0]
+	pathSlice := tmp[1:]
+	path := strings.Join(pathSlice, ":")
+
+	return vdID, path, nil
+}

mod/user/user.go

@@ -68,10 +68,6 @@ func (u *UserHandler) UpdateStoragePool(newpool *storage.StoragePool) {
 	u.basePool = newpool
 }
 
-func (u *User) Parent() *UserHandler {
-	return u.parent
-}
-
 //Get User object from username
 func (u *UserHandler) GetUserInfoFromUsername(username string) (*User, error) {
 	//Check if user exists
@@ -156,26 +152,3 @@ func (u *UserHandler) GetUserInfoFromRequest(w http.ResponseWriter, r *http.Requ
 	}
 	return userObject, nil
 }
-
-//Remove the current user
-func (u *User) RemoveUser() {
-	//Remove the user storage quota settings
-	log.Println("Removing User Quota: ", u.Username)
-	u.StorageQuota.RemoveUserQuota()
-
-	//Remove the user authentication register
-	u.parent.authAgent.UnregisterUser(u.Username)
-}
-
-//Get the current user icon
-func (u *User) GetUserIcon() string {
-	var userIconpath []byte
-	u.parent.database.Read("auth", "profilepic/"+u.Username, &userIconpath)
-	return string(userIconpath)
-}
-
-//Set the current user icon
-func (u *User) SetUserIcon(base64data string) {
-	u.parent.database.Write("auth", "profilepic/"+u.Username, []byte(base64data))
-	return
-}

mod/user/useropr.go

@@ -0,0 +1,31 @@
+package user
+
+import "log"
+
+//Get the user's handler
+func (u *User) Parent() *UserHandler {
+	return u.parent
+}
+
+//Remove the current user
+func (u *User) RemoveUser() {
+	//Remove the user storage quota settings
+	log.Println("Removing User Quota: ", u.Username)
+	u.StorageQuota.RemoveUserQuota()
+
+	//Remove the user authentication register
+	u.parent.authAgent.UnregisterUser(u.Username)
+}
+
+//Get the current user icon
+func (u *User) GetUserIcon() string {
+	var userIconpath []byte
+	u.parent.database.Read("auth", "profilepic/"+u.Username, &userIconpath)
+	return string(userIconpath)
+}
+
+//Set the current user icon
+func (u *User) SetUserIcon(base64data string) {
+	u.parent.database.Write("auth", "profilepic/"+u.Username, []byte(base64data))
+	return
+}

system/share/downloadPageFolder.html

@@ -58,7 +58,6 @@
         }
 
         td{
-          word-wrap: anywhere;
           word-break: break-all;
         }
     </style>
@@ -101,7 +100,7 @@
                     <p style="font-size: 80%;"><b>Depending on folder size, zipping might take a while to complete.</b></p>
                     <p>Request File ID: {{reqid}}<br>
                     Request Timestamp: {{reqtime}}</p>
-                    <small>Double click any item in the list to open or download</small>
+                    <small>📂 Double click any item in the list to open or download</small>
                     
                 </div>
                 <div class="one-half column" id="filelistWrapper" style="overflow-y: auto; padding-right: 0.5em; min-height: 400px;">

web/SystemAO/file_system/file_explorer.html

@@ -1486,12 +1486,12 @@
 
                                 if (data.error !== undefined){
                                     //Parse path error. Try to refresh the page
-                                    console.log("Path parse error! Redirecting to parent directory.", data.error);
-                                    var pdir = currentPath.split("/");
-                                    pdir.pop(); pdir.pop();
-                                    pdir = pdir.join("/");
+                                    //console.log("Path parse error! Redirecting to parent directory.", data.error);
+                                    //var pdir = currentPath.split("/");
+                                    //pdir.pop(); pdir.pop();
+                                    //pdir = pdir.join("/");
                                     //currentPath = pdir;
-
+                                    enableAutoRefresh = false;
                                     //Check if it is already rooted and no more parent ahead
                                     if (currentPath == ""){
                                         currentPath = "user:/";
@@ -1513,6 +1513,7 @@
                                     //});
                                     return;
                                 }else{
+                                    enableAutoRefresh = true;
                                     //Filelist returned. Render it
                                     renderDirectory(data,function(){
                                         //Restore the selected file list
@@ -1524,8 +1525,7 @@
                                                 }
                                             }
                                         });
-                                    
-
+                                        
                                         //Perform the callback
                                         if (callback !== undefined){
                                             callback();