Make backup drive hidden in user list

file_system.go

@@ -1854,10 +1854,12 @@ func system_fs_listRoot(w http.ResponseWriter, r *http.Request) {
 		}
 		var roots []rootObject
 		for _, store := range userinfo.GetAllFileSystemHandler() {
-			var thisDevice = new(rootObject)
-			thisDevice.RootName = store.Name
-			thisDevice.RootPath = store.UUID + ":/"
-			roots = append(roots, *thisDevice)
+			if store.Hierarchy == "user" || store.Hierarchy == "public" {
+				var thisDevice = new(rootObject)
+				thisDevice.RootName = store.Name
+				thisDevice.RootPath = store.UUID + ":/"
+				roots = append(roots, *thisDevice)
+			}
 		}
 		jsonString, _ := json.Marshal(roots)
 		sendJSONResponse(w, string(jsonString))
@@ -1871,6 +1873,8 @@ func system_fs_listRoot(w http.ResponseWriter, r *http.Request) {
 */
 
 func system_fs_specialGlob(path string) ([]string, error) {
+	//Quick fix for foldername containing -] issue
+	path = strings.ReplaceAll(path, "[", "[[]")
 	files, err := filepath.Glob(path)
 	if err != nil {
 		return []string{}, err

mod/filesystem/filesystem.go

@@ -109,12 +109,13 @@ func NewFileSystemHandler(option FileSystemOption) (*FileSystemHandler, error) {
 		if option.Hierarchy == "backup" {
 			//Backup disk. Create an Hierarchy Config for this drive
 			hierarchySpecificConfig = hybridBackup.BackupConfig{
-				CycleCounter:  0,
-				LastCycleTime: time.Now().Unix(),
-				DiskUID:       option.Uuid,
-				DiskPath:      option.Path,
-				ParentUID:     option.Parentuid,
-				Mode:          option.BackupMode,
+				CycleCounter:      0,
+				LastCycleTime:     time.Now().Unix(),
+				DiskUID:           option.Uuid,
+				DiskPath:          option.Path,
+				ParentUID:         option.Parentuid,
+				Mode:              option.BackupMode,
+				DeleteFileMarkers: map[string]int64{},
 			}
 		}
 
@@ -190,7 +191,9 @@ func (fsh *FileSystemHandler) DeleteFileRecord(realpath string) error {
 	return nil
 }
 
+//Close an openeded File System
 func (fsh *FileSystemHandler) Close() {
+	//Close the fsh database
 	fsh.FilesystemDatabase.Close()
 }
 

mod/filesystem/hybridBackup/fileCopy.go

@@ -0,0 +1,50 @@
+package hybridBackup
+
+import (
+	"errors"
+	"io"
+	"os"
+)
+
+func BufferedLargeFileCopy(src string, dst string, BUFFERSIZE int64) error {
+	sourceFileStat, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return errors.New("Invalid file source")
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	buf := make([]byte, BUFFERSIZE)
+	for {
+		n, err := source.Read(buf)
+		if err != nil && err != io.EOF {
+			source.Close()
+			destination.Close()
+			return err
+		}
+		if n == 0 {
+			source.Close()
+			destination.Close()
+			break
+		}
+
+		if _, err := destination.Write(buf[:n]); err != nil {
+			source.Close()
+			destination.Close()
+			return err
+		}
+	}
+	return nil
+}

mod/filesystem/hybridBackup/hybridBackup.go

@@ -1,7 +1,12 @@
 package hybridBackup
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
 	"log"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -14,77 +19,222 @@ import (
 	Backup modes suport in this module currently consists of
 
 	Denote P drive as parent drive and B drive as backup drive.
-	1. Smart (smart):
-		- Any new file created in P will be copied to B within 5 minutes
-		- Any file removed in P will be delete from backup after 24 hours
+	1. Basic (basic):
+		- Any new file created in P will be copied to B within 1 minutes
+		- Any file change will be copied to B within 30 minutes
+		- Any file removed in P will be delete from backup if it is > 24 hours old
 	2. Nightly (nightly):
 		- The whole P drive will be copied to N drive every night
-	3. Append Only (append)
-		- Any new file created in P will be copied to B within 5 minutes
-		- No file will be removed from B unless drive is fulled (Similar to CCTV recorder)
+	3. Versioning (version)
+		- A versioning system will be introduce to this backup drive
+		- Just like the time machine
 
+	Tips when developing this module
+	- This is a sub-module of the current file system. Do not import from arozos file system module
+	- If you need any function from the file system, copy and paste it in this module
 */
 
 type BackupConfig struct {
-	CycleCounter  int64  //The number of backup executed in the background
-	LastCycleTime int64  //The execution time of the last cycle
-	DiskUID       string //The UID of the target fsandlr
-	DiskPath      string //The mount point for the disk
-	ParentUID     string //Parent virtal disk UUID
-	ParentPath    string //Parent disk path
-	Mode          string //Backup mode
+	JobName           string           //The name used by the scheduler for executing this config
+	CycleCounter      int64            //The number of backup executed in the background
+	LastCycleTime     int64            //The execution time of the last cycle
+	DiskUID           string           //The UID of the target fsandlr
+	DiskPath          string           //The mount point for the disk
+	ParentUID         string           //Parent virtal disk UUID
+	ParentPath        string           //Parent disk path
+	DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time
+	Mode              string           //Backup mode
+}
+
+func executeBackup(backupConfig *BackupConfig, deepBackup bool) (string, error) {
+	copiedFileList := []string{}
+
+	rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath))
+
+	//Add file cycles
+	fastWalk(rootPath, func(filename string) error {
+		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
+			//Reserved filename, skipping
+			return nil
+		}
+		//Get the target paste location
+		rootAbs, _ := filepath.Abs(rootPath)
+		fileAbs, _ := filepath.Abs(filename)
+
+		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
+		assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath)
+
+		if !deepBackup {
+			//Shallow copy. Only do copy base on file exists or not
+			//This is used to reduce the time for reading the file metatag
+			if !fileExists(assumedTargetPosition) {
+				//Target file not exists in backup disk. Make a copy
+				if !fileExists(filepath.Dir(assumedTargetPosition)) {
+					//Folder containing this file not exists. Create it
+					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
+				}
+
+				//Copy the file to target
+				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
+				if err != nil {
+					log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+				} else {
+					//No problem. Add this filepath into the list
+					copiedFileList = append(copiedFileList, assumedTargetPosition)
+				}
+			}
+		} else {
+			//Deep copy. Check and match the modtime of each file
+			if !fileExists(assumedTargetPosition) {
+				//Copy the file to target
+				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
+				if err != nil {
+					log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+					return nil
+				} else {
+					//No problem. Add this filepath into the list
+					copiedFileList = append(copiedFileList, assumedTargetPosition)
+				}
+			} else {
+				//Target file already exists. Check if their hash matches
+				srcHash, err := getFileHash(fileAbs)
+				if err != nil {
+					log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+					return nil
+				}
+				targetHash, err := getFileHash(assumedTargetPosition)
+				if err != nil {
+					log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.")
+					return nil
+				}
+
+				if srcHash != targetHash {
+					log.Println("[Debug] Hash mismatch. Copying ", fileAbs)
+					//This file has been recently changed. Copy it to new location
+					err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
+					if err != nil {
+						log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+					} else {
+						//No problem. Add this filepath into the list
+						copiedFileList = append(copiedFileList, assumedTargetPosition)
+					}
+				}
+
+			}
+		}
+
+		///Remove file cycle
+		backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath))
+		fastWalk(backupConfig.DiskPath, func(filename string) error {
+			if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
+				//Reserved filename, skipping
+				return nil
+			}
+			//Get the target paste location
+			rootAbs, _ := filepath.Abs(backupDriveRootPath)
+			fileAbs, _ := filepath.Abs(filename)
+
+			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+			thisFileRel := filename[len(backupDriveRootPath):]
+			originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel)))
+
+			//Check if the taget file not exists and this file has been here for more than 24h
+			if !fileExists(originalFileOnDiskPath) {
+				//This file not exists. Check if it is in the delete file marker for more than 24 hours
+				val, ok := backupConfig.DeleteFileMarkers[thisFileRel]
+				if !ok {
+					//This file is newly deleted. Push into the marker map
+					backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix()
+					log.Println("[Debug] Adding " + filename + " to delete marker")
+				} else {
+					//This file has been marked. Check if it is time to delete
+					if time.Now().Unix()-val > 120 {
+						log.Println("[Debug] Deleting " + filename)
+
+						//Remove the backup file
+						os.RemoveAll(filename)
+
+						//Remove file from delete file markers
+						delete(backupConfig.DeleteFileMarkers, thisFileRel)
+					}
+				}
+			}
+			return nil
+		})
+
+		return nil
+	})
+
+	return "", nil
 }
 
 //Main handler function for hybrid backup
 func HandleBackupProcess(backupConfig *BackupConfig) (string, error) {
-	log.Println(">>>>>> Running backup process: ", backupConfig)
-	rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath))
+	log.Println(">>>>>> [Debug] Running backup process: ", backupConfig)
 
 	//Check if the target disk is writable and mounted
-	if fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock")) {
-		//This filesystem is mounted
+	if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) {
+		//This parent filesystem is mounted
 
 	} else {
-		//File system not mounted. Skipping
+		//File system not mounted even after 3 backup cycle. Terminate backup scheduler
 		log.Println("*HybridBackup* Skipping backup cycle for " + backupConfig.ParentUID + ":/")
-		return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted. Skipping", nil
+		return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil
 	}
 
-	if backupConfig.Mode == "smart" {
-		//Smart backup mode. Scan new files every minutes and compare creation time scan every hour minutes
-		if backupConfig.CycleCounter%5 == 0 {
-			//Perform deep backup, use walk function
+	//Check if the backup disk is mounted. If no, stop the scheulder
+	if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) {
+		log.Println("*HybridBackup* Backup schedule stopped for " + backupConfig.DiskUID + ":/")
+		return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted")
+	}
 
+	deepBackup := true //Default perform deep backup
+	if backupConfig.Mode == "basic" {
+		if backupConfig.CycleCounter%30 == 0 {
+			//Perform deep backup, use walk function
+			deepBackup = true
 		} else {
-			//Perform shallow backup
-			fastWalk(rootPath, func(filename string) error {
-				rootAbs, _ := filepath.Abs(rootPath)
-				fileAbs, _ := filepath.Abs(filename)
-
-				rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
-				fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
-
-				relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
-				assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath)
-				if !fileExists(assumedTargetPosition) {
-					//Target file not exists in backup disk. Make a copy
-					//WIP
-					log.Println("Copying ", assumedTargetPosition)
-				}
-
-				return nil
-			})
+			deepBackup = false
 		}
+		backupConfig.LastCycleTime = time.Now().Unix()
+		return executeBackup(backupConfig, deepBackup)
 	} else if backupConfig.Mode == "nightly" {
+		if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 {
+			//24 hours from last backup. Execute deep backup now
+			executeBackup(backupConfig, true)
+			backupConfig.LastCycleTime = time.Now().Unix()
+		}
 
-	} else if backupConfig.Mode == "append" {
+	} else if backupConfig.Mode == "version" {
+		//Do a versioning backup
 
+		//WIP
 	}
 
 	//Add one to the cycle counter
 	backupConfig.CycleCounter++
-	backupConfig.LastCycleTime = time.Now().Unix()
 
 	//Return the log information
 	return "", nil
 }
+
+//Get and return the file hash for a file
+func getFileHash(filename string) (string, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
+}

mod/time/scheduler/scheduler.go

@@ -128,16 +128,16 @@ func (a *Scheduler) createTicker(duration time.Duration) chan bool {
 			case <-ticker.C:
 				//Run jobs
 				for _, thisJob := range a.jobs {
-					log.Println((time.Now().Unix()-thisJob.BaseTime)%thisJob.ExecutionInterval, thisJob.Name)
 					if (time.Now().Unix()-thisJob.BaseTime)%thisJob.ExecutionInterval == 0 {
 						//Execute this job
 						if thisJob.JobType == "function" {
 							//Execute the script function
 							returnvalue, err := thisJob.ScriptFunc()
 							if err != nil {
+								//Execution error. Kill this scheule
 								log.Println(`*Scheduler* Error occured when running task ` + thisJob.Name + ": " + err.Error())
+								a.RemoveJobFromScheduleList(thisJob.Name)
 								cronlog("[ERROR]: " + err.Error())
-								return
 							}
 
 							//Execution suceed. Log the return value

mod/user/directoryHandler.go

@@ -82,10 +82,19 @@ func (u *User) VirtualPathToRealPath(vpath string) (string, error) {
 			if storage.Closed == true {
 				return "", errors.New("Request Filesystem Handler has been closed by another process")
 			}
+
+			//Check if this is a backup drive
+			if storage.Hierarchy == "backup" {
+				return "", errors.New("Request Filesystem Handler do not allow direct access")
+			}
+
+			//Handle general cases
 			if storage.Hierarchy == "user" {
 				return filepath.ToSlash(filepath.Clean(storage.Path) + "/users/" + u.Username + subpath), nil
-			} else {
+			} else if storage.Hierarchy == "public" {
 				return filepath.ToSlash(filepath.Clean(storage.Path) + subpath), nil
+			} else {
+				return "", errors.New("Unknown Filesystem Handler Hierarchy")
 			}
 
 		}

storage.go

@@ -123,14 +123,20 @@ func FilesystemDaemonInit() {
 				break
 			}
 
+			backupConfig.JobName = "backup-daemon [" + thisHandler.UUID + "]"
 			backupConfig.ParentPath = parentFileSystemHandler.Path
+			backupConfig.CycleCounter = 1
 
 			//Debug backup execution
-			backupConfig.CycleCounter = 1
 			hybridBackup.HandleBackupProcess(&backupConfig)
 
+			//Remove the previous job if it exists
+			if systemScheduler.JobExists(backupConfig.JobName) {
+				systemScheduler.RemoveJobFromScheduleList(backupConfig.JobName)
+			}
+
 			//Create a scheudler for this disk
-			systemScheduler.CreateNewScheduledFunctionJob("backup-daemon ["+thisHandler.UUID+"]",
+			systemScheduler.CreateNewScheduledFunctionJob(backupConfig.JobName,
 				"Backup daemon from "+backupConfig.ParentUID+":/ to "+backupConfig.DiskUID+":/",
 				60,
 				func() (string, error) {

web/SystemAO/storage/fshedit.html

@@ -43,6 +43,10 @@
         .true{
             color: #05b074;
         }
+
+        .backuponly{
+            display:none;
+        }
     </style>
 </head>
 <body>
@@ -95,9 +99,35 @@
                 <div class="menu">
                     <div class="item" data-value="user">Isolated User Folders</div>
                     <div class="item" data-value="public">Public Access Folders</div>
+                    <div class="item" data-value="backup">Backup Storage</div>
                 </div>
                 </div>
             </div>
+            <p class="backuponly">Backup Settings</p>
+            <div class="field backuponly">
+                <label>Backup Virtual Disk UID</label>
+                <div class="ui selection dropdown">
+                    <input type="hidden" autocomplete="false" name="parentuid" value="">
+                    <i class="dropdown icon"></i>
+                    <div class="default text">Storage Hierarchy</div>
+                    <div class="menu" id="backupIdList">
+                        
+                    </div>
+                </div>
+                </div>
+                <div class="field backuponly">
+                <label>Backup Mode</label>
+                <div class="ui selection dropdown">
+                    <input type="hidden" autocomplete="false" name="backupmode" value="">
+                    <i class="dropdown icon"></i>
+                    <div class="default text">Storage Hierarchy</div>
+                    <div class="menu">
+                        <div class="item" data-value="basic">Basic</div>
+                        <div class="item" data-value="nightly">Nightly</div>
+                        <div class="item" data-value="version">Versioning</div>
+                    </div>
+                </div>
+                </div>
             <div class="ui divider"></div>
             <p>Physical Disks Settings</p>
             <div class="field">
@@ -155,9 +185,29 @@
         $(".ui.checkbox").checkbox();
 
         if (window.location.hash.length > 0){
+            //Get a list of vroot from system
+            $("#backupIdList").html(``);
+            $.get("../../system/storage/pool/list", function(data){
+                data.forEach(usergroup => {
+                    if (usergroup.Storages != null){
+                        usergroup.Storages.forEach(storage => {
+                            $("#backupIdList").append(`<div class="item" data-value="${storage.UUID}">${storage.Name} (${storage.UUID}:/)</div>`);
+                        })
+                    }
+                    
+                });
+                $("#backupIdList").parent().dropdown();
+                renderFSHCurrentSettings();
+            });
+
+            
+        }
+
+        function renderFSHCurrentSettings(){
+            //Request server side to provide info on this FSH
             var input = JSON.parse(decodeURIComponent(window.location.hash.substr(1)));
             $("#groupfield").val(input.group);
-            //Request server side to provide info on this FSH
+
             $.ajax({
                 url: "../../system/storage/pool/edit",
                 method: "GET",
@@ -177,6 +227,12 @@
             $("input[name=path]").val(option.path);
             $("#accessfield").dropdown("set selected", option.access);
             $("#hierarchyfield").dropdown("set selected", option.hierarchy);
+            if (option.hierarchy == "backup"){
+                $(".backuponly").slideDown("fast");
+                $("input[name=backupmode]").dropdown("set selected",option.backupmode);
+                $("#backupIdList").parent().dropdown();
+                $("#backupIdList").parent().dropdown("set selected",option.backupmode);
+            }
             $("#fstype").dropdown("set selected",option.filesystem);
             $("input[name=mountdev]").val(option.mountdev);
             $("input[name=mountpt]").val(option.mountpt);

web/SystemAO/storage/poolEditor.html

@@ -135,9 +135,9 @@
                         <i class="dropdown icon"></i>
                         <div class="default text">Storage Hierarchy</div>
                         <div class="menu">
-                            <div class="item" data-value="smart">Smart</div>
+                            <div class="item" data-value="basic">Basic</div>
                             <div class="item" data-value="nightly">Nightly</div>
-                            <div class="item" data-value="append">Append Only</div>
+                            <div class="item" data-value="version">Versioning</div>
                         </div>
                     </div>
                   </div>
@@ -346,9 +346,11 @@
                 $("#backupIdList").html(``);
                 $.get("../../system/storage/pool/list", function(data){
                      data.forEach(usergroup => {
-                         usergroup.Storages.forEach(storage => {
-                             $("#backupIdList").append(`<div class="item" data-value="${storage.UUID}">${storage.Name} (${storage.UUID}:/)</div>`);
-                         })
+                         if (usergroup.Storages != null){
+                            usergroup.Storages.forEach(storage => {
+                                $("#backupIdList").append(`<div class="item" data-value="${storage.UUID}">${storage.Name} (${storage.UUID}:/)</div>`);
+                            })
+                        }
                      });
                      $("#backupIdList").parent().dropdown();
                 });