Added work in progress version backup mechanism

backup.go

@@ -55,12 +55,6 @@ func backup_restoreSelected(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	//Check if the path exists in the backup disk
-	if !fileExists(filepath.Join(fsh.Path, relpath)) {
-		sendErrorResponse(w, "Restore source file not exists")
-		return
-	}
-
 	//Handle restore of the file
 	err = userinfo.HomeDirectories.HyperBackupManager.HandleRestore(fsh.UUID, relpath)
 	if err != nil {

mod/auth/oauth2/oauth2.go

@@ -176,33 +176,45 @@ func (oh *OauthHandler) WriteConfig(w http.ResponseWriter, r *http.Request) {
 
 	oh.coredb.Write("oauth", "enabled", enabled)
 
+	showError := true
 	if enabled != "true" {
-		return
+		showError = false
 	}
 
 	idp, err := mv(r, "idp", true)
 	if err != nil {
-		sendErrorResponse(w, "idp field can't be empty'")
-		return
+		if showError {
+			sendErrorResponse(w, "idp field can't be empty'")
+			return
+		}
 	}
 	redirecturl, err := mv(r, "redirecturl", true)
 	if err != nil {
-		sendErrorResponse(w, "redirecturl field can't be empty'")
-		return
+		if showError {
+			sendErrorResponse(w, "redirecturl field can't be empty'")
+			return
+		}
 	}
 	clientid, err := mv(r, "clientid", true)
 	if err != nil {
-		sendErrorResponse(w, "clientid field can't be empty'")
+		if showError {
+			sendErrorResponse(w, "clientid field can't be empty'")
+			return
+		}
 	}
 	clientsecret, err := mv(r, "clientsecret", true)
 	if err != nil {
-		sendErrorResponse(w, "clientsecret field can't be empty'")
-		return
+		if showError {
+			sendErrorResponse(w, "clientsecret field can't be empty'")
+			return
+		}
 	}
 	defaultusergroup, err := mv(r, "defaultusergroup", true)
 	if err != nil {
-		sendErrorResponse(w, "defaultusergroup field can't be empty'")
-		return
+		if showError {
+			sendErrorResponse(w, "defaultusergroup field can't be empty'")
+			return
+		}
 	}
 
 	oh.coredb.Write("oauth", "idp", idp)

mod/disk/hybridBackup/basicBackup.go

@@ -0,0 +1,186 @@
+package hybridBackup
+
+import (
+	"errors"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+/*
+	Basic Backup
+
+	This script handle basic backup process
+*/
+
+func executeBackup(backupConfig *BackupTask, deepBackup bool) (string, error) {
+	copiedFileList := []string{}
+
+	rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath))
+
+	//Check if the backup parent root is identical / within backup disk
+	parentRootAbs, err := filepath.Abs(backupConfig.ParentPath)
+	if err != nil {
+		return "", errors.New("Unable to resolve parent disk path")
+	}
+
+	backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/backup/"))
+	if err != nil {
+		return "", errors.New("Unable to resolve backup disk path")
+	}
+
+	if len(parentRootAbs) >= len(backupRootAbs) {
+		if parentRootAbs[:len(backupRootAbs)] == backupRootAbs {
+			//parent root is within backup root. Raise configuration error
+			log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive")
+			return "", errors.New("Configuration Error. Skipping backup cycle.")
+		}
+	}
+
+	//Add file cycles
+	fastWalk(rootPath, func(filename string) error {
+		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
+			//Reserved filename, skipping
+			return nil
+		}
+		//Get the target paste location
+		rootAbs, _ := filepath.Abs(rootPath)
+		fileAbs, _ := filepath.Abs(filename)
+
+		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
+		assumedTargetPosition := filepath.Join(backupConfig.DiskPath, "/backup/", relPath)
+
+		if !deepBackup {
+			//Shallow copy. Only do copy base on file exists or not
+			//This is used to reduce the time for reading the file metatag
+			if !fileExists(assumedTargetPosition) {
+				//Target file not exists in backup disk. Make a copy
+				if !fileExists(filepath.Dir(assumedTargetPosition)) {
+					//Folder containing this file not exists. Create it
+					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
+				}
+
+				//Copy the file to target
+				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
+				if err != nil {
+					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+				} else {
+					//No problem. Add this filepath into the list
+					copiedFileList = append(copiedFileList, assumedTargetPosition)
+				}
+
+			}
+		} else {
+			//Deep copy. Check and match the modtime of each file
+			if !fileExists(assumedTargetPosition) {
+				if !fileExists(filepath.Dir(assumedTargetPosition)) {
+					//Folder containing this file not exists. Create it
+					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
+				}
+
+				//Copy the file to target
+				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
+				if err != nil {
+					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+					return nil
+				} else {
+					//No problem. Add this filepath into the list
+					copiedFileList = append(copiedFileList, assumedTargetPosition)
+				}
+			} else {
+				//Target file already exists.
+				//Check if it has been modified since the last cycle time
+				lastModTime := lastModTime(fileAbs)
+				if lastModTime > backupConfig.LastCycleTime {
+					//Check if their hash matches
+					srcHash, err := getFileHash(fileAbs)
+					if err != nil {
+						log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+						return nil
+					}
+					targetHash, err := getFileHash(assumedTargetPosition)
+					if err != nil {
+						log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.")
+						return nil
+					}
+
+					if srcHash != targetHash {
+						log.Println("[Debug] Hash mismatch. Copying ", fileAbs)
+						//This file has been recently changed. Copy it to new location
+						err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
+						if err != nil {
+							log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+						} else {
+							//No problem. Add this filepath into the list
+							copiedFileList = append(copiedFileList, assumedTargetPosition)
+						}
+
+						//Check if this file is in the remove marker list. If yes, pop it from the list
+						_, ok := backupConfig.DeleteFileMarkers[relPath]
+						if ok {
+							//File exists. remove it from delete file amrker
+							delete(backupConfig.DeleteFileMarkers, relPath)
+							log.Println("Removing ", relPath, " from delete marker list")
+						}
+					}
+				}
+
+			}
+		}
+
+		///Remove file cycle
+		backupDriveRootPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.DiskPath, "/backup/")))
+		fastWalk(backupConfig.DiskPath, func(filename string) error {
+			if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
+				//Reserved filename, skipping
+				return nil
+			}
+			//Get the target paste location
+			rootAbs, _ := filepath.Abs(backupDriveRootPath)
+			fileAbs, _ := filepath.Abs(filename)
+
+			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+			thisFileRel := filename[len(backupDriveRootPath):]
+			originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel)))
+
+			//Check if the taget file not exists and this file has been here for more than 24h
+			if !fileExists(originalFileOnDiskPath) {
+				//This file not exists. Check if it is in the delete file marker for more than 24 hours
+				val, ok := backupConfig.DeleteFileMarkers[thisFileRel]
+				if !ok {
+					//This file is newly deleted. Push into the marker map
+					backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix()
+					log.Println("[Debug] Adding " + filename + " to delete marker")
+				} else {
+					//This file has been marked. Check if it is time to delete
+					if time.Now().Unix()-val > 3600*24 {
+						log.Println("[Debug] Deleting " + filename)
+
+						//Remove the backup file
+						os.RemoveAll(filename)
+
+						//Remove file from delete file markers
+						delete(backupConfig.DeleteFileMarkers, thisFileRel)
+					}
+				}
+			}
+			return nil
+		})
+
+		return nil
+	})
+
+	return "", nil
+}
+
+func listBasicRestorables(task *BackupTask) ([]*RestorableFile, error) {
+	restorableFiles, err := task.compareRootPaths()
+	return restorableFiles, err
+}

mod/disk/hybridBackup/compareRoots.go

@@ -1,7 +1,6 @@
 package hybridBackup
 
 import (
-	"errors"
 	"os"
 	"path/filepath"
 	"time"
@@ -16,46 +15,41 @@ import (
 
 */
 
-//This function check which file exists in backup but not source drive
+//This function check which file exists in backup but not source drive.
+//Only usable for basic and nightly backup mode
 func (t *BackupTask) compareRootPaths() ([]*RestorableFile, error) {
 	results := []*RestorableFile{}
 
 	//Check if the source and the backup disk exists
-	if t.Mode == "basic" || t.Mode == "nightly" {
-		for key, value := range t.DeleteFileMarkers {
-			//Check if the source file exists
-			assumedSourcePosition := filepath.Join(t.ParentPath, key)
-			backupFilePosition := filepath.Join(t.DiskPath, key)
-			if !fileExists(assumedSourcePosition) && fileExists(backupFilePosition) {
-				//This is a restorable file
-				var filesize int64 = 0
-				fi, err := os.Stat(backupFilePosition)
-				if err != nil {
-					filesize = 0
-				} else {
-					filesize = fi.Size()
-				}
-
-				fileIsHidden, _ := hidden.IsHidden(backupFilePosition, true)
-
-				//Create the Restorable File
-				thisFile := RestorableFile{
-					Filename:      filepath.Base(key),
-					IsHidden:      fileIsHidden,
-					Filesize:      filesize,
-					RelpathOnDisk: filepath.ToSlash(key),
-					RestorePoint:  filepath.ToSlash(assumedSourcePosition),
-					BackupDiskUID: t.DiskUID,
-					RemainingTime: 86400 - (time.Now().Unix() - value),
-					DeleteTime:    value,
-				}
-				results = append(results, &thisFile)
+	for key, value := range t.DeleteFileMarkers {
+		//Check if the source file exists
+		assumedSourcePosition := filepath.Join(t.ParentPath, key)
+		backupFilePosition := filepath.Join(t.DiskPath, "/backup/", key)
+		if !fileExists(assumedSourcePosition) && fileExists(backupFilePosition) {
+			//This is a restorable file
+			var filesize int64 = 0
+			fi, err := os.Stat(backupFilePosition)
+			if err != nil {
+				filesize = 0
+			} else {
+				filesize = fi.Size()
 			}
-		}
-	} else if t.Mode == "version" {
 
-	} else {
-		return []*RestorableFile{}, errors.New("Uknown backup mode: " + t.Mode)
+			fileIsHidden, _ := hidden.IsHidden(backupFilePosition, true)
+
+			//Create the Restorable File
+			thisFile := RestorableFile{
+				Filename:      filepath.Base(key),
+				IsHidden:      fileIsHidden,
+				Filesize:      filesize,
+				RelpathOnDisk: filepath.ToSlash(key),
+				RestorePoint:  filepath.ToSlash(assumedSourcePosition),
+				BackupDiskUID: t.DiskUID,
+				RemainingTime: 86400 - (time.Now().Unix() - value),
+				DeleteTime:    value,
+			}
+			results = append(results, &thisFile)
+		}
 	}
 
 	return results, nil

mod/disk/hybridBackup/fileCopy.go → mod/disk/hybridBackup/fileUtil.go

@@ -48,3 +48,28 @@ func BufferedLargeFileCopy(src string, dst string, BUFFERSIZE int64) error {
 	}
 	return nil
 }
+
+//Get the last modification tiem of a given file
+func lastModTime(filename string) int64 {
+	file, err := os.Stat(filename)
+
+	if err != nil {
+		return 0
+	}
+
+	modifiedtime := file.ModTime()
+	return modifiedtime.Unix()
+}
+
+func isDir(filename string) bool {
+	file, err := os.Open(filename)
+	if err != nil {
+		return false
+	}
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return false
+	}
+
+	return fileInfo.IsDir()
+}

mod/disk/hybridBackup/hybridBackup.go

@@ -157,165 +157,6 @@ func (m *Manager) Close() error {
 	return nil
 }
 
-func executeBackup(backupConfig *BackupTask, deepBackup bool) (string, error) {
-	copiedFileList := []string{}
-
-	rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath))
-
-	//Check if the backup parent root is identical / within backup disk
-	parentRootAbs, err := filepath.Abs(backupConfig.ParentPath)
-	if err != nil {
-		return "", errors.New("Unable to resolve parent disk path")
-	}
-
-	backupRootAbs, err := filepath.Abs(backupConfig.DiskPath)
-	if err != nil {
-		return "", errors.New("Unable to resolve backup disk path")
-	}
-
-	if len(parentRootAbs) >= len(backupRootAbs) {
-		if parentRootAbs[:len(backupRootAbs)] == backupRootAbs {
-			//parent root is within backup root. Raise configuration error
-			log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive")
-			return "", errors.New("Configuration Error. Skipping backup cycle.")
-		}
-	}
-
-	//Add file cycles
-	fastWalk(rootPath, func(filename string) error {
-		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
-			//Reserved filename, skipping
-			return nil
-		}
-		//Get the target paste location
-		rootAbs, _ := filepath.Abs(rootPath)
-		fileAbs, _ := filepath.Abs(filename)
-
-		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
-		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
-
-		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
-		assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath)
-
-		if !deepBackup {
-			//Shallow copy. Only do copy base on file exists or not
-			//This is used to reduce the time for reading the file metatag
-			if !fileExists(assumedTargetPosition) {
-				//Target file not exists in backup disk. Make a copy
-				if !fileExists(filepath.Dir(assumedTargetPosition)) {
-					//Folder containing this file not exists. Create it
-					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
-				}
-
-				//Copy the file to target
-				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
-				if err != nil {
-					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
-				} else {
-					//No problem. Add this filepath into the list
-					copiedFileList = append(copiedFileList, assumedTargetPosition)
-				}
-
-			}
-		} else {
-			//Deep copy. Check and match the modtime of each file
-			if !fileExists(assumedTargetPosition) {
-				if !fileExists(filepath.Dir(assumedTargetPosition)) {
-					//Folder containing this file not exists. Create it
-					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
-				}
-
-				//Copy the file to target
-				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
-				if err != nil {
-					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
-					return nil
-				} else {
-					//No problem. Add this filepath into the list
-					copiedFileList = append(copiedFileList, assumedTargetPosition)
-				}
-			} else {
-				//Target file already exists. Check if their hash matches
-				srcHash, err := getFileHash(fileAbs)
-				if err != nil {
-					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
-					return nil
-				}
-				targetHash, err := getFileHash(assumedTargetPosition)
-				if err != nil {
-					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.")
-					return nil
-				}
-
-				if srcHash != targetHash {
-					log.Println("[Debug] Hash mismatch. Copying ", fileAbs)
-					//This file has been recently changed. Copy it to new location
-					err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
-					if err != nil {
-						log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
-					} else {
-						//No problem. Add this filepath into the list
-						copiedFileList = append(copiedFileList, assumedTargetPosition)
-					}
-
-					//Check if this file is in the remove marker list. If yes, pop it from the list
-					_, ok := backupConfig.DeleteFileMarkers[relPath]
-					if ok {
-						//File exists. remove it from delete file amrker
-						delete(backupConfig.DeleteFileMarkers, relPath)
-						log.Println("Removing ", relPath, " from delete marker list")
-					}
-				}
-			}
-		}
-
-		///Remove file cycle
-		backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath))
-		fastWalk(backupConfig.DiskPath, func(filename string) error {
-			if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
-				//Reserved filename, skipping
-				return nil
-			}
-			//Get the target paste location
-			rootAbs, _ := filepath.Abs(backupDriveRootPath)
-			fileAbs, _ := filepath.Abs(filename)
-
-			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
-			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
-
-			thisFileRel := filename[len(backupDriveRootPath):]
-			originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel)))
-
-			//Check if the taget file not exists and this file has been here for more than 24h
-			if !fileExists(originalFileOnDiskPath) {
-				//This file not exists. Check if it is in the delete file marker for more than 24 hours
-				val, ok := backupConfig.DeleteFileMarkers[thisFileRel]
-				if !ok {
-					//This file is newly deleted. Push into the marker map
-					backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix()
-					log.Println("[Debug] Adding " + filename + " to delete marker")
-				} else {
-					//This file has been marked. Check if it is time to delete
-					if time.Now().Unix()-val > 3600*24 {
-						log.Println("[Debug] Deleting " + filename)
-
-						//Remove the backup file
-						os.RemoveAll(filename)
-
-						//Remove file from delete file markers
-						delete(backupConfig.DeleteFileMarkers, thisFileRel)
-					}
-				}
-			}
-			return nil
-		})
-
-		return nil
-	})
-
-	return "", nil
-}
-
 //Main handler function for hybrid backup
 func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
 	log.Println(">>>>>> [Debug] Running backup process: ", backupConfig)
@@ -343,8 +184,8 @@ func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
 			deepBackup = true
 		} else {
 			deepBackup = false
+			backupConfig.LastCycleTime = time.Now().Unix()
 		}
-		backupConfig.LastCycleTime = time.Now().Unix()
 		return executeBackup(backupConfig, deepBackup)
 	} else if backupConfig.Mode == "nightly" {
 		if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 {
@@ -355,8 +196,11 @@ func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
 
 	} else if backupConfig.Mode == "version" {
 		//Do a versioning backup
-		log.Println("[WIP] This function is still work in progress. Please do not use version backup for now.")
-		//WIP
+		if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 || backupConfig.CycleCounter == 0 {
+			//Scheduled backup or initial backup
+			executeVersionBackup(backupConfig)
+			backupConfig.LastCycleTime = time.Now().Unix()
+		}
 	}
 
 	//Add one to the cycle counter
@@ -388,6 +232,12 @@ func (m *Manager) HandleRestore(restoreDiskID string, targetFileRelpath string)
 	log.Println("[debug]", backupTask)
 
 	restoreSource := filepath.Join(backupTask.DiskPath, targetFileRelpath)
+	if backupTask.Mode == "basic" || backupTask.Mode == "nightly" {
+		restoreSource = filepath.Join(backupTask.DiskPath, "/backup/", targetFileRelpath)
+	} else if backupTask.Mode == "version" {
+		restoreSource = filepath.Join(backupTask.DiskPath, "/versions/", targetFileRelpath)
+	}
+
 	restoreTarget := filepath.Join(backupTask.ParentPath, targetFileRelpath)
 
 	if !fileExists(restoreSource) {
@@ -427,14 +277,22 @@ func (m *Manager) ListRestorable(parentDiskID string) (RestorableReport, error)
 
 	//Extract all comparasion
 	for _, task := range tasks {
-		restorableFiles, err := task.compareRootPaths()
-		if err != nil {
-			//Unable to list restorable. SKip this
-		} else {
+		if task.Mode == "basic" || task.Mode == "nightly" {
+			restorableFiles, err := listBasicRestorables(task)
+			if err != nil {
+				//Something went wrong. Skip this
+				continue
+			}
 			for _, restorable := range restorableFiles {
 				diffFiles = append(diffFiles, restorable)
 			}
+		} else if task.Mode == "version" {
+
+		} else {
+			//Unknown mode. Skip it
+
 		}
+
 	}
 
 	//Create a Restorable Report

mod/disk/hybridBackup/versionBackup.go

@@ -0,0 +1,247 @@
+package hybridBackup
+
+import (
+	"errors"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+/*
+	VersionBackup.go
+
+	This scirpt file backup the data in the system nightly and create a restore point
+	for the day just like BRTFS
+*/
+
+func executeVersionBackup(backupConfig *BackupTask) (string, error) {
+	//Check if the backup parent root is identical / within backup disk
+	parentRootAbs, err := filepath.Abs(backupConfig.ParentPath)
+	if err != nil {
+		return "", errors.New("Unable to resolve parent disk path")
+	}
+
+	backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/version/"))
+	if err != nil {
+		return "", errors.New("Unable to resolve backup disk path")
+	}
+
+	if len(parentRootAbs) >= len(backupRootAbs) {
+		if parentRootAbs[:len(backupRootAbs)] == backupRootAbs {
+			//parent root is within backup root. Raise configuration error
+			log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive")
+			return "", errors.New("Configuration Error. Skipping backup cycle.")
+		}
+	}
+
+	todayFolderName := time.Now().Format("2006-01-02")
+	previousSnapshotName, _ := getPreviousSnapshotName(backupConfig, todayFolderName)
+	snapshotLocation := filepath.Join(backupConfig.DiskPath, "/version/", todayFolderName)
+	previousSnapshotLocation := filepath.Join(backupConfig.DiskPath, "/version/", previousSnapshotName)
+
+	if !fileExists(snapshotLocation) {
+		//Create today folder if not exist
+		os.MkdirAll(snapshotLocation, 0755)
+	}
+
+	/*
+		Run a three pass compare logic between
+		1. source disk and new backup disk to check any new / modified files (created today)
+		2. yesterday backup and today backup to check any deleted files (created before, deleted today)
+		3. file in today backup disk no longer in the current source disk (created today, deleted today)
+	*/
+	copiedFileList := []string{}
+
+	//First pass: Check if there are any updated file from source and backup it to backup drive
+	fastWalk(parentRootAbs, func(filename string) error {
+		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
+			//Reserved filename, skipping
+			return nil
+		}
+
+		//Get the target paste location
+		rootAbs, _ := filepath.Abs(backupConfig.ParentPath)
+		fileAbs, _ := filepath.Abs(filename)
+
+		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
+		fileBackupLocation := filepath.Join(backupConfig.DiskPath, "/version/", todayFolderName, relPath)
+		yesterdayBackupLocation := filepath.Join(previousSnapshotLocation, relPath)
+
+		//Check if the file exists
+		if !fileExists(fileBackupLocation) && !fileExists(yesterdayBackupLocation) {
+			//File not exists in both current source and yesterday one. Copy it to the target location
+			if !isDir(fileBackupLocation) && fileExists(fileBackupLocation+".deleted") {
+				os.Remove(fileBackupLocation + ".deleted")
+			}
+
+			if !fileExists(filepath.Dir(fileBackupLocation)) {
+				os.MkdirAll(filepath.Dir(fileBackupLocation), 0755)
+			}
+
+			err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
+			if err != nil {
+				log.Println("[HybridBackup] Failed to copy file: ", filepath.Base(filename)+". "+err.Error())
+			}
+
+			copiedFileList = append(copiedFileList, fileBackupLocation)
+
+		} else if fileExists(yesterdayBackupLocation) {
+			//The file exists in the last snapshot
+			//Check if their hash is the same. If no, update it
+			srcHash, err := getFileHash(fileAbs)
+			if err != nil {
+				log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+				return nil
+			}
+			targetHash, err := getFileHash(yesterdayBackupLocation)
+			if err != nil {
+				log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileBackupLocation), err.Error(), " Skipping.")
+				return nil
+			}
+
+			if srcHash != targetHash {
+				//Hash mismatch. Overwrite the file
+				err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
+				if err != nil {
+					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+				} else {
+					//No problem. Add this filepath into the list
+					copiedFileList = append(copiedFileList, fileBackupLocation)
+				}
+			}
+		} else {
+			//Default case
+			lastModTime := lastModTime(fileAbs)
+			if lastModTime > backupConfig.LastCycleTime {
+				//Check if hash the same
+				srcHash, err := getFileHash(fileAbs)
+				if err != nil {
+					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+					return nil
+				}
+				targetHash, err := getFileHash(fileBackupLocation)
+				if err != nil {
+					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileBackupLocation), err.Error(), " Skipping.")
+					return nil
+				}
+
+				if srcHash != targetHash {
+					//Hash mismatch. Overwrite the file
+					err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
+					if err != nil {
+						log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
+					} else {
+						//No problem. Add this filepath into the list
+						copiedFileList = append(copiedFileList, fileBackupLocation)
+					}
+				}
+			}
+		}
+
+		return nil
+	})
+
+	//2nd pass: Check if there are anything exists in the previous backup but no longer exists in the source now
+	//For case where the file is backed up in previous snapshot but now the file has been removed
+	if fileExists(previousSnapshotLocation) {
+		fastWalk(previousSnapshotLocation, func(filename string) error {
+			//Get the target paste location
+			rootAbs, _ := filepath.Abs(previousSnapshotLocation)
+			fileAbs, _ := filepath.Abs(filename)
+
+			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+			relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
+			sourcAssumeLocation := filepath.Join(parentRootAbs, relPath)
+			todaySnapshotLocation := filepath.Join(snapshotLocation, relPath)
+
+			if !fileExists(sourcAssumeLocation) {
+				//File exists in yesterday snapshot but not in the current source
+				//Assume it has been deleted, create a dummy indicator file
+				ioutil.WriteFile(todaySnapshotLocation+".deleted", []byte(""), 0755)
+			}
+			return nil
+		})
+	}
+
+	//3rd pass: Check if there are anything (except file with .deleted) in today backup drive that didn't exists in the source drive
+	//For cases where the backup is applied to overwrite an eariler backup of the same day
+	fastWalk(snapshotLocation, func(filename string) error {
+		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
+			//Reserved filename, skipping
+			return nil
+		}
+
+		if filepath.Ext(filename) == ".deleted" {
+			//Deleted file marker. Skip this
+			return nil
+		}
+
+		//Get the target paste location
+		rootAbs, _ := filepath.Abs(snapshotLocation)
+		fileAbs, _ := filepath.Abs(filename)
+
+		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
+		sourceAssumedLocation := filepath.Join(parentRootAbs, relPath)
+
+		if !fileExists(sourceAssumedLocation) {
+			//File removed from the source. Delete it from backup as well
+			os.Remove(filename)
+		}
+		return nil
+	})
+
+	return "", nil
+}
+
+//Return the previous snapshot for the currentSnspashot
+func getPreviousSnapshotName(backupConfig *BackupTask, currentSnapshotName string) (string, error) {
+	//Resolve the backup root folder
+	backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/version/"))
+	if err != nil {
+		return "", errors.New("Unable to get the previous snapshot directory")
+	}
+
+	//Get the snapshot list and extract the snapshot date from foldername
+	existingSnapshots := []string{}
+	files, _ := filepath.Glob(filepath.ToSlash(filepath.Clean(backupRootAbs)) + "/*")
+	for _, file := range files {
+		if isDir(file) {
+			existingSnapshots = append(existingSnapshots, filepath.Base(file))
+		}
+	}
+
+	if len(existingSnapshots) == 0 {
+		return "", errors.New("No snapshot found")
+	}
+
+	//Check if the current snapshot exists, if not, return the latest one
+	previousSnapshotName := ""
+	if fileExists(filepath.Join(backupRootAbs, currentSnapshotName)) {
+		//Current snapshot exists. Find the one just above it
+		lastSnapshotName := existingSnapshots[0]
+		for _, snapshotName := range existingSnapshots {
+			if snapshotName == currentSnapshotName {
+				//This is the correct snapshot name. Get the last one as previous snapshot
+				previousSnapshotName = lastSnapshotName
+			} else {
+				lastSnapshotName = snapshotName
+			}
+		}
+	} else {
+		//Current snapshot not exists. Use the last item in snapshots list
+		previousSnapshotName = existingSnapshots[len(existingSnapshots)-1]
+	}
+
+	return previousSnapshotName, nil
+}

mod/filesystem/filesystem.go

@@ -110,7 +110,7 @@ func NewFileSystemHandler(option FileSystemOption) (*FileSystemHandler, error) {
 			//Backup disk. Create an Hierarchy Config for this drive
 			hierarchySpecificConfig = hybridBackup.BackupTask{
 				CycleCounter:      0,
-				LastCycleTime:     time.Now().Unix(),
+				LastCycleTime:     0,
 				DiskUID:           option.Uuid,
 				DiskPath:          option.Path,
 				ParentUID:         option.Parentuid,

mod/storage/storage.go

@@ -89,9 +89,13 @@ func (s *StoragePool) HasHigherOrEqualPermissionThan(a *StoragePool) bool {
 
 //Close all fsHandler under this storage pool
 func (s *StoragePool) Close() {
+	//For each storage pool, close it
 	for _, fsh := range s.Storages {
 		fsh.Close()
 	}
+
+	//Close the running backup tasks
+	s.HyperBackupManager.Close()
 }
 
 //Helper function

web/SystemAO/disk/disk_restore.html

@@ -96,7 +96,7 @@
        </div>
        <div id="succ" class="ui green segment" style="display:none;">
         <h4 class="ui header">
-            <i class="checkmark icon"></i>
+            <i class="green checkmark icon"></i>
             <div class="content">
                 File Restored
                 <div class="sub reason header openfolder"><a style="cursor: pointer" onclick="handleOpenRestoreLocation()"><i class="ui folder open icon"></i> Open Location</a></div>

web/SystemAO/storage/fshedit.html

@@ -92,7 +92,7 @@
             </div>
             <div class="field">
                 <label>Storage Hierarchy</label>
-                <div id="hierarchyfield" class="ui selection dropdown">
+                <div id="hierarchyfield" class="ui selection dropdown" onchange="handleHierarchyChange(this);">
                 <input type="hidden" name="hierarchy" value="user">
                 <i class="dropdown icon"></i>
                 <div class="default text">Storage Hierarchy</div>
@@ -118,7 +118,7 @@
                 <div class="field backuponly">
                 <label>Backup Mode</label>
                 <div class="ui selection dropdown">
-                    <input type="hidden" autocomplete="false" name="backupmode" value="">
+                    <input id="backupmode" type="hidden" autocomplete="false" name="backupmode" value="">
                     <i class="dropdown icon"></i>
                     <div class="default text">Storage Hierarchy</div>
                     <div class="menu">
@@ -192,7 +192,7 @@
                     if (usergroup.Storages != null){
                         usergroup.Storages.forEach(storage => {
                             $("#backupIdList").append(`<div class="item" data-value="${storage.UUID}">${storage.Name} (${storage.UUID}:/)</div>`);
-                        })
+                        });
                     }
                     
                 });
@@ -220,6 +220,17 @@
             $("#mainForm").attr("action", "../../system/storage/pool/edit?opr=set&uuid=" + input.uuid + "&group=" + input.group);
         }
 
+        function handleHierarchyChange(object){
+            var newHierarcy = $(object).find("input").val();
+            //Force access mode to readonly if backup mode
+            if (newHierarcy == "backup"){
+                $("#accessfield").dropdown("set selected", "readonly")
+                $("#accessfield").addClass("disabled");
+            }else{
+                $("#accessfield").removeClass("disabled");
+            }
+        }
+
         function renderOptionsToForm(option){
             console.log(option);
             $("input[name=name]").val(option.name);
@@ -228,10 +239,16 @@
             $("#accessfield").dropdown("set selected", option.access);
             $("#hierarchyfield").dropdown("set selected", option.hierarchy);
             if (option.hierarchy == "backup"){
+                //Show backup drive options
                 $(".backuponly").slideDown("fast");
+                //Set backup mode
                 $("input[name=backupmode]").dropdown("set selected",option.backupmode);
                 $("#backupIdList").parent().dropdown();
-                $("#backupIdList").parent().dropdown("set selected",option.backupmode);
+                //Set parent id and backup mode from dropdown
+                $("#backupIdList").parent().dropdown("set selected",option.parentuid);
+                $("#backupmode").parent().dropdown("set selected",option.backupmode);
+                //Disable the readonly settings
+                $("#accessfield").addClass("disabled");
             }
             $("#fstype").dropdown("set selected",option.filesystem);
             $("input[name=mountdev]").val(option.mountdev);