Added snapshot merge function and auto merge if disk out of space

mod/disk/hybridBackup/fileUtil.go

@@ -73,3 +73,13 @@ func isDir(filename string) bool {
 
 	return fileInfo.IsDir()
 }
+
+func fileSize(filename string) int64 {
+	fi, err := os.Stat("/path/to/file")
+	if err != nil {
+		return -1
+	}
+	// get the size
+	size := fi.Size()
+	return size
+}

mod/disk/hybridBackup/hybridBackup.go

@@ -56,6 +56,7 @@ type BackupTask struct {
 	Database          *database.Database //The database for storing requried data
 	Mode              string             //Backup mode
 	PanicStopped      bool               //If the backup process has been stopped due to panic situationc
+	ErrorMessage      string             //Panic stop message
 }
 
 //A snapshot summary
@@ -101,13 +102,18 @@ func NewHyperBackupManager() *Manager {
 
 	///Create task executor
 	go func() {
-		defer log.Println("[HybridBackup] Ticker Stopped")
+		defer log.Println("HybridBackup stopped")
 		for {
 			select {
 			case <-ticker.C:
 				for _, task := range newManager.Tasks {
 					if task.Enabled == true {
-						task.HandleBackupProcess()
+						output, err := task.HandleBackupProcess()
+						if err != nil {
+							task.Enabled = false
+							task.PanicStopped = true
+							task.ErrorMessage = output
+						}
 					}
 				}
 			case <-stopper:
@@ -175,8 +181,16 @@ func (m *Manager) StartTask(jobname string) {
 			//Enable to job
 			task.Enabled = true
 
-			//Run it once
-			task.HandleBackupProcess()
+			//Run it once in go routine
+			go func() {
+				output, err := task.HandleBackupProcess()
+				if err != nil {
+					task.Enabled = false
+					task.PanicStopped = true
+					task.ErrorMessage = output
+				}
+			}()
+
 		}
 	}
 }

mod/disk/hybridBackup/linker.go

@@ -2,8 +2,10 @@ package hybridBackup
 
 import (
 	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 )
 
 /*
@@ -48,11 +50,45 @@ func readLinkFile(snapshotFolder string) (*LinkFileMap, error) {
 				return &lfContent, nil
 			}
 		}
+	} else {
+		return &result, errors.New("Linker file not exists")
 	}
 
 	return &result, nil
 }
 
+//Update the linker by given a snapshot name to a new one
+func updateLinkerPointer(snapshotFolder string, oldSnapshotLink string, newSnapshotLink string) error {
+	oldSnapshotLink = strings.TrimSpace(oldSnapshotLink)
+	newSnapshotLink = strings.TrimSpace(newSnapshotLink)
+
+	//Load the old linker file
+	oldlinkMap, err := readLinkFile(snapshotFolder)
+	if err != nil {
+		return err
+	}
+
+	//Iterate and replace all link that is pointing to the same snapshot
+	newLinkMap := LinkFileMap{
+		UnchangedFile: map[string]string{},
+		DeletedFiles:  map[string]string{},
+	}
+
+	for rel, link := range oldlinkMap.UnchangedFile {
+		if link == oldSnapshotLink {
+			link = newSnapshotLink
+		}
+		newLinkMap.UnchangedFile[rel] = link
+	}
+
+	for rel, ts := range oldlinkMap.DeletedFiles {
+		newLinkMap.DeletedFiles[rel] = ts
+	}
+
+	//Write it back to file
+	return generateLinkFile(snapshotFolder, newLinkMap)
+}
+
 //Check if a file exists in a linkFileMap. return boolean and its linked to snapshot name
 func (lfm *LinkFileMap) fileExists(fileRelPath string) (bool, string) {
 	val, ok := lfm.UnchangedFile[filepath.ToSlash(fileRelPath)]

mod/disk/hybridBackup/restoreSnapshot.go → mod/disk/hybridBackup/snaoshotOpr.go

@@ -2,13 +2,15 @@ package hybridBackup
 
 import (
 	"errors"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 /*
-	restoreSnapshot.go
+	snapshotOpr.go
 
 	Restore snapshot for a certain user in the snapshot
 	The steps basically as follows.
@@ -101,3 +103,102 @@ func restoreSnapshotByName(backupTask *BackupTask, snapshotName string, username
 
 	return nil
 }
+
+/*
+	Merge Snapshot
+
+	This function is used to merge old snapshots if the system is running out of space
+	the two snapshot has to be sequential
+*/
+
+func mergeOldestSnapshots(backupTask *BackupTask) error {
+	//Get all snapshot names from disk path
+	files, err := filepath.Glob(filepath.ToSlash(filepath.Clean(filepath.Join(backupTask.DiskPath, "/version/"))) + "/*")
+	if err != nil {
+		return err
+	}
+
+	snapshots := []string{}
+	for _, file := range files {
+		if isDir(file) && fileExists(filepath.Join(file, "snapshot.datalink")) {
+			//This is a snapshot file
+			snapshots = append(snapshots, file)
+		}
+	}
+
+	if len(snapshots) < 2 {
+		return errors.New("Not enough snapshot to merge")
+	}
+
+	olderSnapshotDir := filepath.ToSlash(snapshots[0])
+	newerSnapshitDir := filepath.ToSlash(snapshots[1])
+
+	//Check if both snapshot exists
+	if !fileExists(olderSnapshotDir) || !fileExists(newerSnapshitDir) {
+		log.Println("[HybridBackup] Snapshot merge failed: Snapshot folder not found")
+		return errors.New("Snapshot folder not found")
+	}
+
+	//Check if link file exists
+	linkFileLocation := filepath.Join(newerSnapshitDir, "snapshot.datalink")
+	if !fileExists(linkFileLocation) {
+		log.Println("[HybridBackup] Snapshot link file not found.")
+		return errors.New("Snapshot link file not found")
+	}
+
+	//Get linker file
+	linkMap, err := readLinkFile(newerSnapshitDir)
+	if err != nil {
+		linkMap = &LinkFileMap{
+			UnchangedFile: map[string]string{},
+			DeletedFiles:  map[string]string{},
+		}
+	}
+
+	log.Println("[HybridBackup] Merging two snapshots in background")
+
+	//All file ready. Merge both snapshots
+	rootAbs, _ := filepath.Abs(olderSnapshotDir)
+	rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
+	fastWalk(olderSnapshotDir, func(filename string) error {
+		fileAbs, _ := filepath.Abs(filename)
+		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
+
+		relPath := filepath.ToSlash(strings.ReplaceAll(fileAbs, rootAbs, ""))
+		mergeAssumedLocation := filepath.Join(newerSnapshitDir, relPath)
+		if !fileExists(mergeAssumedLocation) {
+			//Check if this is in delete marker. If yes, skip this
+			_, ok := linkMap.DeletedFiles[relPath]
+			if !ok {
+				//This is not in delete map. Move it
+				//This must use rename instead of copy because of lack of space issue
+				if !fileExists(filepath.Dir(mergeAssumedLocation)) {
+					os.MkdirAll(filepath.Dir(mergeAssumedLocation), 0775)
+				}
+				err = os.Rename(filename, mergeAssumedLocation)
+				if err != nil {
+					return err
+				}
+			} else {
+				fmt.Println("Disposing file: ", relPath)
+			}
+		}
+
+		return nil
+	})
+
+	//Rewrite all other datalink file to make olderSnapshot name to new snapshot name
+	oldLink := filepath.Base(olderSnapshotDir)
+	newLink := filepath.Base(newerSnapshitDir)
+	for i := 1; i < len(snapshots); i++ {
+		err = updateLinkerPointer(snapshots[i], oldLink, newLink)
+		if err != nil {
+			log.Println("[HybridBackup] Link file update file: " + filepath.Base(snapshots[i]))
+		}
+		fmt.Println("Updating link file for " + filepath.Base(snapshots[i]))
+	}
+
+	//Remove the old snapshot folder structure
+	err = os.RemoveAll(olderSnapshotDir)
+	return err
+}

mod/disk/hybridBackup/versionBackup.go

@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"imuslab.com/arozos/mod/disk/diskcapacity/dftool"
 )
 
 /*
@@ -69,7 +71,7 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 	deletedFileList := map[string]string{}
 
 	//First pass: Check if there are any updated file from source and backup it to backup drive
-	fastWalk(parentRootAbs, func(filename string) error {
+	err = fastWalk(parentRootAbs, func(filename string) error {
 		if filepath.Ext(filename) == ".db" || filepath.Ext(filename) == ".lock" {
 			//Reserved filename, skipping
 			return nil
@@ -107,20 +109,29 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 						linkedFileList[relPath] = nameOfSnapshot
 					} else {
 						//File hash mismatch. Do file copy to renew data
-						copyFileToBackupLocation(filename, fileBackupLocation)
+						err = copyFileToBackupLocation(backupConfig, filename, fileBackupLocation)
+						if err != nil {
+							return err
+						}
 						copiedFileList = append(copiedFileList, fileBackupLocation)
 					}
 				} else {
 					//Invalid snapshot linkage. Assume new and do copy
 					log.Println("[HybridBackup] Link lost. Cloning source file to snapshot.")
-					copyFileToBackupLocation(filename, fileBackupLocation)
+					err = copyFileToBackupLocation(backupConfig, filename, fileBackupLocation)
+					if err != nil {
+						return err
+					}
 					copiedFileList = append(copiedFileList, fileBackupLocation)
 				}
 
 			} else {
 				//This file is not in snapshot link file.
 				//This is new file. Copy it to backup
-				copyFileToBackupLocation(filename, fileBackupLocation)
+				err = copyFileToBackupLocation(backupConfig, filename, fileBackupLocation)
+				if err != nil {
+					return err
+				}
 				copiedFileList = append(copiedFileList, fileBackupLocation)
 			}
 
@@ -185,6 +196,11 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 		return nil
 	})
 
+	if err != nil {
+		//Copy error. Mostly because of disk fulled
+		return err.Error(), err
+	}
+
 	//2nd pass: Check if there are anything exists in the previous backup but no longer exists in the source now
 	//For case where the file is backed up in previous snapshot but now the file has been removed
 	if previousSnapshotExists {
@@ -207,7 +223,6 @@ func executeVersionBackup(backupConfig *BackupTask) (string, error) {
 			if !fileExists(sourcAssumeLocation) {
 				//File exists in yesterday snapshot but not in the current source
 				//Assume it has been deleted, create a dummy indicator file
-				//ioutil.WriteFile(todaySnapshotLocation+".deleted", []byte(""), 0755)
 				deletedFileList[relPath] = todayFolderName
 			}
 			return nil
@@ -308,12 +323,47 @@ func getPreviousSnapshotName(backupConfig *BackupTask, currentSnapshotName strin
 	return previousSnapshotName, nil
 }
 
-func copyFileToBackupLocation(filename string, fileBackupLocation string) error {
+func copyFileToBackupLocation(task *BackupTask, filename string, fileBackupLocation string) error {
+	//Make dir the target dir if not exists
 	if !fileExists(filepath.Dir(fileBackupLocation)) {
 		os.MkdirAll(filepath.Dir(fileBackupLocation), 0755)
 	}
 
-	err := BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
+	//Check if the target disk can fit the new file
+	capinfo, err := dftool.GetCapacityInfoFromPath(filepath.Dir(fileBackupLocation))
+	if err == nil {
+		//Capacity info return normally. Estimate if the file will fit
+		srcSize := fileSize(filename)
+		diskSpace := capinfo.Avilable
+		if diskSpace < srcSize {
+			//Merge older snapshots. Maxium merging is 1 week
+			for i := 0; i < 6; i++ {
+				//Merge the oldest snapshot
+				err = mergeOldestSnapshots(task)
+				if err != nil {
+					log.Println("[HybridBackup] " + err.Error())
+					return errors.New("No space left on device")
+				}
+
+				//Check if there are enough space again
+				capinfo, err := dftool.GetCapacityInfoFromPath(filepath.Dir(fileBackupLocation))
+				if err != nil {
+					log.Println("[HybridBackup] " + err.Error())
+					return errors.New("No space left on device")
+				}
+				srcSize = fileSize(filename)
+				diskSpace = capinfo.Avilable
+				if diskSpace > srcSize {
+					//Space enough. Break out
+					break
+				}
+			}
+			log.Println("[HybridBackup] Error: No space left on device! Require ", srcSize, "bytes but only ", diskSpace, " bytes left")
+			return errors.New("No space left on device")
+		}
+	}
+
+	err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
 	if err != nil {
 		log.Println("[HybridBackup] Failed to copy file: ", filepath.Base(filename)+". "+err.Error())
 		return err