arozos/mod/disk/hybridBackup/versionBackup.go

package hybridBackup

import (
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

/*
	VersionBackup.go

	This scirpt file backup the data in the system nightly and create a restore point
	for the day just like BRTFS
*/

func executeVersionBackup(backupConfig *BackupTask) (string, error) {
	//Check if the backup parent root is identical / within backup disk
	parentRootAbs, err := filepath.Abs(backupConfig.ParentPath)
	if err != nil {
		return "", errors.New("Unable to resolve parent disk path")
	}

	backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/version/"))
	if err != nil {
		return "", errors.New("Unable to resolve backup disk path")
	}

	if len(parentRootAbs) >= len(backupRootAbs) {
		if parentRootAbs[:len(backupRootAbs)] == backupRootAbs {
			//parent root is within backup root. Raise configuration error
			log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive")
			return "", errors.New("Configuration Error. Skipping backup cycle.")
		}
	}

	todayFolderName := time.Now().Format("2006-01-02")
	previousSnapshotName, _ := getPreviousSnapshotName(backupConfig, todayFolderName)
	snapshotLocation := filepath.Join(backupConfig.DiskPath, "/version/", todayFolderName)
	previousSnapshotLocation := filepath.Join(backupConfig.DiskPath, "/version/", previousSnapshotName)

	if !fileExists(snapshotLocation) {
		//Create today folder if not exist
		os.MkdirAll(snapshotLocation, 0755)
	}

	/*
		Run a three pass compare logic between
		1. source disk and new backup disk to check any new / modified files (created today)
		2. yesterday backup and today backup to check any deleted files (created before, deleted today)
		3. file in today backup disk no longer in the current source disk (created today, deleted today)
	*/
	copiedFileList := []string{}

	//First pass: Check if there are any updated file from source and backup it to backup drive
	fastWalk(parentRootAbs, func(filename string) error {
		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
			//Reserved filename, skipping
			return nil
		}

		//Get the target paste location
		rootAbs, _ := filepath.Abs(backupConfig.ParentPath)
		fileAbs, _ := filepath.Abs(filename)

		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
		fileBackupLocation := filepath.Join(backupConfig.DiskPath, "/version/", todayFolderName, relPath)
		yesterdayBackupLocation := filepath.Join(previousSnapshotLocation, relPath)

		//Check if the file exists
		if !fileExists(fileBackupLocation) && !fileExists(yesterdayBackupLocation) {
			//File not exists in both current source and yesterday one. Copy it to the target location
			if !isDir(fileBackupLocation) && fileExists(fileBackupLocation+".deleted") {
				os.Remove(fileBackupLocation + ".deleted")
			}

			if !fileExists(filepath.Dir(fileBackupLocation)) {
				os.MkdirAll(filepath.Dir(fileBackupLocation), 0755)
			}

			err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
			if err != nil {
				log.Println("[HybridBackup] Failed to copy file: ", filepath.Base(filename)+". "+err.Error())
			}

			copiedFileList = append(copiedFileList, fileBackupLocation)

		} else if fileExists(yesterdayBackupLocation) {
			//The file exists in the last snapshot
			//Check if their hash is the same. If no, update it
			srcHash, err := getFileHash(fileAbs)
			if err != nil {
				log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
				return nil
			}
			targetHash, err := getFileHash(yesterdayBackupLocation)
			if err != nil {
				log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileBackupLocation), err.Error(), " Skipping.")
				return nil
			}

			if srcHash != targetHash {
				//Hash mismatch. Overwrite the file
				err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
				if err != nil {
					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
				} else {
					//No problem. Add this filepath into the list
					copiedFileList = append(copiedFileList, fileBackupLocation)
				}
			}
		} else {
			//Default case
			lastModTime := lastModTime(fileAbs)
			if lastModTime > backupConfig.LastCycleTime {
				//Check if hash the same
				srcHash, err := getFileHash(fileAbs)
				if err != nil {
					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					return nil
				}
				targetHash, err := getFileHash(fileBackupLocation)
				if err != nil {
					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileBackupLocation), err.Error(), " Skipping.")
					return nil
				}

				if srcHash != targetHash {
					//Hash mismatch. Overwrite the file
					err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
					if err != nil {
						log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					} else {
						//No problem. Add this filepath into the list
						copiedFileList = append(copiedFileList, fileBackupLocation)
					}
				}
			}
		}

		return nil
	})

	//2nd pass: Check if there are anything exists in the previous backup but no longer exists in the source now
	//For case where the file is backed up in previous snapshot but now the file has been removed
	if fileExists(previousSnapshotLocation) {
		fastWalk(previousSnapshotLocation, func(filename string) error {
			//Get the target paste location
			rootAbs, _ := filepath.Abs(previousSnapshotLocation)
			fileAbs, _ := filepath.Abs(filename)

			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

			relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
			sourcAssumeLocation := filepath.Join(parentRootAbs, relPath)
			todaySnapshotLocation := filepath.Join(snapshotLocation, relPath)

			if !fileExists(sourcAssumeLocation) {
				//File exists in yesterday snapshot but not in the current source
				//Assume it has been deleted, create a dummy indicator file
				ioutil.WriteFile(todaySnapshotLocation+".deleted", []byte(""), 0755)
			}
			return nil
		})
	}

	//3rd pass: Check if there are anything (except file with .deleted) in today backup drive that didn't exists in the source drive
	//For cases where the backup is applied to overwrite an eariler backup of the same day
	fastWalk(snapshotLocation, func(filename string) error {
		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
			//Reserved filename, skipping
			return nil
		}

		if filepath.Ext(filename) == ".deleted" {
			//Deleted file marker. Skip this
			return nil
		}

		//Get the target paste location
		rootAbs, _ := filepath.Abs(snapshotLocation)
		fileAbs, _ := filepath.Abs(filename)

		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
		sourceAssumedLocation := filepath.Join(parentRootAbs, relPath)

		if !fileExists(sourceAssumedLocation) {
			//File removed from the source. Delete it from backup as well
			os.Remove(filename)
		}
		return nil
	})

	return "", nil
}

//Return the previous snapshot for the currentSnspashot
func getPreviousSnapshotName(backupConfig *BackupTask, currentSnapshotName string) (string, error) {
	//Resolve the backup root folder
	backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/version/"))
	if err != nil {
		return "", errors.New("Unable to get the previous snapshot directory")
	}

	//Get the snapshot list and extract the snapshot date from foldername
	existingSnapshots := []string{}
	files, _ := filepath.Glob(filepath.ToSlash(filepath.Clean(backupRootAbs)) + "/*")
	for _, file := range files {
		if isDir(file) {
			existingSnapshots = append(existingSnapshots, filepath.Base(file))
		}
	}

	if len(existingSnapshots) == 0 {
		return "", errors.New("No snapshot found")
	}

	//Check if the current snapshot exists, if not, return the latest one
	previousSnapshotName := ""
	if fileExists(filepath.Join(backupRootAbs, currentSnapshotName)) {
		//Current snapshot exists. Find the one just above it
		lastSnapshotName := existingSnapshots[0]
		for _, snapshotName := range existingSnapshots {
			if snapshotName == currentSnapshotName {
				//This is the correct snapshot name. Get the last one as previous snapshot
				previousSnapshotName = lastSnapshotName
			} else {
				lastSnapshotName = snapshotName
			}
		}
	} else {
		//Current snapshot not exists. Use the last item in snapshots list
		previousSnapshotName = existingSnapshots[len(existingSnapshots)-1]
	}

	return previousSnapshotName, nil
}