arozos/mod/filesystem/hybridBackup/hybridBackup.go

package hybridBackup

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

/*
	Hybrid Backup

	This module handle backup functions from the drive with Hieracchy labeled as "backup"
	Backup modes suport in this module currently consists of

	Denote P drive as parent drive and B drive as backup drive.
	1. Basic (basic):
		- Any new file created in P will be copied to B within 1 minutes
		- Any file change will be copied to B within 30 minutes
		- Any file removed in P will be delete from backup if it is > 24 hours old
	2. Nightly (nightly):
		- The whole P drive will be copied to N drive every night
	3. Versioning (version)
		- A versioning system will be introduce to this backup drive
		- Just like the time machine

	Tips when developing this module
	- This is a sub-module of the current file system. Do not import from arozos file system module
	- If you need any function from the file system, copy and paste it in this module
*/

type BackupConfig struct {
	JobName           string           //The name used by the scheduler for executing this config
	CycleCounter      int64            //The number of backup executed in the background
	LastCycleTime     int64            //The execution time of the last cycle
	DiskUID           string           //The UID of the target fsandlr
	DiskPath          string           //The mount point for the disk
	ParentUID         string           //Parent virtal disk UUID
	ParentPath        string           //Parent disk path
	DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time
	Mode              string           //Backup mode
}

func executeBackup(backupConfig *BackupConfig, deepBackup bool) (string, error) {
	copiedFileList := []string{}

	rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath))

	//Add file cycles
	fastWalk(rootPath, func(filename string) error {
		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
			//Reserved filename, skipping
			return nil
		}
		//Get the target paste location
		rootAbs, _ := filepath.Abs(rootPath)
		fileAbs, _ := filepath.Abs(filename)

		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
		assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath)

		if !deepBackup {
			//Shallow copy. Only do copy base on file exists or not
			//This is used to reduce the time for reading the file metatag
			if !fileExists(assumedTargetPosition) {
				//Target file not exists in backup disk. Make a copy
				if !fileExists(filepath.Dir(assumedTargetPosition)) {
					//Folder containing this file not exists. Create it
					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
				}

				//Copy the file to target
				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
				if err != nil {
					log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
				} else {
					//No problem. Add this filepath into the list
					copiedFileList = append(copiedFileList, assumedTargetPosition)
				}
			}
		} else {
			//Deep copy. Check and match the modtime of each file
			if !fileExists(assumedTargetPosition) {
				//Copy the file to target
				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
				if err != nil {
					log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					return nil
				} else {
					//No problem. Add this filepath into the list
					copiedFileList = append(copiedFileList, assumedTargetPosition)
				}
			} else {
				//Target file already exists. Check if their hash matches
				srcHash, err := getFileHash(fileAbs)
				if err != nil {
					log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					return nil
				}
				targetHash, err := getFileHash(assumedTargetPosition)
				if err != nil {
					log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.")
					return nil
				}

				if srcHash != targetHash {
					log.Println("[Debug] Hash mismatch. Copying ", fileAbs)
					//This file has been recently changed. Copy it to new location
					err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
					if err != nil {
						log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					} else {
						//No problem. Add this filepath into the list
						copiedFileList = append(copiedFileList, assumedTargetPosition)
					}
				}

			}
		}

		///Remove file cycle
		backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath))
		fastWalk(backupConfig.DiskPath, func(filename string) error {
			if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
				//Reserved filename, skipping
				return nil
			}
			//Get the target paste location
			rootAbs, _ := filepath.Abs(backupDriveRootPath)
			fileAbs, _ := filepath.Abs(filename)

			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

			thisFileRel := filename[len(backupDriveRootPath):]
			originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel)))

			//Check if the taget file not exists and this file has been here for more than 24h
			if !fileExists(originalFileOnDiskPath) {
				//This file not exists. Check if it is in the delete file marker for more than 24 hours
				val, ok := backupConfig.DeleteFileMarkers[thisFileRel]
				if !ok {
					//This file is newly deleted. Push into the marker map
					backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix()
					log.Println("[Debug] Adding " + filename + " to delete marker")
				} else {
					//This file has been marked. Check if it is time to delete
					if time.Now().Unix()-val > 120 {
						log.Println("[Debug] Deleting " + filename)

						//Remove the backup file
						os.RemoveAll(filename)

						//Remove file from delete file markers
						delete(backupConfig.DeleteFileMarkers, thisFileRel)
					}
				}
			}
			return nil
		})

		return nil
	})

	return "", nil
}

//Main handler function for hybrid backup
func HandleBackupProcess(backupConfig *BackupConfig) (string, error) {
	log.Println(">>>>>> [Debug] Running backup process: ", backupConfig)

	//Check if the target disk is writable and mounted
	if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) {
		//This parent filesystem is mounted

	} else {
		//File system not mounted even after 3 backup cycle. Terminate backup scheduler
		log.Println("*HybridBackup* Skipping backup cycle for " + backupConfig.ParentUID + ":/")
		return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil
	}

	//Check if the backup disk is mounted. If no, stop the scheulder
	if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) {
		log.Println("*HybridBackup* Backup schedule stopped for " + backupConfig.DiskUID + ":/")
		return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted")
	}

	deepBackup := true //Default perform deep backup
	if backupConfig.Mode == "basic" {
		if backupConfig.CycleCounter%30 == 0 {
			//Perform deep backup, use walk function
			deepBackup = true
		} else {
			deepBackup = false
		}
		backupConfig.LastCycleTime = time.Now().Unix()
		return executeBackup(backupConfig, deepBackup)
	} else if backupConfig.Mode == "nightly" {
		if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 {
			//24 hours from last backup. Execute deep backup now
			executeBackup(backupConfig, true)
			backupConfig.LastCycleTime = time.Now().Unix()
		}

	} else if backupConfig.Mode == "version" {
		//Do a versioning backup

		//WIP
	}

	//Add one to the cycle counter
	backupConfig.CycleCounter++

	//Return the log information
	return "", nil
}

//Get and return the file hash for a file
func getFileHash(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return hex.EncodeToString(h.Sum(nil)), nil
}