arozos/mod/disk/hybridBackup/hybridBackup.go

package hybridBackup

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

/*
	Hybrid Backup

	This module handle backup functions from the drive with Hieracchy labeled as "backup"
	Backup modes suport in this module currently consists of

	Denote P drive as parent drive and B drive as backup drive.
	1. Basic (basic):
		- Any new file created in P will be copied to B within 1 minutes
		- Any file change will be copied to B within 30 minutes
		- Any file removed in P will be delete from backup if it is > 24 hours old
	2. Nightly (nightly):
		- The whole P drive will be copied to N drive every night
	3. Versioning (version)
		- A versioning system will be introduce to this backup drive
		- Just like the time machine

	Tips when developing this module
	- This is a sub-module of the current file system. Do not import from arozos file system module
	- If you need any function from the file system, copy and paste it in this module
*/

type Manager struct {
	Ticker     *time.Ticker  `json:"-"` //The main ticker
	StopTicker chan bool     `json:"-"` //Channel for stopping the backup
	Tasks      []*BackupTask //The backup tasks that is running under this manager
}

type BackupTask struct {
	JobName           string           //The name used by the scheduler for executing this config
	CycleCounter      int64            //The number of backup executed in the background
	LastCycleTime     int64            //The execution time of the last cycle
	Enabled           bool             //Check if the task is enabled. Will not execute if this is set to false
	DiskUID           string           //The UID of the target fsandlr
	DiskPath          string           //The mount point for the disk
	ParentUID         string           //Parent virtal disk UUID
	ParentPath        string           //Parent disk path
	DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time
	Mode              string           //Backup mode
}

//A file in the backup drive that is restorable
type RestorableFile struct {
	Filename      string //Filename of this restorable object
	IsHidden      bool   //Check if the file is hidden or located in a path within hidden folder
	Filesize      int64  //The file size to be restorable
	RelpathOnDisk string //Relative path of this file to the root
	RestorePoint  string //The location this file should restore to
	BackupDiskUID string //The UID of disk that is hold the backup of this file
	RemainingTime int64  //Remaining time till auto remove
	DeleteTime    int64  //Delete time
}

//The restorable report
type RestorableReport struct {
	ParentUID       string            //The Disk ID to be restored to
	RestorableFiles []*RestorableFile //A list of restorable files
}

var (
	internalTickerTime time.Duration = 60
)

func NewHyperBackupManager() *Manager {
	//Create a new minute ticker
	ticker := time.NewTicker(internalTickerTime * time.Second)
	stopper := make(chan bool, 1)

	newManager := &Manager{
		Ticker:     ticker,
		StopTicker: stopper,
		Tasks:      []*BackupTask{},
	}

	///Create task executor
	go func() {
		defer log.Println("[HybridBackup] Ticker Stopped")
		for {
			select {
			case <-ticker.C:
				for _, task := range newManager.Tasks {
					if task.Enabled == true {
						task.HandleBackupProcess()
					}
				}
			case <-stopper:
				return
			}
		}
	}()

	//Return the manager
	return newManager
}

func (m *Manager) AddTask(newtask *BackupTask) error {
	//Create a job for this
	newtask.JobName = "backup-[" + newtask.DiskUID + "]"

	//Check if the same job name exists
	for _, task := range m.Tasks {
		if task.JobName == newtask.JobName {
			return errors.New("Task already exists")
		}
	}

	//Add task to list
	m.Tasks = append(m.Tasks, newtask)

	//Start the task
	m.StartTask(newtask.JobName)

	log.Println(">>>> [Debug] New Backup Tasks added: ", newtask.JobName, newtask)

	return nil
}

//Start a given task given name
func (m *Manager) StartTask(jobname string) {
	for _, task := range m.Tasks {
		if task.JobName == jobname {
			//Enable to job
			task.Enabled = true

			//Run it once
			task.HandleBackupProcess()
		}
	}
}

//Stop a given task given its job name
func (m *Manager) StopTask(jobname string) {
	for _, task := range m.Tasks {
		if task.JobName == jobname {
			task.Enabled = false
		}
	}
}

//Stop all managed handlers
func (m *Manager) Close() error {
	m.StopTicker <- true
	return nil
}

func executeBackup(backupConfig *BackupTask, deepBackup bool) (string, error) {
	copiedFileList := []string{}

	rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath))

	//Check if the backup parent root is identical / within backup disk
	parentRootAbs, err := filepath.Abs(backupConfig.ParentPath)
	if err != nil {
		return "", errors.New("Unable to resolve parent disk path")
	}

	backupRootAbs, err := filepath.Abs(backupConfig.DiskPath)
	if err != nil {
		return "", errors.New("Unable to resolve backup disk path")
	}

	if len(parentRootAbs) >= len(backupRootAbs) {
		if parentRootAbs[:len(backupRootAbs)] == backupRootAbs {
			//parent root is within backup root. Raise configuration error
			log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive")
			return "", errors.New("Configuration Error. Skipping backup cycle.")
		}
	}

	//Add file cycles
	fastWalk(rootPath, func(filename string) error {
		if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
			//Reserved filename, skipping
			return nil
		}
		//Get the target paste location
		rootAbs, _ := filepath.Abs(rootPath)
		fileAbs, _ := filepath.Abs(filename)

		rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
		fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

		relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
		assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath)

		if !deepBackup {
			//Shallow copy. Only do copy base on file exists or not
			//This is used to reduce the time for reading the file metatag
			if !fileExists(assumedTargetPosition) {
				//Target file not exists in backup disk. Make a copy
				if !fileExists(filepath.Dir(assumedTargetPosition)) {
					//Folder containing this file not exists. Create it
					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
				}

				//Copy the file to target
				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
				if err != nil {
					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
				} else {
					//No problem. Add this filepath into the list
					copiedFileList = append(copiedFileList, assumedTargetPosition)
				}

			}
		} else {
			//Deep copy. Check and match the modtime of each file
			if !fileExists(assumedTargetPosition) {
				if !fileExists(filepath.Dir(assumedTargetPosition)) {
					//Folder containing this file not exists. Create it
					os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755)
				}

				//Copy the file to target
				err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
				if err != nil {
					log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					return nil
				} else {
					//No problem. Add this filepath into the list
					copiedFileList = append(copiedFileList, assumedTargetPosition)
				}
			} else {
				//Target file already exists. Check if their hash matches
				srcHash, err := getFileHash(fileAbs)
				if err != nil {
					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					return nil
				}
				targetHash, err := getFileHash(assumedTargetPosition)
				if err != nil {
					log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.")
					return nil
				}

				if srcHash != targetHash {
					log.Println("[Debug] Hash mismatch. Copying ", fileAbs)
					//This file has been recently changed. Copy it to new location
					err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024)
					if err != nil {
						log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
					} else {
						//No problem. Add this filepath into the list
						copiedFileList = append(copiedFileList, assumedTargetPosition)
					}

					//Check if this file is in the remove marker list. If yes, pop it from the list
					_, ok := backupConfig.DeleteFileMarkers[relPath]
					if ok {
						//File exists. remove it from delete file amrker
						delete(backupConfig.DeleteFileMarkers, relPath)
						log.Println("Removing ", relPath, " from delete marker list")
					}
				}
			}
		}

		///Remove file cycle
		backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath))
		fastWalk(backupConfig.DiskPath, func(filename string) error {
			if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" {
				//Reserved filename, skipping
				return nil
			}
			//Get the target paste location
			rootAbs, _ := filepath.Abs(backupDriveRootPath)
			fileAbs, _ := filepath.Abs(filename)

			rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
			fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))

			thisFileRel := filename[len(backupDriveRootPath):]
			originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel)))

			//Check if the taget file not exists and this file has been here for more than 24h
			if !fileExists(originalFileOnDiskPath) {
				//This file not exists. Check if it is in the delete file marker for more than 24 hours
				val, ok := backupConfig.DeleteFileMarkers[thisFileRel]
				if !ok {
					//This file is newly deleted. Push into the marker map
					backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix()
					log.Println("[Debug] Adding " + filename + " to delete marker")
				} else {
					//This file has been marked. Check if it is time to delete
					if time.Now().Unix()-val > 3600*24 {
						log.Println("[Debug] Deleting " + filename)

						//Remove the backup file
						os.RemoveAll(filename)

						//Remove file from delete file markers
						delete(backupConfig.DeleteFileMarkers, thisFileRel)
					}
				}
			}
			return nil
		})

		return nil
	})

	return "", nil
}

//Main handler function for hybrid backup
func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
	log.Println(">>>>>> [Debug] Running backup process: ", backupConfig)

	//Check if the target disk is writable and mounted
	if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) {
		//This parent filesystem is mounted

	} else {
		//File system not mounted even after 3 backup cycle. Terminate backup scheduler
		log.Println("[HybridBackup] Skipping backup cycle for " + backupConfig.ParentUID + ":/")
		return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil
	}

	//Check if the backup disk is mounted. If no, stop the scheulder
	if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) {
		log.Println("[HybridBackup] Backup schedule stopped for " + backupConfig.DiskUID + ":/")
		return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted")
	}

	deepBackup := true //Default perform deep backup
	if backupConfig.Mode == "basic" {
		if backupConfig.CycleCounter%3 == 0 {
			//Perform deep backup, use walk function
			deepBackup = true
		} else {
			deepBackup = false
		}
		backupConfig.LastCycleTime = time.Now().Unix()
		return executeBackup(backupConfig, deepBackup)
	} else if backupConfig.Mode == "nightly" {
		if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 {
			//24 hours from last backup. Execute deep backup now
			executeBackup(backupConfig, true)
			backupConfig.LastCycleTime = time.Now().Unix()
		}

	} else if backupConfig.Mode == "version" {
		//Do a versioning backup
		log.Println("[WIP] This function is still work in progress. Please do not use version backup for now.")
		//WIP
	}

	//Add one to the cycle counter
	backupConfig.CycleCounter++

	//Return the log information
	return "", nil
}

//Restore accidentailly removed file from backup
func HandleRestore(parentDiskID string, restoreDiskID string, targetFileRelpath string) error {

	return nil
}

//List the file that is restorable from the given disk
func (m *Manager) ListRestorable(parentDiskID string) (RestorableReport, error) {
	//List all the backup process that is mirroring this parent disk
	tasks := m.getTaskByParentDiskID(parentDiskID)
	if len(tasks) == 0 {
		return RestorableReport{}, errors.New("No backup root found for this " + parentDiskID + ":/ virtual root.")
	}

	diffFiles := []*RestorableFile{}

	//Extract all comparasion
	for _, task := range tasks {
		restorableFiles, err := task.compareRootPaths()
		if err != nil {
			//Unable to list restorable. SKip this
		} else {
			for _, restorable := range restorableFiles {
				diffFiles = append(diffFiles, restorable)
			}
		}
	}

	//Create a Restorable Report
	thisReport := RestorableReport{
		ParentUID:       parentDiskID,
		RestorableFiles: diffFiles,
	}

	return thisReport, nil
}

func (m *Manager) getTaskByParentDiskID(parentDiskID string) []*BackupTask {
	//Convert ID:/ format to ID
	if strings.Contains(parentDiskID, ":") {
		parentDiskID = strings.Split(parentDiskID, ":")[0]
	}

	possibleTask := []*BackupTask{}
	for _, task := range m.Tasks {
		if task.ParentUID == parentDiskID {
			//This task parent is the target disk. push this to list
			possibleTask = append(possibleTask, task)
		}
	}
	return possibleTask
}

//Get and return the file hash for a file
func getFileHash(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return hex.EncodeToString(h.Sum(nil)), nil
}