package hybridBackup import ( "crypto/sha256" "encoding/hex" "errors" "io" "log" "os" "path/filepath" "strings" "time" ) /* Hybrid Backup This module handle backup functions from the drive with Hieracchy labeled as "backup" Backup modes suport in this module currently consists of Denote P drive as parent drive and B drive as backup drive. 1. Basic (basic): - Any new file created in P will be copied to B within 1 minutes - Any file change will be copied to B within 30 minutes - Any file removed in P will be delete from backup if it is > 24 hours old 2. Nightly (nightly): - The whole P drive will be copied to N drive every night 3. Versioning (version) - A versioning system will be introduce to this backup drive - Just like the time machine Tips when developing this module - This is a sub-module of the current file system. Do not import from arozos file system module - If you need any function from the file system, copy and paste it in this module */ type Manager struct { Ticker *time.Ticker //The main ticker StopTicker chan bool //Channel for stopping the backup Tasks []*BackupTask //The backup tasks that is running under this manager } type BackupTask struct { JobName string //The name used by the scheduler for executing this config CycleCounter int64 //The number of backup executed in the background LastCycleTime int64 //The execution time of the last cycle DiskUID string //The UID of the target fsandlr DiskPath string //The mount point for the disk ParentUID string //Parent virtal disk UUID ParentPath string //Parent disk path DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time Mode string //Backup mode } //A file in the backup drive that is restorable type RestorableFile struct { Filename string //Filename of this restorable object RelpathOnDisk string //Relative path of this file to the root Deleteime int64 //Delete remaining time } //The restorable report type RestorableReport struct { ParentUID string //The Disk ID to be restored to DiskUID string //The Backup disk UID RestorableFiles []RestorableFile //A list of restorable files } var ( internalTickerTime time.Duration = 60 ) func NewHyperBackupManager() *Manager { //Create a new minute ticker ticker := time.NewTicker(internalTickerTime * time.Second) stopper := make(chan bool, 1) newManager := &Manager{ Ticker: ticker, StopTicker: stopper, Tasks: []*BackupTask{}, } ///Create task executor go func() { defer log.Println("[HybridBackup] Ticker Stopped") for { select { case <-ticker.C: for _, task := range newManager.Tasks { task.HandleBackupProcess() } case <-stopper: return } } }() //Return the manager return newManager } func (m *Manager) AddTask(newtask *BackupTask) error { //Create a job for this newtask.JobName = "backup-[" + newtask.DiskUID + "]" //Check if the same job name exists for _, task := range m.Tasks { if task.JobName == newtask.JobName { return errors.New("Task already exists") } } m.Tasks = append(m.Tasks, newtask) log.Println(">>>> [Debug] New Backup Tasks added: ", newtask.JobName, newtask) return nil } func (m *Manager) StopTask(jobname string) error { return nil } //Stop all managed handlers func (m *Manager) Close() error { m.StopTicker <- true return nil } func executeBackup(backupConfig *BackupTask, deepBackup bool) (string, error) { copiedFileList := []string{} rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath)) //Check if the backup parent root is identical / within backup disk parentRootAbs, err := filepath.Abs(backupConfig.ParentPath) if err != nil { return "", errors.New("Unable to resolve parent disk path") } backupRootAbs, err := filepath.Abs(backupConfig.DiskPath) if err != nil { return "", errors.New("Unable to resolve backup disk path") } if len(parentRootAbs) >= len(backupRootAbs) { if parentRootAbs[:len(backupRootAbs)] == backupRootAbs { //parent root is within backup root. Raise configuration error log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive") return "", errors.New("Configuration Error. Skipping backup cycle.") } } //Add file cycles fastWalk(rootPath, func(filename string) error { if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" { //Reserved filename, skipping return nil } //Get the target paste location rootAbs, _ := filepath.Abs(rootPath) fileAbs, _ := filepath.Abs(filename) rootAbs = filepath.ToSlash(filepath.Clean(rootAbs)) fileAbs = filepath.ToSlash(filepath.Clean(fileAbs)) relPath := strings.ReplaceAll(fileAbs, rootAbs, "") assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath) if !deepBackup { //Shallow copy. Only do copy base on file exists or not //This is used to reduce the time for reading the file metatag if !fileExists(assumedTargetPosition) { //Target file not exists in backup disk. Make a copy if !fileExists(filepath.Dir(assumedTargetPosition)) { //Folder containing this file not exists. Create it os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755) } //Copy the file to target err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } } else { //Deep copy. Check and match the modtime of each file if !fileExists(assumedTargetPosition) { if !fileExists(filepath.Dir(assumedTargetPosition)) { //Folder containing this file not exists. Create it os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755) } //Copy the file to target err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") return nil } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } else { //Target file already exists. Check if their hash matches srcHash, err := getFileHash(fileAbs) if err != nil { log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") return nil } targetHash, err := getFileHash(assumedTargetPosition) if err != nil { log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.") return nil } if srcHash != targetHash { log.Println("[Debug] Hash mismatch. Copying ", fileAbs) //This file has been recently changed. Copy it to new location err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } } } ///Remove file cycle backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath)) fastWalk(backupConfig.DiskPath, func(filename string) error { if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" { //Reserved filename, skipping return nil } //Get the target paste location rootAbs, _ := filepath.Abs(backupDriveRootPath) fileAbs, _ := filepath.Abs(filename) rootAbs = filepath.ToSlash(filepath.Clean(rootAbs)) fileAbs = filepath.ToSlash(filepath.Clean(fileAbs)) thisFileRel := filename[len(backupDriveRootPath):] originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel))) //Check if the taget file not exists and this file has been here for more than 24h if !fileExists(originalFileOnDiskPath) { //This file not exists. Check if it is in the delete file marker for more than 24 hours val, ok := backupConfig.DeleteFileMarkers[thisFileRel] if !ok { //This file is newly deleted. Push into the marker map backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix() log.Println("[Debug] Adding " + filename + " to delete marker") } else { //This file has been marked. Check if it is time to delete if time.Now().Unix()-val > 3600*24 { log.Println("[Debug] Deleting " + filename) //Remove the backup file os.RemoveAll(filename) //Remove file from delete file markers delete(backupConfig.DeleteFileMarkers, thisFileRel) } } } return nil }) return nil }) return "", nil } //Main handler function for hybrid backup func (backupConfig *BackupTask) HandleBackupProcess() (string, error) { log.Println(">>>>>> [Debug] Running backup process: ", backupConfig) //Check if the target disk is writable and mounted if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) { //This parent filesystem is mounted } else { //File system not mounted even after 3 backup cycle. Terminate backup scheduler log.Println("[HybridBackup] Skipping backup cycle for " + backupConfig.ParentUID + ":/") return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil } //Check if the backup disk is mounted. If no, stop the scheulder if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) { log.Println("[HybridBackup] Backup schedule stopped for " + backupConfig.DiskUID + ":/") return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted") } deepBackup := true //Default perform deep backup if backupConfig.Mode == "basic" { if backupConfig.CycleCounter%30 == 0 { //Perform deep backup, use walk function deepBackup = true } else { deepBackup = false } backupConfig.LastCycleTime = time.Now().Unix() return executeBackup(backupConfig, deepBackup) } else if backupConfig.Mode == "nightly" { if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 { //24 hours from last backup. Execute deep backup now executeBackup(backupConfig, true) backupConfig.LastCycleTime = time.Now().Unix() } } else if backupConfig.Mode == "version" { //Do a versioning backup log.Println("[WIP] This function is still work in progress. Please do not use version backup for now.") //WIP } //Add one to the cycle counter backupConfig.CycleCounter++ //Return the log information return "", nil } //Restore accidentailly removed file from backup func HandleRestore(parentDiskID string, restoreDiskID string, targetFileRelpath string) error { return nil } //List the file that is restorable from the given disk func (m *Manager) ListRestorable(parentDiskID string) RestorableReport { return RestorableReport{} } //Get and return the file hash for a file func getFileHash(filename string) (string, error) { f, err := os.Open(filename) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil }