package hybridBackup import ( "crypto/sha256" "encoding/hex" "errors" "io" "log" "os" "path/filepath" "strings" "time" ) /* Hybrid Backup This module handle backup functions from the drive with Hieracchy labeled as "backup" Backup modes suport in this module currently consists of Denote P drive as parent drive and B drive as backup drive. 1. Basic (basic): - Any new file created in P will be copied to B within 1 minutes - Any file change will be copied to B within 30 minutes - Any file removed in P will be delete from backup if it is > 24 hours old 2. Nightly (nightly): - The whole P drive will be copied to N drive every night 3. Versioning (version) - A versioning system will be introduce to this backup drive - Just like the time machine Tips when developing this module - This is a sub-module of the current file system. Do not import from arozos file system module - If you need any function from the file system, copy and paste it in this module */ type Manager struct { Tasks []*BackupTask } type BackupTask struct { JobName string //The name used by the scheduler for executing this config CycleCounter int64 //The number of backup executed in the background LastCycleTime int64 //The execution time of the last cycle DiskUID string //The UID of the target fsandlr DiskPath string //The mount point for the disk ParentUID string //Parent virtal disk UUID ParentPath string //Parent disk path DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time Mode string //Backup mode } //A file in the backup drive that is restorable type RestorableFile struct { Filename string //Filename of this restorable object RelpathOnDisk string //Relative path of this file to the root Deleteime int64 //Delete remaining time } //The restorable report type RestorableReport struct { ParentUID string //The Disk ID to be restored to DiskUID string //The Backup disk UID RestorableFiles []RestorableFile //A list of restorable files } func NewHyperBackupManager() *Manager { return &Manager{ Tasks: []*BackupTask{}, } } func (m *Manager) AddTask(newtask *BackupTask) error { log.Println(">>>> [Debug] New Backup Tasks added: ", newtask) /*for _, thisHandler := range fsHandlers { if thisHandler.Hierarchy == "backup" { //This is a backup drive. Generate it handler backupConfig := thisHandler.HierarchyConfig.(hybridBackup.BackupTask) //Get its parent mount point for backup parentFileSystemHandler, err := GetFsHandlerByUUID(backupConfig.ParentUID) if err != nil { log.Println("Virtual Root with UUID: " + backupConfig.ParentUID + " not loaded. Unable to start backup process.") break } backupConfig.JobName = "backup-daemon [" + thisHandler.UUID + "]" backupConfig.ParentPath = parentFileSystemHandler.Path backupConfig.CycleCounter = 1 //Debug backup execution hybridBackup.HandleBackupProcess(&backupConfig) //Remove the previous job if it exists if systemScheduler.JobExists(backupConfig.JobName) { systemScheduler.RemoveJobFromScheduleList(backupConfig.JobName) } //Create a scheudler for this disk systemScheduler.CreateNewScheduledFunctionJob(backupConfig.JobName, "Backup daemon from "+backupConfig.ParentUID+":/ to "+backupConfig.DiskUID+":/", 60, func() (string, error) { return hybridBackup.HandleBackupProcess(&backupConfig) }, ) } }*/ return nil } func executeBackup(backupConfig *BackupTask, deepBackup bool) (string, error) { copiedFileList := []string{} rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath)) //Check if the backup parent root is identical / within backup disk parentRootAbs, err := filepath.Abs(backupConfig.ParentPath) if err != nil { return "", errors.New("Unable to resolve parent disk path") } backupRootAbs, err := filepath.Abs(backupConfig.DiskPath) if err != nil { return "", errors.New("Unable to resolve backup disk path") } if len(parentRootAbs) >= len(backupRootAbs) { if parentRootAbs[:len(backupRootAbs)] == backupRootAbs { //parent root is within backup root. Raise configuration error log.Println("*HyperBackup* Invalid backup cycle: Parent drive is located inside backup drive") return "", errors.New("Configuration Error. Skipping backup cycle.") } } //Add file cycles fastWalk(rootPath, func(filename string) error { if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" { //Reserved filename, skipping return nil } //Get the target paste location rootAbs, _ := filepath.Abs(rootPath) fileAbs, _ := filepath.Abs(filename) rootAbs = filepath.ToSlash(filepath.Clean(rootAbs)) fileAbs = filepath.ToSlash(filepath.Clean(fileAbs)) relPath := strings.ReplaceAll(fileAbs, rootAbs, "") assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath) if !deepBackup { //Shallow copy. Only do copy base on file exists or not //This is used to reduce the time for reading the file metatag if !fileExists(assumedTargetPosition) { //Target file not exists in backup disk. Make a copy if !fileExists(filepath.Dir(assumedTargetPosition)) { //Folder containing this file not exists. Create it os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755) } //Copy the file to target err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } } else { //Deep copy. Check and match the modtime of each file if !fileExists(assumedTargetPosition) { //Copy the file to target err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") return nil } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } else { //Target file already exists. Check if their hash matches srcHash, err := getFileHash(fileAbs) if err != nil { log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") return nil } targetHash, err := getFileHash(assumedTargetPosition) if err != nil { log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.") return nil } if srcHash != targetHash { log.Println("[Debug] Hash mismatch. Copying ", fileAbs) //This file has been recently changed. Copy it to new location err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } } } ///Remove file cycle backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath)) fastWalk(backupConfig.DiskPath, func(filename string) error { if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" { //Reserved filename, skipping return nil } //Get the target paste location rootAbs, _ := filepath.Abs(backupDriveRootPath) fileAbs, _ := filepath.Abs(filename) rootAbs = filepath.ToSlash(filepath.Clean(rootAbs)) fileAbs = filepath.ToSlash(filepath.Clean(fileAbs)) thisFileRel := filename[len(backupDriveRootPath):] originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel))) //Check if the taget file not exists and this file has been here for more than 24h if !fileExists(originalFileOnDiskPath) { //This file not exists. Check if it is in the delete file marker for more than 24 hours val, ok := backupConfig.DeleteFileMarkers[thisFileRel] if !ok { //This file is newly deleted. Push into the marker map backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix() log.Println("[Debug] Adding " + filename + " to delete marker") } else { //This file has been marked. Check if it is time to delete if time.Now().Unix()-val > 3600*24 { log.Println("[Debug] Deleting " + filename) //Remove the backup file os.RemoveAll(filename) //Remove file from delete file markers delete(backupConfig.DeleteFileMarkers, thisFileRel) } } } return nil }) return nil }) return "", nil } //Main handler function for hybrid backup func HandleBackupProcess(backupConfig *BackupTask) (string, error) { log.Println(">>>>>> [Debug] Running backup process: ", backupConfig) //Check if the target disk is writable and mounted if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) { //This parent filesystem is mounted } else { //File system not mounted even after 3 backup cycle. Terminate backup scheduler log.Println("*HybridBackup* Skipping backup cycle for " + backupConfig.ParentUID + ":/") return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil } //Check if the backup disk is mounted. If no, stop the scheulder if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) { log.Println("*HybridBackup* Backup schedule stopped for " + backupConfig.DiskUID + ":/") return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted") } deepBackup := true //Default perform deep backup if backupConfig.Mode == "basic" { if backupConfig.CycleCounter%30 == 0 { //Perform deep backup, use walk function deepBackup = true } else { deepBackup = false } backupConfig.LastCycleTime = time.Now().Unix() return executeBackup(backupConfig, deepBackup) } else if backupConfig.Mode == "nightly" { if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 { //24 hours from last backup. Execute deep backup now executeBackup(backupConfig, true) backupConfig.LastCycleTime = time.Now().Unix() } } else if backupConfig.Mode == "version" { //Do a versioning backup log.Println("[WIP] This function is still work in progress. Please do not use version backup for now.") //WIP } //Add one to the cycle counter backupConfig.CycleCounter++ //Return the log information return "", nil } //Restore accidentailly removed file from backup func HandleRestore(backupConfig *BackupTask, targetFile string) error { return nil } //List the file that is restorable from the given disk func ListRestorable(backupConfig *BackupTask) { } //Get and return the file hash for a file func getFileHash(filename string) (string, error) { f, err := os.Open(filename) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil }