package hybridBackup import ( "crypto/sha256" "encoding/hex" "errors" "io" "log" "os" "path/filepath" "strings" "time" ) /* Hybrid Backup This module handle backup functions from the drive with Hieracchy labeled as "backup" Backup modes suport in this module currently consists of Denote P drive as parent drive and B drive as backup drive. 1. Basic (basic): - Any new file created in P will be copied to B within 1 minutes - Any file change will be copied to B within 30 minutes - Any file removed in P will be delete from backup if it is > 24 hours old 2. Nightly (nightly): - The whole P drive will be copied to N drive every night 3. Versioning (version) - A versioning system will be introduce to this backup drive - Just like the time machine Tips when developing this module - This is a sub-module of the current file system. Do not import from arozos file system module - If you need any function from the file system, copy and paste it in this module */ type BackupConfig struct { JobName string //The name used by the scheduler for executing this config CycleCounter int64 //The number of backup executed in the background LastCycleTime int64 //The execution time of the last cycle DiskUID string //The UID of the target fsandlr DiskPath string //The mount point for the disk ParentUID string //Parent virtal disk UUID ParentPath string //Parent disk path DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time Mode string //Backup mode } func executeBackup(backupConfig *BackupConfig, deepBackup bool) (string, error) { copiedFileList := []string{} rootPath := filepath.ToSlash(filepath.Clean(backupConfig.ParentPath)) //Add file cycles fastWalk(rootPath, func(filename string) error { if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" { //Reserved filename, skipping return nil } //Get the target paste location rootAbs, _ := filepath.Abs(rootPath) fileAbs, _ := filepath.Abs(filename) rootAbs = filepath.ToSlash(filepath.Clean(rootAbs)) fileAbs = filepath.ToSlash(filepath.Clean(fileAbs)) relPath := strings.ReplaceAll(fileAbs, rootAbs, "") assumedTargetPosition := filepath.Join(backupConfig.DiskPath, relPath) if !deepBackup { //Shallow copy. Only do copy base on file exists or not //This is used to reduce the time for reading the file metatag if !fileExists(assumedTargetPosition) { //Target file not exists in backup disk. Make a copy if !fileExists(filepath.Dir(assumedTargetPosition)) { //Folder containing this file not exists. Create it os.MkdirAll(filepath.Dir(assumedTargetPosition), 0755) } //Copy the file to target err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } } else { //Deep copy. Check and match the modtime of each file if !fileExists(assumedTargetPosition) { //Copy the file to target err := BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") return nil } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } else { //Target file already exists. Check if their hash matches srcHash, err := getFileHash(fileAbs) if err != nil { log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") return nil } targetHash, err := getFileHash(assumedTargetPosition) if err != nil { log.Println("*Hybrid Backup* Hash calculation failed for file "+filepath.Base(assumedTargetPosition), err.Error(), " Skipping.") return nil } if srcHash != targetHash { log.Println("[Debug] Hash mismatch. Copying ", fileAbs) //This file has been recently changed. Copy it to new location err = BufferedLargeFileCopy(fileAbs, assumedTargetPosition, 1024) if err != nil { log.Println("*Hybrid Backup* Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.") } else { //No problem. Add this filepath into the list copiedFileList = append(copiedFileList, assumedTargetPosition) } } } } ///Remove file cycle backupDriveRootPath := filepath.ToSlash(filepath.Clean(backupConfig.DiskPath)) fastWalk(backupConfig.DiskPath, func(filename string) error { if filepath.Base(filename) == "aofs.db" || filepath.Base(filename) == "aofs.db.lock" { //Reserved filename, skipping return nil } //Get the target paste location rootAbs, _ := filepath.Abs(backupDriveRootPath) fileAbs, _ := filepath.Abs(filename) rootAbs = filepath.ToSlash(filepath.Clean(rootAbs)) fileAbs = filepath.ToSlash(filepath.Clean(fileAbs)) thisFileRel := filename[len(backupDriveRootPath):] originalFileOnDiskPath := filepath.ToSlash(filepath.Clean(filepath.Join(backupConfig.ParentPath, thisFileRel))) //Check if the taget file not exists and this file has been here for more than 24h if !fileExists(originalFileOnDiskPath) { //This file not exists. Check if it is in the delete file marker for more than 24 hours val, ok := backupConfig.DeleteFileMarkers[thisFileRel] if !ok { //This file is newly deleted. Push into the marker map backupConfig.DeleteFileMarkers[thisFileRel] = time.Now().Unix() log.Println("[Debug] Adding " + filename + " to delete marker") } else { //This file has been marked. Check if it is time to delete if time.Now().Unix()-val > 3600*24 { log.Println("[Debug] Deleting " + filename) //Remove the backup file os.RemoveAll(filename) //Remove file from delete file markers delete(backupConfig.DeleteFileMarkers, thisFileRel) } } } return nil }) return nil }) return "", nil } //Main handler function for hybrid backup func HandleBackupProcess(backupConfig *BackupConfig) (string, error) { log.Println(">>>>>> [Debug] Running backup process: ", backupConfig) //Check if the target disk is writable and mounted if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) { //This parent filesystem is mounted } else { //File system not mounted even after 3 backup cycle. Terminate backup scheduler log.Println("*HybridBackup* Skipping backup cycle for " + backupConfig.ParentUID + ":/") return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil } //Check if the backup disk is mounted. If no, stop the scheulder if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) { log.Println("*HybridBackup* Backup schedule stopped for " + backupConfig.DiskUID + ":/") return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted") } deepBackup := true //Default perform deep backup if backupConfig.Mode == "basic" { if backupConfig.CycleCounter%30 == 0 { //Perform deep backup, use walk function deepBackup = true } else { deepBackup = false } backupConfig.LastCycleTime = time.Now().Unix() return executeBackup(backupConfig, deepBackup) } else if backupConfig.Mode == "nightly" { if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 { //24 hours from last backup. Execute deep backup now executeBackup(backupConfig, true) backupConfig.LastCycleTime = time.Now().Unix() } } else if backupConfig.Mode == "version" { //Do a versioning backup log.Println("[WIP] This function is still work in progress. Please do not use version backup for now.") //WIP } //Add one to the cycle counter backupConfig.CycleCounter++ //Return the log information return "", nil } //Restore accidentailly removed file from backup func HandleRestore(backupConfig *BackupConfig, targetFile string) error { return nil } //Get and return the file hash for a file func getFileHash(filename string) (string, error) { f, err := os.Open(filename) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil }