package hybridBackup import ( "crypto/sha256" "encoding/hex" "errors" "io" "log" "os" "path/filepath" "strings" "time" ) /* Hybrid Backup This module handle backup functions from the drive with Hieracchy labeled as "backup" Backup modes suport in this module currently consists of Denote P drive as parent drive and B drive as backup drive. 1. Basic (basic): - Any new file created in P will be copied to B within 1 minutes - Any file change will be copied to B within 30 minutes - Any file removed in P will be delete from backup if it is > 24 hours old 2. Nightly (nightly): - The whole P drive will be copied to N drive every night 3. Versioning (version) - A versioning system will be introduce to this backup drive - Just like the time machine Tips when developing this module - This is a sub-module of the current file system. Do not import from arozos file system module - If you need any function from the file system, copy and paste it in this module */ type Manager struct { Ticker *time.Ticker `json:"-"` //The main ticker StopTicker chan bool `json:"-"` //Channel for stopping the backup Tasks []*BackupTask //The backup tasks that is running under this manager } type BackupTask struct { JobName string //The name used by the scheduler for executing this config CycleCounter int64 //The number of backup executed in the background LastCycleTime int64 //The execution time of the last cycle Enabled bool //Check if the task is enabled. Will not execute if this is set to false DiskUID string //The UID of the target fsandlr DiskPath string //The mount point for the disk ParentUID string //Parent virtal disk UUID ParentPath string //Parent disk path DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time Mode string //Backup mode } //A file in the backup drive that is restorable type RestorableFile struct { Filename string //Filename of this restorable object IsHidden bool //Check if the file is hidden or located in a path within hidden folder Filesize int64 //The file size to be restorable RelpathOnDisk string //Relative path of this file to the root RestorePoint string //The location this file should restore to BackupDiskUID string //The UID of disk that is hold the backup of this file RemainingTime int64 //Remaining time till auto remove DeleteTime int64 //Delete time } //The restorable report type RestorableReport struct { ParentUID string //The Disk ID to be restored to RestorableFiles []*RestorableFile //A list of restorable files } var ( internalTickerTime time.Duration = 60 ) func NewHyperBackupManager() *Manager { //Create a new minute ticker ticker := time.NewTicker(internalTickerTime * time.Second) stopper := make(chan bool, 1) newManager := &Manager{ Ticker: ticker, StopTicker: stopper, Tasks: []*BackupTask{}, } ///Create task executor go func() { defer log.Println("[HybridBackup] Ticker Stopped") for { select { case <-ticker.C: for _, task := range newManager.Tasks { if task.Enabled == true { task.HandleBackupProcess() } } case <-stopper: return } } }() //Return the manager return newManager } func (m *Manager) AddTask(newtask *BackupTask) error { //Create a job for this newtask.JobName = "backup-[" + newtask.DiskUID + "]" //Check if the same job name exists for _, task := range m.Tasks { if task.JobName == newtask.JobName { return errors.New("Task already exists") } } //Add task to list m.Tasks = append(m.Tasks, newtask) //Start the task m.StartTask(newtask.JobName) log.Println(">>>> [Debug] New Backup Tasks added: ", newtask.JobName, newtask) return nil } //Start a given task given name func (m *Manager) StartTask(jobname string) { for _, task := range m.Tasks { if task.JobName == jobname { //Enable to job task.Enabled = true //Run it once task.HandleBackupProcess() } } } //Stop a given task given its job name func (m *Manager) StopTask(jobname string) { for _, task := range m.Tasks { if task.JobName == jobname { task.Enabled = false } } } //Stop all managed handlers func (m *Manager) Close() error { m.StopTicker <- true return nil } //Main handler function for hybrid backup func (backupConfig *BackupTask) HandleBackupProcess() (string, error) { log.Println(">>>>>> [Debug] Running backup process: ", backupConfig) //Check if the target disk is writable and mounted if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) { //This parent filesystem is mounted } else { //File system not mounted even after 3 backup cycle. Terminate backup scheduler log.Println("[HybridBackup] Skipping backup cycle for " + backupConfig.ParentUID + ":/") return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil } //Check if the backup disk is mounted. If no, stop the scheulder if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) { log.Println("[HybridBackup] Backup schedule stopped for " + backupConfig.DiskUID + ":/") return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted") } deepBackup := true //Default perform deep backup if backupConfig.Mode == "basic" { if backupConfig.CycleCounter%3 == 0 { //Perform deep backup, use walk function deepBackup = true } else { deepBackup = false backupConfig.LastCycleTime = time.Now().Unix() } return executeBackup(backupConfig, deepBackup) } else if backupConfig.Mode == "nightly" { if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 { //24 hours from last backup. Execute deep backup now executeBackup(backupConfig, true) backupConfig.LastCycleTime = time.Now().Unix() } } else if backupConfig.Mode == "version" { //Do a versioning backup if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 || backupConfig.CycleCounter == 0 { //Scheduled backup or initial backup executeVersionBackup(backupConfig) backupConfig.LastCycleTime = time.Now().Unix() } } //Add one to the cycle counter backupConfig.CycleCounter++ //Return the log information return "", nil } //Get the restore parent disk ID by backup disk ID func (m *Manager) GetParentDiskIDByRestoreDiskID(restoreDiskID string) (string, error) { backupTask := m.getTaskByBackupDiskID(restoreDiskID) if backupTask == nil { return "", errors.New("This disk do not have a backup task in this backup maanger") } return backupTask.ParentUID, nil } //Restore accidentailly removed file from backup func (m *Manager) HandleRestore(restoreDiskID string, targetFileRelpath string) error { //Get the backup task from backup disk id backupTask := m.getTaskByBackupDiskID(restoreDiskID) if backupTask == nil { return errors.New("Target disk is not a backup disk") } //Check if source exists and target not exists log.Println("[debug]", backupTask) restoreSource := filepath.Join(backupTask.DiskPath, targetFileRelpath) if backupTask.Mode == "basic" || backupTask.Mode == "nightly" { restoreSource = filepath.Join(backupTask.DiskPath, "/backup/", targetFileRelpath) } else if backupTask.Mode == "version" { restoreSource = filepath.Join(backupTask.DiskPath, "/versions/", targetFileRelpath) } restoreTarget := filepath.Join(backupTask.ParentPath, targetFileRelpath) if !fileExists(restoreSource) { //Restore source not exists return errors.New("Restore source file not exists") } if fileExists(restoreTarget) { //Restore target already exists. return errors.New("Restore target already exists. Cannot overwrite.") } //Check if the restore target parent folder exists. If not, create it if !fileExists(filepath.Dir(restoreTarget)) { os.MkdirAll(filepath.Dir(restoreTarget), 0755) } //Ready to move it back err := BufferedLargeFileCopy(restoreSource, restoreTarget, 4086) if err != nil { return errors.New("Restore failed: " + err.Error()) } //Restore completed return nil } //List the file that is restorable from the given disk func (m *Manager) ListRestorable(parentDiskID string) (RestorableReport, error) { //List all the backup process that is mirroring this parent disk tasks := m.getTaskByParentDiskID(parentDiskID) if len(tasks) == 0 { return RestorableReport{}, errors.New("No backup root found for this " + parentDiskID + ":/ virtual root.") } diffFiles := []*RestorableFile{} //Extract all comparasion for _, task := range tasks { if task.Mode == "basic" || task.Mode == "nightly" { restorableFiles, err := listBasicRestorables(task) if err != nil { //Something went wrong. Skip this continue } for _, restorable := range restorableFiles { diffFiles = append(diffFiles, restorable) } } else if task.Mode == "version" { } else { //Unknown mode. Skip it } } //Create a Restorable Report thisReport := RestorableReport{ ParentUID: parentDiskID, RestorableFiles: diffFiles, } return thisReport, nil } //Get tasks from parent disk id, might return multiple task or no tasks func (m *Manager) getTaskByParentDiskID(parentDiskID string) []*BackupTask { //Convert ID:/ format to ID if strings.Contains(parentDiskID, ":") { parentDiskID = strings.Split(parentDiskID, ":")[0] } possibleTask := []*BackupTask{} for _, task := range m.Tasks { if task.ParentUID == parentDiskID { //This task parent is the target disk. push this to list possibleTask = append(possibleTask, task) } } return possibleTask } //Get task by backup Disk ID, only return 1 task func (m *Manager) getTaskByBackupDiskID(backupDiskID string) *BackupTask { //Trim the :/ parts if strings.Contains(backupDiskID, ":") { backupDiskID = strings.Split(backupDiskID, ":")[0] } for _, task := range m.Tasks { if task.DiskUID == backupDiskID { return task } } return nil } //Get and return the file hash for a file func getFileHash(filename string) (string, error) { f, err := os.Open(filename) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil }