hybridBackup.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396
  1. package hybridBackup
  2. import (
  3. "crypto/sha256"
  4. "encoding/hex"
  5. "errors"
  6. "io"
  7. "log"
  8. "os"
  9. "path/filepath"
  10. "strings"
  11. "time"
  12. )
  13. /*
  14. Hybrid Backup
  15. This module handle backup functions from the drive with Hieracchy labeled as "backup"
  16. Backup modes suport in this module currently consists of
  17. Denote P drive as parent drive and B drive as backup drive.
  18. 1. Basic (basic):
  19. - Any new file created in P will be copied to B within 1 minutes
  20. - Any file change will be copied to B within 30 minutes
  21. - Any file removed in P will be delete from backup if it is > 24 hours old
  22. 2. Nightly (nightly):
  23. - The whole P drive will be copied to N drive every night
  24. 3. Versioning (version)
  25. - A versioning system will be introduce to this backup drive
  26. - Just like the time machine
  27. Tips when developing this module
  28. - This is a sub-module of the current file system. Do not import from arozos file system module
  29. - If you need any function from the file system, copy and paste it in this module
  30. */
  31. type Manager struct {
  32. Ticker *time.Ticker `json:"-"` //The main ticker
  33. StopTicker chan bool `json:"-"` //Channel for stopping the backup
  34. Tasks []*BackupTask //The backup tasks that is running under this manager
  35. }
  36. type BackupTask struct {
  37. JobName string //The name used by the scheduler for executing this config
  38. CycleCounter int64 //The number of backup executed in the background
  39. LastCycleTime int64 //The execution time of the last cycle
  40. Enabled bool //Check if the task is enabled. Will not execute if this is set to false
  41. DiskUID string //The UID of the target fsandlr
  42. DiskPath string //The mount point for the disk
  43. ParentUID string //Parent virtal disk UUID
  44. ParentPath string //Parent disk path
  45. DeleteFileMarkers map[string]int64 //Markers for those files delete pending, [file path (relative)] time
  46. Mode string //Backup mode
  47. }
  48. //A snapshot summary
  49. type SnapshotSummary struct {
  50. ChangedFiles map[string]string
  51. UnchangedFiles map[string]string
  52. DeletedFiles map[string]string
  53. }
  54. //A file in the backup drive that is restorable
  55. type RestorableFile struct {
  56. Filename string //Filename of this restorable object
  57. IsHidden bool //Check if the file is hidden or located in a path within hidden folder
  58. Filesize int64 //The file size to be restorable
  59. RelpathOnDisk string //Relative path of this file to the root
  60. RestorePoint string //The location this file should restore to
  61. BackupDiskUID string //The UID of disk that is hold the backup of this file
  62. RemainingTime int64 //Remaining time till auto remove
  63. DeleteTime int64 //Delete time
  64. IsSnapshot bool //Define is this restorable file point to a snapshot instead
  65. }
  66. //The restorable report
  67. type RestorableReport struct {
  68. ParentUID string //The Disk ID to be restored to
  69. RestorableFiles []*RestorableFile //A list of restorable files
  70. }
  71. var (
  72. internalTickerTime time.Duration = 60
  73. )
  74. func NewHyperBackupManager() *Manager {
  75. //Create a new minute ticker
  76. ticker := time.NewTicker(internalTickerTime * time.Second)
  77. stopper := make(chan bool, 1)
  78. newManager := &Manager{
  79. Ticker: ticker,
  80. StopTicker: stopper,
  81. Tasks: []*BackupTask{},
  82. }
  83. ///Create task executor
  84. go func() {
  85. defer log.Println("[HybridBackup] Ticker Stopped")
  86. for {
  87. select {
  88. case <-ticker.C:
  89. for _, task := range newManager.Tasks {
  90. if task.Enabled == true {
  91. task.HandleBackupProcess()
  92. }
  93. }
  94. case <-stopper:
  95. return
  96. }
  97. }
  98. }()
  99. //Return the manager
  100. return newManager
  101. }
  102. func (m *Manager) AddTask(newtask *BackupTask) error {
  103. //Create a job for this
  104. newtask.JobName = "backup-[" + newtask.DiskUID + "]"
  105. //Check if the same job name exists
  106. for _, task := range m.Tasks {
  107. if task.JobName == newtask.JobName {
  108. return errors.New("Task already exists")
  109. }
  110. }
  111. //Add task to list
  112. m.Tasks = append(m.Tasks, newtask)
  113. //Start the task
  114. m.StartTask(newtask.JobName)
  115. //log.Println(">>>> [Debug] New Backup Tasks added: ", newtask.JobName, newtask)
  116. return nil
  117. }
  118. //Start a given task given name
  119. func (m *Manager) StartTask(jobname string) {
  120. for _, task := range m.Tasks {
  121. if task.JobName == jobname {
  122. //Enable to job
  123. task.Enabled = true
  124. //Run it once
  125. task.HandleBackupProcess()
  126. }
  127. }
  128. }
  129. //Stop a given task given its job name
  130. func (m *Manager) StopTask(jobname string) {
  131. for _, task := range m.Tasks {
  132. if task.JobName == jobname {
  133. task.Enabled = false
  134. }
  135. }
  136. }
  137. //Stop all managed handlers
  138. func (m *Manager) Close() error {
  139. if m != nil {
  140. m.StopTicker <- true
  141. }
  142. return nil
  143. }
  144. //Main handler function for hybrid backup
  145. func (backupConfig *BackupTask) HandleBackupProcess() (string, error) {
  146. //Check if the target disk is writable and mounted
  147. if fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.ParentPath, "aofs.db.lock")) {
  148. //This parent filesystem is mounted
  149. } else {
  150. //File system not mounted even after 3 backup cycle. Terminate backup scheduler
  151. log.Println("[HybridBackup] Skipping backup cycle for " + backupConfig.ParentUID + ":/")
  152. return "Parent drive (" + backupConfig.ParentUID + ":/) not mounted", nil
  153. }
  154. //Check if the backup disk is mounted. If no, stop the scheulder
  155. if backupConfig.CycleCounter > 3 && !(fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db")) && fileExists(filepath.Join(backupConfig.DiskPath, "aofs.db.lock"))) {
  156. log.Println("[HybridBackup] Backup schedule stopped for " + backupConfig.DiskUID + ":/")
  157. return "Backup drive (" + backupConfig.DiskUID + ":/) not mounted", errors.New("Backup File System Handler not mounted")
  158. }
  159. deepBackup := true //Default perform deep backup
  160. if backupConfig.Mode == "basic" {
  161. if backupConfig.CycleCounter%3 == 0 {
  162. //Perform deep backup, use walk function
  163. deepBackup = true
  164. log.Println("[HybridBackup] Basic backup executed: " + backupConfig.ParentUID + ":/ -> " + backupConfig.DiskUID + ":/")
  165. backupConfig.LastCycleTime = time.Now().Unix()
  166. } else {
  167. deepBackup = false
  168. }
  169. //Add one to the cycle counter
  170. backupConfig.CycleCounter++
  171. executeBackup(backupConfig, deepBackup)
  172. } else if backupConfig.Mode == "nightly" {
  173. if time.Now().Unix()-backupConfig.LastCycleTime >= 86400 {
  174. //24 hours from last backup. Execute deep backup now
  175. executeBackup(backupConfig, true)
  176. backupConfig.LastCycleTime = time.Now().Unix()
  177. log.Println("[HybridBackup] Executing nightly backup: " + backupConfig.ParentUID + ":/ -> " + backupConfig.DiskUID + ":/")
  178. //Add one to the cycle counter
  179. backupConfig.CycleCounter++
  180. }
  181. } else if backupConfig.Mode == "version" {
  182. //Do a versioning backup every 6 hours
  183. if time.Now().Unix()-backupConfig.LastCycleTime >= 21600 || backupConfig.CycleCounter == 0 {
  184. //Scheduled backup or initial backup
  185. executeVersionBackup(backupConfig)
  186. backupConfig.LastCycleTime = time.Now().Unix()
  187. log.Println("[HybridBackup] Executing backup schedule: " + backupConfig.ParentUID + ":/ -> " + backupConfig.DiskUID + ":/")
  188. //Add one to the cycle counter
  189. backupConfig.CycleCounter++
  190. }
  191. }
  192. //Return the log information
  193. return "", nil
  194. }
  195. //Get the restore parent disk ID by backup disk ID
  196. func (m *Manager) GetParentDiskIDByRestoreDiskID(restoreDiskID string) (string, error) {
  197. backupTask := m.getTaskByBackupDiskID(restoreDiskID)
  198. if backupTask == nil {
  199. return "", errors.New("This disk do not have a backup task in this backup maanger")
  200. }
  201. return backupTask.ParentUID, nil
  202. }
  203. //Restore accidentailly removed file from backup
  204. func (m *Manager) HandleRestore(restoreDiskID string, targetFileRelpath string, username *string) error {
  205. //Get the backup task from backup disk id
  206. backupTask := m.getTaskByBackupDiskID(restoreDiskID)
  207. if backupTask == nil {
  208. return errors.New("Target disk is not a backup disk")
  209. }
  210. //Check if source exists and target not exists
  211. //log.Println("[debug]", backupTask)
  212. restoreSource := filepath.Join(backupTask.DiskPath, targetFileRelpath)
  213. if backupTask.Mode == "basic" || backupTask.Mode == "nightly" {
  214. restoreSource = filepath.Join(backupTask.DiskPath, "/backup/", targetFileRelpath)
  215. restoreTarget := filepath.Join(backupTask.ParentPath, targetFileRelpath)
  216. if !fileExists(restoreSource) {
  217. //Restore source not exists
  218. return errors.New("Restore source file not exists")
  219. }
  220. if fileExists(restoreTarget) {
  221. //Restore target already exists.
  222. return errors.New("Restore target already exists. Cannot overwrite.")
  223. }
  224. //Check if the restore target parent folder exists. If not, create it
  225. if !fileExists(filepath.Dir(restoreTarget)) {
  226. os.MkdirAll(filepath.Dir(restoreTarget), 0755)
  227. }
  228. //Ready to move it back
  229. err := BufferedLargeFileCopy(restoreSource, restoreTarget, 4086)
  230. if err != nil {
  231. return errors.New("Restore failed: " + err.Error())
  232. }
  233. } else if backupTask.Mode == "version" {
  234. //Check if username is set
  235. if username == nil {
  236. return errors.New("Snapshot mode backup require username to restore")
  237. }
  238. //Restore the snapshot
  239. err := restoreSnapshotByName(backupTask, targetFileRelpath, username)
  240. if err != nil {
  241. return errors.New("Restore failed: " + err.Error())
  242. }
  243. }
  244. //Restore completed
  245. return nil
  246. }
  247. //List the file that is restorable from the given disk
  248. func (m *Manager) ListRestorable(parentDiskID string) (RestorableReport, error) {
  249. //List all the backup process that is mirroring this parent disk
  250. tasks := m.getTaskByParentDiskID(parentDiskID)
  251. if len(tasks) == 0 {
  252. return RestorableReport{}, errors.New("No backup root found for this " + parentDiskID + ":/ virtual root.")
  253. }
  254. diffFiles := []*RestorableFile{}
  255. //Extract all comparasion
  256. for _, task := range tasks {
  257. if task.Mode == "basic" || task.Mode == "nightly" {
  258. restorableFiles, err := listBasicRestorables(task)
  259. if err != nil {
  260. //Something went wrong. Skip this
  261. continue
  262. }
  263. for _, restorable := range restorableFiles {
  264. diffFiles = append(diffFiles, restorable)
  265. }
  266. } else if task.Mode == "version" {
  267. restorableFiles, err := listVersionRestorables(task)
  268. if err != nil {
  269. //Something went wrong. Skip this
  270. continue
  271. }
  272. for _, restorable := range restorableFiles {
  273. diffFiles = append(diffFiles, restorable)
  274. }
  275. } else {
  276. //Unknown mode. Skip it
  277. }
  278. }
  279. //Create a Restorable Report
  280. thisReport := RestorableReport{
  281. ParentUID: parentDiskID,
  282. RestorableFiles: diffFiles,
  283. }
  284. return thisReport, nil
  285. }
  286. //Get tasks from parent disk id, might return multiple task or no tasks
  287. func (m *Manager) getTaskByParentDiskID(parentDiskID string) []*BackupTask {
  288. //Convert ID:/ format to ID
  289. if strings.Contains(parentDiskID, ":") {
  290. parentDiskID = strings.Split(parentDiskID, ":")[0]
  291. }
  292. possibleTask := []*BackupTask{}
  293. for _, task := range m.Tasks {
  294. if task.ParentUID == parentDiskID {
  295. //This task parent is the target disk. push this to list
  296. possibleTask = append(possibleTask, task)
  297. }
  298. }
  299. return possibleTask
  300. }
  301. //Get task by backup Disk ID, only return 1 task
  302. func (m *Manager) getTaskByBackupDiskID(backupDiskID string) *BackupTask {
  303. //Trim the :/ parts
  304. if strings.Contains(backupDiskID, ":") {
  305. backupDiskID = strings.Split(backupDiskID, ":")[0]
  306. }
  307. for _, task := range m.Tasks {
  308. if task.DiskUID == backupDiskID {
  309. return task
  310. }
  311. }
  312. return nil
  313. }
  314. //Get and return the file hash for a file
  315. func getFileHash(filename string) (string, error) {
  316. f, err := os.Open(filename)
  317. if err != nil {
  318. return "", err
  319. }
  320. defer f.Close()
  321. h := sha256.New()
  322. if _, err := io.Copy(h, f); err != nil {
  323. return "", err
  324. }
  325. return hex.EncodeToString(h.Sum(nil)), nil
  326. }
  327. func (m *Manager) GetTaskByBackupDiskID(backupDiskID string) (*BackupTask, error) {
  328. targetTask := m.getTaskByBackupDiskID(backupDiskID)
  329. if targetTask == nil {
  330. return nil, errors.New("Task not found")
  331. }
  332. return targetTask, nil
  333. }