123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568 |
- package hybridBackup
- import (
- "errors"
- "log"
- "os"
- "path/filepath"
- "strings"
- "time"
- "imuslab.com/arozos/mod/disk/diskcapacity/dftool"
- "imuslab.com/arozos/mod/filesystem/hidden"
- )
- /*
- VersionBackup.go
- This scirpt file backup the data in the system nightly and create a restore point
- for the day just like BRTFS
- */
- func executeVersionBackup(backupConfig *BackupTask) (string, error) {
- //Check if the backup parent root is identical / within backup disk
- parentRootAbs, err := filepath.Abs(backupConfig.ParentPath)
- if err != nil {
- backupConfig.PanicStopped = true
- return "", errors.New("Unable to resolve parent disk path")
- }
- backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/version/"))
- if err != nil {
- backupConfig.PanicStopped = true
- return "", errors.New("Unable to resolve backup disk path")
- }
- if len(parentRootAbs) >= len(backupRootAbs) {
- if parentRootAbs[:len(backupRootAbs)] == backupRootAbs {
- //parent root is within backup root. Raise configuration error
- log.Println("[HyperBackup] Invalid backup cycle: Parent drive is located inside backup drive")
- backupConfig.PanicStopped = true
- return "", errors.New("Configuration Error. Skipping backup cycle.")
- }
- }
- backupConfig.PanicStopped = false
- todayFolderName := time.Now().Format("2006-01-02")
- lastSnapshotTime := int64(0)
- previousSnapshotExists := true
- previousSnapshotName, err := getPreviousSnapshotName(backupConfig, todayFolderName)
- if err != nil {
- previousSnapshotExists = false
- }
- snapshotLocation := filepath.Join(backupConfig.DiskPath, "/version/", todayFolderName)
- previousSnapshotLocation := ""
- var previousSnapshotMap *LinkFileMap
- if previousSnapshotExists {
- previousSnapshotLocation = filepath.Join(backupConfig.DiskPath, "/version/", previousSnapshotName)
- previousSnapshotMap, _ = readLinkFile(previousSnapshotLocation)
- lastSnapshotTime = lastModTime(previousSnapshotLocation)
- }
- //Create today folder if not exist
- if !fileExists(snapshotLocation) {
- os.MkdirAll(snapshotLocation, 0755)
- }
- //Read the previous snapshot datalink into a LinkFileMap and use binary search for higher performance
- /*
- Run a three pass compare logic between
- 1. source disk and new backup disk to check any new / modified files (created today)
- 2. yesterday backup and today backup to check any deleted files (created before, deleted today)
- 3. file in today backup disk no longer in the current source disk (created today, deleted today)
- */
- copiedFileList := []string{}
- linkedFileList := map[string]string{}
- deletedFileList := map[string]string{}
- //First pass: Check if there are any updated file from source and backup it to backup drive
- log.Println("[HybridBackup] Snapshot Stage 1 - Started " + backupConfig.JobName)
- rootAbs, _ := filepath.Abs(backupConfig.ParentPath)
- rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
- err = fastWalk(parentRootAbs, func(filename string) error {
- if filepath.Ext(filename) == ".db" || filepath.Ext(filename) == ".lock" {
- //Reserved filename, skipping
- return nil
- }
- if filepath.Ext(filename) == ".datalink" {
- //Reserved filename, skipping
- return nil
- }
- isHiddenFile, _ := hidden.IsHidden(filename, true)
- if isHiddenFile {
- //Do not backup hidden files
- return nil
- }
- //Get the target paste location
- fileAbs, _ := filepath.Abs(filename)
- fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
- relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
- fileBackupLocation := filepath.Join(backupConfig.DiskPath, "/version/", todayFolderName, relPath)
- yesterdayBackupLocation := filepath.Join(previousSnapshotLocation, relPath)
- //Check if the file exists in previous snapshot folder
- if !fileExists(yesterdayBackupLocation) {
- //Not exists in snapshot folder. Search the link file
- //fmt.Println("File not in last snapshot", yesterdayBackupLocation, previousSnapshotLocation, relPath)
- fileFoundInSnapshotLinkFile, nameOfSnapshot := previousSnapshotMap.fileExists(relPath)
- if fileFoundInSnapshotLinkFile {
- //File found in the snapshot link file. Compare the one in snapshot
- linkedSnapshotLocation := filepath.Join(backupConfig.DiskPath, "/version/", nameOfSnapshot)
- linkedSnapshotOriginalFile := filepath.Join(linkedSnapshotLocation, relPath)
- if fileExists(linkedSnapshotOriginalFile) {
- //Linked file exists. Check for changes
- if lastModTime(fileAbs) > lastModTime(linkedSnapshotLocation) {
- //This has changed recently. Match their hash to see if it is identical
- fileHashMatch, err := fileHashIdentical(fileAbs, linkedSnapshotOriginalFile)
- if err != nil {
- return nil
- }
- if fileHashMatch {
- //append this record to this snapshot linkdata file
- linkedFileList[relPath] = nameOfSnapshot
- } else {
- //File hash mismatch. Do file copy to renew data
- err = copyFileToBackupLocation(backupConfig, filename, fileBackupLocation)
- if err != nil {
- return err
- }
- copiedFileList = append(copiedFileList, fileBackupLocation)
- }
- } else {
- //It hasn't been changed since last snapshot
- linkedFileList[relPath] = nameOfSnapshot
- }
- } else {
- //Invalid snapshot linkage. Assume new and do copy
- log.Println("[HybridBackup] Link lost. Cloning source file to snapshot.")
- err = copyFileToBackupLocation(backupConfig, filename, fileBackupLocation)
- if err != nil {
- return err
- }
- copiedFileList = append(copiedFileList, fileBackupLocation)
- }
- } else {
- //This file is not in snapshot link file.
- //This is new file. Copy it to backup
- err = copyFileToBackupLocation(backupConfig, filename, fileBackupLocation)
- if err != nil {
- return err
- }
- copiedFileList = append(copiedFileList, fileBackupLocation)
- }
- } else if fileExists(yesterdayBackupLocation) {
- //The file exists in the last snapshot
- if lastModTime(fileAbs) > lastSnapshotTime {
- //Check if their hash is the same. If no, update it
- fileHashMatch, err := fileHashIdentical(fileAbs, yesterdayBackupLocation)
- if err != nil {
- return nil
- }
- if !fileHashMatch {
- //Hash mismatch. Overwrite the file
- if !fileExists(filepath.Dir(fileBackupLocation)) {
- os.MkdirAll(filepath.Dir(fileBackupLocation), 0755)
- }
- err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
- if err != nil {
- log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
- } else {
- //No problem. Add this filepath into the list
- copiedFileList = append(copiedFileList, fileBackupLocation)
- }
- } else {
- //Create a link file for this relative path
- linkedFileList[relPath] = previousSnapshotName
- }
- } else {
- //Not modified
- linkedFileList[relPath] = previousSnapshotName
- }
- } else {
- //Default case
- lastModTime := lastModTime(fileAbs)
- if lastModTime > backupConfig.LastCycleTime {
- //Check if hash the same
- srcHash, err := getFileHash(fileAbs)
- if err != nil {
- log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
- return nil
- }
- targetHash, err := getFileHash(fileBackupLocation)
- if err != nil {
- log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(fileBackupLocation), err.Error(), " Skipping.")
- return nil
- }
- if srcHash != targetHash {
- //Hash mismatch. Overwrite the file
- if !fileExists(filepath.Dir(fileBackupLocation)) {
- os.MkdirAll(filepath.Dir(fileBackupLocation), 0755)
- }
- err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
- if err != nil {
- log.Println("[HybridBackup] Copy Failed for file "+filepath.Base(fileAbs), err.Error(), " Skipping.")
- } else {
- //No problem. Add this filepath into the list
- copiedFileList = append(copiedFileList, fileBackupLocation)
- }
- }
- }
- }
- return nil
- })
- if err != nil {
- //Copy error. Mostly because of disk fulled
- return err.Error(), err
- }
- //2nd pass: Check if there are anything exists in the previous backup but no longer exists in the source now
- //For case where the file is backed up in previous snapshot but now the file has been removed
- log.Println("[HybridBackup] Snapshot Stage 2 - Started " + backupConfig.JobName)
- if previousSnapshotExists {
- fastWalk(previousSnapshotLocation, func(filename string) error {
- if filepath.Ext(filename) == ".datalink" {
- //System reserved file. Skip this
- return nil
- }
- //Get the target paste location
- rootAbs, _ := filepath.Abs(previousSnapshotLocation)
- fileAbs, _ := filepath.Abs(filename)
- rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
- fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
- relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
- sourcAssumeLocation := filepath.Join(parentRootAbs, relPath)
- //todaySnapshotLocation := filepath.Join(snapshotLocation, relPath)
- if !fileExists(sourcAssumeLocation) {
- //File exists in yesterday snapshot but not in the current source
- //Assume it has been deleted, create a dummy indicator file
- deletedFileList[relPath] = todayFolderName
- }
- return nil
- })
- //Check for deleting of unchanged file as well
- for relPath, _ := range previousSnapshotMap.UnchangedFile {
- sourcAssumeLocation := filepath.Join(parentRootAbs, relPath)
- if !fileExists(sourcAssumeLocation) {
- //The source file no longer exists
- deletedFileList[relPath] = todayFolderName
- }
- }
- }
- //3rd pass: Check if there are anything in today backup drive that didn't exists in the source drive
- //For cases where the backup is applied to overwrite an eariler backup of the same day
- log.Println("[HybridBackup] Snapshot Stage 3 - Started " + backupConfig.JobName)
- fastWalk(snapshotLocation, func(filename string) error {
- if filepath.Ext(filename) == ".db" || filepath.Ext(filename) == ".lock" {
- //Reserved filename, skipping
- return nil
- }
- if filepath.Ext(filename) == ".datalink" {
- //Deleted file marker. Skip this
- return nil
- }
- //Get the target paste location
- rootAbs, _ := filepath.Abs(snapshotLocation)
- fileAbs, _ := filepath.Abs(filename)
- rootAbs = filepath.ToSlash(filepath.Clean(rootAbs))
- fileAbs = filepath.ToSlash(filepath.Clean(fileAbs))
- relPath := strings.ReplaceAll(fileAbs, rootAbs, "")
- sourceAssumedLocation := filepath.Join(parentRootAbs, relPath)
- if !fileExists(sourceAssumedLocation) {
- //File removed from the source. Delete it from backup as well
- os.Remove(filename)
- }
- return nil
- })
- //Generate linkfile for this snapshot
- log.Println("[HybridBackup] Snapshot - Generating Linker File")
- generateLinkFile(snapshotLocation, LinkFileMap{
- UnchangedFile: linkedFileList,
- DeletedFiles: deletedFileList,
- })
- if err != nil {
- log.Println("[HybridBackup] Error! ", err.Error())
- return "", err
- }
- return "", nil
- }
- //Return the previous snapshot for the currentSnspashot
- func getPreviousSnapshotName(backupConfig *BackupTask, currentSnapshotName string) (string, error) {
- //Resolve the backup root folder
- backupRootAbs, err := filepath.Abs(filepath.Join(backupConfig.DiskPath, "/version/"))
- if err != nil {
- return "", errors.New("Unable to get the previous snapshot directory")
- }
- //Get the snapshot list and extract the snapshot date from foldername
- existingSnapshots := []string{}
- files, _ := filepath.Glob(filepath.ToSlash(filepath.Clean(backupRootAbs)) + "/*")
- for _, file := range files {
- if isDir(file) && fileExists(filepath.Join(file, "snapshot.datalink")) {
- existingSnapshots = append(existingSnapshots, filepath.Base(file))
- }
- }
- if len(existingSnapshots) == 0 {
- return "", errors.New("No snapshot found")
- }
- //Check if the current snapshot exists, if not, return the latest one
- previousSnapshotName := ""
- if fileExists(filepath.Join(backupRootAbs, currentSnapshotName)) {
- //Current snapshot exists. Find the one just above it
- lastSnapshotName := existingSnapshots[0]
- for _, snapshotName := range existingSnapshots {
- if snapshotName == currentSnapshotName {
- //This is the correct snapshot name. Get the last one as previous snapshot
- previousSnapshotName = lastSnapshotName
- } else {
- lastSnapshotName = snapshotName
- }
- }
- } else {
- //Current snapshot not exists. Use the last item in snapshots list
- previousSnapshotName = existingSnapshots[len(existingSnapshots)-1]
- }
- return previousSnapshotName, nil
- }
- func copyFileToBackupLocation(task *BackupTask, filename string, fileBackupLocation string) error {
- //Make dir the target dir if not exists
- if !fileExists(filepath.Dir(fileBackupLocation)) {
- os.MkdirAll(filepath.Dir(fileBackupLocation), 0755)
- }
- //Check if the target disk can fit the new file
- capinfo, err := dftool.GetCapacityInfoFromPath(filepath.Dir(fileBackupLocation))
- if err == nil {
- //Capacity info return normally. Estimate if the file will fit
- srcSize := fileSize(filename)
- diskSpace := capinfo.Avilable
- if diskSpace < srcSize {
- //Merge older snapshots. Maxium merging is 1 week
- for i := 0; i < 6; i++ {
- //Merge the oldest snapshot
- err = mergeOldestSnapshots(task)
- if err != nil {
- log.Println("[HybridBackup] " + err.Error())
- return errors.New("No space left on device")
- }
- //Check if there are enough space again
- capinfo, err := dftool.GetCapacityInfoFromPath(filepath.Dir(fileBackupLocation))
- if err != nil {
- log.Println("[HybridBackup] " + err.Error())
- return errors.New("No space left on device")
- }
- srcSize = fileSize(filename)
- diskSpace = capinfo.Avilable
- if diskSpace > srcSize {
- //Space enough. Break out
- break
- }
- }
- log.Println("[HybridBackup] Error: No space left on device! Require ", srcSize, "bytes but only ", diskSpace, " bytes left")
- return errors.New("No space left on device")
- }
- }
- err = BufferedLargeFileCopy(filename, fileBackupLocation, 4096)
- if err != nil {
- log.Println("[HybridBackup] Failed to copy file: ", filepath.Base(filename)+". "+err.Error())
- return err
- }
- return nil
- }
- func fileHashIdentical(srcFile string, matchingFile string) (bool, error) {
- srcHash, err := getFileHash(srcFile)
- if err != nil {
- log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(srcFile), err.Error(), " Skipping.")
- return false, nil
- }
- targetHash, err := getFileHash(matchingFile)
- if err != nil {
- log.Println("[HybridBackup] Hash calculation failed for file "+filepath.Base(matchingFile), err.Error(), " Skipping.")
- return false, nil
- }
- if srcHash != targetHash {
- return false, nil
- } else {
- return true, nil
- }
- }
- //List all restorable for version backup
- func listVersionRestorables(task *BackupTask) ([]*RestorableFile, error) {
- //Check if mode is set correctly
- restorableFiles := []*RestorableFile{}
- if task.Mode != "version" {
- return restorableFiles, errors.New("This task mode is not supported by this list function")
- }
- //List directories of the restorable snapshots
- snapshotPath := filepath.ToSlash(filepath.Clean(filepath.Join(task.DiskPath, "/version/")))
- filesInSnapshotFolder, err := filepath.Glob(snapshotPath + "/*")
- if err != nil {
- return restorableFiles, err
- }
- //Check if the foler is actually a snapshot
- avaibleSnapshot := []string{}
- for _, fileObject := range filesInSnapshotFolder {
- possibleSnapshotDatalinkFile := filepath.Join(fileObject, "snapshot.datalink")
- if fileExists(possibleSnapshotDatalinkFile) {
- //This is a snapshot
- avaibleSnapshot = append(avaibleSnapshot, fileObject)
- }
- }
- //Build restorabe file struct for returning
- for _, snapshot := range avaibleSnapshot {
- thisFile := RestorableFile{
- Filename: filepath.Base(snapshot),
- IsHidden: false,
- Filesize: 0,
- RelpathOnDisk: filepath.Base(snapshot),
- RestorePoint: task.ParentUID,
- BackupDiskUID: task.DiskUID,
- RemainingTime: -1,
- DeleteTime: -1,
- IsSnapshot: true,
- }
- restorableFiles = append(restorableFiles, &thisFile)
- }
- return restorableFiles, nil
- }
- //Check if a file in snapshot relPath (start with /) belongs to a user
- func snapshotFileBelongsToUser(relPath string, username string) bool {
- relPath = filepath.ToSlash(filepath.Clean(relPath))
- userPath := "/users/" + username + "/"
- if len(relPath) > len(userPath) && relPath[:len(userPath)] == userPath {
- return true
- } else {
- return false
- }
- }
- //This function generate and return a snapshot summary. For public drive, leave username as nil
- func (task *BackupTask) GenerateSnapshotSummary(snapshotName string, username *string) (*SnapshotSummary, error) {
- //Check if the task is version
- if task.Mode != "version" {
- return nil, errors.New("Invalid backup mode. This function only support snapshot mode backup task.")
- }
- userSumamryMode := false
- targetUserName := ""
- if username != nil {
- targetUserName = *username
- userSumamryMode = true
- }
- //Check if the snapshot folder exists
- targetSnapshotFolder := filepath.Join(task.DiskPath, "/version/", snapshotName)
- if !fileExists(targetSnapshotFolder) {
- return nil, errors.New("Snapshot not exists")
- }
- if !fileExists(filepath.Join(targetSnapshotFolder, "snapshot.datalink")) {
- return nil, errors.New("Snapshot datalink file not exists")
- }
- summary := SnapshotSummary{
- ChangedFiles: map[string]string{},
- UnchangedFiles: map[string]string{},
- DeletedFiles: map[string]string{},
- }
- fastWalk(targetSnapshotFolder, func(filename string) error {
- if filepath.Base(filename) == "snapshot.datalink" {
- //Exceptional
- return nil
- }
- relPath, err := filepath.Rel(targetSnapshotFolder, filename)
- if err != nil {
- return err
- }
- //Check if user mode, check if folder owned by them
- if userSumamryMode == true {
- if snapshotFileBelongsToUser("/"+filepath.ToSlash(relPath), targetUserName) {
- summary.ChangedFiles["/"+filepath.ToSlash(relPath)] = snapshotName
- }
- } else {
- summary.ChangedFiles["/"+filepath.ToSlash(relPath)] = snapshotName
- }
- return nil
- })
- //Generate the summary
- linkFileMap, err := readLinkFile(targetSnapshotFolder)
- if err != nil {
- return nil, err
- }
- //Move the file map into result
- if userSumamryMode {
- //Only show the files that belongs to this user
- for relPath, linkTarget := range linkFileMap.UnchangedFile {
- if snapshotFileBelongsToUser(filepath.ToSlash(relPath), targetUserName) {
- summary.UnchangedFiles[filepath.ToSlash(relPath)] = linkTarget
- }
- }
- for relPath, linkTarget := range linkFileMap.DeletedFiles {
- if snapshotFileBelongsToUser(filepath.ToSlash(relPath), targetUserName) {
- summary.DeletedFiles[filepath.ToSlash(relPath)] = linkTarget
- }
- }
- } else {
- //Show all files (public mode)
- summary.UnchangedFiles = linkFileMap.UnchangedFile
- summary.DeletedFiles = linkFileMap.DeletedFiles
- }
- return &summary, nil
- }
|