fileOpr.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924
  1. package filesystem
  2. /*
  3. File Operation Wrapper
  4. author: tobychui
  5. This is a module seperated from the aroz online file system script
  6. that allows cleaner code in the main logic handler of the aroz online system.
  7. WARNING! ALL FILE OPERATION USING THIS WRAPPER SHOULD PASS IN REALPATH
  8. DO NOT USE VIRTUAL PATH FOR ANY OPERATIONS WITH THIS WRAPPER
  9. */
  10. import (
  11. "archive/tar"
  12. "archive/zip"
  13. "compress/flate"
  14. "compress/gzip"
  15. "errors"
  16. "fmt"
  17. "io"
  18. "log"
  19. "os"
  20. "path/filepath"
  21. "strconv"
  22. "strings"
  23. "time"
  24. "imuslab.com/arozos/mod/filesystem/arozfs"
  25. "imuslab.com/arozos/mod/filesystem/hidden"
  26. archiver "github.com/mholt/archiver/v3"
  27. )
  28. //A basic file zipping function
  29. func ZipFile(filelist []string, outputfile string, includeTopLevelFolder bool) error {
  30. z := archiver.Zip{
  31. CompressionLevel: flate.DefaultCompression,
  32. MkdirAll: true,
  33. SelectiveCompression: true,
  34. OverwriteExisting: false,
  35. ImplicitTopLevelFolder: includeTopLevelFolder,
  36. }
  37. err := z.Archive(filelist, outputfile)
  38. return err
  39. }
  40. //A basic file unzip function
  41. func Unzip(source, destination string) error {
  42. archive, err := zip.OpenReader(source)
  43. if err != nil {
  44. return err
  45. }
  46. defer archive.Close()
  47. for _, file := range archive.Reader.File {
  48. reader, err := file.Open()
  49. if err != nil {
  50. return err
  51. }
  52. defer reader.Close()
  53. path := filepath.Join(destination, file.Name)
  54. err = os.MkdirAll(path, os.ModePerm)
  55. if err != nil {
  56. return err
  57. }
  58. if file.FileInfo().IsDir() {
  59. continue
  60. }
  61. err = os.Remove(path)
  62. if err != nil {
  63. return err
  64. }
  65. writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
  66. if err != nil {
  67. return err
  68. }
  69. defer writer.Close()
  70. _, err = io.Copy(writer, reader)
  71. if err != nil {
  72. return err
  73. }
  74. }
  75. return nil
  76. }
  77. //Aroz Unzip File with progress update function (current filename / current file count / total file count / progress in percentage)
  78. func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHandler func(string, int, int, float64)) error {
  79. //Gether the total number of files in all zip files
  80. totalFileCounts := 0
  81. unzippedFileCount := 0
  82. for _, srcFile := range filelist {
  83. archive, err := zip.OpenReader(srcFile)
  84. if err != nil {
  85. return err
  86. }
  87. totalFileCounts += len(archive.Reader.File)
  88. archive.Close()
  89. }
  90. //Start extracting
  91. for _, srcFile := range filelist {
  92. archive, err := zip.OpenReader(srcFile)
  93. if err != nil {
  94. return err
  95. }
  96. defer archive.Close()
  97. for _, file := range archive.Reader.File {
  98. reader, err := file.Open()
  99. if err != nil {
  100. return err
  101. }
  102. defer reader.Close()
  103. path := filepath.Join(outputfile, file.Name)
  104. parentFolder := path
  105. if !file.FileInfo().IsDir() {
  106. //Is file. Change target to its parent dir
  107. parentFolder = filepath.Dir(path)
  108. }
  109. err = os.MkdirAll(parentFolder, 0775)
  110. if err != nil {
  111. return err
  112. }
  113. if file.FileInfo().IsDir() {
  114. //Folder is created already be the steps above.
  115. //Update the progress
  116. unzippedFileCount++
  117. progressHandler(file.Name, unzippedFileCount, totalFileCounts, float64(unzippedFileCount)/float64(totalFileCounts)*100.0)
  118. continue
  119. }
  120. //Extrat and write to the target file
  121. writer, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
  122. if err != nil {
  123. return err
  124. }
  125. _, err = io.Copy(writer, reader)
  126. if err != nil {
  127. //Extraction failed. Remove this file if exists
  128. writer.Close()
  129. if FileExists(path) {
  130. os.Remove(path)
  131. }
  132. return err
  133. }
  134. writer.Close()
  135. //Update the progress
  136. unzippedFileCount++
  137. progressHandler(file.Name, unzippedFileCount, totalFileCounts, float64(unzippedFileCount)/float64(totalFileCounts)*100.0)
  138. }
  139. }
  140. return nil
  141. }
  142. /*
  143. Aroz Zip File with progress update function
  144. Returns the following progress: (current filename / current file count / total file count / progress in percentage)
  145. if output is local path that is out of the scope of any fsh, leave outputFsh as nil
  146. */
  147. func ArozZipFileWithProgress(targetFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64)) error {
  148. fmt.Println("WEBSOCKET ZIPPING", targetFshs, filelist)
  149. //Get the file count from the filelist
  150. totalFileCount := 0
  151. for i, srcpath := range filelist {
  152. thisFsh := targetFshs[i]
  153. fshAbs := thisFsh.FileSystemAbstraction
  154. if thisFsh.FileSystemAbstraction.IsDir(srcpath) {
  155. fshAbs.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
  156. if !info.IsDir() {
  157. totalFileCount++
  158. }
  159. return nil
  160. })
  161. } else {
  162. totalFileCount++
  163. }
  164. }
  165. //Create the target zip file
  166. var file arozfs.File
  167. var err error
  168. if outputFsh != nil {
  169. file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
  170. } else {
  171. //Force local fs
  172. file, err = os.Create(outputfile)
  173. }
  174. if err != nil {
  175. return err
  176. }
  177. defer file.Close()
  178. writer := zip.NewWriter(file)
  179. defer writer.Close()
  180. currentFileCount := 0
  181. for i, srcpath := range filelist {
  182. thisFsh := targetFshs[i]
  183. fshAbs := thisFsh.FileSystemAbstraction
  184. //Local File System
  185. if fshAbs.IsDir(srcpath) {
  186. //This is a directory
  187. topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
  188. err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
  189. if err != nil {
  190. return err
  191. }
  192. if info.IsDir() {
  193. return nil
  194. }
  195. if insideHiddenFolder(path) {
  196. //This is hidden file / folder. Skip this
  197. return nil
  198. }
  199. file, err := fshAbs.ReadStream(path)
  200. if err != nil {
  201. return err
  202. }
  203. defer file.Close()
  204. relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
  205. if includeTopLevelFolder {
  206. relativePath = topLevelFolderName + "/" + relativePath
  207. } else {
  208. relativePath = filepath.Base(srcpath) + "/" + relativePath
  209. }
  210. f, err := writer.Create(relativePath)
  211. if err != nil {
  212. return err
  213. }
  214. _, err = io.Copy(f, file)
  215. if err != nil {
  216. return err
  217. }
  218. //Update the zip progress
  219. currentFileCount++
  220. progressHandler(filepath.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
  221. return nil
  222. })
  223. if err != nil {
  224. return err
  225. }
  226. } else {
  227. //This is a file
  228. topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
  229. file, err := fshAbs.ReadStream(srcpath)
  230. if err != nil {
  231. return err
  232. }
  233. defer file.Close()
  234. relativePath := filepath.Base(srcpath)
  235. if includeTopLevelFolder {
  236. relativePath = topLevelFolderName + "/" + relativePath
  237. }
  238. f, err := writer.Create(relativePath)
  239. if err != nil {
  240. return err
  241. }
  242. _, err = io.Copy(f, file)
  243. if err != nil {
  244. return err
  245. }
  246. //Update the zip progress
  247. currentFileCount++
  248. progressHandler(filepath.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
  249. }
  250. }
  251. return nil
  252. }
  253. /*
  254. ArozZipFile
  255. Zip file without progress update, support local file system or buffer space
  256. To use it with local file system, pass in nil in fsh for each item in filelist, e.g.
  257. filesystem.ArozZipFile([]*filesystem.FileSystemHandler{nil}, []string{zippingSource}, nil, targetZipFilename, false)
  258. */
  259. func ArozZipFile(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool) error {
  260. //Create the target zip file
  261. var file arozfs.File
  262. var err error
  263. if outputFsh != nil {
  264. file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
  265. } else {
  266. //Force local fs
  267. file, err = os.Create(outputfile)
  268. }
  269. if err != nil {
  270. return err
  271. }
  272. defer file.Close()
  273. writer := zip.NewWriter(file)
  274. defer writer.Close()
  275. for i, srcpath := range filelist {
  276. thisFsh := sourceFshs[i]
  277. var fshAbs FileSystemAbstraction
  278. if thisFsh == nil {
  279. //Use local fs functions
  280. if IsDir(srcpath) {
  281. //This is a directory
  282. topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
  283. err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
  284. if err != nil {
  285. return err
  286. }
  287. if info.IsDir() {
  288. return nil
  289. }
  290. if insideHiddenFolder(path) {
  291. //This is hidden file / folder. Skip this
  292. return nil
  293. }
  294. file, err := os.Open(path)
  295. if err != nil {
  296. return err
  297. }
  298. defer file.Close()
  299. relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
  300. if includeTopLevelFolder {
  301. relativePath = topLevelFolderName + "/" + relativePath
  302. } else {
  303. relativePath = filepath.Base(srcpath) + "/" + relativePath
  304. }
  305. f, err := writer.Create(relativePath)
  306. if err != nil {
  307. return err
  308. }
  309. _, err = io.Copy(f, file)
  310. if err != nil {
  311. return err
  312. }
  313. return nil
  314. })
  315. if err != nil {
  316. return err
  317. }
  318. } else {
  319. //This is a file
  320. topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
  321. file, err := os.Open(srcpath)
  322. if err != nil {
  323. return err
  324. }
  325. defer file.Close()
  326. relativePath := filepath.Base(srcpath)
  327. if includeTopLevelFolder {
  328. relativePath = topLevelFolderName + "/" + relativePath
  329. }
  330. f, err := writer.Create(relativePath)
  331. if err != nil {
  332. return err
  333. }
  334. _, err = io.Copy(f, file)
  335. if err != nil {
  336. return err
  337. }
  338. }
  339. } else {
  340. //Use file system abstraction
  341. fshAbs = thisFsh.FileSystemAbstraction
  342. if fshAbs.IsDir(srcpath) {
  343. //This is a directory
  344. topLevelFolderName := filepath.ToSlash(filepath.Base(filepath.Dir(srcpath)) + "/" + filepath.Base(srcpath))
  345. err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
  346. if err != nil {
  347. return err
  348. }
  349. if info.IsDir() {
  350. return nil
  351. }
  352. if insideHiddenFolder(path) {
  353. //This is hidden file / folder. Skip this
  354. return nil
  355. }
  356. file, err := fshAbs.ReadStream(path)
  357. if err != nil {
  358. fmt.Println(err)
  359. return err
  360. }
  361. defer file.Close()
  362. relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
  363. if includeTopLevelFolder {
  364. relativePath = topLevelFolderName + "/" + relativePath
  365. } else {
  366. relativePath = filepath.Base(srcpath) + "/" + relativePath
  367. }
  368. f, err := writer.Create(relativePath)
  369. if err != nil {
  370. return err
  371. }
  372. _, err = io.Copy(f, file)
  373. if err != nil {
  374. return err
  375. }
  376. return nil
  377. })
  378. if err != nil {
  379. return err
  380. }
  381. } else {
  382. //This is a file
  383. topLevelFolderName := filepath.Base(filepath.Dir(srcpath))
  384. file, err := fshAbs.ReadStream(srcpath)
  385. if err != nil {
  386. return err
  387. }
  388. defer file.Close()
  389. relativePath := filepath.Base(srcpath)
  390. if includeTopLevelFolder {
  391. relativePath = topLevelFolderName + "/" + relativePath
  392. }
  393. f, err := writer.Create(relativePath)
  394. if err != nil {
  395. return err
  396. }
  397. _, err = io.Copy(f, file)
  398. if err != nil {
  399. return err
  400. }
  401. }
  402. }
  403. }
  404. return nil
  405. }
  406. func insideHiddenFolder(path string) bool {
  407. FileIsHidden, err := hidden.IsHidden(path, true)
  408. if err != nil {
  409. //Read error. Maybe permission issue, assuem is hidden
  410. return true
  411. }
  412. return FileIsHidden
  413. }
  414. func ViewZipFile(filepath string) ([]string, error) {
  415. z := archiver.Zip{}
  416. filelist := []string{}
  417. err := z.Walk(filepath, func(f archiver.File) error {
  418. filelist = append(filelist, f.Name())
  419. return nil
  420. })
  421. return filelist, err
  422. }
  423. func FileCopy(srcFsh *FileSystemHandler, src string, destFsh *FileSystemHandler, dest string, mode string, progressUpdate func(int, string)) error {
  424. srcFshAbs := srcFsh.FileSystemAbstraction
  425. destFshAbs := destFsh.FileSystemAbstraction
  426. if srcFshAbs.IsDir(src) && strings.HasPrefix(dest, src) {
  427. //Recursive operation. Reject
  428. return errors.New("Recursive copy operation.")
  429. }
  430. //Check if the copy destination file already have an identical file
  431. copiedFilename := filepath.Base(src)
  432. if destFshAbs.FileExists(filepath.Join(dest, filepath.Base(src))) {
  433. if mode == "" {
  434. //Do not specific file exists principle
  435. return errors.New("Destination file already exists.")
  436. } else if mode == "skip" {
  437. //Skip this file
  438. return nil
  439. } else if mode == "overwrite" {
  440. //Continue with the following code
  441. //Check if the copy and paste dest are identical
  442. if filepath.ToSlash(filepath.Clean(src)) == filepath.ToSlash(filepath.Clean(filepath.Join(dest, filepath.Base(src)))) {
  443. //Source and target identical. Cannot overwrite.
  444. return errors.New("Source and destination paths are identical.")
  445. }
  446. } else if mode == "keep" {
  447. //Keep the file but saved with 'Copy' suffix
  448. newFilename := strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy" + filepath.Ext(src)
  449. //Check if the newFilename already exists. If yes, continue adding suffix
  450. duplicateCounter := 0
  451. for destFshAbs.FileExists(filepath.Join(dest, newFilename)) {
  452. duplicateCounter++
  453. newFilename = strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy(" + strconv.Itoa(duplicateCounter) + ")" + filepath.Ext(src)
  454. if duplicateCounter > 1024 {
  455. //Maxmium loop encountered. For thread safty, terminate here
  456. return errors.New("Too many copies of identical files.")
  457. }
  458. }
  459. copiedFilename = newFilename
  460. } else {
  461. //This exists opr not supported.
  462. return errors.New("Unknown file exists rules given.")
  463. }
  464. }
  465. //Ready to move. Check if both folder are located in the same root devices. If not, use copy and delete method.
  466. if srcFshAbs.IsDir(src) {
  467. //Source file is directory. CopyFolder
  468. realDest := filepath.Join(dest, copiedFilename)
  469. err := dirCopy(srcFsh, src, destFsh, realDest, progressUpdate)
  470. if err != nil {
  471. return err
  472. }
  473. } else {
  474. //Source is file only. Copy file.
  475. realDest := filepath.Join(dest, copiedFilename)
  476. f, err := srcFshAbs.ReadStream(src)
  477. if err != nil {
  478. return err
  479. }
  480. defer f.Close()
  481. err = destFshAbs.WriteStream(realDest, f, 0775)
  482. if err != nil {
  483. return err
  484. }
  485. if progressUpdate != nil {
  486. //Set progress to 100, leave it to upper level abstraction to handle
  487. progressUpdate(100, filepath.Base(realDest))
  488. }
  489. }
  490. return nil
  491. }
  492. func FileMove(srcFsh *FileSystemHandler, src string, destFsh *FileSystemHandler, dest string, mode string, fastMove bool, progressUpdate func(int, string)) error {
  493. srcAbst := srcFsh.FileSystemAbstraction
  494. destAbst := destFsh.FileSystemAbstraction
  495. src = filepath.ToSlash(src)
  496. dest = filepath.ToSlash(dest)
  497. if srcAbst.IsDir(src) && strings.HasPrefix(dest, src) {
  498. //Recursive operation. Reject
  499. return errors.New("Recursive move operation.")
  500. }
  501. if !destAbst.FileExists(dest) {
  502. if destAbst.FileExists(filepath.Dir(dest)) {
  503. //User pass in the whole path for the folder. Report error usecase.
  504. return errors.New("Dest location should be an existing folder instead of the full path of the moved file.")
  505. }
  506. os.MkdirAll(dest, 0775)
  507. }
  508. //Check if the target file already exists.
  509. movedFilename := filepath.Base(src)
  510. if destAbst.FileExists(filepath.Join(dest, filepath.Base(src))) {
  511. //Handle cases where file already exists
  512. if mode == "" {
  513. //Do not specific file exists principle
  514. return errors.New("Destination file already exists.")
  515. } else if mode == "skip" {
  516. //Skip this file
  517. return nil
  518. } else if mode == "overwrite" {
  519. //Continue with the following code
  520. //Check if the copy and paste dest are identical
  521. if filepath.ToSlash(filepath.Clean(src)) == filepath.ToSlash(filepath.Clean(filepath.Join(dest, filepath.Base(src)))) {
  522. //Source and target identical. Cannot overwrite.
  523. return errors.New("Source and destination paths are identical.")
  524. }
  525. } else if mode == "keep" {
  526. //Keep the file but saved with 'Copy' suffix
  527. newFilename := strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy" + filepath.Ext(src)
  528. //Check if the newFilename already exists. If yes, continue adding suffix
  529. duplicateCounter := 0
  530. for destAbst.FileExists(filepath.Join(dest, newFilename)) {
  531. duplicateCounter++
  532. newFilename = strings.TrimSuffix(filepath.Base(src), filepath.Ext(src)) + " - Copy(" + strconv.Itoa(duplicateCounter) + ")" + filepath.Ext(src)
  533. if duplicateCounter > 1024 {
  534. //Maxmium loop encountered. For thread safty, terminate here
  535. return errors.New("Too many copies of identical files.")
  536. }
  537. }
  538. movedFilename = newFilename
  539. } else {
  540. //This exists opr not supported.
  541. return errors.New("Unknown file exists rules given.")
  542. }
  543. }
  544. if fastMove {
  545. //Ready to move with the quick rename method
  546. realDest := filepath.Join(dest, movedFilename)
  547. err := os.Rename(src, realDest)
  548. if err == nil {
  549. //Fast move success
  550. return nil
  551. }
  552. //Fast move failed. Back to the original copy and move method
  553. }
  554. //Ready to move. Check if both folder are located in the same root devices. If not, use copy and delete method.
  555. if srcAbst.IsDir(src) {
  556. //Source file is directory. CopyFolder
  557. realDest := filepath.Join(dest, movedFilename)
  558. //err := dircpy.Copy(src, realDest)
  559. err := dirCopy(srcFsh, src, destFsh, realDest, progressUpdate)
  560. if err != nil {
  561. return err
  562. } else {
  563. //Move completed. Remove source file.
  564. srcAbst.RemoveAll(src)
  565. return nil
  566. }
  567. } else {
  568. //Source is file only. Copy file.
  569. realDest := filepath.Join(dest, movedFilename)
  570. /*
  571. Updates 20-10-2020, replaced io.Copy to BufferedLargeFileCopy
  572. Legacy code removed.
  573. */
  574. //Update the progress
  575. if progressUpdate != nil {
  576. progressUpdate(100, filepath.Base(src))
  577. }
  578. f, err := srcAbst.ReadStream(src)
  579. if err != nil {
  580. fmt.Println(err)
  581. return err
  582. }
  583. defer f.Close()
  584. err = destAbst.WriteStream(realDest, f, 0755)
  585. if err != nil {
  586. fmt.Println(err)
  587. return err
  588. }
  589. //Delete the source file after copy
  590. err = srcAbst.Remove(src)
  591. counter := 0
  592. for err != nil {
  593. //Sometime Windows need this to prevent windows caching bring problems to file remove
  594. time.Sleep(1 * time.Second)
  595. os.Remove(src)
  596. counter++
  597. log.Println("Retrying to remove file: " + src)
  598. if counter > 10 {
  599. return errors.New("Source file remove failed.")
  600. }
  601. }
  602. }
  603. return nil
  604. }
  605. //Copy a given directory, with no progress udpate
  606. func CopyDir(srcFsh *FileSystemHandler, src string, destFsh *FileSystemHandler, dest string) error {
  607. return dirCopy(srcFsh, src, destFsh, dest, func(progress int, name string) {})
  608. }
  609. //Replacment of the legacy dirCopy plugin with filepath.Walk function. Allowing real time progress update to front end
  610. func dirCopy(srcFsh *FileSystemHandler, src string, destFsh *FileSystemHandler, realDest string, progressUpdate func(int, string)) error {
  611. srcFshAbs := srcFsh.FileSystemAbstraction
  612. destFshAbs := destFsh.FileSystemAbstraction
  613. //Get the total file counts
  614. totalFileCounts := int64(0)
  615. srcFshAbs.Walk(src, func(path string, info os.FileInfo, err error) error {
  616. if !info.IsDir() {
  617. //Updates 22 April 2021, chnaged from file count to file size for progress update
  618. //totalFileCounts++
  619. totalFileCounts += info.Size()
  620. }
  621. return nil
  622. })
  623. //Make the destinaton directory
  624. if !destFshAbs.FileExists(realDest) {
  625. destFshAbs.Mkdir(realDest, 0755)
  626. }
  627. //Start moving
  628. fileCounter := int64(0)
  629. src = filepath.ToSlash(src)
  630. err := srcFshAbs.Walk(src, func(path string, info os.FileInfo, err error) error {
  631. path = filepath.ToSlash(path)
  632. var folderRootRelative string = strings.TrimPrefix(path, src)
  633. if folderRootRelative == "" {
  634. return nil
  635. }
  636. if info.IsDir() {
  637. //Mkdir base on relative path
  638. return destFshAbs.MkdirAll(filepath.Join(realDest, folderRootRelative), 0755)
  639. } else {
  640. //fileCounter++
  641. fileCounter += info.Size()
  642. //Move file base on relative path
  643. fileSrc := filepath.ToSlash(filepath.Join(filepath.Clean(src), folderRootRelative))
  644. fileDest := filepath.ToSlash(filepath.Join(filepath.Clean(realDest), folderRootRelative))
  645. //Update move progress
  646. if progressUpdate != nil {
  647. progressUpdate(int(float64(fileCounter)/float64(totalFileCounts)*100), filepath.Base(fileSrc))
  648. }
  649. //Move the file using BLFC
  650. f, err := srcFshAbs.ReadStream(fileSrc)
  651. if err != nil {
  652. fmt.Println(err)
  653. return err
  654. }
  655. defer f.Close()
  656. err = destFshAbs.WriteStream(fileDest, f, 0755)
  657. if err != nil {
  658. fmt.Println(err)
  659. return err
  660. }
  661. }
  662. return nil
  663. })
  664. return err
  665. }
  666. func BasicFileCopy(src string, dst string) error {
  667. sourceFileStat, err := os.Stat(src)
  668. if err != nil {
  669. return err
  670. }
  671. if !sourceFileStat.Mode().IsRegular() {
  672. return fmt.Errorf("%s is not a regular file", src)
  673. }
  674. source, err := os.Open(src)
  675. if err != nil {
  676. return err
  677. }
  678. defer source.Close()
  679. destination, err := os.Create(dst)
  680. if err != nil {
  681. return err
  682. }
  683. defer destination.Close()
  684. _, err = io.Copy(destination, source)
  685. return err
  686. }
  687. //Use for copying large file using buffering method. Allowing copying large file with little RAM
  688. //Deprecated Since ArozOS v2.000
  689. /*
  690. func BufferedLargeFileCopy(src string, dst string, BUFFERSIZE int64) error {
  691. sourceFileStat, err := os.Stat(src)
  692. if err != nil {
  693. return err
  694. }
  695. if !sourceFileStat.Mode().IsRegular() {
  696. return errors.New("Invalid file source")
  697. }
  698. source, err := os.Open(src)
  699. if err != nil {
  700. return err
  701. }
  702. destination, err := os.Create(dst)
  703. if err != nil {
  704. return err
  705. }
  706. buf := make([]byte, BUFFERSIZE)
  707. for {
  708. n, err := source.Read(buf)
  709. if err != nil && err != io.EOF {
  710. source.Close()
  711. destination.Close()
  712. return err
  713. }
  714. if n == 0 {
  715. source.Close()
  716. destination.Close()
  717. break
  718. }
  719. if _, err := destination.Write(buf[:n]); err != nil {
  720. source.Close()
  721. destination.Close()
  722. return err
  723. }
  724. }
  725. return nil
  726. }
  727. */
  728. //Check if a local path is dir, do not use with file system abstraction realpath
  729. func IsDir(path string) bool {
  730. if !FileExists(path) {
  731. return false
  732. }
  733. fi, err := os.Stat(path)
  734. if err != nil {
  735. log.Fatal(err)
  736. return false
  737. }
  738. switch mode := fi.Mode(); {
  739. case mode.IsDir():
  740. return true
  741. case mode.IsRegular():
  742. return false
  743. }
  744. return false
  745. }
  746. //Unzip tar.gz file, use for unpacking web.tar.gz for lazy people
  747. func ExtractTarGzipFile(filename string, outfile string) error {
  748. f, err := os.Open(filename)
  749. if err != nil {
  750. return err
  751. }
  752. err = ExtractTarGzipByStream(filepath.Clean(outfile), f, true)
  753. if err != nil {
  754. return err
  755. }
  756. return f.Close()
  757. }
  758. func ExtractTarGzipByStream(basedir string, gzipStream io.Reader, onErrorResumeNext bool) error {
  759. uncompressedStream, err := gzip.NewReader(gzipStream)
  760. if err != nil {
  761. return err
  762. }
  763. tarReader := tar.NewReader(uncompressedStream)
  764. for {
  765. header, err := tarReader.Next()
  766. if err == io.EOF {
  767. break
  768. }
  769. if err != nil {
  770. return err
  771. }
  772. switch header.Typeflag {
  773. case tar.TypeDir:
  774. err := os.Mkdir(header.Name, 0755)
  775. if err != nil {
  776. if !onErrorResumeNext {
  777. return err
  778. }
  779. }
  780. case tar.TypeReg:
  781. outFile, err := os.Create(filepath.Join(basedir, header.Name))
  782. if err != nil {
  783. if !onErrorResumeNext {
  784. return err
  785. }
  786. }
  787. _, err = io.Copy(outFile, tarReader)
  788. if err != nil {
  789. if !onErrorResumeNext {
  790. return err
  791. }
  792. }
  793. outFile.Close()
  794. default:
  795. //Unknown filetype, continue
  796. }
  797. }
  798. return nil
  799. }