storage.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. package main
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "log"
  6. "os"
  7. "path/filepath"
  8. "runtime"
  9. "strings"
  10. "time"
  11. "imuslab.com/arozos/mod/filesystem"
  12. "imuslab.com/arozos/mod/filesystem/arozfs"
  13. "imuslab.com/arozos/mod/permission"
  14. "imuslab.com/arozos/mod/storage/bridge"
  15. fs "imuslab.com/arozos/mod/filesystem"
  16. storage "imuslab.com/arozos/mod/storage"
  17. )
  18. var (
  19. baseStoragePool *storage.StoragePool //base storage pool, all user can access these virtual roots
  20. //fsHandlers []*fs.FileSystemHandler //All File system handlers. All opened handles must be registered in here
  21. //storagePools []*storage.StoragePool //All Storage pool opened
  22. bridgeManager *bridge.Record //Manager to handle bridged FSH
  23. storageHeartbeatTickerChan chan bool //Channel to stop the storage heartbeat ticker
  24. )
  25. func StorageInit() {
  26. //Load the default handler for the user storage root
  27. if !fs.FileExists(filepath.Clean(*root_directory) + "/") {
  28. os.MkdirAll(filepath.Clean(*root_directory)+"/", 0755)
  29. }
  30. //Start loading the base storage pool
  31. err := LoadBaseStoragePool()
  32. if err != nil {
  33. panic(err)
  34. }
  35. //Create a brdige record manager
  36. bm := bridge.NewBridgeRecord("system/bridge.json")
  37. bridgeManager = bm
  38. }
  39. func LoadBaseStoragePool() error {
  40. //All fsh for the base pool
  41. fsHandlers := []*fs.FileSystemHandler{}
  42. //Use for Debian buster local file system
  43. localFileSystem := "ext4"
  44. if runtime.GOOS == "windows" {
  45. localFileSystem = "ntfs"
  46. }
  47. baseHandler, err := fs.NewFileSystemHandler(fs.FileSystemOption{
  48. Name: "User",
  49. Uuid: "user",
  50. Path: filepath.ToSlash(filepath.Clean(*root_directory)) + "/",
  51. Hierarchy: "user",
  52. Automount: false,
  53. Filesystem: localFileSystem,
  54. }, fs.RuntimePersistenceConfig{
  55. LocalBufferPath: *tmp_directory,
  56. })
  57. if err != nil {
  58. systemWideLogger.PrintAndLog("Storage", "Failed to initiate user root storage directory: "+*root_directory+err.Error(), err)
  59. return err
  60. }
  61. fsHandlers = append(fsHandlers, baseHandler)
  62. //Load the tmp folder as storage unit
  63. tmpHandler, err := fs.NewFileSystemHandler(fs.FileSystemOption{
  64. Name: "tmp",
  65. Uuid: "tmp",
  66. Path: filepath.ToSlash(filepath.Clean(*tmp_directory)) + "/",
  67. Hierarchy: "user",
  68. Automount: false,
  69. Filesystem: localFileSystem,
  70. }, fs.RuntimePersistenceConfig{
  71. LocalBufferPath: *tmp_directory,
  72. })
  73. if err != nil {
  74. systemWideLogger.PrintAndLog("Storage", "Failed to initiate tmp storage directory: "+*tmp_directory+err.Error(), err)
  75. return err
  76. }
  77. fsHandlers = append(fsHandlers, tmpHandler)
  78. //Load all the storage config from file
  79. rawConfig, err := os.ReadFile(*storage_config_file)
  80. if err != nil {
  81. //File not found. Use internal storage only
  82. systemWideLogger.PrintAndLog("Storage", "Storage configuration file not found. Using internal storage only.", err)
  83. } else {
  84. //Configuration loaded. Initializing handler
  85. externalHandlers, err := fs.NewFileSystemHandlersFromJSON(rawConfig, fs.RuntimePersistenceConfig{
  86. LocalBufferPath: *tmp_directory,
  87. })
  88. if err != nil {
  89. systemWideLogger.PrintAndLog("Storage", "Failed to load storage configuration: "+err.Error()+" -- Skipping", err)
  90. } else {
  91. for _, thisHandler := range externalHandlers {
  92. fsHandlers = append(fsHandlers, thisHandler)
  93. systemWideLogger.PrintAndLog("Storage", thisHandler.Name+" Mounted as "+thisHandler.UUID+":/", err)
  94. }
  95. }
  96. }
  97. //Create a base storage pool for all users
  98. sp, err := storage.NewStoragePool(fsHandlers, "system")
  99. if err != nil {
  100. systemWideLogger.PrintAndLog("Storage", "Failed to create base Storaeg Pool", err)
  101. return err
  102. }
  103. //Update the storage pool permission to readwrite
  104. sp.OtherPermission = arozfs.FsReadWrite
  105. baseStoragePool = sp
  106. return nil
  107. }
  108. // Initialize the storage connection health check for all fsh.
  109. func storageHeartbeatTickerInit() {
  110. ticker := time.NewTicker(60 * time.Second)
  111. done := make(chan bool)
  112. go func() {
  113. for {
  114. select {
  115. case <-done:
  116. return
  117. case <-ticker.C:
  118. StoragePerformFileSystemAbstractionConnectionHeartbeat()
  119. }
  120. }
  121. }()
  122. storageHeartbeatTickerChan = done
  123. }
  124. // Perform heartbeat to all connected file system abstraction.
  125. // Blocking function, use with go routine if needed
  126. func StoragePerformFileSystemAbstractionConnectionHeartbeat() {
  127. allFsh := GetAllLoadedFsh()
  128. for _, thisFsh := range allFsh {
  129. err := thisFsh.FileSystemAbstraction.Heartbeat()
  130. if err != nil {
  131. log.Println("[Storage] File System Abstraction from " + thisFsh.Name + " report an error: " + err.Error())
  132. //Retreive the old startup config and close the pool
  133. originalStartOption := filesystem.FileSystemOption{}
  134. js, _ := json.Marshal(thisFsh.StartOptions)
  135. json.Unmarshal(js, &originalStartOption)
  136. //Create a new fsh from original start options
  137. newfsh, err := filesystem.NewFileSystemHandler(originalStartOption, fs.RuntimePersistenceConfig{
  138. LocalBufferPath: *tmp_directory,
  139. })
  140. if err != nil {
  141. log.Println("[Storage] Unable to reconnect " + thisFsh.Name + ": " + err.Error())
  142. continue
  143. } else {
  144. //New fsh created. Close the old one
  145. thisFsh.Close()
  146. }
  147. //Pop this fsh from all storage pool that mounted this
  148. sp := GetAllStoragePools()
  149. parentsp := []*storage.StoragePool{}
  150. for _, thissp := range sp {
  151. if thissp.ContainDiskID(originalStartOption.Uuid) {
  152. parentsp = append(parentsp, thissp)
  153. thissp.DetachFsHandler(originalStartOption.Uuid)
  154. }
  155. }
  156. //Add the new fsh to all the storage pools that have it originally
  157. for _, pool := range parentsp {
  158. err := pool.AttachFsHandler(newfsh)
  159. if err != nil {
  160. log.Println("[Storage] Attach fsh to pool failed: " + err.Error())
  161. }
  162. }
  163. }
  164. }
  165. }
  166. // Initialize group storage pool
  167. func GroupStoragePoolInit() {
  168. //Mount permission groups
  169. for _, pg := range permissionHandler.PermissionGroups {
  170. //For each group, check does this group has a config file
  171. err := LoadStoragePoolForGroup(pg)
  172. if err != nil {
  173. continue
  174. }
  175. //Do something else, WIP
  176. }
  177. //Start editing interface for Storage Pool Editor
  178. StoragePoolEditorInit()
  179. }
  180. func LoadStoragePoolForGroup(pg *permission.PermissionGroup) error {
  181. expectedConfigPath := "./system/storage/" + pg.Name + ".json"
  182. if fs.FileExists(expectedConfigPath) {
  183. //Read the config file
  184. pgStorageConfig, err := os.ReadFile(expectedConfigPath)
  185. if err != nil {
  186. systemWideLogger.PrintAndLog("Storage", "Failed to read config for "+pg.Name+": "+err.Error(), err)
  187. return errors.New("Failed to read config for " + pg.Name + ": " + err.Error())
  188. }
  189. //Generate fsHandler form json
  190. thisGroupFsHandlers, err := fs.NewFileSystemHandlersFromJSON(pgStorageConfig, fs.RuntimePersistenceConfig{
  191. LocalBufferPath: *tmp_directory,
  192. })
  193. if err != nil {
  194. systemWideLogger.PrintAndLog("Storage", "Failed to load storage configuration: "+err.Error(), err)
  195. return errors.New("Failed to load storage configuration: " + err.Error())
  196. }
  197. //Show debug message
  198. for _, thisHandler := range thisGroupFsHandlers {
  199. systemWideLogger.PrintAndLog("Storage", thisHandler.Name+" Mounted as "+thisHandler.UUID+":/ for group "+pg.Name, err)
  200. }
  201. //Create a storage pool from these handlers
  202. sp, err := storage.NewStoragePool(thisGroupFsHandlers, pg.Name)
  203. if err != nil {
  204. systemWideLogger.PrintAndLog("Storage", "Failed to create storage pool for "+pg.Name, err)
  205. return errors.New("Failed to create storage pool for " + pg.Name)
  206. }
  207. //Set other permission to denied by default
  208. sp.OtherPermission = arozfs.FsDenied
  209. //Assign storage pool to group
  210. pg.StoragePool = sp
  211. } else {
  212. //Storage configuration not exists. Fill in the basic information and move to next storage pool
  213. //Create a new empty storage pool for this group
  214. sp, err := storage.NewStoragePool([]*fs.FileSystemHandler{}, pg.Name)
  215. if err != nil {
  216. systemWideLogger.PrintAndLog("Storage", "Failed to create empty storage pool for group: "+pg.Name, err)
  217. }
  218. pg.StoragePool = sp
  219. pg.StoragePool.OtherPermission = arozfs.FsDenied
  220. }
  221. return nil
  222. }
  223. // Check if a storage pool exists by its group owner name
  224. func StoragePoolExists(poolOwner string) bool {
  225. _, err := GetStoragePoolByOwner(poolOwner)
  226. return err == nil
  227. }
  228. func GetAllStoragePools() []*storage.StoragePool {
  229. //Append the base pool
  230. results := []*storage.StoragePool{baseStoragePool}
  231. //Add each permissionGroup's pool
  232. for _, pg := range permissionHandler.PermissionGroups {
  233. results = append(results, pg.StoragePool)
  234. }
  235. return results
  236. }
  237. func GetStoragePoolByOwner(owner string) (*storage.StoragePool, error) {
  238. sps := GetAllStoragePools()
  239. for _, pool := range sps {
  240. if pool.Owner == owner {
  241. return pool, nil
  242. }
  243. }
  244. return nil, errors.New("Storage pool owned by " + owner + " not found")
  245. }
  246. func GetFSHandlerSubpathFromVpath(vpath string) (*fs.FileSystemHandler, string, error) {
  247. VirtualRootID, subpath, err := fs.GetIDFromVirtualPath(vpath)
  248. if err != nil {
  249. return nil, "", errors.New("Unable to resolve requested path: " + err.Error())
  250. }
  251. fsh, err := GetFsHandlerByUUID(VirtualRootID)
  252. if err != nil {
  253. return nil, "", errors.New("Unable to resolve requested path: " + err.Error())
  254. }
  255. if fsh == nil || fsh.FileSystemAbstraction == nil {
  256. return nil, "", errors.New("Unable to resolve requested path: " + err.Error())
  257. }
  258. if fsh.Closed {
  259. return nil, "", errors.New("Target file system handler already closed")
  260. }
  261. return fsh, subpath, nil
  262. }
  263. func GetFsHandlerByUUID(uuid string) (*fs.FileSystemHandler, error) {
  264. //Filter out the :/ fropm uuid if exists
  265. if strings.Contains(uuid, ":") {
  266. uuid = strings.Split(uuid, ":")[0]
  267. }
  268. var resultFsh *fs.FileSystemHandler = nil
  269. allFsh := GetAllLoadedFsh()
  270. for _, fsh := range allFsh {
  271. if fsh.UUID == uuid && !fsh.Closed {
  272. resultFsh = fsh
  273. }
  274. }
  275. if resultFsh == nil {
  276. return nil, errors.New("Filesystem handler with given UUID not found")
  277. } else {
  278. return resultFsh, nil
  279. }
  280. }
  281. func GetAllLoadedFsh() []*fs.FileSystemHandler {
  282. fshTmp := map[string]*fs.FileSystemHandler{}
  283. allFsh := []*fs.FileSystemHandler{}
  284. allStoragePools := GetAllStoragePools()
  285. for _, thisSP := range allStoragePools {
  286. for _, thisFsh := range thisSP.Storages {
  287. fshPointer := thisFsh
  288. fshTmp[thisFsh.UUID] = fshPointer
  289. }
  290. }
  291. //Restructure the map to slice
  292. for _, fsh := range fshTmp {
  293. allFsh = append(allFsh, fsh)
  294. }
  295. return allFsh
  296. }
  297. func RegisterStorageSettings() {
  298. //Storage Pool Configuration
  299. registerSetting(settingModule{
  300. Name: "Storage Pools",
  301. Desc: "Storage Pool Mounting Configuration",
  302. IconPath: "SystemAO/disk/smart/img/small_icon.png",
  303. Group: "Disk",
  304. StartDir: "SystemAO/storage/poolList.html",
  305. RequireAdmin: true,
  306. })
  307. }
  308. // CloseAllStorages Close all storage database
  309. func CloseAllStorages() {
  310. allFsh := GetAllLoadedFsh()
  311. for _, fsh := range allFsh {
  312. fsh.FilesystemDatabase.Close()
  313. }
  314. }
  315. func closeAllStoragePools() {
  316. //Stop the storage pool heartbeat
  317. storageHeartbeatTickerChan <- true
  318. //Close all storage pools
  319. for _, sp := range GetAllStoragePools() {
  320. sp.Close()
  321. }
  322. }