storage.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372
  1. package main
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "os"
  6. "path/filepath"
  7. "runtime"
  8. "strings"
  9. "time"
  10. "imuslab.com/arozos/mod/filesystem"
  11. "imuslab.com/arozos/mod/permission"
  12. fs "imuslab.com/arozos/mod/filesystem"
  13. storage "imuslab.com/arozos/mod/storage"
  14. )
  15. var (
  16. baseStoragePool *storage.StoragePool //base storage pool, all user can access these virtual roots
  17. //fsHandlers []*fs.FileSystemHandler //All File system handlers. All opened handles must be registered in here
  18. //storagePools []*storage.StoragePool //All Storage pool opened
  19. storageHeartbeatTickerChan chan bool //Channel to stop the storage heartbeat ticker
  20. )
  21. func StorageInit() {
  22. //Load the default handler for the user storage root
  23. if !fs.FileExists(filepath.Clean(*root_directory) + "/") {
  24. os.MkdirAll(filepath.Clean(*root_directory)+"/", 0755)
  25. }
  26. //Start loading the base storage pool
  27. err := LoadBaseStoragePool()
  28. if err != nil {
  29. panic(err)
  30. }
  31. }
  32. func LoadBaseStoragePool() error {
  33. //All fsh for the base pool
  34. fsHandlers := []*fs.FileSystemHandler{}
  35. //Use for Debian buster local file system
  36. localFileSystem := "ext4"
  37. if runtime.GOOS == "windows" {
  38. localFileSystem = "ntfs"
  39. }
  40. baseHandler, err := fs.NewFileSystemHandler(fs.FileSystemOption{
  41. Name: "User",
  42. Uuid: "user",
  43. Path: filepath.ToSlash(filepath.Clean(*root_directory)) + "/",
  44. Hierarchy: "user",
  45. Automount: false,
  46. Filesystem: localFileSystem,
  47. }, fs.RuntimePersistenceConfig{
  48. LocalBufferPath: *tmp_directory,
  49. })
  50. if err != nil {
  51. systemWideLogger.PrintAndLog("Storage", "Failed to initiate user root storage directory: "+*root_directory+err.Error(), err)
  52. return err
  53. }
  54. fsHandlers = append(fsHandlers, baseHandler)
  55. //Load the tmp folder as storage unit
  56. tmpHandler, err := fs.NewFileSystemHandler(fs.FileSystemOption{
  57. Name: "tmp",
  58. Uuid: "tmp",
  59. Path: filepath.ToSlash(filepath.Clean(*tmp_directory)) + "/",
  60. Hierarchy: "user",
  61. Automount: false,
  62. Filesystem: localFileSystem,
  63. }, fs.RuntimePersistenceConfig{
  64. LocalBufferPath: *tmp_directory,
  65. })
  66. if err != nil {
  67. systemWideLogger.PrintAndLog("Storage", "Failed to initiate tmp storage directory: "+*tmp_directory+err.Error(), err)
  68. return err
  69. }
  70. fsHandlers = append(fsHandlers, tmpHandler)
  71. //Load all the storage config from file
  72. rawConfig, err := os.ReadFile(*storage_config_file)
  73. if err != nil {
  74. //File not found. Use internal storage only
  75. systemWideLogger.PrintAndLog("Storage", "Storage configuration file not found. Using internal storage only.", err)
  76. } else {
  77. //Configuration loaded. Initializing handler
  78. externalHandlers, err := fs.NewFileSystemHandlersFromJSON(rawConfig, fs.RuntimePersistenceConfig{
  79. LocalBufferPath: *tmp_directory,
  80. })
  81. if err != nil {
  82. systemWideLogger.PrintAndLog("Storage", "Failed to load storage configuration: "+err.Error()+" -- Skipping", err)
  83. } else {
  84. for _, thisHandler := range externalHandlers {
  85. fsHandlers = append(fsHandlers, thisHandler)
  86. systemWideLogger.PrintAndLog("Storage", thisHandler.Name+" Mounted as "+thisHandler.UUID+":/", err)
  87. }
  88. }
  89. }
  90. //Files other users shared with each user, see file_system.share.go
  91. fsHandlers = sharedDriveAttach(fsHandlers)
  92. //Create a base storage pool for all users
  93. sp, err := storage.NewStoragePool(fsHandlers, "system")
  94. if err != nil {
  95. systemWideLogger.PrintAndLog("Storage", "Failed to create base Storaeg Pool", err)
  96. return err
  97. }
  98. baseStoragePool = sp
  99. //A reload lands here too and cluster:/ is not in any config, so put it
  100. //back. During boot the cluster is not up yet and this changes nothing.
  101. clusterSyncDrive()
  102. return nil
  103. }
  104. // Initialize the storage connection health check for all fsh.
  105. func storageHeartbeatTickerInit() {
  106. ticker := time.NewTicker(60 * time.Second)
  107. done := make(chan bool)
  108. go func() {
  109. for {
  110. select {
  111. case <-done:
  112. return
  113. case <-ticker.C:
  114. StoragePerformFileSystemAbstractionConnectionHeartbeat()
  115. }
  116. }
  117. }()
  118. storageHeartbeatTickerChan = done
  119. }
  120. // Perform heartbeat to all connected file system abstraction.
  121. // Blocking function, use with go routine if needed
  122. func StoragePerformFileSystemAbstractionConnectionHeartbeat() {
  123. allFsh := GetAllLoadedFsh()
  124. for _, thisFsh := range allFsh {
  125. if isClusterOrShareDrive(thisFsh) {
  126. //Built by code and followed by its own subsystem. There are no
  127. //start options to reconnect it from
  128. continue
  129. }
  130. err := thisFsh.FileSystemAbstraction.Heartbeat()
  131. if err != nil {
  132. systemWideLogger.PrintAndLog("System", "[Storage] File System Abstraction from "+thisFsh.Name+" report an error: "+err.Error(), nil)
  133. //Retreive the old startup config and close the pool
  134. originalStartOption := filesystem.FileSystemOption{}
  135. js, _ := json.Marshal(thisFsh.StartOptions)
  136. json.Unmarshal(js, &originalStartOption)
  137. //Create a new fsh from original start options
  138. newfsh, err := filesystem.NewFileSystemHandler(originalStartOption, fs.RuntimePersistenceConfig{
  139. LocalBufferPath: *tmp_directory,
  140. })
  141. if err != nil {
  142. systemWideLogger.PrintAndLog("System", "[Storage] Unable to reconnect "+thisFsh.Name+": "+err.Error(), nil)
  143. continue
  144. } else {
  145. //New fsh created. Close the old one
  146. thisFsh.Close()
  147. }
  148. //Pop this fsh from all storage pool that mounted this
  149. sp := GetAllStoragePools()
  150. parentsp := []*storage.StoragePool{}
  151. for _, thissp := range sp {
  152. if thissp.ContainDiskID(originalStartOption.Uuid) {
  153. parentsp = append(parentsp, thissp)
  154. thissp.DetachFsHandler(originalStartOption.Uuid)
  155. }
  156. }
  157. //Add the new fsh to all the storage pools that have it originally
  158. for _, pool := range parentsp {
  159. err := pool.AttachFsHandler(newfsh)
  160. if err != nil {
  161. systemWideLogger.PrintAndLog("System", "[Storage] Attach fsh to pool failed: "+err.Error(), nil)
  162. }
  163. }
  164. }
  165. }
  166. }
  167. // Initialize group storage pool
  168. func GroupStoragePoolInit() {
  169. //Mount permission groups
  170. for _, pg := range permissionHandler.PermissionGroups {
  171. //For each group, check does this group has a config file
  172. err := LoadStoragePoolForGroup(pg)
  173. if err != nil {
  174. continue
  175. }
  176. //Do something else, WIP
  177. }
  178. //Start editing interface for Storage Pool Editor
  179. StoragePoolEditorInit()
  180. }
  181. func LoadStoragePoolForGroup(pg *permission.PermissionGroup) error {
  182. expectedConfigPath := "./system/storage/" + pg.Name + ".json"
  183. if fs.FileExists(expectedConfigPath) {
  184. //Read the config file
  185. pgStorageConfig, err := os.ReadFile(expectedConfigPath)
  186. if err != nil {
  187. systemWideLogger.PrintAndLog("Storage", "Failed to read config for "+pg.Name+": "+err.Error(), err)
  188. return errors.New("Failed to read config for " + pg.Name + ": " + err.Error())
  189. }
  190. //Generate fsHandler form json
  191. thisGroupFsHandlers, err := fs.NewFileSystemHandlersFromJSON(pgStorageConfig, fs.RuntimePersistenceConfig{
  192. LocalBufferPath: *tmp_directory,
  193. })
  194. if err != nil {
  195. systemWideLogger.PrintAndLog("Storage", "Failed to load storage configuration: "+err.Error(), err)
  196. return errors.New("Failed to load storage configuration: " + err.Error())
  197. }
  198. //Show debug message
  199. for _, thisHandler := range thisGroupFsHandlers {
  200. systemWideLogger.PrintAndLog("Storage", thisHandler.Name+" Mounted as "+thisHandler.UUID+":/ for group "+pg.Name, err)
  201. }
  202. //Create a storage pool from these handlers
  203. sp, err := storage.NewStoragePool(thisGroupFsHandlers, pg.Name)
  204. if err != nil {
  205. systemWideLogger.PrintAndLog("Storage", "Failed to create storage pool for "+pg.Name, err)
  206. return errors.New("Failed to create storage pool for " + pg.Name)
  207. }
  208. //Assign storage pool to group
  209. pg.StoragePool = sp
  210. } else {
  211. //Storage configuration not exists. Fill in the basic information and move to next storage pool
  212. //Create a new empty storage pool for this group
  213. sp, err := storage.NewStoragePool([]*fs.FileSystemHandler{}, pg.Name)
  214. if err != nil {
  215. systemWideLogger.PrintAndLog("Storage", "Failed to create empty storage pool for group: "+pg.Name, err)
  216. }
  217. pg.StoragePool = sp
  218. }
  219. return nil
  220. }
  221. // Check if a storage pool exists by its group owner name
  222. func StoragePoolExists(poolOwner string) bool {
  223. _, err := GetStoragePoolByOwner(poolOwner)
  224. return err == nil
  225. }
  226. func GetAllStoragePools() []*storage.StoragePool {
  227. //Append the base pool
  228. results := []*storage.StoragePool{baseStoragePool}
  229. //Add each permissionGroup's pool
  230. for _, pg := range permissionHandler.PermissionGroups {
  231. results = append(results, pg.StoragePool)
  232. }
  233. return results
  234. }
  235. func GetStoragePoolByOwner(owner string) (*storage.StoragePool, error) {
  236. sps := GetAllStoragePools()
  237. for _, pool := range sps {
  238. if pool.Owner == owner {
  239. return pool, nil
  240. }
  241. }
  242. return nil, errors.New("Storage pool owned by " + owner + " not found")
  243. }
  244. func GetFSHandlerSubpathFromVpath(vpath string) (*fs.FileSystemHandler, string, error) {
  245. VirtualRootID, subpath, err := fs.GetIDFromVirtualPath(vpath)
  246. if err != nil {
  247. return nil, "", errors.New("Unable to resolve requested path: " + err.Error())
  248. }
  249. fsh, err := GetFsHandlerByUUID(VirtualRootID)
  250. if err != nil {
  251. return nil, "", errors.New("Unable to resolve requested path: " + err.Error())
  252. }
  253. if fsh == nil || fsh.FileSystemAbstraction == nil {
  254. //err is nil on this branch, do not dereference it for a message
  255. return nil, "", errors.New("Unable to resolve requested path: file system handler not loaded")
  256. }
  257. if fsh.Closed {
  258. return nil, "", errors.New("Target file system handler already closed")
  259. }
  260. return fsh, subpath, nil
  261. }
  262. func GetFsHandlerByUUID(uuid string) (*fs.FileSystemHandler, error) {
  263. //Filter out the :/ fropm uuid if exists
  264. if strings.Contains(uuid, ":") {
  265. uuid = strings.Split(uuid, ":")[0]
  266. }
  267. var resultFsh *fs.FileSystemHandler = nil
  268. allFsh := GetAllLoadedFsh()
  269. for _, fsh := range allFsh {
  270. if fsh.UUID == uuid && !fsh.Closed {
  271. resultFsh = fsh
  272. }
  273. }
  274. if resultFsh == nil {
  275. return nil, errors.New("Filesystem handler with given UUID not found")
  276. } else {
  277. return resultFsh, nil
  278. }
  279. }
  280. func GetAllLoadedFsh() []*fs.FileSystemHandler {
  281. fshTmp := map[string]*fs.FileSystemHandler{}
  282. allFsh := []*fs.FileSystemHandler{}
  283. allStoragePools := GetAllStoragePools()
  284. for _, thisSP := range allStoragePools {
  285. for _, thisFsh := range thisSP.Storages {
  286. fshPointer := thisFsh
  287. fshTmp[thisFsh.UUID] = fshPointer
  288. }
  289. }
  290. //Restructure the map to slice
  291. for _, fsh := range fshTmp {
  292. allFsh = append(allFsh, fsh)
  293. }
  294. return allFsh
  295. }
  296. func RegisterStorageSettings() {
  297. //Storage Pool Configuration
  298. registerSetting(settingModule{
  299. Name: "Storage Pools",
  300. Desc: "Storage Pool Mounting Configuration",
  301. IconPath: "SystemAO/disk/smart/img/small_icon.png",
  302. Group: "Disk",
  303. StartDir: "SystemAO/storage/poolList.html",
  304. RequireAdmin: true,
  305. })
  306. }
  307. // CloseAllStorages Close all storage database
  308. func CloseAllStorages() {
  309. }
  310. func closeAllStoragePools() {
  311. //Stop the storage pool heartbeat
  312. storageHeartbeatTickerChan <- true
  313. //Close all storage pools
  314. for _, sp := range GetAllStoragePools() {
  315. sp.Close()
  316. }
  317. }