cluster.fsinfo.go 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. package main
  2. /*
  3. Where a cluster:/ file physically lives.
  4. The cluster drive presents one namespace, but every file in it sits on
  5. one or more real volumes on real nodes, and which those are changes over
  6. time as the replication planner works. Nothing in an ordinary file
  7. listing shows that, so this file answers it: given a namespace path, it
  8. reads the metadata record and turns the record, the volumes it points at
  9. and the state of the nodes holding them into the generic shape the File
  10. Manager properties dialog renders (mod/filesystem/arozfs/storageinfo.go).
  11. It is a read of the local replica of the metadata store, so it costs
  12. nothing and works even while some of the nodes involved are offline.
  13. */
  14. import (
  15. "errors"
  16. "path"
  17. "strconv"
  18. "strings"
  19. "time"
  20. "imuslab.com/arozos/mod/cluster/membership"
  21. "imuslab.com/arozos/mod/cluster/metadata"
  22. fs "imuslab.com/arozos/mod/filesystem"
  23. "imuslab.com/arozos/mod/filesystem/arozfs"
  24. )
  25. // Copy states in the words the dialog shows, and the colour each one gets
  26. var clusterCopyStatus = map[string][2]string{
  27. metadata.LocVerified: {"Verified", arozfs.StorageStateOK},
  28. metadata.LocCommitted: {"Committed", arozfs.StorageStateOK},
  29. metadata.LocPending: {"Pending", arozfs.StorageStatePending},
  30. metadata.LocWriting: {"Writing", arozfs.StorageStatePending},
  31. metadata.LocStale: {"Stale", arozfs.StorageStateStale},
  32. metadata.LocFailed: {"Failed", arozfs.StorageStateError},
  33. }
  34. // Node states in the words the dialog shows
  35. var clusterNodeStateWords = map[membership.NodeState]string{
  36. membership.StateOnline: "Online",
  37. membership.StateDegraded: "Degraded",
  38. membership.StateOffline: "Offline",
  39. membership.StateDraining: "Draining",
  40. membership.StateMaintenance: "Maintenance",
  41. membership.StateUnknown: "Unknown",
  42. }
  43. // StorageInfo resolves one namespace path into the copies that hold it. It
  44. // completes the clusterfs.InfoBackend interface.
  45. func (b *clusterBackend) StorageInfo(logical string) (arozfs.StorageInfo, error) {
  46. if clusterManager == nil || clusterMetadata == nil || !clusterManager.InCluster() {
  47. return arozfs.StorageInfo{}, errors.New("this node is not in a cluster")
  48. }
  49. rec, err := clusterMetadata.Stat(logical)
  50. if err != nil {
  51. return arozfs.StorageInfo{}, err
  52. }
  53. info := arozfs.StorageInfo{
  54. Type: "cluster",
  55. Title: "Cluster",
  56. }
  57. //Properties of the file itself
  58. clusterName := ""
  59. if c := clusterManager.Cluster(); c != nil {
  60. clusterName = c.Name
  61. }
  62. policy := clusterMetadata.PolicyFor(rec.Path)
  63. info.Fields = append(info.Fields,
  64. arozfs.StorageInfoField{Key: "Cluster", Value: clusterName},
  65. arozfs.StorageInfoField{Key: "Namespace Path", Value: rec.Path},
  66. )
  67. if rec.IsDir {
  68. //A folder holds no copies of its own: what it decides is how many
  69. //copies the files created inside it are kept at
  70. info.Fields = append(info.Fields,
  71. clusterReplicaPolicyField(policy),
  72. arozfs.StorageInfoField{Key: "Items Inside", Value: clusterFolderItemCount(rec.Path)},
  73. )
  74. return info, nil
  75. }
  76. healthy := len(rec.HealthyLocations())
  77. info.Fields = append(info.Fields,
  78. clusterReplicaPolicyField(policy),
  79. clusterCopyCountField(healthy, len(rec.Locations)),
  80. arozfs.StorageInfoField{Key: "Checksum", Value: clusterChecksumText(rec.Checksum)},
  81. arozfs.StorageInfoField{Key: "File ID", Value: rec.ID},
  82. )
  83. if master := clusterMetadata.Leader(); master != "" {
  84. info.Fields = append(info.Fields,
  85. arozfs.StorageInfoField{Key: "Master Node", Value: clusterNodeLabelWithID(master)})
  86. }
  87. //One entry per physical copy, this node first so the reader sees at a
  88. //glance whether the file is on the host they are logged in to
  89. localNode := clusterManager.NodeID()
  90. for _, loc := range clusterOrderedLocations(rec, localNode) {
  91. info.Items = append(info.Items, clusterCopyItem(rec, loc, localNode))
  92. }
  93. return info, nil
  94. }
  95. // clusterOrderedLocations lists the copies of a file with this node first,
  96. // then the readable ones, keeping the record order within each group.
  97. func clusterOrderedLocations(rec *metadata.FileRecord, localNode string) []metadata.Location {
  98. ordered := []metadata.Location{}
  99. for pass := 0; pass < 3; pass++ {
  100. for _, loc := range rec.Locations {
  101. isLocal := loc.NodeID == localNode
  102. switch pass {
  103. case 0:
  104. if !isLocal {
  105. continue
  106. }
  107. case 1:
  108. if isLocal || !loc.Healthy() {
  109. continue
  110. }
  111. default:
  112. if isLocal || loc.Healthy() {
  113. continue
  114. }
  115. }
  116. ordered = append(ordered, loc)
  117. }
  118. }
  119. return ordered
  120. }
  121. // clusterCopyItem describes one physical copy of a file.
  122. func clusterCopyItem(rec *metadata.FileRecord, loc metadata.Location, localNode string) arozfs.StorageInfoItem {
  123. status, state := "Unknown", arozfs.StorageStateUnknown
  124. if words, ok := clusterCopyStatus[loc.State]; ok {
  125. status, state = words[0], words[1]
  126. }
  127. item := arozfs.StorageInfoItem{
  128. Title: clusterNodeLabel(loc.NodeID),
  129. State: state,
  130. Status: status,
  131. Local: loc.NodeID == localNode,
  132. Fields: []arozfs.StorageInfoField{{Key: "Node ID", Value: loc.NodeID}},
  133. }
  134. //Node health, as an offline node explains a copy that cannot be read
  135. if nodeState, found := clusterNodeState(loc.NodeID); found {
  136. words, ok := clusterNodeStateWords[nodeState]
  137. if !ok {
  138. words = string(nodeState)
  139. }
  140. item.Fields = append(item.Fields, arozfs.StorageInfoField{Key: "Node State", Value: words})
  141. if state == arozfs.StorageStateOK && nodeState != membership.StateOnline &&
  142. nodeState != membership.StateDegraded {
  143. //The copy itself is fine, the node holding it is not
  144. item.State = arozfs.StorageStateStale
  145. }
  146. } else {
  147. item.Fields = append(item.Fields,
  148. arozfs.StorageInfoField{Key: "Node State", Value: "Not a member"})
  149. item.State = arozfs.StorageStateStale
  150. }
  151. //The volume it landed on, and where that volume sits on the holding node
  152. if vol, ok := clusterMetadata.Volume(loc.VolumeID); ok {
  153. item.Subtitle = clusterVolumeRootText(*vol)
  154. item.Fields = append(item.Fields,
  155. arozfs.StorageInfoField{Key: "Volume", Value: vol.Name},
  156. arozfs.StorageInfoField{Key: "Volume ID", Value: vol.ID},
  157. arozfs.StorageInfoField{Key: "Volume Path", Value: clusterVolumePathText(*vol, rec.Path)},
  158. )
  159. if vol.Capacity > 0 {
  160. item.Fields = append(item.Fields, arozfs.StorageInfoField{Key: "Volume Free",
  161. Value: "{0} of {1}",
  162. Args: []string{fs.GetFileDisplaySize(vol.Free, 2), fs.GetFileDisplaySize(vol.Capacity, 2)}})
  163. }
  164. if vol.Evacuating {
  165. item.Fields = append(item.Fields,
  166. arozfs.StorageInfoField{Key: "Volume Access", Value: "Evacuating"})
  167. } else if vol.ReadOnly || vol.LowSpace {
  168. item.Fields = append(item.Fields,
  169. arozfs.StorageInfoField{Key: "Volume Access", Value: "Read only"})
  170. }
  171. } else {
  172. //The volume record is gone, so name the copy by the ids it still has
  173. item.Subtitle = loc.NodeID + ":" + loc.VolumeID
  174. item.Fields = append(item.Fields,
  175. arozfs.StorageInfoField{Key: "Volume ID", Value: loc.VolumeID})
  176. }
  177. if loc.VolumeID == rec.Primary {
  178. item.Fields = append(item.Fields, arozfs.StorageInfoField{Key: "Primary Copy", Value: "Yes"})
  179. }
  180. //A copy whose hash no longer matches the file is worth seeing
  181. if loc.Checksum != "" && rec.Checksum != "" && loc.Checksum != rec.Checksum {
  182. item.Fields = append(item.Fields,
  183. arozfs.StorageInfoField{Key: "Checksum", Value: clusterChecksumText(loc.Checksum)})
  184. }
  185. if loc.Updated > 0 {
  186. item.Fields = append(item.Fields, arozfs.StorageInfoField{Key: "Last Checked",
  187. Value: time.Unix(loc.Updated, 0).Format("2006-01-02 15:04:05")})
  188. }
  189. return item
  190. }
  191. // clusterNodeState returns the computed state of a node, and whether the node
  192. // is still a member at all.
  193. func clusterNodeState(nodeID string) (membership.NodeState, bool) {
  194. for _, n := range clusterManager.NodeViews() {
  195. if n.ID == nodeID {
  196. return n.State, true
  197. }
  198. }
  199. return membership.StateUnknown, false
  200. }
  201. // clusterNodeLabel names a node, falling back to its ID when it has no name.
  202. func clusterNodeLabel(nodeID string) string {
  203. name := clusterManager.NodeName(nodeID)
  204. if strings.TrimSpace(name) == "" {
  205. return nodeID
  206. }
  207. return name
  208. }
  209. // clusterNodeLabelWithID names a node and states its ID. Node names are
  210. // operator set and default to the same "My ArOZ" on every host, so a name on
  211. // its own cannot say which node is meant.
  212. func clusterNodeLabelWithID(nodeID string) string {
  213. name := clusterManager.NodeName(nodeID)
  214. if strings.TrimSpace(name) == "" || name == nodeID {
  215. return nodeID
  216. }
  217. return name + " (" + nodeID + ")"
  218. }
  219. /*
  220. Paths across the cluster
  221. A path is only an answer if it says which node it is on, and the only
  222. thing that identifies a node is its UUID: names repeat, and the default
  223. name is the same on every fresh install. So every path this report shows
  224. is written <node uuid>:<drive uuid>/<path on that drive>, e.g.
  225. 3f7a...c1:user/cluster/photos/a.jpg
  226. which reads as: on node 3f7a…c1, drive user, at /cluster/photos/a.jpg.
  227. */
  228. // clusterVolumeRootText is the folder a volume contributes, node included.
  229. func clusterVolumeRootText(vol metadata.Volume) string {
  230. return vol.NodeID + ":" + vol.FshUUID + path.Join("/", vol.Subpath)
  231. }
  232. // clusterVolumePathText is where one copy of a file sits, node included.
  233. func clusterVolumePathText(vol metadata.Volume, logical string) string {
  234. return vol.NodeID + ":" + vol.FshUUID + path.Join("/", vol.Subpath, logical)
  235. }
  236. // clusterReplicaPolicyField reports how many copies of each file the folder
  237. // policy asks for.
  238. func clusterReplicaPolicyField(policy metadata.Policy) arozfs.StorageInfoField {
  239. if policy.Replicas <= 1 {
  240. return arozfs.StorageInfoField{Key: "Replica Policy", Value: "{0} copy",
  241. Args: []string{"1"}}
  242. }
  243. return arozfs.StorageInfoField{Key: "Replica Policy", Value: "{0} copies",
  244. Args: []string{strconv.Itoa(policy.Replicas)}}
  245. }
  246. // clusterCopyCountField reports how many copies of a file can be read now.
  247. func clusterCopyCountField(healthy int, total int) arozfs.StorageInfoField {
  248. if healthy == total {
  249. return arozfs.StorageInfoField{Key: "Copies", Value: "{0}",
  250. Args: []string{strconv.Itoa(total)}}
  251. }
  252. return arozfs.StorageInfoField{Key: "Copies", Value: "{0} of {1} readable",
  253. Args: []string{strconv.Itoa(healthy), strconv.Itoa(total)}}
  254. }
  255. // clusterChecksumText shortens a hash to something a dialog row can hold.
  256. func clusterChecksumText(checksum string) string {
  257. if checksum == "" {
  258. return ""
  259. }
  260. if len(checksum) > 24 {
  261. return checksum[:24] + "..."
  262. }
  263. return checksum
  264. }
  265. // clusterFolderItemCount counts the records directly inside a folder.
  266. func clusterFolderItemCount(folder string) string {
  267. entries, err := clusterMetadata.ListDir(folder)
  268. if err != nil {
  269. return "0"
  270. }
  271. return strconv.Itoa(len(entries))
  272. }