packed.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455
  1. package office
  2. /*
  3. packed.go - the asset store behind the Office suite's documents.
  4. The editors keep media out of their JSON body as media?file= links into
  5. the host's file system (or, for small pieces, as data URLs). Anything
  6. that has to hold a document by itself - a session snapshot, or the
  7. editor copy embedded in every .docx / .xlsx / .pptx the suite writes
  8. (native.go) - takes those out into binary assets:
  9. document.json the JSON envelope; every media value is replaced
  10. by an "asset://<name>" reference
  11. assets/<name> binary media, deduplicated by content hash
  12. A link is taken out wherever it sits: a whole JSON value (a Slides /
  13. Sheets image src) or inside an HTML string (a Docs body's <img src="...">
  14. or <video poster="...">), where the attribute value becomes the asset
  15. ref. The unpackers resolve "asset://<name>" in both positions the same
  16. way.
  17. PackEnvelope / UnpackEnvelope(ToLinks) write and read that pair as a zip
  18. of its own, which is what the "Restore from previous session" snapshots
  19. (user:/.appdata/Office/session/<app>.osession) are.
  20. */
  21. import (
  22. "archive/zip"
  23. "bytes"
  24. "crypto/sha1"
  25. "encoding/hex"
  26. "encoding/json"
  27. "errors"
  28. "html"
  29. "io"
  30. "path"
  31. "regexp"
  32. "sort"
  33. "strings"
  34. "imuslab.com/arozos/mod/office/internal/media"
  35. )
  36. const packedDocName = "document.json"
  37. // htmlMediaAttrRe matches a src or poster attribute in an HTML fragment -
  38. // the places a Docs body keeps its pictures. href is deliberately left out:
  39. // a hyperlink to a file is a link, not content to copy into the document.
  40. var htmlMediaAttrRe = regexp.MustCompile(`(?i)(\s(?:src|poster)\s*=\s*)("[^"]*"|'[^']*')`)
  41. // embedHTMLMediaLinks rewrites every media?file= link held in a src/poster
  42. // attribute of an HTML string through embed, which returns the asset name
  43. // (and false to leave that link as it is).
  44. func embedHTMLMediaLinks(s string, embed func(vpath string) (string, bool)) string {
  45. if !strings.Contains(s, "media") {
  46. return s
  47. }
  48. return htmlMediaAttrRe.ReplaceAllStringFunc(s, func(m string) string {
  49. sub := htmlMediaAttrRe.FindStringSubmatch(m)
  50. quoted := sub[2]
  51. quote := quoted[:1]
  52. vp := media.LinkVpath(html.UnescapeString(quoted[1 : len(quoted)-1]))
  53. if vp == "" {
  54. return m
  55. }
  56. name, ok := embed(vp)
  57. if !ok {
  58. return m
  59. }
  60. return sub[1] + quote + "asset://" + name + quote
  61. })
  62. }
  63. // htmlDataAttrRe matches the attributes a Docs body may hold a picture in as
  64. // a data URL: src and poster, and the PNG a writer prefers
  65. // (data-export-src, for a picture Word cannot hold)
  66. var htmlDataAttrRe = regexp.MustCompile(`(?i)(\s(?:src|poster|data-export-src)\s*=\s*)("data:[^"]*"|'data:[^']*')`)
  67. // embedHTMLDataURLs moves every data URL held in such an attribute of an
  68. // HTML string out through add, which stores the bytes and names them. A
  69. // Docs body keeps an inline picture there, and left in place it would be
  70. // stored twice in the file: once as base64 in the embedded copy and once as
  71. // the OOXML media part.
  72. func embedHTMLDataURLs(s string, add func(data []byte, ext string) string) string {
  73. if !strings.Contains(s, "data:") {
  74. return s
  75. }
  76. return htmlDataAttrRe.ReplaceAllStringFunc(s, func(m string) string {
  77. sub := htmlDataAttrRe.FindStringSubmatch(m)
  78. quoted := sub[2]
  79. quote := quoted[:1]
  80. data, ext, ok := media.ParseAnyDataURL(html.UnescapeString(quoted[1 : len(quoted)-1]))
  81. if !ok {
  82. return m
  83. }
  84. return sub[1] + quote + "asset://" + add(data, ext) + quote
  85. })
  86. }
  87. /*
  88. InlineMediaLinks turns the picture links in a body (media?file=, as a
  89. whole value or in an <img src>) into data URLs, read through readVpath.
  90. The OpenDocument writers only take inline pictures; doing this on the
  91. server is what spares the browser downloading every picture and sending it
  92. back as base64. Links that are not a png / jpeg / gif, or cannot be read,
  93. stay as they are.
  94. */
  95. func InlineMediaLinks(body string, readVpath func(string) ([]byte, error)) (string, error) {
  96. if readVpath == nil || !strings.Contains(body, "media") {
  97. return body, nil
  98. }
  99. var root interface{}
  100. if err := json.Unmarshal([]byte(body), &root); err != nil {
  101. return "", errors.New("invalid document JSON: " + err.Error())
  102. }
  103. done := map[string]string{}
  104. inline := func(vp string) (string, bool) {
  105. if d, ok := done[vp]; ok {
  106. return d, d != ""
  107. }
  108. data, err := readVpath(vp)
  109. ext := ""
  110. if err == nil {
  111. ext = media.SniffImageExt(data)
  112. }
  113. if ext == "" {
  114. done[vp] = ""
  115. return "", false
  116. }
  117. d := media.EncodeDataURL(data, ext)
  118. done[vp] = d
  119. return d, true
  120. }
  121. root = transformStrings(root, func(s string) string {
  122. if vp := media.LinkVpath(s); vp != "" {
  123. if d, ok := inline(vp); ok {
  124. return d
  125. }
  126. return s
  127. }
  128. if !strings.Contains(s, "media") {
  129. return s
  130. }
  131. return htmlMediaAttrRe.ReplaceAllStringFunc(s, func(m string) string {
  132. sub := htmlMediaAttrRe.FindStringSubmatch(m)
  133. quoted := sub[2]
  134. vp := media.LinkVpath(html.UnescapeString(quoted[1 : len(quoted)-1]))
  135. if vp == "" {
  136. return m
  137. }
  138. d, ok := inline(vp)
  139. if !ok {
  140. return m
  141. }
  142. return sub[1] + quoted[:1] + d + quoted[:1]
  143. })
  144. })
  145. out, err := marshalNoEscape(root)
  146. if err != nil {
  147. return "", err
  148. }
  149. return string(out), nil
  150. }
  151. // assetRefRe matches an asset reference inside a larger string. Asset names
  152. // are written by the packers as <hash>.<ext>, so this character set covers
  153. // every name either of them produces.
  154. var assetRefRe = regexp.MustCompile(`asset://([A-Za-z0-9._-]+)`)
  155. // resolveAssetRefs replaces asset references in s through resolve: the
  156. // whole string when it is one reference (the original form, which accepts
  157. // any name), and every reference embedded in it otherwise.
  158. func resolveAssetRefs(s string, resolve func(name string) (string, bool)) string {
  159. if !strings.Contains(s, "asset://") {
  160. return s
  161. }
  162. if strings.HasPrefix(s, "asset://") {
  163. if out, ok := resolve(strings.TrimPrefix(s, "asset://")); ok {
  164. return out
  165. }
  166. }
  167. return assetRefRe.ReplaceAllStringFunc(s, func(m string) string {
  168. if out, ok := resolve(strings.TrimPrefix(m, "asset://")); ok {
  169. return out
  170. }
  171. return m
  172. })
  173. }
  174. // assetExt is the extension an embedded file is stored under: the source
  175. // path's own, lower-cased, when it is a plain one, and "bin" otherwise.
  176. func assetExt(vpath string) string {
  177. ext := strings.TrimPrefix(strings.ToLower(path.Ext(vpath)), ".")
  178. if ext == "" || len(ext) > 10 {
  179. return "bin"
  180. }
  181. for _, r := range ext {
  182. if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
  183. return "bin"
  184. }
  185. }
  186. return ext
  187. }
  188. // transformStrings walks every string value in decoded JSON
  189. func transformStrings(v interface{}, fn func(string) string) interface{} {
  190. switch t := v.(type) {
  191. case map[string]interface{}:
  192. for k, vv := range t {
  193. t[k] = transformStrings(vv, fn)
  194. }
  195. return t
  196. case []interface{}:
  197. for i, vv := range t {
  198. t[i] = transformStrings(vv, fn)
  199. }
  200. return t
  201. case string:
  202. return fn(t)
  203. }
  204. return v
  205. }
  206. // PackEnvelope converts an envelope JSON string into the zip container.
  207. // readVpath (optional) resolves media?file= links to file bytes.
  208. func PackEnvelope(envelope string, readVpath func(vpath string) ([]byte, error)) ([]byte, error) {
  209. doc, assets, err := collectAssets(envelope, readVpath)
  210. if err != nil {
  211. return nil, err
  212. }
  213. buf := new(bytes.Buffer)
  214. zw := zip.NewWriter(buf)
  215. w, err := zw.Create(packedDocName) // deflate: JSON compresses well
  216. if err != nil {
  217. return nil, err
  218. }
  219. if _, err = w.Write(doc); err != nil {
  220. return nil, err
  221. }
  222. for _, name := range sortedKeys(assets) {
  223. // media is usually pre-compressed - store without recompression
  224. hw, err := zw.CreateHeader(&zip.FileHeader{Name: "assets/" + name, Method: zip.Store})
  225. if err != nil {
  226. return nil, err
  227. }
  228. if _, err = hw.Write(assets[name]); err != nil {
  229. return nil, err
  230. }
  231. }
  232. if err := zw.Close(); err != nil {
  233. return nil, err
  234. }
  235. return buf.Bytes(), nil
  236. }
  237. // sortedKeys keeps the zip entry order stable, so packing the same
  238. // document twice produces the same bytes
  239. func sortedKeys(m map[string][]byte) []string {
  240. keys := make([]string, 0, len(m))
  241. for k := range m {
  242. keys = append(keys, k)
  243. }
  244. sort.Strings(keys)
  245. return keys
  246. }
  247. /*
  248. collectAssets takes the media out of an envelope: every data URL and every
  249. media?file= link (resolved through readVpath) becomes an "asset://<name>"
  250. reference, and the bytes come back beside the rewritten JSON, deduplicated
  251. by content hash. Shared by the session container above and by the asset
  252. store inside the suite's .docx / .xlsx / .pptx files (native.go).
  253. */
  254. func collectAssets(envelope string, readVpath func(vpath string) ([]byte, error)) ([]byte, map[string][]byte, error) {
  255. var root interface{}
  256. if err := json.Unmarshal([]byte(envelope), &root); err != nil {
  257. return nil, nil, errors.New("invalid envelope JSON: " + err.Error())
  258. }
  259. assets := map[string][]byte{} // name -> data
  260. byHash := map[string]string{} // content hash -> name
  261. add := func(data []byte, ext string) string {
  262. h := sha1.Sum(data)
  263. key := hex.EncodeToString(h[:])[:12]
  264. if name, ok := byHash[key]; ok {
  265. return name
  266. }
  267. name := key + "." + ext
  268. byHash[key] = name
  269. assets[name] = data
  270. return name
  271. }
  272. // a linked file, read through the caller; each vpath is read once
  273. embedded := map[string]string{}
  274. embedVpath := func(vp string) (string, bool) {
  275. if readVpath == nil {
  276. return "", false
  277. }
  278. if name, ok := embedded[vp]; ok {
  279. return name, true
  280. }
  281. data, err := readVpath(vp)
  282. if err != nil || len(data) == 0 {
  283. return "", false
  284. }
  285. name := add(data, assetExt(vp))
  286. embedded[vp] = name
  287. return name, true
  288. }
  289. root = transformStrings(root, func(s string) string {
  290. if data, ext, ok := media.ParseAnyDataURL(s); ok {
  291. return "asset://" + add(data, ext)
  292. }
  293. if vp := media.LinkVpath(s); vp != "" {
  294. if name, ok := embedVpath(vp); ok {
  295. return "asset://" + name
  296. }
  297. return s
  298. }
  299. return embedHTMLDataURLs(embedHTMLMediaLinks(s, embedVpath), add)
  300. })
  301. doc, err := marshalNoEscape(root)
  302. if err != nil {
  303. return nil, nil, err
  304. }
  305. return doc, assets, nil
  306. }
  307. // marshalNoEscape is json.Marshal without the HTML escaping: a Docs body is
  308. // mostly markup, and < for every "<" makes it a third bigger for nothing
  309. func marshalNoEscape(v interface{}) ([]byte, error) {
  310. var buf bytes.Buffer
  311. enc := json.NewEncoder(&buf)
  312. enc.SetEscapeHTML(false)
  313. if err := enc.Encode(v); err != nil {
  314. return nil, err
  315. }
  316. return bytes.TrimRight(buf.Bytes(), "\n"), nil
  317. }
  318. // UnpackEnvelopeToLinks restores the envelope JSON from a zip container,
  319. // writing every asset out through saveAsset and rewriting "asset://<name>"
  320. // references with linkFor(name) (typically a media?file= URL into a working
  321. // directory). This keeps the JSON small - the browser streams the media
  322. // instead of carrying megabytes of base64. Legacy plain-JSON files pass
  323. // through unchanged.
  324. func UnpackEnvelopeToLinks(data []byte, saveAsset func(name string, content []byte) error, linkFor func(name string) string) (string, error) {
  325. if len(data) < 4 || data[0] != 'P' || data[1] != 'K' {
  326. return string(data), nil // legacy plain JSON document
  327. }
  328. zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
  329. if err != nil {
  330. return "", errors.New("corrupted document container")
  331. }
  332. var doc []byte
  333. written := map[string]bool{}
  334. for _, f := range zr.File {
  335. name := path.Clean(f.Name)
  336. rc, err := f.Open()
  337. if err != nil {
  338. continue
  339. }
  340. b, err := io.ReadAll(rc)
  341. rc.Close()
  342. if err != nil {
  343. continue
  344. }
  345. if name == packedDocName {
  346. doc = b
  347. } else if strings.HasPrefix(name, "assets/") {
  348. an := strings.TrimPrefix(name, "assets/")
  349. if err := saveAsset(an, b); err == nil {
  350. written[an] = true
  351. }
  352. }
  353. }
  354. if doc == nil {
  355. return "", errors.New("document container is missing " + packedDocName)
  356. }
  357. var root interface{}
  358. if err := json.Unmarshal(doc, &root); err != nil {
  359. return "", errors.New("corrupted document.json: " + err.Error())
  360. }
  361. root = transformStrings(root, func(s string) string {
  362. return resolveAssetRefs(s, func(name string) (string, bool) {
  363. if !written[name] {
  364. return "", false
  365. }
  366. return linkFor(name), true
  367. })
  368. })
  369. out, err := json.Marshal(root)
  370. if err != nil {
  371. return "", err
  372. }
  373. return string(out), nil
  374. }
  375. // UnpackEnvelope restores the envelope JSON from a zip container. Legacy
  376. // plain-JSON files pass through unchanged.
  377. func UnpackEnvelope(data []byte) (string, error) {
  378. if len(data) < 4 || data[0] != 'P' || data[1] != 'K' {
  379. return string(data), nil // legacy plain JSON document
  380. }
  381. zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
  382. if err != nil {
  383. return "", errors.New("corrupted document container")
  384. }
  385. var doc []byte
  386. assets := map[string][]byte{}
  387. for _, f := range zr.File {
  388. rc, err := f.Open()
  389. if err != nil {
  390. continue
  391. }
  392. b, err := io.ReadAll(rc)
  393. rc.Close()
  394. if err != nil {
  395. continue
  396. }
  397. name := path.Clean(f.Name)
  398. if name == packedDocName {
  399. doc = b
  400. } else if strings.HasPrefix(name, "assets/") {
  401. assets[strings.TrimPrefix(name, "assets/")] = b
  402. }
  403. }
  404. if doc == nil {
  405. return "", errors.New("document container is missing " + packedDocName)
  406. }
  407. var root interface{}
  408. if err := json.Unmarshal(doc, &root); err != nil {
  409. return "", errors.New("corrupted document.json: " + err.Error())
  410. }
  411. root = transformStrings(root, func(s string) string {
  412. return resolveAssetRefs(s, func(name string) (string, bool) {
  413. data, ok := assets[name]
  414. if !ok {
  415. return "", false
  416. }
  417. return media.DataURLOf(data, strings.TrimPrefix(path.Ext(name), ".")), true
  418. })
  419. })
  420. out, err := json.Marshal(root)
  421. if err != nil {
  422. return "", err
  423. }
  424. return string(out), nil
  425. }