Przeglądaj źródła

S3 drive: fix copying new files over SMB, cache metadata to cut round trips (#315)

* s3fs: report a missing object as os.ErrNotExist

HeadObject answers a missing key with NotFound and GetObject with
NoSuchKey, and both reached callers as plain SDK errors. Code that tests
errors.Is(err, os.ErrNotExist) took them for real failures.

The built-in SMB server is one such caller: before creating a file,
File Explorer opens the name to see whether it exists, and the server
answered STATUS_UNSUCCESSFUL instead of STATUS_OBJECT_NAME_NOT_FOUND.
Explorer then gives up with "Can't read from the source file or disk",
so no new file could be copied onto an S3 drive over SMB.

Stat, GetModTime, ReadStream and Rename now wrap those answers so they
match os.ErrNotExist while keeping the original error. A missing bucket
is left as it is.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ExjHL5UKQTZYb5tpmZ9v1U

* s3fs: cache object metadata so SMB and listings stop waiting on the bucket

Every Stat on an S3 drive cost three round trips: a HEAD for a directory
marker, a one-key listing, then a HEAD for the object itself. Nothing was
kept, and File Explorer asks about one file ten or more times while it
opens it over SMB (CREATE, QUERY_INFO, FSCTL_CREATE_OR_GET_OBJECT_ID and
CLOSE with post-query each Stat it). Against Cloudflare R2 at ~140 ms a
request, every one of those took ~0.4 s and opening a small file took
several seconds.

- Stat asks for the object first, so a file is one request; only a miss
  goes on to the directory checks. FileExists, GetFileSize and GetModTime
  go through Stat.
- Answers, missing paths included, are kept for 10 seconds (statcache.go),
  the lifetime the Windows SMB client uses for its own cache.
- A complete ReadDir listing fills the cache and also answers for names
  it does not hold, so the per-file questions after Explorer lists a
  folder, and its probes for desktop.ini and the like, need no request.
- Mkdir, Remove, RemoveAll, Rename and WriteStream drop what they could
  make wrong (the path and every folder above it, or everything for a
  folder operation). A generation counter keeps a lookup that started
  before a change from storing its answer after it.

Changes made straight to the bucket, around ArozOS, show within the
10 seconds.

Measured against R2: a cold file Stat went from ~420 ms to ~200 ms, and
repeated Stats and Stats after a listing from ~420 ms to none.

s3fs_bucket_test.go runs against a real S3 service when
S3FS_TEST_ENDPOINT and S3FS_TEST_BUCKET are set (checked with SeaweedFS)
and counts the requests sent; it is skipped otherwise.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ExjHL5UKQTZYb5tpmZ9v1U

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Alan Yeung 1 tydzień temu
rodzic
commit
f3a6700c73

+ 109 - 49
src/mod/filesystem/abstractions/s3fs/s3fs.go

@@ -74,6 +74,7 @@ type S3FSAbstraction struct {
 	useSSL    bool
 	client    *s3.Client
 	uploader  *manager.Uploader
+	cache     *statCache
 }
 
 // NewS3FSAbstraction creates and validates a new S3 filesystem abstraction.
@@ -159,6 +160,7 @@ func NewS3FSAbstraction(uuid, hierarchy, endpoint, bucket, prefix, accessKey, se
 		useSSL:    useSSL,
 		client:    client,
 		uploader:  uploader,
+		cache:     newStatCache(statCacheTTL),
 	}, nil
 }
 
@@ -223,6 +225,7 @@ func (s *S3FSAbstraction) Mkdir(filename string, _ os.FileMode) error {
 		Body:        bytes.NewReader([]byte{}),
 		ContentType: aws.String("application/x-directory"),
 	})
+	s.cache.changed(strings.TrimSuffix(key, "/"))
 	return err
 }
 
@@ -253,11 +256,13 @@ func (s *S3FSAbstraction) Remove(filename string) error {
 		Bucket: aws.String(s.bucket),
 		Key:    aws.String(key),
 	})
+	s.cache.changed(key)
 	return err
 }
 
 // RemoveAll deletes every object whose key starts with the given path prefix.
 func (s *S3FSAbstraction) RemoveAll(path string) error {
+	defer s.cache.clear()
 	prefix := s.realPathToKey(path)
 	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
@@ -296,6 +301,7 @@ func (s *S3FSAbstraction) RemoveAll(path string) error {
 // Rename copies all objects from oldname to newname then removes the originals.
 // S3 has no native rename; this is a copy-then-delete sequence.
 func (s *S3FSAbstraction) Rename(oldname, newname string) error {
+	defer s.cache.clear()
 	ctx := context.Background()
 
 	if s.IsDir(oldname) {
@@ -347,7 +353,7 @@ func (s *S3FSAbstraction) Rename(oldname, newname string) error {
 		Key:        aws.String(newKey),
 	})
 	if err != nil {
-		return err
+		return notExist(err)
 	}
 	_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
 		Bucket: aws.String(s.bucket),
@@ -357,29 +363,54 @@ func (s *S3FSAbstraction) Rename(oldname, newname string) error {
 }
 
 // Stat returns object metadata.  Virtual directories are synthesised when any
-// objects exist under the path as a prefix.
+// objects exist under the path as a prefix.  The object is asked for first,
+// so a file costs one request, and answers are cached (statcache.go).
 func (s *S3FSAbstraction) Stat(filename string) (os.FileInfo, error) {
-	if s.IsDir(filename) {
+	key := s.realPathToKey(filename)
+	if key == "" || key == s.prefix {
 		return NewS3FileInfo(arozfs.Base(filename), 0, true, time.Now()), nil
 	}
-	key := s.realPathToKey(filename)
-	ctx := context.Background()
-	resp, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
+	if info, known := s.cache.get(key); known {
+		if info == nil {
+			return nil, &fs.PathError{Op: "stat", Path: filename, Err: os.ErrNotExist}
+		}
+		return info, nil
+	}
+
+	gen := s.cache.generation()
+	resp, err := s.client.HeadObject(context.Background(), &s3.HeadObjectInput{
 		Bucket: aws.String(s.bucket),
 		Key:    aws.String(key),
 	})
-	if err != nil {
+	if err == nil {
+		size := int64(0)
+		if resp.ContentLength != nil {
+			size = *resp.ContentLength
+		}
+		modTime := time.Now()
+		if resp.LastModified != nil {
+			modTime = *resp.LastModified
+		}
+		info := NewS3FileInfo(arozfs.Base(key), size, false, modTime)
+		s.cache.put(gen, key, info)
+		return info, nil
+	}
+	if err = notExist(err); !errors.Is(err, os.ErrNotExist) {
 		return nil, err
 	}
-	size := int64(0)
-	if resp.ContentLength != nil {
-		size = *resp.ContentLength
+
+	//No such object, but it may be a folder
+	isDir, dirErr := s.dirExists(key)
+	if dirErr != nil {
+		return nil, dirErr
 	}
-	modTime := time.Now()
-	if resp.LastModified != nil {
-		modTime = *resp.LastModified
+	if isDir {
+		info := NewS3FileInfo(arozfs.Base(filename), 0, true, time.Now())
+		s.cache.put(gen, key, info)
+		return info, nil
 	}
-	return NewS3FileInfo(arozfs.Base(key), size, false, modTime), nil
+	s.cache.put(gen, key, nil)
+	return nil, err
 }
 
 func (s *S3FSAbstraction) Close() error { return nil }
@@ -399,16 +430,8 @@ func (s *S3FSAbstraction) RealPathToVirtualPath(fullpath, username string) (stri
 // FileExists returns true if an exact object exists with this key, or if the
 // path is a virtual directory (i.e. objects exist under it as a prefix).
 func (s *S3FSAbstraction) FileExists(realpath string) bool {
-	key := s.realPathToKey(realpath)
-	ctx := context.Background()
-	_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	})
-	if err == nil {
-		return true
-	}
-	return s.IsDir(realpath)
+	_, err := s.Stat(realpath)
+	return err == nil
 }
 
 // IsDir returns true when the path represents a virtual S3 "directory":
@@ -420,12 +443,27 @@ func (s *S3FSAbstraction) IsDir(realpath string) bool {
 	if key == "" || key == s.prefix {
 		return true
 	}
+	if info, known := s.cache.get(key); known {
+		return info != nil && info.IsDir()
+	}
+	gen := s.cache.generation()
+	isDir, err := s.dirExists(key)
+	if err != nil {
+		return false
+	}
+	if isDir {
+		s.cache.put(gen, key, NewS3FileInfo(arozfs.Base(realpath), 0, true, time.Now()))
+	}
+	return isDir
+}
 
+// dirExists asks the bucket whether key is a folder: a directory marker
+// object, or any object under it.
+func (s *S3FSAbstraction) dirExists(key string) (bool, error) {
 	prefix := key
 	if !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
-
 	ctx := context.Background()
 
 	// Check for an explicit directory-marker object.
@@ -434,7 +472,10 @@ func (s *S3FSAbstraction) IsDir(realpath string) bool {
 		Key:    aws.String(prefix),
 	})
 	if err == nil {
-		return true
+		return true, nil
+	}
+	if err = notExist(err); !errors.Is(err, os.ErrNotExist) {
+		return false, err
 	}
 
 	// Check if any objects exist under this prefix.
@@ -443,10 +484,10 @@ func (s *S3FSAbstraction) IsDir(realpath string) bool {
 		Prefix:  aws.String(prefix),
 		MaxKeys: aws.Int32(1),
 	})
-	if err == nil && len(resp.Contents) > 0 {
-		return true
+	if err != nil {
+		return false, err
 	}
-	return false
+	return len(resp.Contents) > 0, nil
 }
 
 // Glob lists all entries in the same directory as realpathWildcard whose names
@@ -472,32 +513,19 @@ func (s *S3FSAbstraction) Glob(realpathWildcard string) ([]string, error) {
 }
 
 func (s *S3FSAbstraction) GetFileSize(realpath string) int64 {
-	key := s.realPathToKey(realpath)
-	ctx := context.Background()
-	resp, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	})
-	if err != nil || resp.ContentLength == nil {
+	info, err := s.Stat(realpath)
+	if err != nil || info.IsDir() {
 		return 0
 	}
-	return *resp.ContentLength
+	return info.Size()
 }
 
 func (s *S3FSAbstraction) GetModTime(realpath string) (int64, error) {
-	key := s.realPathToKey(realpath)
-	ctx := context.Background()
-	resp, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	})
+	info, err := s.Stat(realpath)
 	if err != nil {
 		return 0, err
 	}
-	if resp.LastModified == nil {
-		return time.Now().Unix(), nil
-	}
-	return resp.LastModified.Unix(), nil
+	return info.ModTime().Unix(), nil
 }
 
 func (s *S3FSAbstraction) WriteFile(filename string, content []byte, mode os.FileMode) error {
@@ -519,10 +547,12 @@ func (s *S3FSAbstraction) ReadFile(filename string) ([]byte, error) {
 func (s *S3FSAbstraction) ReadDir(dirname string) ([]fs.DirEntry, error) {
 	results := []fs.DirEntry{}
 
-	prefix := s.realPathToKey(dirname)
+	dirKey := s.realPathToKey(dirname)
+	prefix := dirKey
 	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
+	gen := s.cache.generation()
 
 	ctx := context.Background()
 	paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{
@@ -575,6 +605,14 @@ func (s *S3FSAbstraction) ReadDir(dirname string) ([]fs.DirEntry, error) {
 		}
 	}
 
+	//The listing is complete: keep it to answer Stat for this folder
+	children := make(map[string]os.FileInfo, len(results))
+	for _, entry := range results {
+		if info, err := entry.Info(); err == nil {
+			children[entry.Name()] = info
+		}
+	}
+	s.cache.putListing(gen, strings.TrimSuffix(dirKey, "/"), children)
 	return results, nil
 }
 
@@ -586,6 +624,7 @@ func (s *S3FSAbstraction) ReadDir(dirname string) ([]fs.DirEntry, error) {
 // Raspberry Pi) even for multi-GiB files.
 func (s *S3FSAbstraction) WriteStream(filename string, stream io.Reader, _ os.FileMode) error {
 	key := s.realPathToKey(filename)
+	defer s.cache.changed(key)
 	_, err := s.uploader.Upload(context.Background(), &s3.PutObjectInput{
 		Bucket:      aws.String(s.bucket),
 		Key:         aws.String(key),
@@ -607,7 +646,7 @@ func (s *S3FSAbstraction) ReadStream(filename string) (io.ReadCloser, error) {
 		Key:    aws.String(key),
 	})
 	if err != nil {
-		return nil, err
+		return nil, notExist(err)
 	}
 	return resp.Body, nil // io.ReadCloser backed directly by the S3 HTTP response
 }
@@ -672,6 +711,27 @@ func s3CopySource(bucket, key string) string {
 	return bucket + "/" + strings.Join(encoded, "/")
 }
 
+// notExist turns the answers S3 gives for a missing object into an error
+// matching os.ErrNotExist, keeping the original for the log. Callers test
+// for a missing file with errors.Is, the way they do on a local disk: the SMB
+// server must answer "not found" for File Explorer to create a new file.
+// HEAD answers NotFound and GET NoSuchKey; a missing bucket is not included.
+func notExist(err error) error {
+	if err == nil {
+		return nil
+	}
+	var notFound *types.NotFound
+	var noSuchKey *types.NoSuchKey
+	var coded interface{ ErrorCode() string }
+	switch {
+	case errors.As(err, &notFound), errors.As(err, &noSuchKey):
+	case errors.As(err, &coded) && (coded.ErrorCode() == "NotFound" || coded.ErrorCode() == "NoSuchKey"):
+	default:
+		return err
+	}
+	return fmt.Errorf("%w: %w", os.ErrNotExist, err)
+}
+
 func filterFilepath(rawpath string) string {
 	rawpath = arozfs.ToSlash(filepath.Clean(strings.TrimSpace(rawpath)))
 	if strings.HasPrefix(rawpath, "./") {

+ 198 - 0
src/mod/filesystem/abstractions/s3fs/s3fs_bucket_test.go

@@ -0,0 +1,198 @@
+package s3fs
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+/*
+	Tests against a real S3 compatible service. They run only when one is
+	named, for example a local SeaweedFS or MinIO:
+
+	S3FS_TEST_ENDPOINT=127.0.0.1:8333 S3FS_TEST_BUCKET=test \
+	S3FS_TEST_KEY=key S3FS_TEST_SECRET=secret go test ./mod/filesystem/abstractions/s3fs/
+
+	S3FS_TEST_SSL=1 uses https. Each run works under a prefix of its own and
+	removes it at the end.
+*/
+
+// countingClient counts the requests the drive sends to the bucket.
+type countingClient struct {
+	inner s3.HTTPClient
+	n     atomic.Int64
+}
+
+func (c *countingClient) Do(r *http.Request) (*http.Response, error) {
+	c.n.Add(1)
+	return c.inner.Do(r)
+}
+
+func bucketForTest(t *testing.T) (*S3FSAbstraction, *countingClient) {
+	t.Helper()
+	endpoint := os.Getenv("S3FS_TEST_ENDPOINT")
+	bucket := os.Getenv("S3FS_TEST_BUCKET")
+	if endpoint == "" || bucket == "" {
+		t.Skip("S3FS_TEST_ENDPOINT and S3FS_TEST_BUCKET name no bucket to test against")
+	}
+	prefix := fmt.Sprintf("s3fs-test-%d", time.Now().UnixNano())
+	a, err := NewS3FSAbstraction("s3test", "public", endpoint, bucket, prefix,
+		os.Getenv("S3FS_TEST_KEY"), os.Getenv("S3FS_TEST_SECRET"), os.Getenv("S3FS_TEST_SSL") == "1")
+	if err != nil {
+		t.Fatalf("mount %s/%s: %v", endpoint, bucket, err)
+	}
+	counter := &countingClient{}
+	a.client = s3.New(a.client.Options(), func(o *s3.Options) {
+		counter.inner = o.HTTPClient
+		o.HTTPClient = counter
+	})
+	a.uploader = manager.NewUploader(a.client)
+	t.Cleanup(func() {
+		if err := a.RemoveAll("/"); err != nil {
+			t.Errorf("remove test prefix %s: %v", prefix, err)
+		}
+	})
+	return a, counter
+}
+
+// requests runs f and returns how many requests it sent.
+func requests(c *countingClient, f func()) int64 {
+	before := c.n.Load()
+	f()
+	return c.n.Load() - before
+}
+
+func TestBucketMissingFileIsNotExist(t *testing.T) {
+	a, counter := bucketForTest(t)
+	if _, err := a.Stat("/new.txt"); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("Stat of a missing file = %v, want os.ErrNotExist", err)
+	}
+	if n := requests(counter, func() {
+		if _, err := a.Stat("/new.txt"); !errors.Is(err, os.ErrNotExist) {
+			t.Errorf("second Stat of a missing file = %v", err)
+		}
+	}); n != 0 {
+		t.Errorf("a repeated Stat of a missing file sent %d requests, want 0", n)
+	}
+	if _, err := a.ReadStream("/new.txt"); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("ReadStream of a missing file = %v, want os.ErrNotExist", err)
+	}
+	if _, err := a.GetModTime("/new.txt"); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("GetModTime of a missing file = %v, want os.ErrNotExist", err)
+	}
+}
+
+func TestBucketWriteIsSeenAtOnce(t *testing.T) {
+	a, _ := bucketForTest(t)
+	if _, err := a.Stat("/new.txt"); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("Stat before the write = %v", err)
+	}
+	if err := a.WriteStream("/new.txt", strings.NewReader("hello"), 0644); err != nil {
+		t.Fatalf("WriteStream: %v", err)
+	}
+	info, err := a.Stat("/new.txt")
+	if err != nil || info.Size() != 5 || info.IsDir() {
+		t.Fatalf("Stat right after the write = %v, %v, want a 5 byte file", info, err)
+	}
+	if err := a.WriteStream("/new.txt", strings.NewReader("hello world"), 0644); err != nil {
+		t.Fatalf("second WriteStream: %v", err)
+	}
+	if size := a.GetFileSize("/new.txt"); size != 11 {
+		t.Errorf("size after rewriting = %d, want 11", size)
+	}
+}
+
+func TestBucketFileStatIsOneRequest(t *testing.T) {
+	a, counter := bucketForTest(t)
+	if err := a.WriteStream("/new.txt", strings.NewReader("hello"), 0644); err != nil {
+		t.Fatalf("WriteStream: %v", err)
+	}
+	a.cache.clear()
+	if n := requests(counter, func() {
+		if _, err := a.Stat("/new.txt"); err != nil {
+			t.Errorf("Stat: %v", err)
+		}
+	}); n != 1 {
+		t.Errorf("Stat of a file sent %d requests, want 1", n)
+	}
+}
+
+func TestBucketListingAnswersStat(t *testing.T) {
+	a, counter := bucketForTest(t)
+	for _, p := range []string{"/new.txt", "/photos/a.jpg"} {
+		if err := a.WriteStream(p, strings.NewReader("abc"), 0644); err != nil {
+			t.Fatalf("WriteStream %s: %v", p, err)
+		}
+	}
+	if err := a.Mkdir("/empty", 0755); err != nil {
+		t.Fatalf("Mkdir: %v", err)
+	}
+	a.cache.clear()
+	if _, err := a.ReadDir("/"); err != nil {
+		t.Fatalf("ReadDir /: %v", err)
+	}
+	if _, err := a.ReadDir("/photos"); err != nil {
+		t.Fatalf("ReadDir /photos: %v", err)
+	}
+
+	n := requests(counter, func() {
+		if info, err := a.Stat("/new.txt"); err != nil || info.Size() != 3 {
+			t.Errorf("Stat /new.txt = %v, %v", info, err)
+		}
+		if !a.IsDir("/photos") || !a.IsDir("/empty") || !a.FileExists("/empty") {
+			t.Errorf("a listed folder is not seen as one")
+		}
+		if size := a.GetFileSize("/photos/a.jpg"); size != 3 {
+			t.Errorf("GetFileSize /photos/a.jpg = %d", size)
+		}
+		if _, err := a.Stat("/photos/desktop.ini"); !errors.Is(err, os.ErrNotExist) {
+			t.Errorf("Stat of a name missing from the listing = %v", err)
+		}
+	})
+	if n != 0 {
+		t.Errorf("questions answered by two listings sent %d requests, want 0", n)
+	}
+}
+
+func TestBucketRemoveAndRenameAreSeenAtOnce(t *testing.T) {
+	a, _ := bucketForTest(t)
+	for _, p := range []string{"/new.txt", "/photos/a.jpg"} {
+		if err := a.WriteStream(p, strings.NewReader("abc"), 0644); err != nil {
+			t.Fatalf("WriteStream %s: %v", p, err)
+		}
+	}
+	if _, err := a.ReadDir("/photos"); err != nil {
+		t.Fatalf("ReadDir: %v", err)
+	}
+	if !a.IsDir("/photos") {
+		t.Fatalf("/photos is not a folder")
+	}
+
+	if err := a.Remove("/photos/a.jpg"); err != nil {
+		t.Fatalf("Remove: %v", err)
+	}
+	if _, err := a.Stat("/photos/a.jpg"); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("Stat after Remove = %v, want os.ErrNotExist", err)
+	}
+	if a.IsDir("/photos") {
+		t.Errorf("/photos is still a folder after its only file was removed")
+	}
+
+	if err := a.Rename("/new.txt", "/renamed.txt"); err != nil {
+		t.Fatalf("Rename: %v", err)
+	}
+	if a.FileExists("/new.txt") {
+		t.Errorf("the old name exists after Rename")
+	}
+	if info, err := a.Stat("/renamed.txt"); err != nil || info.Size() != 3 {
+		t.Errorf("Stat of the new name = %v, %v", info, err)
+	}
+}

+ 50 - 0
src/mod/filesystem/abstractions/s3fs/s3fs_test.go

@@ -0,0 +1,50 @@
+package s3fs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+// codedError stands in for an API error the SDK has no type for, such as
+// NoSuchKey from CopyObject.
+type codedError struct{ code string }
+
+func (e codedError) Error() string     { return "api error " + e.code }
+func (e codedError) ErrorCode() string { return e.code }
+
+func TestNotExistMapsMissingObjects(t *testing.T) {
+	cases := []struct {
+		name    string
+		err     error
+		missing bool
+	}{
+		{"HeadObject NotFound", fmt.Errorf("operation error S3: HeadObject: %w", &types.NotFound{}), true},
+		{"GetObject NoSuchKey", fmt.Errorf("operation error S3: GetObject: %w", &types.NoSuchKey{}), true},
+		{"untyped NoSuchKey", fmt.Errorf("operation error S3: CopyObject: %w", codedError{"NoSuchKey"}), true},
+		{"untyped NotFound", codedError{"NotFound"}, true},
+		{"missing bucket", fmt.Errorf("operation error S3: GetObject: %w", &types.NoSuchBucket{}), false},
+		{"access denied", codedError{"AccessDenied"}, false},
+		{"network error", errors.New("dial tcp: connection refused"), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := notExist(tc.err)
+			if errors.Is(got, os.ErrNotExist) != tc.missing {
+				t.Fatalf("errors.Is(notExist(%v), os.ErrNotExist) = %v, want %v", tc.err, !tc.missing, tc.missing)
+			}
+			if !errors.Is(got, tc.err) {
+				t.Errorf("notExist dropped the original error %v", tc.err)
+			}
+		})
+	}
+}
+
+func TestNotExistKeepsNil(t *testing.T) {
+	if err := notExist(nil); err != nil {
+		t.Fatalf("notExist(nil) = %v, want nil", err)
+	}
+}

+ 172 - 0
src/mod/filesystem/abstractions/s3fs/statcache.go

@@ -0,0 +1,172 @@
+package s3fs
+
+import (
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+/*
+	statcache.go
+
+	Every question about a file is a round trip to the bucket, and a client
+	asks many: File Explorer looks at the same file ten or more times while
+	opening it over SMB, and lists a folder before asking about each file in
+	it. The answers are kept here for a few seconds.
+
+	A complete folder listing also answers for names it does not hold, so the
+	probes Explorer makes for desktop.ini and the like need no request either.
+
+	Every change made through this drive drops what it could have made wrong.
+	Changes made around ArozOS, straight to the bucket, show after at most the
+	lifetime of an entry, which matches what the Windows SMB client caches by
+	itself. A lookup that started before a change does not keep its answer.
+*/
+
+// statCacheTTL is how long an answer is kept.
+const statCacheTTL = 10 * time.Second
+
+// statCacheMax bounds the entries kept; past it expired ones are dropped,
+// then everything.
+const statCacheMax = 50000
+
+type statEntry struct {
+	info    os.FileInfo //Nil for a path known to be missing
+	expires time.Time
+}
+
+type statCache struct {
+	mu      sync.Mutex
+	ttl     time.Duration
+	now     func() time.Time
+	gen     uint64               //Counts changes, so a lookup older than one is not kept
+	entries map[string]statEntry //By object key without a trailing slash
+	listed  map[string]time.Time //Folders whose complete listing is in entries
+}
+
+func newStatCache(ttl time.Duration) *statCache {
+	return &statCache{
+		ttl:     ttl,
+		now:     time.Now,
+		entries: map[string]statEntry{},
+		listed:  map[string]time.Time{},
+	}
+}
+
+// parentKey returns the folder holding key, "" being the bucket root.
+func parentKey(key string) string {
+	if i := strings.LastIndexByte(key, '/'); i >= 0 {
+		return key[:i]
+	}
+	return ""
+}
+
+// childKey joins a folder key and a name.
+func childKey(dir, name string) string {
+	if dir == "" {
+		return name
+	}
+	return dir + "/" + name
+}
+
+// generation is taken before asking the bucket and handed to put.
+func (c *statCache) generation() uint64 {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.gen
+}
+
+// get answers from the cache. known is false when the bucket must be asked;
+// otherwise info is nil for a missing path.
+func (c *statCache) get(key string) (info os.FileInfo, known bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := c.now()
+	if e, ok := c.entries[key]; ok && now.Before(e.expires) {
+		return e.info, true
+	}
+	if key == "" {
+		return nil, false
+	}
+	if until, ok := c.listed[parentKey(key)]; ok && now.Before(until) {
+		//The folder was listed whole and nothing changed in it since
+		return nil, true
+	}
+	return nil, false
+}
+
+// put keeps an answer, nil info for a missing path, unless something changed
+// since gen was taken.
+func (c *statCache) put(gen uint64, key string, info os.FileInfo) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if gen != c.gen {
+		return
+	}
+	c.makeRoom(1)
+	c.entries[key] = statEntry{info: info, expires: c.now().Add(c.ttl)}
+}
+
+// putListing keeps a complete listing of the folder dir, by child name.
+func (c *statCache) putListing(gen uint64, dir string, children map[string]os.FileInfo) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if gen != c.gen {
+		return
+	}
+	c.makeRoom(len(children))
+	expires := c.now().Add(c.ttl)
+	for name, info := range children {
+		c.entries[childKey(dir, name)] = statEntry{info: info, expires: expires}
+	}
+	c.listed[dir] = expires
+}
+
+// makeRoom drops expired entries, then all, when n more would not fit.
+// The caller holds c.mu.
+func (c *statCache) makeRoom(n int) {
+	if len(c.entries)+n <= statCacheMax {
+		return
+	}
+	now := c.now()
+	for k, e := range c.entries {
+		if !now.Before(e.expires) {
+			delete(c.entries, k)
+		}
+	}
+	for k, until := range c.listed {
+		if !now.Before(until) {
+			delete(c.listed, k)
+		}
+	}
+	if len(c.entries)+n > statCacheMax {
+		c.entries = map[string]statEntry{}
+		c.listed = map[string]time.Time{}
+	}
+}
+
+// changed drops what a change to one file or empty folder at key can make
+// wrong: the path itself and every folder above it, which may have appeared
+// or gone with it.
+func (c *statCache) changed(key string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.gen++
+	delete(c.entries, key)
+	delete(c.listed, key)
+	for key != "" {
+		key = parentKey(key)
+		delete(c.entries, key)
+		delete(c.listed, key)
+	}
+}
+
+// clear drops everything, after a change to a whole folder.
+func (c *statCache) clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.gen++
+	c.entries = map[string]statEntry{}
+	c.listed = map[string]time.Time{}
+}

+ 189 - 0
src/mod/filesystem/abstractions/s3fs/statcache_test.go

@@ -0,0 +1,189 @@
+package s3fs
+
+import (
+	"os"
+	"testing"
+	"time"
+)
+
+// fakeClock lets a test move time instead of sleeping.
+type fakeClock struct{ t time.Time }
+
+func (c *fakeClock) now() time.Time          { return c.t }
+func (c *fakeClock) add(d time.Duration)     { c.t = c.t.Add(d) }
+func newTestCache() (*statCache, *fakeClock) { return newTestCacheTTL(10 * time.Second) }
+
+func newTestCacheTTL(ttl time.Duration) (*statCache, *fakeClock) {
+	clock := &fakeClock{t: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)}
+	c := newStatCache(ttl)
+	c.now = clock.now
+	return c, clock
+}
+
+func fileInfo(name string, size int64) os.FileInfo {
+	return NewS3FileInfo(name, size, false, time.Unix(0, 0))
+}
+
+func TestParentAndChildKey(t *testing.T) {
+	cases := []struct {
+		key, parent string
+	}{
+		{"a.txt", ""},
+		{"photos/a.jpg", "photos"},
+		{"prefix/photos/2026/a.jpg", "prefix/photos/2026"},
+		{"", ""},
+	}
+	for _, tc := range cases {
+		if got := parentKey(tc.key); got != tc.parent {
+			t.Errorf("parentKey(%q) = %q, want %q", tc.key, got, tc.parent)
+		}
+	}
+	if got := childKey("", "a.txt"); got != "a.txt" {
+		t.Errorf("childKey at the root = %q, want a.txt", got)
+	}
+	if got := childKey("photos", "a.jpg"); got != "photos/a.jpg" {
+		t.Errorf("childKey(photos, a.jpg) = %q", got)
+	}
+}
+
+func TestStatCacheKeepsAnswersUntilTheyExpire(t *testing.T) {
+	c, clock := newTestCache()
+	if _, known := c.get("a.txt"); known {
+		t.Fatalf("an empty cache claims to know a.txt")
+	}
+	c.put(c.generation(), "a.txt", fileInfo("a.txt", 5))
+	c.put(c.generation(), "gone.txt", nil)
+
+	info, known := c.get("a.txt")
+	if !known || info == nil || info.Size() != 5 {
+		t.Fatalf("get(a.txt) = %v, %v, want the stored file", info, known)
+	}
+	if info, known := c.get("gone.txt"); !known || info != nil {
+		t.Fatalf("get(gone.txt) = %v, %v, want known missing", info, known)
+	}
+
+	clock.add(10 * time.Second)
+	if _, known := c.get("a.txt"); known {
+		t.Errorf("a.txt is still known after its lifetime")
+	}
+	if _, known := c.get("gone.txt"); known {
+		t.Errorf("gone.txt is still known missing after its lifetime")
+	}
+}
+
+func TestStatCacheListingAnswersForMissingNames(t *testing.T) {
+	c, clock := newTestCache()
+	c.putListing(c.generation(), "photos", map[string]os.FileInfo{
+		"a.jpg": fileInfo("a.jpg", 1),
+		"2026":  NewS3FileInfo("2026", 0, true, time.Unix(0, 0)),
+	})
+
+	cases := []struct {
+		key     string
+		known   bool
+		present bool
+	}{
+		{"photos/a.jpg", true, true},
+		{"photos/2026", true, true},
+		{"photos/desktop.ini", true, false}, //Not in a complete listing
+		{"photos/2026/b.jpg", false, false}, //Below a folder that was not listed
+		{"other/a.jpg", false, false},
+	}
+	for _, tc := range cases {
+		info, known := c.get(tc.key)
+		if known != tc.known || (info != nil) != tc.present {
+			t.Errorf("get(%q) = present %v known %v, want present %v known %v", tc.key, info != nil, known, tc.present, tc.known)
+		}
+	}
+
+	clock.add(11 * time.Second)
+	if _, known := c.get("photos/desktop.ini"); known {
+		t.Errorf("an expired listing still answers for a missing name")
+	}
+}
+
+func TestStatCacheRootListing(t *testing.T) {
+	c, _ := newTestCache()
+	c.putListing(c.generation(), "", map[string]os.FileInfo{"a.txt": fileInfo("a.txt", 1)})
+	if info, known := c.get("a.txt"); !known || info == nil {
+		t.Errorf("get(a.txt) after a root listing = %v, %v", info, known)
+	}
+	if info, known := c.get("b.txt"); !known || info != nil {
+		t.Errorf("get(b.txt) after a root listing = %v, %v, want known missing", info, known)
+	}
+}
+
+func TestStatCacheChangeDropsPathAndFoldersAbove(t *testing.T) {
+	c, _ := newTestCache()
+	gen := c.generation()
+	c.putListing(gen, "", map[string]os.FileInfo{"photos": NewS3FileInfo("photos", 0, true, time.Unix(0, 0))})
+	c.putListing(gen, "photos", map[string]os.FileInfo{"a.jpg": fileInfo("a.jpg", 1)})
+	c.put(gen, "photos/new.jpg", nil)
+	c.put(gen, "docs/x.txt", fileInfo("x.txt", 1))
+
+	c.changed("photos/new.jpg")
+
+	for _, key := range []string{"photos/new.jpg", "photos", "photos/missing.jpg"} {
+		if _, known := c.get(key); known {
+			t.Errorf("get(%q) is still answered from the cache after photos/new.jpg changed", key)
+		}
+	}
+	//Unrelated entries stay
+	if info, known := c.get("docs/x.txt"); !known || info == nil {
+		t.Errorf("docs/x.txt was dropped by a change in photos")
+	}
+	if info, known := c.get("photos/a.jpg"); !known || info == nil {
+		t.Errorf("photos/a.jpg was dropped although only new.jpg changed")
+	}
+}
+
+func TestStatCacheDropsLookupsOlderThanAChange(t *testing.T) {
+	c, _ := newTestCache()
+	gen := c.generation() //A lookup starts
+	c.changed("a.txt")    //The file is written while it waits for the bucket
+	c.put(gen, "a.txt", nil)
+	c.putListing(gen, "", map[string]os.FileInfo{})
+	if _, known := c.get("a.txt"); known {
+		t.Fatalf("an answer from before the write was kept")
+	}
+
+	gen = c.generation()
+	c.clear()
+	c.put(gen, "b.txt", fileInfo("b.txt", 1))
+	if _, known := c.get("b.txt"); known {
+		t.Fatalf("an answer from before a clear was kept")
+	}
+}
+
+func TestStatCacheClear(t *testing.T) {
+	c, _ := newTestCache()
+	c.put(c.generation(), "a.txt", fileInfo("a.txt", 1))
+	c.putListing(c.generation(), "photos", map[string]os.FileInfo{})
+	c.clear()
+	if _, known := c.get("a.txt"); known {
+		t.Errorf("a.txt survived clear")
+	}
+	if _, known := c.get("photos/x"); known {
+		t.Errorf("the photos listing survived clear")
+	}
+}
+
+func TestStatCacheStaysBounded(t *testing.T) {
+	c, clock := newTestCache()
+	gen := c.generation()
+	for i := 0; i < statCacheMax; i++ {
+		c.put(gen, childKey("old", string(rune('a'+i%26))+time.Duration(i).String()), nil)
+	}
+	clock.add(time.Minute) //All of them expired
+	c.put(gen, "fresh.txt", fileInfo("fresh.txt", 1))
+	if len(c.entries) != 1 {
+		t.Fatalf("%d entries after expired ones should have been dropped, want 1", len(c.entries))
+	}
+
+	for i := 0; i < statCacheMax; i++ {
+		c.put(gen, childKey("live", time.Duration(i).String()), nil)
+	}
+	if len(c.entries) > statCacheMax {
+		t.Fatalf("%d entries, over the limit of %d", len(c.entries), statCacheMax)
+	}
+}