summaryrefslogtreecommitdiffstats
path: root/vendor/github.com/mattermost/rsc/appfs/server/app.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/github.com/mattermost/rsc/appfs/server/app.go')
-rw-r--r--vendor/github.com/mattermost/rsc/appfs/server/app.go982
1 files changed, 982 insertions, 0 deletions
diff --git a/vendor/github.com/mattermost/rsc/appfs/server/app.go b/vendor/github.com/mattermost/rsc/appfs/server/app.go
new file mode 100644
index 000000000..9486eac41
--- /dev/null
+++ b/vendor/github.com/mattermost/rsc/appfs/server/app.go
@@ -0,0 +1,982 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package server implements an appfs server backed by the
+// App Engine datastore.
+package server
+
+import (
+ "bytes"
+ "crypto/sha1"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "appengine"
+ "appengine/datastore"
+ "appengine/memcache"
+ "appengine/user"
+
+ "github.com/mattermost/rsc/appfs/fs"
+ "github.com/mattermost/rsc/appfs/proto"
+)
+
+const pwFile = "/.password"
+var chatty = false
+
+func init() {
+ handle(proto.ReadURL, (*request).read)
+ handle(proto.WriteURL, (*request).write)
+ handle(proto.StatURL, (*request).stat)
+ handle(proto.MkfsURL, (*request).mkfs)
+ handle(proto.CreateURL, (*request).create)
+ handle(proto.RemoveURL, (*request).remove)
+}
+
+type request struct {
+ w http.ResponseWriter
+ req *http.Request
+ c appengine.Context
+ name string
+ mname string
+ key *datastore.Key
+}
+
+func auth(r *request) bool {
+ hdr := r.req.Header.Get("Authorization")
+ if !strings.HasPrefix(hdr, "Basic ") {
+ return false
+ }
+ data, err := base64.StdEncoding.DecodeString(hdr[6:])
+ if err != nil {
+ return false
+ }
+ i := bytes.IndexByte(data, ':')
+ if i < 0 {
+ return false
+ }
+ user, passwd := string(data[:i]), string(data[i+1:])
+
+ _, data, err = read(r.c, pwFile)
+ if err != nil {
+ r.c.Errorf("reading %s: %v", pwFile, err)
+ if _, err := mkfs(r.c); err != nil {
+ r.c.Errorf("creating fs: %v", err)
+ }
+ _, data, err = read(r.c, pwFile)
+ if err != nil {
+ r.c.Errorf("reading %s again: %v", pwFile, err)
+ return false
+ }
+ }
+
+ lines := strings.Split(string(data), "\n")
+ for _, line := range lines {
+ if strings.HasPrefix(line, "#") {
+ continue
+ }
+ f := strings.Fields(line)
+ if len(f) < 3 {
+ continue
+ }
+ if f[0] == user {
+ return hash(f[1]+passwd) == f[2]
+ }
+ }
+ return false
+}
+
+func hash(s string) string {
+ h := sha1.New()
+ h.Write([]byte(s))
+ return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func handle(prefix string, f func(*request)) {
+ http.HandleFunc(prefix, func(w http.ResponseWriter, req *http.Request) {
+ c := appengine.NewContext(req)
+ r := &request{
+ w: w,
+ req: req,
+ c: c,
+ }
+
+ if strings.HasSuffix(prefix, "/") {
+ r.name, r.mname, r.key = mangle(c, req.URL.Path[len(prefix)-1:])
+ } else {
+ req.URL.Path = "/"
+ }
+ defer func() {
+ if err := recover(); err != nil {
+ w.WriteHeader(http.StatusConflict)
+ fmt.Fprintf(w, "%s\n", err)
+ }
+ }()
+
+ if !auth(r) {
+ w.Header().Set("WWW-Authenticate", "Basic realm=\"appfs\"")
+ http.Error(w, "Need auth", http.StatusUnauthorized)
+ return
+ }
+
+ f(r)
+ })
+}
+
+func mangle(c appengine.Context, name string) (string, string, *datastore.Key) {
+ name = path.Clean("/" + name)
+ n := strings.Count(name, "/")
+ if name == "/" {
+ n = 0
+ }
+ mname := fmt.Sprintf("%d%s", n, name)
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ key := datastore.NewKey(c, "FileInfo", mname, 0, root)
+ return name, mname, key
+}
+
+type FileInfo struct {
+ Path string // mangled path
+ Name string
+ Qid int64 // assigned unique id number
+ Seq int64 // modification sequence number in file tree
+ ModTime time.Time
+ Size int64
+ IsDir bool
+}
+
+type FileData struct {
+ Data []byte
+}
+
+func stat(c appengine.Context, name string) (*FileInfo, error) {
+ var fi FileInfo
+ name, _, key := mangle(c, name)
+ c.Infof("DATASTORE Stat %q", name)
+ err := datastore.Get(c, key, &fi)
+ if err != nil {
+ return nil, err
+ }
+ return &fi, nil
+}
+
+func (r *request) saveStat(fi *FileInfo) {
+ jfi, err := json.Marshal(&fi)
+ if err != nil {
+ panic(err)
+ }
+ r.w.Header().Set("X-Appfs-Stat", string(jfi))
+}
+
+func (r *request) tx(f func(c appengine.Context) error) {
+ err := datastore.RunInTransaction(r.c, f, &datastore.TransactionOptions{XG: true})
+ if err != nil {
+ panic(err)
+ }
+}
+
+func (r *request) stat() {
+ var fi *FileInfo
+ r.tx(func(c appengine.Context) error {
+ fi1, err := stat(c, r.name)
+ if err != nil {
+ return err
+ }
+ fi = fi1
+ return nil
+ })
+
+ jfi, err := json.Marshal(&fi)
+ if err != nil {
+ panic(err)
+ }
+ r.w.Write(jfi)
+}
+
+func read(c appengine.Context, name string) (fi *FileInfo, data []byte, err error) {
+ name, _, _ = mangle(c, name)
+ fi1, err := stat(c, name)
+ if err != nil {
+ return nil, nil, err
+ }
+ if fi1.IsDir {
+ dt, err := readdir(c, name)
+ if err != nil {
+ return nil, nil, err
+ }
+ fi = fi1
+ data = dt
+ return fi, data, nil
+ }
+
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root)
+ var fd FileData
+ c.Infof("DATASTORE Read %q", name)
+ if err := datastore.Get(c, dkey, &fd); err != nil {
+ return nil, nil, err
+ }
+ fi = fi1
+ data = fd.Data
+ return fi, data, nil
+}
+
+func (r *request) read() {
+ var (
+ fi *FileInfo
+ data []byte
+ )
+ r.tx(func(c appengine.Context) error {
+ var err error
+ fi, data, err = read(r.c, r.name)
+ return err
+ })
+ r.saveStat(fi)
+ r.w.Write(data)
+}
+
+func readdir(c appengine.Context, name string) ([]byte, error) {
+ name, _, _ = mangle(c, name)
+ var buf bytes.Buffer
+
+ n := strings.Count(name, "/")
+ if name == "/" {
+ name = ""
+ n = 0
+ }
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ first := fmt.Sprintf("%d%s/", n+1, name)
+ limit := fmt.Sprintf("%d%s0", n+1, name)
+ c.Infof("DATASTORE ReadDir %q", name)
+ q := datastore.NewQuery("FileInfo").
+ Filter("Path >=", first).
+ Filter("Path <", limit).
+ Ancestor(root)
+ enc := json.NewEncoder(&buf)
+ it := q.Run(c)
+ var fi FileInfo
+ var pfi proto.FileInfo
+ for {
+ fi = FileInfo{}
+ _, err := it.Next(&fi)
+ if err != nil {
+ if err == datastore.Done {
+ break
+ }
+ return nil, err
+ }
+ pfi = proto.FileInfo{
+ Name: fi.Name,
+ ModTime: fi.ModTime,
+ Size: fi.Size,
+ IsDir: fi.IsDir,
+ }
+ if err := enc.Encode(&pfi); err != nil {
+ return nil, err
+ }
+ }
+
+ return buf.Bytes(), nil
+}
+
+func readdirRaw(c appengine.Context, name string) ([]proto.FileInfo, error) {
+ name, _, _ = mangle(c, name)
+ n := strings.Count(name, "/")
+ if name == "/" {
+ name = ""
+ n = 0
+ }
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ first := fmt.Sprintf("%d%s/", n+1, name)
+ limit := fmt.Sprintf("%d%s0", n+1, name)
+ c.Infof("DATASTORE ReadDir %q", name)
+ q := datastore.NewQuery("FileInfo").
+ Filter("Path >=", first).
+ Filter("Path <", limit).
+ Ancestor(root)
+ it := q.Run(c)
+ var fi FileInfo
+ var pfi proto.FileInfo
+ var out []proto.FileInfo
+ for {
+ fi = FileInfo{}
+ _, err := it.Next(&fi)
+ if err != nil {
+ if err == datastore.Done {
+ break
+ }
+ return nil, err
+ }
+ pfi = proto.FileInfo{
+ Name: fi.Name,
+ ModTime: fi.ModTime,
+ Size: fi.Size,
+ IsDir: fi.IsDir,
+ }
+ out = append(out, pfi)
+ }
+println("READDIR", name, len(out))
+ return out, nil
+}
+
+
+var initPasswd = `# Password file
+# This file controls access to the server.
+# The format is lines of space-separated fields:
+# user salt pwhash
+# The pwhash is the SHA1 of the salt string concatenated with the password.
+
+# user=dummy password=dummy (replace with your own entries)
+dummy 12345 faa863c7d3d41893f80165c704b714d5e31bdd3b
+`
+
+func (r *request) mkfs() {
+ var fi *FileInfo
+ r.tx(func(c appengine.Context) error {
+ var err error
+ fi, err = mkfs(c)
+ return err
+ })
+ r.saveStat(fi)
+}
+
+func mkfs(c appengine.Context) (fi *FileInfo, err error) {
+ fi1, err := stat(c, "/")
+ if err == nil {
+ return fi1, nil
+ }
+
+ // Root needs to be created.
+ // Probably root key does too.
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ _, err = datastore.Put(c, root, &struct{}{})
+ if err != nil {
+ return nil, fmt.Errorf("mkfs put root: %s", err)
+ }
+
+ // Entry for /.
+ _, mpath, key := mangle(c, "/")
+ fi3 := FileInfo{
+ Path: mpath,
+ Name: "/",
+ Seq: 2, // 2, not 1, because we're going to write password file with #2
+ Qid: 1,
+ ModTime: time.Now(),
+ Size: 0,
+ IsDir: true,
+ }
+ _, err = datastore.Put(c, key, &fi3)
+ if err != nil {
+ return nil, fmt.Errorf("mkfs put /: %s", err)
+ }
+
+ /*
+ * Would like to use this code but App Engine apparently
+ * does not let Get observe the effect of a Put in the same
+ * transaction. What planet does that make sense on?
+ * Instead, we have to execute just the datastore writes that this
+ * sequence would.
+ *
+ _, err = create(c, pwFile, false)
+ if err != nil {
+ return nil, fmt.Errorf("mkfs create .password: %s", err)
+ }
+ _, err = write(c, pwFile, []byte(initPasswd))
+ if err != nil {
+ return nil, fmt.Errorf("mkfs write .password: %s", err)
+ }
+ *
+ */
+
+ {
+ name, mname, key := mangle(c, pwFile)
+
+ // Create data object.
+ dataKey := int64(2)
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ dkey := datastore.NewKey(c, "FileData", "", dataKey, root)
+ _, err := datastore.Put(c, dkey, &FileData{[]byte(initPasswd)})
+ if err != nil {
+ return nil, err
+ }
+
+ // Create new directory entry.
+ _, elem := path.Split(name)
+ fi1 = &FileInfo{
+ Path: mname,
+ Name: elem,
+ Qid: 2,
+ Seq: 2,
+ ModTime: time.Now(),
+ Size: int64(len(initPasswd)),
+ IsDir: false,
+ }
+ if _, err := datastore.Put(c, key, fi1); err != nil {
+ return nil, err
+ }
+ }
+
+ return &fi3, nil
+}
+
+func (r *request) write() {
+ data, err := ioutil.ReadAll(r.req.Body)
+ if err != nil {
+ panic(err)
+ }
+
+ var fi *FileInfo
+ var seq int64
+ r.tx(func(c appengine.Context) error {
+ var err error
+ fi, seq, err = write(r.c, r.name, data)
+ return err
+ })
+ updateCacheTime(r.c, seq)
+ r.saveStat(fi)
+}
+
+func write(c appengine.Context, name string, data []byte) (*FileInfo, int64, error) {
+ name, _, key := mangle(c, name)
+
+ // Check that file exists and is not a directory.
+ fi1, err := stat(c, name)
+ if err != nil {
+ return nil, 0, err
+ }
+ if fi1.IsDir {
+ return nil, 0, fmt.Errorf("cannot write to directory")
+ }
+
+ // Fetch and increment root sequence number.
+ rfi, err := stat(c, "/")
+ if err != nil {
+ return nil, 0, err
+ }
+ rfi.Seq++
+
+ // Write data.
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root)
+ fd := &FileData{data}
+ if _, err := datastore.Put(c, dkey, fd); err != nil {
+ return nil, 0, err
+ }
+
+ // Update directory entry.
+ fi1.Seq = rfi.Seq
+ fi1.Size = int64(len(data))
+ fi1.ModTime = time.Now()
+ if _, err := datastore.Put(c, key, fi1); err != nil {
+ return nil, 0, err
+ }
+
+ // Update sequence numbers all the way to the root.
+ if err := updateSeq(c, name, rfi.Seq, 1); err != nil {
+ return nil, 0, err
+ }
+
+ return fi1, rfi.Seq, nil
+}
+
+func updateSeq(c appengine.Context, name string, seq int64, skip int) error {
+ p := path.Clean(name)
+ for i := 0; ; i++ {
+ if i >= skip {
+ _, _, key := mangle(c, p)
+ var fi FileInfo
+ if err := datastore.Get(c, key, &fi); err != nil {
+ return err
+ }
+ fi.Seq = seq
+ if _, err := datastore.Put(c, key, &fi); err != nil {
+ return err
+ }
+ }
+ if p == "/" {
+ break
+ }
+ p, _ = path.Split(p)
+ p = path.Clean(p)
+ }
+ return nil
+}
+
+func (r *request) remove() {
+ panic("remove not implemented")
+}
+
+func (r *request) create() {
+ var fi *FileInfo
+ var seq int64
+ isDir := r.req.FormValue("dir") == "1"
+ r.tx(func(c appengine.Context) error {
+ var err error
+ fi, seq, err = create(r.c, r.name, isDir, nil)
+ return err
+ })
+ updateCacheTime(r.c, seq)
+ r.saveStat(fi)
+}
+
+func create(c appengine.Context, name string, isDir bool, data []byte) (*FileInfo, int64, error) {
+ name, mname, key := mangle(c, name)
+
+ // File must not exist.
+ fi1, err := stat(c, name)
+ if err == nil {
+ return nil, 0, fmt.Errorf("file already exists")
+ }
+ if err != datastore.ErrNoSuchEntity {
+ return nil, 0, err
+ }
+
+ // Parent must exist and be a directory.
+ p, _ := path.Split(name)
+ fi2, err := stat(c, p)
+ if err != nil {
+ if err == datastore.ErrNoSuchEntity {
+ return nil, 0, fmt.Errorf("parent directory %q does not exist", p)
+ }
+ return nil, 0, err
+ }
+ if !fi2.IsDir {
+ return nil, 0, fmt.Errorf("parent %q is not a directory", p)
+ }
+
+ // Fetch and increment root sequence number.
+ rfi, err := stat(c, "/")
+ if err != nil {
+ return nil, 0, err
+ }
+ rfi.Seq++
+
+ var dataKey int64
+ // Create data object.
+ if !isDir {
+ dataKey = rfi.Seq
+ root := datastore.NewKey(c, "RootKey", "v2:", 0, nil)
+ dkey := datastore.NewKey(c, "FileData", "", dataKey, root)
+ _, err := datastore.Put(c, dkey, &FileData{data})
+ if err != nil {
+ return nil, 0, err
+ }
+ }
+
+ // Create new directory entry.
+ _, elem := path.Split(name)
+ fi1 = &FileInfo{
+ Path: mname,
+ Name: elem,
+ Qid: rfi.Seq,
+ Seq: rfi.Seq,
+ ModTime: time.Now(),
+ Size: int64(len(data)),
+ IsDir: isDir,
+ }
+ if _, err := datastore.Put(c, key, fi1); err != nil {
+ return nil, 0, err
+ }
+
+ // Update sequence numbers all the way to root,
+ // but skip entry we just wrote.
+ if err := updateSeq(c, name, rfi.Seq, 1); err != nil {
+ return nil, 0, err
+ }
+
+ return fi1, rfi.Seq, nil
+}
+
+// Implementation of fs.AppEngine.
+
+func init() {
+ fs.Register(ae{})
+}
+
+type ae struct{}
+
+func tx(c interface{}, f func(c appengine.Context) error) error {
+ return datastore.RunInTransaction(c.(appengine.Context), f, &datastore.TransactionOptions{XG: true})
+}
+
+func (ae) NewContext(req *http.Request) interface{} {
+ return appengine.NewContext(req)
+}
+
+func (ae) User(ctxt interface{}) string {
+ c := ctxt.(appengine.Context)
+ u := user.Current(c)
+ if u == nil {
+ return "?"
+ }
+ return u.String()
+}
+
+type cacheKey struct {
+ t int64
+ name string
+}
+
+func (ae) CacheRead(ctxt interface{}, name, path string) (key interface{}, data []byte, found bool) {
+ c := ctxt.(appengine.Context)
+ t, data, _, err := cacheRead(c, "cache", name, path)
+ return &cacheKey{t, name}, data, err == nil
+}
+
+func (ae) CacheWrite(ctxt, key interface{}, data []byte) {
+ c := ctxt.(appengine.Context)
+ k := key.(*cacheKey)
+ cacheWrite(c, k.t, "cache", k.name, data)
+}
+
+func (ae ae) Read(ctxt interface{}, name string) (data []byte, pfi *proto.FileInfo, err error) {
+ c := ctxt.(appengine.Context)
+ name = path.Clean("/"+name)
+ if chatty {
+ c.Infof("AE Read %s", name)
+ }
+ _, data, pfi, err = cacheRead(c, "data", name, name)
+ if err != nil {
+ err = fmt.Errorf("Read %q: %v", name, err)
+ }
+ return
+}
+
+func (ae) Write(ctxt interface{}, path string, data []byte) error {
+ var seq int64
+ err := tx(ctxt, func(c appengine.Context) error {
+ _, err := stat(c, path)
+ if err != nil {
+ _, seq, err = create(c, path, false, data)
+ } else {
+ _, seq, err = write(c, path, data)
+ }
+ return err
+ })
+ if seq != 0 {
+ updateCacheTime(ctxt.(appengine.Context), seq)
+ }
+ if err != nil {
+ err = fmt.Errorf("Write %q: %v", path, err)
+ }
+ return err
+}
+
+func (ae) Remove(ctxt interface{}, path string) error {
+ return fmt.Errorf("remove not implemented")
+}
+
+func (ae) Mkdir(ctxt interface{}, path string) error {
+ var seq int64
+ err := tx(ctxt, func(c appengine.Context) error {
+ var err error
+ _, seq, err = create(c, path, true, nil)
+ return err
+ })
+ if seq != 0 {
+ updateCacheTime(ctxt.(appengine.Context), seq)
+ }
+ if err != nil {
+ err = fmt.Errorf("Mkdir %q: %v", path, err)
+ }
+ return err
+}
+
+func (ae) Criticalf(ctxt interface{}, format string, args ...interface{}) {
+ ctxt.(appengine.Context).Criticalf(format, args...)
+}
+
+type readDirCacheEntry struct {
+ Dir []proto.FileInfo
+ Error string
+}
+
+func (ae) ReadDir(ctxt interface{}, name string) (dir []proto.FileInfo, err error) {
+ c := ctxt.(appengine.Context)
+ name = path.Clean("/"+name)
+ t, data, _, err := cacheRead(c, "dir", name, name)
+ if err == nil {
+ var e readDirCacheEntry
+ if err := json.Unmarshal(data, &e); err == nil {
+ if chatty {
+ c.Infof("cached ReadDir %q", name)
+ }
+ if e.Error != "" {
+ return nil, errors.New(e.Error)
+ }
+ return e.Dir, nil
+ }
+ c.Criticalf("unmarshal cached dir %q: %v", name)
+ }
+ err = tx(ctxt, func(c appengine.Context) error {
+ var err error
+ dir, err = readdirRaw(c, name)
+ return err
+ })
+ var e readDirCacheEntry
+ e.Dir = dir
+ if err != nil {
+ err = fmt.Errorf("ReadDir %q: %v", name, err)
+ e.Error = err.Error()
+ }
+ if data, err := json.Marshal(&e); err != nil {
+ c.Criticalf("json marshal cached dir: %v", err)
+ } else {
+ c.Criticalf("caching dir %q@%d %d bytes", name, t, len(data))
+ cacheWrite(c, t, "dir", name, data)
+ }
+ return
+}
+
+// Caching of file system data.
+//
+// The cache stores entries under keys of the form time,space,name,
+// where time is the time at which the entry is valid for, space is a name
+// space identifier, and name is an arbitrary name.
+//
+// A key of the form t,mtime,path maps to an integer value giving the
+// modification time of the named path at root time t.
+// The special key 0,mtime,/ is an integer giving the current time at the root.
+//
+// A key of the form t,data,path maps to the content of path at time t.
+//
+// Thus, a read from path should first obtain the root time,
+// then obtain the modification time for the path at that root time
+// then obtain the data for that path.
+// t1 = get(0,mtime,/)
+// t2 = get(t1,mtime,path)
+// data = get(t2,data,path)
+//
+// The API allows clients to cache their own data too, with expiry tied to
+// the modification time of a particular path (file or directory). To look
+// up one of those, we use:
+// t1 = get(0,mtime,/)
+// t2 = get(t1,mtime,path)
+// data = get(t2,clientdata,name)
+//
+// To store data in the cache, the t1, t2 should be determined before reading
+// from datastore. Then the data should be saved under t2. This ensures
+// that if a datastore update happens after the read but before the cache write,
+// we'll be writing to an entry that will no longer be used (t2).
+
+const rootMemcacheKey = "0,mtime,/"
+
+func updateCacheTime(c appengine.Context, seq int64) {
+ const key = rootMemcacheKey
+ bseq := []byte(strconv.FormatInt(seq, 10))
+ for tries := 0; tries < 10; tries++ {
+ item, err := memcache.Get(c, key)
+ if err != nil {
+ c.Infof("memcache.Get %q: %v", key, err)
+ err = memcache.Add(c, &memcache.Item{Key: key, Value: bseq})
+ if err == nil {
+ c.Infof("memcache.Add %q %q ok", key, bseq)
+ return
+ }
+ c.Infof("memcache.Add %q %q: %v", key, bseq, err)
+ }
+ v, err := strconv.ParseInt(string(item.Value), 10, 64)
+ if err != nil {
+ c.Criticalf("memcache.Get %q = %q (%v)", key, item.Value, err)
+ return
+ }
+ if v >= seq {
+ return
+ }
+ item.Value = bseq
+ err = memcache.CompareAndSwap(c, item)
+ if err == nil {
+ c.Infof("memcache.CAS %q %d->%d ok", key, v, seq)
+ return
+ }
+ c.Infof("memcache.CAS %q %d->%d: %v", key, v, seq, err)
+ }
+ c.Criticalf("repeatedly failed to update root key")
+}
+
+func cacheTime(c appengine.Context) (t int64, err error) {
+ const key = rootMemcacheKey
+ item, err := memcache.Get(c, key)
+ if err == nil {
+ v, err := strconv.ParseInt(string(item.Value), 10, 64)
+ if err == nil {
+ if chatty {
+ c.Infof("cacheTime %q = %v", key, v)
+ }
+ return v, nil
+ }
+ c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err)
+ memcache.Delete(c, key)
+ }
+ fi, err := stat(c, "/")
+ if err != nil {
+ c.Criticalf("stat /: %v", err)
+ return 0, err
+ }
+ updateCacheTime(c, fi.Seq)
+ return fi.Seq, nil
+}
+
+func cachePathTime(c appengine.Context, path string) (t int64, err error) {
+ t, err = cacheTime(c)
+ if err != nil {
+ return 0, err
+ }
+
+ key := fmt.Sprintf("%d,mtime,%s", t, path)
+ item, err := memcache.Get(c, key)
+ if err == nil {
+ v, err := strconv.ParseInt(string(item.Value), 10, 64)
+ if err == nil {
+ if chatty {
+ c.Infof("cachePathTime %q = %v", key, v)
+ }
+ return v, nil
+ }
+ c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err)
+ memcache.Delete(c, key)
+ }
+
+ var seq int64
+ if fi, err := stat(c, path); err == nil {
+ seq = fi.Seq
+ }
+
+ c.Infof("cachePathTime save %q = %v", key, seq)
+ item = &memcache.Item{Key: key, Value: []byte(strconv.FormatInt(seq, 10))}
+ if err := memcache.Set(c, item); err != nil {
+ c.Criticalf("memcache.Set %q %q: %v", key, item.Value, err)
+ }
+ return seq, nil
+}
+
+type statCacheEntry struct {
+ FileInfo *proto.FileInfo
+ Error string
+}
+
+func cacheRead(c appengine.Context, kind, name, path string) (mtime int64, data []byte, pfi *proto.FileInfo, err error) {
+ for tries := 0; tries < 10; tries++ {
+ t, err := cachePathTime(c, path)
+ if err != nil {
+ return 0, nil, nil, err
+ }
+
+ key := fmt.Sprintf("%d,%s,%s", t, kind, name)
+ item, err := memcache.Get(c, key)
+ var data []byte
+ if item != nil {
+ data = item.Value
+ }
+ if err != nil {
+ c.Infof("memcache miss %q %v", key, err)
+ } else if chatty {
+ c.Infof("memcache hit %q (%d bytes)", key, len(data))
+ }
+ if kind != "data" {
+ // Not a file; whatever memcache says is all we have.
+ return t, data, nil, err
+ }
+
+ // Load stat from cache (includes negative entry).
+ statkey := fmt.Sprintf("%d,stat,%s", t, name)
+ var st statCacheEntry
+ _, err = memcache.JSON.Get(c, statkey, &st)
+ if err == nil {
+ if st.Error != "" {
+ if chatty {
+ c.Infof("memcache hit stat error %q %q", statkey, st.Error)
+ }
+ err = errors.New(st.Error)
+ } else {
+ if chatty {
+ c.Infof("memcache hit stat %q", statkey)
+ }
+ }
+ if err != nil || data != nil {
+ return t, data, st.FileInfo, err
+ }
+ }
+
+ // Need stat, or maybe stat+data.
+ var fi *FileInfo
+ if data != nil {
+ c.Infof("stat %q", name)
+ fi, err = stat(c, name)
+ if err == nil && fi.Seq != t {
+ c.Criticalf("loaded %s but found stat %d", key, fi.Seq)
+ continue
+ }
+ } else {
+ c.Infof("read %q", name)
+ fi, data, err = read(c, name)
+ if err == nil && fi.Seq != t {
+ c.Infof("loaded %s but found read %d", key, fi.Seq)
+ t = fi.Seq
+ key = fmt.Sprintf("%d,data,%s", t, name)
+ statkey = fmt.Sprintf("%d,stat,%s", t, name)
+ }
+
+ // Save data to memcache.
+ if err == nil {
+ if true || chatty {
+ c.Infof("save data in memcache %q", key)
+ }
+ item := &memcache.Item{Key: key, Value: data}
+ if err := memcache.Set(c, item); err != nil {
+ c.Criticalf("failed to cache %s: %v", key, err)
+ }
+ }
+ }
+
+ // Cache stat, including error.
+ st = statCacheEntry{}
+ if fi != nil {
+ st.FileInfo = &proto.FileInfo{
+ Name: fi.Name,
+ ModTime: fi.ModTime,
+ Size: fi.Size,
+ IsDir: fi.IsDir,
+ }
+ }
+ if err != nil {
+ st.Error = err.Error()
+ // If this is a deadline exceeded, do not cache.
+ if strings.Contains(st.Error, "Canceled") || strings.Contains(st.Error, "Deadline") {
+ return t, data, st.FileInfo, err
+ }
+ }
+ if chatty {
+ c.Infof("save stat in memcache %q", statkey)
+ }
+ if err := memcache.JSON.Set(c, &memcache.Item{Key: statkey, Object: &st}); err != nil {
+ c.Criticalf("failed to cache %s: %v", statkey, err)
+ }
+
+ // Done!
+ return t, data, st.FileInfo, err
+ }
+
+ c.Criticalf("failed repeatedly in cacheRead")
+ return 0, nil, nil, errors.New("cacheRead loop failed")
+}
+
+func cacheWrite(c appengine.Context, t int64, kind, name string, data []byte) error {
+ mkey := fmt.Sprintf("%d,%s,%s", t, kind, name)
+ if true || chatty {
+ c.Infof("cacheWrite %s %d bytes", mkey, len(data))
+ }
+ err := memcache.Set(c, &memcache.Item{Key: mkey, Value: data})
+ if err != nil {
+ c.Criticalf("cacheWrite memcache.Set %q: %v", mkey, err)
+ }
+ return err
+}
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-06-27 18:46:50 +0000