From 6e2cb00008cbf09e556b00f87603797fcaa47e09 Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 16 Apr 2018 05:37:14 -0700 Subject: Depenancy upgrades and movign to dep. (#8630) --- .../mattermost/rsc/appfs/appfile/main.go | 156 ---- .../mattermost/rsc/appfs/appmount/main.go | 287 ------ .../mattermost/rsc/appfs/client/client.go | 150 ---- vendor/github.com/mattermost/rsc/appfs/fs/fs.go | 273 ------ vendor/github.com/mattermost/rsc/appfs/fs/local.go | 82 -- .../github.com/mattermost/rsc/appfs/proto/data.go | 55 -- .../github.com/mattermost/rsc/appfs/server/app.go | 982 --------------------- 7 files changed, 1985 deletions(-) delete mode 100644 vendor/github.com/mattermost/rsc/appfs/appfile/main.go delete mode 100644 vendor/github.com/mattermost/rsc/appfs/appmount/main.go delete mode 100644 vendor/github.com/mattermost/rsc/appfs/client/client.go delete mode 100644 vendor/github.com/mattermost/rsc/appfs/fs/fs.go delete mode 100644 vendor/github.com/mattermost/rsc/appfs/fs/local.go delete mode 100644 vendor/github.com/mattermost/rsc/appfs/proto/data.go delete mode 100644 vendor/github.com/mattermost/rsc/appfs/server/app.go (limited to 'vendor/github.com/mattermost/rsc/appfs') diff --git a/vendor/github.com/mattermost/rsc/appfs/appfile/main.go b/vendor/github.com/mattermost/rsc/appfs/appfile/main.go deleted file mode 100644 index 6d77df9aa..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/appfile/main.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// appfile is a command-line interface to an appfs file system. -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "log" - "os" - - "github.com/mattermost/rsc/appfs/client" - "github.com/mattermost/rsc/keychain" -) - -var c client.Client - -func init() { - flag.StringVar(&c.Host, "h", "localhost:8080", "app serving host") - flag.StringVar(&c.User, "u", "", "user name") - flag.StringVar(&c.Password, "p", "", "password") -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage: appfile [-h host] cmd args...\n") - fmt.Fprintf(os.Stderr, "\n") - fmt.Fprintf(os.Stderr, "Commands are:\n") - for _, c := range cmd { - fmt.Fprintf(os.Stderr, "\t%s\n", c.name) - } - os.Exit(2) -} - -func main() { - flag.Usage = usage - flag.Parse() - args := flag.Args() - if len(args) == 0 { - usage() - } - - if c.Password == "" { - var err error - c.User, c.Password, err = keychain.UserPasswd(c.Host, "") - if err != nil { - fmt.Fprintf(os.Stderr, "unable to obtain user and password: %s\n", err) - os.Exit(2) - } - } - - name, args := args[0], args[1:] - for _, c := range cmd { - if name == c.name { - switch c.arg { - case 0, 1: - if len(args) != c.arg { - if c.arg == 0 { - fmt.Fprintf(os.Stderr, "%s takes no arguments\n", name) - os.Exit(2) - } - fmt.Fprintf(os.Stderr, "%s requires 1 argument\n", name) - os.Exit(2) - } - case 2: - if len(args) == 0 { - fmt.Fprintf(os.Stderr, "%s requires at least 1 argument\n", name) - os.Exit(2) - } - } - c.fn(args) - return - } - } - fmt.Fprintf(os.Stderr, "unknown command %s\n", name) - os.Exit(2) -} - -var cmd = []struct { - name string - fn func([]string) - arg int -}{ - {"mkdir", mkdir, 2}, - {"write", write, 1}, - {"read", read, 2}, - {"mkfs", mkfs, 0}, - {"stat", stat, 2}, -} - -func mkdir(args []string) { - for _, name := range args { - if err := c.Create(name, true); err != nil { - log.Printf("mkdir %s: %v", name, err) - } - } -} - -func write(args []string) { - name := args[0] - data, err := ioutil.ReadAll(os.Stdin) - if err != nil { - log.Printf("reading stdin: %v", err) - return - } - c.Create(name, false) - if err := c.Write(name, data); err != nil { - log.Printf("write %s: %v", name, err) - } -} - -func read(args []string) { - for _, name := range args { - fi, err := c.Stat(name) - if err != nil { - log.Printf("stat %s: %v", name, err) - continue - } - if fi.IsDir { - dirs, err := c.ReadDir(name) - if err != nil { - log.Printf("read %s: %v", name, err) - continue - } - for _, fi := range dirs { - fmt.Printf("%+v\n", *fi) - } - } else { - data, err := c.Read(name) - if err != nil { - log.Printf("read %s: %v", name, err) - continue - } - os.Stdout.Write(data) - } - } -} - -func mkfs([]string) { - if err := c.Mkfs(); err != nil { - log.Printf("mkfs: %v", err) - } -} - -func stat(args []string) { - for _, name := range args { - fi, err := c.Stat(name) - if err != nil { - log.Printf("stat %s: %v", name, err) - continue - } - fmt.Printf("%+v\n", *fi) - } -} diff --git a/vendor/github.com/mattermost/rsc/appfs/appmount/main.go b/vendor/github.com/mattermost/rsc/appfs/appmount/main.go deleted file mode 100644 index 2c9f867d3..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/appmount/main.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// appmount mounts an appfs file system. -package main - -import ( - "bytes" - "encoding/gob" - "flag" - "fmt" - "log" - "os" - "os/exec" - "path" - "strings" - "syscall" - "time" - "sync" - "runtime" - - "github.com/mattermost/rsc/appfs/client" - "github.com/mattermost/rsc/appfs/proto" - "github.com/mattermost/rsc/fuse" - "github.com/mattermost/rsc/keychain" -) - -var usageMessage = `usage: appmount [-h host] [-u user] [-p password] /mnt - -Appmount mounts the appfs file system on the named mount point. - -The default host is localhost:8080. -` - -// Shared between master and slave. -var z struct { - Client client.Client - Debug *bool - Mtpt string -} - -var fc *fuse.Conn -var cl = &z.Client - -func init() { - flag.StringVar(&cl.Host, "h", "localhost:8080", "app serving host") - flag.StringVar(&cl.User, "u", "", "user name") - flag.StringVar(&cl.Password, "p", "", "password") - z.Debug = flag.Bool("debug", false, "") -} - -func usage() { - fmt.Fprint(os.Stderr, usageMessage) - os.Exit(2) -} - -func main() { - log.SetFlags(0) - - if len(os.Args) == 2 && os.Args[1] == "MOUNTSLAVE" { - mountslave() - return - } - - flag.Usage = usage - flag.Parse() - args := flag.Args() - if len(args) == 0 { - usage() - } - z.Mtpt = args[0] - - if cl.Password == "" { - var err error - cl.User, cl.Password, err = keychain.UserPasswd(cl.Host, "") - if err != nil { - fmt.Fprintf(os.Stderr, "unable to obtain user and password: %s\n", err) - os.Exit(2) - } - } - - if _, err := cl.Stat("/"); err != nil { - log.Fatal(err) - } - - // Run in child so that we can exit once child is running. - r, w, err := os.Pipe() - if err != nil { - log.Fatal(err) - } - - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - enc.Encode(&z) - - cmd := exec.Command(os.Args[0], "MOUNTSLAVE") - cmd.Stdin = &buf - cmd.Stdout = w - cmd.Stderr = os.Stderr - if err := cmd.Start(); err != nil { - log.Fatalf("mount process: %v", err) - } - w.Close() - - ok := make([]byte, 10) - n, _ := r.Read(ok) - if n != 2 || string(ok[0:2]) != "OK" { - os.Exit(1) - } - - fmt.Fprintf(os.Stderr, "mounted on %s\n", z.Mtpt) -} - -func mountslave() { - stdout, _ := syscall.Dup(1) - syscall.Dup2(2, 1) - - r := gob.NewDecoder(os.Stdin) - if err := r.Decode(&z); err != nil { - log.Fatalf("gob decode: %v", err) - } - - fc, err := fuse.Mount(z.Mtpt) - if err != nil { - log.Fatal(err) - } - defer exec.Command("umount", z.Mtpt).Run() - - if *z.Debug { - fuse.Debugf = log.Printf - } - - syscall.Write(stdout, []byte("OK")) - syscall.Close(stdout) - fc.Serve(FS{}) -} - -type FS struct{} - -func (FS) Root() (fuse.Node, fuse.Error) { - return file("/") -} - -type File struct { - Name string - FileInfo *proto.FileInfo - Data []byte -} - -type statEntry struct { - fi *proto.FileInfo - err error - t time.Time -} - -var statCache struct { - mu sync.Mutex - m map[string] statEntry -} - -func stat(name string) (*proto.FileInfo, error) { - if runtime.GOOS == "darwin" && strings.Contains(name, "/._") { - // Mac resource forks - return nil, fmt.Errorf("file not found") - } - statCache.mu.Lock() - e, ok := statCache.m[name] - statCache.mu.Unlock() - if ok && time.Since(e.t) < 2*time.Minute { - return e.fi, e.err - } - fi, err := cl.Stat(name) - saveStat(name, fi, err) - return fi, err -} - -func saveStat(name string, fi *proto.FileInfo, err error) { - if *z.Debug { -if fi != nil { - fmt.Fprintf(os.Stderr, "savestat %s %+v\n", name, *fi) -} else { - fmt.Fprintf(os.Stderr, "savestat %s %v\n", name, err) -} - } - statCache.mu.Lock() - if statCache.m == nil { - statCache.m = make(map[string]statEntry) - } - statCache.m[name] = statEntry{fi, err, time.Now()} - statCache.mu.Unlock() -} - -func delStat(name string) { - statCache.mu.Lock() - if statCache.m != nil { - delete(statCache.m, name) - } - statCache.mu.Unlock() -} - -func file(name string) (fuse.Node, fuse.Error) { - fi, err := stat(name) - if err != nil { - if strings.Contains(err.Error(), "no such entity") { - return nil, fuse.ENOENT - } - if *z.Debug { - log.Printf("stat %s: %v", name, err) - } - return nil, fuse.EIO - } - return &File{name, fi, nil}, nil -} - -func (f *File) Attr() (attr fuse.Attr) { - fi := f.FileInfo - attr.Mode = 0666 - if fi.IsDir { - attr.Mode |= 0111 | os.ModeDir - } - attr.Mtime = fi.ModTime - attr.Size = uint64(fi.Size) - return -} - -func (f *File) Lookup(name string, intr fuse.Intr) (fuse.Node, fuse.Error) { - return file(path.Join(f.Name, name)) -} - -func (f *File) ReadAll(intr fuse.Intr) ([]byte, fuse.Error) { - data, err := cl.Read(f.Name) - if err != nil { - log.Printf("read %s: %v", f.Name, err) - return nil, fuse.EIO - } - return data, nil -} - -func (f *File) ReadDir(intr fuse.Intr) ([]fuse.Dirent, fuse.Error) { - fis, err := cl.ReadDir(f.Name) - if err != nil { - log.Printf("read %s: %v", f.Name, err) - return nil, fuse.EIO - } - var dirs []fuse.Dirent - for _, fi := range fis { - saveStat(path.Join(f.Name, fi.Name), fi, nil) - dirs = append(dirs, fuse.Dirent{Name: fi.Name}) - } - return dirs, nil -} - -func (f *File) WriteAll(data []byte, intr fuse.Intr) fuse.Error { - defer delStat(f.Name) - if err := cl.Write(f.Name[1:], data); err != nil { - log.Printf("write %s: %v", f.Name, err) - return fuse.EIO - } - return nil -} - -func (f *File) Mkdir(req *fuse.MkdirRequest, intr fuse.Intr) (fuse.Node, fuse.Error) { - defer delStat(f.Name) - p := path.Join(f.Name, req.Name) - if err := cl.Create(p[1:], true); err != nil { - log.Printf("mkdir %s: %v", p, err) - return nil, fuse.EIO - } - delStat(p) - return file(p) -} - -func (f *File) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fuse.Intr) (fuse.Node, fuse.Handle, fuse.Error) { - defer delStat(f.Name) - p := path.Join(f.Name, req.Name) - if err := cl.Create(p[1:], false); err != nil { - log.Printf("create %s: %v", p, err) - return nil, nil, fuse.EIO - } - delStat(p) - n, err := file(p) - if err != nil { - return nil, nil, err - } - return n, n, nil -} diff --git a/vendor/github.com/mattermost/rsc/appfs/client/client.go b/vendor/github.com/mattermost/rsc/appfs/client/client.go deleted file mode 100644 index a1deb291d..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/client/client.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package client implements a basic appfs client. -package client - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "strings" - "time" - - "github.com/mattermost/rsc/appfs/proto" -) - -type Client struct { - Host string - User string - Password string -} - -func (c *Client) url(op, path string) string { - scheme := "https" - if strings.HasPrefix(c.Host, "localhost:") { - scheme = "http" - } - if strings.HasSuffix(op, "/") && strings.HasPrefix(path, "/") { - path = path[1:] - } - return scheme + "://"+ c.User + ":" + c.Password + "@" + c.Host + op + path -} - -func (c *Client) do(u string) error { - _, err := c.get(u) - return err -} - -func (c *Client) get(u string) ([]byte, error) { - tries := 0 - for { - r, err := http.Get(u) - if err != nil { - return nil, err - } - defer r.Body.Close() - data, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, err - } - if r.StatusCode != 200 { - if r.StatusCode == 500 { - if tries++; tries < 3 { - fmt.Printf("%s %s; sleeping\n", r.Status, data) - time.Sleep(5*time.Second) - continue - } - } - return nil, fmt.Errorf("%s %s", r.Status, data) - } - return data, nil - } - panic("unreachable") -} - -func (c *Client) post(u string, data []byte) ([]byte, error) { - tries := 0 - for { - r, err := http.Post(u, proto.PostContentType, bytes.NewBuffer(data)) - if err != nil { - return nil, err - } - defer r.Body.Close() - rdata, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, err - } - if r.StatusCode != 200 { - if r.StatusCode == 500 { - if tries++; tries < 3 { - fmt.Printf("%s %s; sleeping\n", r.Status, rdata) - time.Sleep(5*time.Second) - continue - } - } - return nil, fmt.Errorf("%s %s", r.Status, rdata) - } - return rdata, nil - } - panic("unreachable") -} - -func (c *Client) Create(path string, isdir bool) error { - u := c.url(proto.CreateURL, path) - if isdir { - u += "?dir=1" - } - return c.do(u) -} - -func (c *Client) Read(path string) ([]byte, error) { - return c.get(c.url(proto.ReadURL, path)) -} - -func (c *Client) Write(path string, data []byte) error { - u := c.url(proto.WriteURL, path) - _, err := c.post(u, data) - return err -} - -func (c *Client) Mkfs() error { - return c.do(c.url(proto.MkfsURL, "")) -} - -func (c *Client) Stat(path string) (*proto.FileInfo, error) { - data, err := c.get(c.url(proto.StatURL, path)) - if err != nil { - return nil, err - } - var fi proto.FileInfo - if err := json.Unmarshal(data, &fi); err != nil { - return nil, err - } - return &fi, nil -} - -func (c *Client) ReadDir(path string) ([]*proto.FileInfo, error) { - data, err := c.Read(path) - if err != nil { - return nil, err - } - dec := json.NewDecoder(bytes.NewBuffer(data)) - var out []*proto.FileInfo - for { - var fi proto.FileInfo - err := dec.Decode(&fi) - if err == io.EOF { - break - } - if err != nil { - return out, err - } - out = append(out, &fi) - } - return out, nil -} diff --git a/vendor/github.com/mattermost/rsc/appfs/fs/fs.go b/vendor/github.com/mattermost/rsc/appfs/fs/fs.go deleted file mode 100644 index ac6657393..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/fs/fs.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package fs is an indirection layer, allowing code to use a -// file system without knowing whether it is the host file system -// (running without App Engine) or the datastore-based app -// file system (running on App Engine). -// -// When compiled locally, fs refers to files in the local file system, -// and the cache saves nothing. -// -// When compiled for App Engine, fs uses the appfs file system -// and the memcache-based cache. -package fs - -import ( - "bytes" - "encoding/gob" - "fmt" - "io" - "log" - "net/http" - "os" - "time" - - "github.com/mattermost/rsc/appfs/proto" -) - -type AppEngine interface { - NewContext(req *http.Request) interface{} - CacheRead(ctxt interface{}, name, path string) (key interface{}, data []byte, found bool) - CacheWrite(ctxt, key interface{}, data []byte) - Read(ctxt interface{}, path string) ([]byte, *proto.FileInfo, error) - Write(ctxt interface{}, path string, data []byte) error - Remove(ctxt interface{}, path string) error - Mkdir(ctxt interface{}, path string) error - ReadDir(ctxt interface{}, path string) ([]proto.FileInfo, error) - Criticalf(ctxt interface{}, format string, args ...interface{}) - User(ctxt interface{}) string -} - -var ae AppEngine - -func Register(impl AppEngine) { - ae = impl -} - -// Root is the root of the local file system. It has no effect on App Engine. -var Root = "." - -// A Context is an opaque context that is needed to perform file system -// operations. Each context is associated with a single HTTP request. -type Context struct { - context - ae interface{} -} - -// NewContext returns a context associated with the given HTTP request. -func NewContext(req *http.Request) *Context { - if ae != nil { - ctxt := ae.NewContext(req) - return &Context{ae: ctxt} - } - return newContext(req) -} - -// A CacheKey is an opaque cache key that can be used to store new entries -// in the cache. To ensure that the cache remains consistent with the underlying -// file system, the correct procedure is: -// -// 1. Use CacheRead (or CacheLoad) to attempt to load the entry. If it succeeds, use it. -// If not, continue, saving the CacheKey. -// -// 2. Read from the file system and construct the entry that would have -// been in the cache. In order to be consistent, all the file system reads -// should only refer to parts of the file system in the tree rooted at the path -// passed to CacheRead. -// -// 3. Save the entry using CacheWrite (or CacheStore), using the key that was -// created by the CacheRead (or CacheLoad) executed before reading from the -// file system. -// -type CacheKey struct { - cacheKey - ae interface{} -} - -// CacheRead reads from cache the entry with the given name and path. -// The path specifies the scope of information stored in the cache entry. -// An entry is invalidated by a write to any location in the file tree rooted at path. -// The name is an uninterpreted identifier to distinguish the cache entry -// from other entries using the same path. -// -// If it finds a cache entry, CacheRead returns the data and found=true. -// If it does not find a cache entry, CacheRead returns data=nil and found=false. -// Either way, CacheRead returns an appropriate cache key for storing to the -// cache entry using CacheWrite. -func (c *Context) CacheRead(name, path string) (ckey CacheKey, data []byte, found bool) { - if ae != nil { - key, data, found := ae.CacheRead(c.ae, name, path) - return CacheKey{ae: key}, data, found - } - return c.cacheRead(ckey, path) -} - -// CacheLoad uses CacheRead to load gob-encoded data and decodes it into value. -func (c *Context) CacheLoad(name, path string, value interface{}) (ckey CacheKey, found bool) { - ckey, data, found := c.CacheRead(name, path) - if found { - if err := gob.NewDecoder(bytes.NewBuffer(data)).Decode(value); err != nil { - c.Criticalf("gob Decode: %v", err) - found = false - } - } - return -} - -// CacheWrite writes an entry to the cache with the given key, path, and data. -// The cache entry will be invalidated the next time the file tree rooted at path is -// modified in anyway. -func (c *Context) CacheWrite(ckey CacheKey, data []byte) { - if ae != nil { - ae.CacheWrite(c.ae, ckey.ae, data) - return - } - c.cacheWrite(ckey, data) -} - -// CacheStore uses CacheWrite to save the gob-encoded form of value. -func (c *Context) CacheStore(ckey CacheKey, value interface{}) { - var buf bytes.Buffer - if err := gob.NewEncoder(&buf).Encode(value); err != nil { - c.Criticalf("gob Encode: %v", err) - return - } - c.CacheWrite(ckey, buf.Bytes()) -} - -// Read returns the data associated with the file named by path. -// It is a copy and can be modified without affecting the file. -func (c *Context) Read(path string) ([]byte, *proto.FileInfo, error) { - if ae != nil { - return ae.Read(c.ae, path) - } - return c.read(path) -} - -// Write replaces the data associated with the file named by path. -func (c *Context) Write(path string, data []byte) error { - if ae != nil { - return ae.Write(c.ae, path, data) - } - return c.write(path, data) -} - -// Remove removes the file named by path. -func (c *Context) Remove(path string) error { - if ae != nil { - return ae.Remove(c.ae, path) - } - return c.remove(path) -} - -// Mkdir creates a directory with the given path. -// If the path already exists and is a directory, Mkdir returns no error. -func (c *Context) Mkdir(path string) error { - if ae != nil { - return ae.Mkdir(c.ae, path) - } - return c.mkdir(path) -} - -// ReadDir returns the contents of the directory named by the path. -func (c *Context) ReadDir(path string) ([]proto.FileInfo, error) { - if ae != nil { - return ae.ReadDir(c.ae, path) - } - return c.readdir(path) -} - -// ServeFile serves the named file as the response to the HTTP request. -func (c *Context) ServeFile(w http.ResponseWriter, req *http.Request, name string) { - root := &httpFS{c, name} - http.FileServer(root).ServeHTTP(w, req) -} - -// Criticalf logs the message at critical priority. -func (c *Context) Criticalf(format string, args ...interface{}) { - if ae != nil { - ae.Criticalf(c.ae, format, args...) - } - log.Printf(format, args...) -} - -// User returns the name of the user running the request. -func (c *Context) User() string { - if ae != nil { - return ae.User(c.ae) - } - return os.Getenv("USER") -} - -type httpFS struct { - c *Context - name string -} - -type httpFile struct { - data []byte - fi *proto.FileInfo - off int -} - -func (h *httpFS) Open(_ string) (http.File, error) { - data, fi, err := h.c.Read(h.name) - if err != nil { - return nil, err - } - return &httpFile{data, fi, 0}, nil -} - -func (f *httpFile) Close() error { - return nil -} - -type fileInfo struct { - p *proto.FileInfo -} - -func (f *fileInfo) IsDir() bool { return f.p.IsDir } -func (f *fileInfo) Name() string { return f.p.Name } -func (f *fileInfo) ModTime() time.Time { return f.p.ModTime } -func (f *fileInfo) Size() int64 { return f.p.Size } -func (f *fileInfo) Sys() interface{} { return f.p } -func (f *fileInfo) Mode() os.FileMode { - if f.p.IsDir { - return os.ModeDir | 0777 - } - return 0666 -} - -func (f *httpFile) Stat() (os.FileInfo, error) { - return &fileInfo{f.fi}, nil -} - -func (f *httpFile) Readdir(count int) ([]os.FileInfo, error) { - return nil, fmt.Errorf("no directory") -} - -func (f *httpFile) Read(data []byte) (int, error) { - if f.off >= len(f.data) { - return 0, io.EOF - } - n := copy(data, f.data[f.off:]) - f.off += n - return n, nil -} - -func (f *httpFile) Seek(offset int64, whence int) (int64, error) { - off := int(offset) - if int64(off) != offset { - return 0, fmt.Errorf("invalid offset") - } - switch whence { - case 1: - off += f.off - case 2: - off += len(f.data) - } - f.off = off - return int64(off), nil -} diff --git a/vendor/github.com/mattermost/rsc/appfs/fs/local.go b/vendor/github.com/mattermost/rsc/appfs/fs/local.go deleted file mode 100644 index c78b35b64..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/fs/local.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package fs - -import ( - "io/ioutil" - "net/http" - "os" - "path/filepath" - - "github.com/mattermost/rsc/appfs/proto" -) - -type context struct{} - -type cacheKey struct{} - -func newContext(req *http.Request) *Context { - return &Context{} -} - -func (*context) cacheRead(ckey CacheKey, path string) (CacheKey, []byte, bool) { - return ckey, nil, false -} - -func (*context) cacheWrite(ckey CacheKey, data []byte) { -} - -func (*context) read(path string) ([]byte, *proto.FileInfo, error) { - p := filepath.Join(Root, path) - dir, err := os.Stat(p) - if err != nil { - return nil, nil, err - } - fi := &proto.FileInfo{ - Name: dir.Name(), - ModTime: dir.ModTime(), - Size: dir.Size(), - IsDir: dir.IsDir(), - } - data, err := ioutil.ReadFile(p) - return data, fi, err -} - -func (*context) write(path string, data []byte) error { - p := filepath.Join(Root, path) - return ioutil.WriteFile(p, data, 0666) -} - -func (*context) remove(path string) error { - p := filepath.Join(Root, path) - return os.Remove(p) -} - -func (*context) mkdir(path string) error { - p := filepath.Join(Root, path) - fi, err := os.Stat(p) - if err == nil && fi.IsDir() { - return nil - } - return os.Mkdir(p, 0777) -} - -func (*context) readdir(path string) ([]proto.FileInfo, error) { - p := filepath.Join(Root, path) - dirs, err := ioutil.ReadDir(p) - if err != nil { - return nil, err - } - var out []proto.FileInfo - for _, dir := range dirs { - out = append(out, proto.FileInfo{ - Name: dir.Name(), - ModTime: dir.ModTime(), - Size: dir.Size(), - IsDir: dir.IsDir(), - }) - } - return out, nil -} diff --git a/vendor/github.com/mattermost/rsc/appfs/proto/data.go b/vendor/github.com/mattermost/rsc/appfs/proto/data.go deleted file mode 100644 index ac15411a8..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/proto/data.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package proto defines the protocol between appfs client and server. -package proto - -import "time" - -// An Auth appears, JSON-encoded, as the X-Appfs-Auth header line, -// to authenticate a request made to the file server. -// The authentication scheme could be made more sophisticated, but since -// we are already forcing the use of TLS, a plain password is fine for now. -type Auth struct { - Password string -} - -// GET /.appfs/stat/path returns the metadata for a file or directory, -// a JSON-encoded FileInfo. -const StatURL = "/.appfs/stat/" - -// GET /.appfs/read/path returns the content of the file or directory. -// The body of the response is the raw file or directory content. -// The content of a directory is a sequence of JSON-encoded FileInfo. -const ReadURL = "/.appfs/read/" - -// POST to /.appfs/write/path writes new data to a file. -// The X-Appfs-SHA1 header is the SHA1 hash of the data. -// The body of the request is the raw file content. -const WriteURL = "/.appfs/write/" - -// POST to /.appfs/mount initializes the file system if it does not -// yet exist in the datastore. -const MkfsURL = "/.appfs/mkfs" - -// POST to /.appfs/create/path creates a new file or directory. -// The named path must not already exist; its parent must exist. -// The query parameter dir=1 indicates that a directory should be created. -const CreateURL = "/.appfs/create/" - -// POST to /.appfs/remove/path removes the file or directory. -// A directory must be empty to be removed. -const RemoveURL = "/.appfs/remove/" - -// A FileInfo is a directory entry. -type FileInfo struct { - Name string // final path element - ModTime time.Time - Size int64 - IsDir bool -} - -// PostContentType is the Content-Type for POSTed data. -// There is no encoding or framing: it is just raw data bytes. -const PostContentType = "x-appfs/raw" diff --git a/vendor/github.com/mattermost/rsc/appfs/server/app.go b/vendor/github.com/mattermost/rsc/appfs/server/app.go deleted file mode 100644 index 9486eac41..000000000 --- a/vendor/github.com/mattermost/rsc/appfs/server/app.go +++ /dev/null @@ -1,982 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package server implements an appfs server backed by the -// App Engine datastore. -package server - -import ( - "bytes" - "crypto/sha1" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "path" - "strconv" - "strings" - "time" - - "appengine" - "appengine/datastore" - "appengine/memcache" - "appengine/user" - - "github.com/mattermost/rsc/appfs/fs" - "github.com/mattermost/rsc/appfs/proto" -) - -const pwFile = "/.password" -var chatty = false - -func init() { - handle(proto.ReadURL, (*request).read) - handle(proto.WriteURL, (*request).write) - handle(proto.StatURL, (*request).stat) - handle(proto.MkfsURL, (*request).mkfs) - handle(proto.CreateURL, (*request).create) - handle(proto.RemoveURL, (*request).remove) -} - -type request struct { - w http.ResponseWriter - req *http.Request - c appengine.Context - name string - mname string - key *datastore.Key -} - -func auth(r *request) bool { - hdr := r.req.Header.Get("Authorization") - if !strings.HasPrefix(hdr, "Basic ") { - return false - } - data, err := base64.StdEncoding.DecodeString(hdr[6:]) - if err != nil { - return false - } - i := bytes.IndexByte(data, ':') - if i < 0 { - return false - } - user, passwd := string(data[:i]), string(data[i+1:]) - - _, data, err = read(r.c, pwFile) - if err != nil { - r.c.Errorf("reading %s: %v", pwFile, err) - if _, err := mkfs(r.c); err != nil { - r.c.Errorf("creating fs: %v", err) - } - _, data, err = read(r.c, pwFile) - if err != nil { - r.c.Errorf("reading %s again: %v", pwFile, err) - return false - } - } - - lines := strings.Split(string(data), "\n") - for _, line := range lines { - if strings.HasPrefix(line, "#") { - continue - } - f := strings.Fields(line) - if len(f) < 3 { - continue - } - if f[0] == user { - return hash(f[1]+passwd) == f[2] - } - } - return false -} - -func hash(s string) string { - h := sha1.New() - h.Write([]byte(s)) - return fmt.Sprintf("%x", h.Sum(nil)) -} - -func handle(prefix string, f func(*request)) { - http.HandleFunc(prefix, func(w http.ResponseWriter, req *http.Request) { - c := appengine.NewContext(req) - r := &request{ - w: w, - req: req, - c: c, - } - - if strings.HasSuffix(prefix, "/") { - r.name, r.mname, r.key = mangle(c, req.URL.Path[len(prefix)-1:]) - } else { - req.URL.Path = "/" - } - defer func() { - if err := recover(); err != nil { - w.WriteHeader(http.StatusConflict) - fmt.Fprintf(w, "%s\n", err) - } - }() - - if !auth(r) { - w.Header().Set("WWW-Authenticate", "Basic realm=\"appfs\"") - http.Error(w, "Need auth", http.StatusUnauthorized) - return - } - - f(r) - }) -} - -func mangle(c appengine.Context, name string) (string, string, *datastore.Key) { - name = path.Clean("/" + name) - n := strings.Count(name, "/") - if name == "/" { - n = 0 - } - mname := fmt.Sprintf("%d%s", n, name) - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - key := datastore.NewKey(c, "FileInfo", mname, 0, root) - return name, mname, key -} - -type FileInfo struct { - Path string // mangled path - Name string - Qid int64 // assigned unique id number - Seq int64 // modification sequence number in file tree - ModTime time.Time - Size int64 - IsDir bool -} - -type FileData struct { - Data []byte -} - -func stat(c appengine.Context, name string) (*FileInfo, error) { - var fi FileInfo - name, _, key := mangle(c, name) - c.Infof("DATASTORE Stat %q", name) - err := datastore.Get(c, key, &fi) - if err != nil { - return nil, err - } - return &fi, nil -} - -func (r *request) saveStat(fi *FileInfo) { - jfi, err := json.Marshal(&fi) - if err != nil { - panic(err) - } - r.w.Header().Set("X-Appfs-Stat", string(jfi)) -} - -func (r *request) tx(f func(c appengine.Context) error) { - err := datastore.RunInTransaction(r.c, f, &datastore.TransactionOptions{XG: true}) - if err != nil { - panic(err) - } -} - -func (r *request) stat() { - var fi *FileInfo - r.tx(func(c appengine.Context) error { - fi1, err := stat(c, r.name) - if err != nil { - return err - } - fi = fi1 - return nil - }) - - jfi, err := json.Marshal(&fi) - if err != nil { - panic(err) - } - r.w.Write(jfi) -} - -func read(c appengine.Context, name string) (fi *FileInfo, data []byte, err error) { - name, _, _ = mangle(c, name) - fi1, err := stat(c, name) - if err != nil { - return nil, nil, err - } - if fi1.IsDir { - dt, err := readdir(c, name) - if err != nil { - return nil, nil, err - } - fi = fi1 - data = dt - return fi, data, nil - } - - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root) - var fd FileData - c.Infof("DATASTORE Read %q", name) - if err := datastore.Get(c, dkey, &fd); err != nil { - return nil, nil, err - } - fi = fi1 - data = fd.Data - return fi, data, nil -} - -func (r *request) read() { - var ( - fi *FileInfo - data []byte - ) - r.tx(func(c appengine.Context) error { - var err error - fi, data, err = read(r.c, r.name) - return err - }) - r.saveStat(fi) - r.w.Write(data) -} - -func readdir(c appengine.Context, name string) ([]byte, error) { - name, _, _ = mangle(c, name) - var buf bytes.Buffer - - n := strings.Count(name, "/") - if name == "/" { - name = "" - n = 0 - } - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - first := fmt.Sprintf("%d%s/", n+1, name) - limit := fmt.Sprintf("%d%s0", n+1, name) - c.Infof("DATASTORE ReadDir %q", name) - q := datastore.NewQuery("FileInfo"). - Filter("Path >=", first). - Filter("Path <", limit). - Ancestor(root) - enc := json.NewEncoder(&buf) - it := q.Run(c) - var fi FileInfo - var pfi proto.FileInfo - for { - fi = FileInfo{} - _, err := it.Next(&fi) - if err != nil { - if err == datastore.Done { - break - } - return nil, err - } - pfi = proto.FileInfo{ - Name: fi.Name, - ModTime: fi.ModTime, - Size: fi.Size, - IsDir: fi.IsDir, - } - if err := enc.Encode(&pfi); err != nil { - return nil, err - } - } - - return buf.Bytes(), nil -} - -func readdirRaw(c appengine.Context, name string) ([]proto.FileInfo, error) { - name, _, _ = mangle(c, name) - n := strings.Count(name, "/") - if name == "/" { - name = "" - n = 0 - } - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - first := fmt.Sprintf("%d%s/", n+1, name) - limit := fmt.Sprintf("%d%s0", n+1, name) - c.Infof("DATASTORE ReadDir %q", name) - q := datastore.NewQuery("FileInfo"). - Filter("Path >=", first). - Filter("Path <", limit). - Ancestor(root) - it := q.Run(c) - var fi FileInfo - var pfi proto.FileInfo - var out []proto.FileInfo - for { - fi = FileInfo{} - _, err := it.Next(&fi) - if err != nil { - if err == datastore.Done { - break - } - return nil, err - } - pfi = proto.FileInfo{ - Name: fi.Name, - ModTime: fi.ModTime, - Size: fi.Size, - IsDir: fi.IsDir, - } - out = append(out, pfi) - } -println("READDIR", name, len(out)) - return out, nil -} - - -var initPasswd = `# Password file -# This file controls access to the server. -# The format is lines of space-separated fields: -# user salt pwhash -# The pwhash is the SHA1 of the salt string concatenated with the password. - -# user=dummy password=dummy (replace with your own entries) -dummy 12345 faa863c7d3d41893f80165c704b714d5e31bdd3b -` - -func (r *request) mkfs() { - var fi *FileInfo - r.tx(func(c appengine.Context) error { - var err error - fi, err = mkfs(c) - return err - }) - r.saveStat(fi) -} - -func mkfs(c appengine.Context) (fi *FileInfo, err error) { - fi1, err := stat(c, "/") - if err == nil { - return fi1, nil - } - - // Root needs to be created. - // Probably root key does too. - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - _, err = datastore.Put(c, root, &struct{}{}) - if err != nil { - return nil, fmt.Errorf("mkfs put root: %s", err) - } - - // Entry for /. - _, mpath, key := mangle(c, "/") - fi3 := FileInfo{ - Path: mpath, - Name: "/", - Seq: 2, // 2, not 1, because we're going to write password file with #2 - Qid: 1, - ModTime: time.Now(), - Size: 0, - IsDir: true, - } - _, err = datastore.Put(c, key, &fi3) - if err != nil { - return nil, fmt.Errorf("mkfs put /: %s", err) - } - - /* - * Would like to use this code but App Engine apparently - * does not let Get observe the effect of a Put in the same - * transaction. What planet does that make sense on? - * Instead, we have to execute just the datastore writes that this - * sequence would. - * - _, err = create(c, pwFile, false) - if err != nil { - return nil, fmt.Errorf("mkfs create .password: %s", err) - } - _, err = write(c, pwFile, []byte(initPasswd)) - if err != nil { - return nil, fmt.Errorf("mkfs write .password: %s", err) - } - * - */ - - { - name, mname, key := mangle(c, pwFile) - - // Create data object. - dataKey := int64(2) - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - dkey := datastore.NewKey(c, "FileData", "", dataKey, root) - _, err := datastore.Put(c, dkey, &FileData{[]byte(initPasswd)}) - if err != nil { - return nil, err - } - - // Create new directory entry. - _, elem := path.Split(name) - fi1 = &FileInfo{ - Path: mname, - Name: elem, - Qid: 2, - Seq: 2, - ModTime: time.Now(), - Size: int64(len(initPasswd)), - IsDir: false, - } - if _, err := datastore.Put(c, key, fi1); err != nil { - return nil, err - } - } - - return &fi3, nil -} - -func (r *request) write() { - data, err := ioutil.ReadAll(r.req.Body) - if err != nil { - panic(err) - } - - var fi *FileInfo - var seq int64 - r.tx(func(c appengine.Context) error { - var err error - fi, seq, err = write(r.c, r.name, data) - return err - }) - updateCacheTime(r.c, seq) - r.saveStat(fi) -} - -func write(c appengine.Context, name string, data []byte) (*FileInfo, int64, error) { - name, _, key := mangle(c, name) - - // Check that file exists and is not a directory. - fi1, err := stat(c, name) - if err != nil { - return nil, 0, err - } - if fi1.IsDir { - return nil, 0, fmt.Errorf("cannot write to directory") - } - - // Fetch and increment root sequence number. - rfi, err := stat(c, "/") - if err != nil { - return nil, 0, err - } - rfi.Seq++ - - // Write data. - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - dkey := datastore.NewKey(c, "FileData", "", fi1.Qid, root) - fd := &FileData{data} - if _, err := datastore.Put(c, dkey, fd); err != nil { - return nil, 0, err - } - - // Update directory entry. - fi1.Seq = rfi.Seq - fi1.Size = int64(len(data)) - fi1.ModTime = time.Now() - if _, err := datastore.Put(c, key, fi1); err != nil { - return nil, 0, err - } - - // Update sequence numbers all the way to the root. - if err := updateSeq(c, name, rfi.Seq, 1); err != nil { - return nil, 0, err - } - - return fi1, rfi.Seq, nil -} - -func updateSeq(c appengine.Context, name string, seq int64, skip int) error { - p := path.Clean(name) - for i := 0; ; i++ { - if i >= skip { - _, _, key := mangle(c, p) - var fi FileInfo - if err := datastore.Get(c, key, &fi); err != nil { - return err - } - fi.Seq = seq - if _, err := datastore.Put(c, key, &fi); err != nil { - return err - } - } - if p == "/" { - break - } - p, _ = path.Split(p) - p = path.Clean(p) - } - return nil -} - -func (r *request) remove() { - panic("remove not implemented") -} - -func (r *request) create() { - var fi *FileInfo - var seq int64 - isDir := r.req.FormValue("dir") == "1" - r.tx(func(c appengine.Context) error { - var err error - fi, seq, err = create(r.c, r.name, isDir, nil) - return err - }) - updateCacheTime(r.c, seq) - r.saveStat(fi) -} - -func create(c appengine.Context, name string, isDir bool, data []byte) (*FileInfo, int64, error) { - name, mname, key := mangle(c, name) - - // File must not exist. - fi1, err := stat(c, name) - if err == nil { - return nil, 0, fmt.Errorf("file already exists") - } - if err != datastore.ErrNoSuchEntity { - return nil, 0, err - } - - // Parent must exist and be a directory. - p, _ := path.Split(name) - fi2, err := stat(c, p) - if err != nil { - if err == datastore.ErrNoSuchEntity { - return nil, 0, fmt.Errorf("parent directory %q does not exist", p) - } - return nil, 0, err - } - if !fi2.IsDir { - return nil, 0, fmt.Errorf("parent %q is not a directory", p) - } - - // Fetch and increment root sequence number. - rfi, err := stat(c, "/") - if err != nil { - return nil, 0, err - } - rfi.Seq++ - - var dataKey int64 - // Create data object. - if !isDir { - dataKey = rfi.Seq - root := datastore.NewKey(c, "RootKey", "v2:", 0, nil) - dkey := datastore.NewKey(c, "FileData", "", dataKey, root) - _, err := datastore.Put(c, dkey, &FileData{data}) - if err != nil { - return nil, 0, err - } - } - - // Create new directory entry. - _, elem := path.Split(name) - fi1 = &FileInfo{ - Path: mname, - Name: elem, - Qid: rfi.Seq, - Seq: rfi.Seq, - ModTime: time.Now(), - Size: int64(len(data)), - IsDir: isDir, - } - if _, err := datastore.Put(c, key, fi1); err != nil { - return nil, 0, err - } - - // Update sequence numbers all the way to root, - // but skip entry we just wrote. - if err := updateSeq(c, name, rfi.Seq, 1); err != nil { - return nil, 0, err - } - - return fi1, rfi.Seq, nil -} - -// Implementation of fs.AppEngine. - -func init() { - fs.Register(ae{}) -} - -type ae struct{} - -func tx(c interface{}, f func(c appengine.Context) error) error { - return datastore.RunInTransaction(c.(appengine.Context), f, &datastore.TransactionOptions{XG: true}) -} - -func (ae) NewContext(req *http.Request) interface{} { - return appengine.NewContext(req) -} - -func (ae) User(ctxt interface{}) string { - c := ctxt.(appengine.Context) - u := user.Current(c) - if u == nil { - return "?" - } - return u.String() -} - -type cacheKey struct { - t int64 - name string -} - -func (ae) CacheRead(ctxt interface{}, name, path string) (key interface{}, data []byte, found bool) { - c := ctxt.(appengine.Context) - t, data, _, err := cacheRead(c, "cache", name, path) - return &cacheKey{t, name}, data, err == nil -} - -func (ae) CacheWrite(ctxt, key interface{}, data []byte) { - c := ctxt.(appengine.Context) - k := key.(*cacheKey) - cacheWrite(c, k.t, "cache", k.name, data) -} - -func (ae ae) Read(ctxt interface{}, name string) (data []byte, pfi *proto.FileInfo, err error) { - c := ctxt.(appengine.Context) - name = path.Clean("/"+name) - if chatty { - c.Infof("AE Read %s", name) - } - _, data, pfi, err = cacheRead(c, "data", name, name) - if err != nil { - err = fmt.Errorf("Read %q: %v", name, err) - } - return -} - -func (ae) Write(ctxt interface{}, path string, data []byte) error { - var seq int64 - err := tx(ctxt, func(c appengine.Context) error { - _, err := stat(c, path) - if err != nil { - _, seq, err = create(c, path, false, data) - } else { - _, seq, err = write(c, path, data) - } - return err - }) - if seq != 0 { - updateCacheTime(ctxt.(appengine.Context), seq) - } - if err != nil { - err = fmt.Errorf("Write %q: %v", path, err) - } - return err -} - -func (ae) Remove(ctxt interface{}, path string) error { - return fmt.Errorf("remove not implemented") -} - -func (ae) Mkdir(ctxt interface{}, path string) error { - var seq int64 - err := tx(ctxt, func(c appengine.Context) error { - var err error - _, seq, err = create(c, path, true, nil) - return err - }) - if seq != 0 { - updateCacheTime(ctxt.(appengine.Context), seq) - } - if err != nil { - err = fmt.Errorf("Mkdir %q: %v", path, err) - } - return err -} - -func (ae) Criticalf(ctxt interface{}, format string, args ...interface{}) { - ctxt.(appengine.Context).Criticalf(format, args...) -} - -type readDirCacheEntry struct { - Dir []proto.FileInfo - Error string -} - -func (ae) ReadDir(ctxt interface{}, name string) (dir []proto.FileInfo, err error) { - c := ctxt.(appengine.Context) - name = path.Clean("/"+name) - t, data, _, err := cacheRead(c, "dir", name, name) - if err == nil { - var e readDirCacheEntry - if err := json.Unmarshal(data, &e); err == nil { - if chatty { - c.Infof("cached ReadDir %q", name) - } - if e.Error != "" { - return nil, errors.New(e.Error) - } - return e.Dir, nil - } - c.Criticalf("unmarshal cached dir %q: %v", name) - } - err = tx(ctxt, func(c appengine.Context) error { - var err error - dir, err = readdirRaw(c, name) - return err - }) - var e readDirCacheEntry - e.Dir = dir - if err != nil { - err = fmt.Errorf("ReadDir %q: %v", name, err) - e.Error = err.Error() - } - if data, err := json.Marshal(&e); err != nil { - c.Criticalf("json marshal cached dir: %v", err) - } else { - c.Criticalf("caching dir %q@%d %d bytes", name, t, len(data)) - cacheWrite(c, t, "dir", name, data) - } - return -} - -// Caching of file system data. -// -// The cache stores entries under keys of the form time,space,name, -// where time is the time at which the entry is valid for, space is a name -// space identifier, and name is an arbitrary name. -// -// A key of the form t,mtime,path maps to an integer value giving the -// modification time of the named path at root time t. -// The special key 0,mtime,/ is an integer giving the current time at the root. -// -// A key of the form t,data,path maps to the content of path at time t. -// -// Thus, a read from path should first obtain the root time, -// then obtain the modification time for the path at that root time -// then obtain the data for that path. -// t1 = get(0,mtime,/) -// t2 = get(t1,mtime,path) -// data = get(t2,data,path) -// -// The API allows clients to cache their own data too, with expiry tied to -// the modification time of a particular path (file or directory). To look -// up one of those, we use: -// t1 = get(0,mtime,/) -// t2 = get(t1,mtime,path) -// data = get(t2,clientdata,name) -// -// To store data in the cache, the t1, t2 should be determined before reading -// from datastore. Then the data should be saved under t2. This ensures -// that if a datastore update happens after the read but before the cache write, -// we'll be writing to an entry that will no longer be used (t2). - -const rootMemcacheKey = "0,mtime,/" - -func updateCacheTime(c appengine.Context, seq int64) { - const key = rootMemcacheKey - bseq := []byte(strconv.FormatInt(seq, 10)) - for tries := 0; tries < 10; tries++ { - item, err := memcache.Get(c, key) - if err != nil { - c.Infof("memcache.Get %q: %v", key, err) - err = memcache.Add(c, &memcache.Item{Key: key, Value: bseq}) - if err == nil { - c.Infof("memcache.Add %q %q ok", key, bseq) - return - } - c.Infof("memcache.Add %q %q: %v", key, bseq, err) - } - v, err := strconv.ParseInt(string(item.Value), 10, 64) - if err != nil { - c.Criticalf("memcache.Get %q = %q (%v)", key, item.Value, err) - return - } - if v >= seq { - return - } - item.Value = bseq - err = memcache.CompareAndSwap(c, item) - if err == nil { - c.Infof("memcache.CAS %q %d->%d ok", key, v, seq) - return - } - c.Infof("memcache.CAS %q %d->%d: %v", key, v, seq, err) - } - c.Criticalf("repeatedly failed to update root key") -} - -func cacheTime(c appengine.Context) (t int64, err error) { - const key = rootMemcacheKey - item, err := memcache.Get(c, key) - if err == nil { - v, err := strconv.ParseInt(string(item.Value), 10, 64) - if err == nil { - if chatty { - c.Infof("cacheTime %q = %v", key, v) - } - return v, nil - } - c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err) - memcache.Delete(c, key) - } - fi, err := stat(c, "/") - if err != nil { - c.Criticalf("stat /: %v", err) - return 0, err - } - updateCacheTime(c, fi.Seq) - return fi.Seq, nil -} - -func cachePathTime(c appengine.Context, path string) (t int64, err error) { - t, err = cacheTime(c) - if err != nil { - return 0, err - } - - key := fmt.Sprintf("%d,mtime,%s", t, path) - item, err := memcache.Get(c, key) - if err == nil { - v, err := strconv.ParseInt(string(item.Value), 10, 64) - if err == nil { - if chatty { - c.Infof("cachePathTime %q = %v", key, v) - } - return v, nil - } - c.Criticalf("memcache.Get %q = %q (%v) - deleting", key, item.Value, err) - memcache.Delete(c, key) - } - - var seq int64 - if fi, err := stat(c, path); err == nil { - seq = fi.Seq - } - - c.Infof("cachePathTime save %q = %v", key, seq) - item = &memcache.Item{Key: key, Value: []byte(strconv.FormatInt(seq, 10))} - if err := memcache.Set(c, item); err != nil { - c.Criticalf("memcache.Set %q %q: %v", key, item.Value, err) - } - return seq, nil -} - -type statCacheEntry struct { - FileInfo *proto.FileInfo - Error string -} - -func cacheRead(c appengine.Context, kind, name, path string) (mtime int64, data []byte, pfi *proto.FileInfo, err error) { - for tries := 0; tries < 10; tries++ { - t, err := cachePathTime(c, path) - if err != nil { - return 0, nil, nil, err - } - - key := fmt.Sprintf("%d,%s,%s", t, kind, name) - item, err := memcache.Get(c, key) - var data []byte - if item != nil { - data = item.Value - } - if err != nil { - c.Infof("memcache miss %q %v", key, err) - } else if chatty { - c.Infof("memcache hit %q (%d bytes)", key, len(data)) - } - if kind != "data" { - // Not a file; whatever memcache says is all we have. - return t, data, nil, err - } - - // Load stat from cache (includes negative entry). - statkey := fmt.Sprintf("%d,stat,%s", t, name) - var st statCacheEntry - _, err = memcache.JSON.Get(c, statkey, &st) - if err == nil { - if st.Error != "" { - if chatty { - c.Infof("memcache hit stat error %q %q", statkey, st.Error) - } - err = errors.New(st.Error) - } else { - if chatty { - c.Infof("memcache hit stat %q", statkey) - } - } - if err != nil || data != nil { - return t, data, st.FileInfo, err - } - } - - // Need stat, or maybe stat+data. - var fi *FileInfo - if data != nil { - c.Infof("stat %q", name) - fi, err = stat(c, name) - if err == nil && fi.Seq != t { - c.Criticalf("loaded %s but found stat %d", key, fi.Seq) - continue - } - } else { - c.Infof("read %q", name) - fi, data, err = read(c, name) - if err == nil && fi.Seq != t { - c.Infof("loaded %s but found read %d", key, fi.Seq) - t = fi.Seq - key = fmt.Sprintf("%d,data,%s", t, name) - statkey = fmt.Sprintf("%d,stat,%s", t, name) - } - - // Save data to memcache. - if err == nil { - if true || chatty { - c.Infof("save data in memcache %q", key) - } - item := &memcache.Item{Key: key, Value: data} - if err := memcache.Set(c, item); err != nil { - c.Criticalf("failed to cache %s: %v", key, err) - } - } - } - - // Cache stat, including error. - st = statCacheEntry{} - if fi != nil { - st.FileInfo = &proto.FileInfo{ - Name: fi.Name, - ModTime: fi.ModTime, - Size: fi.Size, - IsDir: fi.IsDir, - } - } - if err != nil { - st.Error = err.Error() - // If this is a deadline exceeded, do not cache. - if strings.Contains(st.Error, "Canceled") || strings.Contains(st.Error, "Deadline") { - return t, data, st.FileInfo, err - } - } - if chatty { - c.Infof("save stat in memcache %q", statkey) - } - if err := memcache.JSON.Set(c, &memcache.Item{Key: statkey, Object: &st}); err != nil { - c.Criticalf("failed to cache %s: %v", statkey, err) - } - - // Done! - return t, data, st.FileInfo, err - } - - c.Criticalf("failed repeatedly in cacheRead") - return 0, nil, nil, errors.New("cacheRead loop failed") -} - -func cacheWrite(c appengine.Context, t int64, kind, name string, data []byte) error { - mkey := fmt.Sprintf("%d,%s,%s", t, kind, name) - if true || chatty { - c.Infof("cacheWrite %s %d bytes", mkey, len(data)) - } - err := memcache.Set(c, &memcache.Item{Key: mkey, Value: data}) - if err != nil { - c.Criticalf("cacheWrite memcache.Set %q: %v", mkey, err) - } - return err -} -- cgit v1.2.3-1-g7c22