diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a3a6f2fa7..89063a842 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -23,11 +23,11 @@ }, { "ImportPath": "github.com/juju/ratelimit", - "Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56" + "Rev": "c5abe513796336ee2869745bff0638508450e9c5" }, { "ImportPath": "github.com/kardianos/osext", - "Rev": "91292666f7e40f03185cdd1da7d85633c973eca7" + "Rev": "efacde03154693404c65e7aa7d461ac9014acd0c" }, { "ImportPath": "github.com/syncthing/protocol", @@ -35,11 +35,15 @@ }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", - "Rev": "e3f32eb300aa1e514fe8ba58d008da90a062273d" + "Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e" }, { "ImportPath": "github.com/syndtr/gosnappy/snappy", - "Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862" + "Rev": "156a073208e131d7d2e212cb749feae7c339e846" + }, + { + "ImportPath": "github.com/thejerf/suture", + "Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c" }, { "ImportPath": "github.com/vitrun/qart/coding", @@ -55,19 +59,19 @@ }, { "ImportPath": "golang.org/x/crypto/bcrypt", - "Rev": "4ed45ec682102c643324fae5dff8dab085b6c300" + "Rev": "c57d4a71915a248dbad846d60825145062b4c18e" }, { "ImportPath": "golang.org/x/crypto/blowfish", - "Rev": "4ed45ec682102c643324fae5dff8dab085b6c300" + "Rev": "c57d4a71915a248dbad846d60825145062b4c18e" }, { "ImportPath": "golang.org/x/text/transform", - "Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a" + "Rev": "2076e9cab4147459c82bc81169e46c139d358547" }, { "ImportPath": "golang.org/x/text/unicode/norm", - "Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a" + "Rev": "2076e9cab4147459c82bc81169e46c139d358547" } ] } diff --git a/Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE b/Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE index 53320c352..bce6bdd4f 100644 --- a/Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE +++ b/Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE @@ -1,3 +1,6 @@ +This package contains an efficient token-bucket-based rate limiter. +Copyright (C) 2015 Canonical Ltd. + This software is licensed under the LGPLv3, included below. As a special exception to the GNU Lesser General Public License version 3 diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go b/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go index a50021ad5..07a2a09e7 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go +++ b/Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go @@ -11,12 +11,18 @@ import ( "fmt" "os" "runtime" + "strings" ) func executable() (string, error) { switch runtime.GOOS { case "linux": - return os.Readlink("/proc/self/exe") + const deletedSuffix = " (deleted)" + execpath, err := os.Readlink("/proc/self/exe") + if err != nil { + return execpath, err + } + return strings.TrimSuffix(execpath, deletedSuffix), nil case "netbsd": return os.Readlink("/proc/curproc/exe") case "openbsd", "dragonfly": diff --git a/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go b/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go index dc661dbc2..5aafa3af2 100644 --- a/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go +++ b/Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go @@ -7,35 +7,42 @@ package osext import ( + "bytes" "fmt" + "io" "os" - oexec "os/exec" + "os/exec" "path/filepath" "runtime" "testing" ) -const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH" +const ( + executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE" -func TestExecPath(t *testing.T) { + executableEnvValueMatch = "match" + executableEnvValueDelete = "delete" +) + +func TestExecutableMatch(t *testing.T) { ep, err := Executable() if err != nil { - t.Fatalf("ExecPath failed: %v", err) + t.Fatalf("Executable failed: %v", err) } - // we want fn to be of the form "dir/prog" + + // fullpath to be of the form "dir/prog". dir := filepath.Dir(filepath.Dir(ep)) - fn, err := filepath.Rel(dir, ep) + fullpath, err := filepath.Rel(dir, ep) if err != nil { t.Fatalf("filepath.Rel: %v", err) } - cmd := &oexec.Cmd{} - // make child start with a relative program path - cmd.Dir = dir - cmd.Path = fn - // forge argv[0] for child, so that we can verify we could correctly - // get real path of the executable without influenced by argv[0]. - cmd.Args = []string{"-", "-test.run=XXXX"} - cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)} + // Make child start with a relative program path. + // Alter argv[0] for child to verify getting real path without argv[0]. + cmd := &exec.Cmd{ + Dir: dir, + Path: fullpath, + Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)}, + } out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("exec(self) failed: %v", err) @@ -49,6 +56,63 @@ func TestExecPath(t *testing.T) { } } +func TestExecutableDelete(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + fpath, err := Executable() + if err != nil { + t.Fatalf("Executable failed: %v", err) + } + + r, w := io.Pipe() + stderrBuff := &bytes.Buffer{} + stdoutBuff := &bytes.Buffer{} + cmd := &exec.Cmd{ + Path: fpath, + Env: []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)}, + Stdin: r, + Stderr: stderrBuff, + Stdout: stdoutBuff, + } + err = cmd.Start() + if err != nil { + t.Fatalf("exec(self) start failed: %v", err) + } + + tempPath := fpath + "_copy" + _ = os.Remove(tempPath) + + err = copyFile(tempPath, fpath) + if err != nil { + t.Fatalf("copy file failed: %v", err) + } + err = os.Remove(fpath) + if err != nil { + t.Fatalf("remove running test file failed: %v", err) + } + err = os.Rename(tempPath, fpath) + if err != nil { + t.Fatalf("rename copy to previous name failed: %v", err) + } + + w.Write([]byte{0}) + w.Close() + + err = cmd.Wait() + if err != nil { + t.Fatalf("exec wait failed: %v", err) + } + + childPath := stderrBuff.String() + if !filepath.IsAbs(childPath) { + t.Fatalf("Child returned %q, want an absolute path", childPath) + } + if !sameFile(childPath, fpath) { + t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath) + } +} + func sameFile(fn1, fn2 string) bool { fi1, err := os.Stat(fn1) if err != nil { @@ -60,10 +124,30 @@ func sameFile(fn1, fn2 string) bool { } return os.SameFile(fi1, fi2) } +func copyFile(dest, src string) error { + df, err := os.Create(dest) + if err != nil { + return err + } + defer df.Close() -func init() { - if e := os.Getenv(execPath_EnvVar); e != "" { - // first chdir to another path + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + + _, err = io.Copy(df, sf) + return err +} + +func TestMain(m *testing.M) { + env := os.Getenv(executableEnvVar) + switch env { + case "": + os.Exit(m.Run()) + case executableEnvValueMatch: + // First chdir to another path. dir := "/" if runtime.GOOS == "windows" { dir = filepath.VolumeName(".") @@ -74,6 +158,23 @@ func init() { } else { fmt.Fprint(os.Stderr, ep) } - os.Exit(0) + case executableEnvValueDelete: + bb := make([]byte, 1) + var err error + n, err := os.Stdin.Read(bb) + if err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + os.Exit(2) + } + if n != 1 { + fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n) + os.Exit(2) + } + if ep, err := Executable(); err != nil { + fmt.Fprint(os.Stderr, "ERROR: ", err) + } else { + fmt.Fprint(os.Stderr, ep) + } } + os.Exit(0) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go index 86828f47b..2b4289744 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go @@ -153,7 +153,7 @@ type Options struct { BlockCacher Cacher // BlockCacheCapacity defines the capacity of the 'sorted table' block caching. - // Use -1 for zero, this has same effect with specifying NoCacher to BlockCacher. + // Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher. // // The default value is 8MiB. BlockCacheCapacity int @@ -308,7 +308,7 @@ type Options struct { OpenFilesCacher Cacher // OpenFilesCacheCapacity defines the capacity of the open files caching. - // Use -1 for zero, this has same effect with specifying NoCacher to OpenFilesCacher. + // Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher. // // The default value is 500. OpenFilesCacheCapacity int @@ -355,9 +355,9 @@ func (o *Options) GetBlockCacher() Cacher { } func (o *Options) GetBlockCacheCapacity() int { - if o == nil || o.BlockCacheCapacity <= 0 { + if o == nil || o.BlockCacheCapacity == 0 { return DefaultBlockCacheCapacity - } else if o.BlockCacheCapacity == -1 { + } else if o.BlockCacheCapacity < 0 { return 0 } return o.BlockCacheCapacity @@ -497,7 +497,7 @@ func (o *Options) GetMaxMemCompationLevel() int { if o != nil { if o.MaxMemCompationLevel > 0 { level = o.MaxMemCompationLevel - } else if o.MaxMemCompationLevel == -1 { + } else if o.MaxMemCompationLevel < 0 { level = 0 } } @@ -525,9 +525,9 @@ func (o *Options) GetOpenFilesCacher() Cacher { } func (o *Options) GetOpenFilesCacheCapacity() int { - if o == nil || o.OpenFilesCacheCapacity <= 0 { + if o == nil || o.OpenFilesCacheCapacity == 0 { return DefaultOpenFilesCacheCapacity - } else if o.OpenFilesCacheCapacity == -1 { + } else if o.OpenFilesCacheCapacity < 0 { return 0 } return o.OpenFilesCacheCapacity diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go index d93c1b9db..552a17bfb 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go @@ -7,10 +7,15 @@ package snappy import ( "encoding/binary" "errors" + "io" ) -// ErrCorrupt reports that the input is invalid. -var ErrCorrupt = errors.New("snappy: corrupt input") +var ( + // ErrCorrupt reports that the input is invalid. + ErrCorrupt = errors.New("snappy: corrupt input") + // ErrUnsupported reports that the input isn't supported. + ErrUnsupported = errors.New("snappy: unsupported input") +) // DecodedLen returns the length of the decoded block. func DecodedLen(src []byte) (int, error) { @@ -122,3 +127,166 @@ func Decode(dst, src []byte) ([]byte, error) { } return dst[:d], nil } + +// NewReader returns a new Reader that decompresses from r, using the framing +// format described at +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func NewReader(r io.Reader) *Reader { + return &Reader{ + r: r, + decoded: make([]byte, maxUncompressedChunkLen), + buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize), + } +} + +// Reader is an io.Reader than can read Snappy-compressed bytes. +type Reader struct { + r io.Reader + err error + decoded []byte + buf []byte + // decoded[i:j] contains decoded bytes that have not yet been passed on. + i, j int + readHeader bool +} + +// Reset discards any buffered data, resets all state, and switches the Snappy +// reader to read from r. This permits reusing a Reader rather than allocating +// a new one. +func (r *Reader) Reset(reader io.Reader) { + r.r = reader + r.err = nil + r.i = 0 + r.j = 0 + r.readHeader = false +} + +func (r *Reader) readFull(p []byte) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF { + r.err = ErrCorrupt + } + return false + } + return true +} + +// Read satisfies the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, r.err + } + for { + if r.i < r.j { + n := copy(p, r.decoded[r.i:r.j]) + r.i += n + return n, nil + } + if !r.readFull(r.buf[:4]) { + return 0, r.err + } + chunkType := r.buf[0] + if !r.readHeader { + if chunkType != chunkTypeStreamIdentifier { + r.err = ErrCorrupt + return 0, r.err + } + r.readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + r.err = ErrUnsupported + return 0, r.err + } + + // The chunk types are specified at + // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[checksumSize:] + + n, err := DecodedLen(buf) + if err != nil { + r.err = err + return 0, r.err + } + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if _, err := Decode(r.decoded, buf); err != nil { + r.err = err + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeUncompressedData: + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:checksumSize] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - checksumSize + if !r.readFull(r.decoded[:n]) { + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeStreamIdentifier: + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(magicBody) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.buf[:len(magicBody)]) { + return 0, r.err + } + for i := 0; i < len(magicBody); i++ { + if r.buf[i] != magicBody[i] { + r.err = ErrCorrupt + return 0, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + r.err = ErrUnsupported + return 0, r.err + + } else { + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen]) { + return 0, r.err + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go index b2371db11..dda372422 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go @@ -6,6 +6,7 @@ package snappy import ( "encoding/binary" + "io" ) // We limit how far copy back-references can go, the same as the C++ code. @@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int { // This last factor dominates the blowup, so the final estimate is: return 32 + srcLen + srcLen/6 } + +// NewWriter returns a new Writer that compresses to w, using the framing +// format described at +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)), + } +} + +// Writer is an io.Writer than can write Snappy-compressed bytes. +type Writer struct { + w io.Writer + err error + enc []byte + buf [checksumSize + chunkHeaderSize]byte + wroteHeader bool +} + +// Reset discards the writer's state and switches the Snappy writer to write to +// w. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(writer io.Writer) { + w.w = writer + w.err = nil + w.wroteHeader = false +} + +// Write satisfies the io.Writer interface. +func (w *Writer) Write(p []byte) (n int, errRet error) { + if w.err != nil { + return 0, w.err + } + if !w.wroteHeader { + copy(w.enc, magicChunk) + if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil { + w.err = err + return n, err + } + w.wroteHeader = true + } + for len(p) > 0 { + var uncompressed []byte + if len(p) > maxUncompressedChunkLen { + uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:] + } else { + uncompressed, p = p, nil + } + checksum := crc(uncompressed) + + // Compress the buffer, discarding the result if the improvement + // isn't at least 12.5%. + chunkType := uint8(chunkTypeCompressedData) + chunkBody, err := Encode(w.enc, uncompressed) + if err != nil { + w.err = err + return n, err + } + if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 { + chunkType, chunkBody = chunkTypeUncompressedData, uncompressed + } + + chunkLen := 4 + len(chunkBody) + w.buf[0] = chunkType + w.buf[1] = uint8(chunkLen >> 0) + w.buf[2] = uint8(chunkLen >> 8) + w.buf[3] = uint8(chunkLen >> 16) + w.buf[4] = uint8(checksum >> 0) + w.buf[5] = uint8(checksum >> 8) + w.buf[6] = uint8(checksum >> 16) + w.buf[7] = uint8(checksum >> 24) + if _, err = w.w.Write(w.buf[:]); err != nil { + w.err = err + return n, err + } + if _, err = w.w.Write(chunkBody); err != nil { + w.err = err + return n, err + } + n += len(uncompressed) + } + return n, nil +} diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go index 2f1b790d0..043bf3d81 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go @@ -8,6 +8,10 @@ // The C++ snappy implementation is at http://code.google.com/p/snappy/ package snappy +import ( + "hash/crc32" +) + /* Each encoded block begins with the varint-encoded length of the decoded data, followed by a sequence of chunks. Chunks begin and end on byte boundaries. The @@ -36,3 +40,29 @@ const ( tagCopy2 = 0x02 tagCopy4 = 0x03 ) + +const ( + checksumSize = 4 + chunkHeaderSize = 4 + magicChunk = "\xff\x06\x00\x00" + magicBody + magicBody = "sNaPpY" + // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 bytes". + maxUncompressedChunkLen = 65536 +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func crc(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go index 7ba839244..0623385b7 100644 --- a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go @@ -18,7 +18,10 @@ import ( "testing" ) -var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") +var ( + download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") + testdata = flag.String("testdata", "testdata", "Directory containing the test data") +) func roundtrip(b, ebuf, dbuf []byte) error { e, err := Encode(ebuf, b) @@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) { } func TestSmallRand(t *testing.T) { - rand.Seed(27354294) + rng := rand.New(rand.NewSource(27354294)) for n := 1; n < 20000; n += 23 { b := make([]byte, n) - for i, _ := range b { - b[i] = uint8(rand.Uint32()) + for i := range b { + b[i] = uint8(rng.Uint32()) } if err := roundtrip(b, nil, nil); err != nil { t.Fatal(err) @@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) { func TestSmallRegular(t *testing.T) { for n := 1; n < 20000; n += 23 { b := make([]byte, n) - for i, _ := range b { + for i := range b { b[i] = uint8(i%10 + 'a') } if err := roundtrip(b, nil, nil); err != nil { @@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) { } } +func cmp(a, b []byte) error { + if len(a) != len(b) { + return fmt.Errorf("got %d bytes, want %d", len(a), len(b)) + } + for i := range a { + if a[i] != b[i] { + return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i]) + } + } + return nil +} + +func TestFramingFormat(t *testing.T) { + // src is comprised of alternating 1e5-sized sequences of random + // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen + // because it is larger than maxUncompressedChunkLen (64k). + src := make([]byte, 1e6) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 10; i++ { + if i%2 == 0 { + for j := 0; j < 1e5; j++ { + src[1e5*i+j] = uint8(rng.Intn(256)) + } + } else { + for j := 0; j < 1e5; j++ { + src[1e5*i+j] = uint8(i) + } + } + } + + buf := new(bytes.Buffer) + if _, err := NewWriter(buf).Write(src); err != nil { + t.Fatalf("Write: encoding: %v", err) + } + dst, err := ioutil.ReadAll(NewReader(buf)) + if err != nil { + t.Fatalf("ReadAll: decoding: %v", err) + } + if err := cmp(dst, src); err != nil { + t.Fatal(err) + } +} + +func TestReaderReset(t *testing.T) { + gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000) + buf := new(bytes.Buffer) + if _, err := NewWriter(buf).Write(gold); err != nil { + t.Fatalf("Write: %v", err) + } + encoded, invalid, partial := buf.String(), "invalid", "partial" + r := NewReader(nil) + for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} { + if s == partial { + r.Reset(strings.NewReader(encoded)) + if _, err := r.Read(make([]byte, 101)); err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + continue + } + r.Reset(strings.NewReader(s)) + got, err := ioutil.ReadAll(r) + switch s { + case encoded: + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + if err := cmp(got, gold); err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + case invalid: + if err == nil { + t.Errorf("#%d: got nil error, want non-nil", i) + continue + } + } + } +} + +func TestWriterReset(t *testing.T) { + gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000) + var gots, wants [][]byte + const n = 20 + w, failed := NewWriter(nil), false + for i := 0; i <= n; i++ { + buf := new(bytes.Buffer) + w.Reset(buf) + want := gold[:len(gold)*i/n] + if _, err := w.Write(want); err != nil { + t.Errorf("#%d: Write: %v", i, err) + failed = true + continue + } + got, err := ioutil.ReadAll(NewReader(buf)) + if err != nil { + t.Errorf("#%d: ReadAll: %v", i, err) + failed = true + continue + } + gots = append(gots, got) + wants = append(wants, want) + } + if failed { + return + } + for i := range gots { + if err := cmp(gots[i], wants[i]); err != nil { + t.Errorf("#%d: %v", i, err) + } + } +} + func benchDecode(b *testing.B, src []byte) { encoded, err := Encode(nil, src) if err != nil { @@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) { } } -func readFile(b *testing.B, filename string) []byte { +func readFile(b testing.TB, filename string) []byte { src, err := ioutil.ReadFile(filename) if err != nil { b.Fatalf("failed reading %s: %s", filename, err) @@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) } func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) } // testFiles' values are copied directly from -// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc. +// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc // The label field is unused in snappy-go. var testFiles = []struct { label string @@ -152,29 +269,36 @@ var testFiles = []struct { }{ {"html", "html"}, {"urls", "urls.10K"}, - {"jpg", "house.jpg"}, - {"pdf", "mapreduce-osdi-1.pdf"}, + {"jpg", "fireworks.jpeg"}, + {"jpg_200", "fireworks.jpeg"}, + {"pdf", "paper-100k.pdf"}, {"html4", "html_x_4"}, - {"cp", "cp.html"}, - {"c", "fields.c"}, - {"lsp", "grammar.lsp"}, - {"xls", "kennedy.xls"}, {"txt1", "alice29.txt"}, {"txt2", "asyoulik.txt"}, {"txt3", "lcet10.txt"}, {"txt4", "plrabn12.txt"}, - {"bin", "ptt5"}, - {"sum", "sum"}, - {"man", "xargs.1"}, {"pb", "geo.protodata"}, {"gaviota", "kppkn.gtb"}, } // The test data files are present at this canonical URL. -const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/" +const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/" func downloadTestdata(basename string) (errRet error) { - filename := filepath.Join("testdata", basename) + filename := filepath.Join(*testdata, basename) + if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 { + return nil + } + + if !*download { + return fmt.Errorf("test data not found; skipping benchmark without the -download flag") + } + // Download the official snappy C++ implementation reference test data + // files for benchmarking. + if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create testdata: %s", err) + } + f, err := os.Create(filename) if err != nil { return fmt.Errorf("failed to create %s: %s", filename, err) @@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) { os.Remove(filename) } }() - resp, err := http.Get(baseURL + basename) + url := baseURL + basename + resp, err := http.Get(url) if err != nil { - return fmt.Errorf("failed to download %s: %s", baseURL+basename, err) + return fmt.Errorf("failed to download %s: %s", url, err) } defer resp.Body.Close() + if s := resp.StatusCode; s != http.StatusOK { + return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s)) + } _, err = io.Copy(f, resp.Body) if err != nil { - return fmt.Errorf("failed to write %s: %s", filename, err) + return fmt.Errorf("failed to download %s to %s: %s", url, filename, err) } return nil } func benchFile(b *testing.B, n int, decode bool) { - filename := filepath.Join("testdata", testFiles[n].filename) - if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 { - if !*download { - b.Fatal("test data not found; skipping benchmark without the -download flag") - } - // Download the official snappy C++ implementation reference test data - // files for benchmarking. - if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) { - b.Fatalf("failed to create testdata: %s", err) - } - for _, tf := range testFiles { - if err := downloadTestdata(tf.filename); err != nil { - b.Fatalf("failed to download testdata: %s", err) - } - } + if err := downloadTestdata(testFiles[n].filename); err != nil { + b.Fatalf("failed to download testdata: %s", err) } - data := readFile(b, filename) + data := readFile(b, filepath.Join(*testdata, testFiles[n].filename)) if decode { benchDecode(b, data) } else { @@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) } func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) } func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) } func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) } -func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) } -func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) } -func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) } -func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) } -func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) } -func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) } func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) } func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) } func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) } @@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) } func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) } func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) } func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) } -func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) } -func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) } -func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) } -func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) } -func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) } -func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) } diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/.travis.yml b/Godeps/_workspace/src/github.com/thejerf/suture/.travis.yml new file mode 100644 index 000000000..814710d4e --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/.travis.yml @@ -0,0 +1,7 @@ +language: go +go: + - 1.1 + - 1.2 + - 1.3 + - 1.4 + - tip diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/LICENSE b/Godeps/_workspace/src/github.com/thejerf/suture/LICENSE new file mode 100644 index 000000000..fb1cf4ae3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2014 Barracuda Networks, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/README.md b/Godeps/_workspace/src/github.com/thejerf/suture/README.md new file mode 100644 index 000000000..3bbd9875a --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/README.md @@ -0,0 +1,45 @@ +Suture +====== + +[![Build Status](https://travis-ci.org/thejerf/suture.png?branch=master)](https://travis-ci.org/thejerf/suture) + +Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" -> +"sutree" -> "suture" -> holds your code together when it's trying to die. + +This is intended to be a production-quality library going into code that I +will be very early on the phone tree to support when it goes down. However, +it has not been deployed into something quite that serious yet. (I will +update this statement when that changes.) + +It is intended to deal gracefully with the real failure cases that can +occur with supervision trees (such as burning all your CPU time endlessly +restarting dead services), while also making no unnecessary demands on the +"service" code, and providing hooks to perform adequate logging with in a +production environment. + +[A blog post describing the design decisions](http://www.jerf.org/iri/post/2930) +is available. + +This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/suture), +including an example, usage, and everything else you might expect from a +README.md on GitHub. (DRY.) + +This is not currently tagged with particular git tags for Go as this is +currently considered to be alpha code. As I move this into production and +feel more confident about it, I'll give it relevant tags. + +Code Signing +------------ + +Starting with the commit after ac7cf8591b, I will be signing this repository +with the ["jerf" keybase account](https://keybase.io/jerf). + +Aspiration +---------- + +One of the big wins the Erlang community has with their pervasive OTP +support is that it makes it easy for them to distribute libraries that +easily fit into the OTP paradigm. It ought to someday be considered a good +idea to distribute libraries that provide some sort of supervisor tree +functionality out of the box. It is possible to provide this functionality +without explicitly depending on the Suture library. diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/pre-commit b/Godeps/_workspace/src/github.com/thejerf/suture/pre-commit new file mode 100644 index 000000000..c88ec0fe4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/pre-commit @@ -0,0 +1,11 @@ +#!/bin/bash + +GOLINTOUT=$(golint *go) + +if [ ! -z "$GOLINTOUT" -o "$?" != 0 ]; then + echo golint failed: + echo $GOLINTOUT + exit 1 +fi + +go test diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/suture.go b/Godeps/_workspace/src/github.com/thejerf/suture/suture.go new file mode 100644 index 000000000..3d98de13a --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/suture.go @@ -0,0 +1,650 @@ +/* + +Package suture provides Erlang-like supervisor trees. + +This implements Erlang-esque supervisor trees, as adapted for Go. This is +intended to be an industrial-strength implementation, but it has not yet +been deployed in a hostile environment. (It's headed there, though.) + +Supervisor Tree -> SuTree -> suture -> holds your code together when it's +trying to fall apart. + +Why use Suture? + + * You want to write bullet-resistant services that will remain available + despite unforeseen failure. + * You need the code to be smart enough not to consume 100% of the CPU + restarting things. + * You want to easily compose multiple such services in one program. + * You want the Erlang programmers to stop lording their supervision + trees over you. + +Suture has 100% test coverage, and is golint clean. This doesn't prove it +free of bugs, but it shows I care. + +A blog post describing the design decisions is available at +http://www.jerf.org/iri/post/2930 . + +Using Suture + +To idiomatically use Suture, create a Supervisor which is your top level +"application" supervisor. This will often occur in your program's "main" +function. + +Create "Service"s, which implement the Service interface. .Add() them +to your Supervisor. Supervisors are also services, so you can create a +tree structure here, depending on the exact combination of restarts +you want to create. + +Finally, as what is probably the last line of your main() function, call +.Serve() on your top level supervisor. This will start all the services +you've defined. + +See the Example for an example, using a simple service that serves out +incrementing integers. + +*/ +package suture + +import ( + "errors" + "fmt" + "log" + "math" + "runtime" + "sync/atomic" + "time" +) + +const ( + notRunning = iota + normal + paused +) + +type supervisorID uint32 +type serviceID uint32 + +var currentSupervisorID uint32 + +// ErrWrongSupervisor is returned by the (*Supervisor).Remove method +// if you pass a ServiceToken from the wrong Supervisor. +var ErrWrongSupervisor = errors.New("wrong supervisor for this service token, no service removed") + +// ServiceToken is an opaque identifier that can be used to terminate a service that +// has been Add()ed to a Supervisor. +type ServiceToken struct { + id uint64 +} + +/* +Supervisor is the core type of the module that represents a Supervisor. + +Supervisors should be constructed either by New or NewSimple. + +Once constructed, a Supervisor should be started in one of three ways: + + 1. Calling .Serve(). + 2. Calling .ServeBackground(). + 3. Adding it to an existing Supervisor. + +Calling Serve will cause the supervisor to run until it is shut down by +an external user calling Stop() on it. If that never happens, it simply +runs forever. I suggest creating your services in Supervisors, then making +a Serve() call on your top-level Supervisor be the last line of your main +func. + +Calling ServeBackground will CORRECTLY start the supervisor running in a +new goroutine. You do not want to just: + + go supervisor.Serve() + +because that will briefly create a race condition as it starts up, if you +try to .Add() services immediately afterward. + +*/ +type Supervisor struct { + Name string + id supervisorID + + failureDecay float64 + failureThreshold float64 + failureBackoff time.Duration + timeout time.Duration + log func(string) + services map[serviceID]Service + lastFail time.Time + failures float64 + restartQueue []serviceID + state uint8 + serviceCounter serviceID + control chan supervisorMessage + resumeTimer <-chan time.Time + + // The testing uses the ability to grab these individual logging functions + // and get inside of suture's handling at a deep level. + // If you ever come up with some need to get into these, submit a pull + // request to make them public and some smidge of justification, and + // I'll happily do it. + logBadStop func(Service) + logFailure func(service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte) + logBackoff func(*Supervisor, bool) + + // avoid a dependency on github.com/thejerf/abtime by just implementing + // a minimal chunk. + getNow func() time.Time + getResume func(time.Duration) <-chan time.Time +} + +// Spec is used to pass arguments to the New function to create a +// supervisor. See the New function for full documentation. +type Spec struct { + Log func(string) + FailureDecay float64 + FailureThreshold float64 + FailureBackoff time.Duration + Timeout time.Duration +} + +/* + +New is the full constructor function for a supervisor. + +The name is a friendly human name for the supervisor, used in logging. Suture +does not care if this is unique, but it is good for your sanity if it is. + +If not set, the following values are used: + + * Log: A function is created that uses log.Print. + * FailureDecay: 30 seconds + * FailureThreshold: 5 failures + * FailureBackoff: 15 seconds + * Timeout: 10 seconds + +The Log function will be called when errors occur. Suture will log the +following: + + * When a service has failed, with a descriptive message about the + current backoff status, and whether it was immediately restarted + * When the supervisor has gone into its backoff mode, and when it + exits it + * When a service fails to stop + +The failureRate, failureThreshold, and failureBackoff controls how failures +are handled, in order to avoid the supervisor failure case where the +program does nothing but restarting failed services. If you do not +care how failures behave, the default values should be fine for the +vast majority of services, but if you want the details: + +The supervisor tracks the number of failures that have occurred, with an +exponential decay on the count. Every FailureDecay seconds, the number of +failures that have occurred is cut in half. (This is done smoothly with an +exponential function.) When a failure occurs, the number of failures +is incremented by one. When the number of failures passes the +FailureThreshold, the entire service waits for FailureBackoff seconds +before attempting any further restarts, at which point it resets its +failure count to zero. + +Timeout is how long Suture will wait for a service to properly terminate. + +*/ +func New(name string, spec Spec) (s *Supervisor) { + s = new(Supervisor) + + s.Name = name + s.id = supervisorID(atomic.AddUint32(¤tSupervisorID, 1)) + + if spec.Log == nil { + s.log = func(msg string) { + log.Print(fmt.Sprintf("Supervisor %s: %s", s.Name, msg)) + } + } else { + s.log = spec.Log + } + + if spec.FailureDecay == 0 { + s.failureDecay = 30 + } else { + s.failureDecay = spec.FailureDecay + } + if spec.FailureThreshold == 0 { + s.failureThreshold = 5 + } else { + s.failureThreshold = spec.FailureThreshold + } + if spec.FailureBackoff == 0 { + s.failureBackoff = time.Second * 15 + } else { + s.failureBackoff = spec.FailureBackoff + } + if spec.Timeout == 0 { + s.timeout = time.Second * 10 + } else { + s.timeout = spec.Timeout + } + + // overriding these allows for testing the threshold behavior + s.getNow = time.Now + s.getResume = time.After + + s.control = make(chan supervisorMessage) + s.services = make(map[serviceID]Service) + s.restartQueue = make([]serviceID, 0, 1) + s.resumeTimer = make(chan time.Time) + + // set up the default logging handlers + s.logBadStop = func(service Service) { + s.log(fmt.Sprintf("Service %s failed to terminate in a timely manner", serviceName(service))) + } + s.logFailure = func(service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) { + var errString string + + e, canError := err.(error) + if canError { + errString = e.Error() + } else { + errString = fmt.Sprintf("%#v", err) + } + + s.log(fmt.Sprintf("Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(service), failures, threshold, restarting, errString, string(st))) + } + s.logBackoff = func(s *Supervisor, entering bool) { + if entering { + s.log("Entering the backoff state.") + } else { + s.log("Exiting backoff state.") + } + } + + return +} + +func serviceName(service Service) (serviceName string) { + stringer, canStringer := service.(fmt.Stringer) + if canStringer { + serviceName = stringer.String() + } else { + serviceName = fmt.Sprintf("%#v", service) + } + return +} + +// NewSimple is a convenience function to create a service with just a name +// and the sensible defaults. +func NewSimple(name string) *Supervisor { + return New(name, Spec{}) +} + +/* +Service is the interface that describes a service to a Supervisor. + +Serve Method + +The Serve method is called by a Supervisor to start the service. +The service should execute within the goroutine that this is +called in. If this function either returns or panics, the Supervisor +will call it again. + +A Serve method SHOULD do as much cleanup of the state as possible, +to prevent any corruption in the previous state from crashing the +service again. + +Stop Method + +This method is used by the supervisor to stop the service. Calling this +directly on a Service given to a Supervisor will simply result in the +Service being restarted; use the Supervisor's .Remove(ServiceToken) method +to stop a service. A supervisor will call .Stop() only once. Thus, it may +be as destructive as it likes to get the service to stop. + +Once Stop has been called on a Service, the Service SHOULD NOT be +reused in any other supervisor! Because of the impossibility of +guaranteeing that the service has actually stopped in Go, you can't +prove that you won't be starting two goroutines using the exact +same memory to store state, causing completely unpredictable behavior. + +Stop should not return until the service has actually stopped. +"Stopped" here is defined as "the service will stop servicing any +further requests in the future". For instance, a common implementation +is to receive a message on a dedicated "stop" channel and immediately +returning. Once the stop command has been processed, the service is +stopped. + +Another common Stop implementation is to forcibly close an open socket +or other resource, which will cause detectable errors to manifest in the +service code. Bear in mind that to perfectly correctly use this +approach requires a bit more work to handle the chance of a Stop +command coming in before the resource has been created. + +If a service does not Stop within the supervisor's timeout duration, a log +entry will be made with a descriptive string to that effect. This does +not guarantee that the service is hung; it may still get around to being +properly stopped in the future. Until the service is fully stopped, +both the service and the spawned goroutine trying to stop it will be +"leaked". + +Stringer Interface + +It is not mandatory to implement the fmt.Stringer interface on your +service, but if your Service does happen to implement that, the log +messages that describe your service will use that when naming the +service. Otherwise, you'll see the GoString of your service object, +obtained via fmt.Sprintf("%#v", service). + +*/ +type Service interface { + Serve() + Stop() +} + +/* +Add adds a service to this supervisor. + +If the supervisor is currently running, the service will be started +immediately. If the supervisor is not currently running, the service +will be started when the supervisor is. + +The returned ServiceID may be passed to the Remove method of the Supervisor +to terminate the service. +*/ +func (s *Supervisor) Add(service Service) ServiceToken { + if s == nil { + panic("can't add service to nil *suture.Supervisor") + } + + if s.state == notRunning { + id := s.serviceCounter + s.serviceCounter++ + + s.services[id] = service + s.restartQueue = append(s.restartQueue, id) + + return ServiceToken{uint64(s.id)<<32 | uint64(id)} + } + + response := make(chan serviceID) + s.control <- addService{service, response} + return ServiceToken{uint64(s.id)<<32 | uint64(<-response)} +} + +// ServeBackground starts running a supervisor in its own goroutine. This +// method does not return until it is safe to use .Add() on the Supervisor. +func (s *Supervisor) ServeBackground() { + go s.Serve() + s.sync() +} + +/* +Serve starts the supervisor. You should call this on the top-level supervisor, +but nothing else. +*/ +func (s *Supervisor) Serve() { + if s == nil { + panic("Can't serve with a nil *suture.Supervisor") + } + if s.id == 0 { + panic("Can't call Serve on an incorrectly-constructed *suture.Supervisor") + } + + defer func() { + s.state = notRunning + }() + + if s.state != notRunning { + // FIXME: Don't explain why I don't need a semaphore, just use one + // This doesn't use a semaphore because it's just a sanity check. + panic("Running a supervisor while it is already running?") + } + + s.state = normal + + // for all the services I currently know about, start them + for _, id := range s.restartQueue { + service, present := s.services[id] + if present { + s.runService(service, id) + } + } + s.restartQueue = make([]serviceID, 0, 1) + + for { + select { + case m := <-s.control: + switch msg := m.(type) { + case serviceFailed: + s.handleFailedService(msg.id, msg.err, msg.stacktrace) + case serviceEnded: + service, monitored := s.services[msg.id] + if monitored { + s.handleFailedService(msg.id, fmt.Sprintf("%s returned unexpectedly", service), []byte("[unknown stack trace]")) + } + case addService: + id := s.serviceCounter + s.serviceCounter++ + + s.services[id] = msg.service + s.runService(msg.service, id) + + msg.response <- id + case removeService: + s.removeService(msg.id) + case stopSupervisor: + for id := range s.services { + s.removeService(id) + } + return + case listServices: + services := []Service{} + for _, service := range s.services { + services = append(services, service) + } + msg.c <- services + case syncSupervisor: + // this does nothing on purpose; its sole purpose is to + // introduce a sync point via the channel receive + case panicSupervisor: + // used only by tests + panic("Panicking as requested!") + } + case _ = <-s.resumeTimer: + // We're resuming normal operation after a pause due to + // excessive thrashing + // FIXME: Ought to permit some spacing of these functions, rather + // than simply hammering through them + s.state = normal + s.failures = 0 + s.logBackoff(s, false) + for _, id := range s.restartQueue { + service, present := s.services[id] + if present { + s.runService(service, id) + } + } + s.restartQueue = make([]serviceID, 0, 1) + } + } +} + +func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktrace []byte) { + now := s.getNow() + + if s.lastFail.IsZero() { + s.lastFail = now + s.failures = 1.0 + } else { + sinceLastFail := now.Sub(s.lastFail).Seconds() + intervals := sinceLastFail / s.failureDecay + s.failures = s.failures*math.Pow(.5, intervals) + 1 + } + + if s.failures > s.failureThreshold { + s.state = paused + s.logBackoff(s, true) + s.resumeTimer = s.getResume(s.failureBackoff) + } + + s.lastFail = now + + failedService, monitored := s.services[id] + + // It is possible for a service to be no longer monitored + // by the time we get here. In that case, just ignore it. + if monitored { + if s.state == normal { + s.runService(failedService, id) + s.logFailure(failedService, s.failures, s.failureThreshold, true, err, stacktrace) + } else { + // FIXME: When restarting, check that the service still + // exists (it may have been stopped in the meantime) + s.restartQueue = append(s.restartQueue, id) + s.logFailure(failedService, s.failures, s.failureThreshold, false, err, stacktrace) + } + } +} + +func (s *Supervisor) runService(service Service, id serviceID) { + go func() { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 65535, 65535) + written := runtime.Stack(buf, false) + buf = buf[:written] + s.fail(id, r, buf) + } + }() + + service.Serve() + + s.serviceEnded(id) + }() +} + +func (s *Supervisor) removeService(id serviceID) { + service, present := s.services[id] + if present { + delete(s.services, id) + go func() { + successChan := make(chan bool) + go func() { + service.Stop() + successChan <- true + }() + + failChan := s.getResume(s.timeout) + + select { + case <-successChan: + // Life is good! + case <-failChan: + s.logBadStop(service) + } + }() + } +} + +// String implements the fmt.Stringer interface. +func (s *Supervisor) String() string { + return s.Name +} + +// sum type pattern for type-safe message passing; see +// http://www.jerf.org/iri/post/2917 + +type supervisorMessage interface { + isSupervisorMessage() +} + +/* +Remove will remove the given service from the Supervisor, and attempt to Stop() it. +The ServiceID token comes from the Add() call. +*/ +func (s *Supervisor) Remove(id ServiceToken) error { + sID := supervisorID(id.id >> 32) + if sID != s.id { + return ErrWrongSupervisor + } + s.control <- removeService{serviceID(id.id & 0xffffffff)} + return nil +} + +/* + +Services returns a []Service containing a snapshot of the services this +Supervisor is managing. + +*/ +func (s *Supervisor) Services() []Service { + ls := listServices{make(chan []Service)} + s.control <- ls + return <-ls.c +} + +type listServices struct { + c chan []Service +} + +func (ls listServices) isSupervisorMessage() {} + +type removeService struct { + id serviceID +} + +func (rs removeService) isSupervisorMessage() {} + +func (s *Supervisor) sync() { + s.control <- syncSupervisor{} +} + +type syncSupervisor struct { +} + +func (ss syncSupervisor) isSupervisorMessage() {} + +func (s *Supervisor) fail(id serviceID, err interface{}, stacktrace []byte) { + s.control <- serviceFailed{id, err, stacktrace} +} + +type serviceFailed struct { + id serviceID + err interface{} + stacktrace []byte +} + +func (sf serviceFailed) isSupervisorMessage() {} + +func (s *Supervisor) serviceEnded(id serviceID) { + s.control <- serviceEnded{id} +} + +type serviceEnded struct { + id serviceID +} + +func (s serviceEnded) isSupervisorMessage() {} + +// added by the Add() method +type addService struct { + service Service + response chan serviceID +} + +func (as addService) isSupervisorMessage() {} + +// Stop stops the Supervisor. +func (s *Supervisor) Stop() { + s.control <- stopSupervisor{} +} + +type stopSupervisor struct { +} + +func (ss stopSupervisor) isSupervisorMessage() {} + +func (s *Supervisor) panic() { + s.control <- panicSupervisor{} +} + +type panicSupervisor struct { +} + +func (ps panicSupervisor) isSupervisorMessage() {} diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/suture_simple_test.go b/Godeps/_workspace/src/github.com/thejerf/suture/suture_simple_test.go new file mode 100644 index 000000000..718e72967 --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/suture_simple_test.go @@ -0,0 +1,49 @@ +package suture + +import "fmt" + +type Incrementor struct { + current int + next chan int + stop chan bool +} + +func (i *Incrementor) Stop() { + fmt.Println("Stopping the service") + i.stop <- true +} + +func (i *Incrementor) Serve() { + for { + select { + case i.next <- i.current: + i.current += 1 + case <-i.stop: + // We sync here just to guarantee the output of "Stopping the service", + // so this passes the test reliably. + // Most services would simply "return" here. + i.stop <- true + return + } + } +} + +func ExampleNew_simple() { + supervisor := NewSimple("Supervisor") + service := &Incrementor{0, make(chan int), make(chan bool)} + supervisor.Add(service) + + go supervisor.ServeBackground() + + fmt.Println("Got:", <-service.next) + fmt.Println("Got:", <-service.next) + supervisor.Stop() + + // We sync here just to guarantee the output of "Stopping the service" + <-service.stop + + // Output: + // Got: 0 + // Got: 1 + // Stopping the service +} diff --git a/Godeps/_workspace/src/github.com/thejerf/suture/suture_test.go b/Godeps/_workspace/src/github.com/thejerf/suture/suture_test.go new file mode 100644 index 000000000..b61bd4fb7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/thejerf/suture/suture_test.go @@ -0,0 +1,592 @@ +package suture + +import ( + "errors" + "fmt" + "reflect" + "sync" + "testing" + "time" +) + +const ( + Happy = iota + Fail + Panic + Hang + UseStopChan +) + +var everMultistarted = false + +// Test that supervisors work perfectly when everything is hunky dory. +func TestTheHappyCase(t *testing.T) { + t.Parallel() + + s := NewSimple("A") + if s.String() != "A" { + t.Fatal("Can't get name from a supervisor") + } + service := NewService("B") + + s.Add(service) + + go s.Serve() + + <-service.started + + // If we stop the service, it just gets restarted + service.Stop() + <-service.started + + // And it is shut down when we stop the supervisor + service.take <- UseStopChan + s.Stop() + <-service.stop +} + +// Test that adding to a running supervisor does indeed start the service. +func TestAddingToRunningSupervisor(t *testing.T) { + t.Parallel() + + s := NewSimple("A1") + + s.ServeBackground() + defer s.Stop() + + service := NewService("B1") + s.Add(service) + + <-service.started + + services := s.Services() + if !reflect.DeepEqual([]Service{service}, services) { + t.Fatal("Can't get list of services as expected.") + } +} + +// Test what happens when services fail. +func TestFailures(t *testing.T) { + t.Parallel() + + s := NewSimple("A2") + s.failureThreshold = 3.5 + + go s.Serve() + defer func() { + // to avoid deadlocks during shutdown, we have to not try to send + // things out on channels while we're shutting down (this undoes the + // logFailure overide about 25 lines down) + s.logFailure = func(Service, float64, float64, bool, interface{}, []byte) {} + s.Stop() + }() + s.sync() + + service1 := NewService("B2") + service2 := NewService("C2") + + s.Add(service1) + <-service1.started + s.Add(service2) + <-service2.started + + nowFeeder := NewNowFeeder() + pastVal := time.Unix(1000000, 0) + nowFeeder.appendTimes(pastVal) + s.getNow = nowFeeder.getter + + resumeChan := make(chan time.Time) + s.getResume = func(d time.Duration) <-chan time.Time { + return resumeChan + } + + failNotify := make(chan bool) + // use this to synchronize on here + s.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) { + failNotify <- r + } + + // All that setup was for this: Service1, please return now. + service1.take <- Fail + restarted := <-failNotify + <-service1.started + + if !restarted || s.failures != 1 || s.lastFail != pastVal { + t.Fatal("Did not fail in the expected manner") + } + // Getting past this means the service was restarted. + service1.take <- Happy + + // Service2, your turn. + service2.take <- Fail + nowFeeder.appendTimes(pastVal) + restarted = <-failNotify + <-service2.started + if !restarted || s.failures != 2 || s.lastFail != pastVal { + t.Fatal("Did not fail in the expected manner") + } + // And you're back. (That is, the correct service was restarted.) + service2.take <- Happy + + // Now, one failureDecay later, is everything working correctly? + oneDecayLater := time.Unix(1000030, 0) + nowFeeder.appendTimes(oneDecayLater) + service2.take <- Fail + restarted = <-failNotify + <-service2.started + // playing a bit fast and loose here with floating point, but... + // we get 2 by taking the current failure value of 2, decaying it + // by one interval, which cuts it in half to 1, then adding 1 again, + // all of which "should" be precise + if !restarted || s.failures != 2 || s.lastFail != oneDecayLater { + t.Fatal("Did not decay properly", s.lastFail, oneDecayLater) + } + + // For a change of pace, service1 would you be so kind as to panic? + nowFeeder.appendTimes(oneDecayLater) + service1.take <- Panic + restarted = <-failNotify + <-service1.started + if !restarted || s.failures != 3 || s.lastFail != oneDecayLater { + t.Fatal("Did not correctly recover from a panic") + } + + nowFeeder.appendTimes(oneDecayLater) + backingoff := make(chan bool) + s.logBackoff = func(s *Supervisor, backingOff bool) { + backingoff <- backingOff + } + + // And with this failure, we trigger the backoff code. + service1.take <- Fail + backoff := <-backingoff + restarted = <-failNotify + + if !backoff || restarted || s.failures != 4 { + t.Fatal("Broke past the threshold but did not log correctly", s.failures) + } + if service1.existing != 0 { + t.Fatal("service1 still exists according to itself?") + } + + // service2 is still running, because we don't shut anything down in a + // backoff, we just stop restarting. + service2.take <- Happy + + var correct bool + timer := time.NewTimer(time.Millisecond * 10) + // verify the service has not been restarted + // hard to get around race conditions here without simply using a timer... + select { + case service1.take <- Happy: + correct = false + case <-timer.C: + correct = true + } + if !correct { + t.Fatal("Restarted the service during the backoff interval") + } + + // tell the supervisor the restart interval has passed + resumeChan <- time.Time{} + backoff = <-backingoff + <-service1.started + s.sync() + if s.failures != 0 { + t.Fatal("Did not reset failure count after coming back from timeout.") + } + + nowFeeder.appendTimes(oneDecayLater) + service1.take <- Fail + restarted = <-failNotify + <-service1.started + if !restarted || backoff { + t.Fatal("For some reason, got that we were backing off again.", restarted, backoff) + } +} + +func TestRunningAlreadyRunning(t *testing.T) { + t.Parallel() + + s := NewSimple("A3") + go s.Serve() + defer s.Stop() + + // ensure the supervisor has made it to its main loop + s.sync() + var errored bool + func() { + defer func() { + if r := recover(); r != nil { + errored = true + } + }() + + s.Serve() + }() + if !errored { + t.Fatal("Supervisor failed to prevent itself from double-running.") + } +} + +func TestFullConstruction(t *testing.T) { + t.Parallel() + + s := New("Moo", Spec{ + Log: func(string) {}, + FailureDecay: 1, + FailureThreshold: 2, + FailureBackoff: 3, + Timeout: time.Second * 29, + }) + if s.String() != "Moo" || s.failureDecay != 1 || s.failureThreshold != 2 || s.failureBackoff != 3 || s.timeout != time.Second*29 { + t.Fatal("Full construction failed somehow") + } +} + +// This is mostly for coverage testing. +func TestDefaultLogging(t *testing.T) { + t.Parallel() + + s := NewSimple("A4") + + service := NewService("B4") + s.Add(service) + + s.failureThreshold = .5 + s.failureBackoff = time.Millisecond * 25 + go s.Serve() + s.sync() + + <-service.started + + resumeChan := make(chan time.Time) + s.getResume = func(d time.Duration) <-chan time.Time { + return resumeChan + } + + service.take <- UseStopChan + service.take <- Fail + <-service.stop + resumeChan <- time.Time{} + + <-service.started + + service.take <- Happy + + serviceName(&BarelyService{}) + + s.logBadStop(service) + s.logFailure(service, 1, 1, true, errors.New("test error"), []byte{}) + + s.Stop() +} + +func TestNestedSupervisors(t *testing.T) { + t.Parallel() + + super1 := NewSimple("Top5") + super2 := NewSimple("Nested5") + service := NewService("Service5") + + super1.Add(super2) + super2.Add(service) + + go super1.Serve() + super1.sync() + + <-service.started + service.take <- Happy + + super1.Stop() +} + +func TestStoppingSupervisorStopsServices(t *testing.T) { + t.Parallel() + + s := NewSimple("Top6") + service := NewService("Service 6") + + s.Add(service) + + go s.Serve() + s.sync() + + <-service.started + + service.take <- UseStopChan + + s.Stop() + <-service.stop +} + +func TestStoppingStillWorksWithHungServices(t *testing.T) { + t.Parallel() + + s := NewSimple("Top7") + service := NewService("Service WillHang7") + + s.Add(service) + + go s.Serve() + + <-service.started + + service.take <- UseStopChan + service.take <- Hang + + resumeChan := make(chan time.Time) + s.getResume = func(d time.Duration) <-chan time.Time { + return resumeChan + } + failNotify := make(chan struct{}) + s.logBadStop = func(s Service) { + failNotify <- struct{}{} + } + + s.Stop() + + resumeChan <- time.Time{} + <-failNotify + service.release <- true + <-service.stop +} + +func TestRemoveService(t *testing.T) { + t.Parallel() + + s := NewSimple("Top") + service := NewService("ServiceToRemove8") + + id := s.Add(service) + + go s.Serve() + + <-service.started + service.take <- UseStopChan + + err := s.Remove(id) + if err != nil { + t.Fatal("Removing service somehow failed") + } + <-service.stop + + err = s.Remove(ServiceToken{1<<36 + 1}) + if err != ErrWrongSupervisor { + t.Fatal("Did not detect that the ServiceToken was wrong") + } +} + +func TestFailureToConstruct(t *testing.T) { + t.Parallel() + + var s *Supervisor + + panics(func() { + s.Serve() + }) + + s = new(Supervisor) + panics(func() { + s.Serve() + }) +} + +func TestFailingSupervisors(t *testing.T) { + t.Parallel() + + // This is a bit of a complicated test, so let me explain what + // all this is doing: + // 1. Set up a top-level supervisor with a hair-trigger backoff. + // 2. Add a supervisor to that. + // 3. To that supervisor, add a service. + // 4. Panic the supervisor in the middle, sending the top-level into + // backoff. + // 5. Kill the lower level service too. + // 6. Verify that when the top-level service comes out of backoff, + // the service ends up restarted as expected. + + // Ultimately, we can't have more than a best-effort recovery here. + // A panic'ed supervisor can't really be trusted to have consistent state, + // and without *that*, we can't trust it to do anything sensible with + // the children it may have been running. So unlike Erlang, we can't + // can't really expect to be able to safely restart them or anything. + // Really, the "correct" answer is that the Supervisor must never panic, + // but in the event that it does, this verifies that it at least tries + // to get on with life. + + // This also tests that if a Supervisor itself panics, and one of its + // monitored services goes down in the meantime, that the monitored + // service also gets correctly restarted when the supervisor does. + + s1 := NewSimple("Top9") + s2 := NewSimple("Nested9") + service := NewService("Service9") + + s1.Add(s2) + s2.Add(service) + + go s1.Serve() + <-service.started + + s1.failureThreshold = .5 + + // let us control precisely when s1 comes back + resumeChan := make(chan time.Time) + s1.getResume = func(d time.Duration) <-chan time.Time { + return resumeChan + } + failNotify := make(chan string) + // use this to synchronize on here + s1.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) { + failNotify <- fmt.Sprintf("%s", s) + } + + s2.panic() + + failing := <-failNotify + // that's enough sync to guarantee this: + if failing != "Nested9" || s1.state != paused { + t.Fatal("Top-level supervisor did not go into backoff as expected") + } + + service.take <- Fail + + resumeChan <- time.Time{} + <-service.started +} + +func TestNilSupervisorAdd(t *testing.T) { + t.Parallel() + + var s *Supervisor + + defer func() { + if r := recover(); r == nil { + t.Fatal("did not panic as expected on nil add") + } + }() + + s.Add(s) +} + +// http://golangtutorials.blogspot.com/2011/10/gotest-unit-testing-and-benchmarking-go.html +// claims test function are run in the same order as the source file... +// I'm not sure if this is part of the contract, though. Especially in the +// face of "t.Parallel()"... +func TestEverMultistarted(t *testing.T) { + if everMultistarted { + t.Fatal("Seem to have multistarted a service at some point, bummer.") + } +} + +// A test service that can be induced to fail, panic, or hang on demand. +func NewService(name string) *FailableService { + return &FailableService{name, make(chan bool), make(chan int), + make(chan bool, 1), make(chan bool), make(chan bool), 0} +} + +type FailableService struct { + name string + started chan bool + take chan int + shutdown chan bool + release chan bool + stop chan bool + existing int +} + +func (s *FailableService) Serve() { + if s.existing != 0 { + everMultistarted = true + panic("Multi-started the same service! " + s.name) + } + s.existing += 1 + + s.started <- true + + useStopChan := false + + for { + select { + case val := <-s.take: + switch val { + case Happy: + // Do nothing on purpose. Life is good! + case Fail: + s.existing -= 1 + if useStopChan { + s.stop <- true + } + return + case Panic: + s.existing -= 1 + panic("Panic!") + case Hang: + // or more specifically, "hang until I release you" + <-s.release + case UseStopChan: + useStopChan = true + } + case <-s.shutdown: + s.existing -= 1 + if useStopChan { + s.stop <- true + } + return + } + } +} + +func (s *FailableService) String() string { + return s.name +} + +func (s *FailableService) Stop() { + s.shutdown <- true +} + +type NowFeeder struct { + values []time.Time + getter func() time.Time + m sync.Mutex +} + +// This is used to test serviceName; it's a service without a Stringer. +type BarelyService struct{} + +func (bs *BarelyService) Serve() {} +func (bs *BarelyService) Stop() {} + +func NewNowFeeder() (nf *NowFeeder) { + nf = new(NowFeeder) + nf.getter = func() time.Time { + nf.m.Lock() + defer nf.m.Unlock() + if len(nf.values) > 0 { + ret := nf.values[0] + nf.values = nf.values[1:] + return ret + } + panic("Ran out of values for NowFeeder") + } + return +} + +func (nf *NowFeeder) appendTimes(t ...time.Time) { + nf.m.Lock() + defer nf.m.Unlock() + nf.values = append(nf.values, t...) +} + +func panics(doesItPanic func()) (panics bool) { + defer func() { + if r := recover(); r != nil { + panics = true + } + }() + + doesItPanic() + + return +} diff --git a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/Makefile b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/Makefile deleted file mode 100644 index b4f5d3590..000000000 --- a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2011 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -maketables: maketables.go triegen.go - go build $^ - -normregtest: normregtest.go - go build $^ - -tables: maketables - ./maketables > tables.go - gofmt -w tables.go - -# Downloads from www.unicode.org, so not part -# of standard test scripts. -test: testtables regtest - -testtables: maketables - ./maketables -test > data_test.go && go test -tags=test - -regtest: normregtest - ./normregtest diff --git a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go index 3524e8c0c..9f9a00eea 100644 --- a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go +++ b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go @@ -16,20 +16,17 @@ import ( "fmt" "io" "log" - "net/http" - "os" - "regexp" "sort" "strconv" "strings" - "unicode" + "golang.org/x/text/internal/gen" "golang.org/x/text/internal/triegen" "golang.org/x/text/internal/ucd" ) func main() { - flag.Parse() + gen.Init() loadUnicodeData() compactCCC() loadCompositionExclusions() @@ -46,24 +43,18 @@ func main() { } } -var url = flag.String("url", - "http://www.unicode.org/Public/"+unicode.Version+"/ucd/", - "URL of Unicode database directory") -var tablelist = flag.String("tables", - "all", - "comma-separated list of which tables to generate; "+ - "can be 'decomp', 'recomp', 'info' and 'all'") -var test = flag.Bool("test", - false, - "test existing tables against DerivedNormalizationProps and generate test data for regression testing") -var verbose = flag.Bool("verbose", - false, - "write data to stdout as it is parsed") -var localFiles = flag.Bool("local", - false, - "data files have been copied to the current directory; for debugging only") - -var logger = log.New(os.Stderr, "", log.Lshortfile) +var ( + tablelist = flag.String("tables", + "all", + "comma-separated list of which tables to generate; "+ + "can be 'decomp', 'recomp', 'info' and 'all'") + test = flag.Bool("test", + false, + "test existing tables against DerivedNormalizationProps and generate test data for regression testing") + verbose = flag.Bool("verbose", + false, + "write data to stdout as it is parsed") +) const MaxChar = 0x10FFFF // anything above this shouldn't exist @@ -189,27 +180,6 @@ func (f FormInfo) String() string { type Decomposition []rune -func openReader(file string) (input io.ReadCloser) { - if *localFiles { - f, err := os.Open(file) - if err != nil { - logger.Fatal(err) - } - input = f - } else { - path := *url + file - resp, err := http.Get(path) - if err != nil { - logger.Fatal(err) - } - if resp.StatusCode != 200 { - logger.Fatal("bad GET status for "+file, resp.Status) - } - input = resp.Body - } - return -} - func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { decomp := strings.Split(s, " ") if len(decomp) > 0 && skipfirst { @@ -226,7 +196,7 @@ func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { } func loadUnicodeData() { - f := openReader("UnicodeData.txt") + f := gen.OpenUCDFile("UnicodeData.txt") defer f.Close() p := ucd.New(f) for p.Next() { @@ -242,7 +212,7 @@ func loadUnicodeData() { if len(decmap) > 0 { exp, err = parseDecomposition(decmap, true) if err != nil { - logger.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) + log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) } isCompat = true } @@ -261,7 +231,7 @@ func loadUnicodeData() { } } if err := p.Err(); err != nil { - logger.Fatal(err) + log.Fatal(err) } } @@ -296,18 +266,18 @@ func compactCCC() { // 0958 # ... // See http://unicode.org/reports/tr44/ for full explanation func loadCompositionExclusions() { - f := openReader("CompositionExclusions.txt") + f := gen.OpenUCDFile("CompositionExclusions.txt") defer f.Close() p := ucd.New(f) for p.Next() { c := &chars[p.Rune(0)] if c.excludeInComp { - logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) + log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) } c.excludeInComp = true } if e := p.Err(); e != nil { - logger.Fatal(e) + log.Fatal(e) } } @@ -542,19 +512,19 @@ func computeNonStarterCounts() { } } -func printBytes(b []byte, name string) { - fmt.Printf("// %s: %d bytes\n", name, len(b)) - fmt.Printf("var %s = [...]byte {", name) +func printBytes(w io.Writer, b []byte, name string) { + fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) + fmt.Fprintf(w, "var %s = [...]byte {", name) for i, c := range b { switch { case i%64 == 0: - fmt.Printf("\n// Bytes %x - %x\n", i, i+63) + fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) case i%8 == 0: - fmt.Printf("\n") + fmt.Fprintf(w, "\n") } - fmt.Printf("0x%.2X, ", c) + fmt.Fprintf(w, "0x%.2X, ", c) } - fmt.Print("\n}\n\n") + fmt.Fprint(w, "\n}\n\n") } // See forminfo.go for format. @@ -610,13 +580,13 @@ func (m *decompSet) insert(key int, s string) { m[key][s] = true } -func printCharInfoTables() int { +func printCharInfoTables(w io.Writer) int { mkstr := func(r rune, f *FormInfo) (int, string) { d := f.expandedDecomp s := string([]rune(d)) if max := 1 << 6; len(s) >= max { const msg = "%U: too many bytes in decomposition: %d >= %d" - logger.Fatalf(msg, r, len(s), max) + log.Fatalf(msg, r, len(s), max) } head := uint8(len(s)) if f.quickCheck[MComposed] != QCYes { @@ -631,11 +601,11 @@ func printCharInfoTables() int { tccc := ccc(d[len(d)-1]) cc := ccc(r) if cc != 0 && lccc == 0 && tccc == 0 { - logger.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) + log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) } if tccc < lccc && lccc != 0 { const msg = "%U: lccc (%d) must be <= tcc (%d)" - logger.Fatalf(msg, r, lccc, tccc) + log.Fatalf(msg, r, lccc, tccc) } index := normalDecomp nTrail := chars[r].nTrailingNonStarters @@ -652,13 +622,13 @@ func printCharInfoTables() int { if lccc > 0 { s += string([]byte{lccc}) if index == firstCCC { - logger.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) + log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) } index = firstLeadingCCC } if cc != lccc { if cc != 0 { - logger.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) + log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) } index = firstCCCZeroExcept } @@ -680,7 +650,7 @@ func printCharInfoTables() int { continue } if f.combinesBackward { - logger.Fatalf("%U: combinesBackward and decompose", c.codePoint) + log.Fatalf("%U: combinesBackward and decompose", c.codePoint) } index, s := mkstr(c.codePoint, &f) decompSet.insert(index, s) @@ -691,7 +661,7 @@ func printCharInfoTables() int { size := 0 positionMap := make(map[string]uint16) decompositions.WriteString("\000") - fmt.Println("const (") + fmt.Fprintln(w, "const (") for i, m := range decompSet { sa := []string{} for s := range m { @@ -704,13 +674,13 @@ func printCharInfoTables() int { positionMap[s] = uint16(p) } if cname[i] != "" { - fmt.Printf("%s = 0x%X\n", cname[i], decompositions.Len()) + fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) } } - fmt.Println("maxDecomp = 0x8000") - fmt.Println(")") + fmt.Fprintln(w, "maxDecomp = 0x8000") + fmt.Fprintln(w, ")") b := decompositions.Bytes() - printBytes(b, "decomps") + printBytes(w, b, "decomps") size += len(b) varnames := []string{"nfc", "nfkc"} @@ -726,7 +696,7 @@ func printCharInfoTables() int { if c.ccc != ccc(d[0]) { // We assume the lead ccc of a decomposition !=0 in this case. if ccc(d[0]) == 0 { - logger.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) + log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) } } } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { @@ -737,9 +707,9 @@ func printCharInfoTables() int { trie.Insert(c.codePoint, uint64(0x8000|v)) } } - sz, err := trie.Gen(os.Stdout, triegen.Compact(&normCompacter{name: varnames[i]})) + sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) if err != nil { - logger.Fatal(err) + log.Fatal(err) } size += sz } @@ -755,30 +725,9 @@ func contains(sa []string, s string) bool { return false } -// Extract the version number from the URL. -func version() string { - // From http://www.unicode.org/standard/versions/#Version_Numbering: - // for the later Unicode versions, data files are located in - // versioned directories. - fields := strings.Split(*url, "/") - for _, f := range fields { - if match, _ := regexp.MatchString(`[0-9]\.[0-9]\.[0-9]`, f); match { - return f - } - } - logger.Fatal("unknown version") - return "Unknown" -} - -const fileHeader = `// Generated by running -// maketables --tables=%s --url=%s -// DO NOT EDIT - -package norm - -` - func makeTables() { + w := &bytes.Buffer{} + size := 0 if *tablelist == "" { return @@ -787,7 +736,6 @@ func makeTables() { if *tablelist == "all" { list = []string{"recomp", "info"} } - fmt.Printf(fileHeader, *tablelist, *url) // Compute maximum decomposition size. max := 0 @@ -797,30 +745,30 @@ func makeTables() { } } - fmt.Println("const (") - fmt.Println("\t// Version is the Unicode edition from which the tables are derived.") - fmt.Printf("\tVersion = %q\n", version()) - fmt.Println() - fmt.Println("\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") - fmt.Println("\t// may need to write atomically for any Form. Making a destination buffer at") - fmt.Println("\t// least this size ensures that Transform can always make progress and that") - fmt.Println("\t// the user does not need to grow the buffer on an ErrShortDst.") - fmt.Printf("\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) - fmt.Println(")\n") + fmt.Fprintln(w, "const (") + fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") + fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) + fmt.Fprintln(w) + fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") + fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") + fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") + fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") + fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) + fmt.Fprintln(w, ")\n") // Print the CCC remap table. size += len(cccMap) - fmt.Printf("var ccc = [%d]uint8{", len(cccMap)) + fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) for i := 0; i < len(cccMap); i++ { if i%8 == 0 { - fmt.Println() + fmt.Fprintln(w) } - fmt.Printf("%3d, ", cccMap[uint8(i)]) + fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) } - fmt.Println("\n}\n") + fmt.Fprintln(w, "\n}\n") if contains(list, "info") { - size += printCharInfoTables() + size += printCharInfoTables(w) } if contains(list, "recomp") { @@ -842,20 +790,21 @@ func makeTables() { } sz := nrentries * 8 size += sz - fmt.Printf("// recompMap: %d bytes (entries only)\n", sz) - fmt.Println("var recompMap = map[uint32]rune{") + fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) + fmt.Fprintln(w, "var recompMap = map[uint32]rune{") for i, c := range chars { f := c.forms[FCanonical] d := f.decomp if !f.isOneWay && len(d) > 0 { key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) - fmt.Printf("0x%.8X: 0x%.4X,\n", key, i) + fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i) } } - fmt.Printf("}\n\n") + fmt.Fprintf(w, "}\n\n") } - fmt.Printf("// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) + fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) + gen.WriteGoFile("tables.go", "norm", w.Bytes()) } func printChars() { @@ -901,7 +850,7 @@ func verifyComputed() { nfc := c.forms[FCanonical] nfkc := c.forms[FCompatibility] if nfc.combinesBackward != nfkc.combinesBackward { - logger.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) + log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) } } } @@ -913,7 +862,7 @@ func verifyComputed() { // 0374 ; NFD_QC; N # ... // See http://unicode.org/reports/tr44/ for full explanation func testDerived() { - f := openReader("DerivedNormalizationProps.txt") + f := gen.OpenUCDFile("DerivedNormalizationProps.txt") defer f.Close() p := ucd.New(f) for p.Next() { @@ -946,12 +895,12 @@ func testDerived() { log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) } if got := c.forms[ftype].quickCheck[mode]; got != qr { - logger.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) + log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) } c.forms[ftype].verified[mode] = true } if err := p.Err(); err != nil { - logger.Fatal(err) + log.Fatal(err) } // Any unspecified value must be QCYes. Verify this. for i, c := range chars { @@ -959,20 +908,14 @@ func testDerived() { for k, qr := range fd.quickCheck { if !fd.verified[k] && qr != QCYes { m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" - logger.Printf(m, i, j, k, qr, c.name) + log.Printf(m, i, j, k, qr, c.name) } } } } } -var testHeader = `// Generated by running -// maketables --test --url=%s -// +build test - -package norm - -const ( +var testHeader = `const ( Yes = iota No Maybe @@ -1010,8 +953,10 @@ func printTestdata() { nTrail uint8 f string } + last := lastInfo{} - fmt.Printf(testHeader, *url) + w := &bytes.Buffer{} + fmt.Fprintf(w, testHeader) for r, c := range chars { f := c.forms[FCanonical] qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) @@ -1025,9 +970,10 @@ func printTestdata() { } current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} if last != current { - fmt.Printf("\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) + fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) last = current } } - fmt.Println("}") + fmt.Fprintln(w, "}") + gen.WriteGoFile("data_test.go", "norm", w.Bytes()) } diff --git a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize.go b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize.go index d81728494..bcf8450d0 100644 --- a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize.go +++ b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize.go @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:generate go run maketables.go triegen.go +//go:generate go run maketables.go triegen.go -test + // Package norm contains types and functions for normalizing Unicode strings. package norm diff --git a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go index 932c4ec52..1ca93889e 100644 --- a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go +++ b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go @@ -1,6 +1,4 @@ -// Generated by running -// maketables --tables=all --url=http://www.unicode.org/Public/7.0.0/ucd/ -// DO NOT EDIT +// This file was generated by go generate; DO NOT EDIT package norm diff --git a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/normregtest.go b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/ucd_test.go similarity index 60% rename from Godeps/_workspace/src/golang.org/x/text/unicode/norm/normregtest.go rename to Godeps/_workspace/src/golang.org/x/text/unicode/norm/ucd_test.go index 654b25b16..a949a298a 100644 --- a/Godeps/_workspace/src/golang.org/x/text/unicode/norm/normregtest.go +++ b/Godeps/_workspace/src/golang.org/x/text/unicode/norm/ucd_test.go @@ -2,52 +2,37 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build ignore - -package main +package norm import ( "bufio" "bytes" "flag" "fmt" - "log" - "net/http" - "os" - "path" "regexp" "runtime" "strconv" "strings" + "sync" + "testing" "time" - "unicode" "unicode/utf8" - "golang.org/x/text/unicode/norm" + "golang.org/x/text/internal/gen" ) -func main() { - flag.Parse() - loadTestData() - CharacterByCharacterTests() - StandardTests() - PerformanceTest() - if errorCount == 0 { - fmt.Println("PASS") +var long = flag.Bool("long", false, + "run time-consuming tests, such as tests that fetch data online") + +var once sync.Once + +func skipShort(t *testing.T) { + if !gen.IsLocal() && !*long { + t.Skip("skipping test to prevent downloading; to run use -long or use -local to specify a local source") } + once.Do(func() { loadTestData(t) }) } -const file = "NormalizationTest.txt" - -var url = flag.String("url", - "http://www.unicode.org/Public/"+unicode.Version+"/ucd/"+file, - "URL of Unicode database directory") -var localFiles = flag.Bool("local", - false, - "data files have been copied to the current directory; for debugging only") - -var logger = log.New(os.Stderr, "", log.Lshortfile) - // This regression test runs the test set in NormalizationTest.txt // (taken from http://www.unicode.org/Public//ucd/). // @@ -124,22 +109,8 @@ var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (. var counter int // Load the data form NormalizationTest.txt -func loadTestData() { - if *localFiles { - pwd, _ := os.Getwd() - *url = "file://" + path.Join(pwd, file) - } - t := &http.Transport{} - t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) - c := &http.Client{Transport: t} - resp, err := c.Get(*url) - if err != nil { - logger.Fatal(err) - } - if resp.StatusCode != 200 { - logger.Fatal("bad GET status for "+file, resp.Status) - } - f := resp.Body +func loadTestData(t *testing.T) { + f := gen.OpenUCDFile("NormalizationTest.txt") defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { @@ -150,11 +121,11 @@ func loadTestData() { m := partRe.FindStringSubmatch(line) if m != nil { if len(m) < 3 { - logger.Fatal("Failed to parse Part: ", line) + t.Fatal("Failed to parse Part: ", line) } i, err := strconv.Atoi(m[1]) if err != nil { - logger.Fatal(err) + t.Fatal(err) } name := m[2] part = append(part, Part{name: name[:len(name)-1], number: i}) @@ -162,7 +133,7 @@ func loadTestData() { } m = testRe.FindStringSubmatch(line) if m == nil || len(m) < 7 { - logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m) + t.Fatalf(`Failed to parse: "%s" result: %#v`, line, m) } test := Test{name: m[6], partnr: len(part) - 1, number: counter} counter++ @@ -170,7 +141,7 @@ func loadTestData() { for _, split := range strings.Split(m[j], " ") { r, err := strconv.ParseUint(split, 16, 64) if err != nil { - logger.Fatal(err) + t.Fatal(err) } if test.r == 0 { // save for CharacterByCharacterTests @@ -185,50 +156,38 @@ func loadTestData() { part.tests = append(part.tests, test) } if scanner.Err() != nil { - logger.Fatal(scanner.Err()) + t.Fatal(scanner.Err()) } } -var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"} - -var errorCount int - -func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) { +func cmpResult(t *testing.T, tc *Test, name string, f Form, gold, test, result string) { if gold != result { - errorCount++ - if errorCount > 20 { - return - } - logger.Printf("%s:%s: %s(%+q)=%+q; want %+q: %s", - t.Name(), name, fstr[f], test, result, gold, t.name) + t.Errorf("%s:%s: %s(%+q)=%+q; want %+q: %s", + tc.Name(), name, fstr[f], test, result, gold, tc.name) } } -func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) { +func cmpIsNormal(t *testing.T, tc *Test, name string, f Form, test string, result, want bool) { if result != want { - errorCount++ - if errorCount > 20 { - return - } - logger.Printf("%s:%s: %s(%+q)=%v; want %v", t.Name(), name, fstr[f], test, result, want) + t.Errorf("%s:%s: %s(%+q)=%v; want %v", tc.Name(), name, fstr[f], test, result, want) } } -func doTest(t *Test, f norm.Form, gold, test string) { +func doTest(t *testing.T, tc *Test, f Form, gold, test string) { testb := []byte(test) result := f.Bytes(testb) - cmpResult(t, "Bytes", f, gold, test, string(result)) + cmpResult(t, tc, "Bytes", f, gold, test, string(result)) sresult := f.String(test) - cmpResult(t, "String", f, gold, test, sresult) + cmpResult(t, tc, "String", f, gold, test, sresult) acc := []byte{} - i := norm.Iter{} + i := Iter{} i.InitString(f, test) for !i.Done() { acc = append(acc, i.Next()...) } - cmpResult(t, "Iter.Next", f, gold, test, string(acc)) + cmpResult(t, tc, "Iter.Next", f, gold, test, string(acc)) buf := make([]byte, 128) acc = nil @@ -237,32 +196,33 @@ func doTest(t *Test, f norm.Form, gold, test string) { acc = append(acc, buf[:nDst]...) p += nSrc } - cmpResult(t, "Transform", f, gold, test, string(acc)) + cmpResult(t, tc, "Transform", f, gold, test, string(acc)) for i := range test { out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...) - cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out)) + cmpResult(t, tc, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out)) } - cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold) - cmpIsNormal(t, "IsNormalString", f, test, f.IsNormalString(test), test == gold) + cmpIsNormal(t, tc, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold) + cmpIsNormal(t, tc, "IsNormalString", f, test, f.IsNormalString(test), test == gold) } -func doConformanceTests(t *Test, partn int) { +func doConformanceTests(t *testing.T, tc *Test, partn int) { for i := 0; i <= 2; i++ { - doTest(t, norm.NFC, t.cols[1], t.cols[i]) - doTest(t, norm.NFD, t.cols[2], t.cols[i]) - doTest(t, norm.NFKC, t.cols[3], t.cols[i]) - doTest(t, norm.NFKD, t.cols[4], t.cols[i]) + doTest(t, tc, NFC, tc.cols[1], tc.cols[i]) + doTest(t, tc, NFD, tc.cols[2], tc.cols[i]) + doTest(t, tc, NFKC, tc.cols[3], tc.cols[i]) + doTest(t, tc, NFKD, tc.cols[4], tc.cols[i]) } for i := 3; i <= 4; i++ { - doTest(t, norm.NFC, t.cols[3], t.cols[i]) - doTest(t, norm.NFD, t.cols[4], t.cols[i]) - doTest(t, norm.NFKC, t.cols[3], t.cols[i]) - doTest(t, norm.NFKD, t.cols[4], t.cols[i]) + doTest(t, tc, NFC, tc.cols[3], tc.cols[i]) + doTest(t, tc, NFD, tc.cols[4], tc.cols[i]) + doTest(t, tc, NFKC, tc.cols[3], tc.cols[i]) + doTest(t, tc, NFKD, tc.cols[4], tc.cols[i]) } } -func CharacterByCharacterTests() { +func TestCharacterByCharacter(t *testing.T) { + skipShort(t) tests := part[1].tests var last rune = 0 for i := 0; i <= len(tests); i++ { // last one is special case @@ -274,37 +234,39 @@ func CharacterByCharacterTests() { } for last++; last < r; last++ { // Check all characters that were not explicitly listed in the test. - t := &Test{partnr: 1, number: -1} + tc := &Test{partnr: 1, number: -1} char := string(last) - doTest(t, norm.NFC, char, char) - doTest(t, norm.NFD, char, char) - doTest(t, norm.NFKC, char, char) - doTest(t, norm.NFKD, char, char) + doTest(t, tc, NFC, char, char) + doTest(t, tc, NFD, char, char) + doTest(t, tc, NFKC, char, char) + doTest(t, tc, NFKD, char, char) } if i < len(tests) { - doConformanceTests(&tests[i], 1) + doConformanceTests(t, &tests[i], 1) } } } -func StandardTests() { +func TestStandardTests(t *testing.T) { + skipShort(t) for _, j := range []int{0, 2, 3} { for _, test := range part[j].tests { - doConformanceTests(&test, j) + doConformanceTests(t, &test, j) } } } -// PerformanceTest verifies that normalization is O(n). If any of the +// TestPerformance verifies that normalization is O(n). If any of the // code does not properly check for maxCombiningChars, normalization // may exhibit O(n**2) behavior. -func PerformanceTest() { +func TestPerformance(t *testing.T) { + skipShort(t) runtime.GOMAXPROCS(2) success := make(chan bool, 1) go func() { buf := bytes.Repeat([]byte("\u035D"), 1024*1024) buf = append(buf, "\u035B"...) - norm.NFC.Append(nil, buf...) + NFC.Append(nil, buf...) success <- true }() timeout := time.After(1 * time.Second) @@ -312,7 +274,6 @@ func PerformanceTest() { case <-success: // test completed before the timeout case <-timeout: - errorCount++ - logger.Printf(`unexpectedly long time to complete PerformanceTest`) + t.Errorf(`unexpectedly long time to complete PerformanceTest`) } } diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 9a4b6f930..7966668ec 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -49,6 +49,11 @@ var ( eventSub *events.BufferedSubscription ) +var ( + lastEventRequest time.Time + lastEventRequestMut sync.Mutex +) + func init() { l.AddHandler(logger.LevelWarn, showGuiError) sub := events.Default.Subscribe(events.AllEvents) @@ -179,6 +184,9 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro ReadTimeout: 10 * time.Second, } + csrv := &folderSummarySvc{model: m} + go csrv.Serve() + go func() { err := srv.Serve(listener) if err != nil { @@ -293,8 +301,14 @@ func restGetCompletion(m *model.Model, w http.ResponseWriter, r *http.Request) { } func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) { - var qs = r.URL.Query() - var folder = qs.Get("folder") + qs := r.URL.Query() + folder := qs.Get("folder") + res := folderSummary(m, folder) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + json.NewEncoder(w).Encode(res) +} + +func folderSummary(m *model.Model, folder string) map[string]interface{} { var res = make(map[string]interface{}) res["invalid"] = cfg.Folders()[folder].Invalid @@ -322,8 +336,7 @@ func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) { } } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(res) + return res } func restPostOverride(m *model.Model, w http.ResponseWriter, r *http.Request) { @@ -598,6 +611,10 @@ func restGetEvents(w http.ResponseWriter, r *http.Request) { since, _ := strconv.Atoi(sinceStr) limit, _ := strconv.Atoi(limitStr) + lastEventRequestMut.Lock() + lastEventRequest = time.Now() + lastEventRequestMut.Unlock() + w.Header().Set("Content-Type", "application/json; charset=utf-8") // Flush before blocking, to indicate that we've received the request diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index f15e847bb..94b65336d 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -61,7 +61,10 @@ const ( exitUpgrading = 4 ) -const bepProtocolName = "bep/1.0" +const ( + bepProtocolName = "bep/1.0" + pingEventInterval = time.Minute +) var l = logger.DefaultLogger @@ -613,7 +616,7 @@ func syncthingMain() { } events.Default.Log(events.StartupComplete, nil) - go generateEvents() + go generatePingEvents() code := <-stop @@ -701,9 +704,9 @@ func defaultConfig(myName string) config.Configuration { return newCfg } -func generateEvents() { +func generatePingEvents() { for { - time.Sleep(300 * time.Second) + time.Sleep(pingEventInterval) events.Default.Log(events.Ping, nil) } } diff --git a/cmd/syncthing/summarysvc.go b/cmd/syncthing/summarysvc.go new file mode 100644 index 000000000..8f252e4ab --- /dev/null +++ b/cmd/syncthing/summarysvc.go @@ -0,0 +1,190 @@ +// Copyright (C) 2015 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package main + +import ( + "sync" + "time" + + "github.com/syncthing/syncthing/internal/events" + "github.com/syncthing/syncthing/internal/model" + "github.com/thejerf/suture" +) + +// The folderSummarySvc adds summary information events (FolderSummary and +// FolderCompletion) into the event stream at certain intervals. +type folderSummarySvc struct { + model *model.Model + srv suture.Service + stop chan struct{} + immediate chan string + + // For keeping track of folders to recalculate for + foldersMut sync.Mutex + folders map[string]struct{} +} + +func (c *folderSummarySvc) Serve() { + srv := suture.NewSimple("folderSummarySvc") + srv.Add(serviceFunc(c.listenForUpdates)) + srv.Add(serviceFunc(c.calculateSummaries)) + + c.immediate = make(chan string) + c.stop = make(chan struct{}) + c.folders = make(map[string]struct{}) + c.srv = srv + + srv.Serve() +} + +func (c *folderSummarySvc) Stop() { + // c.srv.Stop() is mostly a no-op here, but we need to call it anyway so + // c.srv doesn't try to restart the serviceFuncs when they exit after we + // close the stop channel. + c.srv.Stop() + close(c.stop) +} + +// listenForUpdates subscribes to the event bus and makes note of folders that +// need their data recalculated. +func (c *folderSummarySvc) listenForUpdates() { + sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged) + defer events.Default.Unsubscribe(sub) + + for { + // This loop needs to be fast so we don't miss too many events. + + select { + case ev := <-sub.C(): + // Whenever the local or remote index is updated for a given + // folder we make a note of it. + + data := ev.Data.(map[string]interface{}) + folder := data["folder"].(string) + + if ev.Type == events.StateChanged && data["to"].(string) == "idle" && data["from"].(string) == "syncing" { + // The folder changed to idle from syncing. We should do an + // immediate refresh to update the GUI. The send to + // c.immediate must be nonblocking so that we can continue + // handling events. + + select { + case c.immediate <- folder: + c.foldersMut.Lock() + delete(c.folders, folder) + c.foldersMut.Unlock() + + default: + } + } else { + // This folder needs to be refreshed whenever we do the next + // refresh. + + c.foldersMut.Lock() + c.folders[folder] = struct{}{} + c.foldersMut.Unlock() + } + + case <-c.stop: + return + } + } +} + +// calculateSummaries periodically recalculates folder summaries and +// completion percentage, and sends the results on the event bus. +func (c *folderSummarySvc) calculateSummaries() { + const pumpInterval = 2 * time.Second + pump := time.NewTimer(pumpInterval) + + for { + select { + case <-pump.C: + t0 := time.Now() + for _, folder := range c.foldersToHandle() { + c.sendSummary(folder) + } + + // We don't want to spend all our time calculating summaries. Lets + // set an arbitrary limit at not spending more than about 30% of + // our time here... + wait := 2*time.Since(t0) + pumpInterval + pump.Reset(wait) + + case folder := <-c.immediate: + c.sendSummary(folder) + + case <-c.stop: + return + } + } +} + +// foldersToHandle returns the list of folders needing a summary update, and +// clears the list. +func (c *folderSummarySvc) foldersToHandle() []string { + // We only recalculate sumamries if someone is listening to events + // (a request to /rest/events has been made within the last + // pingEventInterval). + + lastEventRequestMut.Lock() + // XXX: Reaching out to a global var here is very ugly :( Should + // we make the gui stuff a proper object with methods on it that + // we can query about this kind of thing? + last := lastEventRequest + lastEventRequestMut.Unlock() + if time.Since(last) < pingEventInterval { + return nil + } + + c.foldersMut.Lock() + res := make([]string, 0, len(c.folders)) + for folder := range c.folders { + res = append(res, folder) + delete(c.folders, folder) + } + c.foldersMut.Unlock() + return res +} + +// sendSummary send the summary events for a single folder +func (c *folderSummarySvc) sendSummary(folder string) { + // The folder summary contains how many bytes, files etc + // are in the folder and how in sync we are. + data := folderSummary(c.model, folder) + events.Default.Log(events.FolderSummary, map[string]interface{}{ + "folder": folder, + "summary": data, + }) + + for _, devCfg := range cfg.Folders()[folder].Devices { + if devCfg.DeviceID.Equals(myID) { + // We already know about ourselves. + continue + } + if !c.model.ConnectedTo(devCfg.DeviceID) { + // We're not interested in disconnected devices. + continue + } + + // Get completion percentage of this folder for the + // remote device. + comp := c.model.Completion(devCfg.DeviceID, folder) + events.Default.Log(events.FolderCompletion, map[string]interface{}{ + "folder": folder, + "device": devCfg.DeviceID.String(), + "completion": comp, + }) + } +} + +// serviceFunc wraps a function to create a suture.Service without stop +// functionality. +type serviceFunc func() + +func (f serviceFunc) Serve() { f() } +func (f serviceFunc) Stop() {} diff --git a/gui/scripts/syncthing/core/controllers/eventController.js b/gui/scripts/syncthing/core/controllers/eventController.js index 85c30763d..f72da69ba 100644 --- a/gui/scripts/syncthing/core/controllers/eventController.js +++ b/gui/scripts/syncthing/core/controllers/eventController.js @@ -1,3 +1,5 @@ +var debugEvents = false; + angular.module('syncthing.core') .controller('EventController', function ($scope, $http) { 'use strict'; @@ -20,7 +22,9 @@ angular.module('syncthing.core') if (lastID > 0) { data.forEach(function (event) { - console.log("event", event.id, event.type, event.data); + if (debugEvents) { + console.log("event", event.id, event.type, event.data); + } $scope.$emit(event.type, event); }); } diff --git a/gui/scripts/syncthing/core/controllers/syncthingController.js b/gui/scripts/syncthing/core/controllers/syncthingController.js index 610f830ba..e7615d82e 100644 --- a/gui/scripts/syncthing/core/controllers/syncthingController.js +++ b/gui/scripts/syncthing/core/controllers/syncthingController.js @@ -140,19 +140,11 @@ angular.module('syncthing.core') $scope.$on('LocalIndexUpdated', function (event, arg) { var data = arg.data; - refreshFolder(data.folder); refreshFolderStats(); - - // Update completion status for all devices that we share this folder with. - $scope.folders[data.folder].devices.forEach(function (deviceCfg) { - refreshCompletion(deviceCfg.deviceID, data.folder); - }); }); $scope.$on('RemoteIndexUpdated', function (event, arg) { - var data = arg.data; - refreshFolder(data.folder); - refreshCompletion(data.device, data.folder); + // Nothing }); $scope.$on('DeviceDisconnected', function (event, arg) { @@ -215,7 +207,6 @@ angular.module('syncthing.core') var stats = arg.data; var progress = {}; for (var folder in stats) { - refreshFolder(folder); progress[folder] = {}; for (var file in stats[folder]) { var s = stats[folder][file]; @@ -241,7 +232,6 @@ angular.module('syncthing.core') } for (var folder in $scope.progress) { if (!(folder in progress)) { - refreshFolder(folder); if ($scope.neededFolder == folder) { refreshNeed(folder); } @@ -258,6 +248,30 @@ angular.module('syncthing.core') console.log("DownloadProgress", $scope.progress); }); + $scope.$on('FolderSummary', function (event, arg) { + var data = arg.data; + $scope.model[data.folder] = data.summary; + }); + + $scope.$on('FolderCompletion', function (event, arg) { + var data = arg.data; + if (!$scope.completion[data.device]) { + $scope.completion[data.device] = {}; + } + $scope.completion[data.device][data.folder] = data.completion; + + var tot = 0, + cnt = 0; + for (var cmp in $scope.completion[data.device]) { + if (cmp === "_total") { + continue; + } + tot += $scope.completion[data.device][cmp]; + cnt += 1; + } + $scope.completion[data.device]._total = tot / cnt; + }); + $scope.emitHTTPError = function (data, status, headers, config) { $scope.$emit('HTTPError', {data: data, status: status, headers: headers, config: config}); }; @@ -325,31 +339,25 @@ angular.module('syncthing.core') return; } - var key = "refreshCompletion" + device + folder; - if (!debouncedFuncs[key]) { - debouncedFuncs[key] = debounce(function () { - $http.get(urlbase + '/completion?device=' + device + '&folder=' + encodeURIComponent(folder)).success(function (data) { - if (!$scope.completion[device]) { - $scope.completion[device] = {}; - } - $scope.completion[device][folder] = data.completion; + $http.get(urlbase + '/completion?device=' + device + '&folder=' + encodeURIComponent(folder)).success(function (data) { + if (!$scope.completion[device]) { + $scope.completion[device] = {}; + } + $scope.completion[device][folder] = data.completion; - var tot = 0, - cnt = 0; - for (var cmp in $scope.completion[device]) { - if (cmp === "_total") { - continue; - } - tot += $scope.completion[device][cmp]; - cnt += 1; - } - $scope.completion[device]._total = tot / cnt; + var tot = 0, + cnt = 0; + for (var cmp in $scope.completion[device]) { + if (cmp === "_total") { + continue; + } + tot += $scope.completion[device][cmp]; + cnt += 1; + } + $scope.completion[device]._total = tot / cnt; - console.log("refreshCompletion", device, folder, $scope.completion[device]); - }).error($scope.emitHTTPError); - }, 1000, true); - } - debouncedFuncs[key](); + console.log("refreshCompletion", device, folder, $scope.completion[device]); + }).error($scope.emitHTTPError); } function refreshConnectionStats() { @@ -412,7 +420,7 @@ angular.module('syncthing.core') } console.log("refreshDeviceStats", data); }).error($scope.emitHTTPError); - }, 500); + }, 2500); var refreshFolderStats = debounce(function () { $http.get(urlbase + "/stats/folder").success(function (data) { @@ -424,7 +432,7 @@ angular.module('syncthing.core') } console.log("refreshfolderStats", data); }).error($scope.emitHTTPError); - }, 500); + }, 2500); $scope.refresh = function () { refreshSystem(); diff --git a/internal/auto/gui.files.go b/internal/auto/gui.files.go index 5da44d1a5..1eca14ea3 100644 --- a/internal/auto/gui.files.go +++ b/internal/auto/gui.files.go @@ -187,12 +187,12 @@ func Assets() map[string][]byte { bs, _ = ioutil.ReadAll(gr) assets["scripts/syncthing/app.js"] = bs - bs, _ = base64.StdEncoding.DecodeString("H4sIAAAJbogA/7RUT4/aPhC98ynmt0JK0C8NUKknRHtod6U99dCteqh6MMkkseTYyB4vQhXfvWOHDRCCtqpaXwjz/z2/sdC1V8LmrSm9wjRxe11QI3WdF8ZiMpsAH/7WZI1SaNPk/hk1fewNSQaV5xxpNKRTV5gtZjBtiLYz+Bmzw0m8Q3BkZUHJatKbu/hcCUexLKxBe6VWfcCzsBC8j5/YtTjLDA7niwKde9DsO81QChLnrcOZz+Fbgxq+vKADi46EJQe7RioEahCUYfuWQQU/I9bYVZQOpB6W21pTcw0XMzfW7Bxa4GBnWoStElQZ2zpuQ95qBwLeLhaQOqmL2GxYrkFRonXQCAcb5Ekr5V2DJewkNbFHV4nnKjHUmmWdSxsIgPNhwaeGx94IJwuh1B5aFDrMKigWO0PXd2Q0SBkIXXYhnDcsuuNYbQhEQT6W5RtgDiqvLvvLCtL/xu4hHLTW2Afd3dPqyt3hvLQfJhd/j6qZYispTb4+ftZ8ZazV1eRqiqN23sNibJTIHF/UvSia9KQgDFIciw+HqXNGsWZNnd7FyLsM4m8uy5cv2oct6L5v4LwCcpU5knSY/Q4x5+sUun+POBXqmgXzBpY/Lov0+zVMZ0ADSlkhT7JF4+mMrjGm4gOQ10ipt4pliPA/JPMIy32IW7BO2NS1no2Skx/XO+3X/EZcFFR6lNWQoAze8bKcjIfBG3JMe/UFudJcVY2K7u8wpCS3WS+Tf8vMcnGTmj+Y7rWpRqcJgv4FAAD//wEAAP//QKHce4MGAAA=") + bs, _ = base64.StdEncoding.DecodeString("H4sIAAAJbogA/7RUQYvbPBC951fMtyzY4XO9SaGnkPbQ7sKeeuiWHkoPij22BbIUNNKGUPLfO5ITr+M4bCmtLnFGM2/ePD3pWVgocePr+2fUjmANlVCEq9lM6NorYfPWlF5hmtBeF66Rus4LYzGZz4AXf2tnjVJo0yRCfOwDSQaV5xppNKS3VJgtZnDbOLedw89YHVbiCYGclYVLuOsp3OXnSpCLsExMe6VWfUIgHnYfP/HWYlAZNsgXBRI96DBPz6EUTgxbh3V3B98a1PDlNB1YJCcsS7FrpEJwDYIyHN/yUGGfJ9bYIUoCqcdwW2tqxqBYubFmR2iBk8m0CFslXGVsS9zGeasJBLxdLCAlqYvYbAzXoCjREjSCYIPMtFKeGixhJ10Te3RIzKvEgDXPui1tIAycjwGfGqa9ESQLodQeWhQ6cBUugg2m6zvyNOgyELrsUrhuDLrjXG0ciML5CMsnwBpUXp33lxWk/02dQ1horbEPujun1cV2N+d5/DA7+3t0zS220qXJ18fPmo+MvbqaXbA4euc9LKaoROX4oO5F0aQvDsJgxan8E+rgKl1LC4tVJqPY3qZObyLoTQbxN5fl6cvtw4Xpvq9I0kkwFT1T4gJvAuow/x1lh/cxcPoehVKoa3bcG1j+OAfpL+i4nMccnQlb7Em2aLwb6D2lYXxB8hpd6q1iHyP8D8ldHIs+xGu0TjjUtZ5PipMf34e0fyeu5EVHpkdfjgXK4B3ftpfgYfQIHctefYIuTFtVk679OwopyW3Wy+TfKrNcXJXmD9i9xmqSTTD0LwAAAP//AQAA//8bMA983gYAAA==") gr, _ = gzip.NewReader(bytes.NewReader(bs)) bs, _ = ioutil.ReadAll(gr) assets["scripts/syncthing/core/controllers/eventController.js"] = bs - bs, _ = base64.StdEncoding.DecodeString("H4sIAAAJbogA/+w9a3PbOJLf8ysYXy6SxorszKt27XimvHZy65tHUuPJ7gev94qWIIsbitTyYcU7yX+/bjxIAmiAoO3MzlQtKxVLItBoNBrdjUajEWfXdRoXs3W+qFM2HpW32bxaJdn1bJ4XbDR5FMEDn7Nlcj1e1vAyybPxkzSfx/jpTZHfJAtWTKJfeEl8rJezVbVOv/ohX7BxVdRsMlvF5epNwZbJ+/Ho8WhyyKt+bNuqijxNWTEenStsTpofR9NIoRGNn5TzfMOm0ZNVVW2mbcvT6Hv4lLJzVtwkc9bFblSXLCqrIplXo8NHzc97e9GmSG7iiu2tWLphRbQA/LIEoZVtsZu4gHLs5hQKRkfR/qH2JotvkmtAILs+3sa38H4ZpyXTy+RZmmSMflewsooLrN++bwo0vUasWnqMu53DR+v5LK6r/ISPnvh9PDnUSpesOssqKBynkpozGBhAZDWNnu/D0yn/8ZFOr/oqme/xOjSxJLx5vt6kjKN+FP3y8dB+j+j53p1lyAhQAtmHKpMxTpqSBsKKIi/w3cWl9Q74nqV0tfXt2Sm8GY2sNwuGtHVAFC9/Yv/w4rTMU5gZfaU2RV7l8zw9WcE8ZQuba5oh2+RFBUwZ03DEe5hyNwnbOqEInByolIxlL5GQNEnqzXURL9hZtsyhQFanqYMw51VceWniKQDkuAbWVG/b1+Ntki3y7WR2BX/Hoyu2BOFVZ2keLzR5Yc4Va8LqDPZxcmhx8xMQfzuNoBEDc17PgR3KHV9T+rQEISTnYwMKKBwX89V4MksB5sSPRcbY4rjlHK2lUbEeHUSjU5aOpuaLRVLId9EYPk/MEij+sQDON/NdldfzFb58u1mA+Bu1CFoDheidzT3IFWyd3zAnfvRrhRyMtBxaB4JxCRItKd9pKFIDOXp79prLY41L2A3LqmkUF9fmICbLaCwF+NOn0eNWXpsF8SlYVReZLm4/PtK+IonylM3S/LqDS3e8BSAukM9voVtrU4DLl0LGu19KCcknl6PUaTs/HSVetRN0bCLJVfDsmlXjukivYlCyu9Fo7wbkCTQ7msxKMUfGLZmBiWKKbnJ8ZFXgICyovhrknAjhrlQXWyfVn3/++Q0XVGEYCtF4FwQ1oYtlHxw3KVTvgpwujz3YuSWWE5ou3QU0oz+NlaMLVHwoI0drcjz6r4xV27x4x0kFvQclDebJaJUs2Ghil24h9pctV3WF8sNd0iX0cX4ulwOFhaFgPnyIHgvSPJDAkAgZHXXZmAqrHsHlHIGJiVsvzRqGD6QZmJUvYZ5UYOtsVyyL4gghRKApU+hMkpaz6K8sQhO+WqEZH1d1CURZwPcceO3WBAa/glisCxbldQUg42pURtsih/XNo193sLqUwI4/6IhhEQA6kwQ5OoKlCVVOUuU4ksMbcRkwjbK8iuJM0Jr/RNZUA4uSzMN9nBIRg36YeH1zFH25v4+qs/Pji6Poqz/+0YvtNctqJE8ffoMkR7dWqATp1umXJGYNlPHeCcUpN3SKoTpmcoEQOMtwrbkQegsHwtYOOG7dBdIFV8HCPr/06Am7MB9lppR4lQ+WINxMPssW7L0wOx+sj5pNM+7gPNz2ASYVyEWdta5kb1iJRCi91JqxAhkUbUF2reICpViCRRB0tE2q1ewRQVW5LtPpKuHB9+JlDMuGjmXA35wsHUaptAgVnm1xCfLsdBq5qfExRFP+BOZ7xX4Dg9btJpYVPXT1z9kfYRmfJqV0NQT3B2YDA66wHRUXqnuzZHE5xBrvwfFkIIJct/Sg55nvrirWsq9pL7valAfR/pR8Czra9zrJ/nRbsfLnvIpTH4yAUvFigY6Eg4bNZviLLYxt+Ww5tcI6/n+VwOf5/n5fKwFCUXn0wCYnRpoaZM2fNss3fMxmdXEM64oNmltOmwGE29kSTK74mkViyYOm+youoysGFlrGQGyB9IoVoCxH3+kcLYOFTXyABsJvG2cV2mZx+Y4bcmDQFfh9Hb9jYPPNVzk6SaI/1VxULmABWfE6FDiodlVfI5h1tKgLRI67IuMUnZv1ZhqVORe4rELQef4uYVzWksAAmSpZg7m45J+XSVFW0U1SJhVYnmiRcvktoYDs3gAbMRqxOFu08KDoOucCH+ysZV4X0Qr+K6P4Op8idpISFJx/1mCdoIPokfUW5SVH8S+IIWrZfF6vAZ+ZwBDXqGk8Z+O98bcH8O/vH2afHf6t/GzSVoJvfzuC/8YXfz+8/Gwy++zJ5MPf4f+9abTz5PkOYaFwmdECcFluBiqA3E5b6WgHFrjoRJ9l+RZWnrvRzuE6fv8MmIy/+mI/+iz6/Ev474uv9/ed9qVDyACCux2yvOi29CxSUOEPerhd+OODVlvtNdc4Mvc34k47TuNg4e1wOLcCSby5FNo0QIe86jilh2JhOrRbLKR5A0P8DIf2ztgJoXce3wSjVnPjg1uQ0j+mGg/zvwhpucf9jnfwwRibF7zT3d+G+Yv6eUi6Rd9IL/kAm6uUPnfa6BKbXprrvfsaLdwxl0RioBNh+5Ye81Pab6Tpho9q7UJau0SrestJypp2LzzrlKbDAFErfIEgLun5LbbmQEfhBsxzLjjKmfxhDz5y1e6uCiOUsMWrIl+/LpLrJOsAsV4NAvcSROAWdC8jIbZvA4Bu6jTVuid/CKwqnHnduvhLT2VQbX9l6LRBFV7kNajLetOAQ+ugSllcotbesGIO/IuanOtfEOrbvE4XYIJ0Sl0lQlVfpWzmVAwK/ovoOToiZD93SbLu2mO3qzjhxVHk9lrg06FLqNrg1QzWF4zptCvxERgdyL+0wYuP2ZkD65eQug15Dqgf3RAEpQ/kX385oNuB+uAuedWx88tZ+62nxmmesaYCfqHLE/LGpdbxIaSgsWvpctw9HrdVmrIuzgoUoAp2Zz8OBko0A4a+rOflKd7Oj1DR3wph/LS+vzs1zympJLpBwz7Zrvr9uAFg1fRVHdRz9VwVLH7nLkJP83CbER97/1t9PHR6nHdMe2BnanGk36rQLBD0TWuGz1R6uKbRisXon5pGwrhxGIjSbdz1g/+CcA6iLrQDE+qBCf5A/tU8UYd6iM6CXYE+mQPPwf9WuEDTD3IyEebRO4axATtacTRlRQ3CW6+3fwH1SY4liqGdKH/t2ZrjpCXNVu6F/VYgdzSCn1iGWyRvfzpDfxgIvKxSnR1i1RrjKXy9rYFm243dR+NMnZBTSUfhlnOJmUDruCkvwpemfAPSuW/lGISxHvSkPjYkstcVNOcj76zi8kQFOD1OypfrTXX7+gpXSbpPxlqRGMFR4sOhu0zj1EmTsmLZsfBwnVc4cwMKzv6RJ7CImEbWrilZ9zrNr+L0OMs47TCsBaaorzGygtaoe21bWlDl7ySmd/SM24492zP+MP49msAK7TIvKoUp4BIXJvta0Vri0w/xxvDyyRJGdcF8M+DzcqyDmhAkc6vqQEvE2MO45/ZFp+GeLQwnQib5rdgFLsGbaethFanRNGese4vcrX9UYI+pN0nhXvLCd3FHyHhG7obAz86xEk04RbosFetTmVveCjx7X6mZ/vq76FuN5+wCk1nKsutqFR1EhKuP29RxIpanF8QSvbG8S44JGn1EGx5f4WO7+IWA5bUzBVKw2i1XY1H8jv5BN1FfqX6LtmzwlFoV7LRDqtNgJ5ObW615N3XY80haUYJvLnS48I7xFLYp1uKC5phs7Ddrl7Wq5VuBKjfOGqxHTz+ZyaZv8xkKzsvkvDeuig63XDt8gyEa9mSnxKG9+6Ee5AtQvxiP71764zPPKj1o33waUTJfbzqrz+EkQ4IjDGT8HWEb7PTV4RjmWZVkZsic+bgJiw+SYvfIQ2NAzOHpbNDIOAiH48qPgrPdmaADBgUChnvYiGdQKcnWme7TSBc+U89Y/Q5WFK5A3SCToLMLP8gu4EdWcjwP0O6P0ROoQh00xrLPmuMvExhCThBary6Ise2cnAFYDk2eLJQG79fZYKW93uI5ow0rqluo6vXu+CcXzdHonnZDRCQuksXljMc1QLd+iKvVbB2/HwOfjDtv25AEoCARPGGWQuJWC4/jqYEtYiacTWvhEO62tWLexj9G87iar6IxGRXpJIxH6NpdcRQeYEfpZ4JoO5aWME29T2ZA8aKhc1ucXbqLud+cehLWLf8WRgSB4Cc0IMVRhVDZBoUH9d/2zjj8Sg4GgBp363twd35rO8rusep6wQPGa2cPXf3K9bjzYHbsnfcSqCHGPoU5HbUmnXLk432JLvaXzai/sBUHPQp8a3tP2D87d+Az/age3e3GVpYLmNZc7tR2LpCsko2VmMZldY6RZWAksC03TcYBxf1D6Kt5Gt+W3LxRrU1aPeltURpA8OcPX39JRikFiZzOmN9T5k6jr/iBXYqzXmmHK+/JWWLu3IWz9DOePZxlbaV2avuMQ7t043fEwXuVpF7jJaD6LK4I/uypcQ/3EMU3ndYemm/0k+Da1p9JuMDDiQHnD5VdRG/rWSSuSw0v8YJwGyNDVLcbli+pnSt0GvO4XVAlozrj59jRjev0UGGpdxme+/D6qrDVtgXl7ZZ+zhewtPa3wE8HLPqbcPcILO+bOIXF1GPsmbdDZZVvNn2tqcLonPK0yo97BIzgCcyLX3sA8ZiGGDsnKZJsmX+CgeUtizF1Nr2NiwwPIH3aMccDdjjPPIgs8ChR4cdDBQ0yng1gEEuobsjqUD9ZpMzPoELLWCi5QaKBzU9zeaBuimQdF7dDoM7jLHsIsA52I2YLduSNiH7DAHx7wgybLUME3XPTpLFJQ7Ugd3q5O8F9qsDdhsVnm3nVBBeSDSZ87SMa3KOLdHAydQ/HgztPlmkOyhKa86sgucc452fTA/Yt9eMXjduF2F3uWfr0bE0/fRq2h934YY84Vd3brIJH83cjwrL1xd+rmlKvU9VDNM06yerSPzta89ywBkIGQ5QRdnHZqaGEeR/vwizCeMtggf17HPwk466KOzKAksIDGQB0VPesHckci+5hvAAese2NT8oiXOEjdzyYuv89cg+tt3k/Q9iH1qSCTB6ihbBPv+IVfcbgqDCuEaH5WaaFKvlGzLaIsIJH3uDrmTy4GGJcfBs0L7rJuX473eyco96NRv8dZEz5uwvmzuJURiNYHbVDErCfa9zo6AafNUFLSVqxbuoUX3+yhhZoQ6rP5krdJpNsXYmax/41Y2PRBVBKQr7YvwxgkB/jNUUx/zqtKTPA3NzZ6bM2W+7KACkPKL1gAEVsvp2V9VVZFbiZ9rXfHMQj/Kc0qcjj7YoHreqmW8Si5xBijqWXQladBBI3jLLDyHoHmrJFUp2zCk8al16Kgqz/QZwZxqwfeNwG6utZYBSd15vXG7UhqLJeYmH6aDQdc9kCmdXFyyy+EtFZY+p1c7L6m2i/F5o21wjOIChuA8Fcj29FpiYvbp1yKvfjn71Y/s/bMz/RruuEyrUkR5A6P0stdeMb1oRK+yfRHCVL9L/nr3+cYSrP7DpZ3lrh1GYlGFk7Hx0+zUkD2hzBAM8K1t/PfobJyNPLbTZpIpL27f2jzLNRn31ikIY71jd5Se50TrFzU44s5V4nXesPutXYGZFh2428q3d1QdPsMEgGlJjmqczXjGeMjeYiJY7NO00uzceKo9k/6zgtaUEwtSfahKeG8laGCTHVJxAh3CUmjvXDyYrN32E5Z+oFJic5JlxISv7ZtwYghVdr+dPC64Unk5S3Il8guE/udyMS74TYN3fG6xkR1vbRDlbqDgFKzGcyCd5D0N+W1OF9IaX3UWhB4Nznnw9cjwXjQG6BegmrEtxG2yRNI9xo5/k8WDNLeSgCT87lX96qWYeZVX/+/px7wLX5J1/00NlOuGsnMHR36xgUwy3fFVSqL8Lsrml6a8tsh44nR7K1DmxMSKHl0NYdEdYHCEjpBiLEmQXhYqSdKMJjPeSpn9ElcezjHbsN0kaqiyoq2/kSt6vPq2LnclaCuq7GexfR9HJ3D2yReNNp973/FCy3Y9/PwMiwNjcbXnAcMnFRuLVz7ISK/TbUDmaZ2/GrTjljevZtO2k4bSZ3ZMUzrTzalJEVvSfJjJzepkY/3yYYZbhlVxvUfo2cgEkvwnFcfjJjBrviXhuAsKLCToyI8fIJGBfXmHBJ55fdFAzzz8ma5XUVcLpAJLueNYmjO22qj0Sj0+hzY5dfPU7ZRySj/OjjO6Uh78l3AsxwtvOlyyXt5iAed2NFJnwMTaw7BKLfTpZ5KO9FdZqgnRSX4RS18mLecUnQ6aL3FgCH+8Dp6yM8V6qZuihgndfUfIJnwli2GP/ycdq6iGgUsUmg7sv3oAGdVNaKnrN0ietz2/cUGaelaJp0kVVuYSaP4LYwmxfO871dS7yDmi8KVWu6ZCn3s78ibw8wquobKhS8YYdQQ7HqnCtyGHTeEx0a9i+BSHkxewIy+03Bh9ty3eE8aJkw0P+RUP5pi1UBcrL4Zye3bbnKtyM/ZGCCENCkG4FvtIHVdivvguk+hFQo2DWQpOiz5RqogJGqQg2K3WS1YlnICTx6SrtNve4MwjsGbsEaT8w7ELoPblFgDUADyq9ZFaN/xFMhwStbFvWcAXyuWd1FDe49gClFm560ReqUSYRCp+tI4dRfPnhKNPV6p0bTub4wDMytGjhliBZJXUsIQ0W+Ox5a1beOHm4vqbPA1SVf2A6Tbte6sDs0zfKzLJqjZNiyKI0rTPu5wNB4THMp9yjWcVbjincaiUs8+LvkOsuLODWhFWyeFwsy57FEjFdkUnrZaSD01zYtE9L+l8SUcXIDiaj90EQqi0NrWoQjGaGsl+CnnizaEy8DueRBOIXiFpqn6YWsw5X7wBPVgiwRIU2KPuX4I9ueNraXZTC6d/H4Bid5flwAO1CHVK0SwdomWMsEaJc+rWLuUzhI3GMJg2pfJ6X0ajX5QBUtA/mE2AfyhiQsMI1blOgIEVYwbRvDIEgn0Wg6CnIQeR1DlphA7GhtKk68ctdplEQvTOUg9v7h1e5uT4CPrHGRXHaXEUfEznbQkRwE1KUVrctlv2iLGh9HajRv+I443ZvTt24Y9OG5N1z86FXEwWl27qEhMcE/8BjPZvk7VpA2C/Ww+4AVpaFA26YMWeU77eOsdOFIXq8eccQI8456Ld0GT3ueUqq6Z86a2LshGXN5wFTWMRcddE/S7tOTyxAfd+YHwq+pHpEzHFEJTDZCUZbP9v4em9q3SzF/zzzn7R3rL9+2mb83D2XvqYey+3qnbkjv+zNV3ss2pM2FADNMv1tjoddvTmv6PS5cOqp8506PoctdSIrYjkpym6z09s/9Dagcr2JodYHPH8y55eHXonZisZ5lfLsD2hv1k0wjMw3Q/eynzFLnIHPtWcDvBtPNqq57tmch1m/+eKgDhkLIcLZLko5c6TKDyWqaSCX2oq0Tmp14vtLve0c3/vdJ2bf72OU/kZzCZj9fJCLgi/dqfNNMK3U/7SDum6csLl6qTBmepanRSEtlgfuF3hMZs/ssen7J0QzabeF19zhGPf7cZZGwbJHeUqxRVpajvDWy7jpR2pi7gOlS8i0I+L+5fWSuZWbUQGBQA743OY6MJwWYfrpwTRrKfG1pMw9mj8ICq7zKi1vZDqYkNMs82WKI9Xgk3RDCHp1t4mqlXcqQse1NnNa2giE98Rh0I+Pi2V6DBaZRtoYAljPxGhf4kcTgIFJt2ekqhmeHMCgw/O7Tthy9eafSe4QcD9Y9Pk1NLXSlrWy7e6n6fKhmZYqy8NnzSWdDTuTG5O/PGZIZ6NC/VdaBGpnOOLvF/WkEjQbsR+kQ1OqnlQrmRhxZrVH+1rYbqf1Dmr7ItIy59tKDOu9AApa3/uISunNGyVFkhtHyIuVemeA8IVPu0R3g5TFPw1/aFl2LJhLCUqt7zgnC9YTCJRCOKP0dYxuou9vXYzHRZ++guEFhKwn/AxG2iq+vGdi7A2irqnwi8jYYhVJYVfghfn/Mz1R3Tv+GUnwt6srUL55wOrrpE9DymQqdHDDQ8269oY1K+pVvPGLIbvKmU+vX4TGMuigyR8ZMEoyq8Wk4rMEnEJIqf5Kv1zF3voSOrqhAUjmQEu5eZHlmCqEQpdIVR30lPnyIvgrQOM5ZMKQ0tGVfDXd//qcLQ2N4PM10z5qSZB5nEU/Akd5iDPO/WIGX+60SHsgYlSt+ZRHetSwNYxMe3kooXcXLuE6rSFhs+RK6+tUsOs/xnuhbfpNhtxRMwKQalSa0eF5xR3N7NnBmKV09U0SPnOw/fRYob6E3gxkxcFKZxaiRc4ZHOEO2hP04IPBHYBMY+BMvFoTBG2jn2oNgGGOB23w6GQuGuU7UZDuHdr4OmWqDpE+/vBkkTNzcNUwI3VOqmMdYA5k5qBquAH/MizWImH/RG29DIn9+Xd4+zhbn/Mpre1GngisdHC/9oQJM6w816vaTj5wsuE0gIX2SiWSujKRbmo57/89U/M9U/HRTkecfEzX+CoYG4XQPmo5LPaK2b3fO3I3z+gH0nZVlE1Ec4qrr9jRwiyVYpDhiVQK0dpvG8gSP9TZxKiHDSQY+NcAcFpCjdGfb0bxDpS1jCqvOLgftxzFywJ6d4ia+E54rhMWNQMeLdCTad+222Cn5enaLTb507A2Th7s8tqvclXR2iQiVaMu6ZHaPR6sFcNNd+9J9H6lT7AKgI1559IavSEeuM/G8DHqdENKouXKFd7hRGzRBbXLaFLapSPjngqt95/OOBVLf6/YaNgCVGgAF0zUGm4AxEH4oNQrdqWQo48+kp8oddD7SnEvEwJJq2gOv6znygOtq7YdkGdrtOKSmIN2QGid+/5y74l80L9v9mNXrP7ubsFAg7yMupHfL5gTDBHtAHqD9ggMqngzyyFlgbhxt0jEzumXVJLZt4d3fWAq3ltyhOobVNMCC1EN2ljoc1W88t/2svc7LH8HDbdpyyHYhv/QmXvdYQu5NMbpn+HSyKHVjsAJCRTg+WixGZ3NcUavJX2Y2MjH3zjmtvHa2aJCHwNKRHqIAPzK4M4126HsLNFxftfwXFrLdMqw9DFr+d2FVGqzsCFJwBtcpUM16hRCMdpTEgJjOdvTvGbgpMRbsoND2RCZ6QjZ7Q/esaCrRtIcxZBG/nODTPMSlOWDd8/Ansyhp1Kx59NU+yOF/s+zFLp/xSEN/oNInOMAmB0m2/qe6qnI83B1XVTEeqcQ3eLC4+Uwe1zfDWUTcZNlzT7VjOCZBh1F9Ka54MiuBgwxhUd8+fNBjeihihHCsevj1hHi7KExUFEw6PfkrUGKxdQWyelTdGZiU4w6i8kT332AwXFWNtlpHkVOazPKM92bBstlVKYprMUu98du9rinzIU75atjgAec7ItOQbpnPaysIUkeC0KH3OH7smDTiyMexPnWGn31FsfFvkgZ07OL9JjMVwiYgHnimC58PE3VwC6eB0+ahSMiq4zdn3/HrZFsCzm1bBYMX400iShawGMjX5zzn4PiLz/scrvn27U9vClD6bNurCK1MP/S5R8wbUkiY7ey607Tty9EYQMWYp1V7+1PQzqV+IbydKw43rb+PC1gPR9Uqzvged4rfyypiyn8u8uBFcmUVrlNb4rkJTNoy8zTJ2P17aGad+xTIIsPh7WbESsiBsX6bGnlxMnETnI2ogHNffiRxcmQ2tiq4i/bwMFY/JpayxC1RfFFyKuzGo+jL/T9+fUi8V+GxyNdff/GHLwmfL4c+W6bxdRk9hbYkzN1O7cmEe3PIV+6w+FGxXiTW5S2tB0lvWALXWvIDD4fcIiuAhyBf5fXcvC/C6W3p3jLgTBBOHZIB2VEki5DLVEhtt7PH7xbZU3ACrzv0y9GrvPYHreMs46UGZX7BIj9xiRmkhkSlwqjQm29JbFEfpynlBAikKULoT2sHhZzOngENPcSQXdXrDfoVSf8XJTwc+CCcMHyw+FOE7CyHrf7692oqSvwGLtXsDBMYE5uS1Ys8WsdJNoswD6dIKAofIq4oEpEsMy5LsDY7KQXxDSZ9LnKoUijFjGrk/wEAAP//AQAA///OgFqdL68AAA==") + bs, _ = base64.StdEncoding.DecodeString("H4sIAAAJbogA/+w9/XPbuLG/31/B+OVF0lmhnfua1jnfTRonr373kcw5aX9w3Te0BFlsKFLlhx33kv/97eKDJIAFCMrOtZ0pJxNLArBYLBa7i8VimeRXTZaU8aZYNhmbTqrbfFGv0/wqXhQlm8w+i+CBz/kqvZquGihMi3z6MCsWCX56XRbX6ZKVs+hXXhMfqzBe15vs65+KJZvWZcNm8Tqp1q9LtkrfTycPJrOnvOnHrq+6LLKMldPJmcLmefvjZB4pNKLpw2pRbNk8eriu6+2863ke/QifMnbGyut0wfrYTZqKRVVdpot68vSz9ueDg2hbptdJzQ7WLNuyMloCfnmK0Kqu2nVSQj12fQIVo+Po8KlWkifX6RUgkF89u0luoXyVZBXT6xR5luaMLitZVScltu/K2wrtqBGrjh7T/uDw0UYeJ01dPOezJ36fzp5qtStWn+Y1VE4ySc0YJgYQWc+jJ4fw9Op//EynV3OZLg54G5pYEt6i2GwzxlE/jn79+NQuR/R8Zac5MgLUQPah6uSMk6aigbCyLEosO7+wyoDvWUY329yenkDJZGKVLBnS1gFRFP7C/ubFaVVksDKGam3Loi4WRfZ8DeuULW2uaadsW5Q1MGVCwxHlsOSuU3bjhCJwcqBSMZa/QELSJGm2V2WyZKf5qoAKeZNlDsKc1UntpYmnApDjClhTlXbF05s0XxY3s/gS/k4nl2wFwqvJsyJZavLCXCvWgtUZ7OPsqcXND0H87bWCRkzMWbMAdqj2fF3pyxKEkFyPLSigcFIu1tNZnAHMmR+LnLHls45ztJ4m5WZyFE1OWDaZmwXLtJRl0RQ+z8waKP6xAq43s6wumsUaC99ulyD+Jh2C1kQheqcLD3Il2xTXzIkfXayQg5mWU+tAMKlAoqXVOw1FaiInb09fcXmscQm7Znk9j5LyypzEdBVNpQB/9Ch60MlrsyI+JaubMtfF7cfPtK9IoiJjcVZc9XDpz7cAxAXy2S0Ma2MKcFkoZLy7UEpIvrgctU669emo8bJboFMTSa6C4ytWT5syu0xAye5Hk4NrkCfQ7WQWV2KNTDsyAxMlFN3k/MimwEFYUX01yDkTwl2pLrZJ6z++efOaC6owDIVo3AVBTehi3XvHTQrVXZDT5bEHO7fEckLTpbuAZoyntXJ0gYoPZeRoXU4n/5Wz+qYo33FSwehBSYN5MlmnSzaZ2bU7iMN1q3VTo/xw13QJfVyfq9VIYWEomA8fogeCNPckMCRCxkBdNqbCakBwOWdgZuI2SLOW4QNpBmblC1gnNdg6N2uWR0mEECLQlBkMJs2qOPozi9CEr9doxid1UwFRlvC9AF67NYHBryAWm5JFRVMDyKSeVNFNWcD+5rPfdrL6lMCB3+uMYRUAGkuCHB/D1oSqJ6nyLJLTG3EZMI/yoo6SXNCa/0S2VBOLkszDfZwSEYNxmHh9dxx9dXiIqrP347fH0de//70X2yuWN0ieIfxGSY5+q1AJ0m8zLEnMFijjvQuKU27sEkN1zOQGIXCV4V5zKfQWToStHXDe+hukc66ChX1+4dETdmU+y0wp8boYLUG4mXyaL9l7YXbe2xhpm2YQn1/ANK3ZDggBJ/9ccGfGcCfCFDtJK7m3De4EyM+A3PbO+FwRIU6XF2PMvwEcn49EkAuzAfQ8DOZqYu0z2v7yy211FB3OyVJQCr7iNP/Dbc2qN0WdZD4YAbWS5RJ3rkctM8b4i736bYFgeVHCBv5/tcDnyeHhUC8Bq1C5kMAIJGaammTNgRMXWz5ncVM+A0N2i/rdqaRgoZyuQMcnVywSNjbaiuukii4ZmAQ5S0H3l1GiAOUFOusWqIqWNvEB2g2LbpK8RmMgqd5xywEsiBK/b5J3DIyMxbrAXXn0BzQSWLSEHUvN21DgoNllc4VgNtGyKRE57vtKMvSmNdt5VBUIBb4g6KJ4lwICgDQJDJCp0w3YJyv+eZWWVR1dp1Vag6mDJlANRouCklbofawYjViSLzt4UHVTlGgngWJfFU0ZreG/KkquijliJylBwfl7A+oQPRKfWaUoVTmKf0IMUawXi2YD+MQCQ9wUZcmCTQ+m3x/Bv79+iD9/+pfq81nXCL795Rj+m57/9enF57P484ezD3+F/w/m0d7DJ3uESuQyowPgMhUMVAC5va7R8R7sqNBrG+fFDWx19qO9p5vk/WNgMl705WH0efTFV/Dfl98cHjoNGoeQAQT3e2T5tt/T40hBhT/oUnXhjw+aCY3XPuDI3N1qOOl5KYOFt8PD2QkkUXIhdG6ADnnZ84KOxcL0oHZYiBKc4sc4tTtjJ4TeWXIdjFrDLQJuskiHjOo8bMMvpOUBd3TtsOk3vOV80P3fxjkohnlI+uFeS7fsCMuskk5e2jQTpyyar7dfvAKBP+WSSEx0mgt4FGUUmHNpjxLgdJBpxlqA5x6Ltx0JQNQqnyOIC3rhikMeUD7oyn/CJUIVyx8O4CPX2e6mQPqULV+WxeZVmV6leQ+IVTQK3AuQbbDfLhkJsSsNALptskwbnvwhsKlwC/Xb4i8DjUFn/Znh9h91c1k0oAebbQsO1X6dsaRCdbxl5QIYE1U0V6wgrW+KJluCbdGrdZkKHXwJu3enxFfwv42e4JZWjnOfJOu+PXf7ihNgA+ze/+LTo0uoPuDNDNYXjOk0GPERGB3Jv7Qli485mCPrl5C2LXmOqB/dEASlj+Rffz2g25H64K552TPgq7j7NtDipMhZ2wC/0PUJeePS1/gQ4s04/3K5gB5MuyZtXRdn9Yx0PK4B6ou2YJYLKD6WlHvGn6Gh7DPcVOlcQzt1z8mjxLRBmCGB3SOUBGC19DUdNXL1XJYseeeuQq/dcAsPH/t4VH186nRI7pnae29usVmwBXfWbDZJeXtfrhmnN0kZNZXoLxi/5+3m+T49ZA/srXnfzvQaamQLwjghJ9rRnCRVV9ewQHGIoFMxisUWW4u81sNb8GkF02Kz7S2+EcNHqmFj3PvvCQ/FnmvBYRxQmpunRzZN8MFx7B8PkQd6JuwyHOq+pVrH0D0WI8HTLsDjACH6OVOztNHprxn4c3muMY/WLMGAjHkkjHjHRkj64/sHDL8inKOoD+3IhHpkgj+Sfz/21/1TPfZpyS7BvFqAtIb/rTiMdhyab3VKy3ME945h0MWeVh23bKIFseL0/s+hPclnRDVcD/LXgTNPTlpye8YF0vcCueMJ/MRyPHt6+8spChjQ/3mtBjtm92bMpxB7+jJ26w9NpuuEnEs6ziO5A6Xah+4C2/oiLmzOT3adB4KOSZjq0WTqY0sie/9Mcz7yzjqpnqvIsQdp9WKzrW9fXaI3QPc9WjtvI+pMfCAVkOG8zNKqZvkz4ck9q3HlBlSM/1akoIjmkXUcTba9yorLJHuW55x2GC8ES9TXGdlA69Ttw6ksqPJ3ElNZBhqmfJEs1n3OFmcBK/o42RaeqroEeXpyP35smsAK7aooa4Up4JKUJvtaYXDi00/J1vBmyxpGc8F8MfB5NdVBzQiSuY1cUng6zwVkF0pe7DpJvY47Y2lqT5SSKZRkMKWByXpcgrfL1sMqUqNphw7u2AO3/lERU6beJIV7xSvv4naTgaLcJMDPzrkSXThFuqyV6Ev5jbQtOHj2vlYr/dUP0fcaz9kVZnHG8qt6HR2Zphw+fIuZpMJbc05YRq29V3FM0OQj+vBsLx/Y1c8FLO8OTSAVb5tqPRXVd/SDu4n6Uo1b9EVYhYRaFey0R6rTYGeqm1utdTd37ISRtKIGN6R7XLhjoIrLGa3w+V70xk0e2TFUeHTvhpBrU+XeUPRm2G7g8PU6ecSGcB66mcLHu6HCh9xU4ROwsfKTYMy+imPi3FvRBMLHvcfybq/UwO0tFt2VEz65ywpat93CwrWrLa25h9afYIEbYbhBeqkX8jBKOfELKQVG+3eHkTRf1igIp1j3cXu5ZQYU5nY+zW9LgvS9ezEAy8Hl6VKpkWHFAabCqxu8RbRlZX0LTb3OuZ14urz1QEQkztPlRcyDSGBYPyX1Ot4k76ew/Zn2Srv4D6AgEali1kLi1kuP37CFLQJUnF1rsSfuvrVq3s4/RoukXqyjKRnz6CQMIdPcQ3FUHqHM9Rs/tDFFC4K23SfT4rxq6NoWN5N2sTnbO03CxOLfwoggEPyEVoy4iBAq26DyqPHbLgKHc8PBANBit7EHD+df7fjePVf9Q4yA+do7wJMa5f/au1ezb6ejIGqKcUxhni+tS6cc+XhXooszfzPEMswdSc8CDzc4EGbK3g58pl/Eo4fdmqTS3u+s0l7rAYu8V7M14rKkqs8wjA+MBHbDTZNpQHX/FPpaniS3FTdvVG+zTk96e5QGEPz53TdfkSFhQSKnN+d3lLnz6Iuv+X1cirVeancn78haYvHswlr6Fc4B1rLOt3utA06te7Vb7xfO3kvY0Qe42X3N46QmGHSgxR2cFBTj9Hq7d8bRb3prJ1Am5QIvHwbcL1SWEX26ZNG4qTS8RAHhvUSOqG+3rFhRByjou+Rh0qBMJk3O76mjN9HpKMFa73K81+F1mWCvXQ/K6Srdbd/CXtffQ7VOSkBjsAv3iMD2vk4y2E49wJF5B1TVxXY71JuqjN4cT6/8OkfADD6HhfFbT+DBQfRWzJ2TFGm+Kj7BxPKexZw6u75JyhwvGH3aOccLdLjOPIgs8apQ6cdDxWgyftt/FEuoYcjm0D5dZszPoELNWCi5QaKJzW9reaBuy5QHpYyAukjy/D7AOtiNWC04kNciJhHvO9gLZtxqGSPonphGjU0aqgd54MgdCu5LHO4+LD7bLuo25JPsMOW7H9HhAV2lh5Opezge3H2yygrQltCdXwXJo64Fv3secHym33ZpHS/EIefA5mfghPTRo7Cj1NZResyp6j7tEzxavJsQtq3vuoNqKfU61TxE02zSvKn8q6Mz0A1rIGQyRB1hGFe9FkqYD/EurCKMgg0W2P+Ok5/m3FmxIwMoKTySAUBH9a82ksyx7N99DOAR2974pCzCFT5yx72p+39H7qH1Nh9nCPvQmlSQyUO0EPYZVrxizBijE8Y14sJEnmsRM74Zsy0ibOCRN1gcy3uiIcbF90Hrop98619nmL2cYPvR5L+DjCn/cMHcWZ7IQ3FroPbJOI5zg0cd/RioNnYmzWrWT43iG0/e0gJtSPXZ3KrbZJK9K1HzwL9nbC26AEpJyOeHFwEM8nOyoSjm36e1dUaYm3t7Q9Zmx105IOUBpVcMoIjNt3HVXFZ1icdp3/jNwXqdVic0qcioccWDVnPTLWLRcwwxp9JLIZvOAokbRtlxZN2BpmyZ1mesxovdlZeiIOt/Ele0MasHXoKC9nqWF0XnzfbVVh0JqqyWWJm+iU6H/nVA4qZ8kSeXIkhoShW3F9m/iw4HoWlrjeAMguI2EMzl+FZkYvLi1quncjv+0Yvl/7w99RPtqkmpXEpyBqnrytRWN7lmbcSufxEtULJE/3v26ucYU3XmV+nq1orqNRvBzNr55vBpA95pcwTjDGvYfz9+A4uRp4/bbrNUJOU7+FtV5JMh+8QgDfesb4uKPOuc4+DmHFnKv0761u/1sLE3I+MOHPlQd/VB0+wwSgZUmMapKjaMJ1GJFiLljc07ba7MB4qj2d+bJKtoQTC3F9qMp37yNoYFMdcXECHcJSaO/cPzNVu8w3rOTBdMLnLMb5FW/LNvD0AKr87yp4XXt55MUd6GfIPgTpTQD+XbCbHvdsbrMRln5p0ClJiPZZK7+6C/LanDx0JK7+PQisC5T74YuR8LxoE8BPUSViWwjW7SLIvwqJ2nT2HtKuXBCDz5ln97q1YdZk598+MZ94Br608WDNDZTqhrJyh0D+sZKIZbfiyoVF+E2Vuz7NaW2Q4dT85kZx3YmJBCy6GteyJsCBCQ0g1EiDMLwvlEu9iCt0vIyyeTC+L2wTt2G6SN1BDVpS1nIZ5Xn9Xl3kVcgbqupwfn0fxi/wBskWTb6/e9/xozt2Pfx2BkWIebLS847jq4KNzZOXbCxGEbag+zyO35VadcMQPntr00mzaTO7LemVYebcrIht4LTUbOblOjn92kGGd4wy63qP1aOQGLXgTkuPxkxgp2Rb62AGFHhYOYEPPlEzAurjHhks4vuyuY5jfphhVNHXD5UCSzjtvE0L0+1UeiU/uUXz1O2Uckm/zo4zulIe/IdwLMeLbzpcMl7eYgHndjRSZ0DE2cOwai306WeSbvRHWaoL0UluEUtfJe7rgl6A3Rm+Xf4T5w+voIz5XqpilL2Oe1LR/i1SSWL6e/fpx3LiIaRewSqPviPWhAJ5W1qmcsW+H+3PY9RcalHZomfWSVW5jJm6AdzLbAec20b4n3UPPFoWpdVyzjfvaX5NsBjKb6gQoFb9xdyFCsehd1HAad96qyhv0LIFJRxg9BZr8u+XRbrjtcBx0TBvo/Uso/bbEqQE6Xf+/lrq3Wxc3EDxmYIAQ06UbgB21gtd3Kd730H0IqlOwKSFIO2XItVMBINaEmxe6yXrM85II+vaTdpl5/BeE7BG7BGk/Ndxz0HzyiwBaABtTfsDpB/4inQYqvZFk2CwbwuWZ1VzW49wiWFG160hapUyYRCp1uI4XTcP3gJdG2G1wa7eCGwjAwlW3gkiF6JHUtIQwV+Xa9O6kdHd3fWVJvg6tLvrATJt2udWH31DTLT/NogZLhhkVZUmOW1SUGx2NWUXlGsUnyBne880i8pIOXpVd5USaZCa1ki6Jcxh7EeEMmpZedjUAvtmmZkva/JKaMkxtJRO2HNlRZXFvTIhzJEGW9Br/3ZNGeKAzkknvhFIpbaJ6mN7IOV+49L1QLskSENCmGlOPP7Oaktb0sg9F9iscPOMn0MgLYkbpNatUI1jbBWiZAuwxpFfOcwkHiAUsYVPsmraRXq02/qmgZyCfEOZA3JGGJyfWiVEeIsIJp2xgmQTqJJvNJkIPI6xiyxARiR2tTceeVu06jNPrWVA7i7B+K9vcHAnxki/P0or+NOCZOtoMu5SCgPq1oXS7HRVvU+Dhy23nDd8T93oJ+q4ZBH54CwsWPXkUcnO3lDhoSYOJC5zlG/40VpM1CA+w+YkdpKNCuK0NW+a77OBudO94VoB5xxwizwXot3RZPe51SqnpgzZrYuyEZa3nEUtYxFwN0L9L+M5CMEh96JyJK3HsfnqIdURnC2UNZvtqHR2xq3z7F/CPz3Lh37L98x2b+0dyXvaceyu4bXLohox9ONXon25A2FwLMMP1VJku9fXtf0+9x4dJRpZd3egxd7kJSxPZUkttkpY9/7m5AFfjmi04X+PzBnFvufy9q57ca2MZ3J6CDUT/pPDLz6tzNfsotdQ4y114F/N1fulnVd88ObMSGzR8PdcBQCJnObkvSkyt9ZjBZTROpxFm0dUOzF89X+X3v6Mb/Ma2GTh/7/CfSU9js54tEBHzxNSbftctKvX92FPctMpaUL1SuDM/W1Oiko7LA/VwfiYzZfRw9ueBoBp228LYHHKMBf+6qTFm+zG4p1qhqy1HeGVm7LpQu5i5guVT8CAL+b1/2stASBGogMKgBy02OI+NJAaafLlyThjJfV9tMxzigsMAqr4vyVvaDmfHMOg9vMMR6OpFuCGGPxtukXmv5pXN2c51kja1gSE88Bt3IuHh20GKB2XytKYDtTLLBDX4kMTiKVF92worx+SEMCox/t2lXjz68Uwk+Qq4H6x6ftqUWutI1tt29VHs+VXGVoSx8/GTWO5ATKRp5+RlDMgMdho/KelAj0xln93g4j6DTgPMoHYLa/XRSwTyII5u1yt86diO1f0jX57mWuNXeelD3HUjA8q2+uIXu3VFyVIkxWl7kuKtSXCdkjjt6ALw+Jmr4U9eja9NEQlhpbc84QbieULgEwhG1f2BsC233h0YsFnr8DqobFLbeonBPhK2TqysG9u4I2qomn4i8LUahFFYNfkreP+N3qnu3f0MpvhFtZfIXTzgd3fVz0PK5Cp0cMdGLfruxnUr6Va89Ysju8rrX6rfhMYy6KHNHikoSjGrxaTisxScQkqr/vNhsEu58CZ1d0YCkciAl3KPIi9wUQiFKpS+Ohmp8+BB9HaBxnKtgTG3oy34T3935n64MneH1NNM9a0qSRZJHPAFHdosxzP9gJb5LcZ3yQMaoWvMXSeG7lKVhbMLDl0BKV/EqabI6EhZbsYKhfh1HZwW+B/qWvziyXwsWYFpPKhNasqi5o7m7GxhbSlfPFDEgJ4dvnwXKWxjNaEYMXFRmNWrmnOERzpAtYT+OCPwR2AQG/iTLJWHwBtq59iQYxljgMZ9OxpJhrhO12M6gn29Cltoo6TMsb0YJEzd3jRNCd5Qq5jXWQGYOaoY7wJ+LcgMi5h/0wduYyJ/flref5cszTIBEbOpUcKWD46U/VIDp/KFG22HykYsFjwkkpE+ykMydkXRL03Hv/1mK/1mKn24p8vxjosWfwdAgnO5By3GlR9QOnc6Zp3FeP4B+srJqI4pDXHX9kQYesQSLFEesSoDW7vJYPsdrvW2cSsh0koFPLTCHBeSo3Tt2NF/l0dUxhVXvlIP24xhZYE9P8BDfCc8VwuJGoOdFOhb9u05b7JR8A6fFJl86zobJy10e21WeSjqHRIRKdHVdMnvAo9UBuO7vfemxT9QtdgHQEa88ec13pBPXnXheB71OCGnSvpGND7hVGzRBbXLaFLapSPjngpv94POOBVLf6/YaNwG1mgAF0zUH24A5EH4oNQv9pWQo48+lp8oddD7RnEvExJJq2gOv7znygOtr7ftkGdrtOKalIN2YFs/9/jl3wz9pXra7MavXf7absFAg7yIupHfL5gTDBLtHHqD9giMaPh/lkbPAXDv6pGNmdMuqTWzbwbu7sRRuLblDdQyraYQFqYfsrHQ4atx4b/vxXvsmKX8ED7dpqzHHhfy1N8lmwBJyH4rRI8Onl0WpH4MVECrC8dFiMXqH44pabf4ys5OZeXbOaeW1s0WHPASWjvQQFfiVwb15tEe/uUDD9WXHf2Eh2x3D2tOgJYAXVqXByo4gBWdwnQLV7lcIwWhHSYyI6exm/46BmxJjwQ4KbU9koidkczB0z4qmEl17GENW8csJvsxDXJoj9j33fzOLkkbtnkff7YMc/ifLXhzyKY809AcqfYILbHKSZO9/aOq6wMvdSV2X04lKfIMXi9vP5HV9M5xFxE1WA69LdkzHLOgyqi/FFU9mJXCQISzq24cPekwPRYwQjlUPf+8fvuQSFioKJp2evAiUWGK9iVc9qm0MJuW0h6i80f0XmAxXU6OvzlHklCYxvhQfRrNkeXxZiepazNJg/Paga8p8iFu+GjZ4wXlHZFrSrYpFYwVB6kgQOvQO148di0Zc+XimL53xd19RbPyTpAEdu3i3xUyFsAmIR57lwtfDTF3cwmXgtHkoErL62evTH/gL5jsCLmxbBYMXk20qapawGSg2Zzzn4PTLL4YcrsXN219el6D02c2gIrQy/dD3HjFvSClhdqtrp2U7lKMxgIoJT6v29pegk0v9veR2rjg8tP4xKWE/HNXrJOdn3Bl+r+qIKf+5yIMXyZ1VuE7tiOcmMGnLLLI0Z3cfoZl17lMgiwyH7zcjdkIOjPX3qUnzxTT8rHfB2YgKOHflRxInR2Zjq4G76gAPY/NnxFaWeE0U35ScCLvxOPrq8PffPCXKVXgs8vU3X/7uK8Lny6HHqyy5qqJH0JeEud9rPZtxbw5Z5A6Ln5SbZWq9vKXzIOkdS+BaT37g4ZA7ZAXwEOTrolmY74twelv6bxlwJginLsmA7CjTZcjLVEhtt3fA3y1yoOAEvvDQL0cvi8YftI6rjNcalfkFq/zCJWaQGhKNSqPBYL4lcUT9LMsoJ0AgTRHCcFo7qOR09ozo6D6m7LLZbNGvSPq/KOHhwAfhhOGD1R8hZGc97PW3f7OmosS/wGs1e9MExsS2Ys2yiDZJmscR5uEUCUXhQ8QVRSqSZSZVBdZmL6UglmDS57KAJqVSzKhG/h8AAP//AQAA//9GbuCxD68AAA==") gr, _ = gzip.NewReader(bytes.NewReader(bs)) bs, _ = ioutil.ReadAll(gr) assets["scripts/syncthing/core/controllers/syncthingController.js"] = bs diff --git a/internal/events/events.go b/internal/events/events.go index 7ccadd3ca..3f22c4f41 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -31,6 +31,8 @@ const ( FolderRejected ConfigSaved DownloadProgress + FolderSummary + FolderCompletion AllEvents = (1 << iota) - 1 ) @@ -67,6 +69,10 @@ func (t EventType) String() string { return "ConfigSaved" case DownloadProgress: return "DownloadProgress" + case FolderSummary: + return "FolderSummary" + case FolderCompletion: + return "FolderCompletion" default: return "Unknown" }