From 655acb4cb2cd32f39889ed8e1c22e7e481794827 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 12 Jul 2014 23:06:48 +0200 Subject: [PATCH] Deprecate scanner.Block & File --- build.sh | 2 +- config/config.go | 5 +- config/config_test.go | 7 +- files/cmd/pidx/main.go | 5 +- files/leveldb.go | 39 ++++--- files/set.go | 13 ++- files/set_test.go | 230 ++++++++++++++++++++--------------------- files/sort.go | 11 +- model/blockqueue.go | 14 +-- model/model.go | 101 ++++++------------ model/model_test.go | 22 ++-- model/puller.go | 4 +- model/util.go | 46 --------- protocol/message.go | 31 +++++- scanner/blocks.go | 32 +++--- scanner/blocks_test.go | 26 ++--- scanner/blocks_xdr.go | 49 --------- scanner/file.go | 30 ------ scanner/file_xdr.go | 64 ------------ scanner/walk.go | 15 ++- xdr/cmd/genxdr/main.go | 3 + 21 files changed, 269 insertions(+), 480 deletions(-) delete mode 100644 scanner/blocks_xdr.go delete mode 100644 scanner/file.go delete mode 100644 scanner/file_xdr.go diff --git a/build.sh b/build.sh index 2bf81fe6e..33b89263b 100755 --- a/build.sh +++ b/build.sh @@ -96,7 +96,7 @@ setup() { } xdr() { - for f in discover/packets files/leveldb protocol/message scanner/blocks scanner/file ; do + for f in discover/packets files/leveldb protocol/message ; do go run xdr/cmd/genxdr/main.go -- "${f}.go" > "${f}_xdr.go" done } diff --git a/config/config.go b/config/config.go index 710201654..0b3e27028 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,6 @@ import ( "code.google.com/p/go.crypto/bcrypt" "github.com/calmh/syncthing/logger" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" ) var l = logger.DefaultLogger @@ -122,11 +121,11 @@ func (r *RepositoryConfiguration) NodeIDs() []protocol.NodeID { return r.nodeIDs } -func (r RepositoryConfiguration) FileRanker() func(scanner.File) int { +func (r RepositoryConfiguration) FileRanker() func(protocol.FileInfo) int { if len(r.SyncOrderPatterns) <= 0 { return nil } - return func(f scanner.File) int { + return func(f protocol.FileInfo) int { ret := 0 for _, v := range r.SyncOrderPatterns { if v.CompiledPattern().MatchString(f.Name) { diff --git a/config/config_test.go b/config/config_test.go index cacb45fe7..d01a6580f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -13,7 +13,6 @@ import ( "github.com/calmh/syncthing/files" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" ) var node1, node2, node3, node4 protocol.NodeID @@ -274,7 +273,7 @@ func TestFileSorter(t *testing.T) { }, } - f := []scanner.File{ + f := []protocol.FileInfo{ {Name: "bar.mov"}, {Name: "baz.txt"}, {Name: "foo.jpg"}, @@ -290,7 +289,7 @@ func TestFileSorter(t *testing.T) { files.SortBy(rcfg.FileRanker()).Sort(f) - expected := []scanner.File{ + expected := []protocol.FileInfo{ {Name: "camera-uploads/foo.jpg"}, {Name: "camera-uploads/herp.mov"}, {Name: "camera-uploads/hurr.pl"}, @@ -314,7 +313,7 @@ func TestFileSorter(t *testing.T) { } } -func formatFiles(f []scanner.File) string { +func formatFiles(f []protocol.FileInfo) string { ret := "" for _, v := range f { diff --git a/files/cmd/pidx/main.go b/files/cmd/pidx/main.go index ae0ea91ca..7a4b630b0 100644 --- a/files/cmd/pidx/main.go +++ b/files/cmd/pidx/main.go @@ -8,7 +8,6 @@ import ( "github.com/calmh/syncthing/files" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" "github.com/syndtr/goleveldb/leveldb" ) @@ -29,7 +28,7 @@ func main() { if *node == "" { log.Printf("*** Global index for repo %q", *repo) - fs.WithGlobal(func(f scanner.File) bool { + fs.WithGlobal(func(f protocol.FileInfo) bool { fmt.Println(f) fmt.Println("\t", fs.Availability(f.Name)) return true @@ -40,7 +39,7 @@ func main() { log.Fatal(err) } log.Printf("*** Have index for repo %q node %q", *repo, n) - fs.WithHave(n, func(f scanner.File) bool { + fs.WithHave(n, func(f protocol.FileInfo) bool { fmt.Println(f) return true }) diff --git a/files/leveldb.go b/files/leveldb.go index a338d6671..28f88ee7b 100644 --- a/files/leveldb.go +++ b/files/leveldb.go @@ -6,7 +6,6 @@ import ( "github.com/calmh/syncthing/lamport" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" @@ -27,7 +26,7 @@ type versionList struct { versions []fileVersion } -type fileList []scanner.File +type fileList []protocol.FileInfo func (l fileList) Len() int { return len(l) @@ -94,9 +93,9 @@ func globalKeyName(key []byte) []byte { type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool -type fileIterator func(f scanner.File) bool +type fileIterator func(f protocol.FileInfo) bool -func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, deleteFn deletionHandler) bool { +func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo, deleteFn deletionHandler) bool { sort.Sort(fileList(fs)) // sort list on name, same as on disk start := nodeKey(repo, node, nil) // before all repo/node files @@ -153,7 +152,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, del case cmp == 0: // File exists on both sides - compare versions. - var ef scanner.File + var ef protocol.FileInfo ef.UnmarshalXDR(dbi.Value()) if fs[fsi].Version > ef.Version { ldbInsert(batch, repo, node, newName, fs[fsi]) @@ -182,7 +181,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, del return changed } -func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { +func ldbReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool { return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool { // Disk has files that we are missing. Remove it. if debug { @@ -194,9 +193,9 @@ func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { }) } -func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { +func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool { return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool { - var f scanner.File + var f protocol.FileInfo err := f.UnmarshalXDR(dbi.Value()) if err != nil { panic(err) @@ -216,7 +215,7 @@ func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File) }) } -func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { +func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool { batch := new(leveldb.Batch) snap, err := db.GetSnapshot() if err != nil { @@ -234,7 +233,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { continue } - var ef scanner.File + var ef protocol.FileInfo err = ef.UnmarshalXDR(bs) if err != nil { panic(err) @@ -253,7 +252,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { return true } -func ldbInsert(batch dbWriter, repo, node, name []byte, file scanner.File) { +func ldbInsert(batch dbWriter, repo, node, name []byte, file protocol.FileInfo) { if debug { l.Debugf("insert; repo=%q node=%x %v", repo, node, file) } @@ -362,7 +361,7 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) { defer dbi.Release() for dbi.Next() { - var f scanner.File + var f protocol.FileInfo err := f.UnmarshalXDR(dbi.Value()) if err != nil { panic(err) @@ -373,17 +372,17 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) { } } -func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File { +func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo { nk := nodeKey(repo, node, file) bs, err := db.Get(nk, nil) if err == leveldb.ErrNotFound { - return scanner.File{} + return protocol.FileInfo{} } if err != nil { panic(err) } - var f scanner.File + var f protocol.FileInfo err = f.UnmarshalXDR(bs) if err != nil { panic(err) @@ -391,7 +390,7 @@ func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File { return f } -func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File { +func ldbGetGlobal(db *leveldb.DB, repo, file []byte) protocol.FileInfo { k := globalKey(repo, file) snap, err := db.GetSnapshot() if err != nil { @@ -401,7 +400,7 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File { bs, err := snap.Get(k, nil) if err == leveldb.ErrNotFound { - return scanner.File{} + return protocol.FileInfo{} } if err != nil { panic(err) @@ -423,7 +422,7 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File { panic(err) } - var f scanner.File + var f protocol.FileInfo err = f.UnmarshalXDR(bs) if err != nil { panic(err) @@ -458,7 +457,7 @@ func ldbWithGlobal(db *leveldb.DB, repo []byte, fn fileIterator) { panic(err) } - var f scanner.File + var f protocol.FileInfo err = f.UnmarshalXDR(bs) if err != nil { panic(err) @@ -544,7 +543,7 @@ func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) { panic(err) } - var gf scanner.File + var gf protocol.FileInfo err = gf.UnmarshalXDR(bs) if err != nil { panic(err) diff --git a/files/set.go b/files/set.go index 47969e1e6..387227702 100644 --- a/files/set.go +++ b/files/set.go @@ -9,12 +9,11 @@ import ( "sync" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" "github.com/syndtr/goleveldb/leveldb" ) type fileRecord struct { - File scanner.File + File protocol.FileInfo Usage int Global bool } @@ -37,7 +36,7 @@ func NewSet(repo string, db *leveldb.DB) *Set { return &s } -func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) { +func (s *Set) Replace(node protocol.NodeID, fs []protocol.FileInfo) { if debug { l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs)) } @@ -48,7 +47,7 @@ func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) { } } -func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) { +func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []protocol.FileInfo) { if debug { l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs)) } @@ -59,7 +58,7 @@ func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) { } } -func (s *Set) Update(node protocol.NodeID, fs []scanner.File) { +func (s *Set) Update(node protocol.NodeID, fs []protocol.FileInfo) { if debug { l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs)) } @@ -91,11 +90,11 @@ func (s *Set) WithGlobal(fn fileIterator) { ldbWithGlobal(s.db, []byte(s.repo), fn) } -func (s *Set) Get(node protocol.NodeID, file string) scanner.File { +func (s *Set) Get(node protocol.NodeID, file string) protocol.FileInfo { return ldbGet(s.db, []byte(s.repo), node[:], []byte(file)) } -func (s *Set) GetGlobal(file string) scanner.File { +func (s *Set) GetGlobal(file string) protocol.FileInfo { return ldbGetGlobal(s.db, []byte(s.repo), []byte(file)) } diff --git a/files/set_test.go b/files/set_test.go index 2110e9a46..93932a33a 100644 --- a/files/set_test.go +++ b/files/set_test.go @@ -12,7 +12,6 @@ import ( "github.com/calmh/syncthing/files" "github.com/calmh/syncthing/lamport" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" ) @@ -23,48 +22,47 @@ func init() { remoteNode, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") } -func genBlocks(n int) []scanner.Block { - b := make([]scanner.Block, n) +func genBlocks(n int) []protocol.BlockInfo { + b := make([]protocol.BlockInfo, n) for i := range b { h := make([]byte, 32) for j := range h { h[j] = byte(i + j) } b[i].Size = uint32(i) - b[i].Offset = int64(i) b[i].Hash = h } return b } -func globalList(s *files.Set) []scanner.File { - var fs []scanner.File - s.WithGlobal(func(f scanner.File) bool { +func globalList(s *files.Set) []protocol.FileInfo { + var fs []protocol.FileInfo + s.WithGlobal(func(f protocol.FileInfo) bool { fs = append(fs, f) return true }) return fs } -func haveList(s *files.Set, n protocol.NodeID) []scanner.File { - var fs []scanner.File - s.WithHave(n, func(f scanner.File) bool { +func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo { + var fs []protocol.FileInfo + s.WithHave(n, func(f protocol.FileInfo) bool { fs = append(fs, f) return true }) return fs } -func needList(s *files.Set, n protocol.NodeID) []scanner.File { - var fs []scanner.File - s.WithNeed(n, func(f scanner.File) bool { +func needList(s *files.Set, n protocol.NodeID) []protocol.FileInfo { + var fs []protocol.FileInfo + s.WithNeed(n, func(f protocol.FileInfo) bool { fs = append(fs, f) return true }) return fs } -type fileList []scanner.File +type fileList []protocol.FileInfo func (l fileList) Len() int { return len(l) @@ -88,44 +86,44 @@ func TestGlobalSet(t *testing.T) { m := files.NewSet("test", db) - local0 := []scanner.File{ - scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)}, - scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)}, - scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)}, - scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)}, - scanner.File{Name: "z", Version: 1000, Blocks: genBlocks(8)}, + local0 := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)}, + protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)}, + protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)}, + protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)}, + protocol.FileInfo{Name: "z", Version: 1000, Blocks: genBlocks(8)}, } - local1 := []scanner.File{ - scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)}, - scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)}, - scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)}, - scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)}, + local1 := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)}, + protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)}, + protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)}, + protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)}, } - localTot := []scanner.File{ + localTot := []protocol.FileInfo{ local0[0], local0[1], local0[2], local0[3], - scanner.File{Name: "z", Version: 1001, Flags: protocol.FlagDeleted}, + protocol.FileInfo{Name: "z", Version: 1001, Flags: protocol.FlagDeleted}, } - remote0 := []scanner.File{ - scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)}, - scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)}, - scanner.File{Name: "c", Version: 1002, Blocks: genBlocks(5)}, + remote0 := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)}, + protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)}, + protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5)}, } - remote1 := []scanner.File{ - scanner.File{Name: "b", Version: 1001, Blocks: genBlocks(6)}, - scanner.File{Name: "e", Version: 1000, Blocks: genBlocks(7)}, + remote1 := []protocol.FileInfo{ + protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(6)}, + protocol.FileInfo{Name: "e", Version: 1000, Blocks: genBlocks(7)}, } - remoteTot := []scanner.File{ + remoteTot := []protocol.FileInfo{ remote0[0], remote1[0], remote0[2], remote1[1], } - expectedGlobal := []scanner.File{ + expectedGlobal := []protocol.FileInfo{ remote0[0], remote1[0], remote0[2], @@ -134,13 +132,13 @@ func TestGlobalSet(t *testing.T) { localTot[4], } - expectedLocalNeed := []scanner.File{ + expectedLocalNeed := []protocol.FileInfo{ remote1[0], remote0[2], remote1[1], } - expectedRemoteNeed := []scanner.File{ + expectedRemoteNeed := []protocol.FileInfo{ local0[3], } @@ -201,12 +199,12 @@ func TestGlobalSet(t *testing.T) { f = m.Get(protocol.LocalNodeID, "zz") if f.Name != "" { - t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, scanner.File{}) + t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{}) } f = m.GetGlobal("zz") if f.Name != "" { - t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, scanner.File{}) + t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{}) } av := []protocol.NodeID{protocol.LocalNodeID, remoteNode} @@ -232,41 +230,41 @@ func TestLocalDeleted(t *testing.T) { m := files.NewSet("test", db) lamport.Default = lamport.Clock{} - local1 := []scanner.File{ - scanner.File{Name: "a", Version: 1000}, - scanner.File{Name: "b", Version: 1000}, - scanner.File{Name: "c", Version: 1000}, - scanner.File{Name: "d", Version: 1000}, - scanner.File{Name: "z", Version: 1000, Flags: protocol.FlagDirectory}, + local1 := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000}, + protocol.FileInfo{Name: "b", Version: 1000}, + protocol.FileInfo{Name: "c", Version: 1000}, + protocol.FileInfo{Name: "d", Version: 1000}, + protocol.FileInfo{Name: "z", Version: 1000, Flags: protocol.FlagDirectory}, } m.ReplaceWithDelete(protocol.LocalNodeID, local1) - m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ + m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{ local1[0], // [1] removed local1[2], local1[3], local1[4], }) - m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ + m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{ local1[0], local1[2], // [3] removed local1[4], }) - m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ + m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{ local1[0], local1[2], // [4] removed }) - expectedGlobal1 := []scanner.File{ + expectedGlobal1 := []protocol.FileInfo{ local1[0], - scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted}, + protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted}, local1[2], - scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted}, - scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory}, + protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted}, + protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory}, } g := globalList(m) @@ -277,17 +275,17 @@ func TestLocalDeleted(t *testing.T) { t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1) } - m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ + m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{ local1[0], // [2] removed }) - expectedGlobal2 := []scanner.File{ + expectedGlobal2 := []protocol.FileInfo{ local1[0], - scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted}, - scanner.File{Name: "c", Version: 1004, Flags: protocol.FlagDeleted}, - scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted}, - scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory}, + protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted}, + protocol.FileInfo{Name: "c", Version: 1004, Flags: protocol.FlagDeleted}, + protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted}, + protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory}, } g = globalList(m) @@ -305,9 +303,9 @@ func Benchmark10kReplace(b *testing.B) { b.Fatal(err) } - var local []scanner.File + var local []protocol.FileInfo for i := 0; i < 10000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } b.ResetTimer() @@ -318,9 +316,9 @@ func Benchmark10kReplace(b *testing.B) { } func Benchmark10kUpdateChg(b *testing.B) { - var remote []scanner.File + var remote []protocol.FileInfo for i := 0; i < 10000; i++ { - remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } db, err := leveldb.Open(storage.NewMemStorage(), nil) @@ -331,9 +329,9 @@ func Benchmark10kUpdateChg(b *testing.B) { m := files.NewSet("test", db) m.Replace(remoteNode, remote) - var local []scanner.File + var local []protocol.FileInfo for i := 0; i < 10000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -350,9 +348,9 @@ func Benchmark10kUpdateChg(b *testing.B) { } func Benchmark10kUpdateSme(b *testing.B) { - var remote []scanner.File + var remote []protocol.FileInfo for i := 0; i < 10000; i++ { - remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } db, err := leveldb.Open(storage.NewMemStorage(), nil) @@ -362,9 +360,9 @@ func Benchmark10kUpdateSme(b *testing.B) { m := files.NewSet("test", db) m.Replace(remoteNode, remote) - var local []scanner.File + var local []protocol.FileInfo for i := 0; i < 10000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -376,9 +374,9 @@ func Benchmark10kUpdateSme(b *testing.B) { } func Benchmark10kNeed2k(b *testing.B) { - var remote []scanner.File + var remote []protocol.FileInfo for i := 0; i < 10000; i++ { - remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } db, err := leveldb.Open(storage.NewMemStorage(), nil) @@ -389,12 +387,12 @@ func Benchmark10kNeed2k(b *testing.B) { m := files.NewSet("test", db) m.Replace(remoteNode, remote) - var local []scanner.File + var local []protocol.FileInfo for i := 0; i < 8000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } for i := 8000; i < 10000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980}) } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -409,9 +407,9 @@ func Benchmark10kNeed2k(b *testing.B) { } func Benchmark10kHaveFullList(b *testing.B) { - var remote []scanner.File + var remote []protocol.FileInfo for i := 0; i < 10000; i++ { - remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } db, err := leveldb.Open(storage.NewMemStorage(), nil) @@ -422,12 +420,12 @@ func Benchmark10kHaveFullList(b *testing.B) { m := files.NewSet("test", db) m.Replace(remoteNode, remote) - var local []scanner.File + var local []protocol.FileInfo for i := 0; i < 2000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } for i := 2000; i < 10000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980}) } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -442,9 +440,9 @@ func Benchmark10kHaveFullList(b *testing.B) { } func Benchmark10kGlobal(b *testing.B) { - var remote []scanner.File + var remote []protocol.FileInfo for i := 0; i < 10000; i++ { - remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } db, err := leveldb.Open(storage.NewMemStorage(), nil) @@ -455,12 +453,12 @@ func Benchmark10kGlobal(b *testing.B) { m := files.NewSet("test", db) m.Replace(remoteNode, remote) - var local []scanner.File + var local []protocol.FileInfo for i := 0; i < 2000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000}) } for i := 2000; i < 10000; i++ { - local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980}) + local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980}) } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -482,18 +480,18 @@ func TestGlobalReset(t *testing.T) { m := files.NewSet("test", db) - local := []scanner.File{ - scanner.File{Name: "a", Version: 1000}, - scanner.File{Name: "b", Version: 1000}, - scanner.File{Name: "c", Version: 1000}, - scanner.File{Name: "d", Version: 1000}, + local := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000}, + protocol.FileInfo{Name: "b", Version: 1000}, + protocol.FileInfo{Name: "c", Version: 1000}, + protocol.FileInfo{Name: "d", Version: 1000}, } - remote := []scanner.File{ - scanner.File{Name: "a", Version: 1000}, - scanner.File{Name: "b", Version: 1001}, - scanner.File{Name: "c", Version: 1002}, - scanner.File{Name: "e", Version: 1000}, + remote := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000}, + protocol.FileInfo{Name: "b", Version: 1001}, + protocol.FileInfo{Name: "c", Version: 1002}, + protocol.FileInfo{Name: "e", Version: 1000}, } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -523,24 +521,24 @@ func TestNeed(t *testing.T) { m := files.NewSet("test", db) - local := []scanner.File{ - scanner.File{Name: "a", Version: 1000}, - scanner.File{Name: "b", Version: 1000}, - scanner.File{Name: "c", Version: 1000}, - scanner.File{Name: "d", Version: 1000}, + local := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000}, + protocol.FileInfo{Name: "b", Version: 1000}, + protocol.FileInfo{Name: "c", Version: 1000}, + protocol.FileInfo{Name: "d", Version: 1000}, } - remote := []scanner.File{ - scanner.File{Name: "a", Version: 1000}, - scanner.File{Name: "b", Version: 1001}, - scanner.File{Name: "c", Version: 1002}, - scanner.File{Name: "e", Version: 1000}, + remote := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000}, + protocol.FileInfo{Name: "b", Version: 1001}, + protocol.FileInfo{Name: "c", Version: 1002}, + protocol.FileInfo{Name: "e", Version: 1000}, } - shouldNeed := []scanner.File{ - scanner.File{Name: "b", Version: 1001}, - scanner.File{Name: "c", Version: 1002}, - scanner.File{Name: "e", Version: 1000}, + shouldNeed := []protocol.FileInfo{ + protocol.FileInfo{Name: "b", Version: 1001}, + protocol.FileInfo{Name: "c", Version: 1002}, + protocol.FileInfo{Name: "e", Version: 1000}, } m.ReplaceWithDelete(protocol.LocalNodeID, local) @@ -564,19 +562,19 @@ func TestChanges(t *testing.T) { m := files.NewSet("test", db) - local1 := []scanner.File{ - scanner.File{Name: "a", Version: 1000}, - scanner.File{Name: "b", Version: 1000}, - scanner.File{Name: "c", Version: 1000}, - scanner.File{Name: "d", Version: 1000}, + local1 := []protocol.FileInfo{ + protocol.FileInfo{Name: "a", Version: 1000}, + protocol.FileInfo{Name: "b", Version: 1000}, + protocol.FileInfo{Name: "c", Version: 1000}, + protocol.FileInfo{Name: "d", Version: 1000}, } - local2 := []scanner.File{ + local2 := []protocol.FileInfo{ local1[0], // [1] deleted local1[2], - scanner.File{Name: "d", Version: 1002}, - scanner.File{Name: "e", Version: 1000}, + protocol.FileInfo{Name: "d", Version: 1002}, + protocol.FileInfo{Name: "e", Version: 1000}, } m.ReplaceWithDelete(protocol.LocalNodeID, local1) diff --git a/files/sort.go b/files/sort.go index f5bcdfcff..e8002dfff 100644 --- a/files/sort.go +++ b/files/sort.go @@ -2,13 +2,12 @@ package files import ( "sort" - - "github.com/calmh/syncthing/scanner" + "github.com/calmh/syncthing/protocol" ) -type SortBy func(p scanner.File) int +type SortBy func(p protocol.FileInfo) int -func (by SortBy) Sort(files []scanner.File) { +func (by SortBy) Sort(files []protocol.FileInfo) { ps := &fileSorter{ files: files, by: by, @@ -17,8 +16,8 @@ func (by SortBy) Sort(files []scanner.File) { } type fileSorter struct { - files []scanner.File - by func(p1 scanner.File) int + files []protocol.FileInfo + by func(p1 protocol.FileInfo) int } func (s *fileSorter) Len() int { diff --git a/model/blockqueue.go b/model/blockqueue.go index 8742959ff..48330afcf 100644 --- a/model/blockqueue.go +++ b/model/blockqueue.go @@ -7,19 +7,19 @@ package model import ( "sync" - "github.com/calmh/syncthing/scanner" + "github.com/calmh/syncthing/protocol" ) type bqAdd struct { - file scanner.File - have []scanner.Block - need []scanner.Block + file protocol.FileInfo + have []protocol.BlockInfo + need []protocol.BlockInfo } type bqBlock struct { - file scanner.File - block scanner.Block // get this block from the network - copy []scanner.Block // copy these blocks from the old version of the file + file protocol.FileInfo + block protocol.BlockInfo // get this block from the network + copy []protocol.BlockInfo // copy these blocks from the old version of the file first bool last bool } diff --git a/model/model.go b/model/model.go index fc69e581c..7cffe6474 100644 --- a/model/model.go +++ b/model/model.go @@ -162,11 +162,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo { var have int64 for _, repo := range m.nodeRepos[node] { - m.repoFiles[repo].WithGlobal(func(f scanner.File) bool { + m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool { if !protocol.IsDeleted(f.Flags) { - size := f.Size + var size int64 if protocol.IsDirectory(f.Flags) { size = zeroEntrySize + } else { + size = f.Size() } tot += size have += size @@ -174,11 +176,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo { return true }) - m.repoFiles[repo].WithNeed(node, func(f scanner.File) bool { + m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool { if !protocol.IsDeleted(f.Flags) { - size := f.Size + var size int64 if protocol.IsDirectory(f.Flags) { size = zeroEntrySize + } else { + size = f.Size() } have -= size } @@ -209,7 +213,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo { return res } -func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) { +func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) { for _, f := range fs { fs, de, by := sizeOfFile(f) files += fs @@ -219,11 +223,11 @@ func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) { return } -func sizeOfFile(f scanner.File) (files, deleted int, bytes int64) { +func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) { if !protocol.IsDeleted(f.Flags) { files++ if !protocol.IsDirectory(f.Flags) { - bytes += f.Size + bytes += f.Size() } else { bytes += zeroEntrySize } @@ -240,7 +244,7 @@ func (m *Model) GlobalSize(repo string) (files, deleted int, bytes int64) { m.rmut.RLock() defer m.rmut.RUnlock() if rf, ok := m.repoFiles[repo]; ok { - rf.WithGlobal(func(f scanner.File) bool { + rf.WithGlobal(func(f protocol.FileInfo) bool { fs, de, by := sizeOfFile(f) files += fs deleted += de @@ -257,7 +261,7 @@ func (m *Model) LocalSize(repo string) (files, deleted int, bytes int64) { m.rmut.RLock() defer m.rmut.RUnlock() if rf, ok := m.repoFiles[repo]; ok { - rf.WithHave(protocol.LocalNodeID, func(f scanner.File) bool { + rf.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool { fs, de, by := sizeOfFile(f) files += fs deleted += de @@ -275,12 +279,12 @@ func (m *Model) NeedSize(repo string) (files int, bytes int64) { } // NeedFiles returns the list of currently needed files -func (m *Model) NeedFilesRepo(repo string) []scanner.File { +func (m *Model) NeedFilesRepo(repo string) []protocol.FileInfo { m.rmut.RLock() defer m.rmut.RUnlock() if rf, ok := m.repoFiles[repo]; ok { - var fs []scanner.File - rf.WithNeed(protocol.LocalNodeID, func(f scanner.File) bool { + var fs []protocol.FileInfo + rf.WithNeed(protocol.LocalNodeID, func(f protocol.FileInfo) bool { fs = append(fs, f) return true }) @@ -304,23 +308,9 @@ func (m *Model) Index(nodeID protocol.NodeID, repo string, fs []protocol.FileInf return } - var files = make([]scanner.File, len(fs)) - for i := range fs { - f := fs[i] - lamport.Default.Tick(f.Version) - if debug { - var flagComment string - if protocol.IsDeleted(f.Flags) { - flagComment = " (deleted)" - } - l.Debugf("IDX(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks)) - } - files[i] = fileFromFileInfo(f) - } - m.rmut.RLock() if r, ok := m.repoFiles[repo]; ok { - r.Replace(nodeID, files) + r.Replace(nodeID, fs) } else { l.Fatalf("Index for nonexistant repo %q", repo) } @@ -339,23 +329,9 @@ func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.F return } - var files = make([]scanner.File, len(fs)) - for i := range fs { - f := fs[i] - lamport.Default.Tick(f.Version) - if debug { - var flagComment string - if protocol.IsDeleted(f.Flags) { - flagComment = " (deleted)" - } - l.Debugf("IDXUP(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks)) - } - files[i] = fileFromFileInfo(f) - } - m.rmut.RLock() if r, ok := m.repoFiles[repo]; ok { - r.Update(nodeID, files) + r.Update(nodeID, fs) } else { l.Fatalf("IndexUpdate for nonexistant repo %q", repo) } @@ -432,14 +408,14 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64, } lf := r.Get(protocol.LocalNodeID, name) - if lf.Suppressed || protocol.IsDeleted(lf.Flags) { + if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) { if debug { l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf) } return nil, ErrInvalid } - if offset > lf.Size { + if offset > lf.Size() { if debug { l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size) } @@ -468,20 +444,20 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64, } // ReplaceLocal replaces the local repository index with the given list of files. -func (m *Model) ReplaceLocal(repo string, fs []scanner.File) { +func (m *Model) ReplaceLocal(repo string, fs []protocol.FileInfo) { m.rmut.RLock() m.repoFiles[repo].ReplaceWithDelete(protocol.LocalNodeID, fs) m.rmut.RUnlock() } -func (m *Model) CurrentRepoFile(repo string, file string) scanner.File { +func (m *Model) CurrentRepoFile(repo string, file string) protocol.FileInfo { m.rmut.RLock() f := m.repoFiles[repo].Get(protocol.LocalNodeID, file) m.rmut.RUnlock() return f } -func (m *Model) CurrentGlobalFile(repo string, file string) scanner.File { +func (m *Model) CurrentGlobalFile(repo string, file string) protocol.FileInfo { m.rmut.RLock() f := m.repoFiles[repo].GetGlobal(file) m.rmut.RUnlock() @@ -494,7 +470,7 @@ type cFiler struct { } // Implements scanner.CurrentFiler -func (cf cFiler) CurrentFile(file string) scanner.File { +func (cf cFiler) CurrentFile(file string) protocol.FileInfo { return cf.m.CurrentRepoFile(cf.r, file) } @@ -552,32 +528,18 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) // protocolIndex returns the current local index in protocol data types. func (m *Model) protocolIndex(repo string) []protocol.FileInfo { - var index []protocol.FileInfo - - var fs []scanner.File - m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f scanner.File) bool { + var fs []protocol.FileInfo + m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool { fs = append(fs, f) return true }) - for _, f := range fs { - mf := fileInfoFromFile(f) - if debug { - var flagComment string - if protocol.IsDeleted(mf.Flags) { - flagComment = " (deleted)" - } - l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks)) - } - index = append(index, mf) - } - - return index + return fs } -func (m *Model) updateLocal(repo string, f scanner.File) { +func (m *Model) updateLocal(repo string, f protocol.FileInfo) { m.rmut.RLock() - m.repoFiles[repo].Update(protocol.LocalNodeID, []scanner.File{f}) + m.repoFiles[repo].Update(protocol.LocalNodeID, []protocol.FileInfo{f}) m.rmut.RUnlock() } @@ -736,11 +698,10 @@ func (m *Model) LoadIndexes(dir string) { for repo := range m.repoCfgs { fs := m.loadIndex(repo, dir) - var sfs = make([]scanner.File, len(fs)) + var sfs = make([]protocol.FileInfo, len(fs)) for i := 0; i < len(fs); i++ { lamport.Default.Tick(fs[i].Version) - sfs[i] = fileFromFileInfo(fs[i]) - sfs[i].Suppressed = false // we might have saved an index with files that were suppressed; the should not be on startup + fs[i].Flags &= ^uint32(protocol.FlagInvalid) // we might have saved an index with files that were suppressed; the should not be on startup } m.repoFiles[repo].Replace(protocol.LocalNodeID, sfs) diff --git a/model/model_test.go b/model/model_test.go index 921e0f598..672975e9c 100644 --- a/model/model_test.go +++ b/model/model_test.go @@ -13,7 +13,6 @@ import ( "github.com/calmh/syncthing/config" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" ) @@ -25,27 +24,24 @@ func init() { node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY") } -var testDataExpected = map[string]scanner.File{ - "foo": scanner.File{ +var testDataExpected = map[string]protocol.FileInfo{ + "foo": protocol.FileInfo{ Name: "foo", Flags: 0, Modified: 0, - Size: 7, - Blocks: []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}}, + Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}}, }, - "empty": scanner.File{ + "empty": protocol.FileInfo{ Name: "empty", Flags: 0, Modified: 0, - Size: 0, - Blocks: []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}}, + Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}}, }, - "bar": scanner.File{ + "bar": protocol.FileInfo{ Name: "bar", Flags: 0, Modified: 0, - Size: 10, - Blocks: []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}, + Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}, }, } @@ -89,7 +85,7 @@ func genFiles(n int) []protocol.FileInfo { files[i] = protocol.FileInfo{ Name: fmt.Sprintf("file%d", i), Modified: t, - Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}}, + Blocks: []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}}, } } @@ -212,7 +208,7 @@ func BenchmarkRequest(b *testing.B) { files[i] = protocol.FileInfo{ Name: fmt.Sprintf("file%d", i), Modified: t, - Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}}, + Blocks: []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}}, } } diff --git a/model/puller.go b/model/puller.go index 912dbafd4..41e0a3418 100644 --- a/model/puller.go +++ b/model/puller.go @@ -21,7 +21,7 @@ import ( type requestResult struct { node protocol.NodeID - file scanner.File + file protocol.FileInfo filepath string // full filepath name offset int64 data []byte @@ -621,7 +621,7 @@ func (p *puller) queueNeededBlocks() { } } -func (p *puller) closeFile(f scanner.File) { +func (p *puller) closeFile(f protocol.FileInfo) { if debug { l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name) } diff --git a/model/util.go b/model/util.go index c4a05e332..86c156c48 100644 --- a/model/util.go +++ b/model/util.go @@ -6,58 +6,12 @@ package model import ( "fmt" - "path/filepath" "sync" "time" "github.com/calmh/syncthing/protocol" - "github.com/calmh/syncthing/scanner" ) -func fileFromFileInfo(f protocol.FileInfo) scanner.File { - var blocks = make([]scanner.Block, len(f.Blocks)) - var offset int64 - for i, b := range f.Blocks { - blocks[i] = scanner.Block{ - Offset: offset, - Size: b.Size, - Hash: b.Hash, - } - offset += int64(b.Size) - } - return scanner.File{ - // Name is with native separator and normalization - Name: filepath.FromSlash(f.Name), - Size: offset, - Flags: f.Flags &^ protocol.FlagInvalid, - Modified: f.Modified, - Version: f.Version, - Blocks: blocks, - Suppressed: f.Flags&protocol.FlagInvalid != 0, - } -} - -func fileInfoFromFile(f scanner.File) protocol.FileInfo { - var blocks = make([]protocol.BlockInfo, len(f.Blocks)) - for i, b := range f.Blocks { - blocks[i] = protocol.BlockInfo{ - Size: b.Size, - Hash: b.Hash, - } - } - pf := protocol.FileInfo{ - Name: filepath.ToSlash(f.Name), - Flags: f.Flags, - Modified: f.Modified, - Version: f.Version, - Blocks: blocks, - } - if f.Suppressed { - pf.Flags |= protocol.FlagInvalid - } - return pf -} - func cmMap(cm protocol.ClusterConfigMessage) map[string]map[protocol.NodeID]uint32 { m := make(map[string]map[protocol.NodeID]uint32) for _, repo := range cm.Repositories { diff --git a/protocol/message.go b/protocol/message.go index f4bc93423..6613b911f 100644 --- a/protocol/message.go +++ b/protocol/message.go @@ -4,6 +4,8 @@ package protocol +import "fmt" + type IndexMessage struct { Repository string // max:64 Files []FileInfo // max:10000000 @@ -17,9 +19,34 @@ type FileInfo struct { Blocks []BlockInfo // max:1000000 } +func (f FileInfo) String() string { + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}", + f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks) +} + +func (f FileInfo) Size() (bytes int64) { + for _, b := range f.Blocks { + bytes += int64(b.Size) + } + return +} + +func (a FileInfo) Equals(b FileInfo) bool { + return a.Version == b.Version +} + +func (a FileInfo) NewerThan(b FileInfo) bool { + return a.Version > b.Version +} + type BlockInfo struct { - Size uint32 - Hash []byte // max:64 + Offset int64 // noencode (cache only) + Size uint32 + Hash []byte // max:64 +} + +func (b BlockInfo) String() string { + return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash) } type RequestMessage struct { diff --git a/scanner/blocks.go b/scanner/blocks.go index fd6f1d3c9..fa1cf1c18 100644 --- a/scanner/blocks.go +++ b/scanner/blocks.go @@ -7,25 +7,16 @@ package scanner import ( "bytes" "crypto/sha256" - "fmt" "io" + + "github.com/calmh/syncthing/protocol" ) const StandardBlockSize = 128 * 1024 -type Block struct { - Offset int64 - Size uint32 - Hash []byte -} - -func (b Block) String() string { - return fmt.Sprintf("%d/%d/%x", b.Offset, b.Size, b.Hash) -} - // Blocks returns the blockwise hash of the reader. -func Blocks(r io.Reader, blocksize int) ([]Block, error) { - var blocks []Block +func Blocks(r io.Reader, blocksize int) ([]protocol.BlockInfo, error) { + var blocks []protocol.BlockInfo var offset int64 for { lr := &io.LimitedReader{R: r, N: int64(blocksize)} @@ -39,9 +30,9 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) { break } - b := Block{ - Offset: offset, + b := protocol.BlockInfo{ Size: uint32(n), + Offset: offset, Hash: hf.Sum(nil), } blocks = append(blocks, b) @@ -50,7 +41,7 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) { if len(blocks) == 0 { // Empty file - blocks = append(blocks, Block{ + blocks = append(blocks, protocol.BlockInfo{ Offset: 0, Size: 0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}, @@ -62,11 +53,18 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) { // BlockDiff returns lists of common and missing (to transform src into tgt) // blocks. Both block lists must have been created with the same block size. -func BlockDiff(src, tgt []Block) (have, need []Block) { +func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) { if len(tgt) == 0 && len(src) != 0 { return nil, nil } + // Set the Offset field on each target block + var offset int64 + for i := range tgt { + tgt[i].Offset = offset + offset += int64(tgt[i].Size) + } + if len(tgt) != 0 && len(src) == 0 { // Copy the entire file return nil, tgt diff --git a/scanner/blocks_test.go b/scanner/blocks_test.go index 685d52a1f..c4fbc5740 100644 --- a/scanner/blocks_test.go +++ b/scanner/blocks_test.go @@ -8,6 +8,8 @@ import ( "bytes" "fmt" "testing" + + "github.com/calmh/syncthing/protocol" ) var blocksTestData = []struct { @@ -83,20 +85,20 @@ var diffTestData = []struct { a string b string s int - d []Block + d []protocol.BlockInfo }{ - {"contents", "contents", 1024, []Block{}}, - {"", "", 1024, []Block{}}, - {"contents", "contents", 3, []Block{}}, - {"contents", "cantents", 3, []Block{{0, 3, nil}}}, - {"contents", "contants", 3, []Block{{3, 3, nil}}}, - {"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}}, - {"contents", "", 3, []Block{{0, 0, nil}}}, - {"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}}, - {"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}}, + {"contents", "contents", 1024, []protocol.BlockInfo{}}, + {"", "", 1024, []protocol.BlockInfo{}}, + {"contents", "contents", 3, []protocol.BlockInfo{}}, + {"contents", "cantents", 3, []protocol.BlockInfo{{0, 3, nil}}}, + {"contents", "contants", 3, []protocol.BlockInfo{{3, 3, nil}}}, + {"contents", "cantants", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}}}, + {"contents", "", 3, []protocol.BlockInfo{{0, 0, nil}}}, + {"", "contents", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}}, + {"con", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}}, {"contents", "con", 3, nil}, - {"contents", "cont", 3, []Block{{3, 1, nil}}}, - {"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}}, + {"contents", "cont", 3, []protocol.BlockInfo{{3, 1, nil}}}, + {"cont", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}}, } func TestDiff(t *testing.T) { diff --git a/scanner/blocks_xdr.go b/scanner/blocks_xdr.go deleted file mode 100644 index 92ec1b6f9..000000000 --- a/scanner/blocks_xdr.go +++ /dev/null @@ -1,49 +0,0 @@ -package scanner - -import ( - "bytes" - "io" - - "github.com/calmh/syncthing/xdr" -) - -func (o Block) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) -} - -func (o Block) MarshalXDR() []byte { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Block) AppendXDR(bs []byte) []byte { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) -} - -func (o Block) encodeXDR(xw *xdr.Writer) (int, error) { - xw.WriteUint64(uint64(o.Offset)) - xw.WriteUint32(o.Size) - xw.WriteBytes(o.Hash) - return xw.Tot(), xw.Error() -} - -func (o *Block) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.decodeXDR(xr) -} - -func (o *Block) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.decodeXDR(xr) -} - -func (o *Block) decodeXDR(xr *xdr.Reader) error { - o.Offset = int64(xr.ReadUint64()) - o.Size = xr.ReadUint32() - o.Hash = xr.ReadBytes() - return xr.Error() -} diff --git a/scanner/file.go b/scanner/file.go deleted file mode 100644 index d875f2194..000000000 --- a/scanner/file.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved. -// Use of this source code is governed by an MIT-style license that can be -// found in the LICENSE file. - -package scanner - -import "fmt" - -type File struct { - Name string - Flags uint32 - Modified int64 - Version uint64 - Size int64 - Blocks []Block - Suppressed bool -} - -func (f File) String() string { - return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v, Sup:%v}", - f.Name, f.Flags, f.Modified, f.Version, f.Size, f.Blocks, f.Suppressed) -} - -func (f File) Equals(o File) bool { - return f.Modified == o.Modified && f.Version == o.Version -} - -func (f File) NewerThan(o File) bool { - return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version) -} diff --git a/scanner/file_xdr.go b/scanner/file_xdr.go deleted file mode 100644 index af7ab4296..000000000 --- a/scanner/file_xdr.go +++ /dev/null @@ -1,64 +0,0 @@ -package scanner - -import ( - "bytes" - "io" - - "github.com/calmh/syncthing/xdr" -) - -func (o File) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) -} - -func (o File) MarshalXDR() []byte { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o File) AppendXDR(bs []byte) []byte { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) -} - -func (o File) encodeXDR(xw *xdr.Writer) (int, error) { - xw.WriteString(o.Name) - xw.WriteUint32(o.Flags) - xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(uint64(o.Size)) - xw.WriteUint32(uint32(len(o.Blocks))) - for i := range o.Blocks { - o.Blocks[i].encodeXDR(xw) - } - xw.WriteBool(o.Suppressed) - return xw.Tot(), xw.Error() -} - -func (o *File) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.decodeXDR(xr) -} - -func (o *File) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.decodeXDR(xr) -} - -func (o *File) decodeXDR(xr *xdr.Reader) error { - o.Name = xr.ReadString() - o.Flags = xr.ReadUint32() - o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.Size = int64(xr.ReadUint64()) - _BlocksSize := int(xr.ReadUint32()) - o.Blocks = make([]Block, _BlocksSize) - for i := range o.Blocks { - (&o.Blocks[i]).decodeXDR(xr) - } - o.Suppressed = xr.ReadBool() - return xr.Error() -} diff --git a/scanner/walk.go b/scanner/walk.go index 0cf11ee6b..8b9b0c0ee 100644 --- a/scanner/walk.go +++ b/scanner/walk.go @@ -55,12 +55,12 @@ type Suppressor interface { type CurrentFiler interface { // CurrentFile returns the file as seen at last scan. - CurrentFile(name string) File + CurrentFile(name string) protocol.FileInfo } // Walk returns the list of files found in the local repository by scanning the // file system. Files are blockwise hashed. -func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) { +func (w *Walker) Walk() (files []protocol.FileInfo, ignore map[string][]string, err error) { if debug { l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile) } @@ -122,7 +122,7 @@ func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.W } } -func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc { +func (w *Walker) walkAndHashFiles(res *[]protocol.FileInfo, ign map[string][]string) filepath.WalkFunc { return func(p string, info os.FileInfo, err error) error { if err != nil { if debug { @@ -183,7 +183,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath } else { flags |= uint32(info.Mode() & os.ModePerm) } - f := File{ + f := protocol.FileInfo{ Name: rn, Version: lamport.Default.Tick(0), Flags: flags, @@ -213,8 +213,8 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath if w.Suppressor != nil { if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev { l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p) - cf.Suppressed = true - cf.Version++ + cf.Flags |= protocol.FlagInvalid + cf.Version = lamport.Default.Tick(cf.Version) if debug { l.Debugln("suppressed:", cf) } @@ -256,10 +256,9 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath if w.IgnorePerms { flags = protocol.FlagNoPermBits | 0666 } - f := File{ + f := protocol.FileInfo{ Name: rn, Version: lamport.Default.Tick(0), - Size: info.Size(), Flags: flags, Modified: info.ModTime().Unix(), Blocks: blocks, diff --git a/xdr/cmd/genxdr/main.go b/xdr/cmd/genxdr/main.go index dac3fec5b..16155a440 100644 --- a/xdr/cmd/genxdr/main.go +++ b/xdr/cmd/genxdr/main.go @@ -178,6 +178,9 @@ func handleStruct(name string, t *ast.StructType) { if m := maxRe.FindStringSubmatch(c); m != nil { max, _ = strconv.Atoi(m[1]) } + if strings.Contains(c, "noencode") { + continue + } } var f field