lib/db: Add local need check&repair (#6950)

lib/db/db_test.go

@@ -843,6 +843,81 @@ func TestFlushRecursion(t *testing.T) {
 	}
 }
 
+func TestCheckLocalNeed(t *testing.T) {
+	db := NewLowlevel(backend.OpenMemory())
+	defer db.Close()
+
+	folderStr := "test"
+	fs := NewFileSet(folderStr, fs.NewFilesystem(fs.FilesystemTypeFake, ""), db)
+
+	// Add files such that we are in sync for a and b, and need c and d.
+	files := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}},
+	}
+	fs.Update(protocol.LocalDeviceID, files)
+	files[2].Version = files[2].Version.Update(remoteDevice0.Short())
+	files[3].Version = files[2].Version.Update(remoteDevice0.Short())
+	fs.Update(remoteDevice0, files)
+
+	checkNeed := func() {
+		snap := fs.Snapshot()
+		defer snap.Release()
+		c := snap.NeedSize(protocol.LocalDeviceID)
+		if c.Files != 2 {
+			t.Errorf("Expected 2 needed files locally, got %v in meta", c.Files)
+		}
+		needed := make([]protocol.FileInfo, 0, 2)
+		snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
+			needed = append(needed, fi.(protocol.FileInfo))
+			return true
+		})
+		if l := len(needed); l != 2 {
+			t.Errorf("Expected 2 needed files locally, got %v in db", l)
+		} else if needed[0].Name != "c" || needed[1].Name != "d" {
+			t.Errorf("Expected files c and d to be needed, got %v and %v", needed[0].Name, needed[1].Name)
+		}
+	}
+
+	checkNeed()
+
+	trans, err := db.newReadWriteTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer trans.close()
+
+	// Add "b" to needed and remove "d"
+	folder := []byte(folderStr)
+	key, err := trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[1].Name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = trans.Put(key, nil); err != nil {
+		t.Fatal(err)
+	}
+	key, err = trans.keyer.GenerateNeedFileKey(nil, folder, []byte(files[3].Name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = trans.Delete(key); err != nil {
+		t.Fatal(err)
+	}
+	if err := trans.Commit(); err != nil {
+		t.Fatal(err)
+	}
+
+	if repaired, err := db.checkLocalNeed(folder); err != nil {
+		t.Fatal(err)
+	} else if repaired != 2 {
+		t.Error("Expected 2 repaired local need items, got", repaired)
+	}
+
+	checkNeed()
+}
+
 func numBlockLists(db *Lowlevel) (int, error) {
 	it, err := db.Backend.NewPrefixIterator([]byte{KeyTypeBlockList})
 	if err != nil {

lib/db/lowlevel.go

@@ -801,19 +801,33 @@ func (db *Lowlevel) getMetaAndCheck(folder string) *metadataTracker {
 	db.gcMut.RLock()
 	defer db.gcMut.RUnlock()
 
-	meta, err := db.recalcMeta(folder)
-	if err == nil {
-		var fixed int
-		fixed, err = db.repairSequenceGCLocked(folder, meta)
-		if fixed != 0 {
-			l.Infof("Repaired %d sequence entries in database", fixed)
+	var err error
+	defer func() {
+		if err != nil && !backend.IsClosed(err) {
+			panic(err)
 		}
+	}()
+
+	var fixed int
+	fixed, err = db.checkLocalNeed([]byte(folder))
+	if err != nil {
+		return nil
+	}
+	if fixed != 0 {
+		l.Infof("Repaired %d local need entries for folder %v in database", fixed, folder)
 	}
 
-	if backend.IsClosed(err) {
+	meta, err := db.recalcMeta(folder)
+	if err != nil {
 		return nil
-	} else if err != nil {
-		panic(err)
+	}
+
+	fixed, err = db.repairSequenceGCLocked(folder, meta)
+	if err != nil {
+		return nil
+	}
+	if fixed != 0 {
+		l.Infof("Repaired %d sequence entries for folder %v in database", fixed, folder)
 	}
 
 	return meta
@@ -1035,6 +1049,86 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack
 	return fixed, t.Commit()
 }
 
+// Does not take care of metadata - if anything is repaired, the need count
+// needs to be recalculated.
+func (db *Lowlevel) checkLocalNeed(folder []byte) (int, error) {
+	repaired := 0
+
+	t, err := db.newReadWriteTransaction()
+	if err != nil {
+		return 0, err
+	}
+	defer t.close()
+
+	key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
+	if err != nil {
+		return 0, err
+	}
+	dbi, err := t.NewPrefixIterator(key.WithoutName())
+	if err != nil {
+		return 0, err
+	}
+	defer dbi.Release()
+
+	var needName string
+	var needDone bool
+	next := func() {
+		needDone = !dbi.Next()
+		if !needDone {
+			needName = string(t.keyer.NameFromGlobalVersionKey(dbi.Key()))
+		}
+	}
+	next()
+	t.withNeedIteratingGlobal(folder, protocol.LocalDeviceID[:], true, func(fi protocol.FileIntf) bool {
+		f := fi.(FileInfoTruncated)
+		for !needDone && needName < f.Name {
+			repaired++
+			if err = t.Delete(dbi.Key()); err != nil {
+				return false
+			}
+			l.Debugln("check local need: removing", needName)
+			next()
+		}
+		if needName == f.Name {
+			next()
+		} else {
+			repaired++
+			key, err = t.keyer.GenerateNeedFileKey(key, folder, []byte(f.Name))
+			if err != nil {
+				return false
+			}
+			if err = t.Put(key, nil); err != nil {
+				return false
+			}
+			l.Debugln("check local need: adding", f.Name)
+		}
+		return true
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	for !needDone {
+		repaired++
+		if err := t.Delete(dbi.Key()); err != nil {
+			return 0, err
+		}
+		l.Debugln("check local need: removing", needName)
+		next()
+	}
+
+	if err := dbi.Error(); err != nil {
+		return 0, err
+	}
+	dbi.Release()
+
+	if err = t.Commit(); err != nil {
+		return 0, err
+	}
+
+	return repaired, nil
+}
+
 // unchanged checks if two files are the same and thus don't need to be updated.
 // Local flags or the invalid bit might change without the version
 // being bumped.

lib/db/transactions.go

@@ -429,7 +429,10 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn
 	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
 		return t.withNeedLocal(folder, truncate, fn)
 	}
+	return t.withNeedIteratingGlobal(folder, device, truncate, fn)
+}
 
+func (t *readOnlyTransaction) withNeedIteratingGlobal(folder, device []byte, truncate bool, fn Iterator) error {
 	key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
 	if err != nil {
 		return err
@@ -468,11 +471,12 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn
 			return err
 		}
 
-		globalDev, ok := globalFV.FirstDevice()
-		if !ok {
-			return errEmptyFileVersion
+		if shouldDebug() {
+			if globalDev, ok := globalFV.FirstDevice(); ok {
+				globalID, _ := protocol.DeviceIDFromBytes(globalDev)
+				l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalID)
+			}
 		}
-		l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalDev)
 		if !fn(gf) {
 			return dbi.Error()
 		}