Simple file versioning (fixes #218)

cmd/syncthing/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/calmh/syncthing/discover"
 	"github.com/calmh/syncthing/logger"
 	"github.com/calmh/syncthing/model"
+	"github.com/calmh/syncthing/osutil"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/upnp"
 	"github.com/juju/ratelimit"
@@ -498,7 +499,7 @@ func saveConfigLoop(cfgFile string) {
 			continue
 		}
 
-		err = model.Rename(cfgFile+".tmp", cfgFile)
+		err = osutil.Rename(cfgFile+".tmp", cfgFile)
 		if err != nil {
 			l.Warnln(err)
 		}

config/config.go

@@ -27,13 +27,56 @@ type Configuration struct {
 }
 
 type RepositoryConfiguration struct {
-	ID          string              `xml:"id,attr"`
+	ID          string                  `xml:"id,attr"`
-	Directory   string              `xml:"directory,attr"`
+	Directory   string                  `xml:"directory,attr"`
-	Nodes       []NodeConfiguration `xml:"node"`
+	Nodes       []NodeConfiguration     `xml:"node"`
-	ReadOnly    bool                `xml:"ro,attr"`
+	ReadOnly    bool                    `xml:"ro,attr"`
-	IgnorePerms bool                `xml:"ignorePerms,attr"`
+	IgnorePerms bool                    `xml:"ignorePerms,attr"`
-	Invalid     string              `xml:"-"` // Set at runtime when there is an error, not saved
+	Invalid     string                  `xml:"-"` // Set at runtime when there is an error, not saved
-	nodeIDs     []string
+	Versioning  VersioningConfiguration `xml:"versioning"`
+
+	nodeIDs []string
+}
+
+type VersioningConfiguration struct {
+	Type   string `xml:"type,attr"`
+	Params map[string]string
+}
+
+type InternalVersioningConfiguration struct {
+	Type   string          `xml:"type,attr,omitempty"`
+	Params []InternalParam `xml:"param"`
+}
+
+type InternalParam struct {
+	Key string `xml:"key,attr"`
+	Val string `xml:"val,attr"`
+}
+
+func (c *VersioningConfiguration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	var tmp InternalVersioningConfiguration
+	tmp.Type = c.Type
+	for k, v := range c.Params {
+		tmp.Params = append(tmp.Params, InternalParam{k, v})
+	}
+
+	return e.EncodeElement(tmp, start)
+
+}
+
+func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var tmp InternalVersioningConfiguration
+	err := d.DecodeElement(&tmp, &start)
+	if err != nil {
+		return err
+	}
+
+	c.Type = tmp.Type
+	c.Params = make(map[string]string, len(tmp.Params))
+	for _, p := range tmp.Params {
+		c.Params[p.Key] = p.Val
+	}
+	return nil
 }
 
 func (r *RepositoryConfiguration) NodeIDs() []string {

gui/app.js

@@ -405,10 +405,16 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     }
 
     $scope.editRepo = function (nodeCfg) {
-        $scope.currentRepo = $.extend({selectedNodes: {}}, nodeCfg);
+        $scope.currentRepo = angular.copy(nodeCfg);
+        $scope.currentRepo.selectedNodes = {};
         $scope.currentRepo.Nodes.forEach(function (n) {
             $scope.currentRepo.selectedNodes[n.NodeID] = true;
         });
+        if ($scope.currentRepo.Versioning && $scope.currentRepo.Versioning.Type === "simple") {
+            $scope.currentRepo.simpleFileVersioning = true;
+            $scope.currentRepo.simpleKeep = +$scope.currentRepo.Versioning.Params.keep;
+        }
+        $scope.currentRepo.simpleKeep = $scope.currentRepo.simpleKeep || 5;
         $scope.editingExisting = true;
         $scope.repoEditor.$setPristine();
         $('#editRepo').modal({backdrop: 'static', keyboard: true});
@@ -436,6 +442,19 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
         delete repoCfg.selectedNodes;
 
+        if (repoCfg.simpleFileVersioning) {
+            repoCfg.Versioning = {
+                'Type': 'simple',
+                'Params': {
+                    'keep': '' + repoCfg.simpleKeep,
+                }
+            };
+            delete repoCfg.simpleFileVersioning;
+            delete repoCfg.simpleKeep;
+        } else {
+            delete repoCfg.Versioning;
+        }
+
         $scope.repos[repoCfg.ID] = repoCfg;
         $scope.config.Repositories = repoList($scope.repos);
 

gui/index.html

@@ -455,47 +455,75 @@
         </div>
         <div class="modal-body">
           <form role="form" name="repoEditor">
-            <div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
+            <div class="row">
-              <label for="repoID">Repository ID</label>
+              <div class="col-md-12">
-              <input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
+                <div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
-              <p class="help-block">
+                  <label for="repoID">Repository ID</label>
-                <span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
+                  <input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
-                <span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
+                  <p class="help-block">
-                <span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
+                    <span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
-              </p>
+                    <span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
-            </div>
+                    <span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
-            <div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
+                  </p>
-              <label for="repoPath">Repository Path</label>
+                </div>
-              <input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
+                <div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
-              <p class="help-block">
+                  <label for="repoPath">Repository Path</label>
-                <span ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist. The tilde character <code>~</code> can be used as a shortcut for <code>{{system.tilde}}</code>.</span>
+                  <input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
-                <span ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
+                  <p class="help-block">
-              </p>
+                    <span ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist. The tilde character <code>~</code> can be used as a shortcut for <code>{{system.tilde}}</code>.</span>
-            </div>
+                    <span ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
-            <div class="form-group">
+                  </p>
-              <div class="checkbox">
+                </div>
-                <label>
-                  <input type="checkbox" ng-model="currentRepo.ReadOnly"> Repository Master
-                </label>
               </div>
-              <p class="help-block">Files are protected from changes made on other nodes, but changes made on <em>this</em> node will be sent to the rest of the cluster.</p>
             </div>
-            <div class="form-group">
+            <div class="row">
-              <div class="checkbox">
+              <div class="col-md-6">
-                <label>
+                <div class="form-group">
-                  <input type="checkbox" ng-model="currentRepo.IgnorePerms"> Ignore Permissions
+                  <div class="checkbox">
-                </label>
+                    <label>
+                      <input type="checkbox" ng-model="currentRepo.ReadOnly"> Repository Master
+                    </label>
+                  </div>
+                  <p class="help-block">Files are protected from changes made on other nodes, but changes made on <em>this</em> node will be sent to the rest of the cluster.</p>
+                </div>
+                <div class="form-group">
+                  <div class="checkbox">
+                    <label>
+                      <input type="checkbox" ng-model="currentRepo.IgnorePerms"> Ignore Permissions
+                    </label>
+                  </div>
+                  <p class="help-block">File permission bits are ignored when looking for changes. Use on FAT filesystems.</p>
+                </div>
+                <div class="form-group">
+                  <label for="nodes">Nodes</label>
+                  <div class="checkbox" ng-repeat="node in otherNodes()">
+                    <label>
+                      <input type="checkbox" ng-model="currentRepo.selectedNodes[node.NodeID]"> {{nodeName(node)}}
+                    </label>
+                  </div>
+                  <p class="help-block">Select the nodes to share this repository with.</p>
+                </div>
               </div>
-              <p class="help-block">File permission bits are ignored when looking for changes. Use on FAT filesystems.</p>
+              <div class="col-md-6">
-            </div>
+                <div class="form-group">
-            <div class="form-group">
+                  <div class="checkbox">
-              <label for="nodes">Nodes</label>
+                    <label>
-              <div class="checkbox" ng-repeat="node in otherNodes()">
+                      <input type="checkbox" ng-model="currentRepo.simpleFileVersioning"> Simple File Versioning
-                <label>
+                    </label>
-                  <input type="checkbox" ng-model="currentRepo.selectedNodes[node.NodeID]"> {{nodeName(node)}}
+                  </div>
-                </label>
+                  <p class="help-block">Files are moved to date stamped versions in a <code>.stversions</code> folder when replaced or deleted by syncthing.</p>
+                </div>
+                <div class="form-group" ng-if="currentRepo.simpleFileVersioning" ng-class="{'has-error': repoEditor.simpleKeep.$invalid && repoEditor.simpleKeep.$dirty}">
+                  <label for="simpleKeep">Keep Versions</label>
+                  <input name="simpleKeep" id="simpleKeep" class="form-control" type="number" ng-model="currentRepo.simpleKeep" required min="1"></input>
+                  <p class="help-block">
+                    <span ng-if="repoEditor.simpleKeep.$valid || repoEditor.simpleKeep.$pristine">The number of old versions to keep, per file.</span>
+                    <span ng-if="repoEditor.simpleKeep.$error.required && repoEditor.simpleKeep.$dirty">The number of versions must be a number and cannot be blank.</span>
+                    <span ng-if="repoEditor.simpleKeep.$error.min && repoEditor.simpleKeep.$dirty">You must keep at least one version.</span>
+                  </p>
+                </div>
+
               </div>
-              <p class="help-block">Select the nodes to share this repository with.</p>
             </div>
           </form>
           <div ng-show="!editingExisting">

model/model.go

@@ -17,6 +17,7 @@ import (
 	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/lamport"
+	"github.com/calmh/syncthing/osutil"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
 )
@@ -716,7 +717,7 @@ func (m *Model) saveIndex(repo string, dir string, fs []protocol.FileInfo) {
 	gzw.Close()
 	idxf.Close()
 
-	Rename(name+".tmp", name)
+	osutil.Rename(name+".tmp", name)
 }
 
 func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {

model/puller.go

@@ -10,8 +10,10 @@ import (
 	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/cid"
 	"github.com/calmh/syncthing/config"
+	"github.com/calmh/syncthing/osutil"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
+	"github.com/calmh/syncthing/versioner"
 )
 
 type requestResult struct {
@@ -71,6 +73,7 @@ type puller struct {
 	requestSlots      chan bool
 	blocks            chan bqBlock
 	requestResults    chan requestResult
+	versioner         versioner.Versioner
 }
 
 func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int, cfg *config.Configuration) *puller {
@@ -86,6 +89,14 @@ func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int,
 		requestResults:    make(chan requestResult),
 	}
 
+	if len(repoCfg.Versioning.Type) > 0 {
+		factory, ok := versioner.Factories[repoCfg.Versioning.Type]
+		if !ok {
+			l.Fatalf("Requested versioning type %q that does not exist", repoCfg.Versioning.Type)
+		}
+		p.versioner = factory(repoCfg.Versioning.Params)
+	}
+
 	if slots > 0 {
 		// Read/write
 		for i := 0; i < slots; i++ {
@@ -221,6 +232,10 @@ func (p *puller) fixupDirectories() {
 			return nil
 		}
 
+		if filepath.Base(rn) == ".stversions" {
+			return nil
+		}
+
 		cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
 		if cur.Name != rn {
 			// No matching dir in current list; weird
@@ -284,10 +299,10 @@ func (p *puller) fixupDirectories() {
 				l.Debugln("delete dir:", dir)
 			}
 			err := os.Remove(dir)
-			if err != nil {
+			if err == nil {
-				l.Warnln(err)
-			} else {
 				deleted++
+			} else if p.versioner == nil { // Failures are expected in the presence of versioning
+				l.Warnln(err)
 			}
 		}
 
@@ -385,7 +400,7 @@ func (p *puller) handleBlock(b bqBlock) bool {
 			}
 			return true
 		}
-		defTempNamer.Hide(of.temp)
+		osutil.HideFile(of.temp)
 	}
 
 	if of.err != nil {
@@ -524,7 +539,11 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
 		}
 		os.Remove(of.temp)
 		os.Chmod(of.filepath, 0666)
-		if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
+		if p.versioner != nil {
+			if err := p.versioner.Archive(of.filepath); err == nil {
+				p.model.updateLocal(p.repoCfg.ID, f)
+			}
+		} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
 			p.model.updateLocal(p.repoCfg.ID, f)
 		}
 	} else {
@@ -540,8 +559,8 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
 			delete(p.openFiles, f.Name)
 			return
 		}
-		defTempNamer.Show(of.temp)
+		osutil.ShowFile(of.temp)
-		if Rename(of.temp, of.filepath) == nil {
+		if osutil.Rename(of.temp, of.filepath) == nil {
 			p.model.updateLocal(p.repoCfg.ID, f)
 		}
 	}
@@ -614,11 +633,23 @@ func (p *puller) closeFile(f scanner.File) {
 			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
 		}
 	}
-	defTempNamer.Show(of.temp)
+
+	osutil.ShowFile(of.temp)
+
+	if p.versioner != nil {
+		err := p.versioner.Archive(of.filepath)
+		if err != nil {
+			if debug {
+				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+			}
+			return
+		}
+	}
+
 	if debug {
 		l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath)
 	}
-	if err := Rename(of.temp, of.filepath); err == nil {
+	if err := osutil.Rename(of.temp, of.filepath); err == nil {
 		p.model.updateLocal(p.repoCfg.ID, f)
 	} else {
 		l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)

model/tempname.go

@@ -23,11 +23,3 @@ func (t tempNamer) TempName(name string) string {
 	tname := fmt.Sprintf("%s.%s", t.prefix, filepath.Base(name))
 	return filepath.Join(tdir, tname)
 }
-
-func (t tempNamer) Hide(path string) error {
-	return nil
-}
-
-func (t tempNamer) Show(path string) error {
-	return nil
-}

model/tempname_windows.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
-	"syscall"
 )
 
 type tempNamer struct {
@@ -24,33 +23,3 @@ func (t tempNamer) TempName(name string) string {
 	tname := fmt.Sprintf("%s.%s.tmp", t.prefix, filepath.Base(name))
 	return filepath.Join(tdir, tname)
 }
-
-func (t tempNamer) Hide(path string) error {
-	p, err := syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return err
-	}
-
-	attrs, err := syscall.GetFileAttributes(p)
-	if err != nil {
-		return err
-	}
-
-	attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
-	return syscall.SetFileAttributes(p, attrs)
-}
-
-func (t tempNamer) Show(path string) error {
-	p, err := syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return err
-	}
-
-	attrs, err := syscall.GetFileAttributes(p)
-	if err != nil {
-		return err
-	}
-
-	attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
-	return syscall.SetFileAttributes(p, attrs)
-}

model/util.go

@@ -2,26 +2,12 @@ package model
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
 )
 
-func Rename(from, to string) error {
-	if runtime.GOOS == "windows" {
-		os.Chmod(to, 0666) // Make sure the file is user writeable
-		err := os.Remove(to)
-		if err != nil && !os.IsNotExist(err) {
-			l.Warnln(err)
-		}
-	}
-	defer os.Remove(from) // Don't leave a dangling temp file in case of rename error
-	return os.Rename(from, to)
-}
-
 func fileFromFileInfo(f protocol.FileInfo) scanner.File {
 	var blocks = make([]scanner.Block, len(f.Blocks))
 	var offset int64

osutil/hidden_unix.go

@@ -0,0 +1,11 @@
+// +build !windows
+
+package osutil
+
+func HideFile(path string) error {
+	return nil
+}
+
+func ShowFile(path string) error {
+	return nil
+}

osutil/hidden_windows.go

@@ -0,0 +1,35 @@
+// +build windows
+
+package osutil
+
+import "syscall"
+
+func HideFile(path string) error {
+	p, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return err
+	}
+
+	attrs, err := syscall.GetFileAttributes(p)
+	if err != nil {
+		return err
+	}
+
+	attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
+	return syscall.SetFileAttributes(p, attrs)
+}
+
+func ShowFile(path string) error {
+	p, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return err
+	}
+
+	attrs, err := syscall.GetFileAttributes(p)
+	if err != nil {
+		return err
+	}
+
+	attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
+	return syscall.SetFileAttributes(p, attrs)
+}

osutil/osutil.go

@@ -0,0 +1,18 @@
+package osutil
+
+import (
+	"os"
+	"runtime"
+)
+
+func Rename(from, to string) error {
+	if runtime.GOOS == "windows" {
+		os.Chmod(to, 0666) // Make sure the file is user writeable
+		err := os.Remove(to)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+	defer os.Remove(from) // Don't leave a dangling temp file in case of rename error
+	return os.Rename(from, to)
+}

scanner/walk.go

@@ -144,15 +144,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 			return nil
 		}
 
-		if _, sn := filepath.Split(rn); sn == w.IgnoreFile {
+		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
-			// An ignore-file; these are ignored themselves
-			if debug {
-				l.Debugln("ignorefile:", rn)
-			}
-			return nil
-		}
-
-		if w.ignoreFile(ign, rn) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)

versioner/debug.go

@@ -0,0 +1,13 @@
+package versioner
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/syncthing/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "versioner") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

versioner/simple.go

@@ -0,0 +1,84 @@
+package versioner
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/calmh/syncthing/osutil"
+)
+
+func init() {
+	// Register the constructor for this type of versioner with the name "simple"
+	Factories["simple"] = NewSimple
+}
+
+// The type holds our configuration
+type Simple struct {
+	keep int
+}
+
+// The constructor function takes a map of parameters and creates the type.
+func NewSimple(params map[string]string) Versioner {
+	keep, err := strconv.Atoi(params["keep"])
+	if err != nil {
+		keep = 5 // A reasonable default
+	}
+
+	s := Simple{
+		keep: keep,
+	}
+
+	if debug {
+		l.Debugf("instantiated %#v", s)
+	}
+	return s
+}
+
+// Move away the named file to a version archive. If this function returns
+// nil, the named file does not exist any more (has been archived).
+func (v Simple) Archive(path string) error {
+	_, err := os.Stat(path)
+	if err != nil && os.IsNotExist(err) {
+		return nil
+	}
+
+	if debug {
+		l.Debugln("archiving", path)
+	}
+
+	file := filepath.Base(path)
+	dir := filepath.Join(filepath.Dir(path), ".stversions")
+	err = os.MkdirAll(dir, 0755)
+	if err != nil && !os.IsExist(err) {
+		return err
+	} else {
+		osutil.HideFile(dir)
+	}
+
+	ver := file + "~" + time.Now().Format("20060102-150405")
+	err = osutil.Rename(path, filepath.Join(dir, ver))
+	if err != nil {
+		return err
+	}
+
+	versions, err := filepath.Glob(filepath.Join(dir, file+"~*"))
+	if err != nil {
+		l.Warnln(err)
+		return nil
+	}
+
+	if len(versions) > v.keep {
+		sort.Strings(versions)
+		for _, toRemove := range versions[:len(versions)-v.keep] {
+			err = os.Remove(toRemove)
+			if err != nil {
+				l.Warnln(err)
+			}
+		}
+	}
+
+	return nil
+}

versioner/versioner.go

@@ -0,0 +1,7 @@
+package versioner
+
+type Versioner interface {
+	Archive(path string) error
+}
+
+var Factories = map[string]func(map[string]string) Versioner{}