lib/fs: Add case insensitivity to MtimeFS

This is step one of a hundred fifty on the path to case insensitivity.
It brings in the basic case folding mechanism and adds it to the
mtimefs, as this is something outside the fileset that touches stuff in
the database based on name. No effort to convert or handle existing
entries when the insensitivity is changed, I don't think we need it...

Useless by itself but includes tests and will reduce the review load
along the way.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4521

lib/fs/folding.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"unicode"
+)
+
+func UnicodeLowercase(s string) string {
+	rs := []rune(s)
+	for i, r := range rs {
+		rs[i] = unicode.ToLower(unicode.ToUpper(r))
+	}
+	return string(rs)
+}

lib/fs/folding_test.go

@@ -0,0 +1,48 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+import "testing"
+
+func TestUnicodeLowercase(t *testing.T) {
+	cases := [][2]string{
+		{"", ""},
+		{"hej", "hej"},
+		{"HeJ!@#", "hej!@#"},
+		// Western Europe diacritical stuff is trivial
+		{"ÜBERRÄKSMÖRGÅS", "überräksmörgås"},
+		// Cyrillic seems regular as well
+		{"Привет", "привет"},
+		// Greek has multiple lower case characters for things depending on
+		// context; we should always choose the right one.
+		{"Ὀδυσσεύς", "ὀδυσσεύσ"},
+		{"ὈΔΥΣΣΕΎΣ", "ὀδυσσεύσ"},
+		// German ß doesn't really have an upper case variant, and we
+		// shouldn't mess things up when lower casing it either. We don't
+		// attempt to make ß equivalant to "ss".
+		{"Reichwaldstraße", "reichwaldstraße"},
+		// The Turks do their thing with the Is.... Like the Greek example
+		// we pick just the one canonicalized "i" although you can argue
+		// with this... From what I understand most operating systems don't
+		// get this right anyway.
+		{"İI", "ii"},
+		// Arabic doesn't do case folding.
+		{"العَرَبِيَّة", "العَرَبِيَّة"},
+		// Neither does Hebrew.
+		{"עברית", "עברית"},
+		// Nor Chinese, in any variant.
+		{"汉语/漢語 or 中文", "汉语/漢語 or 中文"},
+		// Niether katakana as far as I can tell.
+		{"チャーハン", "チャーハン"},
+	}
+	for _, tc := range cases {
+		res := UnicodeLowercase(tc[0])
+		if res != tc[1] {
+			t.Errorf("UnicodeLowercase(%q) => %q, expected %q", tc[0], res, tc[1])
+		}
+	}
+}

lib/fs/mtimefs.go

@@ -6,7 +6,9 @@
 
 package fs
 
-import "time"
+import (
+	"time"
+)
 
 // The database is where we store the virtual mtimes
 type database interface {
@@ -20,16 +22,29 @@ type database interface {
 // just does the underlying operations with no additions.
 type MtimeFS struct {
 	Filesystem
-	chtimes func(string, time.Time, time.Time) error
-	db      database
+	chtimes         func(string, time.Time, time.Time) error
+	db              database
+	caseInsensitive bool
 }
 
-func NewMtimeFS(underlying Filesystem, db database) *MtimeFS {
-	return &MtimeFS{
+type MtimeFSOption func(*MtimeFS)
+
+func WithCaseInsensitivity(v bool) MtimeFSOption {
+	return func(f *MtimeFS) {
+		f.caseInsensitive = v
+	}
+}
+
+func NewMtimeFS(underlying Filesystem, db database, options ...MtimeFSOption) *MtimeFS {
+	f := &MtimeFS{
 		Filesystem: underlying,
 		chtimes:    underlying.Chtimes, // for mocking it out in the tests
 		db:         db,
 	}
+	for _, opt := range options {
+		opt(f)
+	}
+	return f
 }
 
 func (f *MtimeFS) Chtimes(name string, atime, mtime time.Time) error {
@@ -72,6 +87,10 @@ func (f *MtimeFS) Lstat(name string) (FileInfo, error) {
 // "virtual" is what want the timestamp to be
 
 func (f *MtimeFS) save(name string, real, virtual time.Time) {
+	if f.caseInsensitive {
+		name = UnicodeLowercase(name)
+	}
+
 	if real.Equal(virtual) {
 		// If the virtual time and the real on disk time are equal we don't
 		// need to store anything.
@@ -88,6 +107,10 @@ func (f *MtimeFS) save(name string, real, virtual time.Time) {
 }
 
 func (f *MtimeFS) load(name string) (real, virtual time.Time) {
+	if f.caseInsensitive {
+		name = UnicodeLowercase(name)
+	}
+
 	data, exists := f.db.Bytes(name)
 	if !exists {
 		return

lib/fs/mtimefs_test.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"io/ioutil"
 	"os"
+	"runtime"
 	"testing"
 	"time"
 )
@@ -80,6 +81,51 @@ func TestMtimeFS(t *testing.T) {
 	}
 }
 
+func TestMtimeFSInsensitive(t *testing.T) {
+	switch runtime.GOOS {
+	case "darwin", "windows":
+		// blatantly assume file systems here are case insensitive. Might be
+		// a spurious failure on oddly configured systems.
+	default:
+		t.Skip("need case insensitive FS")
+	}
+
+	theTest := func(t *testing.T, fs *MtimeFS, shouldSucceed bool) {
+		os.RemoveAll("testdata")
+		defer os.RemoveAll("testdata")
+		os.Mkdir("testdata", 0755)
+		ioutil.WriteFile("testdata/FiLe", []byte("hello"), 0644)
+
+		// a random time with nanosecond precision
+		testTime := time.Unix(1234567890, 123456789)
+
+		// Do one call that gets struck by an exceptionally evil Chtimes, with a
+		// different case from what is on disk.
+		fs.chtimes = evilChtimes
+		if err := fs.Chtimes("testdata/fIlE", testTime, testTime); err != nil {
+			t.Error("Should not have failed:", err)
+		}
+
+		// Check that we get back the mtime we set, if we were supposed to succed.
+		info, err := fs.Lstat("testdata/FILE")
+		if err != nil {
+			t.Error("Lstat shouldn't fail:", err)
+		} else if info.ModTime().Equal(testTime) != shouldSucceed {
+			t.Errorf("Time mismatch; got %v, comparison %v, expected equal=%v", info.ModTime(), testTime, shouldSucceed)
+		}
+	}
+
+	// The test should fail with a case sensitive mtimefs
+	t.Run("with case sensitive mtimefs", func(t *testing.T) {
+		theTest(t, NewMtimeFS(newBasicFilesystem("."), make(mapStore)), false)
+	})
+
+	// And succeed with a case insensitive one.
+	t.Run("with case insensitive mtimefs", func(t *testing.T) {
+		theTest(t, NewMtimeFS(newBasicFilesystem("."), make(mapStore), WithCaseInsensitivity(true)), true)
+	})
+}
+
 // The mapStore is a simple database
 
 type mapStore map[string][]byte