Merge branch 'rfjakob:master' into badnamecontent

Author: DerDonut

Documentation/performance.txt

@@ -59,18 +59,19 @@ v1.7.1            450      697      11.5   9.5  1.5    3.6
 **********************************************
 * CPU = Core i5-3470, governor = performance *
 **********************************************
-v1.7.1            556     1000       9.0   4.2  0.9    2.0    go1.13.6, Linux 5.4.17
-v1.7.1            577     1100       8.3   4.2  0.9    2.0    go1.14.2, Linux 5.6.7
-v1.7.1-60-gb23f77c 472    1100      12.7   4.2  0.8    2.0
-v1.8.0            410     1000      17.5   6.7  5.4    7.8    go1.15.3, Linux 5.8.13
-v2.0-beta1        387     1100      36.2  14.4 12.8   19.3
-v2.0-beta1-5-gc943ed3 417 1000      30.4  12.7  9.9   16.4
-v2.0-beta1-6      529     1100      17.5   9.0  3.6    9.0
+v1.7.1                 556 1000      9.0   4.2  0.9    2.0    go1.13.6, Linux 5.4.17
+v1.7.1                 577 1100      8.3   4.2  0.9    2.0    go1.14.2, Linux 5.6.7
+v1.7.1-60-gb23f77c     472 1100     12.7   4.2  0.8    2.0
+v1.8.0                 410 1000     17.5   6.7  5.4    7.8    go1.15.3, Linux 5.8.13
+v2.0-beta1             387 1100     36.2  14.4 12.8   19.3
+v2.0-beta1-5-gc943ed3  417 1000     30.4  12.7  9.9   16.4
+v2.0-beta1-6           529 1100     17.5   9.0  3.6    9.0
 v2.0-beta1-9-g029e44d  477 1000     15.5   8.7  2.8    7.6
 v2.0-beta2-16-geaca820 542  997     15.9   8.8  6.2    7.8    go1.16.2, Linux 5.11.10    fusefrontend: do not encrypt ACLs
 v2.0-beta2-36-g6aae2aa 505 1000     16.1   8.2  6.3    7.7
 v2.0-beta2-37-g24d5d39 558 1000     12.3   6.4  4.4    2.8                               fs: add initial dirfd caching
 v2.0-beta2-42-g4a07d65 549 1000      8.2   4.7  1.8    2.4                               fusefrontend: make dirCache work for "node itself"
+v2.0                   420 1000      8.5   4.5  1.8    2.3    go1.16.5, Linux 5.11.21
 
 Results for EncFS for comparison (benchmark.bash -encfs):
 

README.md

@@ -96,17 +96,28 @@ as well as in the go-fuse library.
 Compile
 -------
 
-With go 1.11 or higher:
+Install Go 1.11 or higher:
+
+* Debian/Ubuntu: `apt install golang`
+* Fedora: `dnf install golang`
+
+Then, download the source code and compile:
 
 	$ git clone https://github.com/rfjakob/gocryptfs.git
 	$ cd gocryptfs
-	$ ./build.bash
+	$ ./build-without-openssl.bash
 
-build.bash needs the OpenSSL headers installed (Debian: `apt install libssl-dev`,
-Fedora: `dnf install openssl-devel`). Alternatively, you can compile
-without OpenSSL using
+This will compile a static binary that uses the Go stdlib crypto backend.
 
-	$ ./build-without-openssl.bash
+If you want to use the OpenSSL crypto backend (faster on
+old CPUs lacking AES-NI), you have to install a few dependencies:
+
+* Debian/Ubuntu: `apt install libssl-dev gcc pkg-config`
+* Fedora: `dnf install openssl-devel gcc pkg-config`
+
+Then, run:
+
+	$ ./build.bash
 
 Use
 ---
@@ -166,9 +177,11 @@ Example for a CPU without AES-NI:
 
 ```
 $ ./gocryptfs -speed
-AES-GCM-256-OpenSSL    165.67 MB/s  (selected in auto mode)
-AES-GCM-256-Go          49.62 MB/s
-AES-SIV-512-Go          39.98 MB/s
+gocryptfs v2.0; go-fuse v2.1.1-0.20210508151621-62c5aa1919a7; 2021-06-06 go1.16.5 linux/amd64
+AES-GCM-256-OpenSSL    536.63 MB/s  
+AES-GCM-256-Go         831.84 MB/s  (selected in auto mode)
+AES-SIV-512-Go         155.85 MB/s  
+XChaCha20-Poly1305-Go  700.02 MB/s  (benchmark only, not selectable yet)
 ```
 
 You can run `./benchmark.bash` to run gocryptfs' canonical set of
@@ -177,22 +190,30 @@ tarball, recursively listing and finally deleting it. The output will
 look like this:
 
 ```
-$ ./benchmark.bash
-Testing gocryptfs at /tmp/benchmark.bash.DwL: gocryptfs v1.6; go-fuse v20170619-45-g95c6370; 2018-08-18 go1.10.3
-WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 1.1033 s, 238 MB/s
-READ:  262144000 bytes (262 MB, 250 MiB) copied, 0.945291 s, 277 MB/s
-UNTAR: 17.768
-MD5:   8.459
-LS:    1.460
-RM:    3.379
+$ ./benchmark.bash 
+Testing gocryptfs at /tmp/benchmark.bash.xFD: gocryptfs v2.0; go-fuse v2.1.1-0.20210508151621-62c5aa1919a7; 2021-06-06 go1.16.5 linux/amd64
+WRITE: 262144000 bytes (262 MB, 250 MiB) copied, 0,698174 s, 375 MB/s
+READ:  262144000 bytes (262 MB, 250 MiB) copied, 0,268916 s, 975 MB/s
+UNTAR: 8,970
+MD5:   4,846
+LS:    1,851
+RM:    2,367
 ```
 
 Changelog
 ---------
 
+v2.0.1, 2021-06-07
+* Fix symlink creation reporting the wrong size, causing git to report it as modified
+  ([#574](https://github.com/rfjakob/gocryptfs/issues/574))
+
+v2.0, 2021-06-05
+* Fix a few [issues discovered by xfstests](https://github.com/rfjakob/fuse-xfstests/wiki/results_2021-05-19)
+  * Biggest change: rewrite SEEK_HOLE / SEEK_DATA logic (now emulates 4k alignment)
+
 v2.0-beta4, 2021-05-15
-* Make ACLs *actually* work (pass `-acl` to enable) ([#536](https://github.com/rfjakob/gocryptfs/issues/536))
-* Blocklist RENAME_EXCHANGE and RENAME_WHITEOUT (broken as discovered by [fuse-xfstest/gocryptfs-2019-12](https://github.com/rfjakob/fuse-xfstests/tree/gocryptfs-2019-12))
+* **Make ACLs *actually* work (pass `-acl` to enable)** ([#536](https://github.com/rfjakob/gocryptfs/issues/536))
+* Blocklist `RENAME_EXCHANGE` and `RENAME_WHITEOUT` (broken as discovered by [fuse-xfstest/gocryptfs-2019-12](https://github.com/rfjakob/fuse-xfstests/tree/gocryptfs-2019-12))
 
 v2.0-beta3, 2021-04-24
 * MANPAGE: Split options into sections acc. to where they apply ([#517](https://github.com/rfjakob/gocryptfs/issues/517))

build.bash

@@ -94,8 +94,9 @@ fi
 # Actual "go build" call for gocryptfs
 go build "-ldflags=$GO_LDFLAGS" "$@"
 # Additional binaries
-(cd gocryptfs-xray; go build "-ldflags=$GO_LDFLAGS" "$@")
-(cd contrib/statfs; go build "-ldflags=$GO_LDFLAGS" "$@")
+for d in gocryptfs-xray contrib/statfs contrib/findholes contrib/atomicrename ; do
+	(cd "$d"; go build "-ldflags=$GO_LDFLAGS" "$@")
+done
 
 ./gocryptfs -version
 

contrib/atomicrename/.gitignore

@@ -0,0 +1 @@
+/atomicrename

contrib/atomicrename/main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"sync/atomic"
+	"syscall"
+)
+
+const fileCount = 100
+
+type stats struct {
+	renameOk            int
+	renameError         int
+	readOk              int
+	readError           int
+	readContentMismatch int
+}
+
+func usage() {
+	fmt.Printf(`atomicrename creates %d "src" files in the current directory, renames
+them in random order over a single "dst" file while reading the "dst"
+file concurrently in a loop.
+
+Progress and errors are reported as they occour in addition to a summary
+printed at the end. cifs and fuse filesystems are known to fail, local
+filesystems and nfs seem ok.
+
+See https://github.com/hanwen/go-fuse/issues/398 for background info.
+`, fileCount)
+	os.Exit(1)
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+
+	hello := []byte("hello world")
+	srcFiles := make(map[string]struct{})
+
+	// prepare source files
+	fmt.Print("creating files")
+	for i := 0; i < fileCount; i++ {
+		srcName := fmt.Sprintf("src.atomicrename.%d", i)
+		srcFiles[srcName] = struct{}{}
+		buf := bytes.Repeat([]byte("_"), i)
+		buf = append(buf, hello...)
+		if err := ioutil.WriteFile(srcName, buf, 0600); err != nil {
+			panic(err)
+		}
+		fmt.Print(".")
+	}
+	fmt.Print("\n")
+
+	// prepare destination file
+	const dstName = "dst.atomicrename"
+	if err := ioutil.WriteFile(dstName, hello, 0600); err != nil {
+		panic(err)
+	}
+
+	var running int32 = 1
+
+	stats := stats{}
+
+	// read thread
+	go func() {
+		for atomic.LoadInt32(&running) == 1 {
+			have, err := ioutil.ReadFile(dstName)
+			if err != nil {
+				fmt.Println(err)
+				stats.readError++
+				continue
+			}
+			if !strings.HasSuffix(string(have), string(hello)) {
+				fmt.Printf("content mismatch: have %q\n", have)
+				stats.readContentMismatch++
+				continue
+			}
+			fmt.Printf("content ok len=%d\n", len(have))
+			stats.readOk++
+		}
+	}()
+
+	// rename thread = main thread
+	for srcName := range srcFiles {
+		if err := os.Rename(srcName, dstName); err != nil {
+			fmt.Println(err)
+			stats.renameError++
+		}
+		stats.renameOk++
+	}
+	// Signal the Read goroutine to stop when loop is done
+	atomic.StoreInt32(&running, 0)
+
+	syscall.Unlink(dstName)
+	fmt.Printf("stats: %#v\n", stats)
+}

contrib/findholes/.gitignore

@@ -0,0 +1 @@
+/findholes