~kvik/clone: Fix directory read serialization bottleneck

94d6eed5cfd4c6950d66dca15f6424fcafcc7abc — kvik 4 years ago 4745a86 master

Fix directory read serialization bottleneck

The directory walker used dirreadall call for loading directory
entries into memory before assigning them to workers -- causing
severe and unwanted serialization that was especially noticeable
on long directories.

The solution is simple: chunk up the directory read and assign
work in between the chunks.

patch browse .tar.gz

1 files changed, 20 insertions(+), 23 deletions(-)

M clone.c

M clone.c => clone.c +20 -23

@@ 263,32 263,29 @@ clonedir(char *src, char *dst)
 		error("can't open: %r");
 		return;
 	}
-	n = dirreadall(fd, &dirs);
-	if(n < 0){
-		error("can't read directory: %r");
-		close(fd);
-		return;
-	}
-	close(fd);
-
-	for(d = dirs; n; n--, d++){
-		if(d->mode & DMDIR && same(skipdir, d))
-			continue;
-
-		sn = smprint("%s/%s", src, d->name);
-		dn = smprint("%s/%s", dst, d->name);
-		if(d->mode & DMDIR){
-			if(mkdir(sn, dn, d, nil) < 0)
+	while((n = dirread(fd, &dirs)) > 0){
+		for(d = dirs; n; n--, d++){
+			if(d->mode & DMDIR && same(skipdir, d))
 				continue;
-			clonedir(sn, dn);
-		}else{
-			f = filenew(sn, dn, d);
-			sendp(filechan, f);
+
+			sn = smprint("%s/%s", src, d->name);
+			dn = smprint("%s/%s", dst, d->name);
+			if(d->mode & DMDIR){
+				if(mkdir(sn, dn, d, nil) < 0)
+					continue;
+				clonedir(sn, dn);
+			}else{
+				f = filenew(sn, dn, d);
+				sendp(filechan, f);
+			}
+			free(sn);
+			free(dn);
 		}
-		free(sn);
-		free(dn);
+		free(dirs);
 	}
-	free(dirs);
+	if(n < 0)
+		error("can't read directory: %r");
+	close(fd);
 }
 
 void