[prev in list] [next in list] [prev in thread] [next in thread]
List: openbsd-tech
Subject: more nfs patches
From: marius aamodt eriksen <marius () monkey ! org>
Date: 2004-06-25 2:46:16
Message-ID: 20040625024616.GB3646 () umich ! edu
[Download RAW message or body]
here is another one, it collates nfs commits so that we no longer
have to wait for the rtt to the server for every block we write,
instead we commit blocks in bulk, and return early when they have
already been commited.
this will make a high impact for write performance on high latency
connections, but the difference is noticeable even on low latency
ones.
for example, through loopback:
# dd if=/dev/zero of=test count=1 bs=12m
tcp on lo0; without RANGECOMMIT
# time sh -c "cp test /mnt/nfs/ && sync"
34.11s real 0.00s user 0.02s system
tcp on lo0; with RANGECOMMIT
# time sh -c "cp test /mnt/nfs/ && sync"
22.93s real 0.00s user 0.02s system
idea and some code yanked from netbsd.
marius.
Index: kern/vfs_conf.c
===================================================================
RCS file: /cvs/src/sys/kern/vfs_conf.c,v
retrieving revision 1.23
diff -u -r1.23 vfs_conf.c
--- kern/vfs_conf.c 13 Apr 2004 00:15:28 -0000 1.23
+++ kern/vfs_conf.c 25 Jun 2004 02:11:11 -0000
@@ -60,6 +60,7 @@
#endif
#ifdef NFSCLIENT
+#include <sys/rwlock.h> /* XXX*/
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
#include <nfs/nfsnode.h>
Index: nfs/nfs_bio.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_bio.c,v
retrieving revision 1.38
diff -u -r1.38 nfs_bio.c
--- nfs/nfs_bio.c 2 Jun 2003 23:28:19 -0000 1.38
+++ nfs/nfs_bio.c 25 Jun 2004 02:11:12 -0000
@@ -35,7 +35,6 @@
* @(#)nfs_bio.c 8.9 (Berkeley) 3/30/95
*/
-
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/resourcevar.h>
@@ -46,6 +45,7 @@
#include <sys/mount.h>
#include <sys/kernel.h>
#include <sys/namei.h>
+#include <sys/rwlock.h>
#include <uvm/uvm_extern.h>
@@ -411,7 +411,17 @@
* Since this block is being modified, it must be written
* again and not just committed.
*/
- bp->b_flags &= ~B_NEEDCOMMIT;
+
+ if (NFS_ISV3(vp)) {
+ rw_enter_write(&np->n_commitlock, p);
+ if (bp->b_flags & B_NEEDCOMMIT) {
+ bp->b_flags &= ~B_NEEDCOMMIT;
+ nfs_del_tobecommitted_range(vp, bp);
+ }
+ nfs_del_committed_range(vp, bp);
+ rw_exit_write(&np->n_commitlock);
+ } else
+ bp->b_flags &= ~B_NEEDCOMMIT;
/*
* If the lease is non-cachable or IO_SYNC do bwrite().
@@ -661,7 +671,7 @@
error = nfs_readlinkrpc(vp, uiop, curproc->p_ucred);
break;
default:
- printf("nfs_doio: type %x unexpected\n",vp->v_type);
+ printf("nfs_doio: type %x unexpected\n", vp->v_type);
break;
};
if (error) {
@@ -686,10 +696,17 @@
vp, bp, bp->b_dirtyoff, bp->b_dirtyend);
#endif
error = nfs_writerpc(vp, uiop, &iomode, &must_commit);
- if (!error && iomode == NFSV3WRITE_UNSTABLE)
+
+ rw_enter_write(&np->n_commitlock, p != NULL ? p : curproc);
+ if (!error && iomode == NFSV3WRITE_UNSTABLE) {
bp->b_flags |= B_NEEDCOMMIT;
- else
+ nfs_add_tobecommitted_range(vp, bp);
+ } else {
bp->b_flags &= ~B_NEEDCOMMIT;
+ nfs_del_committed_range(vp, bp);
+ }
+ rw_exit_write(&np->n_commitlock);
+
bp->b_flags &= ~B_WRITEINPROG;
/*
Index: nfs/nfs_node.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_node.c,v
retrieving revision 1.26
diff -u -r1.26 nfs_node.c
--- nfs/nfs_node.c 2 Jun 2003 23:28:19 -0000 1.26
+++ nfs/nfs_node.c 25 Jun 2004 02:11:12 -0000
@@ -46,6 +46,7 @@
#include <sys/malloc.h>
#include <sys/pool.h>
#include <sys/hash.h>
+#include <sys/rwlock.h>
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
@@ -127,6 +128,8 @@
bzero((caddr_t)np, sizeof *np);
vp->v_data = np;
np->n_vnode = vp;
+
+ rw_init(&np->n_commitlock);
/*
* Are we getting the root? If so, make sure the vnode flags
Index: nfs/nfs_socket.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_socket.c,v
retrieving revision 1.37
diff -u -r1.37 nfs_socket.c
--- nfs/nfs_socket.c 24 Jun 2004 19:35:26 -0000 1.37
+++ nfs/nfs_socket.c 25 Jun 2004 02:11:12 -0000
@@ -53,6 +53,7 @@
#include <sys/syslog.h>
#include <sys/tprintf.h>
#include <sys/namei.h>
+#include <sys/rwlock.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
Index: nfs/nfs_subs.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_subs.c,v
retrieving revision 1.45
diff -u -r1.45 nfs_subs.c
--- nfs/nfs_subs.c 21 Jun 2004 23:50:38 -0000 1.45
+++ nfs/nfs_subs.c 25 Jun 2004 02:11:12 -0000
@@ -55,6 +55,7 @@
#include <sys/malloc.h>
#include <sys/pool.h>
#include <sys/time.h>
+#include <sys/rwlock.h>
#include <uvm/uvm_extern.h>
@@ -1756,6 +1757,193 @@
}
}
splx(s);
+}
+
+void
+nfs_merge_commit_ranges(vp)
+ struct vnode *vp;
+{
+ struct nfsnode *np = VTONFS(vp);
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSHED_VALID)) {
+ np->n_pushedlo = np->n_pushlo;
+ np->n_pushedhi = np->n_pushhi;
+ np->n_commitflags |= NFS_COMMIT_PUSHED_VALID;
+ } else {
+ if (np->n_pushlo < np->n_pushedlo)
+ np->n_pushedlo = np->n_pushlo;
+ if (np->n_pushhi > np->n_pushedhi)
+ np->n_pushedhi = np->n_pushhi;
+ }
+
+ np->n_pushlo = np->n_pushhi = 0;
+ np->n_commitflags &= ~NFS_COMMIT_PUSH_VALID;
+
+#ifdef fvdl_debug
+ printf("merge: committed: %u - %u\n", (unsigned)np->n_pushedlo,
+ (unsigned)np->n_pushedhi);
+#endif
+}
+
+int
+nfs_in_committed_range(vp, bp)
+ struct vnode *vp;
+ struct buf *bp;
+{
+ struct nfsnode *np = VTONFS(vp);
+ off_t lo, hi;
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSHED_VALID))
+ return 0;
+ lo = (off_t)bp->b_blkno * DEV_BSIZE;
+ hi = lo + bp->b_dirtyend;
+
+ return (lo >= np->n_pushedlo && hi <= np->n_pushedhi);
+}
+
+int
+nfs_in_tobecommitted_range(vp, bp)
+ struct vnode *vp;
+ struct buf *bp;
+{
+ struct nfsnode *np = VTONFS(vp);
+ off_t lo, hi;
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSH_VALID))
+ return 0;
+ lo = (off_t)bp->b_blkno * DEV_BSIZE;
+ hi = lo + bp->b_dirtyend;
+
+ return (lo >= np->n_pushlo && hi <= np->n_pushhi);
+}
+
+void
+nfs_add_committed_range(vp, bp)
+ struct vnode *vp;
+ struct buf *bp;
+{
+ struct nfsnode *np = VTONFS(vp);
+ off_t lo, hi;
+
+ lo = (off_t)bp->b_blkno * DEV_BSIZE;
+ hi = lo + bp->b_dirtyend;
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSHED_VALID)) {
+ np->n_pushedlo = lo;
+ np->n_pushedhi = hi;
+ np->n_commitflags |= NFS_COMMIT_PUSHED_VALID;
+ } else {
+ if (hi > np->n_pushedhi)
+ np->n_pushedhi = hi;
+ if (lo < np->n_pushedlo)
+ np->n_pushedlo = lo;
+ }
+#ifdef fvdl_debug
+ printf("add: committed: %u - %u\n", (unsigned)np->n_pushedlo,
+ (unsigned)np->n_pushedhi);
+#endif
+}
+
+void
+nfs_del_committed_range(vp, bp)
+ struct vnode *vp;
+ struct buf *bp;
+{
+ struct nfsnode *np = VTONFS(vp);
+ off_t lo, hi;
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSHED_VALID))
+ return;
+
+ lo = (off_t)bp->b_blkno * DEV_BSIZE;
+ hi = lo + bp->b_dirtyend;
+
+ if (lo > np->n_pushedhi || hi < np->n_pushedlo)
+ return;
+ if (lo <= np->n_pushedlo)
+ np->n_pushedlo = hi;
+ else if (hi >= np->n_pushedhi)
+ np->n_pushedhi = lo;
+ else {
+ /*
+ * XXX There's only one range. If the deleted range
+ * is in the middle, pick the largest of the
+ * contiguous ranges that it leaves.
+ */
+ if ((np->n_pushedlo - lo) > (hi - np->n_pushedhi))
+ np->n_pushedhi = lo;
+ else
+ np->n_pushedlo = hi;
+ }
+#ifdef fvdl_debug
+ printf("del: committed: %u - %u\n", (unsigned)np->n_pushedlo,
+ (unsigned)np->n_pushedhi);
+#endif
+}
+
+void
+nfs_add_tobecommitted_range(vp, bp)
+ struct vnode *vp;
+ struct buf *bp;
+{
+ struct nfsnode *np = VTONFS(vp);
+ off_t lo, hi;
+
+ lo = (off_t)bp->b_blkno * DEV_BSIZE;
+ hi = lo + bp->b_dirtyend;
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSH_VALID)) {
+ np->n_pushlo = lo;
+ np->n_pushhi = hi;
+ np->n_commitflags |= NFS_COMMIT_PUSH_VALID;
+ } else {
+ if (lo < np->n_pushlo)
+ np->n_pushlo = lo;
+ if (hi > np->n_pushhi)
+ np->n_pushhi = hi;
+ }
+#ifdef fvdl_debug
+ printf("add: tobecommitted: %u - %u\n", (unsigned)np->n_pushlo,
+ (unsigned)np->n_pushhi);
+#endif
+}
+
+void
+nfs_del_tobecommitted_range(vp, bp)
+ struct vnode *vp;
+ struct buf *bp;
+{
+ struct nfsnode *np = VTONFS(vp);
+ off_t lo, hi;
+
+ if (!(np->n_commitflags & NFS_COMMIT_PUSH_VALID))
+ return;
+
+ lo = (off_t)bp->b_blkno * DEV_BSIZE;
+ hi = lo + bp->b_dirtyend;
+
+ if (lo > np->n_pushhi || hi < np->n_pushlo)
+ return;
+
+ if (lo <= np->n_pushlo)
+ np->n_pushlo = hi;
+ else if (hi >= np->n_pushhi)
+ np->n_pushhi = lo;
+ else {
+ /*
+ * XXX There's only one range. If the deleted range
+ * is in the middle, pick the largest of the
+ * contiguous ranges that it leaves.
+ */
+ if ((np->n_pushlo - lo) > (hi - np->n_pushhi))
+ np->n_pushhi = lo;
+ else
+ np->n_pushlo = hi;
+ }
+#ifdef fvdl_debug
+ printf("del: tobecommitted: %u - %u\n", (unsigned)np->n_pushlo,
+ (unsigned)np->n_pushhi);
+#endif
}
/*
Index: nfs/nfs_syscalls.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_syscalls.c,v
retrieving revision 1.41
diff -u -r1.41 nfs_syscalls.c
--- nfs/nfs_syscalls.c 25 Jun 2004 00:54:28 -0000 1.41
+++ nfs/nfs_syscalls.c 25 Jun 2004 02:11:12 -0000
@@ -56,6 +56,7 @@
#include <sys/filedesc.h>
#include <sys/signalvar.h>
#include <sys/kthread.h>
+#include <sys/rwlock.h>
#include <sys/syscallargs.h>
Index: nfs/nfs_var.h
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_var.h,v
retrieving revision 1.21
diff -u -r1.21 nfs_var.h
--- nfs/nfs_var.h 3 Jul 2002 20:57:00 -0000 1.21
+++ nfs/nfs_var.h 25 Jun 2004 02:11:13 -0000
@@ -257,6 +257,13 @@
struct nfssvc_sock *, struct mbuf *, int *, int);
int netaddr_match(int, union nethostaddr *, struct mbuf *);
void nfs_clearcommit(struct mount *);
+int nfs_in_committed_range(struct vnode *, struct buf *);
+int nfs_in_tobecommitted_range(struct vnode *, struct buf *);
+void nfs_add_committed_range(struct vnode *, struct buf *);
+void nfs_del_committed_range(struct vnode *, struct buf *);
+void nfs_add_tobecommitted_range(struct vnode *, struct buf *);
+void nfs_del_tobecommitted_range(struct vnode *, struct buf *);
+void nfs_merge_commit_ranges(struct vnode *);
int nfsrv_errmap(struct nfsrv_descript *, int);
void nfsrvw_sort(gid_t *, int);
void nfsrv_setcred(struct ucred *, struct ucred *);
Index: nfs/nfs_vfsops.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_vfsops.c,v
retrieving revision 1.53
diff -u -r1.53 nfs_vfsops.c
--- nfs/nfs_vfsops.c 21 Jun 2004 23:50:38 -0000 1.53
+++ nfs/nfs_vfsops.c 25 Jun 2004 02:11:13 -0000
@@ -54,6 +54,7 @@
#include <net/if.h>
#include <net/route.h>
#include <netinet/in.h>
+#include <sys/rwlock.h>
#include <nfs/rpcv2.h>
#include <nfs/nfsproto.h>
Index: nfs/nfs_vnops.c
===================================================================
RCS file: /cvs/src/sys/nfs/nfs_vnops.c,v
retrieving revision 1.61
diff -u -r1.61 nfs_vnops.c
--- nfs/nfs_vnops.c 24 Jun 2004 19:35:26 -0000 1.61
+++ nfs/nfs_vnops.c 25 Jun 2004 02:11:13 -0000
@@ -58,6 +58,7 @@
#include <sys/dirent.h>
#include <sys/fcntl.h>
#include <sys/lockf.h>
+#include <sys/rwlock.h>
#include <uvm/uvm_extern.h>
@@ -2964,11 +2965,17 @@
int oldflags = bp->b_flags, retv = 1;
struct proc *p = curproc; /* XXX */
off_t off;
+ size_t cnt;
int s;
+ struct vnode *vp;
+ struct nfsnode *np;
if(!(bp->b_flags & B_BUSY))
panic("bwrite: buffer is not busy???");
+ vp = bp->b_vp;
+ np = VTONFS(vp);
+
#ifdef fvdl_debug
printf("nfs_writebp(%x): vp %x voff %d vend %d doff %d dend %d\n",
bp, bp->b_vp, bp->b_validoff, bp->b_validend, bp->b_dirtyoff,
@@ -2992,10 +2999,45 @@
*/
if ((oldflags & (B_NEEDCOMMIT | B_WRITEINPROG)) == B_NEEDCOMMIT) {
off = ((u_quad_t)bp->b_blkno) * DEV_BSIZE + bp->b_dirtyoff;
- bp->b_flags |= B_WRITEINPROG;
- retv = nfs_commit(bp->b_vp, off, bp->b_dirtyend-bp->b_dirtyoff,
- bp->b_proc);
- bp->b_flags &= ~B_WRITEINPROG;
+ cnt = bp->b_dirtyend - bp->b_dirtyoff;
+
+ rw_enter_write(&np->n_commitlock, p);
+ if (!(bp->b_flags & B_NEEDCOMMIT)) {
+ rw_exit_write(&np->n_commitlock);
+ return (0);
+ }
+
+ /*
+ * If it's already been commited by somebody else,
+ * bail.
+ */
+ if (!nfs_in_committed_range(vp, bp)) {
+ int pushedrange = 0;
+ /*
+ * Since we're going to do this, push as much
+ * as we can.
+ */
+
+ if (nfs_in_tobecommitted_range(vp, bp)) {
+ pushedrange = 1;
+ off = np->n_pushlo;
+ cnt = np->n_pushhi - np->n_pushlo;
+ }
+
+ bp->b_flags |= B_WRITEINPROG;
+ retv = nfs_commit(bp->b_vp, off, cnt, bp->b_proc);
+ bp->b_flags &= ~B_WRITEINPROG;
+
+ if (retv == 0) {
+ if (pushedrange)
+ nfs_merge_commit_ranges(vp);
+ else
+ nfs_add_committed_range(vp, bp);
+ }
+ } else
+ retv = 0; /* It has already been commited. */
+
+ rw_exit_write(&np->n_commitlock);
if (!retv) {
bp->b_dirtyoff = bp->b_dirtyend = 0;
bp->b_flags &= ~B_NEEDCOMMIT;
Index: nfs/nfsnode.h
===================================================================
RCS file: /cvs/src/sys/nfs/nfsnode.h,v
retrieving revision 1.21
diff -u -r1.21 nfsnode.h
--- nfs/nfsnode.h 26 Apr 2004 18:57:36 -0000 1.21
+++ nfs/nfsnode.h 25 Jun 2004 02:11:13 -0000
@@ -115,7 +115,20 @@
nfsfh_t n_fh; /* Small File Handle */
struct ucred *n_rcred;
struct ucred *n_wcred;
+
+ off_t n_pushedlo; /* 1st blk in commited range */
+ off_t n_pushedhi; /* Last block in range */
+ off_t n_pushlo; /* 1st block in commit range */
+ off_t n_pushhi; /* Last block in range */
+ struct rwlock n_commitlock; /* Serialize commits */
+ int n_commitflags;
};
+
+/*
+ * Values for n_commitflags
+ */
+#define NFS_COMMIT_PUSH_VALID 0x0001 /* push range valid */
+#define NFS_COMMIT_PUSHED_VALID 0x0002 /* pushed range valid */
#define n_atim n_un1.nf_atim
#define n_mtim n_un2.nf_mtim
--
marius a eriksen <marius@monkey.org> | http://monkey.org/~marius/
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic