This is the first non-RFC iteration of the NFS re-export
improvement series for nfs-utils.
While the kernel side[0] didn't change at all and is still small,
the userspace side saw much more changes.
The core idea is adding new export option: reeport=
Using reexport= it is possible to mark an export entry in the exports
file explicitly as NFS re-export and select a strategy how unique
identifiers should be provided.
Currently two strategies are supported, "auto-fsidnum" and
"predefined-fsidnum", both use a SQLite database as backend to keep
track of generated ids.
For a more detailed description see patch "exports: Implement new export option reexport=".
I choose SQLite because nfs-utils already uses it and using SQL ids can nicely
generated and maintained. It will also scale for large setups where the amount
of subvolumes is high.
Beside of id generation this series also addresses the reboot problem.
If the re-exporting NFS server reboots, uncovered NFS subvolumes are not yet
mounted and file handles become stale.
Now mountd/exportd keeps track of uncovered subvolumes and makes sure they get
uncovered while nfsd starts.
The whole set of features is currently opt-in via --enable-reexport.
I'm also not sure about the rearrangement of the reexport code,
currently it is a helper library.
A typical export entry on a re-exporting server looks like:
/nfs *(rw,no_root_squash,no_subtree_check,crossmnt,reexport=auto-fsidnum)
reexport=auto-fsidnum will automatically assign an fsid= to /nfs and all
uncovered subvolumes.
Richard Weinberger (5):
Implement reexport helper library
exports: Implement new export option reexport=
export: Implement logic behind reexport=
export: Avoid fsid= conflicts
reexport: Make state database location configurable
[0] https://git.kernel.org/pub/scm/linux/kernel/git/rw/misc.git/log/?h=nfs_reexport_clean
configure.ac | 12 ++
nfs.conf | 3 +
support/Makefile.am | 4 +
support/export/Makefile.am | 2 +
support/export/cache.c | 71 ++++++-
support/export/export.c | 27 ++-
support/include/nfslib.h | 1 +
support/nfs/Makefile.am | 1 +
support/nfs/exports.c | 68 +++++++
support/reexport/Makefile.am | 6 +
support/reexport/reexport.c | 354 +++++++++++++++++++++++++++++++++
support/reexport/reexport.h | 39 ++++
systemd/Makefile.am | 4 +
systemd/nfs-server-generator.c | 14 +-
systemd/nfs.conf.man | 6 +
utils/exportd/Makefile.am | 8 +-
utils/exportd/exportd.c | 5 +
utils/exportfs/Makefile.am | 6 +
utils/exportfs/exportfs.c | 21 +-
utils/exportfs/exports.man | 31 +++
utils/mount/Makefile.am | 7 +
utils/mountd/Makefile.am | 6 +
utils/mountd/mountd.c | 1 +
utils/mountd/svc_run.c | 6 +
24 files changed, 690 insertions(+), 13 deletions(-)
create mode 100644 support/reexport/Makefile.am
create mode 100644 support/reexport/reexport.c
create mode 100644 support/reexport/reexport.h
--
2.31.1
This covers the cross mount case. When mountd/exportd detect
a cross mount on a re-exported NFS volume a identifier has to
be found to make nfsd happy.
Signed-off-by: Richard Weinberger <[email protected]>
---
support/export/cache.c | 71 +++++++++++++++++++++++++++++++++----
support/reexport/reexport.c | 1 +
utils/exportd/Makefile.am | 8 ++++-
utils/exportd/exportd.c | 5 +++
utils/mountd/Makefile.am | 6 ++++
utils/mountd/mountd.c | 1 +
utils/mountd/svc_run.c | 6 ++++
7 files changed, 91 insertions(+), 7 deletions(-)
diff --git a/support/export/cache.c b/support/export/cache.c
index a5823e92..307d183b 100644
--- a/support/export/cache.c
+++ b/support/export/cache.c
@@ -33,6 +33,7 @@
#include "export.h"
#include "pseudoflavors.h"
#include "xcommon.h"
+#include "reexport.h"
#ifdef HAVE_JUNCTION_SUPPORT
#include "fsloc.h"
@@ -235,6 +236,16 @@ static void auth_unix_gid(int f)
xlog(L_ERROR, "auth_unix_gid: error writing reply");
}
+static int match_crossmnt_fsidnum(uint32_t parsed_fsidnum, char *path)
+{
+ uint32_t fsidnum;
+
+ if (reexpdb_fsidnum_by_path(path, &fsidnum, 0) == 0)
+ return 0;
+
+ return fsidnum == parsed_fsidnum;
+}
+
#ifdef USE_BLKID
static const char *get_uuid_blkdev(char *path)
{
@@ -684,8 +695,13 @@ static int match_fsid(struct parsed_fsid *parsed, nfs_export *exp, char *path)
goto match;
case FSID_NUM:
if (((exp->m_export.e_flags & NFSEXP_FSID) == 0 ||
- exp->m_export.e_fsid != parsed->fsidnum))
+ exp->m_export.e_fsid != parsed->fsidnum)) {
+ if (exp->m_export.e_flags & NFSEXP_CROSSMOUNT &&
+ match_crossmnt_fsidnum(parsed->fsidnum, path))
+ goto match;
+
goto nomatch;
+ }
goto match;
case FSID_UUID4_INUM:
case FSID_UUID16_INUM:
@@ -789,6 +805,9 @@ static void nfsd_fh(int f)
goto out;
}
+ if (parsed.fsidtype == FSID_NUM)
+ reexpdb_uncover_subvolume(parsed.fsidnum);
+
/* Now determine export point for this fsid/domain */
for (i=0 ; i < MCL_MAXTYPES; i++) {
nfs_export *next_exp;
@@ -932,7 +951,7 @@ static void write_fsloc(char **bp, int *blen, struct exportent *ep)
release_replicas(servers);
}
#endif
-static void write_secinfo(char **bp, int *blen, struct exportent *ep, int flag_mask)
+static void write_secinfo(char **bp, int *blen, struct exportent *ep, int flag_mask, int extra_flag)
{
struct sec_entry *p;
@@ -947,11 +966,20 @@ static void write_secinfo(char **bp, int *blen, struct exportent *ep, int flag_m
qword_addint(bp, blen, p - ep->e_secinfo);
for (p = ep->e_secinfo; p->flav; p++) {
qword_addint(bp, blen, p->flav->fnum);
- qword_addint(bp, blen, p->flags & flag_mask);
+ qword_addint(bp, blen, (p->flags | extra_flag) & flag_mask);
}
}
+static int can_reexport_via_fsidnum(struct exportent *exp, struct statfs64 *st)
+{
+ if (st->f_type != 0x6969 /* NFS_SUPER_MAGIC */)
+ return 0;
+
+ return exp->e_reexport == REEXP_PREDEFINED_FSIDNUM ||
+ exp->e_reexport == REEXP_AUTO_FSIDNUM;
+}
+
static int dump_to_cache(int f, char *buf, int blen, char *domain,
char *path, struct exportent *exp, int ttl)
{
@@ -968,17 +996,48 @@ static int dump_to_cache(int f, char *buf, int blen, char *domain,
if (exp) {
int different_fs = strcmp(path, exp->e_path) != 0;
int flag_mask = different_fs ? ~NFSEXP_FSID : ~0;
+ int rc, do_fsidnum = 0;
+ uint32_t fsidnum = exp->e_fsid;
+
+ if (different_fs) {
+ struct statfs64 st;
+
+ rc = nfsd_path_statfs64(path, &st);
+ if (rc) {
+ xlog(L_WARNING, "unable to statfs %s", path);
+ errno = EINVAL;
+ return -1;
+ }
+
+ if (can_reexport_via_fsidnum(exp, &st)) {
+ do_fsidnum = 1;
+ flag_mask = ~0;
+ }
+ }
qword_adduint(&bp, &blen, now + exp->e_ttl);
- qword_addint(&bp, &blen, exp->e_flags & flag_mask);
+
+ if (do_fsidnum) {
+ uint32_t search_fsidnum = 0;
+ if (reexpdb_fsidnum_by_path(path, &search_fsidnum,
+ exp->e_reexport == REEXP_AUTO_FSIDNUM) == 0) {
+ errno = EINVAL;
+ return -1;
+ }
+ fsidnum = search_fsidnum;
+ qword_addint(&bp, &blen, exp->e_flags | NFSEXP_FSID);
+ } else {
+ qword_addint(&bp, &blen, exp->e_flags & flag_mask);
+ }
+
qword_addint(&bp, &blen, exp->e_anonuid);
qword_addint(&bp, &blen, exp->e_anongid);
- qword_addint(&bp, &blen, exp->e_fsid);
+ qword_addint(&bp, &blen, fsidnum);
#ifdef HAVE_JUNCTION_SUPPORT
write_fsloc(&bp, &blen, exp);
#endif
- write_secinfo(&bp, &blen, exp, flag_mask);
+ write_secinfo(&bp, &blen, exp, flag_mask, do_fsidnum ? NFSEXP_FSID : 0);
if (exp->e_uuid == NULL || different_fs) {
char u[16];
if ((exp->e_flags & flag_mask & NFSEXP_FSID) == 0 &&
diff --git a/support/reexport/reexport.c b/support/reexport/reexport.c
index a9529b2b..51e49834 100644
--- a/support/reexport/reexport.c
+++ b/support/reexport/reexport.c
@@ -346,5 +346,6 @@ int reexpdb_apply_reexport_settings(struct exportent *ep, char *flname, int flli
ep->e_fsid = fsidnum;
}
+out:
return ret;
}
diff --git a/utils/exportd/Makefile.am b/utils/exportd/Makefile.am
index c95bdee7..b0ec9034 100644
--- a/utils/exportd/Makefile.am
+++ b/utils/exportd/Makefile.am
@@ -16,11 +16,17 @@ exportd_SOURCES = exportd.c
exportd_LDADD = ../../support/export/libexport.a \
../../support/nfs/libnfs.la \
../../support/misc/libmisc.a \
- $(OPTLIBS) $(LIBBLKID) $(LIBPTHREAD) -luuid
+ $(OPTLIBS) $(LIBBLKID) $(LIBPTHREAD) \
+ -luuid
+if CONFIG_REEXPORT
+exportd_LDADD += ../../support/reexport/libreexport.a $(LIBSQLITE) -lrt
+endif
exportd_CPPFLAGS = $(AM_CPPFLAGS) $(CPPFLAGS) \
-I$(top_srcdir)/support/export
+exportd_CPPFLAGS += -I$(top_srcdir)/support/reexport
+
MAINTAINERCLEANFILES = Makefile.in
#######################################################################
diff --git a/utils/exportd/exportd.c b/utils/exportd/exportd.c
index 2dd12cb6..e076a295 100644
--- a/utils/exportd/exportd.c
+++ b/utils/exportd/exportd.c
@@ -22,6 +22,7 @@
#include "conffile.h"
#include "exportfs.h"
#include "export.h"
+#include "reexport.h"
extern void my_svc_run(void);
@@ -296,6 +297,10 @@ main(int argc, char **argv)
/* Open files now to avoid sharing descriptors among forked processes */
cache_open();
v4clients_init();
+ if (reexpdb_init() != 0) {
+ xlog(L_ERROR, "%s: Failed to init reexport database", __func__);
+ exit(1);
+ }
/* Process incoming upcalls */
cache_process_loop();
diff --git a/utils/mountd/Makefile.am b/utils/mountd/Makefile.am
index 13b25c90..569d335a 100644
--- a/utils/mountd/Makefile.am
+++ b/utils/mountd/Makefile.am
@@ -20,10 +20,16 @@ mountd_LDADD = ../../support/export/libexport.a \
$(OPTLIBS) \
$(LIBBSD) $(LIBWRAP) $(LIBNSL) $(LIBBLKID) -luuid $(LIBTIRPC) \
$(LIBPTHREAD)
+if CONFIG_REEXPORT
+mountd_LDADD += ../../support/reexport/libreexport.a $(LIBSQLITE) -lrt
+endif
+
mountd_CPPFLAGS = $(AM_CPPFLAGS) $(CPPFLAGS) \
-I$(top_builddir)/support/include \
-I$(top_srcdir)/support/export
+mountd_CPPFLAGS += -I$(top_srcdir)/support/reexport
+
MAINTAINERCLEANFILES = Makefile.in
#######################################################################
diff --git a/utils/mountd/mountd.c b/utils/mountd/mountd.c
index bcf749fa..8555d746 100644
--- a/utils/mountd/mountd.c
+++ b/utils/mountd/mountd.c
@@ -32,6 +32,7 @@
#include "nfsd_path.h"
#include "nfslib.h"
#include "export.h"
+#include "reexport.h"
extern void my_svc_run(void);
diff --git a/utils/mountd/svc_run.c b/utils/mountd/svc_run.c
index 167b9757..6b5f47f1 100644
--- a/utils/mountd/svc_run.c
+++ b/utils/mountd/svc_run.c
@@ -57,6 +57,7 @@
#include <rpc/rpc_com.h>
#endif
#include "export.h"
+#include "reexport.h"
void my_svc_run(void);
@@ -96,6 +97,11 @@ my_svc_run(void)
fd_set readfds;
int selret;
+ if (reexpdb_init() != 0) {
+ xlog(L_ERROR, "%s: Failed to init reexport database", __func__);
+ return;
+ }
+
for (;;) {
readfds = svc_fdset;
--
2.31.1