First of nine tmpfs patches, equivalent to (but aesthetic differences
from) what I sent wli a few days ago: starting from 2.5.40-mm1 base.
Diffstat for the whole lot:
include/linux/shmem_fs.h | 6
mm/shmem.c | 1269 ++++++++++++++++++++++++++---------------------
2 files changed, 726 insertions(+), 549 deletions(-)
This is not the end of tmpfs mods for 2.5: I should have six or seven
more patches to come (including loopable unless loop.c does it first);
but let's publish these before they go stale. I know a swap_free goes
missing occasionally, before and after these patches: I wanted to sort
that out before sending these, but it's elusive so far: tests running.
tmpfs 1/9 shmem_vm_writeback
Give tmpfs its own shmem_vm_writeback (and empty shmem_writepages):
going through the default mpage_writepages is very wrong for tmpfs,
since that may write nearby pages while still mapped into mms, but
"writing" converts pages from tmpfs file identity to swap backing
identity: doing so while mapped breaks assumptions throughout e.g.
the shared file is liable to disintegrate into private instances.
--- 2.5.40-mm1/mm/shmem.c Tue Oct 1 15:35:33 2002
+++ tmpfs1/mm/shmem.c Tue Oct 1 19:48:54 2002
@@ -515,9 +515,8 @@
if (!PageLocked(page))
BUG();
-
- if (!(current->flags & PF_MEMALLOC))
- return fail_writepage(page);
+ if (page_mapped(page))
+ BUG();
mapping = page->mapping;
index = page->index;
@@ -551,6 +550,19 @@
return fail_writepage(page);
}
+static int shmem_writepages(struct address_space *mapping, struct writeback_control *wbc)
+{
+ return 0;
+}
+
+static int shmem_vm_writeback(struct page *page, struct writeback_control *wbc)
+{
+ clear_page_dirty(page);
+ if (shmem_writepage(page) < 0)
+ set_page_dirty(page);
+ return 0;
+}
+
/*
* shmem_getpage_locked - either get the page from swap or allocate a new one
*
@@ -1524,6 +1536,8 @@
static struct address_space_operations shmem_aops = {
.writepage = shmem_writepage,
+ .writepages = shmem_writepages,
+ .vm_writeback = shmem_vm_writeback,
.set_page_dirty = __set_page_dirty_nobuffers,
};
The earlier partial truncation fix in shmem_truncate admits it is racy,
and I've now seen that (though perhaps more likely when mpage_writepages
was writing pages it shouldn't). A cleaner fix is, not to repeat the
memclear in shmem_truncate, but to hold the partial page in memory
throughout truncation, by shmem_holdpage from shmem_notify_change.
--- tmpfs1/mm/shmem.c Tue Oct 1 19:48:54 2002
+++ tmpfs2/mm/shmem.c Tue Oct 1 19:48:59 2002
@@ -68,8 +68,6 @@
static spinlock_t shmem_ilock = SPIN_LOCK_UNLOCKED;
atomic_t shmem_nrpages = ATOMIC_INIT(0); /* Not used right now */
-static struct page *shmem_getpage_locked(struct shmem_inode_info *, struct inode *, unsigned long);
-
/*
* shmem_recalc_inode - recalculate the size of an inode
*
@@ -329,7 +327,6 @@
static void shmem_truncate (struct inode * inode)
{
unsigned long index;
- unsigned long partial;
unsigned long freed = 0;
struct shmem_inode_info * info = SHMEM_I(inode);
@@ -337,29 +334,6 @@
inode->i_ctime = inode->i_mtime = CURRENT_TIME;
spin_lock (&info->lock);
index = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
- partial = inode->i_size & ~PAGE_CACHE_MASK;
-
- if (partial) {
- swp_entry_t *entry = shmem_swp_entry(info, index-1, 0);
- struct page *page;
- /*
- * This check is racy: it's faintly possible that page
- * was assigned to swap during truncate_inode_pages,
- * and now assigned to file; but better than nothing.
- */
- if (!IS_ERR(entry) && entry->val) {
- spin_unlock(&info->lock);
- page = shmem_getpage_locked(info, inode, index-1);
- if (!IS_ERR(page)) {
- memclear_highpage_flush(page, partial,
- PAGE_CACHE_SIZE - partial);
- unlock_page(page);
- page_cache_release(page);
- }
- spin_lock(&info->lock);
- }
- }
-
while (index < info->next_index)
freed += shmem_truncate_indirect(info, index);
@@ -369,9 +343,11 @@
up(&info->sem);
}
-static int shmem_notify_change(struct dentry * dentry, struct iattr *attr)
+static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
{
+ static struct page *shmem_holdpage(struct inode *, unsigned long);
struct inode *inode = dentry->d_inode;
+ struct page *page = NULL;
long change = 0;
int error;
@@ -384,13 +360,27 @@
if (change > 0) {
if (!vm_enough_memory(change))
return -ENOMEM;
- } else
+ } else if (attr->ia_size < inode->i_size) {
vm_unacct_memory(-change);
+ /*
+ * If truncating down to a partial page, then
+ * if that page is already allocated, hold it
+ * in memory until the truncation is over, so
+ * truncate_partial_page cannnot miss it were
+ * it assigned to swap.
+ */
+ if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
+ page = shmem_holdpage(inode,
+ attr->ia_size >> PAGE_CACHE_SHIFT);
+ }
+ }
}
error = inode_change_ok(inode, attr);
if (!error)
error = inode_setattr(inode, attr);
+ if (page)
+ page_cache_release(page);
if (error)
vm_unacct_memory(change);
return error;
@@ -729,6 +719,33 @@
return error;
}
+static struct page *shmem_holdpage(struct inode *inode, unsigned long idx)
+{
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ struct page *page;
+ swp_entry_t *entry;
+ swp_entry_t swap = {0};
+
+ /*
+ * Somehow, it feels wrong for truncation down to cause any
+ * allocation: so instead of a blind shmem_getpage, check that
+ * the page has actually been instantiated before holding it.
+ */
+ spin_lock(&info->lock);
+ page = find_get_page(inode->i_mapping, idx);
+ if (!page) {
+ entry = shmem_swp_entry(info, idx, 0);
+ if (!IS_ERR(entry))
+ swap = *entry;
+ }
+ spin_unlock(&info->lock);
+ if (swap.val) {
+ if (shmem_getpage(inode, idx, &page) < 0)
+ page = NULL;
+ }
+ return page;
+}
+
struct page * shmem_nopage(struct vm_area_struct * vma, unsigned long address, int unused)
{
struct page * page;
Between inode->i_sem and info->lock comes info->sem; but it doesn't
guard thoroughly against the difficult races (truncate during read),
and serializes reads from tmpfs unlike other filesystems. I'd prefer
to work with just i_sem and info->lock, backtracking when necessary
(when another task allocates block or metablock at the same time).
(I am not satisfied with the locked setting of next_index at the
start of shmem_getpage_locked: it's one lock hold too many, and
it doesn't really fix races against truncate better than before:
another patch in a later batch will resolve that.)
--- tmpfs2/include/linux/shmem_fs.h Thu Aug 1 23:58:33 2002
+++ tmpfs3/include/linux/shmem_fs.h Tue Oct 1 19:49:04 2002
@@ -11,7 +11,6 @@
struct shmem_inode_info {
spinlock_t lock;
- struct semaphore sem;
unsigned long next_index;
swp_entry_t i_direct[SHMEM_NR_DIRECT]; /* for the first blocks */
void **i_indirect; /* indirect blocks */
--- tmpfs2/mm/shmem.c Tue Oct 1 19:48:59 2002
+++ tmpfs3/mm/shmem.c Tue Oct 1 19:49:04 2002
@@ -111,11 +111,9 @@
* @page: optional page to add to the structure. Has to be preset to
* all zeros
*
- * If there is no space allocated yet it will return -ENOMEM when
- * page == 0 else it will use the page for the needed block.
- *
- * returns -EFBIG if the index is too big.
- *
+ * If there is no space allocated yet it will return NULL when
+ * page is 0, else it will use the page for the needed block,
+ * setting it to 0 on return to indicate that it has been used.
*
* The swap vector is organized the following way:
*
@@ -143,70 +141,80 @@
* +-> 48-51
* +-> 52-55
*/
-static swp_entry_t * shmem_swp_entry (struct shmem_inode_info *info, unsigned long index, unsigned long page)
+static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, unsigned long *page)
{
unsigned long offset;
void **dir;
+ if (index >= info->next_index)
+ return NULL;
if (index < SHMEM_NR_DIRECT)
return info->i_direct+index;
+ if (!info->i_indirect) {
+ if (page) {
+ info->i_indirect = (void *) *page;
+ *page = 0;
+ }
+ return NULL; /* need another page */
+ }
index -= SHMEM_NR_DIRECT;
offset = index % ENTRIES_PER_PAGE;
index /= ENTRIES_PER_PAGE;
-
- if (!info->i_indirect) {
- info->i_indirect = (void *) page;
- return ERR_PTR(-ENOMEM);
- }
-
dir = info->i_indirect + index;
+
if (index >= ENTRIES_PER_PAGE/2) {
index -= ENTRIES_PER_PAGE/2;
dir = info->i_indirect + ENTRIES_PER_PAGE/2
+ index/ENTRIES_PER_PAGE;
index %= ENTRIES_PER_PAGE;
-
- if(!*dir) {
- *dir = (void *) page;
- /* We return since we will need another page
- in the next step */
- return ERR_PTR(-ENOMEM);
+ if (!*dir) {
+ if (page) {
+ *dir = (void *) *page;
+ *page = 0;
+ }
+ return NULL; /* need another page */
}
dir = ((void **)*dir) + index;
}
+
if (!*dir) {
- if (!page)
- return ERR_PTR(-ENOMEM);
- *dir = (void *)page;
+ if (!page || !*page)
+ return NULL; /* need a page */
+ *dir = (void *) *page;
+ *page = 0;
}
return ((swp_entry_t *)*dir) + offset;
}
/*
- * shmem_alloc_entry - get the position of the swap entry for the
- * page. If it does not exist allocate the entry
+ * shmem_swp_alloc - get the position of the swap entry for the page.
+ * If it does not exist allocate the entry.
*
* @info: info structure for the inode
* @index: index of the page to find
*/
-static inline swp_entry_t * shmem_alloc_entry (struct shmem_inode_info *info, unsigned long index)
+static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index)
{
unsigned long page = 0;
- swp_entry_t * res;
-
- if (index >= SHMEM_MAX_INDEX)
- return ERR_PTR(-EFBIG);
-
- if (info->next_index <= index)
- info->next_index = index + 1;
+ swp_entry_t *entry;
- while ((res = shmem_swp_entry(info,index,page)) == ERR_PTR(-ENOMEM)) {
+ while (!(entry = shmem_swp_entry(info, index, &page))) {
+ if (index >= info->next_index) {
+ entry = ERR_PTR(-EFAULT);
+ break;
+ }
+ spin_unlock(&info->lock);
page = get_zeroed_page(GFP_USER);
+ spin_lock(&info->lock);
if (!page)
- break;
+ return ERR_PTR(-ENOMEM);
+ }
+ if (page) {
+ /* another task gave its page, or truncated the file */
+ free_page(page);
}
- return res;
+ return entry;
}
/*
@@ -330,17 +338,15 @@
unsigned long freed = 0;
struct shmem_inode_info * info = SHMEM_I(inode);
- down(&info->sem);
inode->i_ctime = inode->i_mtime = CURRENT_TIME;
- spin_lock (&info->lock);
index = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+ spin_lock (&info->lock);
while (index < info->next_index)
freed += shmem_truncate_indirect(info, index);
info->swapped -= freed;
shmem_recalc_inode(inode);
spin_unlock (&info->lock);
- up(&info->sem);
}
static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
@@ -436,8 +442,8 @@
for (idx = SHMEM_NR_DIRECT; idx < info->next_index;
idx += ENTRIES_PER_PAGE) {
- ptr = shmem_swp_entry(info, idx, 0);
- if (IS_ERR(ptr))
+ ptr = shmem_swp_entry(info, idx, NULL);
+ if (!ptr)
continue;
offset = info->next_index - idx;
if (offset > ENTRIES_PER_PAGE)
@@ -519,10 +525,10 @@
return fail_writepage(page);
spin_lock(&info->lock);
- entry = shmem_swp_entry(info, index, 0);
- if (IS_ERR(entry)) /* this had been allocated on page allocation */
- BUG();
shmem_recalc_inode(inode);
+ entry = shmem_swp_entry(info, index, NULL);
+ if (!entry)
+ BUG();
if (entry->val)
BUG();
@@ -570,61 +576,68 @@
struct shmem_sb_info *sbinfo;
struct page *page;
swp_entry_t *entry;
- int error;
+ swp_entry_t swap;
+ int error = 0;
+
+ if (idx >= SHMEM_MAX_INDEX)
+ return ERR_PTR(-EFBIG);
+
+ /*
+ * When writing, i_sem is held against truncation and other
+ * writing, so next_index will remain as set here; but when
+ * reading, idx must always be checked against next_index
+ * after sleeping, lest truncation occurred meanwhile.
+ */
+ spin_lock(&info->lock);
+ if (info->next_index <= idx)
+ info->next_index = idx + 1;
+ spin_unlock(&info->lock);
repeat:
page = find_lock_page(mapping, idx);
if (page)
return page;
- entry = shmem_alloc_entry (info, idx);
- if (IS_ERR(entry))
+ spin_lock(&info->lock);
+ shmem_recalc_inode(inode);
+ entry = shmem_swp_alloc(info, idx);
+ if (IS_ERR(entry)) {
+ spin_unlock(&info->lock);
return (void *)entry;
-
- spin_lock (&info->lock);
-
- /* The shmem_alloc_entry() call may have blocked, and
- * shmem_writepage may have been moving a page between the page
- * cache and swap cache. We need to recheck the page cache
- * under the protection of the info->lock spinlock. */
-
- page = find_get_page(mapping, idx);
- if (page) {
- if (TestSetPageLocked(page))
- goto wait_retry;
- spin_unlock (&info->lock);
- return page;
}
-
- shmem_recalc_inode(inode);
- if (entry->val) {
+ swap = *entry;
+
+ if (swap.val) {
/* Look it up and read it in.. */
- page = lookup_swap_cache(*entry);
+ page = lookup_swap_cache(swap);
if (!page) {
- swp_entry_t swap = *entry;
- spin_unlock (&info->lock);
- swapin_readahead(*entry);
- page = read_swap_cache_async(*entry);
+ spin_unlock(&info->lock);
+ swapin_readahead(swap);
+ page = read_swap_cache_async(swap);
if (!page) {
- if (entry->val != swap.val)
- goto repeat;
- return ERR_PTR(-ENOMEM);
+ spin_lock(&info->lock);
+ entry = shmem_swp_alloc(info, idx);
+ if (IS_ERR(entry))
+ error = PTR_ERR(entry);
+ else if (entry->val == swap.val)
+ error = -ENOMEM;
+ spin_unlock(&info->lock);
+ if (error)
+ return ERR_PTR(error);
+ goto repeat;
}
wait_on_page_locked(page);
- if (!PageUptodate(page) && entry->val == swap.val) {
- page_cache_release(page);
- return ERR_PTR(-EIO);
- }
-
- /* Too bad we can't trust this page, because we
- * dropped the info->lock spinlock */
page_cache_release(page);
goto repeat;
}
/* We have to do this with page locked to prevent races */
- if (TestSetPageLocked(page))
- goto wait_retry;
+ if (TestSetPageLocked(page)) {
+ spin_unlock(&info->lock);
+ wait_on_page_locked(page);
+ page_cache_release(page);
+ goto repeat;
+ }
if (PageWriteback(page)) {
spin_unlock(&info->lock);
wait_on_page_writeback(page);
@@ -632,42 +645,55 @@
page_cache_release(page);
goto repeat;
}
- error = move_from_swap_cache(page, idx, mapping);
- if (error < 0) {
+
+ error = PageUptodate(page)?
+ move_from_swap_cache(page, idx, mapping): -EIO;
+ if (error) {
spin_unlock(&info->lock);
unlock_page(page);
page_cache_release(page);
return ERR_PTR(error);
}
- swap_free(*entry);
*entry = (swp_entry_t) {0};
info->swapped--;
spin_unlock (&info->lock);
+ swap_free(swap);
} else {
+ spin_unlock(&info->lock);
sbinfo = SHMEM_SB(inode->i_sb);
- spin_unlock (&info->lock);
- spin_lock (&sbinfo->stat_lock);
- if (sbinfo->free_blocks == 0)
- goto no_space;
+ spin_lock(&sbinfo->stat_lock);
+ if (sbinfo->free_blocks == 0) {
+ spin_unlock(&sbinfo->stat_lock);
+ return ERR_PTR(-ENOSPC);
+ }
sbinfo->free_blocks--;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
- /* Ok, get a new page. We don't have to worry about the
- * info->lock spinlock here: we cannot race against
- * shm_writepage because we have already verified that
- * there is no page present either in memory or in the
- * swap cache, so we are guaranteed to be populating a
- * new shm entry. The inode semaphore we already hold
- * is enough to make this atomic. */
page = page_cache_alloc(mapping);
- if (!page)
- goto no_mem;
- error = add_to_page_cache_lru(page, mapping, idx);
- if (error < 0) {
+ if (!page) {
+ spin_lock(&sbinfo->stat_lock);
+ sbinfo->free_blocks++;
+ spin_unlock(&sbinfo->stat_lock);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ spin_lock(&info->lock);
+ entry = shmem_swp_alloc(info, idx);
+ if (IS_ERR(entry))
+ error = PTR_ERR(entry);
+ if (error || entry->val ||
+ add_to_page_cache_lru(page, mapping, idx) < 0) {
+ spin_unlock(&info->lock);
page_cache_release(page);
- goto no_mem;
+ spin_lock(&sbinfo->stat_lock);
+ sbinfo->free_blocks++;
+ spin_unlock(&sbinfo->stat_lock);
+ if (error)
+ return ERR_PTR(error);
+ goto repeat;
}
+ spin_unlock(&info->lock);
clear_highpage(page);
inode->i_blocks += BLOCKS_PER_PAGE;
}
@@ -675,22 +701,6 @@
/* We have the page */
SetPageUptodate(page);
return page;
-
-no_mem:
- spin_lock(&sbinfo->stat_lock);
- sbinfo->free_blocks++;
- spin_unlock(&sbinfo->stat_lock);
- return ERR_PTR(-ENOMEM);
-
-no_space:
- spin_unlock (&sbinfo->stat_lock);
- return ERR_PTR(-ENOSPC);
-
-wait_retry:
- spin_unlock(&info->lock);
- wait_on_page_locked(page);
- page_cache_release(page);
- goto repeat;
}
static int shmem_getpage(struct inode * inode, unsigned long idx, struct page **ptr)
@@ -698,7 +708,6 @@
struct shmem_inode_info *info = SHMEM_I(inode);
int error;
- down (&info->sem);
*ptr = ERR_PTR(-EFAULT);
if (inode->i_size <= (loff_t) idx * PAGE_CACHE_SIZE)
goto failed;
@@ -708,10 +717,8 @@
goto failed;
unlock_page(*ptr);
- up (&info->sem);
return 0;
failed:
- up (&info->sem);
error = PTR_ERR(*ptr);
*ptr = NOPAGE_SIGBUS;
if (error == -ENOMEM)
@@ -734,8 +741,8 @@
spin_lock(&info->lock);
page = find_get_page(inode->i_mapping, idx);
if (!page) {
- entry = shmem_swp_entry(info, idx, 0);
- if (!IS_ERR(entry))
+ entry = shmem_swp_entry(info, idx, NULL);
+ if (entry)
swap = *entry;
}
spin_unlock(&info->lock);
@@ -814,12 +821,8 @@
inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
info = SHMEM_I(inode);
- spin_lock_init (&info->lock);
- sema_init (&info->sem, 1);
- info->next_index = 0;
- memset (info->i_direct, 0, sizeof(info->i_direct));
- info->i_indirect = NULL;
- info->swapped = 0;
+ memset(info, 0, (char *)inode - (char *)info);
+ spin_lock_init(&info->lock);
info->flags = VM_ACCOUNT;
switch (mode & S_IFMT) {
default:
@@ -971,9 +974,7 @@
}
info = SHMEM_I(inode);
- down (&info->sem);
page = shmem_getpage_locked(info, inode, index);
- up (&info->sem);
status = PTR_ERR(page);
if (IS_ERR(page))
@@ -1041,17 +1042,33 @@
end_index = inode->i_size >> PAGE_CACHE_SHIFT;
if (index > end_index)
break;
- nr = PAGE_CACHE_SIZE;
if (index == end_index) {
nr = inode->i_size & ~PAGE_CACHE_MASK;
if (nr <= offset)
break;
}
- nr = nr - offset;
-
- if ((desc->error = shmem_getpage(inode, index, &page)))
+ desc->error = shmem_getpage(inode, index, &page);
+ if (desc->error) {
+ if (desc->error == -EFAULT)
+ desc->error = 0;
break;
+ }
+
+ /*
+ * We must evaluate after, since reads (unlike writes)
+ * are called without i_sem protection against truncate
+ */
+ nr = PAGE_CACHE_SIZE;
+ end_index = inode->i_size >> PAGE_CACHE_SHIFT;
+ if (index == end_index) {
+ nr = inode->i_size & ~PAGE_CACHE_MASK;
+ if (nr <= offset) {
+ page_cache_release(page);
+ break;
+ }
+ }
+ nr -= offset;
if (!list_empty(&mapping->i_mmap_shared))
flush_dcache_page(page);
@@ -1279,10 +1296,8 @@
iput(inode);
return -ENOMEM;
}
- down(&info->sem);
page = shmem_getpage_locked(info, inode, 0);
if (IS_ERR(page)) {
- up(&info->sem);
vm_unacct_memory(VM_ACCT(1));
iput(inode);
return PTR_ERR(page);
@@ -1297,7 +1312,6 @@
set_page_dirty(page);
unlock_page(page);
page_cache_release(page);
- up(&info->sem);
}
dir->i_size += BOGO_DIRENT_SIZE;
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
If PAGE_CACHE_SIZE were to differ from PAGE_SIZE, the VM_ACCT macro,
and shmem_nopage's vm_pgoff manipulation, were still not quite right.
Slip a cond_resched_lock into shmem_truncate's long loop; but not into
shmem_unuse_inode's, since other locks held, and swapoff awful anyway.
Move SetPageUptodate to where it's not already set. Replace
copy_from_user by __copy_from_user since access already verified.
Replace BUG()s by BUG_ON()s. Remove an uninteresting PAGE_BUG().
--- tmpfs7/mm/shmem.c Tue Oct 1 19:49:24 2002
+++ tmpfs8/mm/shmem.c Tue Oct 1 19:49:29 2002
@@ -43,7 +43,7 @@
#define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
#define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
-#define VM_ACCT(size) (((size) + PAGE_CACHE_SIZE - 1) >> PAGE_SHIFT)
+#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
/* Pretend that each entry is of this size in directory's i_size */
#define BOGO_DIRENT_SIZE 20
@@ -428,6 +428,7 @@
info->alloced++;
}
empty = subdir;
+ cond_resched_lock(&info->lock);
dir = shmem_dir_map(subdir);
}
subdir = *dir;
@@ -657,10 +658,8 @@
unsigned long index;
struct inode *inode;
- if (!PageLocked(page))
- BUG();
- if (page_mapped(page))
- BUG();
+ BUG_ON(!PageLocked(page));
+ BUG_ON(page_mapped(page));
mapping = page->mapping;
index = page->index;
@@ -675,10 +674,8 @@
spin_lock(&info->lock);
shmem_recalc_inode(inode);
entry = shmem_swp_entry(info, index, NULL);
- if (!entry)
- BUG();
- if (entry->val)
- BUG();
+ BUG_ON(!entry);
+ BUG_ON(entry->val);
if (move_to_swap_cache(page, swap) == 0) {
shmem_swp_set(info, entry, swap.val);
@@ -852,10 +849,10 @@
info->alloced++;
spin_unlock(&info->lock);
clear_highpage(page);
+ SetPageUptodate(page);
}
/* We have the page */
- SetPageUptodate(page);
*pagep = page;
return 0;
}
@@ -896,8 +893,9 @@
unsigned long idx;
int error;
- idx = (address - vma->vm_start) >> PAGE_CACHE_SHIFT;
+ idx = (address - vma->vm_start) >> PAGE_SHIFT;
idx += vma->vm_pgoff;
+ idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
if (((loff_t) idx << PAGE_CACHE_SHIFT) >= inode->i_size)
return NOPAGE_SIGBUS;
@@ -1118,13 +1116,8 @@
if (status)
break;
- /* We have exclusive IO access to the page.. */
- if (!PageLocked(page)) {
- PAGE_BUG(page);
- }
-
kaddr = kmap(page);
- status = copy_from_user(kaddr+offset, buf, bytes);
+ status = __copy_from_user(kaddr+offset, buf, bytes);
kunmap(page);
if (status)
goto fail_write;
The distinction between shmem_getpage and shmem_getpage_locked is
not helpful, particularly now info->sem is gone; and shmem_getpage
confusingly tailored to shmem_nopage's expectations. Put the code
of shmem_getpage_locked into the frame of shmem_getpage, leaving
its callers to unlock_page afterwards.
--- tmpfs3/mm/shmem.c Tue Oct 1 19:49:04 2002
+++ tmpfs4/mm/shmem.c Tue Oct 1 19:49:09 2002
@@ -495,10 +495,6 @@
/*
* Move the page from the page cache to the swap cache.
- *
- * The page lock prevents multiple occurences of shmem_writepage at
- * once. We still need to guard against racing with
- * shmem_getpage_locked().
*/
static int shmem_writepage(struct page * page)
{
@@ -560,19 +556,16 @@
}
/*
- * shmem_getpage_locked - either get the page from swap or allocate a new one
+ * shmem_getpage - either get the page from swap or allocate a new one
*
* If we allocate a new one we do not mark it dirty. That's up to the
* vm. If we swap it in we mark it dirty since we also free the swap
* entry since a page cannot live in both the swap and page cache
- *
- * Called with the inode locked, so it cannot race with itself, but we
- * still need to guard against racing with shm_writepage(), which might
- * be trying to move the page to the swap cache as we run.
*/
-static struct page * shmem_getpage_locked(struct shmem_inode_info *info, struct inode * inode, unsigned long idx)
+static int shmem_getpage(struct inode *inode, unsigned long idx, struct page **pagep)
{
struct address_space *mapping = inode->i_mapping;
+ struct shmem_inode_info *info = SHMEM_I(inode);
struct shmem_sb_info *sbinfo;
struct page *page;
swp_entry_t *entry;
@@ -580,7 +573,7 @@
int error = 0;
if (idx >= SHMEM_MAX_INDEX)
- return ERR_PTR(-EFBIG);
+ return -EFBIG;
/*
* When writing, i_sem is held against truncation and other
@@ -595,15 +588,17 @@
repeat:
page = find_lock_page(mapping, idx);
- if (page)
- return page;
+ if (page) {
+ *pagep = page;
+ return 0;
+ }
spin_lock(&info->lock);
shmem_recalc_inode(inode);
entry = shmem_swp_alloc(info, idx);
if (IS_ERR(entry)) {
spin_unlock(&info->lock);
- return (void *)entry;
+ return PTR_ERR(entry);
}
swap = *entry;
@@ -623,7 +618,7 @@
error = -ENOMEM;
spin_unlock(&info->lock);
if (error)
- return ERR_PTR(error);
+ return error;
goto repeat;
}
wait_on_page_locked(page);
@@ -652,7 +647,7 @@
spin_unlock(&info->lock);
unlock_page(page);
page_cache_release(page);
- return ERR_PTR(error);
+ return error;
}
*entry = (swp_entry_t) {0};
@@ -665,7 +660,7 @@
spin_lock(&sbinfo->stat_lock);
if (sbinfo->free_blocks == 0) {
spin_unlock(&sbinfo->stat_lock);
- return ERR_PTR(-ENOSPC);
+ return -ENOSPC;
}
sbinfo->free_blocks--;
spin_unlock(&sbinfo->stat_lock);
@@ -675,7 +670,7 @@
spin_lock(&sbinfo->stat_lock);
sbinfo->free_blocks++;
spin_unlock(&sbinfo->stat_lock);
- return ERR_PTR(-ENOMEM);
+ return -ENOMEM;
}
spin_lock(&info->lock);
@@ -690,7 +685,7 @@
sbinfo->free_blocks++;
spin_unlock(&sbinfo->stat_lock);
if (error)
- return ERR_PTR(error);
+ return error;
goto repeat;
}
spin_unlock(&info->lock);
@@ -700,30 +695,8 @@
/* We have the page */
SetPageUptodate(page);
- return page;
-}
-
-static int shmem_getpage(struct inode * inode, unsigned long idx, struct page **ptr)
-{
- struct shmem_inode_info *info = SHMEM_I(inode);
- int error;
-
- *ptr = ERR_PTR(-EFAULT);
- if (inode->i_size <= (loff_t) idx * PAGE_CACHE_SIZE)
- goto failed;
-
- *ptr = shmem_getpage_locked(info, inode, idx);
- if (IS_ERR (*ptr))
- goto failed;
-
- unlock_page(*ptr);
+ *pagep = page;
return 0;
-failed:
- error = PTR_ERR(*ptr);
- *ptr = NOPAGE_SIGBUS;
- if (error == -ENOMEM)
- *ptr = NOPAGE_OOM;
- return error;
}
static struct page *shmem_holdpage(struct inode *inode, unsigned long idx)
@@ -747,26 +720,32 @@
}
spin_unlock(&info->lock);
if (swap.val) {
- if (shmem_getpage(inode, idx, &page) < 0)
- page = NULL;
+ if (shmem_getpage(inode, idx, &page) == 0)
+ unlock_page(page);
}
return page;
}
-struct page * shmem_nopage(struct vm_area_struct * vma, unsigned long address, int unused)
+struct page *shmem_nopage(struct vm_area_struct *vma, unsigned long address, int unused)
{
- struct page * page;
- unsigned int idx;
- struct inode * inode = vma->vm_file->f_dentry->d_inode;
+ struct inode *inode = vma->vm_file->f_dentry->d_inode;
+ struct page *page;
+ unsigned long idx;
+ int error;
idx = (address - vma->vm_start) >> PAGE_CACHE_SHIFT;
idx += vma->vm_pgoff;
- if (shmem_getpage(inode, idx, &page))
- return page;
+ if (((loff_t) idx << PAGE_CACHE_SHIFT) >= inode->i_size)
+ return NOPAGE_SIGBUS;
+ error = shmem_getpage(inode, idx, &page);
+ if (error)
+ return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS;
+
+ unlock_page(page);
flush_page_to_ram(page);
- return(page);
+ return page;
}
void shmem_lock(struct file * file, int lock)
@@ -882,7 +861,6 @@
shmem_file_write(struct file *file,const char *buf,size_t count,loff_t *ppos)
{
struct inode *inode = file->f_dentry->d_inode;
- struct shmem_inode_info *info;
unsigned long limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
loff_t pos;
struct page *page;
@@ -973,11 +951,8 @@
__get_user(dummy, buf+bytes-1);
}
- info = SHMEM_I(inode);
- page = shmem_getpage_locked(info, inode, index);
-
- status = PTR_ERR(page);
- if (IS_ERR(page))
+ status = shmem_getpage(inode, index, &page);
+ if (status)
break;
/* We have exclusive IO access to the page.. */
@@ -1064,10 +1039,12 @@
if (index == end_index) {
nr = inode->i_size & ~PAGE_CACHE_MASK;
if (nr <= offset) {
+ unlock_page(page);
page_cache_release(page);
break;
}
}
+ unlock_page(page);
nr -= offset;
if (!list_empty(&mapping->i_mmap_shared))
@@ -1271,6 +1248,7 @@
static int shmem_symlink(struct inode * dir, struct dentry *dentry, const char * symname)
{
+ int error;
int len;
struct inode *inode;
struct page *page;
@@ -1296,11 +1274,11 @@
iput(inode);
return -ENOMEM;
}
- page = shmem_getpage_locked(info, inode, 0);
- if (IS_ERR(page)) {
+ error = shmem_getpage(inode, 0, &page);
+ if (error) {
vm_unacct_memory(VM_ACCT(1));
iput(inode);
- return PTR_ERR(page);
+ return error;
}
inode->i_op = &shmem_symlink_inode_operations;
spin_lock (&shmem_ilock);
@@ -1332,27 +1310,26 @@
static int shmem_readlink(struct dentry *dentry, char *buffer, int buflen)
{
- struct page * page;
+ struct page *page;
int res = shmem_getpage(dentry->d_inode, 0, &page);
-
if (res)
return res;
-
- res = vfs_readlink(dentry,buffer,buflen, kmap(page));
+ res = vfs_readlink(dentry, buffer, buflen, kmap(page));
kunmap(page);
+ unlock_page(page);
page_cache_release(page);
return res;
}
static int shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
{
- struct page * page;
+ struct page *page;
int res = shmem_getpage(dentry->d_inode, 0, &page);
if (res)
return res;
-
res = vfs_follow_link(nd, kmap(page));
kunmap(page);
+ unlock_page(page);
page_cache_release(page);
return res;
}
wli suffered OOMs because tmpfs was allocating GFP_USER, for its
metadata pages. This patch allocates them GFP_HIGHUSER (default
mapping->gfp_mask) and uses atomic kmaps to access (KM_USER0 for
upper levels, KM_USER1 for lowest level). shmem_unuse_inode and
shmem_truncate rewritten alike to avoid repeated maps and unmaps
of the same page: cr's truncate was much more elegant,
but I couldn't quite see how to convert it.
I do wonder whether this patch is a bloat too far for tmpfs, and
even non-highmem configs will be penalised by page_address overhead
(perhaps a further patch could get over that). There is an attractive
alternative (keep swp_entry_ts in the existing radix-tree, no metadata
pages at all), but we haven't worked out an unhacky interface to that.
For now at least, let's give tmpfs highmem metadata a spin.
--- tmpfs5/include/linux/shmem_fs.h Tue Oct 1 19:49:14 2002
+++ tmpfs6/include/linux/shmem_fs.h Tue Oct 1 19:49:19 2002
@@ -13,7 +13,7 @@
spinlock_t lock;
unsigned long next_index;
swp_entry_t i_direct[SHMEM_NR_DIRECT]; /* for the first blocks */
- void **i_indirect; /* indirect blocks */
+ struct page *i_indirect; /* indirect blocks */
unsigned long alloced; /* data pages allocated to file */
unsigned long swapped; /* subtotal assigned to swap */
unsigned long flags;
--- tmpfs5/mm/shmem.c Tue Oct 1 19:49:14 2002
+++ tmpfs6/mm/shmem.c Tue Oct 1 19:49:19 2002
@@ -37,9 +37,10 @@
#define TMPFS_MAGIC 0x01021994
#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
+#define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
-#define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + ENTRIES_PER_PAGE * (ENTRIES_PER_PAGE/2) * (ENTRIES_PER_PAGE+1))
+#define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
#define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
#define VM_ACCT(size) (((size) + PAGE_CACHE_SIZE - 1) >> PAGE_SHIFT)
@@ -47,6 +48,51 @@
/* Pretend that each entry is of this size in directory's i_size */
#define BOGO_DIRENT_SIZE 20
+static inline struct page *shmem_dir_alloc(unsigned int gfp_mask)
+{
+ /*
+ * The above definition of ENTRIES_PER_PAGE, and the use of
+ * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
+ * might be reconsidered if it ever diverges from PAGE_SIZE.
+ */
+ return alloc_pages(gfp_mask, PAGE_CACHE_SHIFT-PAGE_SHIFT);
+}
+
+static inline void shmem_dir_free(struct page *page)
+{
+ __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
+}
+
+static struct page **shmem_dir_map(struct page *page)
+{
+ return (struct page **)kmap_atomic(page, KM_USER0);
+}
+
+static inline void shmem_dir_unmap(struct page **dir)
+{
+ kunmap_atomic(dir, KM_USER0);
+}
+
+static swp_entry_t *shmem_swp_map(struct page *page)
+{
+ /*
+ * We have to avoid the unconditional inc_preempt_count()
+ * in kmap_atomic(), since shmem_swp_unmap() will also be
+ * applied to the low memory addresses within i_direct[].
+ * PageHighMem and high_memory tests are good for all arches
+ * and configs: highmem_start_page and FIXADDR_START are not.
+ */
+ return PageHighMem(page)?
+ (swp_entry_t *)kmap_atomic(page, KM_USER1):
+ (swp_entry_t *)page_address(page);
+}
+
+static inline void shmem_swp_unmap(swp_entry_t *entry)
+{
+ if (entry >= (swp_entry_t *)high_memory)
+ kunmap_atomic(entry, KM_USER1);
+}
+
static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
{
return sb->u.generic_sbp;
@@ -116,8 +162,8 @@
* all zeros
*
* If there is no space allocated yet it will return NULL when
- * page is 0, else it will use the page for the needed block,
- * setting it to 0 on return to indicate that it has been used.
+ * page is NULL, else it will use the page for the needed block,
+ * setting it to NULL on return to indicate that it has been used.
*
* The swap vector is organized the following way:
*
@@ -145,10 +191,11 @@
* +-> 48-51
* +-> 52-55
*/
-static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, unsigned long *page)
+static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
{
unsigned long offset;
- void **dir;
+ struct page **dir;
+ struct page *subdir;
if (index >= info->next_index)
return NULL;
@@ -156,8 +203,8 @@
return info->i_direct+index;
if (!info->i_indirect) {
if (page) {
- info->i_indirect = (void *) *page;
- *page = 0;
+ info->i_indirect = *page;
+ *page = NULL;
}
return NULL; /* need another page */
}
@@ -165,30 +212,37 @@
index -= SHMEM_NR_DIRECT;
offset = index % ENTRIES_PER_PAGE;
index /= ENTRIES_PER_PAGE;
- dir = info->i_indirect + index;
+ dir = shmem_dir_map(info->i_indirect);
if (index >= ENTRIES_PER_PAGE/2) {
index -= ENTRIES_PER_PAGE/2;
- dir = info->i_indirect + ENTRIES_PER_PAGE/2
- + index/ENTRIES_PER_PAGE;
+ dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
index %= ENTRIES_PER_PAGE;
- if (!*dir) {
+ subdir = *dir;
+ if (!subdir) {
if (page) {
- *dir = (void *) *page;
- *page = 0;
+ *dir = *page;
+ *page = NULL;
}
+ shmem_dir_unmap(dir);
return NULL; /* need another page */
}
- dir = ((void **)*dir) + index;
+ shmem_dir_unmap(dir);
+ dir = shmem_dir_map(subdir);
}
- if (!*dir) {
- if (!page || !*page)
+ dir += index;
+ subdir = *dir;
+ if (!subdir) {
+ if (!page || !(subdir = *page)) {
+ shmem_dir_unmap(dir);
return NULL; /* need a page */
- *dir = (void *) *page;
- *page = 0;
+ }
+ *dir = subdir;
+ *page = NULL;
}
- return ((swp_entry_t *)*dir) + offset;
+ shmem_dir_unmap(dir);
+ return shmem_swp_map(subdir) + offset;
}
/*
@@ -202,7 +256,7 @@
{
struct inode *inode = &info->vfs_inode;
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
- unsigned long page = 0;
+ struct page *page = NULL;
swp_entry_t *entry;
while (!(entry = shmem_swp_entry(info, index, &page))) {
@@ -226,7 +280,9 @@
spin_unlock(&sbinfo->stat_lock);
spin_unlock(&info->lock);
- page = get_zeroed_page(GFP_USER);
+ page = shmem_dir_alloc(inode->i_mapping->gfp_mask);
+ if (page)
+ clear_highpage(page);
spin_lock(&info->lock);
if (!page) {
@@ -237,7 +293,7 @@
if (page) {
/* another task gave its page, or truncated the file */
shmem_free_block(inode);
- free_page(page);
+ shmem_dir_free(page);
}
return entry;
}
@@ -246,133 +302,138 @@
* shmem_free_swp - free some swap entries in a directory
*
* @dir: pointer to the directory
- * @count: number of entries to scan
+ * @edir: pointer after last entry of the directory
*/
-static int shmem_free_swp(swp_entry_t *dir, unsigned int count)
+static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir)
{
- swp_entry_t *ptr, entry;
+ swp_entry_t *ptr;
int freed = 0;
- for (ptr = dir; ptr < dir + count; ptr++) {
- if (!ptr->val)
- continue;
- entry = *ptr;
- *ptr = (swp_entry_t){0};
- freed++;
- free_swap_and_cache(entry);
+ for (ptr = dir; ptr < edir; ptr++) {
+ if (ptr->val) {
+ free_swap_and_cache(*ptr);
+ *ptr = (swp_entry_t){0};
+ freed++;
+ }
}
return freed;
}
-/*
- * shmem_truncate_direct - free the swap entries of a whole doubly
- * indirect block
- *
- * @info: the info structure of the inode
- * @dir: pointer to the pointer to the block
- * @start: offset to start from (in pages)
- * @len: how many pages are stored in this block
- *
- * Returns the number of freed swap entries.
- */
-static inline unsigned long
-shmem_truncate_direct(struct shmem_inode_info *info, swp_entry_t ***dir, unsigned long start, unsigned long len)
+static void shmem_truncate(struct inode *inode)
{
- swp_entry_t **last, **ptr;
- unsigned long off, freed_swp, freed = 0;
-
- last = *dir + (len + ENTRIES_PER_PAGE-1) / ENTRIES_PER_PAGE;
- off = start % ENTRIES_PER_PAGE;
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ unsigned long idx;
+ unsigned long size;
+ unsigned long limit;
+ unsigned long stage;
+ struct page **dir;
+ struct page *subdir;
+ struct page *empty;
+ swp_entry_t *ptr;
+ int offset;
- for (ptr = *dir + start/ENTRIES_PER_PAGE; ptr < last; ptr++, off = 0) {
- if (!*ptr)
- continue;
+ inode->i_ctime = inode->i_mtime = CURRENT_TIME;
+ idx = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+ if (idx >= info->next_index)
+ return;
- if (info->swapped) {
- freed_swp = shmem_free_swp(*ptr + off,
- ENTRIES_PER_PAGE - off);
- info->swapped -= freed_swp;
- freed += freed_swp;
+ spin_lock(&info->lock);
+ limit = info->next_index;
+ info->next_index = idx;
+ if (info->swapped && idx < SHMEM_NR_DIRECT) {
+ ptr = info->i_direct;
+ size = limit;
+ if (size > SHMEM_NR_DIRECT)
+ size = SHMEM_NR_DIRECT;
+ info->swapped -= shmem_free_swp(ptr+idx, ptr+size);
+ }
+ if (!info->i_indirect)
+ goto done2;
+
+ BUG_ON(limit <= SHMEM_NR_DIRECT);
+ limit -= SHMEM_NR_DIRECT;
+ idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
+ offset = idx % ENTRIES_PER_PAGE;
+ idx -= offset;
+
+ empty = NULL;
+ dir = shmem_dir_map(info->i_indirect);
+ stage = ENTRIES_PER_PAGEPAGE/2;
+ if (idx < ENTRIES_PER_PAGEPAGE/2)
+ dir += idx/ENTRIES_PER_PAGE;
+ else {
+ dir += ENTRIES_PER_PAGE/2;
+ dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
+ while (stage <= idx)
+ stage += ENTRIES_PER_PAGEPAGE;
+ if (*dir) {
+ subdir = *dir;
+ size = ((idx - ENTRIES_PER_PAGEPAGE/2) %
+ ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
+ if (!size && !offset) {
+ empty = subdir;
+ *dir = NULL;
+ }
+ shmem_dir_unmap(dir);
+ dir = shmem_dir_map(subdir) + size;
+ } else {
+ offset = 0;
+ idx = stage;
}
+ }
- if (!off) {
+ for (; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
+ if (unlikely(idx == stage)) {
+ shmem_dir_unmap(dir-1);
+ dir = shmem_dir_map(info->i_indirect) +
+ ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
+ while (!*dir) {
+ dir++;
+ idx += ENTRIES_PER_PAGEPAGE;
+ if (idx >= limit)
+ goto done1;
+ }
+ stage = idx + ENTRIES_PER_PAGEPAGE;
+ subdir = *dir;
+ *dir = NULL;
+ shmem_dir_unmap(dir);
+ if (empty) {
+ shmem_dir_free(empty);
+ info->alloced++;
+ }
+ empty = subdir;
+ dir = shmem_dir_map(subdir);
+ }
+ subdir = *dir;
+ if (subdir) {
+ ptr = shmem_swp_map(subdir);
+ size = limit - idx;
+ if (size > ENTRIES_PER_PAGE)
+ size = ENTRIES_PER_PAGE;
+ info->swapped -= shmem_free_swp(ptr+offset, ptr+size);
+ shmem_swp_unmap(ptr);
+ }
+ if (offset)
+ offset = 0;
+ else if (subdir) {
+ *dir = NULL;
+ shmem_dir_free(subdir);
info->alloced++;
- free_page((unsigned long) *ptr);
- *ptr = 0;
}
}
-
- if (!start) {
+done1:
+ shmem_dir_unmap(dir-1);
+ if (empty) {
+ shmem_dir_free(empty);
info->alloced++;
- free_page((unsigned long) *dir);
- *dir = 0;
- }
- return freed;
-}
-
-/*
- * shmem_truncate_indirect - truncate an inode
- *
- * @info: the info structure of the inode
- * @index: the index to truncate
- *
- * This function locates the last doubly indirect block and calls
- * then shmem_truncate_direct to do the real work
- */
-static inline unsigned long
-shmem_truncate_indirect(struct shmem_inode_info *info, unsigned long index)
-{
- swp_entry_t ***base;
- unsigned long baseidx, len, start;
- unsigned long max = info->next_index-1;
- unsigned long freed;
-
- if (max < SHMEM_NR_DIRECT) {
- info->next_index = index;
- if (!info->swapped)
- return 0;
- freed = shmem_free_swp(info->i_direct + index,
- SHMEM_NR_DIRECT - index);
- info->swapped -= freed;
- return freed;
- }
-
- if (max < ENTRIES_PER_PAGE * ENTRIES_PER_PAGE/2 + SHMEM_NR_DIRECT) {
- max -= SHMEM_NR_DIRECT;
- base = (swp_entry_t ***) &info->i_indirect;
- baseidx = SHMEM_NR_DIRECT;
- len = max+1;
- } else {
- max -= ENTRIES_PER_PAGE*ENTRIES_PER_PAGE/2+SHMEM_NR_DIRECT;
- if (max >= ENTRIES_PER_PAGE*ENTRIES_PER_PAGE*ENTRIES_PER_PAGE/2)
- BUG();
-
- baseidx = max & ~(ENTRIES_PER_PAGE*ENTRIES_PER_PAGE-1);
- base = (swp_entry_t ***) info->i_indirect + ENTRIES_PER_PAGE/2 + baseidx/ENTRIES_PER_PAGE/ENTRIES_PER_PAGE ;
- len = max - baseidx + 1;
- baseidx += ENTRIES_PER_PAGE*ENTRIES_PER_PAGE/2+SHMEM_NR_DIRECT;
}
-
- if (index > baseidx) {
- info->next_index = index;
- start = index - baseidx;
- } else {
- info->next_index = baseidx;
- start = 0;
+ if (info->next_index <= SHMEM_NR_DIRECT) {
+ shmem_dir_free(info->i_indirect);
+ info->i_indirect = NULL;
+ info->alloced++;
}
- return *base? shmem_truncate_direct(info, base, start, len): 0;
-}
-
-static void shmem_truncate(struct inode *inode)
-{
- struct shmem_inode_info *info = SHMEM_I(inode);
- unsigned long index;
-
- inode->i_ctime = inode->i_mtime = CURRENT_TIME;
- index = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
- spin_lock(&info->lock);
- while (index < info->next_index)
- (void) shmem_truncate_indirect(info, index);
+done2:
+ BUG_ON(info->swapped > info->next_index);
shmem_recalc_inode(inode);
spin_unlock(&info->lock);
}
@@ -442,46 +503,81 @@
clear_inode(inode);
}
-static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *ptr, swp_entry_t *eptr)
+static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
{
- swp_entry_t *test;
+ swp_entry_t *ptr;
- for (test = ptr; test < eptr; test++) {
- if (test->val == entry.val)
- return test - ptr;
+ for (ptr = dir; ptr < edir; ptr++) {
+ if (ptr->val == entry.val)
+ return ptr - dir;
}
return -1;
}
static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
{
- swp_entry_t *ptr;
unsigned long idx;
+ unsigned long size;
+ unsigned long limit;
+ unsigned long stage;
+ struct page **dir;
+ struct page *subdir;
+ swp_entry_t *ptr;
int offset;
idx = 0;
ptr = info->i_direct;
- spin_lock (&info->lock);
- offset = info->next_index;
- if (offset > SHMEM_NR_DIRECT)
- offset = SHMEM_NR_DIRECT;
- offset = shmem_find_swp(entry, ptr, ptr + offset);
+ spin_lock(&info->lock);
+ limit = info->next_index;
+ size = limit;
+ if (size > SHMEM_NR_DIRECT)
+ size = SHMEM_NR_DIRECT;
+ offset = shmem_find_swp(entry, ptr, ptr+size);
if (offset >= 0)
goto found;
-
- for (idx = SHMEM_NR_DIRECT; idx < info->next_index;
- idx += ENTRIES_PER_PAGE) {
- ptr = shmem_swp_entry(info, idx, NULL);
- if (!ptr)
- continue;
- offset = info->next_index - idx;
- if (offset > ENTRIES_PER_PAGE)
- offset = ENTRIES_PER_PAGE;
- offset = shmem_find_swp(entry, ptr, ptr + offset);
- if (offset >= 0)
- goto found;
+ if (!info->i_indirect)
+ goto lost2;
+ /* we might be racing with shmem_truncate */
+ if (limit <= SHMEM_NR_DIRECT)
+ goto lost2;
+
+ dir = shmem_dir_map(info->i_indirect);
+ stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
+
+ for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
+ if (unlikely(idx == stage)) {
+ shmem_dir_unmap(dir-1);
+ dir = shmem_dir_map(info->i_indirect) +
+ ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
+ while (!*dir) {
+ dir++;
+ idx += ENTRIES_PER_PAGEPAGE;
+ if (idx >= limit)
+ goto lost1;
+ }
+ stage = idx + ENTRIES_PER_PAGEPAGE;
+ subdir = *dir;
+ shmem_dir_unmap(dir);
+ dir = shmem_dir_map(subdir);
+ }
+ subdir = *dir;
+ if (subdir) {
+ ptr = shmem_swp_map(subdir);
+ size = limit - idx;
+ if (size > ENTRIES_PER_PAGE)
+ size = ENTRIES_PER_PAGE;
+ offset = shmem_find_swp(entry, ptr, ptr+size);
+ if (offset >= 0) {
+ shmem_dir_unmap(dir);
+ goto found;
+ }
+ shmem_swp_unmap(ptr);
+ }
}
- spin_unlock (&info->lock);
+lost1:
+ shmem_dir_unmap(dir-1);
+lost2:
+ spin_unlock(&info->lock);
return 0;
found:
if (move_from_swap_cache(page, idx + offset,
@@ -489,6 +585,7 @@
ptr[offset] = (swp_entry_t) {0};
info->swapped--;
}
+ shmem_swp_unmap(ptr);
spin_unlock(&info->lock);
/*
* Decrement swap count even when the entry is left behind:
@@ -561,11 +658,13 @@
if (!err) {
*entry = swap;
info->swapped++;
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
unlock_page(page);
return 0;
}
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
swap_free(swap);
return fail_writepage(page);
@@ -635,6 +734,7 @@
/* Look it up and read it in.. */
page = lookup_swap_cache(swap);
if (!page) {
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
swapin_readahead(swap);
page = read_swap_cache_async(swap);
@@ -643,8 +743,11 @@
entry = shmem_swp_alloc(info, idx);
if (IS_ERR(entry))
error = PTR_ERR(entry);
- else if (entry->val == swap.val)
- error = -ENOMEM;
+ else {
+ if (entry->val == swap.val)
+ error = -ENOMEM;
+ shmem_swp_unmap(entry);
+ }
spin_unlock(&info->lock);
if (error)
return error;
@@ -657,12 +760,14 @@
/* We have to do this with page locked to prevent races */
if (TestSetPageLocked(page)) {
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
wait_on_page_locked(page);
page_cache_release(page);
goto repeat;
}
if (PageWriteback(page)) {
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
wait_on_page_writeback(page);
unlock_page(page);
@@ -673,6 +778,7 @@
error = PageUptodate(page)?
move_from_swap_cache(page, idx, mapping): -EIO;
if (error) {
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
unlock_page(page);
page_cache_release(page);
@@ -681,9 +787,11 @@
*entry = (swp_entry_t) {0};
info->swapped--;
- spin_unlock (&info->lock);
+ shmem_swp_unmap(entry);
+ spin_unlock(&info->lock);
swap_free(swap);
} else {
+ shmem_swp_unmap(entry);
spin_unlock(&info->lock);
sbinfo = SHMEM_SB(inode->i_sb);
spin_lock(&sbinfo->stat_lock);
@@ -705,7 +813,11 @@
entry = shmem_swp_alloc(info, idx);
if (IS_ERR(entry))
error = PTR_ERR(entry);
- if (error || entry->val ||
+ else {
+ swap = *entry;
+ shmem_swp_unmap(entry);
+ }
+ if (error || swap.val ||
add_to_page_cache_lru(page, mapping, idx) < 0) {
spin_unlock(&info->lock);
page_cache_release(page);
@@ -741,8 +853,10 @@
page = find_get_page(inode->i_mapping, idx);
if (!page) {
entry = shmem_swp_entry(info, idx, NULL);
- if (entry)
+ if (entry) {
swap = *entry;
+ shmem_swp_unmap(entry);
+ }
}
spin_unlock(&info->lock);
if (swap.val) {
If we're going to rely on struct page *s rather than virtual addresses
for the metadata pages, let's count nr_swapped in the private field:
these pages are only for storing swp_entry_ts, and need not be examined
at all when nr_swapped is zero.
--- tmpfs6/mm/shmem.c Tue Oct 1 19:49:19 2002
+++ tmpfs7/mm/shmem.c Tue Oct 1 19:49:24 2002
@@ -48,6 +48,9 @@
/* Pretend that each entry is of this size in directory's i_size */
#define BOGO_DIRENT_SIZE 20
+/* Keep swapped page count in private field of indirect struct page */
+#define nr_swapped private
+
static inline struct page *shmem_dir_alloc(unsigned int gfp_mask)
{
/*
@@ -242,7 +245,27 @@
*page = NULL;
}
shmem_dir_unmap(dir);
- return shmem_swp_map(subdir) + offset;
+
+ /*
+ * With apologies... caller shmem_swp_alloc passes non-NULL
+ * page (though perhaps NULL *page); and now we know that this
+ * indirect page has been allocated, we can shortcut the final
+ * kmap if we know it contains no swap entries, as is commonly
+ * the case: return pointer to a 0 which doesn't need kmapping.
+ */
+ return (page && !subdir->nr_swapped)?
+ (swp_entry_t *)&subdir->nr_swapped:
+ shmem_swp_map(subdir) + offset;
+}
+
+static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
+{
+ long incdec = value? 1: -1;
+
+ entry->val = value;
+ info->swapped += incdec;
+ if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT)
+ kmap_atomic_to_page(entry)->nr_swapped += incdec;
}
/*
@@ -281,8 +304,10 @@
spin_unlock(&info->lock);
page = shmem_dir_alloc(inode->i_mapping->gfp_mask);
- if (page)
+ if (page) {
clear_highpage(page);
+ page->nr_swapped = 0;
+ }
spin_lock(&info->lock);
if (!page) {
@@ -331,6 +356,7 @@
struct page *empty;
swp_entry_t *ptr;
int offset;
+ int freed;
inode->i_ctime = inode->i_mtime = CURRENT_TIME;
idx = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
@@ -405,13 +431,16 @@
dir = shmem_dir_map(subdir);
}
subdir = *dir;
- if (subdir) {
+ if (subdir && subdir->nr_swapped) {
ptr = shmem_swp_map(subdir);
size = limit - idx;
if (size > ENTRIES_PER_PAGE)
size = ENTRIES_PER_PAGE;
- info->swapped -= shmem_free_swp(ptr+offset, ptr+size);
+ freed = shmem_free_swp(ptr+offset, ptr+size);
shmem_swp_unmap(ptr);
+ info->swapped -= freed;
+ subdir->nr_swapped -= freed;
+ BUG_ON(subdir->nr_swapped > offset);
}
if (offset)
offset = 0;
@@ -561,7 +590,7 @@
dir = shmem_dir_map(subdir);
}
subdir = *dir;
- if (subdir) {
+ if (subdir && subdir->nr_swapped) {
ptr = shmem_swp_map(subdir);
size = limit - idx;
if (size > ENTRIES_PER_PAGE)
@@ -581,10 +610,8 @@
return 0;
found:
if (move_from_swap_cache(page, idx + offset,
- info->vfs_inode.i_mapping) == 0) {
- ptr[offset] = (swp_entry_t) {0};
- info->swapped--;
- }
+ info->vfs_inode.i_mapping) == 0)
+ shmem_swp_set(info, ptr + offset, 0);
shmem_swp_unmap(ptr);
spin_unlock(&info->lock);
/*
@@ -624,7 +651,6 @@
*/
static int shmem_writepage(struct page * page)
{
- int err;
struct shmem_inode_info *info;
swp_entry_t *entry, swap;
struct address_space *mapping;
@@ -654,10 +680,8 @@
if (entry->val)
BUG();
- err = move_to_swap_cache(page, swap);
- if (!err) {
- *entry = swap;
- info->swapped++;
+ if (move_to_swap_cache(page, swap) == 0) {
+ shmem_swp_set(info, entry, swap.val);
shmem_swp_unmap(entry);
spin_unlock(&info->lock);
unlock_page(page);
@@ -785,8 +809,7 @@
return error;
}
- *entry = (swp_entry_t) {0};
- info->swapped--;
+ shmem_swp_set(info, entry, 0);
shmem_swp_unmap(entry);
spin_unlock(&info->lock);
swap_free(swap);
akpm and wli each discovered unfortunate behaviour of dbench on tmpfs:
after tmpfs has reached its data memory limit, dbench continues to lseek
and write, and tmpfs carries on allocating unlimited metadata blocks to
accommodate the data it then refuses. That particular behaviour could
be simply fixed by checking earlier; but I think tmpfs metablocks should
be subject to the memory limit, and included in df and du accounting.
Also, manipulate inode->i_blocks under lock, was missed before.
--- tmpfs4/include/linux/shmem_fs.h Tue Oct 1 19:49:04 2002
+++ tmpfs5/include/linux/shmem_fs.h Tue Oct 1 19:49:14 2002
@@ -14,7 +14,8 @@
unsigned long next_index;
swp_entry_t i_direct[SHMEM_NR_DIRECT]; /* for the first blocks */
void **i_indirect; /* indirect blocks */
- unsigned long swapped;
+ unsigned long alloced; /* data pages allocated to file */
+ unsigned long swapped; /* subtotal assigned to swap */
unsigned long flags;
struct list_head list;
struct inode vfs_inode;
--- tmpfs4/mm/shmem.c Tue Oct 1 19:49:09 2002
+++ tmpfs5/mm/shmem.c Tue Oct 1 19:49:14 2002
@@ -68,38 +68,42 @@
static spinlock_t shmem_ilock = SPIN_LOCK_UNLOCKED;
atomic_t shmem_nrpages = ATOMIC_INIT(0); /* Not used right now */
+static void shmem_free_block(struct inode *inode)
+{
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ spin_lock(&sbinfo->stat_lock);
+ sbinfo->free_blocks++;
+ inode->i_blocks -= BLOCKS_PER_PAGE;
+ spin_unlock(&sbinfo->stat_lock);
+}
+
/*
* shmem_recalc_inode - recalculate the size of an inode
*
* @inode: inode to recalc
- * @swap: additional swap pages freed externally
*
- * We have to calculate the free blocks since the mm can drop pages
- * behind our back
+ * We have to calculate the free blocks since the mm can drop
+ * undirtied hole pages behind our back. Later we should be
+ * able to use the releasepage method to handle this better.
*
- * But we know that normally
- * inodes->i_blocks/BLOCKS_PER_PAGE ==
- * inode->i_mapping->nrpages + info->swapped
- *
- * So the mm freed
- * inodes->i_blocks/BLOCKS_PER_PAGE -
- * (inode->i_mapping->nrpages + info->swapped)
+ * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
+ * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
*
* It has to be called with the spinlock held.
*/
-
static void shmem_recalc_inode(struct inode * inode)
{
- unsigned long freed;
+ struct shmem_inode_info *info = SHMEM_I(inode);
+ long freed;
- freed = (inode->i_blocks/BLOCKS_PER_PAGE) -
- (inode->i_mapping->nrpages + SHMEM_I(inode)->swapped);
- if (freed){
- struct shmem_sb_info * sbinfo = SHMEM_SB(inode->i_sb);
- inode->i_blocks -= freed*BLOCKS_PER_PAGE;
- spin_lock (&sbinfo->stat_lock);
+ freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
+ if (freed > 0) {
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ info->alloced -= freed;
+ spin_lock(&sbinfo->stat_lock);
sbinfo->free_blocks += freed;
- spin_unlock (&sbinfo->stat_lock);
+ inode->i_blocks -= freed*BLOCKS_PER_PAGE;
+ spin_unlock(&sbinfo->stat_lock);
}
}
@@ -196,6 +200,8 @@
*/
static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index)
{
+ struct inode *inode = &info->vfs_inode;
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
unsigned long page = 0;
swp_entry_t *entry;
@@ -204,14 +210,33 @@
entry = ERR_PTR(-EFAULT);
break;
}
+
+ /*
+ * Test free_blocks against 1 not 0, since we have 1 data
+ * page (and perhaps indirect index pages) yet to allocate:
+ * a waste to allocate index if we cannot allocate data.
+ */
+ spin_lock(&sbinfo->stat_lock);
+ if (sbinfo->free_blocks <= 1) {
+ spin_unlock(&sbinfo->stat_lock);
+ return ERR_PTR(-ENOSPC);
+ }
+ sbinfo->free_blocks--;
+ inode->i_blocks += BLOCKS_PER_PAGE;
+ spin_unlock(&sbinfo->stat_lock);
+
spin_unlock(&info->lock);
page = get_zeroed_page(GFP_USER);
spin_lock(&info->lock);
- if (!page)
+
+ if (!page) {
+ shmem_free_block(inode);
return ERR_PTR(-ENOMEM);
+ }
}
if (page) {
/* another task gave its page, or truncated the file */
+ shmem_free_block(inode);
free_page(page);
}
return entry;
@@ -243,41 +268,42 @@
* shmem_truncate_direct - free the swap entries of a whole doubly
* indirect block
*
+ * @info: the info structure of the inode
* @dir: pointer to the pointer to the block
* @start: offset to start from (in pages)
* @len: how many pages are stored in this block
*
* Returns the number of freed swap entries.
*/
-
-static inline unsigned long
-shmem_truncate_direct(swp_entry_t *** dir, unsigned long start, unsigned long len) {
+static inline unsigned long
+shmem_truncate_direct(struct shmem_inode_info *info, swp_entry_t ***dir, unsigned long start, unsigned long len)
+{
swp_entry_t **last, **ptr;
- unsigned long off, freed = 0;
-
- if (!*dir)
- return 0;
+ unsigned long off, freed_swp, freed = 0;
last = *dir + (len + ENTRIES_PER_PAGE-1) / ENTRIES_PER_PAGE;
off = start % ENTRIES_PER_PAGE;
- for (ptr = *dir + start/ENTRIES_PER_PAGE; ptr < last; ptr++) {
- if (!*ptr) {
- off = 0;
+ for (ptr = *dir + start/ENTRIES_PER_PAGE; ptr < last; ptr++, off = 0) {
+ if (!*ptr)
continue;
+
+ if (info->swapped) {
+ freed_swp = shmem_free_swp(*ptr + off,
+ ENTRIES_PER_PAGE - off);
+ info->swapped -= freed_swp;
+ freed += freed_swp;
}
if (!off) {
- freed += shmem_free_swp(*ptr, ENTRIES_PER_PAGE);
- free_page ((unsigned long) *ptr);
+ info->alloced++;
+ free_page((unsigned long) *ptr);
*ptr = 0;
- } else {
- freed += shmem_free_swp(*ptr+off,ENTRIES_PER_PAGE-off);
- off = 0;
}
}
-
+
if (!start) {
+ info->alloced++;
free_page((unsigned long) *dir);
*dir = 0;
}
@@ -299,11 +325,16 @@
swp_entry_t ***base;
unsigned long baseidx, len, start;
unsigned long max = info->next_index-1;
+ unsigned long freed;
if (max < SHMEM_NR_DIRECT) {
info->next_index = index;
- return shmem_free_swp(info->i_direct + index,
- SHMEM_NR_DIRECT - index);
+ if (!info->swapped)
+ return 0;
+ freed = shmem_free_swp(info->i_direct + index,
+ SHMEM_NR_DIRECT - index);
+ info->swapped -= freed;
+ return freed;
}
if (max < ENTRIES_PER_PAGE * ENTRIES_PER_PAGE/2 + SHMEM_NR_DIRECT) {
@@ -329,24 +360,21 @@
info->next_index = baseidx;
start = 0;
}
- return shmem_truncate_direct(base, start, len);
+ return *base? shmem_truncate_direct(info, base, start, len): 0;
}
-static void shmem_truncate (struct inode * inode)
+static void shmem_truncate(struct inode *inode)
{
+ struct shmem_inode_info *info = SHMEM_I(inode);
unsigned long index;
- unsigned long freed = 0;
- struct shmem_inode_info * info = SHMEM_I(inode);
inode->i_ctime = inode->i_mtime = CURRENT_TIME;
index = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
- spin_lock (&info->lock);
- while (index < info->next_index)
- freed += shmem_truncate_indirect(info, index);
-
- info->swapped -= freed;
+ spin_lock(&info->lock);
+ while (index < info->next_index)
+ (void) shmem_truncate_indirect(info, index);
shmem_recalc_inode(inode);
- spin_unlock (&info->lock);
+ spin_unlock(&info->lock);
}
static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
@@ -407,6 +435,7 @@
inode->i_size = 0;
shmem_truncate (inode);
}
+ BUG_ON(inode->i_blocks);
spin_lock (&sbinfo->stat_lock);
sbinfo->free_inodes++;
spin_unlock (&sbinfo->stat_lock);
@@ -663,13 +692,12 @@
return -ENOSPC;
}
sbinfo->free_blocks--;
+ inode->i_blocks += BLOCKS_PER_PAGE;
spin_unlock(&sbinfo->stat_lock);
page = page_cache_alloc(mapping);
if (!page) {
- spin_lock(&sbinfo->stat_lock);
- sbinfo->free_blocks++;
- spin_unlock(&sbinfo->stat_lock);
+ shmem_free_block(inode);
return -ENOMEM;
}
@@ -681,16 +709,14 @@
add_to_page_cache_lru(page, mapping, idx) < 0) {
spin_unlock(&info->lock);
page_cache_release(page);
- spin_lock(&sbinfo->stat_lock);
- sbinfo->free_blocks++;
- spin_unlock(&sbinfo->stat_lock);
+ shmem_free_block(inode);
if (error)
return error;
goto repeat;
}
+ info->alloced++;
spin_unlock(&info->lock);
clear_highpage(page);
- inode->i_blocks += BLOCKS_PER_PAGE;
}
/* We have the page */
Regularize the erratic whitespace conventions in mm/shmem.c. Removal
of blank line changes BUG_ON line numbers, otherwise builds the same.
--- tmpfs8/mm/shmem.c Tue Oct 1 19:49:29 2002
+++ tmpfs9/mm/shmem.c Tue Oct 1 19:49:34 2002
@@ -140,7 +140,7 @@
*
* It has to be called with the spinlock held.
*/
-static void shmem_recalc_inode(struct inode * inode)
+static void shmem_recalc_inode(struct inode *inode)
{
struct shmem_inode_info *info = SHMEM_I(inode);
long freed;
@@ -511,8 +511,7 @@
return error;
}
-
-static void shmem_delete_inode(struct inode * inode)
+static void shmem_delete_inode(struct inode *inode)
{
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
struct shmem_inode_info *info = SHMEM_I(inode);
@@ -524,12 +523,12 @@
if (info->flags & VM_ACCOUNT)
vm_unacct_memory(VM_ACCT(inode->i_size));
inode->i_size = 0;
- shmem_truncate (inode);
+ shmem_truncate(inode);
}
BUG_ON(inode->i_blocks);
- spin_lock (&sbinfo->stat_lock);
+ spin_lock(&sbinfo->stat_lock);
sbinfo->free_inodes++;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
clear_inode(inode);
}
@@ -629,10 +628,10 @@
int shmem_unuse(swp_entry_t entry, struct page *page)
{
struct list_head *p;
- struct shmem_inode_info * info;
+ struct shmem_inode_info *info;
int found = 0;
- spin_lock (&shmem_ilock);
+ spin_lock(&shmem_ilock);
list_for_each(p, &shmem_inodes) {
info = list_entry(p, struct shmem_inode_info, list);
@@ -643,14 +642,14 @@
break;
}
}
- spin_unlock (&shmem_ilock);
+ spin_unlock(&shmem_ilock);
return found;
}
/*
* Move the page from the page cache to the swap cache.
*/
-static int shmem_writepage(struct page * page)
+static int shmem_writepage(struct page *page)
{
struct shmem_inode_info *info;
swp_entry_t *entry, swap;
@@ -909,10 +908,10 @@
return page;
}
-void shmem_lock(struct file * file, int lock)
+void shmem_lock(struct file *file, int lock)
{
- struct inode * inode = file->f_dentry->d_inode;
- struct shmem_inode_info * info = SHMEM_I(inode);
+ struct inode *inode = file->f_dentry->d_inode;
+ struct shmem_inode_info *info = SHMEM_I(inode);
spin_lock(&info->lock);
if (lock)
@@ -922,9 +921,9 @@
spin_unlock(&info->lock);
}
-static int shmem_mmap(struct file * file, struct vm_area_struct * vma)
+static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
{
- struct vm_operations_struct * ops;
+ struct vm_operations_struct *ops;
struct inode *inode = file->f_dentry->d_inode;
ops = &shmem_vm_ops;
@@ -937,17 +936,17 @@
struct inode *shmem_get_inode(struct super_block *sb, int mode, int dev)
{
- struct inode * inode;
+ struct inode *inode;
struct shmem_inode_info *info;
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
- spin_lock (&sbinfo->stat_lock);
+ spin_lock(&sbinfo->stat_lock);
if (!sbinfo->free_inodes) {
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
return NULL;
}
sbinfo->free_inodes--;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
inode = new_inode(sb);
if (inode) {
@@ -971,9 +970,9 @@
case S_IFREG:
inode->i_op = &shmem_inode_operations;
inode->i_fop = &shmem_file_operations;
- spin_lock (&shmem_ilock);
+ spin_lock(&shmem_ilock);
list_add_tail(&info->list, &shmem_inodes);
- spin_unlock (&shmem_ilock);
+ spin_unlock(&shmem_ilock);
break;
case S_IFDIR:
inode->i_nlink++;
@@ -1019,9 +1018,9 @@
static struct inode_operations shmem_symlink_inline_operations;
static ssize_t
-shmem_file_write(struct file *file,const char *buf,size_t count,loff_t *ppos)
+shmem_file_write(struct file *file, const char *buf, size_t count, loff_t *ppos)
{
- struct inode *inode = file->f_dentry->d_inode;
+ struct inode *inode = file->f_dentry->d_inode;
unsigned long limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
loff_t pos;
struct page *page;
@@ -1129,7 +1128,7 @@
count -= bytes;
pos += bytes;
buf += bytes;
- if (pos > inode->i_size)
+ if (pos > inode->i_size)
inode->i_size = pos;
}
unlock:
@@ -1156,7 +1155,7 @@
goto unlock;
}
-static void do_shmem_file_read(struct file * filp, loff_t *ppos, read_descriptor_t * desc)
+static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc)
{
struct inode *inode = filp->f_dentry->d_inode;
struct address_space *mapping = inode->i_mapping;
@@ -1220,7 +1219,7 @@
offset += nr;
index += offset >> PAGE_CACHE_SHIFT;
offset &= ~PAGE_CACHE_MASK;
-
+
page_cache_release(page);
}
@@ -1228,7 +1227,7 @@
UPDATE_ATIME(inode);
}
-static ssize_t shmem_file_read(struct file * filp, char * buf, size_t count, loff_t *ppos)
+static ssize_t shmem_file_read(struct file *filp, char *buf, size_t count, loff_t *ppos)
{
ssize_t retval;
@@ -1259,12 +1258,12 @@
buf->f_type = TMPFS_MAGIC;
buf->f_bsize = PAGE_CACHE_SIZE;
- spin_lock (&sbinfo->stat_lock);
+ spin_lock(&sbinfo->stat_lock);
buf->f_blocks = sbinfo->max_blocks;
buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
buf->f_files = sbinfo->max_inodes;
buf->f_ffree = sbinfo->free_inodes;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
buf->f_namelen = NAME_MAX;
return 0;
}
@@ -1274,7 +1273,7 @@
*/
static int shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, int dev)
{
- struct inode * inode = shmem_get_inode(dir->i_sb, mode, dev);
+ struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
int error = -ENOSPC;
if (inode) {
@@ -1287,7 +1286,7 @@
return error;
}
-static int shmem_mkdir(struct inode * dir, struct dentry * dentry, int mode)
+static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
{
int error;
@@ -1305,7 +1304,7 @@
/*
* Link a file..
*/
-static int shmem_link(struct dentry *old_dentry, struct inode * dir, struct dentry * dentry)
+static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
{
struct inode *inode = old_dentry->d_inode;
@@ -1351,7 +1350,7 @@
return 1;
}
-static int shmem_unlink(struct inode * dir, struct dentry *dentry)
+static int shmem_unlink(struct inode *dir, struct dentry *dentry)
{
struct inode *inode = dentry->d_inode;
@@ -1362,7 +1361,7 @@
return 0;
}
-static int shmem_rmdir(struct inode * dir, struct dentry *dentry)
+static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
{
if (!shmem_empty(dentry))
return -ENOTEMPTY;
@@ -1377,12 +1376,12 @@
* it exists so that the VFS layer correctly free's it when it
* gets overwritten.
*/
-static int shmem_rename(struct inode * old_dir, struct dentry *old_dentry, struct inode * new_dir,struct dentry *new_dentry)
+static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
{
struct inode *inode = old_dentry->d_inode;
int they_are_dirs = S_ISDIR(inode->i_mode);
- if (!shmem_empty(new_dentry))
+ if (!shmem_empty(new_dentry))
return -ENOTEMPTY;
if (new_dentry->d_inode) {
@@ -1402,14 +1401,14 @@
return 0;
}
-static int shmem_symlink(struct inode * dir, struct dentry *dentry, const char * symname)
+static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
{
int error;
int len;
struct inode *inode;
struct page *page;
char *kaddr;
- struct shmem_inode_info * info;
+ struct shmem_inode_info *info;
len = strlen(symname) + 1;
if (len > PAGE_CACHE_SIZE)
@@ -1437,9 +1436,9 @@
return error;
}
inode->i_op = &shmem_symlink_inode_operations;
- spin_lock (&shmem_ilock);
+ spin_lock(&shmem_ilock);
list_add_tail(&info->list, &shmem_inodes);
- spin_unlock (&shmem_ilock);
+ spin_unlock(&shmem_ilock);
kaddr = kmap(page);
memcpy(kaddr, symname, len);
kunmap(page);
@@ -1456,7 +1455,7 @@
static int shmem_readlink_inline(struct dentry *dentry, char *buffer, int buflen)
{
- return vfs_readlink(dentry,buffer,buflen, (const char *)SHMEM_I(dentry->d_inode));
+ return vfs_readlink(dentry, buffer, buflen, (const char *)SHMEM_I(dentry->d_inode));
}
static int shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
@@ -1501,7 +1500,7 @@
.follow_link = shmem_follow_link,
};
-static int shmem_parse_options(char *options, int *mode, uid_t *uid, gid_t *gid, unsigned long * blocks, unsigned long *inodes)
+static int shmem_parse_options(char *options, int *mode, uid_t *uid, gid_t *gid, unsigned long *blocks, unsigned long *inodes)
{
char *this_char, *value, *rest;
@@ -1511,8 +1510,8 @@
if ((value = strchr(this_char,'=')) != NULL) {
*value++ = 0;
} else {
- printk(KERN_ERR
- "tmpfs: No value for mount option '%s'\n",
+ printk(KERN_ERR
+ "tmpfs: No value for mount option '%s'\n",
this_char);
return 1;
}
@@ -1558,13 +1557,13 @@
return 0;
bad_val:
- printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
+ printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
value, this_char);
return 1;
}
-static int shmem_remount_fs (struct super_block *sb, int *flags, char *data)
+static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
{
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
unsigned long max_blocks = sbinfo->max_blocks;
@@ -1575,16 +1574,16 @@
return shmem_set_size(sbinfo, max_blocks, max_inodes);
}
-int shmem_sync_file(struct file * file, struct dentry *dentry, int datasync)
+int shmem_sync_file(struct file *file, struct dentry *dentry, int datasync)
{
return 0;
}
#endif
-static int shmem_fill_super(struct super_block * sb, void * data, int silent)
+static int shmem_fill_super(struct super_block *sb, void *data, int silent)
{
- struct inode * inode;
- struct dentry * root;
+ struct inode *inode;
+ struct dentry *root;
unsigned long blocks, inodes;
int mode = S_IRWXUGO | S_ISVTX;
uid_t uid = current->fsuid;
@@ -1615,7 +1614,7 @@
sb->s_flags |= MS_NOUSER;
#endif
- spin_lock_init (&sbinfo->stat_lock);
+ spin_lock_init(&sbinfo->stat_lock);
sbinfo->max_blocks = blocks;
sbinfo->free_blocks = blocks;
sbinfo->max_inodes = inodes;
@@ -1655,7 +1654,7 @@
sb->u.generic_sbp = NULL;
}
-static kmem_cache_t * shmem_inode_cachep;
+static kmem_cache_t *shmem_inode_cachep;
static struct inode *shmem_alloc_inode(struct super_block *sb)
{
@@ -1671,7 +1670,7 @@
kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
}
-static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags)
+static void init_once(void *foo, kmem_cache_t *cachep, unsigned long flags)
{
struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
@@ -1680,7 +1679,7 @@
inode_init_once(&p->vfs_inode);
}
}
-
+
static int init_inodecache(void)
{
shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
@@ -1775,7 +1774,7 @@
static int __init init_shmem_fs(void)
{
int error;
- struct vfsmount * res;
+ struct vfsmount *res;
error = init_inodecache();
if (error)
@@ -1783,21 +1782,21 @@
error = register_filesystem(&tmpfs_fs_type);
if (error) {
- printk (KERN_ERR "Could not register tmpfs\n");
+ printk(KERN_ERR "Could not register tmpfs\n");
goto out2;
}
#ifdef CONFIG_TMPFS
error = register_filesystem(&shmem_fs_type);
if (error) {
- printk (KERN_ERR "Could not register shm fs\n");
+ printk(KERN_ERR "Could not register shm fs\n");
goto out1;
}
- devfs_mk_dir (NULL, "shm", NULL);
+ devfs_mk_dir(NULL, "shm", NULL);
#endif
res = kern_mount(&tmpfs_fs_type);
if (IS_ERR (res)) {
error = PTR_ERR(res);
- printk (KERN_ERR "could not kern_mount tmpfs\n");
+ printk(KERN_ERR "could not kern_mount tmpfs\n");
goto out;
}
shm_mnt = res;
@@ -1838,11 +1837,11 @@
* @size: size to be set for the file
*
*/
-struct file *shmem_file_setup(char * name, loff_t size, unsigned long flags)
+struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
{
int error;
struct file *file;
- struct inode * inode;
+ struct inode *inode;
struct dentry *dentry, *root;
struct qstr this;
@@ -1868,7 +1867,7 @@
error = -ENOSPC;
inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
- if (!inode)
+ if (!inode)
goto close_file;
SHMEM_I(inode)->flags &= flags;
@@ -1884,11 +1883,11 @@
close_file:
put_filp(file);
put_dentry:
- dput (dentry);
+ dput(dentry);
put_memory:
if (flags & VM_ACCOUNT)
vm_unacct_memory(VM_ACCT(size));
- return ERR_PTR(error);
+ return ERR_PTR(error);
}
/*
@@ -1900,13 +1899,13 @@
{
struct file *file;
loff_t size = vma->vm_end - vma->vm_start;
-
+
file = shmem_file_setup("dev/zero", size, vma->vm_flags);
if (IS_ERR(file))
return PTR_ERR(file);
if (vma->vm_file)
- fput (vma->vm_file);
+ fput(vma->vm_file);
vma->vm_file = file;
vma->vm_ops = &shmem_vm_ops;
return 0;