From: tao zeng Date: Wed, 7 Nov 2018 08:51:50 +0000 (+0800) Subject: mm: optimize stack usage for functions [1/1] X-Git-Tag: hardkernel-4.9.236-104~2047 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=08fbfd263a0b7f8ef9a0684791a4bb02487c4c83;p=platform%2Fkernel%2Flinux-amlogic.git mm: optimize stack usage for functions [1/1] PD#SWPL-1773 Problem: After adding optimization of vmap stack, we can found stack usage of each functions when handle vmap fault. From test log we see some functions using large stack size which over 256bytes. Especially common call path from fs. We need to optimize stack usage of these functions to reduce stack fault probability and save stack memory usage. Solution: 1. remove CONFIG_CC_STACKPROTECTOR_STRONG and set STACKPROTECTOR to NONE. This can save stack usage add by compiler for most functions. Kernel code size can also save over 1MB. 2. Add some noinline functions for android_fs_data rw trace calls. In these trace call it allcated a 256 bytes local buffer. 3. Add a wrap function for mem abort handler. By default, it defined a siginfo struct(size over 100 bytes) in local but only used when fault can't be handled. 4. reduce cached page size for vmap stack since probability of page fault caused by stack overflow is reduced after function stack usage optimized. Monkey test show real stack usage ratio compared with 1st vmap implementation reduced from 35% ~ 38% to 26 ~ 27%. Which is very close to 25%, theory limit. Verify: P212 Change-Id: I5505cacc1cab51f88654052902852fd648b6a036 Signed-off-by: tao zeng --- diff --git a/arch/arm64/configs/meson64_defconfig b/arch/arm64/configs/meson64_defconfig index 8858f73..2e6a998 100644 --- a/arch/arm64/configs/meson64_defconfig +++ b/arch/arm64/configs/meson64_defconfig @@ -30,7 +30,6 @@ CONFIG_EMBEDDED=y # CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_JUMP_LABEL=y -CONFIG_CC_STACKPROTECTOR_STRONG=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2a976fc..df09e98 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -662,6 +662,20 @@ static const struct fault_info fault_info[] = { { do_bad, SIGBUS, 0, "unknown 63" }, }; +#ifdef CONFIG_AMLOGIC_VMAP +asmlinkage static void die_wrap(const struct fault_info *inf, + struct pt_regs *regs, unsigned int esr, + unsigned long addr) +{ + struct siginfo info; + + info.si_signo = inf->sig; + info.si_errno = 0; + info.si_code = inf->code; + info.si_addr = (void __user *)addr; + arm64_notify_die("", regs, &info, esr); +} +#endif /* * Dispatch a data abort to the relevant handler. */ @@ -669,7 +683,9 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) { const struct fault_info *inf = esr_to_fault_info(esr); +#ifndef CONFIG_AMLOGIC_VMAP struct siginfo info; +#endif if (!inf->fn(addr, esr, regs)) return; @@ -677,11 +693,15 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", inf->name, esr, addr); +#ifndef CONFIG_AMLOGIC_VMAP info.si_signo = inf->sig; info.si_errno = 0; info.si_code = inf->code; info.si_addr = (void __user *)addr; arm64_notify_die("", regs, &info, esr); +#else + die_wrap(inf, regs, esr, addr); +#endif } asmlinkage void __exception do_el0_irq_bp_hardening(void) diff --git a/drivers/amlogic/memory_ext/vmap_stack.c b/drivers/amlogic/memory_ext/vmap_stack.c index 687a1a6..511cfc7 100644 --- a/drivers/amlogic/memory_ext/vmap_stack.c +++ b/drivers/amlogic/memory_ext/vmap_stack.c @@ -352,11 +352,17 @@ void aml_account_task_stack(struct task_struct *tsk, int account) memcg_kmem_update_page_stat(first_page, MEMCG_KERNEL_STACK_KB, account * (THREAD_SIZE / 1024)); if (time_after(jiffies, vmap_debug_jiff + HZ * 5)) { + int ratio, rem; + vmap_debug_jiff = jiffies; - D("KERNEL_STACK:%ld KB, vmap stack:%d KB, cached:%d KB\n", + ratio = ((get_vmap_stack_size() << (PAGE_SHIFT - 10)) * 10000) / + global_page_state(NR_KERNEL_STACK_KB); + rem = ratio % 100; + D("STACK:%ld KB, vmap:%d KB, cached:%d KB, rate:%2d.%02d%%\n", global_page_state(NR_KERNEL_STACK_KB), get_vmap_stack_size() << (PAGE_SHIFT - 10), - avmap->cached_pages << (PAGE_SHIFT - 10)); + avmap->cached_pages << (PAGE_SHIFT - 10), + ratio / 100, rem); } } diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 949499d..5543cdf 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1172,6 +1172,38 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, } #endif +#ifdef CONFIG_AMLOGIC_VMAP +noinline void trace_android_fs_datawrite_wrap(struct inode *inode, + loff_t pos, unsigned int len) +{ + if (trace_android_fs_datawrite_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_datawrite_start(inode, pos, len, + current->pid, path, + current->comm); + } +} + +noinline void trace_android_fs_dataread_wrap(struct inode *inode, + loff_t pos, unsigned int len) +{ + if (trace_android_fs_dataread_start_enabled()) { + char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; + + path = android_fstrace_get_pathname(pathbuf, + MAX_TRACE_PATHBUF_LEN, + inode); + trace_android_fs_dataread_start(inode, pos, len, + current->pid, path, + current->comm); + } +} +#endif + static int ext4_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) @@ -1184,6 +1216,9 @@ static int ext4_write_begin(struct file *file, struct address_space *mapping, pgoff_t index; unsigned from, to; +#ifdef CONFIG_AMLOGIC_VMAP + trace_android_fs_datawrite_wrap(inode, pos, len); +#else if (trace_android_fs_datawrite_start_enabled()) { char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; @@ -1194,6 +1229,7 @@ static int ext4_write_begin(struct file *file, struct address_space *mapping, current->pid, path, current->comm); } +#endif trace_ext4_write_begin(inode, pos, len, flags); /* * Reserve one block more for addition to orphan list in case @@ -2937,6 +2973,9 @@ static int ext4_da_write_begin(struct file *file, struct address_space *mapping, len, flags, pagep, fsdata); } *fsdata = (void *)0; +#ifdef CONFIG_AMLOGIC_VMAP + trace_android_fs_datawrite_wrap(inode, pos, len); +#else if (trace_android_fs_datawrite_start_enabled()) { char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; @@ -2947,6 +2986,7 @@ static int ext4_da_write_begin(struct file *file, struct address_space *mapping, current->pid, path, current->comm); } +#endif trace_ext4_da_write_begin(inode, pos, len, flags); if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { @@ -3646,6 +3686,12 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (ext4_has_inline_data(inode)) return 0; +#ifdef CONFIG_AMLOGIC_VMAP + if (rw == READ) + trace_android_fs_dataread_wrap(inode, offset, count); + if (rw == WRITE) + trace_android_fs_datawrite_wrap(inode, offset, count); +#else if (trace_android_fs_dataread_start_enabled() && (rw == READ)) { char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; @@ -3668,6 +3714,7 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter) current->pid, path, current->comm); } +#endif trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); if (iov_iter_rw(iter) == READ) ret = ext4_direct_IO_read(iocb, iter); diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index c39a12d..483c22b 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -117,6 +117,12 @@ ext4_submit_bio_read(struct bio *bio) struct page *first_page = bio->bi_io_vec[0].bv_page; if (first_page != NULL) { + #ifdef CONFIG_AMLOGIC_VMAP + trace_android_fs_dataread_wrap( + first_page->mapping->host, + page_offset(first_page), + bio->bi_iter.bi_size); + #else char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; path = android_fstrace_get_pathname(pathbuf, @@ -129,6 +135,7 @@ ext4_submit_bio_read(struct bio *bio) current->pid, path, current->comm); + #endif } } submit_bio(bio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index a481dde..9c87753 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -2243,6 +2243,9 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping, block_t blkaddr = NULL_ADDR; int err = 0; +#ifdef CONFIG_AMLOGIC_VMAP + trace_android_fs_datawrite_wrap(inode, pos, len); +#else if (trace_android_fs_datawrite_start_enabled()) { char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; @@ -2253,6 +2256,7 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping, current->pid, path, current->comm); } +#endif trace_f2fs_write_begin(inode, pos, len, flags); if (f2fs_is_atomic_file(inode) && @@ -2413,6 +2417,12 @@ static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) trace_f2fs_direct_IO_enter(inode, offset, count, rw); +#ifdef CONFIG_AMLOGIC_VMAP + if (rw == READ) + trace_android_fs_dataread_wrap(inode, offset, count); + if (rw == WRITE) + trace_android_fs_datawrite_wrap(inode, offset, count); +#else if (trace_android_fs_dataread_start_enabled() && (rw == READ)) { char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; @@ -2435,6 +2445,7 @@ static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) current->pid, path, current->comm); } +#endif if (rw == WRITE && whint_mode == WHINT_MODE_OFF) iocb->ki_hint = WRITE_LIFE_NOT_SET; diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 156ac4f..aed9c0d 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -86,6 +86,9 @@ int f2fs_read_inline_data(struct inode *inode, struct page *page) { struct page *ipage; +#ifdef CONFIG_AMLOGIC_VMAP + trace_android_fs_dataread_wrap(inode, page_offset(page), PAGE_SIZE); +#else if (trace_android_fs_dataread_start_enabled()) { char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; @@ -96,6 +99,7 @@ int f2fs_read_inline_data(struct inode *inode, struct page *page) PAGE_SIZE, current->pid, path, current->comm); } +#endif ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); if (IS_ERR(ipage)) { diff --git a/fs/mpage.c b/fs/mpage.c index d4e17c8..dc0e052 100644 --- a/fs/mpage.c +++ b/fs/mpage.c @@ -80,6 +80,12 @@ static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio) struct page *first_page = bio->bi_io_vec[0].bv_page; if (first_page != NULL) { + #ifdef CONFIG_AMLOGIC_VMAP + trace_android_fs_dataread_wrap( + first_page->mapping->host, + page_offset(first_page), + bio->bi_iter.bi_size); + #else char *path, pathbuf[MAX_TRACE_PATHBUF_LEN]; path = android_fstrace_get_pathname(pathbuf, @@ -92,6 +98,7 @@ static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio) current->pid, path, current->comm); + #endif } } bio->bi_end_io = mpage_end_io; diff --git a/fs/sdcardfs/inode.c b/fs/sdcardfs/inode.c index 81fe9e6..84df23ac0 100644 --- a/fs/sdcardfs/inode.c +++ b/fs/sdcardfs/inode.c @@ -694,6 +694,153 @@ static int sdcardfs_setattr_wrn(struct dentry *dentry, struct iattr *ia) return -EINVAL; } +#ifdef CONFIG_AMLOGIC_VMAP +/* a save stack version */ +static int sdcardfs_setattr(struct vfsmount *mnt, struct dentry *dentry, + struct iattr *ia) +{ + int err; + struct dentry *lower_dentry; + struct vfsmount *lower_mnt; + struct inode *inode; + struct inode *lower_inode; + struct path lower_path; + struct iattr lower_ia; + struct dentry *parent; + struct inode *tmp; + struct dentry *tmp_d; + struct sdcardfs_inode_data *top; + + const struct cred *saved_cred = NULL; + + inode = d_inode(dentry); + top = top_data_get(SDCARDFS_I(inode)); + + if (!top) + return -EINVAL; + + tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + tmp_d = kzalloc(sizeof(*tmp_d), GFP_KERNEL); + if (!tmp_d) { + kfree(tmp); + return -ENOMEM; + } + /* + * Permission check on sdcardfs inode. + * Calling process should have AID_SDCARD_RW permission + * Since generic_permission only needs i_mode, i_uid, + * i_gid, and i_sb, we can create a fake inode to pass + * this information down in. + * + * The underlying code may attempt to take locks in some + * cases for features we're not using, but if that changes, + * locks must be dealt with to avoid undefined behavior. + * + */ + copy_attrs(tmp, inode); + tmp->i_uid = make_kuid(&init_user_ns, top->d_uid); + tmp->i_gid = make_kgid(&init_user_ns, get_gid(mnt, dentry->d_sb, top)); + tmp->i_mode = (inode->i_mode & S_IFMT) + | get_mode(mnt, SDCARDFS_I(inode), top); + tmp->i_size = i_size_read(inode); + data_put(top); + tmp->i_sb = inode->i_sb; + tmp_d->d_inode = tmp; + + /* + * Check if user has permission to change dentry. We don't check if + * this user can change the lower inode: that should happen when + * calling notify_change on the lower inode. + */ + /* prepare our own lower struct iattr (with the lower file) */ + memcpy(&lower_ia, ia, sizeof(lower_ia)); + /* Allow touch updating timestamps. A previous permission check ensures + * we have write access. Changes to mode, owner, and group are ignored + */ + ia->ia_valid |= ATTR_FORCE; + err = setattr_prepare(tmp_d, ia); + + if (!err) { + /* check the Android group ID */ + parent = dget_parent(dentry); + if (!check_caller_access_to_name(d_inode(parent), + &dentry->d_name)) + err = -EACCES; + dput(parent); + } + + if (err) + goto out_err; + + /* save current_cred and override it */ + OVERRIDE_CRED(SDCARDFS_SB(dentry->d_sb), saved_cred, SDCARDFS_I(inode)); + + sdcardfs_get_lower_path(dentry, &lower_path); + lower_dentry = lower_path.dentry; + lower_mnt = lower_path.mnt; + lower_inode = sdcardfs_lower_inode(inode); + + if (ia->ia_valid & ATTR_FILE) + lower_ia.ia_file = sdcardfs_lower_file(ia->ia_file); + + lower_ia.ia_valid &= ~(ATTR_UID | ATTR_GID | ATTR_MODE); + + /* + * If shrinking, first truncate upper level to cancel writing dirty + * pages beyond the new eof; and also if its' maxbytes is more + * limiting (fail with -EFBIG before making any change to the lower + * level). There is no need to vmtruncate the upper level + * afterwards in the other cases: we fsstack_copy_inode_size from + * the lower level. + */ + if (ia->ia_valid & ATTR_SIZE) { + err = inode_newsize_ok(tmp, ia->ia_size); + if (err) + goto out; + truncate_setsize(inode, ia->ia_size); + } + + /* + * mode change is for clearing setuid/setgid bits. Allow lower fs + * to interpret this in its own way. + */ + if (lower_ia.ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) + lower_ia.ia_valid &= ~ATTR_MODE; + + /* notify the (possibly copied-up) lower inode */ + /* + * Note: we use d_inode(lower_dentry), because lower_inode may be + * unlinked (no inode->i_sb and i_ino==0. This happens if someone + * tries to open(), unlink(), then ftruncate() a file. + */ + inode_lock(d_inode(lower_dentry)); + err = notify_change2(lower_mnt, lower_dentry, &lower_ia, + NULL); + inode_unlock(d_inode(lower_dentry)); + if (err) + goto out; + + /* get attributes from the lower inode and update derived permissions */ + sdcardfs_copy_and_fix_attrs(inode, lower_inode); + + /* + * Not running fsstack_copy_inode_size(inode, lower_inode), because + * VFS should update our inode size, and notify_change on + * lower_inode should update its size. + */ + +out: + sdcardfs_put_lower_path(dentry, &lower_path); + REVERT_CRED(saved_cred); +out_err: + kfree(tmp); + kfree(tmp_d); + return err; +} +#else static int sdcardfs_setattr(struct vfsmount *mnt, struct dentry *dentry, struct iattr *ia) { int err; @@ -826,6 +973,7 @@ out: out_err: return err; } +#endif static int sdcardfs_fillattr(struct vfsmount *mnt, struct inode *inode, struct kstat *lower_stat, struct kstat *stat) diff --git a/include/linux/amlogic/vmap_stack.h b/include/linux/amlogic/vmap_stack.h index 8f7a36f..1b4081e 100644 --- a/include/linux/amlogic/vmap_stack.h +++ b/include/linux/amlogic/vmap_stack.h @@ -29,7 +29,7 @@ #define VMAP_PAGE_FLAG (__GFP_ZERO | __GFP_HIGH |\ __GFP_ATOMIC | __GFP_REPEAT) -#define VMAP_CACHE_PAGE_ORDER 6 +#define VMAP_CACHE_PAGE_ORDER 5 #define VMAP_CACHE_PAGE (1 << VMAP_CACHE_PAGE_ORDER) #define CACHE_MAINTAIN_DELAY (HZ) diff --git a/include/trace/events/android_fs.h b/include/trace/events/android_fs.h index 4950953..9fc5534 100644 --- a/include/trace/events/android_fs.h +++ b/include/trace/events/android_fs.h @@ -63,3 +63,9 @@ android_fstrace_get_pathname(char *buf, int buflen, struct inode *inode) return path; } #endif +#ifdef CONFIG_AMLOGIC_VMAP +extern void trace_android_fs_datawrite_wrap(struct inode *inode, + loff_t pos, unsigned int len); +extern void trace_android_fs_dataread_wrap(struct inode *inode, + loff_t pos, unsigned int len); +#endif