diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs new file mode 100644 index 00000000000..2c4cc42006e --- /dev/null +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -0,0 +1,82 @@ +What: /sys/fs/f2fs//gc_max_sleep_time +Date: July 2013 +Contact: "Namjae Jeon" +Description: + Controls the maximun sleep time for gc_thread. Time + is in milliseconds. + +What: /sys/fs/f2fs//gc_min_sleep_time +Date: July 2013 +Contact: "Namjae Jeon" +Description: + Controls the minimum sleep time for gc_thread. Time + is in milliseconds. + +What: /sys/fs/f2fs//gc_no_gc_sleep_time +Date: July 2013 +Contact: "Namjae Jeon" +Description: + Controls the default sleep time for gc_thread. Time + is in milliseconds. + +What: /sys/fs/f2fs//gc_idle +Date: July 2013 +Contact: "Namjae Jeon" +Description: + Controls the victim selection policy for garbage collection. + +What: /sys/fs/f2fs//reclaim_segments +Date: October 2013 +Contact: "Jaegeuk Kim" +Description: + Controls the issue rate of segment discard commands. + +What: /sys/fs/f2fs//ipu_policy +Date: November 2013 +Contact: "Jaegeuk Kim" +Description: + Controls the in-place-update policy. + +What: /sys/fs/f2fs//min_ipu_util +Date: November 2013 +Contact: "Jaegeuk Kim" +Description: + Controls the FS utilization condition for the in-place-update + policies. + +What: /sys/fs/f2fs//min_fsync_blocks +Date: September 2014 +Contact: "Jaegeuk Kim" +Description: + Controls the dirty page count condition for the in-place-update + policies. + +What: /sys/fs/f2fs//max_small_discards +Date: November 2013 +Contact: "Jaegeuk Kim" +Description: + Controls the issue rate of small discard commands. + +What: /sys/fs/f2fs//max_victim_search +Date: January 2014 +Contact: "Jaegeuk Kim" +Description: + Controls the number of trials to find a victim segment. + +What: /sys/fs/f2fs//dir_level +Date: March 2014 +Contact: "Jaegeuk Kim" +Description: + Controls the directory level for large directory. + +What: /sys/fs/f2fs//ram_thresh +Date: March 2014 +Contact: "Jaegeuk Kim" +Description: + Controls the memory footprint used by f2fs. + +What: /sys/fs/f2fs//trim_sections +Date: February 2015 +Contact: "Jaegeuk Kim" +Description: + Controls the trimming rate in batch mode. diff --git a/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons b/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons new file mode 100644 index 00000000000..acb19b91c19 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-kernel-wakeup_reasons @@ -0,0 +1,16 @@ +What: /sys/kernel/wakeup_reasons/last_resume_reason +Date: February 2014 +Contact: Ruchi Kandoi +Description: + The /sys/kernel/wakeup_reasons/last_resume_reason is + used to report wakeup reasons after system exited suspend. + +What: /sys/kernel/wakeup_reasons/last_suspend_time +Date: March 2015 +Contact: jinqian +Description: + The /sys/kernel/wakeup_reasons/last_suspend_time is + used to report time spent in last suspend cycle. It contains + two numbers (in seconds) separated by space. First number is + the time spent in suspend and resume processes. Second number + is the time spent in sleep state. \ No newline at end of file diff --git a/Documentation/devicetree/bindings/arm/msm/rpm-master-stats.txt b/Documentation/devicetree/bindings/arm/msm/rpm-master-stats.txt new file mode 100644 index 00000000000..02396745dd5 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/msm/rpm-master-stats.txt @@ -0,0 +1,29 @@ +* RPM Master Stats + +RPM maintains each master data in RPM message RAM at a specific +offset. It tells about the individual masters information at +any given time like "number of active cores in sub system", +"number of shutdowns" and "wakeup reason for SS" etc. These stats +can be show to the user using the debugfs interface of the kernel. +To achieve this device tree node has been added and it will hold +the address of the RPM RAM from where master stats are read. +Added version number to distinguish the type of data structure +being read from the RAM for different targets. + +The required properties for rpm-master-stats are: + +- compatible: "qcom,rpm-master-stats". +- reg: The address on the RPM RAM from where stats are read. +- qcom,masters: Each master name. +- qcom,master-offset: Offset required to access each master stats area. +- qcom,master-stats-version: Version number. + +Example: + +qcom,rpm-stats@fc428150 { + compatible = "qcom,rpm-stats"; + reg = <0xfc428150 0x1000>; + qcom,masters = "APSS", "MPSS", "LPSS", "PRONTO"; + qcom,master-offset = <2560>; + qcom,master-stats-version = <2>; +}; diff --git a/Documentation/filesystems/00-INDEX b/Documentation/filesystems/00-INDEX index 8c624a18f67..a328432d563 100644 --- a/Documentation/filesystems/00-INDEX +++ b/Documentation/filesystems/00-INDEX @@ -46,6 +46,8 @@ ext3.txt - info, mount options and specifications for the Ext3 filesystem. ext4.txt - info, mount options and specifications for the Ext4 filesystem. +f2fs.txt + - info and mount options for the F2FS filesystem. files.txt - info on file management in the Linux kernel. fuse.txt diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking index 4fca82e5276..d5a269a51a9 100644 --- a/Documentation/filesystems/Locking +++ b/Documentation/filesystems/Locking @@ -62,6 +62,7 @@ ata *); int (*removexattr) (struct dentry *, const char *); void (*truncate_range)(struct inode *, loff_t, loff_t); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len); + void (*update_time)(struct inode *, struct timespec *, int); locking rules: all may block @@ -89,6 +90,8 @@ listxattr: no removexattr: yes truncate_range: yes fiemap: no +update_time: no + Additionally, ->rmdir(), ->unlink() and ->rename() have ->i_mutex on victim. cross-directory ->rename() has (per-superblock) ->s_vfs_rename_sem. diff --git a/Documentation/filesystems/f2fs.txt b/Documentation/filesystems/f2fs.txt new file mode 100644 index 00000000000..b102b436563 --- /dev/null +++ b/Documentation/filesystems/f2fs.txt @@ -0,0 +1,579 @@ +================================================================================ +WHAT IS Flash-Friendly File System (F2FS)? +================================================================================ + +NAND flash memory-based storage devices, such as SSD, eMMC, and SD cards, have +been equipped on a variety systems ranging from mobile to server systems. Since +they are known to have different characteristics from the conventional rotating +disks, a file system, an upper layer to the storage device, should adapt to the +changes from the sketch in the design level. + +F2FS is a file system exploiting NAND flash memory-based storage devices, which +is based on Log-structured File System (LFS). The design has been focused on +addressing the fundamental issues in LFS, which are snowball effect of wandering +tree and high cleaning overhead. + +Since a NAND flash memory-based storage device shows different characteristic +according to its internal geometry or flash memory management scheme, namely FTL, +F2FS and its tools support various parameters not only for configuring on-disk +layout, but also for selecting allocation and cleaning algorithms. + +The following git tree provides the file system formatting tool (mkfs.f2fs), +a consistency checking tool (fsck.f2fs), and a debugging tool (dump.f2fs). +>> git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git + +For reporting bugs and sending patches, please use the following mailing list: +>> linux-f2fs-devel@lists.sourceforge.net + +================================================================================ +BACKGROUND AND DESIGN ISSUES +================================================================================ + +Log-structured File System (LFS) +-------------------------------- +"A log-structured file system writes all modifications to disk sequentially in +a log-like structure, thereby speeding up both file writing and crash recovery. +The log is the only structure on disk; it contains indexing information so that +files can be read back from the log efficiently. In order to maintain large free +areas on disk for fast writing, we divide the log into segments and use a +segment cleaner to compress the live information from heavily fragmented +segments." from Rosenblum, M. and Ousterhout, J. K., 1992, "The design and +implementation of a log-structured file system", ACM Trans. Computer Systems +10, 1, 26–52. + +Wandering Tree Problem +---------------------- +In LFS, when a file data is updated and written to the end of log, its direct +pointer block is updated due to the changed location. Then the indirect pointer +block is also updated due to the direct pointer block update. In this manner, +the upper index structures such as inode, inode map, and checkpoint block are +also updated recursively. This problem is called as wandering tree problem [1], +and in order to enhance the performance, it should eliminate or relax the update +propagation as much as possible. + +[1] Bityutskiy, A. 2005. JFFS3 design issues. http://www.linux-mtd.infradead.org/ + +Cleaning Overhead +----------------- +Since LFS is based on out-of-place writes, it produces so many obsolete blocks +scattered across the whole storage. In order to serve new empty log space, it +needs to reclaim these obsolete blocks seamlessly to users. This job is called +as a cleaning process. + +The process consists of three operations as follows. +1. A victim segment is selected through referencing segment usage table. +2. It loads parent index structures of all the data in the victim identified by + segment summary blocks. +3. It checks the cross-reference between the data and its parent index structure. +4. It moves valid data selectively. + +This cleaning job may cause unexpected long delays, so the most important goal +is to hide the latencies to users. And also definitely, it should reduce the +amount of valid data to be moved, and move them quickly as well. + +================================================================================ +KEY FEATURES +================================================================================ + +Flash Awareness +--------------- +- Enlarge the random write area for better performance, but provide the high + spatial locality +- Align FS data structures to the operational units in FTL as best efforts + +Wandering Tree Problem +---------------------- +- Use a term, “node”, that represents inodes as well as various pointer blocks +- Introduce Node Address Table (NAT) containing the locations of all the “node” + blocks; this will cut off the update propagation. + +Cleaning Overhead +----------------- +- Support a background cleaning process +- Support greedy and cost-benefit algorithms for victim selection policies +- Support multi-head logs for static/dynamic hot and cold data separation +- Introduce adaptive logging for efficient block allocation + +================================================================================ +MOUNT OPTIONS +================================================================================ + +background_gc=%s Turn on/off cleaning operations, namely garbage + collection, triggered in background when I/O subsystem is + idle. If background_gc=on, it will turn on the garbage + collection and if background_gc=off, garbage collection + will be truned off. If background_gc=sync, it will turn + on synchronous garbage collection running in background. + Default value for this option is on. So garbage + collection is on by default. +disable_roll_forward Disable the roll-forward recovery routine +norecovery Disable the roll-forward recovery routine, mounted read- + only (i.e., -o ro,disable_roll_forward) +discard Issue discard/TRIM commands when a segment is cleaned. +no_heap Disable heap-style segment allocation which finds free + segments for data from the beginning of main area, while + for node from the end of main area. +nouser_xattr Disable Extended User Attributes. Note: xattr is enabled + by default if CONFIG_F2FS_FS_XATTR is selected. +noacl Disable POSIX Access Control List. Note: acl is enabled + by default if CONFIG_F2FS_FS_POSIX_ACL is selected. +active_logs=%u Support configuring the number of active logs. In the + current design, f2fs supports only 2, 4, and 6 logs. + Default number is 6. +disable_ext_identify Disable the extension list configured by mkfs, so f2fs + does not aware of cold files such as media files. +inline_xattr Enable the inline xattrs feature. +inline_data Enable the inline data feature: New created small(<~3.4k) + files can be written into inode block. +inline_dentry Enable the inline dir feature: data in new created + directory entries can be written into inode block. The + space of inode block which is used to store inline + dentries is limited to ~3.4k. +flush_merge Merge concurrent cache_flush commands as much as possible + to eliminate redundant command issues. If the underlying + device handles the cache_flush command relatively slowly, + recommend to enable this option. +nobarrier This option can be used if underlying storage guarantees + its cached data should be written to the novolatile area. + If this option is set, no cache_flush commands are issued + but f2fs still guarantees the write ordering of all the + data writes. +fastboot This option is used when a system wants to reduce mount + time as much as possible, even though normal performance + can be sacrificed. +extent_cache Enable an extent cache based on rb-tree, it can cache + as many as extent which map between contiguous logical + address and physical address per inode, resulting in + increasing the cache hit ratio. Set by default. +noextent_cache Diable an extent cache based on rb-tree explicitly, see + the above extent_cache mount option. +noinline_data Disable the inline data feature, inline data feature is + enabled by default. + +================================================================================ +DEBUGFS ENTRIES +================================================================================ + +/sys/kernel/debug/f2fs/ contains information about all the partitions mounted as +f2fs. Each file shows the whole f2fs information. + +/sys/kernel/debug/f2fs/status includes: + - major file system information managed by f2fs currently + - average SIT information about whole segments + - current memory footprint consumed by f2fs. + +================================================================================ +SYSFS ENTRIES +================================================================================ + +Information about mounted f2f2 file systems can be found in +/sys/fs/f2fs. Each mounted filesystem will have a directory in +/sys/fs/f2fs based on its device name (i.e., /sys/fs/f2fs/sda). +The files in each per-device directory are shown in table below. + +Files in /sys/fs/f2fs/ +(see also Documentation/ABI/testing/sysfs-fs-f2fs) +.............................................................................. + File Content + + gc_max_sleep_time This tuning parameter controls the maximum sleep + time for the garbage collection thread. Time is + in milliseconds. + + gc_min_sleep_time This tuning parameter controls the minimum sleep + time for the garbage collection thread. Time is + in milliseconds. + + gc_no_gc_sleep_time This tuning parameter controls the default sleep + time for the garbage collection thread. Time is + in milliseconds. + + gc_idle This parameter controls the selection of victim + policy for garbage collection. Setting gc_idle = 0 + (default) will disable this option. Setting + gc_idle = 1 will select the Cost Benefit approach + & setting gc_idle = 2 will select the greedy aproach. + + reclaim_segments This parameter controls the number of prefree + segments to be reclaimed. If the number of prefree + segments is larger than the number of segments + in the proportion to the percentage over total + volume size, f2fs tries to conduct checkpoint to + reclaim the prefree segments to free segments. + By default, 5% over total # of segments. + + max_small_discards This parameter controls the number of discard + commands that consist small blocks less than 2MB. + The candidates to be discarded are cached until + checkpoint is triggered, and issued during the + checkpoint. By default, it is disabled with 0. + + trim_sections This parameter controls the number of sections + to be trimmed out in batch mode when FITRIM + conducts. 32 sections is set by default. + + ipu_policy This parameter controls the policy of in-place + updates in f2fs. There are five policies: + 0x01: F2FS_IPU_FORCE, 0x02: F2FS_IPU_SSR, + 0x04: F2FS_IPU_UTIL, 0x08: F2FS_IPU_SSR_UTIL, + 0x10: F2FS_IPU_FSYNC. + + min_ipu_util This parameter controls the threshold to trigger + in-place-updates. The number indicates percentage + of the filesystem utilization, and used by + F2FS_IPU_UTIL and F2FS_IPU_SSR_UTIL policies. + + min_fsync_blocks This parameter controls the threshold to trigger + in-place-updates when F2FS_IPU_FSYNC mode is set. + The number indicates the number of dirty pages + when fsync needs to flush on its call path. If + the number is less than this value, it triggers + in-place-updates. + + max_victim_search This parameter controls the number of trials to + find a victim segment when conducting SSR and + cleaning operations. The default value is 4096 + which covers 8GB block address range. + + dir_level This parameter controls the directory level to + support large directory. If a directory has a + number of files, it can reduce the file lookup + latency by increasing this dir_level value. + Otherwise, it needs to decrease this value to + reduce the space overhead. The default value is 0. + + ram_thresh This parameter controls the memory footprint used + by free nids and cached nat entries. By default, + 10 is set, which indicates 10 MB / 1 GB RAM. + +================================================================================ +USAGE +================================================================================ + +1. Download userland tools and compile them. + +2. Skip, if f2fs was compiled statically inside kernel. + Otherwise, insert the f2fs.ko module. + # insmod f2fs.ko + +3. Create a directory trying to mount + # mkdir /mnt/f2fs + +4. Format the block device, and then mount as f2fs + # mkfs.f2fs -l label /dev/block_device + # mount -t f2fs /dev/block_device /mnt/f2fs + +mkfs.f2fs +--------- +The mkfs.f2fs is for the use of formatting a partition as the f2fs filesystem, +which builds a basic on-disk layout. + +The options consist of: +-l [label] : Give a volume label, up to 512 unicode name. +-a [0 or 1] : Split start location of each area for heap-based allocation. + 1 is set by default, which performs this. +-o [int] : Set overprovision ratio in percent over volume size. + 5 is set by default. +-s [int] : Set the number of segments per section. + 1 is set by default. +-z [int] : Set the number of sections per zone. + 1 is set by default. +-e [str] : Set basic extension list. e.g. "mp3,gif,mov" +-t [0 or 1] : Disable discard command or not. + 1 is set by default, which conducts discard. + +fsck.f2fs +--------- +The fsck.f2fs is a tool to check the consistency of an f2fs-formatted +partition, which examines whether the filesystem metadata and user-made data +are cross-referenced correctly or not. +Note that, initial version of the tool does not fix any inconsistency. + +The options consist of: + -d debug level [default:0] + +dump.f2fs +--------- +The dump.f2fs shows the information of specific inode and dumps SSA and SIT to +file. Each file is dump_ssa and dump_sit. + +The dump.f2fs is used to debug on-disk data structures of the f2fs filesystem. +It shows on-disk inode information reconized by a given inode number, and is +able to dump all the SSA and SIT entries into predefined files, ./dump_ssa and +./dump_sit respectively. + +The options consist of: + -d debug level [default:0] + -i inode no (hex) + -s [SIT dump segno from #1~#2 (decimal), for all 0~-1] + -a [SSA dump segno from #1~#2 (decimal), for all 0~-1] + +Examples: +# dump.f2fs -i [ino] /dev/sdx +# dump.f2fs -s 0~-1 /dev/sdx (SIT dump) +# dump.f2fs -a 0~-1 /dev/sdx (SSA dump) + +================================================================================ +DESIGN +================================================================================ + +On-disk Layout +-------------- + +F2FS divides the whole volume into a number of segments, each of which is fixed +to 2MB in size. A section is composed of consecutive segments, and a zone +consists of a set of sections. By default, section and zone sizes are set to one +segment size identically, but users can easily modify the sizes by mkfs. + +F2FS splits the entire volume into six areas, and all the areas except superblock +consists of multiple segments as described below. + + align with the zone size <-| + |-> align with the segment size + _________________________________________________________________________ + | | | Segment | Node | Segment | | + | Superblock | Checkpoint | Info. | Address | Summary | Main | + | (SB) | (CP) | Table (SIT) | Table (NAT) | Area (SSA) | | + |____________|_____2______|______N______|______N______|______N_____|__N___| + . . + . . + . . + ._________________________________________. + |_Segment_|_..._|_Segment_|_..._|_Segment_| + . . + ._________._________ + |_section_|__...__|_ + . . + .________. + |__zone__| + +- Superblock (SB) + : It is located at the beginning of the partition, and there exist two copies + to avoid file system crash. It contains basic partition information and some + default parameters of f2fs. + +- Checkpoint (CP) + : It contains file system information, bitmaps for valid NAT/SIT sets, orphan + inode lists, and summary entries of current active segments. + +- Segment Information Table (SIT) + : It contains segment information such as valid block count and bitmap for the + validity of all the blocks. + +- Node Address Table (NAT) + : It is composed of a block address table for all the node blocks stored in + Main area. + +- Segment Summary Area (SSA) + : It contains summary entries which contains the owner information of all the + data and node blocks stored in Main area. + +- Main Area + : It contains file and directory data including their indices. + +In order to avoid misalignment between file system and flash-based storage, F2FS +aligns the start block address of CP with the segment size. Also, it aligns the +start block address of Main area with the zone size by reserving some segments +in SSA area. + +Reference the following survey for additional technical details. +https://wiki.linaro.org/WorkingGroups/Kernel/Projects/FlashCardSurvey + +File System Metadata Structure +------------------------------ + +F2FS adopts the checkpointing scheme to maintain file system consistency. At +mount time, F2FS first tries to find the last valid checkpoint data by scanning +CP area. In order to reduce the scanning time, F2FS uses only two copies of CP. +One of them always indicates the last valid data, which is called as shadow copy +mechanism. In addition to CP, NAT and SIT also adopt the shadow copy mechanism. + +For file system consistency, each CP points to which NAT and SIT copies are +valid, as shown as below. + + +--------+----------+---------+ + | CP | SIT | NAT | + +--------+----------+---------+ + . . . . + . . . . + . . . . + +-------+-------+--------+--------+--------+--------+ + | CP #0 | CP #1 | SIT #0 | SIT #1 | NAT #0 | NAT #1 | + +-------+-------+--------+--------+--------+--------+ + | ^ ^ + | | | + `----------------------------------------' + +Index Structure +--------------- + +The key data structure to manage the data locations is a "node". Similar to +traditional file structures, F2FS has three types of node: inode, direct node, +indirect node. F2FS assigns 4KB to an inode block which contains 923 data block +indices, two direct node pointers, two indirect node pointers, and one double +indirect node pointer as described below. One direct node block contains 1018 +data blocks, and one indirect node block contains also 1018 node blocks. Thus, +one inode block (i.e., a file) covers: + + 4KB * (923 + 2 * 1018 + 2 * 1018 * 1018 + 1018 * 1018 * 1018) := 3.94TB. + + Inode block (4KB) + |- data (923) + |- direct node (2) + | `- data (1018) + |- indirect node (2) + | `- direct node (1018) + | `- data (1018) + `- double indirect node (1) + `- indirect node (1018) + `- direct node (1018) + `- data (1018) + +Note that, all the node blocks are mapped by NAT which means the location of +each node is translated by the NAT table. In the consideration of the wandering +tree problem, F2FS is able to cut off the propagation of node updates caused by +leaf data writes. + +Directory Structure +------------------- + +A directory entry occupies 11 bytes, which consists of the following attributes. + +- hash hash value of the file name +- ino inode number +- len the length of file name +- type file type such as directory, symlink, etc + +A dentry block consists of 214 dentry slots and file names. Therein a bitmap is +used to represent whether each dentry is valid or not. A dentry block occupies +4KB with the following composition. + + Dentry Block(4 K) = bitmap (27 bytes) + reserved (3 bytes) + + dentries(11 * 214 bytes) + file name (8 * 214 bytes) + + [Bucket] + +--------------------------------+ + |dentry block 1 | dentry block 2 | + +--------------------------------+ + . . + . . + . [Dentry Block Structure: 4KB] . + +--------+----------+----------+------------+ + | bitmap | reserved | dentries | file names | + +--------+----------+----------+------------+ + [Dentry Block: 4KB] . . + . . + . . + +------+------+-----+------+ + | hash | ino | len | type | + +------+------+-----+------+ + [Dentry Structure: 11 bytes] + +F2FS implements multi-level hash tables for directory structure. Each level has +a hash table with dedicated number of hash buckets as shown below. Note that +"A(2B)" means a bucket includes 2 data blocks. + +---------------------- +A : bucket +B : block +N : MAX_DIR_HASH_DEPTH +---------------------- + +level #0 | A(2B) + | +level #1 | A(2B) - A(2B) + | +level #2 | A(2B) - A(2B) - A(2B) - A(2B) + . | . . . . +level #N/2 | A(2B) - A(2B) - A(2B) - A(2B) - A(2B) - ... - A(2B) + . | . . . . +level #N | A(4B) - A(4B) - A(4B) - A(4B) - A(4B) - ... - A(4B) + +The number of blocks and buckets are determined by, + + ,- 2, if n < MAX_DIR_HASH_DEPTH / 2, + # of blocks in level #n = | + `- 4, Otherwise + + ,- 2^(n + dir_level), + | if n + dir_level < MAX_DIR_HASH_DEPTH / 2, + # of buckets in level #n = | + `- 2^((MAX_DIR_HASH_DEPTH / 2) - 1), + Otherwise + +When F2FS finds a file name in a directory, at first a hash value of the file +name is calculated. Then, F2FS scans the hash table in level #0 to find the +dentry consisting of the file name and its inode number. If not found, F2FS +scans the next hash table in level #1. In this way, F2FS scans hash tables in +each levels incrementally from 1 to N. In each levels F2FS needs to scan only +one bucket determined by the following equation, which shows O(log(# of files)) +complexity. + + bucket number to scan in level #n = (hash value) % (# of buckets in level #n) + +In the case of file creation, F2FS finds empty consecutive slots that cover the +file name. F2FS searches the empty slots in the hash tables of whole levels from +1 to N in the same way as the lookup operation. + +The following figure shows an example of two cases holding children. + --------------> Dir <-------------- + | | + child child + + child - child [hole] - child + + child - child - child [hole] - [hole] - child + + Case 1: Case 2: + Number of children = 6, Number of children = 3, + File size = 7 File size = 7 + +Default Block Allocation +------------------------ + +At runtime, F2FS manages six active logs inside "Main" area: Hot/Warm/Cold node +and Hot/Warm/Cold data. + +- Hot node contains direct node blocks of directories. +- Warm node contains direct node blocks except hot node blocks. +- Cold node contains indirect node blocks +- Hot data contains dentry blocks +- Warm data contains data blocks except hot and cold data blocks +- Cold data contains multimedia data or migrated data blocks + +LFS has two schemes for free space management: threaded log and copy-and-compac- +tion. The copy-and-compaction scheme which is known as cleaning, is well-suited +for devices showing very good sequential write performance, since free segments +are served all the time for writing new data. However, it suffers from cleaning +overhead under high utilization. Contrarily, the threaded log scheme suffers +from random writes, but no cleaning process is needed. F2FS adopts a hybrid +scheme where the copy-and-compaction scheme is adopted by default, but the +policy is dynamically changed to the threaded log scheme according to the file +system status. + +In order to align F2FS with underlying flash-based storage, F2FS allocates a +segment in a unit of section. F2FS expects that the section size would be the +same as the unit size of garbage collection in FTL. Furthermore, with respect +to the mapping granularity in FTL, F2FS allocates each section of the active +logs from different zones as much as possible, since FTL can write the data in +the active logs into one allocation unit according to its mapping granularity. + +Cleaning process +---------------- + +F2FS does cleaning both on demand and in the background. On-demand cleaning is +triggered when there are not enough free segments to serve VFS calls. Background +cleaner is operated by a kernel thread, and triggers the cleaning job when the +system is idle. + +F2FS supports two victim selection policies: greedy and cost-benefit algorithms. +In the greedy algorithm, F2FS selects a victim segment having the smallest number +of valid blocks. In the cost-benefit algorithm, F2FS selects a victim segment +according to the segment age and the number of valid blocks in order to address +log block thrashing problem in the greedy algorithm. F2FS adopts the greedy +algorithm for on-demand cleaner, while background cleaner adopts cost-benefit +algorithm. + +In order to identify whether the data in the victim segment are valid or not, +F2FS manages a bitmap. Each bit represents the validity of a block, and the +bitmap is composed of a bit stream covering whole blocks in main area. diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt index 87fb3e9ccd7..8051b91296c 100644 --- a/Documentation/filesystems/proc.txt +++ b/Documentation/filesystems/proc.txt @@ -445,6 +445,10 @@ To clear the bits for the file mapped pages associated with the process > echo 3 > /proc/PID/clear_refs Any other value written to /proc/PID/clear_refs will have no effect. +To reset the peak resident set size ("high water mark") to the process's +current value: + > echo 5 > /proc/PID/clear_refs + The /proc/pid/pagemap gives the PFN, which can be used to find the pageflags using /proc/kpageflags and number of times a page is mapped using /proc/kpagecount. For detailed explanation, see Documentation/vm/pagemap.txt. diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt index 0d049202808..b2aa722e5ea 100644 --- a/Documentation/filesystems/vfs.txt +++ b/Documentation/filesystems/vfs.txt @@ -364,6 +364,7 @@ struct inode_operations { ssize_t (*listxattr) (struct dentry *, char *, size_t); int (*removexattr) (struct dentry *, const char *); void (*truncate_range)(struct inode *, loff_t, loff_t); + void (*update_time)(struct inode *, struct timespec *, int); }; Again, all methods are called without any locks being held, unless @@ -475,6 +476,9 @@ otherwise noted. truncate_range: a method provided by the underlying filesystem to truncate a range of blocks , i.e. punch a hole somewhere in a file. + update_time: called by the VFS to update a specific time or the i_version of + an inode. If this is not defined the VFS will update the inode itself + and call mark_inode_dirty_sync. The Address Space Object ======================== diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt index 4af594604e9..c1d563479dc 100644 --- a/Documentation/networking/ip-sysctl.txt +++ b/Documentation/networking/ip-sysctl.txt @@ -1204,6 +1204,13 @@ router_solicitations - INTEGER routers are present. Default: 3 +use_oif_addrs_only - BOOLEAN + When enabled, the candidate source addresses for destinations + routed via this interface are restricted to the set of addresses + configured on this interface (vis. RFC 6724, section 4). + + Default: false + use_tempaddr - INTEGER Preference for Privacy Extensions (RFC3041). <= 0 : disable Privacy Extensions @@ -1279,6 +1286,19 @@ force_tllao - BOOLEAN race condition where the sender deletes the cached link-layer address prior to receiving a response to a previous solicitation." +optimistic_dad - BOOLEAN + Whether to perform Optimistic Duplicate Address Detection (RFC 4429). + 0: disabled (default) + 1: enabled + +use_optimistic - BOOLEAN + If enabled, do not classify optimistic addresses as deprecated during + source address selection. Preferred addresses will still be chosen + before optimistic addresses, subject to other ranking in the source + address selection algorithm. + 0: disabled (default) + 1: enabled + icmp/*: ratelimit - INTEGER Limit the maximal rates for sending ICMPv6 packets. diff --git a/Documentation/prctl/seccomp_filter.txt b/Documentation/prctl/seccomp_filter.txt new file mode 100644 index 00000000000..597c3c58137 --- /dev/null +++ b/Documentation/prctl/seccomp_filter.txt @@ -0,0 +1,163 @@ + SECure COMPuting with filters + ============================= + +Introduction +------------ + +A large number of system calls are exposed to every userland process +with many of them going unused for the entire lifetime of the process. +As system calls change and mature, bugs are found and eradicated. A +certain subset of userland applications benefit by having a reduced set +of available system calls. The resulting set reduces the total kernel +surface exposed to the application. System call filtering is meant for +use with those applications. + +Seccomp filtering provides a means for a process to specify a filter for +incoming system calls. The filter is expressed as a Berkeley Packet +Filter (BPF) program, as with socket filters, except that the data +operated on is related to the system call being made: system call +number and the system call arguments. This allows for expressive +filtering of system calls using a filter program language with a long +history of being exposed to userland and a straightforward data set. + +Additionally, BPF makes it impossible for users of seccomp to fall prey +to time-of-check-time-of-use (TOCTOU) attacks that are common in system +call interposition frameworks. BPF programs may not dereference +pointers which constrains all filters to solely evaluating the system +call arguments directly. + +What it isn't +------------- + +System call filtering isn't a sandbox. It provides a clearly defined +mechanism for minimizing the exposed kernel surface. It is meant to be +a tool for sandbox developers to use. Beyond that, policy for logical +behavior and information flow should be managed with a combination of +other system hardening techniques and, potentially, an LSM of your +choosing. Expressive, dynamic filters provide further options down this +path (avoiding pathological sizes or selecting which of the multiplexed +system calls in socketcall() is allowed, for instance) which could be +construed, incorrectly, as a more complete sandboxing solution. + +Usage +----- + +An additional seccomp mode is added and is enabled using the same +prctl(2) call as the strict seccomp. If the architecture has +CONFIG_HAVE_ARCH_SECCOMP_FILTER, then filters may be added as below: + +PR_SET_SECCOMP: + Now takes an additional argument which specifies a new filter + using a BPF program. + The BPF program will be executed over struct seccomp_data + reflecting the system call number, arguments, and other + metadata. The BPF program must then return one of the + acceptable values to inform the kernel which action should be + taken. + + Usage: + prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, prog); + + The 'prog' argument is a pointer to a struct sock_fprog which + will contain the filter program. If the program is invalid, the + call will return -1 and set errno to EINVAL. + + If fork/clone and execve are allowed by @prog, any child + processes will be constrained to the same filters and system + call ABI as the parent. + + Prior to use, the task must call prctl(PR_SET_NO_NEW_PRIVS, 1) or + run with CAP_SYS_ADMIN privileges in its namespace. If these are not + true, -EACCES will be returned. This requirement ensures that filter + programs cannot be applied to child processes with greater privileges + than the task that installed them. + + Additionally, if prctl(2) is allowed by the attached filter, + additional filters may be layered on which will increase evaluation + time, but allow for further decreasing the attack surface during + execution of a process. + +The above call returns 0 on success and non-zero on error. + +Return values +------------- +A seccomp filter may return any of the following values. If multiple +filters exist, the return value for the evaluation of a given system +call will always use the highest precedent value. (For example, +SECCOMP_RET_KILL will always take precedence.) + +In precedence order, they are: + +SECCOMP_RET_KILL: + Results in the task exiting immediately without executing the + system call. The exit status of the task (status & 0x7f) will + be SIGSYS, not SIGKILL. + +SECCOMP_RET_TRAP: + Results in the kernel sending a SIGSYS signal to the triggering + task without executing the system call. The kernel will + rollback the register state to just before the system call + entry such that a signal handler in the task will be able to + inspect the ucontext_t->uc_mcontext registers and emulate + system call success or failure upon return from the signal + handler. + + The SECCOMP_RET_DATA portion of the return value will be passed + as si_errno. + + SIGSYS triggered by seccomp will have a si_code of SYS_SECCOMP. + +SECCOMP_RET_ERRNO: + Results in the lower 16-bits of the return value being passed + to userland as the errno without executing the system call. + +SECCOMP_RET_TRACE: + When returned, this value will cause the kernel to attempt to + notify a ptrace()-based tracer prior to executing the system + call. If there is no tracer present, -ENOSYS is returned to + userland and the system call is not executed. + + A tracer will be notified if it requests PTRACE_O_TRACESECCOMP + using ptrace(PTRACE_SETOPTIONS). The tracer will be notified + of a PTRACE_EVENT_SECCOMP and the SECCOMP_RET_DATA portion of + the BPF program return value will be available to the tracer + via PTRACE_GETEVENTMSG. + +SECCOMP_RET_ALLOW: + Results in the system call being executed. + +If multiple filters exist, the return value for the evaluation of a +given system call will always use the highest precedent value. + +Precedence is only determined using the SECCOMP_RET_ACTION mask. When +multiple filters return values of the same precedence, only the +SECCOMP_RET_DATA from the most recently installed filter will be +returned. + +Pitfalls +-------- + +The biggest pitfall to avoid during use is filtering on system call +number without checking the architecture value. Why? On any +architecture that supports multiple system call invocation conventions, +the system call numbers may vary based on the specific invocation. If +the numbers in the different calling conventions overlap, then checks in +the filters may be abused. Always check the arch value! + +Example +------- + +The samples/seccomp/ directory contains both an x86-specific example +and a more generic example of a higher level macro interface for BPF +program generation. + + + +Adding architecture support +----------------------- + +See arch/Kconfig for the authoritative requirements. In general, if an +architecture supports both ptrace_event and seccomp, it will be able to +support seccomp filter with minor fixup: SIGSYS support and seccomp return +value checking. Then it must just add CONFIG_HAVE_ARCH_SECCOMP_FILTER +to its arch-specific Kconfig. diff --git a/Documentation/sysctl/fs.txt b/Documentation/sysctl/fs.txt index 88fd7f5c8dc..9330fba7d35 100644 --- a/Documentation/sysctl/fs.txt +++ b/Documentation/sysctl/fs.txt @@ -32,6 +32,8 @@ Currently, these files are in /proc/sys/fs: - nr_open - overflowuid - overflowgid +- pipe-user-pages-hard +- pipe-user-pages-soft - suid_dumpable - super-max - super-nr @@ -157,6 +159,27 @@ The default is 65534. ============================================================== +pipe-user-pages-hard: + +Maximum total number of pages a non-privileged user may allocate for pipes. +Once this limit is reached, no new pipes may be allocated until usage goes +below the limit again. When set to 0, no limit is applied, which is the default +setting. + +============================================================== + +pipe-user-pages-soft: + +Maximum total number of pages a non-privileged user may allocate for pipes +before the pipe size gets limited to a single page. Once this limit is reached, +new pipes will be limited to a single page in size for this user in order to +limit total memory usage, and trying to increase them using fcntl() will be +denied until usage goes below the limit again. The default value allows to +allocate up to 1024 pipes at their default size. When set to 0, no limit is +applied. + +============================================================== + suid_dumpable: This value can be used to query and set the core dump mode for setuid diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt index a419f5e3143..86b8bc503af 100644 --- a/Documentation/sysctl/kernel.txt +++ b/Documentation/sysctl/kernel.txt @@ -49,8 +49,9 @@ show up in /proc/sys/kernel: - overflowuid - panic - panic_on_oops -- panic_on_unrecovered_nmi - panic_on_stackoverflow +- panic_on_unrecovered_nmi +- perf_event_paranoid - pid_max - powersave-nap [ PPC only ] - printk @@ -384,19 +385,6 @@ the recommended setting is 60. ============================================================== -panic_on_unrecovered_nmi: - -The default Linux behaviour on an NMI of either memory or unknown is -to continue operation. For many environments such as scientific -computing it is preferable that the box is taken out and the error -dealt with than an uncorrected parity/ECC error get propagated. - -A small number of systems do generate NMI's for bizarre random reasons -such as power management so the default is off. That sysctl works like -the existing panic controls already in that directory. - -============================================================== - panic_on_oops: Controls the kernel's behaviour when an oops or BUG is encountered. @@ -420,6 +408,32 @@ This file shows up if CONFIG_DEBUG_STACKOVERFLOW is enabled. ============================================================== +panic_on_unrecovered_nmi: + +The default Linux behaviour on an NMI of either memory or unknown is +to continue operation. For many environments such as scientific +computing it is preferable that the box is taken out and the error +dealt with than an uncorrected parity/ECC error get propagated. + +A small number of systems do generate NMI's for bizarre random reasons +such as power management so the default is off. That sysctl works like +the existing panic controls already in that directory. + +============================================================== + +perf_event_paranoid: + +Controls use of the performance events system by unprivileged +users (without CAP_SYS_ADMIN). The default value is 3 if +CONFIG_SECURITY_PERF_EVENTS_RESTRICT is set, or 1 otherwise. + + -1: Allow use of (almost) all events by all users +>=0: Disallow raw tracepoint access by users without CAP_IOC_LOCK +>=1: Disallow CPU event access by users without CAP_SYS_ADMIN +>=2: Disallow kernel profiling by users without CAP_SYS_ADMIN +>=3: Disallow all event access by users without CAP_SYS_ADMIN + +============================================================== pid_max: diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt index 96f0ee825be..5f96262a09a 100644 --- a/Documentation/sysctl/vm.txt +++ b/Documentation/sysctl/vm.txt @@ -40,6 +40,8 @@ Currently, these files are in /proc/sys/vm: - min_slab_ratio - min_unmapped_ratio - mmap_min_addr +- mmap_rnd_bits +- mmap_rnd_compat_bits - nr_hugepages - nr_overcommit_hugepages - nr_pdflush_threads @@ -409,6 +411,33 @@ against future potential kernel bugs. ============================================================== +mmap_rnd_bits: + +This value can be used to select the number of bits to use to +determine the random offset to the base address of vma regions +resulting from mmap allocations on architectures which support +tuning address space randomization. This value will be bounded +by the architecture's minimum and maximum supported values. + +This value can be changed after boot using the +/proc/sys/vm/mmap_rnd_bits tunable + +============================================================== + +mmap_rnd_compat_bits: + +This value can be used to select the number of bits to use to +determine the random offset to the base address of vma regions +resulting from mmap allocations for applications run in +compatibility mode on architectures which support tuning address +space randomization. This value will be bounded by the +architecture's minimum and maximum supported values. + +This value can be changed after boot using the +/proc/sys/vm/mmap_rnd_compat_bits tunable + +============================================================== + nr_hugepages Change the minimum size of the hugepage pool. diff --git a/MAINTAINERS b/MAINTAINERS index 5f8ab499444..5968f291422 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2688,6 +2688,20 @@ S: Maintained F: drivers/video/exynos/exynos_mipi* F: include/video/exynos_mipi* +F2FS FILE SYSTEM +M: Jaegeuk Kim +M: Changman Lee +R: Chao Yu +L: linux-f2fs-devel@lists.sourceforge.net +W: http://en.wikipedia.org/wiki/F2FS +T: git git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git +S: Maintained +F: Documentation/filesystems/f2fs.txt +F: Documentation/ABI/testing/sysfs-fs-f2fs +F: fs/f2fs/ +F: include/linux/f2fs_fs.h +F: include/trace/events/f2fs.h + F71805F HARDWARE MONITORING DRIVER M: Jean Delvare L: lm-sensors@lm-sensors.org @@ -5937,6 +5951,16 @@ S: Maintained F: drivers/mmc/host/sdhci.* F: drivers/mmc/host/sdhci-pltfm.[ch] +SECURE COMPUTING +M: Kees Cook +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp +S: Supported +F: kernel/seccomp.c +F: include/uapi/linux/seccomp.h +F: include/linux/seccomp.h +K: \bsecure_computing +K: \bTIF_SECCOMP\b + SECURE DIGITAL HOST CONTROLLER INTERFACE, OPEN FIRMWARE BINDINGS (SDHCI-OF) M: Anton Vorontsov L: linuxppc-dev@lists.ozlabs.org diff --git a/android/configs/android-base.cfg b/android/configs/android-base.cfg index 225f3e28590..4630fc632ce 100644 --- a/android/configs/android-base.cfg +++ b/android/configs/android-base.cfg @@ -129,6 +129,10 @@ CONFIG_PREEMPT=y CONFIG_RESOURCE_COUNTERS=y CONFIG_RTC_CLASS=y CONFIG_RT_GROUP_SCHED=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y +CONFIG_SECURITY_SELINUX=y CONFIG_STAGING=y CONFIG_SWITCH=y CONFIG_SYNC=y diff --git a/arch/Kconfig b/arch/Kconfig index 0a3ffe46e56..569d1b2aea1 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -219,4 +219,96 @@ config HAVE_CMPXCHG_DOUBLE config ARCH_WANT_OLD_COMPAT_IPC bool +config HAVE_ARCH_SECCOMP_FILTER + bool + help + An arch should select this symbol if it provides all of these things: + - syscall_get_arch() + - syscall_get_arguments() + - syscall_rollback() + - syscall_set_return_value() + - SIGSYS siginfo_t support + - secure_computing is called from a ptrace_event()-safe context + - secure_computing return value is checked and a return value of -1 + results in the system call being skipped immediately. + - seccomp syscall wired up + +config SECCOMP_FILTER + def_bool y + depends on HAVE_ARCH_SECCOMP_FILTER && SECCOMP && NET + help + Enable tasks to build secure computing environments defined + in terms of Berkeley Packet Filter programs which implement + task-defined system call filtering polices. + + See Documentation/prctl/seccomp_filter.txt for details. + +config HAVE_ARCH_MMAP_RND_BITS + bool + help + An arch should select this symbol if it supports setting a variable + number of bits for use in establishing the base address for mmap + allocations, has MMU enabled and provides values for both: + - ARCH_MMAP_RND_BITS_MIN + - ARCH_MMAP_RND_BITS_MAX + +config ARCH_MMAP_RND_BITS_MIN + int + +config ARCH_MMAP_RND_BITS_MAX + int + +config ARCH_MMAP_RND_BITS_DEFAULT + int + +config ARCH_MMAP_RND_BITS + int "Number of bits to use for ASLR of mmap base address" if EXPERT + range ARCH_MMAP_RND_BITS_MIN ARCH_MMAP_RND_BITS_MAX + default ARCH_MMAP_RND_BITS_DEFAULT if ARCH_MMAP_RND_BITS_DEFAULT + default ARCH_MMAP_RND_BITS_MIN + depends on HAVE_ARCH_MMAP_RND_BITS + help + This value can be used to select the number of bits to use to + determine the random offset to the base address of vma regions + resulting from mmap allocations. This value will be bounded + by the architecture's minimum and maximum supported values. + + This value can be changed after boot using the + /proc/sys/vm/mmap_rnd_bits tunable + +config HAVE_ARCH_MMAP_RND_COMPAT_BITS + bool + help + An arch should select this symbol if it supports running applications + in compatibility mode, supports setting a variable number of bits for + use in establishing the base address for mmap allocations, has MMU + enabled and provides values for both: + - ARCH_MMAP_RND_COMPAT_BITS_MIN + - ARCH_MMAP_RND_COMPAT_BITS_MAX + +config ARCH_MMAP_RND_COMPAT_BITS_MIN + int + +config ARCH_MMAP_RND_COMPAT_BITS_MAX + int + +config ARCH_MMAP_RND_COMPAT_BITS_DEFAULT + int + +config ARCH_MMAP_RND_COMPAT_BITS + int "Number of bits to use for ASLR of mmap base address for compatible applications" if EXPERT + range ARCH_MMAP_RND_COMPAT_BITS_MIN ARCH_MMAP_RND_COMPAT_BITS_MAX + default ARCH_MMAP_RND_COMPAT_BITS_DEFAULT if ARCH_MMAP_RND_COMPAT_BITS_DEFAULT + default ARCH_MMAP_RND_COMPAT_BITS_MIN + depends on HAVE_ARCH_MMAP_RND_COMPAT_BITS + help + This value can be used to select the number of bits to use to + determine the random offset to the base address of vma regions + resulting from mmap allocations for compatible applications This + value will be bounded by the architecture's minimum and maximum + supported values. + + This value can be changed after boot using the + /proc/sys/vm/mmap_rnd_compat_bits tunable + source "kernel/gcov/Kconfig" diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 69b2cd2a5ec..ad76d6c7f0c 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -16,6 +16,7 @@ config ARM select HAVE_KPROBES if !XIP_KERNEL select HAVE_KRETPROBES if (HAVE_KPROBES) select HAVE_FUNCTION_TRACER if (!XIP_KERNEL) + select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_FTRACE_MCOUNT_RECORD if (!XIP_KERNEL) select HAVE_DYNAMIC_FTRACE if (!XIP_KERNEL) select HAVE_FUNCTION_GRAPH_TRACER if (!THUMB2_KERNEL) @@ -32,6 +33,8 @@ config ARM select HAVE_HW_BREAKPOINT if (PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7)) select HAVE_C_RECORDMCOUNT select HAVE_GENERIC_HARDIRQS + select HAVE_SPARSE_IRQ + select HAVE_ARCH_SECCOMP_FILTER select GENERIC_IRQ_SHOW select CPU_PM if (SUSPEND || CPU_IDLE) select GENERIC_PCI_IOMAP @@ -303,6 +306,14 @@ config MMU Select if you want MMU-based virtualised addressing space support by paged memory management. If unsure, say 'Y'. +config ARCH_MMAP_RND_BITS_MIN + default 8 + +config ARCH_MMAP_RND_BITS_MAX + default 14 if PAGE_OFFSET=0x40000000 + default 15 if PAGE_OFFSET=0x80000000 + default 16 + # # The "ARM system type" choice list is ordered alphabetically by option # text. Please add new entries in the option alphabetic order. diff --git a/arch/arm/Makefile b/arch/arm/Makefile index a1f42d95907..4e240ccf0c5 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -251,6 +251,7 @@ core-$(CONFIG_VFP) += arch/arm/vfp/ # If we have a machine-specific directory, then include it in the build. core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/ core-y += arch/arm/net/ +core-y += arch/arm/crypto/ core-y += $(machdirs) $(platdirs) drivers-$(CONFIG_OPROFILE) += arch/arm/oprofile/ diff --git a/arch/arm/boot/dts/msm8974-hammerhead/msm8974-hammerhead-misc.dtsi b/arch/arm/boot/dts/msm8974-hammerhead/msm8974-hammerhead-misc.dtsi index ff71ce73950..99d150ca774 100644 --- a/arch/arm/boot/dts/msm8974-hammerhead/msm8974-hammerhead-misc.dtsi +++ b/arch/arm/boot/dts/msm8974-hammerhead/msm8974-hammerhead-misc.dtsi @@ -15,7 +15,9 @@ compatible = "msm,pwm_vibrator"; haptic-pwr-gpio = <&msmgpio 60 0x00>; motor-pwm-gpio = <&msmgpio 27 0x00>; - motor-amp = <63>; + vtg_default = <63>; + vtg_min = <0>; + vtg_max = <100>; n-value = <127>; vibe-warmup-delay = <20>; vibe-braking-gain = <100>; diff --git a/arch/arm/boot/dts/msm8974-v1-pm.dtsi b/arch/arm/boot/dts/msm8974-v1-pm.dtsi index 6041b3affcf..63dfa01ce71 100644 --- a/arch/arm/boot/dts/msm8974-v1-pm.dtsi +++ b/arch/arm/boot/dts/msm8974-v1-pm.dtsi @@ -451,4 +451,13 @@ reg-names = "phys_addr_base"; qcom,sleep-stats-version = <2>; }; + + qcom,rpm-master-stats@fc428150 { + compatible = "qcom,rpm-master-stats"; + reg = <0xfc428150 0x3200>; + qcom,masters = "APSS", "MPSS", "LPSS", "PRONTO"; + qcom,master-stats-version = <2>; + qcom,master-offset = <2560>; + }; + }; diff --git a/arch/arm/boot/dts/msm8974-v2-pm.dtsi b/arch/arm/boot/dts/msm8974-v2-pm.dtsi index da1d3b280d1..12487646aed 100644 --- a/arch/arm/boot/dts/msm8974-v2-pm.dtsi +++ b/arch/arm/boot/dts/msm8974-v2-pm.dtsi @@ -477,4 +477,12 @@ reg = <0xfc000000 0x1a0000>; qcom,start-offset = <0x190010>; }; + + qcom,rpm-master-stats@fc428150 { + compatible = "qcom,rpm-master-stats"; + reg = <0xfc428150 0x3200>; + qcom,masters = "APSS", "MPSS", "LPSS", "PRONTO"; + qcom,master-stats-version = <2>; + qcom,master-offset = <2560>; + }; }; diff --git a/arch/arm/boot/dts/msm8974.dtsi b/arch/arm/boot/dts/msm8974.dtsi index b23b37cfe88..226c0f85a66 100644 --- a/arch/arm/boot/dts/msm8974.dtsi +++ b/arch/arm/boot/dts/msm8974.dtsi @@ -34,24 +34,88 @@ device_type = "cpu"; compatible = "qcom,krait"; reg = <0x0>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 57900 //300000 kHz + 88200 //422400 kHz + 99600 //652800 kHz + 138800 //729600 kHz + 149600 //883200 kHz + 170200 //960000 kHz + 178300 //1036800 kHz + 189100 //1190400 kHz + 232100 //1267200 kHz + 256500 //1497600 kHz + 266400 //1574000 kHz + 287700 //1728000 kHz + 325700 //1958400 kHz + 386200>; //2265600 kHz }; CPU1: cpu@1 { device_type = "cpu"; compatible = "qcom,krait"; reg = <0x1>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 23739 //300000 kHz + 36162 //422400 kHz + 40836 //652800 kHz + 56908 //729600 kHz + 61335 //883200 kHz + 69782 //960000 kHz + 73103 //1036800 kHz + 77531 //1190400 kHz + 95161 //1267200 kHz + 105165 //1497600 kHz + 109224 //1574000 kHz + 117957 //1728000 kHz + 133537 //1958400 kHz + 158342>; //2265600 kHz }; CPU2: cpu@2 { device_type = "cpu"; compatible = "qcom,krait"; reg = <0x2>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 27213 //300000 kHz + 41454 //422400 kHz + 46812 //652800 kHz + 65235 //729600 kHz + 70312 //883200 kHz + 79994 //960000 kHz + 83801 //1036800 kHz + 88877 //1190400 kHz + 109087 //1267200 kHz + 120555 //1497600 kHz + 125208 //1574000 kHz + 135219 //1728000 kHz + 153079 //1958400 kHz + 181514>; //2265600 kHz }; CPU3: cpu@3 { device_type = "cpu"; compatible = "qcom,krait"; reg = <0x3>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 31266 //300000 kHz + 47628 //422400 kHz + 53784 //652800 kHz + 74952 //729600 kHz + 80784 //883200 kHz + 91908 //960000 kHz + 96282 //1036800 kHz + 102114 //1190400 kHz + 125334 //1267200 kHz + 138510 //1497600 kHz + 143856 //1574000 kHz + 155358 //1728000 kHz + 175878 //1958400 kHz + 208548>; //2265600 kHz }; }; diff --git a/arch/arm/common/gic.c b/arch/arm/common/gic.c index 296ccb331e6..f9b9e6797e7 100644 --- a/arch/arm/common/gic.c +++ b/arch/arm/common/gic.c @@ -251,13 +251,14 @@ static void gic_show_resume_irq(struct gic_chip_data *gic) enabled = readl_relaxed(base + GIC_DIST_ENABLE_CLEAR + i * 4); pending[i] = readl_relaxed(base + GIC_DIST_PENDING_SET + i * 4); pending[i] &= enabled; + pending[i] &= gic->wakeup_irqs[i]; } raw_spin_unlock(&irq_controller_lock); for (i = find_first_bit(pending, gic->max_irq); i < gic->max_irq; i = find_next_bit(pending, gic->max_irq, i+1)) { - log_wakeup_reason(i + gic->irq_offset); + log_base_wakeup_reason(i + gic->irq_offset); } } @@ -459,12 +460,13 @@ asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) } while (1); } -static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) +static bool gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) { struct gic_chip_data *chip_data = irq_get_handler_data(irq); struct irq_chip *chip = irq_get_chip(irq); unsigned int cascade_irq, gic_irq; unsigned long status; + int handled = false; chained_irq_enter(chip, desc); @@ -480,10 +482,11 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) if (unlikely(gic_irq < 32 || gic_irq > 1020)) do_bad_IRQ(cascade_irq, desc); else - generic_handle_irq(cascade_irq); + handled = generic_handle_irq(cascade_irq); out: chained_irq_exit(chip, desc); + return handled == true; } static struct irq_chip gic_chip = { diff --git a/arch/arm/configs/aopp_hammerhead_defconfig b/arch/arm/configs/aopp_hammerhead_defconfig new file mode 100644 index 00000000000..76cae9fb4b6 --- /dev/null +++ b/arch/arm/configs/aopp_hammerhead_defconfig @@ -0,0 +1,3596 @@ +# +# Automatically generated file; DO NOT EDIT. +# Linux/arm 3.4.0 Kernel Configuration +# +CONFIG_ARM=y +CONFIG_ARM_HAS_SG_CHAIN=y +CONFIG_SYS_SUPPORTS_APM_EMULATION=y +CONFIG_GENERIC_GPIO=y +# CONFIG_ARCH_USES_GETTIMEOFFSET is not set +CONFIG_GENERIC_CLOCKEVENTS=y +CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y +CONFIG_KTIME_SCALAR=y +CONFIG_HAVE_PROC_CPU=y +CONFIG_STACKTRACE_SUPPORT=y +CONFIG_LOCKDEP_SUPPORT=y +CONFIG_TRACE_IRQFLAGS_SUPPORT=y +CONFIG_HARDIRQS_SW_RESEND=y +CONFIG_GENERIC_IRQ_PROBE=y +CONFIG_ARM_TICKET_LOCKS=y +CONFIG_RWSEM_GENERIC_SPINLOCK=y +CONFIG_ARCH_HAS_CPUFREQ=y +CONFIG_ARCH_HAS_CPU_IDLE_WAIT=y +CONFIG_GENERIC_HWEIGHT=y +CONFIG_GENERIC_CALIBRATE_DELAY=y +CONFIG_NEED_DMA_MAP_STATE=y +CONFIG_ARCH_WANT_KMAP_ATOMIC_FLUSH=y +CONFIG_VECTORS_BASE=0xffff0000 +# CONFIG_ARM_PATCH_PHYS_VIRT is not set +CONFIG_NEED_MACH_IO_H=y +CONFIG_NEED_MACH_MEMORY_H=y +CONFIG_PHYS_OFFSET=0x00000000 +CONFIG_GENERIC_BUG=y +# CONFIG_GENERIC_TIME_VSYSCALL is not set +CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config" +CONFIG_HAVE_IRQ_WORK=y +CONFIG_IRQ_WORK=y + +# +# General setup +# +CONFIG_EXPERIMENTAL=y +CONFIG_INIT_ENV_ARG_LIMIT=32 +CONFIG_CROSS_COMPILE="" +CONFIG_LOCALVERSION="" +CONFIG_LOCALVERSION_AUTO=y +CONFIG_HAVE_KERNEL_GZIP=y +CONFIG_HAVE_KERNEL_LZMA=y +CONFIG_HAVE_KERNEL_XZ=y +CONFIG_HAVE_KERNEL_LZO=y +CONFIG_KERNEL_GZIP=y +# CONFIG_KERNEL_LZMA is not set +# CONFIG_KERNEL_XZ is not set +# CONFIG_KERNEL_LZO is not set +CONFIG_DEFAULT_HOSTNAME="(none)" +# CONFIG_SWAP is not set +# CONFIG_SYSVIPC is not set +# CONFIG_POSIX_MQUEUE is not set +# CONFIG_BSD_PROCESS_ACCT is not set +# CONFIG_FHANDLE is not set +# CONFIG_TASKSTATS is not set +CONFIG_AUDIT=y +CONFIG_AUDITSYSCALL=y +CONFIG_AUDIT_WATCH=y +CONFIG_AUDIT_TREE=y +# CONFIG_AUDIT_LOGINUID_IMMUTABLE is not set +CONFIG_HAVE_GENERIC_HARDIRQS=y + +# +# IRQ subsystem +# +CONFIG_GENERIC_HARDIRQS=y +CONFIG_MAY_HAVE_SPARSE_IRQ=y +CONFIG_GENERIC_IRQ_SHOW=y +CONFIG_IRQ_DOMAIN=y +# CONFIG_IRQ_DOMAIN_DEBUG is not set +CONFIG_SPARSE_IRQ=y + +# +# RCU Subsystem +# +CONFIG_TREE_PREEMPT_RCU=y +CONFIG_PREEMPT_RCU=y +CONFIG_RCU_FANOUT=32 +# CONFIG_RCU_FANOUT_EXACT is not set +CONFIG_RCU_FAST_NO_HZ=y +# CONFIG_TREE_RCU_TRACE is not set +# CONFIG_RCU_BOOST is not set +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=19 +CONFIG_CGROUPS=y +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +# CONFIG_CGROUP_DEVICE is not set +# CONFIG_CPUSETS is not set +CONFIG_CGROUP_CPUACCT=y +CONFIG_RESOURCE_COUNTERS=y +# CONFIG_CGROUP_MEM_RES_CTLR is not set +# CONFIG_CGROUP_PERF is not set +CONFIG_CGROUP_SCHED=y +CONFIG_FAIR_GROUP_SCHED=y +# CONFIG_CFS_BANDWIDTH is not set +CONFIG_RT_GROUP_SCHED=y +# CONFIG_BLK_CGROUP is not set +# CONFIG_CHECKPOINT_RESTORE is not set +CONFIG_NAMESPACES=y +# CONFIG_UTS_NS is not set +# CONFIG_USER_NS is not set +# CONFIG_PID_NS is not set +CONFIG_NET_NS=y +# CONFIG_SCHED_AUTOGROUP is not set +# CONFIG_SYSFS_DEPRECATED is not set +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_INITRAMFS_SOURCE="" +CONFIG_RD_GZIP=y +CONFIG_RD_BZIP2=y +CONFIG_RD_LZMA=y +# CONFIG_RD_XZ is not set +# CONFIG_RD_LZO is not set +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_SYSCTL=y +CONFIG_ANON_INODES=y +CONFIG_PANIC_TIMEOUT=5 +CONFIG_EXPERT=y +CONFIG_UID16=y +# CONFIG_SYSCTL_SYSCALL is not set +CONFIG_KALLSYMS=y +CONFIG_KALLSYMS_ALL=y +CONFIG_HOTPLUG=y +CONFIG_PRINTK=y +CONFIG_BUG=y +CONFIG_ELF_CORE=y +CONFIG_BASE_FULL=y +CONFIG_FUTEX=y +CONFIG_EPOLL=y +CONFIG_SIGNALFD=y +CONFIG_TIMERFD=y +CONFIG_EVENTFD=y +CONFIG_SHMEM=y +CONFIG_AIO=y +CONFIG_EMBEDDED=y +CONFIG_HAVE_PERF_EVENTS=y +CONFIG_PERF_USE_VMALLOC=y + +# +# Kernel Performance Events And Counters +# +CONFIG_PERF_EVENTS=y +# CONFIG_PERF_COUNTERS is not set +# CONFIG_DEBUG_PERF_USE_VMALLOC is not set +CONFIG_VM_EVENT_COUNTERS=y +CONFIG_SLUB_DEBUG=y +CONFIG_COMPAT_BRK=y +# CONFIG_SLAB is not set +CONFIG_SLUB=y +# CONFIG_SLOB is not set +CONFIG_PROFILING=y +CONFIG_TRACEPOINTS=y +# CONFIG_OPROFILE is not set +CONFIG_HAVE_OPROFILE=y +# CONFIG_JUMP_LABEL is not set +CONFIG_HAVE_KPROBES=y +CONFIG_HAVE_KRETPROBES=y +CONFIG_HAVE_DMA_ATTRS=y +CONFIG_HAVE_DMA_CONTIGUOUS=y +CONFIG_USE_GENERIC_SMP_HELPERS=y +CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y +CONFIG_HAVE_CLK=y +CONFIG_HAVE_DMA_API_DEBUG=y +CONFIG_HAVE_HW_BREAKPOINT=y +CONFIG_HAVE_ARCH_JUMP_LABEL=y +CONFIG_HAVE_ARCH_SECCOMP_FILTER=y +CONFIG_SECCOMP_FILTER=y +CONFIG_HAVE_ARCH_MMAP_RND_BITS=y +CONFIG_ARCH_MMAP_RND_BITS_MIN=8 +CONFIG_ARCH_MMAP_RND_BITS_MAX=16 +CONFIG_ARCH_MMAP_RND_BITS=8 + +# +# GCOV-based kernel profiling +# +# CONFIG_GCOV_KERNEL is not set +CONFIG_HAVE_GENERIC_DMA_COHERENT=y +CONFIG_SLABINFO=y +CONFIG_RT_MUTEXES=y +CONFIG_BASE_SMALL=0 +# CONFIG_MODULES is not set +CONFIG_STOP_MACHINE=y +CONFIG_BLOCK=y +CONFIG_LBDAF=y +CONFIG_BLK_DEV_BSG=y +# CONFIG_BLK_DEV_BSGLIB is not set +# CONFIG_BLK_DEV_INTEGRITY is not set + +# +# Partition Types +# +CONFIG_PARTITION_ADVANCED=y +# CONFIG_ACORN_PARTITION is not set +# CONFIG_OSF_PARTITION is not set +# CONFIG_AMIGA_PARTITION is not set +# CONFIG_ATARI_PARTITION is not set +# CONFIG_MAC_PARTITION is not set +CONFIG_MSDOS_PARTITION=y +# CONFIG_BSD_DISKLABEL is not set +# CONFIG_MINIX_SUBPARTITION is not set +# CONFIG_SOLARIS_X86_PARTITION is not set +# CONFIG_UNIXWARE_DISKLABEL is not set +# CONFIG_LDM_PARTITION is not set +# CONFIG_SGI_PARTITION is not set +# CONFIG_ULTRIX_PARTITION is not set +# CONFIG_SUN_PARTITION is not set +# CONFIG_KARMA_PARTITION is not set +CONFIG_EFI_PARTITION=y +# CONFIG_SYSV68_PARTITION is not set + +# +# IO Schedulers +# +CONFIG_IOSCHED_NOOP=y +CONFIG_IOSCHED_TEST=y +CONFIG_IOSCHED_DEADLINE=y +CONFIG_IOSCHED_ROW=y +CONFIG_IOSCHED_CFQ=y +# CONFIG_DEFAULT_DEADLINE is not set +# CONFIG_DEFAULT_ROW is not set +CONFIG_DEFAULT_CFQ=y +# CONFIG_DEFAULT_NOOP is not set +CONFIG_DEFAULT_IOSCHED="cfq" +# CONFIG_INLINE_SPIN_TRYLOCK is not set +# CONFIG_INLINE_SPIN_TRYLOCK_BH is not set +# CONFIG_INLINE_SPIN_LOCK is not set +# CONFIG_INLINE_SPIN_LOCK_BH is not set +# CONFIG_INLINE_SPIN_LOCK_IRQ is not set +# CONFIG_INLINE_SPIN_LOCK_IRQSAVE is not set +CONFIG_UNINLINE_SPIN_UNLOCK=y +# CONFIG_INLINE_SPIN_UNLOCK_BH is not set +# CONFIG_INLINE_SPIN_UNLOCK_IRQ is not set +# CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE is not set +# CONFIG_INLINE_READ_TRYLOCK is not set +# CONFIG_INLINE_READ_LOCK is not set +# CONFIG_INLINE_READ_LOCK_BH is not set +# CONFIG_INLINE_READ_LOCK_IRQ is not set +# CONFIG_INLINE_READ_LOCK_IRQSAVE is not set +# CONFIG_INLINE_READ_UNLOCK is not set +# CONFIG_INLINE_READ_UNLOCK_BH is not set +# CONFIG_INLINE_READ_UNLOCK_IRQ is not set +# CONFIG_INLINE_READ_UNLOCK_IRQRESTORE is not set +# CONFIG_INLINE_WRITE_TRYLOCK is not set +# CONFIG_INLINE_WRITE_LOCK is not set +# CONFIG_INLINE_WRITE_LOCK_BH is not set +# CONFIG_INLINE_WRITE_LOCK_IRQ is not set +# CONFIG_INLINE_WRITE_LOCK_IRQSAVE is not set +# CONFIG_INLINE_WRITE_UNLOCK is not set +# CONFIG_INLINE_WRITE_UNLOCK_BH is not set +# CONFIG_INLINE_WRITE_UNLOCK_IRQ is not set +# CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE is not set +# CONFIG_MUTEX_SPIN_ON_OWNER is not set +CONFIG_FREEZER=y + +# +# System Type +# +CONFIG_MMU=y +# CONFIG_ARCH_INTEGRATOR is not set +# CONFIG_ARCH_REALVIEW is not set +# CONFIG_ARCH_VERSATILE is not set +# CONFIG_ARCH_VEXPRESS is not set +# CONFIG_ARCH_AT91 is not set +# CONFIG_ARCH_BCMRING is not set +# CONFIG_ARCH_HIGHBANK is not set +# CONFIG_ARCH_CLPS711X is not set +# CONFIG_ARCH_CNS3XXX is not set +# CONFIG_ARCH_GEMINI is not set +# CONFIG_ARCH_PRIMA2 is not set +# CONFIG_ARCH_EBSA110 is not set +# CONFIG_ARCH_EP93XX is not set +# CONFIG_ARCH_FOOTBRIDGE is not set +# CONFIG_ARCH_MXC is not set +# CONFIG_ARCH_MXS is not set +# CONFIG_ARCH_NETX is not set +# CONFIG_ARCH_H720X is not set +# CONFIG_ARCH_IOP13XX is not set +# CONFIG_ARCH_IOP32X is not set +# CONFIG_ARCH_IOP33X is not set +# CONFIG_ARCH_IXP23XX is not set +# CONFIG_ARCH_IXP2000 is not set +# CONFIG_ARCH_IXP4XX is not set +# CONFIG_ARCH_DOVE is not set +# CONFIG_ARCH_KIRKWOOD is not set +# CONFIG_ARCH_LPC32XX is not set +# CONFIG_ARCH_MV78XX0 is not set +# CONFIG_ARCH_ORION5X is not set +# CONFIG_ARCH_MMP is not set +# CONFIG_ARCH_KS8695 is not set +# CONFIG_ARCH_W90X900 is not set +# CONFIG_ARCH_TEGRA is not set +# CONFIG_ARCH_PICOXCELL is not set +# CONFIG_ARCH_PNX4008 is not set +# CONFIG_ARCH_PXA is not set +CONFIG_ARCH_MSM=y +# CONFIG_ARCH_SHMOBILE is not set +# CONFIG_ARCH_RPC is not set +# CONFIG_ARCH_SA1100 is not set +# CONFIG_ARCH_S3C24XX is not set +# CONFIG_ARCH_S3C64XX is not set +# CONFIG_ARCH_S5P64X0 is not set +# CONFIG_ARCH_S5PC100 is not set +# CONFIG_ARCH_S5PV210 is not set +# CONFIG_ARCH_EXYNOS is not set +# CONFIG_ARCH_SHARK is not set +# CONFIG_ARCH_U300 is not set +# CONFIG_ARCH_U8500 is not set +# CONFIG_ARCH_NOMADIK is not set +# CONFIG_ARCH_DAVINCI is not set +# CONFIG_ARCH_OMAP is not set +# CONFIG_PLAT_SPEAR is not set +# CONFIG_ARCH_VT8500 is not set +# CONFIG_ARCH_ZYNQ is not set +# CONFIG_GPIO_PCA953X is not set +# CONFIG_KEYBOARD_GPIO_POLLED is not set + +# +# MSM SoC Type +# +# CONFIG_ARCH_MSM7X01A is not set +# CONFIG_ARCH_MSM7X25 is not set +# CONFIG_ARCH_MSM7X27 is not set +# CONFIG_ARCH_MSM7X30 is not set +# CONFIG_ARCH_QSD8X50 is not set +# CONFIG_ARCH_MSM8X60 is not set +# CONFIG_ARCH_MSM8960 is not set +# CONFIG_ARCH_MSM8930 is not set +# CONFIG_ARCH_APQ8064 is not set +CONFIG_ARCH_MSM8974=y +# CONFIG_ARCH_APQ8084 is not set +# CONFIG_ARCH_MPQ8092 is not set +# CONFIG_ARCH_FSM9900 is not set +# CONFIG_ARCH_FSM9XXX is not set +# CONFIG_ARCH_MSM9615 is not set +# CONFIG_ARCH_MSM8625 is not set +# CONFIG_ARCH_MSM9625 is not set +# CONFIG_ARCH_MSMKRYPTON is not set +# CONFIG_ARCH_MSM8610 is not set +# CONFIG_ARCH_MSM8226 is not set +# CONFIG_ARCH_MSMSAMARIUM is not set +CONFIG_MSM_SOC_REV_NONE=y +# CONFIG_MSM_SOC_REV_A is not set +CONFIG_MSM_KRAIT_TBB_ABORT_HANDLER=y +CONFIG_ARCH_MSM_KRAIT=y +CONFIG_MSM_SMP=y +CONFIG_ARCH_MSM_KRAITMP=y +CONFIG_MSM_RESTART_V2=y +# CONFIG_MSM_RPM is not set +CONFIG_MSM_RPM_SMD=y +# CONFIG_MSM_MPM is not set +CONFIG_MSM_MPM_OF=y +CONFIG_MSM_LPM_TEST=y + +# +# MSM Board Selection +# +CONFIG_MACH_LGE=y + +# +# LGE Board Selection +# +CONFIG_MACH_MSM8974_HAMMERHEAD=y + +# +# LGE Specific Patches +# +CONFIG_LGE_HANDLE_PANIC=y +CONFIG_LGE_BLUETOOTH=y +# CONFIG_MSM_STACKED_MEMORY is not set +# CONFIG_KERNEL_MSM_CONTIG_MEM_REGION is not set +CONFIG_MSM_AMSS_VERSION=6225 +# CONFIG_MSM_AMSS_VERSION_6210 is not set +# CONFIG_MSM_AMSS_VERSION_6220 is not set +CONFIG_MSM_AMSS_VERSION_6225=y +CONFIG_MSM7X00A_USE_GP_TIMER=y +# CONFIG_MSM7X00A_USE_DG_TIMER is not set +CONFIG_MSM7X00A_SLEEP_MODE_POWER_COLLAPSE_SUSPEND=y +# CONFIG_MSM7X00A_SLEEP_MODE_POWER_COLLAPSE is not set +# CONFIG_MSM7X00A_SLEEP_MODE_APPS_SLEEP is not set +# CONFIG_MSM7X00A_SLEEP_MODE_RAMP_DOWN_AND_WAIT_FOR_INTERRUPT is not set +# CONFIG_MSM7X00A_SLEEP_WAIT_FOR_INTERRUPT is not set +CONFIG_MSM7X00A_SLEEP_MODE=0 +# CONFIG_MSM7X00A_IDLE_SLEEP_MODE_POWER_COLLAPSE_SUSPEND is not set +CONFIG_MSM7X00A_IDLE_SLEEP_MODE_POWER_COLLAPSE=y +# CONFIG_MSM7X00A_IDLE_SLEEP_MODE_APPS_SLEEP is not set +# CONFIG_MSM7X00A_IDLE_SLEEP_MODE_RAMP_DOWN_AND_WAIT_FOR_INTERRUPT is not set +# CONFIG_MSM7X00A_IDLE_SLEEP_WAIT_FOR_INTERRUPT is not set +CONFIG_MSM7X00A_IDLE_SLEEP_MODE=1 +CONFIG_MSM7X00A_IDLE_SLEEP_MIN_TIME=20000000 +CONFIG_MSM7X00A_IDLE_SPIN_TIME=80000 +CONFIG_MSM_IDLE_STATS=y +CONFIG_MSM_IDLE_STATS_FIRST_BUCKET=62500 +CONFIG_MSM_IDLE_STATS_BUCKET_SHIFT=2 +CONFIG_MSM_IDLE_STATS_BUCKET_COUNT=10 +CONFIG_MSM_SUSPEND_STATS_FIRST_BUCKET=1000000000 +CONFIG_CPU_HAS_L2_PMU=y +# CONFIG_HTC_HEADSET is not set +# CONFIG_HTC_PWRSINK is not set +# CONFIG_MSM_FIQ_SUPPORT is not set +# CONFIG_MSM_SERIAL_DEBUGGER is not set +# CONFIG_MSM_PROC_COMM is not set +CONFIG_MSM_SMD=y +# CONFIG_MSM_SMD_PKG3 is not set +CONFIG_MSM_SMD_PKG4=y +CONFIG_MSM_SMD_DEBUG=y +CONFIG_MSM_BAM_DMUX=y +CONFIG_MSM_N_WAY_SMD=y +CONFIG_MSM_N_WAY_SMSM=y +CONFIG_MSM_SMP2P=y +CONFIG_MSM_SMP2P_TEST=y +# CONFIG_MSM_RESET_MODEM is not set +CONFIG_MSM_SMD_LOGGING=y +CONFIG_MSM_IPC_LOGGING=y +CONFIG_MSM_SMD_NMEA=y +# CONFIG_MSM_SMD_TTY is not set +CONFIG_MSM_SMD_QMI=y +CONFIG_MSM_SMD_PKT=y +# CONFIG_MSM_ONCRPCROUTER is not set +CONFIG_MSM_IPC_ROUTER=y +CONFIG_MSM_IPC_ROUTER_SMD_XPRT=y +CONFIG_MSM_IPC_ROUTER_SECURITY=y +CONFIG_MSM_QMI_INTERFACE=y +# CONFIG_MSM_TEST_QMI_CLIENT is not set +# CONFIG_MSM_DALRPC is not set +# CONFIG_MSM_CPU_FREQ_SET_MIN_MAX is not set +# CONFIG_MSM_AVS_HW is not set +# CONFIG_MSM_HW3D is not set +CONFIG_AMSS_7X25_VERSION_2009=y +# CONFIG_AMSS_7X25_VERSION_2008 is not set +CONFIG_RTAC=y +# CONFIG_MSM_VREG_SWITCH_INVERTED is not set +# CONFIG_MSM_DMA_TEST is not set +CONFIG_WIFI_CONTROL_FUNC=y +# CONFIG_WIFI_MEM_PREALLOC is not set +CONFIG_MSM_SLEEP_TIME_OVERRIDE=y +# CONFIG_MSM_MEMORY_LOW_POWER_MODE is not set +CONFIG_MSM_PM_TIMEOUT_HALT=y +# CONFIG_MSM_PM_TIMEOUT_RESET_MODEM is not set +# CONFIG_MSM_PM_TIMEOUT_RESET_CHIP is not set +CONFIG_MSM_IDLE_WAIT_ON_MODEM=0 +CONFIG_MSM_RPM_REGULATOR_SMD=y +# CONFIG_MSM_SPM_REGULATOR is not set +# CONFIG_MSM_SMCMOD is not set +CONFIG_MSM_SUBSYSTEM_RESTART=y +CONFIG_MSM_SYSMON_COMM=y +CONFIG_MSM_PIL=y +# CONFIG_MSM_PIL_MODEM is not set +# CONFIG_MSM_PIL_QDSP6V3 is not set +# CONFIG_MSM_PIL_LPASS_QDSP6V4 is not set +# CONFIG_MSM_PIL_MODEM_QDSP6V4 is not set +CONFIG_MSM_PIL_LPASS_QDSP6V5=y +CONFIG_MSM_PIL_MSS_QDSP6V5=y +# CONFIG_MSM_PIL_RIVA is not set +# CONFIG_MSM_PIL_TZAPPS is not set +# CONFIG_MSM_PIL_DSPS is not set +# CONFIG_MSM_PIL_VIDC is not set +CONFIG_MSM_PIL_VENUS=y +# CONFIG_MSM_PIL_GSS is not set +# CONFIG_MSM_PIL_PRONTO is not set +CONFIG_MSM_SCM=y +# CONFIG_MSM_BUSPM_DEV is not set +CONFIG_MSM_TZ_LOG=y +CONFIG_MSM_RPM_LOG=y +CONFIG_MSM_RPM_STATS_LOG=y +CONFIG_MSM_RPM_RBCPR_STATS_V2_LOG=y +CONFIG_MSM_DIRECT_SCLK_ACCESS=y +CONFIG_IOMMU_API=y +CONFIG_MSM_GPIOMUX=y +CONFIG_MSM_NATIVE_RESTART=y +CONFIG_MSM_PM8X60=y +CONFIG_MSM_EVENT_TIMER=y +CONFIG_MSM_BUS_SCALING=y +# CONFIG_MSM_BUS_RPM_MULTI_TIER_ENABLED is not set +CONFIG_MSM_WATCHDOG_V2=y +CONFIG_MSM_MEMORY_DUMP=y +CONFIG_MSM_DLOAD_MODE=y +# CONFIG_MSM_JTAG is not set +# CONFIG_MSM_JTAG_MM is not set +# CONFIG_MSM_SLEEP_STATS_DEVICE is not set +CONFIG_MSM_RUN_QUEUE_STATS=y +# CONFIG_MSM_STANDALONE_POWER_COLLAPSE is not set +# CONFIG_MSM_GSBI9_UART is not set +CONFIG_MSM_SHOW_RESUME_IRQ=y +# CONFIG_MSM_FAKE_BATTERY is not set +# CONFIG_MSM_QDSP6_APR is not set +CONFIG_MSM_QDSP6_APRV2=y +# CONFIG_MSM_QDSP6_CODECS is not set +CONFIG_MSM_QDSP6V2_CODECS=y +# CONFIG_MSM_AUDIO_QDSP6 is not set +CONFIG_MSM_AUDIO_QDSP6V2=y +CONFIG_MSM_ADSP_LOADER=y +# CONFIG_MSM_ULTRASOUND_A is not set +CONFIG_MSM_ULTRASOUND_B=y +# CONFIG_MSM_SPM_V1 is not set +CONFIG_MSM_SPM_V2=y +CONFIG_MSM_L2_SPM=y +CONFIG_MSM_MULTIMEDIA_USE_ION=y +CONFIG_MSM_OCMEM=y +CONFIG_MSM_OCMEM_LOCAL_POWER_CTRL=y +CONFIG_MSM_OCMEM_DEBUG=y +# CONFIG_MSM_OCMEM_NONSECURE is not set +# CONFIG_MSM_OCMEM_POWER_DEBUG is not set +# CONFIG_MSM_OCMEM_DEBUG_ALWAYS_ON is not set +# CONFIG_MSM_OCMEM_POWER_DISABLE is not set +CONFIG_SENSORS_ADSP=y +# CONFIG_MSM_RTB is not set +# CONFIG_MSM_EBI_ERP is not set +CONFIG_MSM_CACHE_ERP=y +CONFIG_MSM_L1_ERR_PANIC=y +CONFIG_MSM_L1_RECOV_ERR_PANIC=y +CONFIG_MSM_L1_ERR_LOG=y +# CONFIG_MSM_L2_ERP_PRINT_ACCESS_ERRORS is not set +# CONFIG_MSM_L2_ERP_1BIT_PANIC is not set +# CONFIG_MSM_L2_ERP_2BIT_PANIC is not set +# CONFIG_MSM_DCVS is not set +# CONFIG_MSM_CPR is not set +CONFIG_HAVE_ARCH_HAS_CURRENT_TIMER=y +# CONFIG_MSM_CACHE_DUMP is not set +# CONFIG_MSM_HSIC_SYSMON is not set +CONFIG_MSM_ENABLE_WDOG_DEBUG_CONTROL=y +# CONFIG_USE_DEV_CTRL_VOLUME is not set +# CONFIG_MSM_CPU_PWRCTL is not set +CONFIG_MSM_UARTDM_Core_v14=y +# CONFIG_MSM_BOOT_STATS is not set +# CONFIG_MSM_XPU_ERR_FATAL is not set +# CONFIG_MSM_CPR_REGULATOR is not set + +# +# System MMU +# + +# +# Processor Type +# +CONFIG_CPU_V7=y +CONFIG_CPU_32v6K=y +CONFIG_CPU_32v7=y +CONFIG_CPU_ABRT_EV7=y +CONFIG_CPU_PABRT_V7=y +CONFIG_CPU_CACHE_V7=y +CONFIG_CPU_CACHE_VIPT=y +CONFIG_CPU_COPY_V6=y +CONFIG_CPU_TLB_V7=y +CONFIG_CPU_HAS_ASID=y +CONFIG_CPU_CP15=y +CONFIG_CPU_CP15_MMU=y + +# +# Processor Features +# +# CONFIG_ARM_LPAE is not set +# CONFIG_ARCH_PHYS_ADDR_T_64BIT is not set +CONFIG_ARM_THUMB=y +# CONFIG_ARM_THUMBEE is not set +CONFIG_SWP_EMULATE=y +# CONFIG_CPU_ICACHE_DISABLE is not set +# CONFIG_CPU_DCACHE_DISABLE is not set +# CONFIG_CPU_BPREDICT_DISABLE is not set +# CONFIG_CACHE_L2X0 is not set +CONFIG_ARM_L1_CACHE_SHIFT_6=y +CONFIG_ARM_L1_CACHE_SHIFT=6 +CONFIG_ARM_DMA_MEM_BUFFERABLE=y +CONFIG_STRICT_MEMORY_RWX=y +CONFIG_ARM_NR_BANKS=8 +# CONFIG_RESERVE_FIRST_PAGE is not set +CONFIG_CPU_HAS_PMU=y +CONFIG_MULTI_IRQ_HANDLER=y +# CONFIG_ARM_ERRATA_430973 is not set +# CONFIG_ARM_ERRATA_458693 is not set +# CONFIG_ARM_ERRATA_460075 is not set +# CONFIG_ARM_ERRATA_742230 is not set +# CONFIG_ARM_ERRATA_742231 is not set +# CONFIG_ARM_ERRATA_720789 is not set +# CONFIG_ARM_ERRATA_743622 is not set +# CONFIG_ARM_ERRATA_751472 is not set +# CONFIG_ARM_ERRATA_754322 is not set +# CONFIG_ARM_ERRATA_754327 is not set +# CONFIG_ARM_ERRATA_764369 is not set +# CONFIG_KSAPI is not set +CONFIG_ARM_GIC=y +# CONFIG_FIQ_DEBUGGER is not set + +# +# Bus support +# +# CONFIG_PCI_SYSCALL is not set +# CONFIG_ARCH_SUPPORTS_MSI is not set +# CONFIG_PCCARD is not set + +# +# Kernel Features +# +CONFIG_TICK_ONESHOT=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_GENERIC_CLOCKEVENTS_BUILD=y +CONFIG_HAVE_SMP=y +CONFIG_SMP=y +# CONFIG_SMP_ON_UP is not set +CONFIG_ARM_CPU_TOPOLOGY=y +CONFIG_SCHED_MC=y +# CONFIG_SCHED_SMT is not set +CONFIG_HAVE_ARM_SCU=y +CONFIG_ARM_ARCH_TIMER=y +CONFIG_VMSPLIT_3G=y +# CONFIG_VMSPLIT_2G is not set +# CONFIG_VMSPLIT_1G is not set +CONFIG_PAGE_OFFSET=0xC0000000 +CONFIG_NR_CPUS=4 +CONFIG_HOTPLUG_CPU=y +CONFIG_LOCAL_TIMERS=y +CONFIG_ARCH_NR_GPIO=0 +# CONFIG_PREEMPT_NONE is not set +# CONFIG_PREEMPT_VOLUNTARY is not set +CONFIG_PREEMPT=y +CONFIG_PREEMPT_COUNT=y +CONFIG_HZ=100 +# CONFIG_THUMB2_KERNEL is not set +CONFIG_AEABI=y +# CONFIG_OABI_COMPAT is not set +# CONFIG_ARCH_SPARSEMEM_DEFAULT is not set +# CONFIG_ARCH_SELECT_MEMORY_MODEL is not set +CONFIG_HAVE_ARCH_PFN_VALID=y +CONFIG_HIGHMEM=y +# CONFIG_HIGHPTE is not set +CONFIG_HW_PERF_EVENTS=y +CONFIG_SELECT_MEMORY_MODEL=y +CONFIG_FLATMEM_MANUAL=y +CONFIG_FLATMEM=y +CONFIG_FLAT_NODE_MEM_MAP=y +CONFIG_HAVE_MEMBLOCK=y +CONFIG_PAGEFLAGS_EXTENDED=y +CONFIG_SPLIT_PTLOCK_CPUS=4 +CONFIG_COMPACTION=y +CONFIG_MIGRATION=y +# CONFIG_PHYS_ADDR_T_64BIT is not set +CONFIG_ZONE_DMA_FLAG=0 +CONFIG_BOUNCE=y +CONFIG_VIRT_TO_BUS=y +# CONFIG_KSM is not set +CONFIG_DEFAULT_MMAP_MIN_ADDR=4096 +# CONFIG_CLEANCACHE is not set +CONFIG_MEMORY_HOLE_CARVEOUT=y +# CONFIG_USE_USER_ACCESSIBLE_TIMERS is not set +# CONFIG_ARCH_MEMORY_PROBE is not set +# CONFIG_ARCH_MEMORY_REMOVE is not set +# CONFIG_ENABLE_DMM is not set +CONFIG_DONT_MAP_HOLE_AFTER_MEMBANK0=y +CONFIG_FORCE_MAX_ZONEORDER=11 +CONFIG_ALIGNMENT_TRAP=y +# CONFIG_UACCESS_WITH_MEMCPY is not set +CONFIG_SECCOMP=y +CONFIG_CC_STACKPROTECTOR=y +# CONFIG_DEPRECATED_PARAM_STRUCT is not set +# CONFIG_ARM_FLUSH_CONSOLE_ON_RESTART is not set +CONFIG_CP_ACCESS=y + +# +# Boot options +# +CONFIG_USE_OF=y +CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE=y +CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE_NAMES="" +CONFIG_ZBOOT_ROM_TEXT=0 +CONFIG_ZBOOT_ROM_BSS=0 +# CONFIG_ARM_APPENDED_DTB is not set +CONFIG_CMDLINE="" +# CONFIG_XIP_KERNEL is not set +# CONFIG_KEXEC is not set +# CONFIG_CRASH_DUMP is not set +# CONFIG_AUTO_ZRELADDR is not set + +# +# CPU Power Management +# + +# +# CPU Frequency scaling +# +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_TABLE=y +CONFIG_CPU_FREQ_STAT=y +# CONFIG_CPU_FREQ_STAT_DETAILS is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE is not set +# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set +# CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND is not set +# CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE is not set +# CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE is not set +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_INTERACTIVE=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y + +# +# ARM CPU frequency scaling drivers +# +# CONFIG_ARM_EXYNOS4210_CPUFREQ is not set +# CONFIG_ARM_EXYNOS4X12_CPUFREQ is not set +# CONFIG_ARM_EXYNOS5250_CPUFREQ is not set +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y +# CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED is not set +CONFIG_CPU_FREQ_MSM=y + +# +# Floating point emulation +# + +# +# At least one emulation must be selected +# +CONFIG_VFP=y +CONFIG_VFPv3=y +CONFIG_NEON=y + +# +# Userspace binary formats +# +CONFIG_BINFMT_ELF=y +CONFIG_ARCH_BINFMT_ELF_RANDOMIZE_PIE=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_HAVE_AOUT=y +# CONFIG_BINFMT_AOUT is not set +# CONFIG_BINFMT_MISC is not set + +# +# Power management options +# +CONFIG_SUSPEND=y +CONFIG_SUSPEND_FREEZER=y +CONFIG_HAS_WAKELOCK=y +CONFIG_WAKELOCK=y +CONFIG_PM_SLEEP=y +CONFIG_PM_SLEEP_SMP=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_RUNTIME=y +CONFIG_PM=y +# CONFIG_PM_DEBUG is not set +# CONFIG_APM_EMULATION is not set +CONFIG_PM_CLK=y +CONFIG_CPU_PM=y +# CONFIG_SUSPEND_TIME is not set +CONFIG_DEDUCE_WAKEUP_REASONS=y +CONFIG_PARTIALRESUME=y +CONFIG_ARCH_SUSPEND_POSSIBLE=y +CONFIG_ARM_CPU_SUSPEND=y +CONFIG_NET=y + +# +# Networking options +# +CONFIG_PACKET=y +CONFIG_UNIX=y +# CONFIG_UNIX_DIAG is not set +CONFIG_XFRM=y +CONFIG_XFRM_USER=y +# CONFIG_XFRM_SUB_POLICY is not set +# CONFIG_XFRM_MIGRATE is not set +# CONFIG_XFRM_STATISTICS is not set +CONFIG_XFRM_IPCOMP=y +CONFIG_NET_KEY=y +# CONFIG_NET_KEY_MIGRATE is not set +CONFIG_INET=y +# CONFIG_IP_MULTICAST is not set +CONFIG_IP_ADVANCED_ROUTER=y +# CONFIG_IP_FIB_TRIE_STATS is not set +CONFIG_IP_MULTIPLE_TABLES=y +# CONFIG_IP_ROUTE_MULTIPATH is not set +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +# CONFIG_IP_PNP_BOOTP is not set +# CONFIG_IP_PNP_RARP is not set +# CONFIG_NET_IPIP is not set +# CONFIG_NET_IPGRE_DEMUX is not set +# CONFIG_ARPD is not set +# CONFIG_SYN_COOKIES is not set +CONFIG_INET_AH=y +CONFIG_INET_ESP=y +CONFIG_INET_IPCOMP=y +CONFIG_INET_XFRM_TUNNEL=y +CONFIG_INET_TUNNEL=y +CONFIG_INET_XFRM_MODE_TRANSPORT=y +CONFIG_INET_XFRM_MODE_TUNNEL=y +CONFIG_INET_XFRM_MODE_BEET=y +# CONFIG_INET_LRO is not set +CONFIG_INET_DIAG=y +CONFIG_INET_TCP_DIAG=y +# CONFIG_INET_UDP_DIAG is not set +# CONFIG_TCP_CONG_ADVANCED is not set +CONFIG_TCP_CONG_CUBIC=y +CONFIG_DEFAULT_TCP_CONG="cubic" +# CONFIG_TCP_MD5SIG is not set +CONFIG_IPV6=y +CONFIG_IPV6_PRIVACY=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_INET6_XFRM_TUNNEL=y +CONFIG_INET6_TUNNEL=y +CONFIG_INET6_XFRM_MODE_TRANSPORT=y +CONFIG_INET6_XFRM_MODE_TUNNEL=y +CONFIG_INET6_XFRM_MODE_BEET=y +# CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION is not set +CONFIG_IPV6_SIT=y +# CONFIG_IPV6_SIT_6RD is not set +CONFIG_IPV6_NDISC_NODETYPE=y +# CONFIG_IPV6_TUNNEL is not set +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPV6_SUBTREES=y +# CONFIG_IPV6_MROUTE is not set +# CONFIG_NETLABEL is not set +CONFIG_ANDROID_PARANOID_NETWORK=y +CONFIG_NET_ACTIVITY_STATS=y +CONFIG_NETWORK_SECMARK=y +# CONFIG_NETWORK_PHY_TIMESTAMPING is not set +CONFIG_NETFILTER=y +# CONFIG_NETFILTER_DEBUG is not set +CONFIG_NETFILTER_ADVANCED=y +CONFIG_BRIDGE_NETFILTER=y + +# +# Core Netfilter Configuration +# +CONFIG_NETFILTER_NETLINK=y +# CONFIG_NETFILTER_NETLINK_ACCT is not set +CONFIG_NETFILTER_NETLINK_QUEUE=y +CONFIG_NETFILTER_NETLINK_LOG=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_CONNTRACK_EVENTS=y +# CONFIG_NF_CONNTRACK_TIMEOUT is not set +# CONFIG_NF_CONNTRACK_TIMESTAMP is not set +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_GRE=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_BROADCAST=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +# CONFIG_NF_CONNTRACK_SNMP is not set +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +# CONFIG_NF_CONNTRACK_SIP is not set +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +# CONFIG_NF_CT_NETLINK_TIMEOUT is not set +CONFIG_NETFILTER_TPROXY=y +CONFIG_NETFILTER_XTABLES=y + +# +# Xtables combined modules +# +CONFIG_NETFILTER_XT_MARK=y +CONFIG_NETFILTER_XT_CONNMARK=y + +# +# Xtables targets +# +# CONFIG_NETFILTER_XT_TARGET_AUDIT is not set +# CONFIG_NETFILTER_XT_TARGET_CHECKSUM is not set +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +# CONFIG_NETFILTER_XT_TARGET_CT is not set +# CONFIG_NETFILTER_XT_TARGET_DSCP is not set +# CONFIG_NETFILTER_XT_TARGET_HL is not set +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +# CONFIG_NETFILTER_XT_TARGET_LED is not set +CONFIG_NETFILTER_XT_TARGET_LOG=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +# CONFIG_NETFILTER_XT_TARGET_NOTRACK is not set +# CONFIG_NETFILTER_XT_TARGET_RATEEST is not set +# CONFIG_NETFILTER_XT_TARGET_TEE is not set +# CONFIG_NETFILTER_XT_TARGET_TPROXY is not set +# CONFIG_NETFILTER_XT_TARGET_TRACE is not set +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +# CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP is not set + +# +# Xtables matches +# +# CONFIG_NETFILTER_XT_MATCH_ADDRTYPE is not set +# CONFIG_NETFILTER_XT_MATCH_CLUSTER is not set +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +# CONFIG_NETFILTER_XT_MATCH_CONNBYTES is not set +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +# CONFIG_NETFILTER_XT_MATCH_CPU is not set +# CONFIG_NETFILTER_XT_MATCH_DCCP is not set +# CONFIG_NETFILTER_XT_MATCH_DEVGROUP is not set +# CONFIG_NETFILTER_XT_MATCH_DSCP is not set +CONFIG_NETFILTER_XT_MATCH_ECN=y +# CONFIG_NETFILTER_XT_MATCH_ESP is not set +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_HL=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +# CONFIG_NETFILTER_XT_MATCH_NFACCT is not set +# CONFIG_NETFILTER_XT_MATCH_OSF is not set +# CONFIG_NETFILTER_XT_MATCH_OWNER is not set +CONFIG_NETFILTER_XT_MATCH_POLICY=y +# CONFIG_NETFILTER_XT_MATCH_PHYSDEV is not set +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y +# CONFIG_NETFILTER_XT_MATCH_RATEEST is not set +# CONFIG_NETFILTER_XT_MATCH_REALM is not set +# CONFIG_NETFILTER_XT_MATCH_RECENT is not set +# CONFIG_NETFILTER_XT_MATCH_SCTP is not set +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +# CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +# CONFIG_IP_SET is not set +# CONFIG_IP_VS is not set + +# +# IP: Netfilter Configuration +# +CONFIG_NF_DEFRAG_IPV4=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_NF_CONNTRACK_PROC_COMPAT=y +# CONFIG_IP_NF_QUEUE is not set +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +# CONFIG_IP_NF_MATCH_RPFILTER is not set +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_TARGET_REJECT_SKERR=y +# CONFIG_IP_NF_TARGET_ULOG is not set +CONFIG_NF_NAT=y +CONFIG_NF_NAT_NEEDED=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +# CONFIG_IP_NF_TARGET_NATTYPE_MODULE is not set +CONFIG_IP_NF_TARGET_NETMAP=y +CONFIG_IP_NF_TARGET_REDIRECT=y +CONFIG_NF_NAT_PROTO_DCCP=y +CONFIG_NF_NAT_PROTO_GRE=y +CONFIG_NF_NAT_PROTO_UDPLITE=y +CONFIG_NF_NAT_PROTO_SCTP=y +CONFIG_NF_NAT_FTP=y +CONFIG_NF_NAT_IRC=y +CONFIG_NF_NAT_TFTP=y +CONFIG_NF_NAT_AMANDA=y +CONFIG_NF_NAT_PPTP=y +CONFIG_NF_NAT_H323=y +# CONFIG_NF_NAT_SIP is not set +CONFIG_IP_NF_MANGLE=y +# CONFIG_IP_NF_TARGET_CLUSTERIP is not set +# CONFIG_IP_NF_TARGET_ECN is not set +# CONFIG_IP_NF_TARGET_TTL is not set +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y + +# +# IPv6: Netfilter Configuration +# +CONFIG_NF_DEFRAG_IPV6=y +CONFIG_NF_CONNTRACK_IPV6=y +# CONFIG_IP6_NF_QUEUE is not set +CONFIG_IP6_NF_IPTABLES=y +# CONFIG_IP6_NF_MATCH_AH is not set +# CONFIG_IP6_NF_MATCH_EUI64 is not set +# CONFIG_IP6_NF_MATCH_FRAG is not set +# CONFIG_IP6_NF_MATCH_OPTS is not set +# CONFIG_IP6_NF_MATCH_HL is not set +# CONFIG_IP6_NF_MATCH_IPV6HEADER is not set +# CONFIG_IP6_NF_MATCH_MH is not set +CONFIG_IP6_NF_MATCH_RPFILTER=y +# CONFIG_IP6_NF_MATCH_RT is not set +# CONFIG_IP6_NF_TARGET_HL is not set +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_TARGET_REJECT_SKERR=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +# CONFIG_IP6_NF_SECURITY is not set +CONFIG_BRIDGE_NF_EBTABLES=y +CONFIG_BRIDGE_EBT_BROUTE=y +# CONFIG_BRIDGE_EBT_T_FILTER is not set +# CONFIG_BRIDGE_EBT_T_NAT is not set +# CONFIG_BRIDGE_EBT_802_3 is not set +# CONFIG_BRIDGE_EBT_AMONG is not set +# CONFIG_BRIDGE_EBT_ARP is not set +# CONFIG_BRIDGE_EBT_IP is not set +# CONFIG_BRIDGE_EBT_IP6 is not set +# CONFIG_BRIDGE_EBT_LIMIT is not set +# CONFIG_BRIDGE_EBT_MARK is not set +# CONFIG_BRIDGE_EBT_PKTTYPE is not set +# CONFIG_BRIDGE_EBT_STP is not set +# CONFIG_BRIDGE_EBT_VLAN is not set +# CONFIG_BRIDGE_EBT_ARPREPLY is not set +# CONFIG_BRIDGE_EBT_DNAT is not set +# CONFIG_BRIDGE_EBT_MARK_T is not set +# CONFIG_BRIDGE_EBT_REDIRECT is not set +# CONFIG_BRIDGE_EBT_SNAT is not set +# CONFIG_BRIDGE_EBT_LOG is not set +# CONFIG_BRIDGE_EBT_ULOG is not set +# CONFIG_BRIDGE_EBT_NFLOG is not set +# CONFIG_IP_DCCP is not set +# CONFIG_IP_SCTP is not set +# CONFIG_RDS is not set +# CONFIG_TIPC is not set +# CONFIG_ATM is not set +CONFIG_L2TP=y +# CONFIG_L2TP_DEBUGFS is not set +# CONFIG_L2TP_V3 is not set +CONFIG_STP=y +CONFIG_BRIDGE=y +CONFIG_BRIDGE_IGMP_SNOOPING=y +# CONFIG_NET_DSA is not set +# CONFIG_VLAN_8021Q is not set +# CONFIG_DECNET is not set +CONFIG_LLC=y +# CONFIG_LLC2 is not set +# CONFIG_IPX is not set +# CONFIG_ATALK is not set +# CONFIG_X25 is not set +# CONFIG_LAPB is not set +# CONFIG_ECONET is not set +# CONFIG_WAN_ROUTER is not set +# CONFIG_PHONET is not set +# CONFIG_IEEE802154 is not set +CONFIG_NET_SCHED=y + +# +# Queueing/Scheduling +# +# CONFIG_NET_SCH_CBQ is not set +CONFIG_NET_SCH_HTB=y +# CONFIG_NET_SCH_HFSC is not set +CONFIG_NET_SCH_PRIO=y +# CONFIG_NET_SCH_MULTIQ is not set +# CONFIG_NET_SCH_RED is not set +# CONFIG_NET_SCH_SFB is not set +# CONFIG_NET_SCH_SFQ is not set +# CONFIG_NET_SCH_TEQL is not set +# CONFIG_NET_SCH_TBF is not set +# CONFIG_NET_SCH_GRED is not set +# CONFIG_NET_SCH_DSMARK is not set +# CONFIG_NET_SCH_NETEM is not set +# CONFIG_NET_SCH_DRR is not set +# CONFIG_NET_SCH_MQPRIO is not set +# CONFIG_NET_SCH_CHOKE is not set +# CONFIG_NET_SCH_QFQ is not set +# CONFIG_NET_SCH_INGRESS is not set +# CONFIG_NET_SCH_PLUG is not set + +# +# Classification +# +CONFIG_NET_CLS=y +# CONFIG_NET_CLS_BASIC is not set +# CONFIG_NET_CLS_TCINDEX is not set +# CONFIG_NET_CLS_ROUTE4 is not set +CONFIG_NET_CLS_FW=y +CONFIG_NET_CLS_U32=y +# CONFIG_CLS_U32_PERF is not set +CONFIG_CLS_U32_MARK=y +# CONFIG_NET_CLS_RSVP is not set +# CONFIG_NET_CLS_RSVP6 is not set +CONFIG_NET_CLS_FLOW=y +# CONFIG_NET_CLS_CGROUP is not set +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_STACK=32 +CONFIG_NET_EMATCH_CMP=y +CONFIG_NET_EMATCH_NBYTE=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_EMATCH_META=y +CONFIG_NET_EMATCH_TEXT=y +CONFIG_NET_CLS_ACT=y +# CONFIG_NET_ACT_POLICE is not set +# CONFIG_NET_ACT_GACT is not set +# CONFIG_NET_ACT_MIRRED is not set +# CONFIG_NET_ACT_IPT is not set +# CONFIG_NET_ACT_NAT is not set +# CONFIG_NET_ACT_PEDIT is not set +# CONFIG_NET_ACT_SIMP is not set +# CONFIG_NET_ACT_SKBEDIT is not set +# CONFIG_NET_ACT_CSUM is not set +# CONFIG_NET_CLS_IND is not set +CONFIG_NET_SCH_FIFO=y +# CONFIG_DCB is not set +# CONFIG_DNS_RESOLVER is not set +# CONFIG_BATMAN_ADV is not set +# CONFIG_OPENVSWITCH is not set +CONFIG_RPS=y +CONFIG_RFS_ACCEL=y +CONFIG_XPS=y +# CONFIG_NETPRIO_CGROUP is not set +CONFIG_BQL=y +CONFIG_HAVE_BPF_JIT=y + +# +# Network testing +# +# CONFIG_NET_PKTGEN is not set +# CONFIG_NET_DROP_MONITOR is not set +# CONFIG_HAMRADIO is not set +# CONFIG_CAN is not set +# CONFIG_IRDA is not set +CONFIG_BT=y +CONFIG_BT_RFCOMM=y +# CONFIG_BT_RFCOMM_TTY is not set +CONFIG_BT_BNEP=y +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=y + +# +# Bluetooth device drivers +# +# CONFIG_BT_HCISMD is not set +CONFIG_BT_HCIBTUSB=y +# CONFIG_BT_HCIBTSDIO is not set +# CONFIG_BT_HCIUART is not set +# CONFIG_BT_HCIBCM203X is not set +# CONFIG_BT_HCIBPA10X is not set +CONFIG_BT_MSM_SLEEP=y +# CONFIG_BT_HCIBFUSB is not set +# CONFIG_BT_HCIVHCI is not set +# CONFIG_BT_MRVL is not set +# CONFIG_MSM_BT_POWER is not set +# CONFIG_BT_ATH3K is not set +# CONFIG_AF_RXRPC is not set +CONFIG_FIB_RULES=y +CONFIG_WIRELESS=y +CONFIG_WEXT_CORE=y +CONFIG_WEXT_PROC=y +CONFIG_CFG80211=y +CONFIG_NL80211_TESTMODE=y +# CONFIG_CFG80211_DEVELOPER_WARNINGS is not set +CONFIG_CFG80211_REG_DEBUG=y +CONFIG_CFG80211_DEFAULT_PS=y +# CONFIG_CFG80211_DEBUGFS is not set +# CONFIG_CFG80211_INTERNAL_REGDB is not set +CONFIG_CFG80211_WEXT=y +# CONFIG_WIRELESS_EXT_SYSFS is not set +# CONFIG_LIB80211 is not set +# CONFIG_CFG80211_ALLOW_RECONNECT is not set +CONFIG_MAC80211=y +CONFIG_MAC80211_HAS_RC=y +# CONFIG_MAC80211_RC_PID is not set +CONFIG_MAC80211_RC_MINSTREL=y +CONFIG_MAC80211_RC_MINSTREL_HT=y +CONFIG_MAC80211_RC_DEFAULT_MINSTREL=y +CONFIG_MAC80211_RC_DEFAULT="minstrel_ht" +# CONFIG_MAC80211_MESH is not set +CONFIG_MAC80211_LEDS=y +# CONFIG_MAC80211_DEBUGFS is not set +# CONFIG_MAC80211_DEBUG_MENU is not set +# CONFIG_WIMAX is not set +CONFIG_RFKILL=y +CONFIG_RFKILL_PM=y +CONFIG_RFKILL_LEDS=y +# CONFIG_RFKILL_INPUT is not set +# CONFIG_RFKILL_REGULATOR is not set +# CONFIG_RFKILL_GPIO is not set +# CONFIG_NET_9P is not set +# CONFIG_CAIF is not set +# CONFIG_CEPH_LIB is not set +# CONFIG_NFC is not set +CONFIG_NFC_BCM2079X=y + +# +# Device Drivers +# + +# +# Generic Driver Options +# +CONFIG_UEVENT_HELPER_PATH="" +# CONFIG_DEVTMPFS is not set +CONFIG_STANDALONE=y +CONFIG_PREVENT_FIRMWARE_BUILD=y +CONFIG_FW_LOADER=y +CONFIG_FIRMWARE_IN_KERNEL=y +CONFIG_EXTRA_FIRMWARE="htc_7010.fw htc_9271.fw" +CONFIG_EXTRA_FIRMWARE_DIR="/lib/firmware" +# CONFIG_DEBUG_DRIVER is not set +# CONFIG_DEBUG_DEVRES is not set +# CONFIG_SYS_HYPERVISOR is not set +# CONFIG_GENERIC_CPU_DEVICES is not set +CONFIG_SOC_BUS=y +CONFIG_REGMAP=y +CONFIG_REGMAP_I2C=y +CONFIG_REGMAP_SPI=y +CONFIG_DMA_SHARED_BUFFER=y +# CONFIG_GENLOCK is not set +CONFIG_SYNC=y +CONFIG_SW_SYNC=y +# CONFIG_SW_SYNC_USER is not set +CONFIG_CMA=y +# CONFIG_CMA_DEBUG is not set + +# +# Default contiguous memory area size: +# +CONFIG_CMA_SIZE_MBYTES=16 +CONFIG_CMA_SIZE_SEL_MBYTES=y +# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set +# CONFIG_CMA_SIZE_SEL_MIN is not set +# CONFIG_CMA_SIZE_SEL_MAX is not set +CONFIG_CMA_ALIGNMENT=8 +CONFIG_CMA_AREAS=7 +# CONFIG_CONNECTOR is not set +# CONFIG_MTD is not set +CONFIG_DTC=y +CONFIG_OF=y + +# +# Device Tree and Open Firmware support +# +# CONFIG_PROC_DEVICETREE is not set +# CONFIG_OF_SELFTEST is not set +CONFIG_OF_FLATTREE=y +CONFIG_OF_EARLY_FLATTREE=y +CONFIG_OF_ADDRESS=y +CONFIG_OF_IRQ=y +CONFIG_OF_DEVICE=y +CONFIG_OF_GPIO=y +CONFIG_OF_I2C=y +CONFIG_OF_NET=y +CONFIG_OF_SPI=y +CONFIG_OF_SPMI=y +CONFIG_OF_SLIMBUS=y +# CONFIG_PARPORT is not set +CONFIG_BLK_DEV=y +# CONFIG_BLK_DEV_COW_COMMON is not set +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_LOOP_MIN_COUNT=8 +# CONFIG_BLK_DEV_CRYPTOLOOP is not set + +# +# DRBD disabled because PROC_FS, INET or CONNECTOR not selected +# +# CONFIG_BLK_DEV_NBD is not set +# CONFIG_BLK_DEV_UB is not set +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_COUNT=16 +CONFIG_BLK_DEV_RAM_SIZE=4096 +# CONFIG_BLK_DEV_XIP is not set +# CONFIG_CDROM_PKTCDVD is not set +# CONFIG_ATA_OVER_ETH is not set +# CONFIG_MG_DISK is not set +# CONFIG_BLK_DEV_RBD is not set + +# +# Misc devices +# +# CONFIG_SENSORS_LIS3LV02D is not set +# CONFIG_AD525X_DPOT is not set +# CONFIG_ATMEL_PWM is not set +# CONFIG_ICS932S401 is not set +# CONFIG_ENCLOSURE_SERVICES is not set +# CONFIG_APDS9802ALS is not set +# CONFIG_APDS9930 is not set +# CONFIG_ISL29003 is not set +# CONFIG_ISL29020 is not set +# CONFIG_SENSORS_TSL2550 is not set +# CONFIG_SENSORS_BH1780 is not set +# CONFIG_SENSORS_BH1770 is not set +# CONFIG_SENSORS_APDS990X is not set +# CONFIG_HMC6352 is not set +# CONFIG_SENSORS_AK8975 is not set +# CONFIG_DS1682 is not set +# CONFIG_TI_DAC7512 is not set +CONFIG_UID_STAT=y +# CONFIG_BMP085 is not set +# CONFIG_USB_SWITCH_FSA9480 is not set +# CONFIG_WL127X_RFKILL is not set +# CONFIG_TSPP is not set +# CONFIG_CI_BRIDGE_SPI is not set +CONFIG_HAPTIC_ISA1200=y +CONFIG_MSM8974_PWM_VIBRATOR=y +CONFIG_QSEECOM=y +# CONFIG_QFP_FUSE is not set +CONFIG_QPNP_MISC=y +CONFIG_TI_DRV2667=y +CONFIG_EARJACK_DEBUGGER=y +CONFIG_FAN48632_BOOST=y +CONFIG_UID_CPUTIME=y +# CONFIG_C2PORT is not set + +# +# EEPROM support +# +# CONFIG_EEPROM_AT24 is not set +# CONFIG_EEPROM_AT25 is not set +# CONFIG_EEPROM_LEGACY is not set +# CONFIG_EEPROM_MAX6875 is not set +# CONFIG_EEPROM_93CX6 is not set +# CONFIG_EEPROM_93XX46 is not set +# CONFIG_IWMC3200TOP is not set + +# +# Texas Instruments shared transport line discipline +# +# CONFIG_TI_ST is not set +# CONFIG_SENSORS_LIS3_SPI is not set +# CONFIG_SENSORS_LIS3_I2C is not set + +# +# Altera FPGA firmware download module +# +# CONFIG_ALTERA_STAPL is not set + +# +# SCSI device support +# +CONFIG_SCSI_MOD=y +# CONFIG_RAID_ATTRS is not set +CONFIG_SCSI=y +CONFIG_SCSI_DMA=y +CONFIG_SCSI_TGT=y +# CONFIG_SCSI_NETLINK is not set +CONFIG_SCSI_PROC_FS=y + +# +# SCSI support type (disk, tape, CD-ROM) +# +CONFIG_BLK_DEV_SD=y +# CONFIG_CHR_DEV_ST is not set +# CONFIG_CHR_DEV_OSST is not set +# CONFIG_BLK_DEV_SR is not set +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_MULTI_LUN=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y + +# +# SCSI Transports +# +# CONFIG_SCSI_SPI_ATTRS is not set +# CONFIG_SCSI_FC_ATTRS is not set +# CONFIG_SCSI_ISCSI_ATTRS is not set +# CONFIG_SCSI_SAS_ATTRS is not set +# CONFIG_SCSI_SAS_LIBSAS is not set +# CONFIG_SCSI_SRP_ATTRS is not set +CONFIG_SCSI_LOWLEVEL=y +# CONFIG_ISCSI_TCP is not set +# CONFIG_ISCSI_BOOT_SYSFS is not set +# CONFIG_SCSI_UFSHCD is not set +# CONFIG_LIBFC is not set +# CONFIG_LIBFCOE is not set +# CONFIG_SCSI_DEBUG is not set +# CONFIG_SCSI_DH is not set +# CONFIG_SCSI_OSD_INITIATOR is not set +# CONFIG_ATA is not set +CONFIG_MD=y +# CONFIG_BLK_DEV_MD is not set +CONFIG_BLK_DEV_DM=y +# CONFIG_DM_DEBUG is not set +CONFIG_DM_BUFIO=y +CONFIG_DM_CRYPT=y +# CONFIG_DM_SNAPSHOT is not set +# CONFIG_DM_THIN_PROVISIONING is not set +# CONFIG_DM_MIRROR is not set +# CONFIG_DM_RAID is not set +# CONFIG_DM_ZERO is not set +# CONFIG_DM_MULTIPATH is not set +# CONFIG_DM_DELAY is not set +# CONFIG_DM_UEVENT is not set +# CONFIG_DM_FLAKEY is not set +CONFIG_DM_VERITY=y +# CONFIG_TARGET_CORE is not set +CONFIG_NETDEVICES=y +CONFIG_NET_CORE=y +# CONFIG_BONDING is not set +# CONFIG_DUMMY is not set +# CONFIG_EQUALIZER is not set +CONFIG_MII=y +# CONFIG_IFB is not set +# CONFIG_NET_TEAM is not set +# CONFIG_MACVLAN is not set +# CONFIG_NETCONSOLE is not set +# CONFIG_NETPOLL is not set +# CONFIG_NET_POLL_CONTROLLER is not set +CONFIG_TUN=y +# CONFIG_VETH is not set + +# +# CAIF transport drivers +# +CONFIG_ETHERNET=y +# CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CALXEDA_XGMAC is not set +# CONFIG_NET_VENDOR_CHELSIO is not set +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_DM9000 is not set +# CONFIG_DNET is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_INTEL is not set +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_NET_VENDOR_MICREL=y +# CONFIG_KS8851 is not set +# CONFIG_KS8851_MLL is not set +# CONFIG_NET_VENDOR_MICROCHIP is not set +# CONFIG_MSM_RMNET is not set +CONFIG_MSM_RMNET_BAM=y +# CONFIG_QFEC is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_ETHOC is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_PHYLIB is not set +# CONFIG_MICREL_KS8995MA is not set +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=y +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOE=y +CONFIG_PPPOL2TP=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y +CONFIG_PPP_ASYNC=y +CONFIG_PPP_SYNC_TTY=y +CONFIG_SLIP=y +CONFIG_SLHC=y +CONFIG_SLIP_COMPRESSED=y +# CONFIG_SLIP_SMART is not set +CONFIG_SLIP_MODE_SLIP6=y + +# +# USB Network Adapters +# +CONFIG_USB_CATC=y +CONFIG_USB_KAWETH=y +CONFIG_USB_PEGASUS=y +CONFIG_USB_RTL8150=y +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=y +CONFIG_USB_NET_CDCETHER=y +CONFIG_USB_NET_CDC_EEM=y +CONFIG_USB_NET_CDC_NCM=y +CONFIG_USB_NET_DM9601=y +CONFIG_USB_NET_SMSC75XX=y +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_NET_GL620A=y +CONFIG_USB_NET_NET1080=y +CONFIG_USB_NET_PLUSB=y +CONFIG_USB_NET_MCS7830=y +CONFIG_USB_NET_RNDIS_HOST=y +CONFIG_USB_NET_CDC_SUBSET=y +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_BELKIN=y +CONFIG_USB_ARMLINUX=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_ZAURUS=y +CONFIG_USB_NET_CX82310_ETH=y +CONFIG_USB_NET_KALMIA=y +CONFIG_USB_NET_QMI_WWAN=y +CONFIG_USB_HSO=y +CONFIG_USB_NET_INT51X1=y +CONFIG_USB_IPHETH=y +CONFIG_USB_SIERRA_NET=y +# CONFIG_USB_VL600 is not set +# CONFIG_MSM_RMNET_USB is not set +CONFIG_WLAN=y +# CONFIG_LIBERTAS_THINFIRM is not set +# CONFIG_AT76C50X_USB is not set +# CONFIG_USB_ZD1201 is not set +# CONFIG_USB_NET_RNDIS_WLAN is not set +# CONFIG_RTL8187 is not set +# CONFIG_MAC80211_HWSIM is not set +# CONFIG_LIBRA_SDIOIF is not set +# CONFIG_ATH6K_LEGACY_EXT is not set +# CONFIG_WCNSS_CORE is not set +CONFIG_ATH_COMMON=y +# CONFIG_ATH_DEBUG is not set +CONFIG_ATH9K_HW=y +CONFIG_ATH9K_COMMON=y +CONFIG_ATH9K_BTCOEX_SUPPORT=y +# CONFIG_ATH9K is not set +CONFIG_ATH9K_HTC=y +# CONFIG_ATH9K_HTC_DEBUGFS is not set +# CONFIG_CARL9170 is not set +# CONFIG_ATH6KL is not set +# CONFIG_B43 is not set +# CONFIG_B43LEGACY is not set +CONFIG_BCMDHD=y +CONFIG_BCM4339=y +CONFIG_BCMDHD_FW_PATH="/vendor/firmware/fw_bcmdhd.bin" +CONFIG_BCMDHD_NVRAM_PATH="/etc/wifi/bcmdhd.cal" +# CONFIG_DHD_USE_STATIC_BUF is not set +CONFIG_DHD_USE_SCHED_SCAN=y +# CONFIG_BRCMFMAC is not set +# CONFIG_HOSTAP is not set +# CONFIG_IWM is not set +# CONFIG_LIBERTAS is not set +# CONFIG_P54_COMMON is not set +# CONFIG_RT2X00 is not set +# CONFIG_RTL8192CU is not set +# CONFIG_WL1251 is not set +# CONFIG_WL12XX_MENU is not set +# CONFIG_ZD1211RW is not set +# CONFIG_MWIFIEX is not set + +# +# Enable WiMAX (Networking options) to see the WiMAX drivers +# +# CONFIG_WAN is not set +# CONFIG_ISDN is not set + +# +# Input device support +# +CONFIG_INPUT=y +CONFIG_INPUT_FF_MEMLESS=y +# CONFIG_INPUT_POLLDEV is not set +# CONFIG_INPUT_SPARSEKMAP is not set + +# +# Userland interfaces +# +# CONFIG_INPUT_MOUSEDEV is not set +# CONFIG_INPUT_JOYDEV is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_INPUT_EVBUG is not set +CONFIG_INPUT_KEYRESET=y + +# +# Input Device Drivers +# +CONFIG_INPUT_KEYBOARD=y +# CONFIG_KEYBOARD_ADP5588 is not set +# CONFIG_KEYBOARD_ADP5589 is not set +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_KEYBOARD_QT1070 is not set +# CONFIG_KEYBOARD_QT2160 is not set +# CONFIG_KEYBOARD_LKKBD is not set +CONFIG_KEYBOARD_GPIO=y +# CONFIG_KEYBOARD_TCA6416 is not set +# CONFIG_KEYBOARD_TCA8418 is not set +# CONFIG_KEYBOARD_MATRIX is not set +# CONFIG_KEYBOARD_LM8323 is not set +# CONFIG_KEYBOARD_MAX7359 is not set +# CONFIG_KEYBOARD_MCS is not set +# CONFIG_KEYBOARD_MPR121 is not set +# CONFIG_KEYBOARD_NEWTON is not set +# CONFIG_KEYBOARD_OPENCORES is not set +# CONFIG_KEYBOARD_QPNP is not set +# CONFIG_KEYBOARD_SAMSUNG is not set +# CONFIG_KEYBOARD_STOWAWAY is not set +# CONFIG_KEYBOARD_SUNKBD is not set +# CONFIG_KEYBOARD_OMAP4 is not set +# CONFIG_KEYBOARD_XTKBD is not set +# CONFIG_KEYBOARD_QCIKBD is not set +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +# CONFIG_JOYSTICK_ANALOG is not set +# CONFIG_JOYSTICK_A3D is not set +# CONFIG_JOYSTICK_ADI is not set +# CONFIG_JOYSTICK_COBRA is not set +# CONFIG_JOYSTICK_GF2K is not set +# CONFIG_JOYSTICK_GRIP is not set +# CONFIG_JOYSTICK_GRIP_MP is not set +# CONFIG_JOYSTICK_GUILLEMOT is not set +# CONFIG_JOYSTICK_INTERACT is not set +# CONFIG_JOYSTICK_SIDEWINDER is not set +# CONFIG_JOYSTICK_TMDC is not set +# CONFIG_JOYSTICK_IFORCE is not set +# CONFIG_JOYSTICK_WARRIOR is not set +# CONFIG_JOYSTICK_MAGELLAN is not set +# CONFIG_JOYSTICK_SPACEORB is not set +# CONFIG_JOYSTICK_SPACEBALL is not set +# CONFIG_JOYSTICK_STINGER is not set +# CONFIG_JOYSTICK_TWIDJOY is not set +# CONFIG_JOYSTICK_ZHENHUA is not set +# CONFIG_JOYSTICK_AS5011 is not set +# CONFIG_JOYSTICK_JOYDUMP is not set +CONFIG_JOYSTICK_XPAD=y +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +# CONFIG_TOUCHDISC_VTD518_SHINETSU is not set +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=y +CONFIG_TABLET_USB_AIPTEK=y +CONFIG_TABLET_USB_GTCO=y +CONFIG_TABLET_USB_HANWANG=y +CONFIG_TABLET_USB_KBTAB=y +CONFIG_TABLET_USB_WACOM=y +CONFIG_INPUT_TOUCHSCREEN=y +# CONFIG_TOUCHSCREEN_ADS7846 is not set +# CONFIG_TOUCHSCREEN_AD7877 is not set +# CONFIG_TOUCHSCREEN_ATMEL_MAXTOUCH is not set +# CONFIG_TOUCHSCREEN_AD7879 is not set +# CONFIG_TOUCHSCREEN_ATMEL_MXT is not set +# CONFIG_TOUCHSCREEN_AUO_PIXCIR is not set +# CONFIG_TOUCHSCREEN_BU21013 is not set +# CONFIG_TOUCHSCREEN_CY8CTMG110 is not set +# CONFIG_TOUCHSCREEN_CYTTSP_CORE is not set +# CONFIG_TOUCHSCREEN_DYNAPRO is not set +# CONFIG_TOUCHSCREEN_HAMPSHIRE is not set +# CONFIG_TOUCHSCREEN_EETI is not set +# CONFIG_TOUCHSCREEN_EGALAX is not set +# CONFIG_TOUCHSCREEN_FUJITSU is not set +# CONFIG_TOUCHSCREEN_ILI210X is not set +# CONFIG_TOUCHSCREEN_GUNZE is not set +# CONFIG_TOUCHSCREEN_ELO is not set +# CONFIG_TOUCHSCREEN_WACOM_W8001 is not set +# CONFIG_TOUCHSCREEN_MAX11801 is not set +# CONFIG_TOUCHSCREEN_MCS5000 is not set +# CONFIG_TOUCHSCREEN_MTOUCH is not set +# CONFIG_TOUCHSCREEN_INEXIO is not set +# CONFIG_TOUCHSCREEN_MK712 is not set +# CONFIG_TOUCHSCREEN_PENMOUNT is not set +# CONFIG_TOUCHSCREEN_SYNAPTICS_I2C_RMI is not set +# CONFIG_TOUCHSCREEN_SYNAPTICS_RMI4_I2C is not set +# CONFIG_TOUCHSCREEN_SYNAPTICS_I2C_RMI4 is not set +# CONFIG_TOUCHSCREEN_TOUCHRIGHT is not set +# CONFIG_TOUCHSCREEN_TOUCHWIN is not set +# CONFIG_TOUCHSCREEN_PIXCIR is not set +# CONFIG_TOUCHSCREEN_USB_COMPOSITE is not set +# CONFIG_TOUCHSCREEN_TOUCHIT213 is not set +# CONFIG_TOUCHSCREEN_TSC_SERIO is not set +# CONFIG_TOUCHSCREEN_TSC2005 is not set +# CONFIG_TOUCHSCREEN_TSC2007 is not set +# CONFIG_TOUCHSCREEN_MSM_LEGACY is not set +# CONFIG_TOUCHSCREEN_W90X900 is not set +# CONFIG_TOUCHSCREEN_ST1232 is not set +# CONFIG_TOUCHSCREEN_TPS6507X is not set +# CONFIG_TOUCHSCREEN_CY8C_TS is not set +# CONFIG_TOUCHSCREEN_CYTTSP_I2C_QC is not set +# CONFIG_TOUCHSCREEN_FT5X06 is not set +# CONFIG_TOUCHSCREEN_GEN_VKEYS is not set +CONFIG_TOUCHSCREEN_LGE_SYNAPTICS=y +CONFIG_TOUCHSCREEN_CHARGER_NOTIFY=y +CONFIG_INPUT_MISC=y +# CONFIG_INPUT_AD714X is not set +# CONFIG_INPUT_BMA150 is not set +# CONFIG_INPUT_MMA8450 is not set +# CONFIG_INPUT_MPU3050 is not set +# CONFIG_INPUT_GP2A is not set +# CONFIG_INPUT_GPIO_TILT_POLLED is not set +# CONFIG_INPUT_ATI_REMOTE2 is not set +CONFIG_INPUT_KEYCHORD=y +# CONFIG_INPUT_KEYSPAN_REMOTE is not set +# CONFIG_INPUT_KXTJ9 is not set +# CONFIG_INPUT_POWERMATE is not set +# CONFIG_INPUT_YEALINK is not set +# CONFIG_INPUT_CM109 is not set +CONFIG_INPUT_UINPUT=y +# CONFIG_INPUT_GPIO is not set +# CONFIG_INPUT_ISA1200_FF_MEMLESS is not set +# CONFIG_INPUT_PCF8574 is not set +# CONFIG_INPUT_GPIO_ROTARY_ENCODER is not set +# CONFIG_INPUT_ADXL34X is not set +# CONFIG_INPUT_CMA3000 is not set +# CONFIG_BOSCH_BMA150 is not set +# CONFIG_STM_LIS3DH is not set + +# +# Hardware I/O ports +# +# CONFIG_SERIO is not set +# CONFIG_GAMEPORT is not set + +# +# Character devices +# +CONFIG_VT=y +CONFIG_CONSOLE_TRANSLATIONS=y +CONFIG_VT_CONSOLE=y +CONFIG_VT_CONSOLE_SLEEP=y +CONFIG_HW_CONSOLE=y +# CONFIG_VT_HW_CONSOLE_BINDING is not set +CONFIG_UNIX98_PTYS=y +# CONFIG_DEVPTS_MULTIPLE_INSTANCES is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_SERIAL_NONSTANDARD is not set +# CONFIG_N_GSM is not set +# CONFIG_N_SMUX is not set +# CONFIG_TRACE_SINK is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set + +# +# Serial drivers +# +# CONFIG_SERIAL_8250 is not set + +# +# Non-8250 serial port support +# +# CONFIG_SERIAL_MAX3100 is not set +# CONFIG_SERIAL_MAX3107 is not set +CONFIG_SERIAL_CORE=y +CONFIG_SERIAL_CORE_CONSOLE=y +# CONFIG_SERIAL_MSM is not set +CONFIG_SERIAL_MSM_HS=y +CONFIG_SERIAL_MSM_HSL=y +CONFIG_SERIAL_MSM_HSL_CONSOLE=y +# CONFIG_SERIAL_BCM_BT_LPM is not set +# CONFIG_SERIAL_TIMBERDALE is not set +# CONFIG_SERIAL_ALTERA_JTAGUART is not set +# CONFIG_SERIAL_ALTERA_UART is not set +# CONFIG_SERIAL_IFX6X60 is not set +# CONFIG_SERIAL_MSM_SMD is not set +# CONFIG_SERIAL_XILINX_PS_UART is not set + +# +# Diag Support +# +CONFIG_DIAG_CHAR=y + +# +# DIAG traffic over USB +# +CONFIG_DIAG_OVER_USB=y + +# +# SDIO support for DIAG +# + +# +# HSIC/SMUX support for DIAG +# +# CONFIG_TTY_PRINTK is not set +# CONFIG_HVC_DCC is not set +# CONFIG_IPMI_HANDLER is not set +CONFIG_HW_RANDOM=y +# CONFIG_HW_RANDOM_TIMERIOMEM is not set +CONFIG_HW_RANDOM_MSM=y +# CONFIG_R3964 is not set +# CONFIG_RAW_DRIVER is not set +# CONFIG_TCG_TPM is not set +# CONFIG_DCC_TTY is not set +# CONFIG_RAMOOPS is not set +CONFIG_MSM_ADSPRPC=y +# CONFIG_MMC_GENERIC_CSDIO is not set +CONFIG_I2C=y +CONFIG_I2C_BOARDINFO=y +CONFIG_I2C_COMPAT=y +CONFIG_I2C_CHARDEV=y +# CONFIG_I2C_MUX is not set +CONFIG_I2C_HELPER_AUTO=y + +# +# I2C Hardware Bus support +# + +# +# I2C system bus drivers (mostly embedded / system-on-chip) +# +# CONFIG_I2C_DESIGNWARE_PLATFORM is not set +# CONFIG_I2C_GPIO is not set +CONFIG_I2C_MSM=y +CONFIG_I2C_QUP=y +# CONFIG_I2C_OCORES is not set +# CONFIG_I2C_PCA_PLATFORM is not set +# CONFIG_I2C_PXA_PCI is not set +# CONFIG_I2C_SIMTEC is not set +# CONFIG_I2C_XILINX is not set + +# +# External I2C/SMBus adapter drivers +# +# CONFIG_I2C_DIOLAN_U2C is not set +# CONFIG_I2C_PARPORT_LIGHT is not set +# CONFIG_I2C_TAOS_EVM is not set +# CONFIG_I2C_TINY_USB is not set + +# +# Other I2C/SMBus bus drivers +# +# CONFIG_I2C_DEBUG_CORE is not set +# CONFIG_I2C_DEBUG_ALGO is not set +# CONFIG_I2C_DEBUG_BUS is not set +CONFIG_SPI=y +# CONFIG_SPI_DEBUG is not set +CONFIG_SPI_MASTER=y + +# +# SPI Master Controller Drivers +# +# CONFIG_SPI_ALTERA is not set +# CONFIG_SPI_BITBANG is not set +# CONFIG_SPI_GPIO is not set +# CONFIG_SPI_OC_TINY is not set +# CONFIG_SPI_PXA2XX_PCI is not set +# CONFIG_SPI_XILINX is not set +CONFIG_SPI_QUP=y +# CONFIG_SPI_DESIGNWARE is not set + +# +# SPI Protocol Masters +# +# CONFIG_SPI_SPIDEV is not set +# CONFIG_SPI_TLE62X0 is not set +CONFIG_SPMI=y +CONFIG_SPMI_MSM_PMIC_ARB=y +CONFIG_MSM_QPNP_INT=y +CONFIG_SLIMBUS=y +# CONFIG_SLIMBUS_MSM_CTRL is not set +CONFIG_SLIMBUS_MSM_NGD=y +# CONFIG_HSI is not set + +# +# PPS support +# +# CONFIG_PPS is not set + +# +# PPS generators support +# + +# +# PTP clock support +# + +# +# Enable Device Drivers -> PPS to see the PTP clock options. +# +CONFIG_ARCH_REQUIRE_GPIOLIB=y +CONFIG_GPIOLIB=y +CONFIG_DEBUG_GPIO=y +CONFIG_GPIO_SYSFS=y + +# +# Memory mapped GPIO drivers: +# +# CONFIG_GPIO_GENERIC_PLATFORM is not set +# CONFIG_GPIO_MSM_V1 is not set +# CONFIG_GPIO_MSM_V2 is not set +CONFIG_GPIO_MSM_V3=y +# CONFIG_GPIO_FSM9XXX is not set + +# +# I2C GPIO expanders: +# +# CONFIG_GPIO_MAX7300 is not set +# CONFIG_GPIO_MAX732X is not set +# CONFIG_GPIO_PCF857X is not set +# CONFIG_GPIO_SX150X is not set +# CONFIG_GPIO_ADP5588 is not set + +# +# PCI GPIO expanders: +# + +# +# SPI GPIO expanders: +# +# CONFIG_GPIO_MAX7301 is not set +# CONFIG_GPIO_MCP23S08 is not set +# CONFIG_GPIO_MC33880 is not set +# CONFIG_GPIO_74X164 is not set + +# +# AC97 GPIO expanders: +# + +# +# MODULbus GPIO expanders: +# +# CONFIG_GPIO_PM8XXX_RPC is not set +CONFIG_GPIO_QPNP_PIN=y +CONFIG_GPIO_QPNP_PIN_DEBUG=y +# CONFIG_W1 is not set +CONFIG_POWER_SUPPLY=y +# CONFIG_POWER_SUPPLY_DEBUG is not set +# CONFIG_PDA_POWER is not set +# CONFIG_TEST_POWER is not set +# CONFIG_BATTERY_DS2780 is not set +# CONFIG_BATTERY_DS2781 is not set +# CONFIG_BATTERY_DS2782 is not set +# CONFIG_BATTERY_SBS is not set +# CONFIG_BATTERY_BQ27x00 is not set +# CONFIG_BATTERY_MAX17040 is not set +# CONFIG_BATTERY_MAX17042 is not set +# CONFIG_CHARGER_ISP1704 is not set +# CONFIG_CHARGER_MAX8903 is not set +# CONFIG_CHARGER_LP8727 is not set +# CONFIG_CHARGER_GPIO is not set +# CONFIG_CHARGER_MANAGER is not set +# CONFIG_BATTERY_MSM8X60 is not set +# CONFIG_SMB137B_CHARGER is not set +# CONFIG_SMB137C_CHARGER is not set +CONFIG_SMB349_CHARGER=y +# CONFIG_SMB350_CHARGER is not set +# CONFIG_BATTERY_BQ27520 is not set +# CONFIG_BATTERY_BQ27541 is not set +# CONFIG_BATTERY_BQ28400 is not set +CONFIG_QPNP_CHARGER=y +# CONFIG_LTC4088_CHARGER is not set +# CONFIG_BATTERY_BCL is not set +# CONFIG_CHARGER_SMB347 is not set +# CONFIG_QPNP_BMS is not set +CONFIG_MAX17048_FUELGAUGE=y +CONFIG_BQ51013B_CHARGER=y +CONFIG_BQ24192_CHARGER=y +CONFIG_BATTERY_TEMP_CONTROL=y +CONFIG_HWMON=y +# CONFIG_HWMON_VID is not set +# CONFIG_HWMON_DEBUG_CHIP is not set + +# +# Native drivers +# +# CONFIG_SENSORS_AD7314 is not set +# CONFIG_SENSORS_AD7414 is not set +# CONFIG_SENSORS_AD7418 is not set +# CONFIG_SENSORS_ADCXX is not set +# CONFIG_SENSORS_ADM1021 is not set +# CONFIG_SENSORS_ADM1025 is not set +# CONFIG_SENSORS_ADM1026 is not set +# CONFIG_SENSORS_ADM1029 is not set +# CONFIG_SENSORS_ADM1031 is not set +# CONFIG_SENSORS_ADM9240 is not set +# CONFIG_SENSORS_ADT7411 is not set +# CONFIG_SENSORS_ADT7462 is not set +# CONFIG_SENSORS_ADT7470 is not set +# CONFIG_SENSORS_ADT7475 is not set +# CONFIG_SENSORS_ASC7621 is not set +# CONFIG_SENSORS_ATXP1 is not set +# CONFIG_SENSORS_DS620 is not set +# CONFIG_SENSORS_DS1621 is not set +# CONFIG_SENSORS_F71805F is not set +# CONFIG_SENSORS_F71882FG is not set +# CONFIG_SENSORS_F75375S is not set +# CONFIG_SENSORS_G760A is not set +# CONFIG_SENSORS_GL518SM is not set +# CONFIG_SENSORS_GL520SM is not set +# CONFIG_SENSORS_GPIO_FAN is not set +# CONFIG_SENSORS_IT87 is not set +# CONFIG_SENSORS_JC42 is not set +# CONFIG_SENSORS_LINEAGE is not set +# CONFIG_SENSORS_LM63 is not set +# CONFIG_SENSORS_LM70 is not set +# CONFIG_SENSORS_LM73 is not set +# CONFIG_SENSORS_LM75 is not set +# CONFIG_SENSORS_LM77 is not set +# CONFIG_SENSORS_LM78 is not set +# CONFIG_SENSORS_LM80 is not set +# CONFIG_SENSORS_LM83 is not set +# CONFIG_SENSORS_LM85 is not set +# CONFIG_SENSORS_LM87 is not set +# CONFIG_SENSORS_LM90 is not set +# CONFIG_SENSORS_LM92 is not set +# CONFIG_SENSORS_LM93 is not set +# CONFIG_SENSORS_LTC4151 is not set +# CONFIG_SENSORS_LTC4215 is not set +# CONFIG_SENSORS_LTC4245 is not set +# CONFIG_SENSORS_LTC4261 is not set +# CONFIG_SENSORS_LM95241 is not set +# CONFIG_SENSORS_LM95245 is not set +# CONFIG_SENSORS_MAX1111 is not set +# CONFIG_SENSORS_MAX16065 is not set +# CONFIG_SENSORS_MAX1619 is not set +# CONFIG_SENSORS_MAX1668 is not set +# CONFIG_SENSORS_MAX6639 is not set +# CONFIG_SENSORS_MAX6642 is not set +# CONFIG_SENSORS_MAX6650 is not set +# CONFIG_SENSORS_MCP3021 is not set +# CONFIG_SENSORS_NTC_THERMISTOR is not set +CONFIG_SENSORS_EPM_ADC=y +CONFIG_SENSORS_QPNP_ADC_VOLTAGE=y +CONFIG_SENSORS_QPNP_ADC_CURRENT=y +# CONFIG_SENSORS_PC87360 is not set +# CONFIG_SENSORS_PC87427 is not set +# CONFIG_SENSORS_PCF8591 is not set +# CONFIG_PMBUS is not set +# CONFIG_SENSORS_SHT15 is not set +# CONFIG_SENSORS_SHT21 is not set +# CONFIG_SENSORS_SMM665 is not set +# CONFIG_SENSORS_DME1737 is not set +# CONFIG_SENSORS_EMC1403 is not set +# CONFIG_SENSORS_EMC2103 is not set +# CONFIG_SENSORS_EMC6W201 is not set +# CONFIG_SENSORS_SMSC47M1 is not set +# CONFIG_SENSORS_SMSC47M192 is not set +# CONFIG_SENSORS_SMSC47B397 is not set +# CONFIG_SENSORS_SCH56XX_COMMON is not set +# CONFIG_SENSORS_SCH5627 is not set +# CONFIG_SENSORS_SCH5636 is not set +# CONFIG_SENSORS_ADS1015 is not set +# CONFIG_SENSORS_ADS7828 is not set +# CONFIG_SENSORS_ADS7871 is not set +# CONFIG_SENSORS_AMC6821 is not set +# CONFIG_SENSORS_THMC50 is not set +# CONFIG_SENSORS_TMP102 is not set +# CONFIG_SENSORS_TMP401 is not set +# CONFIG_SENSORS_TMP421 is not set +# CONFIG_SENSORS_VT1211 is not set +# CONFIG_SENSORS_W83781D is not set +# CONFIG_SENSORS_W83791D is not set +# CONFIG_SENSORS_W83792D is not set +# CONFIG_SENSORS_W83793 is not set +# CONFIG_SENSORS_W83795 is not set +# CONFIG_SENSORS_W83L785TS is not set +# CONFIG_SENSORS_W83L786NG is not set +# CONFIG_SENSORS_W83627HF is not set +# CONFIG_SENSORS_W83627EHF is not set +CONFIG_THERMAL=y +CONFIG_THERMAL_HWMON=y +# CONFIG_THERMAL_MSM_POPMEM is not set +# CONFIG_THERMAL_TSENS is not set +# CONFIG_THERMAL_TSENS8960 is not set +CONFIG_THERMAL_TSENS8974=y +CONFIG_THERMAL_MONITOR=y +CONFIG_THERMAL_QPNP=y +CONFIG_THERMAL_QPNP_ADC_TM=y +# CONFIG_WATCHDOG is not set +CONFIG_SSB_POSSIBLE=y + +# +# Sonics Silicon Backplane +# +# CONFIG_SSB is not set +CONFIG_BCMA_POSSIBLE=y + +# +# Broadcom specific AMBA +# +# CONFIG_BCMA is not set + +# +# Multifunction device drivers +# +CONFIG_MFD_CORE=y +# CONFIG_MFD_88PM860X is not set +# CONFIG_MFD_SM501 is not set +# CONFIG_MFD_ASIC3 is not set +# CONFIG_HTC_EGPIO is not set +# CONFIG_HTC_PASIC3 is not set +# CONFIG_HTC_I2CPLD is not set +# CONFIG_TPS6105X is not set +# CONFIG_TPS65010 is not set +# CONFIG_TPS6507X is not set +# CONFIG_MFD_TPS65217 is not set +# CONFIG_MFD_TPS6586X is not set +# CONFIG_MFD_TPS65910 is not set +# CONFIG_MFD_TPS65912_I2C is not set +# CONFIG_MFD_TPS65912_SPI is not set +# CONFIG_TWL4030_CORE is not set +# CONFIG_TWL6040_CORE is not set +# CONFIG_MFD_STMPE is not set +# CONFIG_MFD_TC3589X is not set +# CONFIG_MFD_TMIO is not set +# CONFIG_MFD_T7L66XB is not set +# CONFIG_MFD_TC6387XB is not set +# CONFIG_MFD_TC6393XB is not set +# CONFIG_PMIC_DA903X is not set +# CONFIG_MFD_DA9052_SPI is not set +# CONFIG_MFD_DA9052_I2C is not set +# CONFIG_PMIC_ADP5520 is not set +# CONFIG_MFD_MAX8925 is not set +# CONFIG_MFD_MAX8997 is not set +# CONFIG_MFD_MAX8998 is not set +# CONFIG_MFD_S5M_CORE is not set +# CONFIG_MFD_WM8400 is not set +# CONFIG_MFD_WM831X_I2C is not set +# CONFIG_MFD_WM831X_SPI is not set +# CONFIG_MFD_WM8350_I2C is not set +# CONFIG_MFD_WM8994 is not set +# CONFIG_MFD_PCF50633 is not set +# CONFIG_MFD_MC13XXX is not set +# CONFIG_ABX500_CORE is not set +# CONFIG_EZX_PCAP is not set +# CONFIG_MFD_WL1273_CORE is not set +# CONFIG_MFD_PM8821_IRQ is not set +# CONFIG_MFD_TPS65090 is not set +# CONFIG_MFD_AAT2870_CORE is not set +# CONFIG_WCD9304_CODEC is not set +# CONFIG_WCD9310_CODEC is not set +CONFIG_WCD9320_CODEC=y +# CONFIG_WCD9306_CODEC is not set +# CONFIG_MFD_RC5T583 is not set +CONFIG_REGULATOR=y +# CONFIG_REGULATOR_DEBUG is not set +# CONFIG_REGULATOR_DUMMY is not set +CONFIG_REGULATOR_FIXED_VOLTAGE=y +# CONFIG_REGULATOR_VIRTUAL_CONSUMER is not set +# CONFIG_REGULATOR_USERSPACE_CONSUMER is not set +# CONFIG_REGULATOR_GPIO is not set +# CONFIG_REGULATOR_AD5398 is not set +# CONFIG_REGULATOR_ISL6271A is not set +# CONFIG_REGULATOR_MAX1586 is not set +# CONFIG_REGULATOR_MAX8649 is not set +# CONFIG_REGULATOR_MAX8660 is not set +# CONFIG_REGULATOR_MAX8952 is not set +# CONFIG_REGULATOR_ONSEMI_NCP6335D is not set +# CONFIG_REGULATOR_LP3971 is not set +# CONFIG_REGULATOR_LP3972 is not set +# CONFIG_REGULATOR_TPS62360 is not set +# CONFIG_REGULATOR_TPS65023 is not set +# CONFIG_REGULATOR_TPS6507X is not set +# CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_REGULATOR_MSM_GPIO is not set +CONFIG_REGULATOR_STUB=y +CONFIG_REGULATOR_QPNP=y +CONFIG_MEDIA_SUPPORT=y + +# +# Multimedia core support +# +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_DEV=y +CONFIG_VIDEO_V4L2_COMMON=y +CONFIG_VIDEO_V4L2_SUBDEV_API=y +# CONFIG_DVB_CORE is not set +CONFIG_VIDEO_MEDIA=y + +# +# Multimedia drivers +# +CONFIG_RC_CORE=y +CONFIG_LIRC=y +# CONFIG_USER_RC_INPUT is not set +# CONFIG_USER_SP_RC_INPUT is not set +CONFIG_RC_MAP=y +CONFIG_IR_NEC_DECODER=y +CONFIG_IR_RC5_DECODER=y +CONFIG_IR_RC6_DECODER=y +CONFIG_IR_JVC_DECODER=y +CONFIG_IR_SONY_DECODER=y +CONFIG_IR_RC5_SZ_DECODER=y +CONFIG_IR_SANYO_DECODER=y +CONFIG_IR_MCE_KBD_DECODER=y +CONFIG_IR_LIRC_CODEC=y +# CONFIG_RC_ATI_REMOTE is not set +# CONFIG_IR_IMON is not set +# CONFIG_IR_MCEUSB is not set +# CONFIG_IR_REDRAT3 is not set +# CONFIG_IR_STREAMZAP is not set +# CONFIG_RC_LOOPBACK is not set +# CONFIG_IR_GPIO_CIR is not set + +# +# Qualcomm MSM Camera And Video +# +# CONFIG_MSM_CAMERA is not set +# CONFIG_MT9M114 is not set +# CONFIG_OV2720 is not set +# CONFIG_OV8825 is not set +# CONFIG_IMX135 is not set +CONFIG_MSM_CAMERA_SENSOR=y +CONFIG_MSM_EEPROM=y +CONFIG_MSM_CPP=y +CONFIG_MSM_CCI=y +# CONFIG_MSM_CSI20_HEADER is not set +CONFIG_MSM_CSI30_HEADER=y +CONFIG_MSM_CSIPHY=y +CONFIG_MSM_CSID=y +CONFIG_MSM_ISPIF=y +# CONFIG_S5K3L1YX is not set +# CONFIG_MSM_V4L2_VIDEO_OVERLAY_DEVICE is not set +CONFIG_MSMB_CAMERA=y +# CONFIG_MSMB_CAMERA_DEBUG is not set +CONFIG_IMX179=y +CONFIG_OIS_CONTROLLER=y +CONFIG_OIS_ROHM_BU24205GWL=y +CONFIG_OIS_ONSEMI_LC898111A=y +# CONFIG_MSM_CSI22_HEADER is not set +# CONFIG_MSM_ISPIF_V1 is not set +# CONFIG_OV9724 is not set +CONFIG_MT9M114B=y +CONFIG_MSMB_JPEG=y +CONFIG_MSM_VIDC_V4L2=y +# CONFIG_MSM_WFD is not set +CONFIG_MSM_VCAP=y +CONFIG_MEDIA_TUNER=y +CONFIG_MEDIA_TUNER_CUSTOMISE=y + +# +# Customize TV tuners +# +# CONFIG_MEDIA_TUNER_SIMPLE is not set +# CONFIG_MEDIA_TUNER_TDA8290 is not set +# CONFIG_MEDIA_TUNER_TDA827X is not set +# CONFIG_MEDIA_TUNER_TDA18271 is not set +# CONFIG_MEDIA_TUNER_TDA9887 is not set +# CONFIG_MEDIA_TUNER_TEA5761 is not set +# CONFIG_MEDIA_TUNER_TEA5767 is not set +# CONFIG_MEDIA_TUNER_MT20XX is not set +# CONFIG_MEDIA_TUNER_MT2060 is not set +# CONFIG_MEDIA_TUNER_MT2063 is not set +# CONFIG_MEDIA_TUNER_MT2266 is not set +# CONFIG_MEDIA_TUNER_MT2131 is not set +# CONFIG_MEDIA_TUNER_QT1010 is not set +# CONFIG_MEDIA_TUNER_XC2028 is not set +# CONFIG_MEDIA_TUNER_XC5000 is not set +# CONFIG_MEDIA_TUNER_XC4000 is not set +# CONFIG_MEDIA_TUNER_MXL5005S is not set +# CONFIG_MEDIA_TUNER_MXL5007T is not set +# CONFIG_MEDIA_TUNER_MC44S803 is not set +# CONFIG_MEDIA_TUNER_MAX2165 is not set +# CONFIG_MEDIA_TUNER_TDA18218 is not set +# CONFIG_MEDIA_TUNER_TDA18212 is not set +CONFIG_VIDEO_V4L2=y +CONFIG_VIDEOBUF2_CORE=y +CONFIG_VIDEOBUF2_MEMOPS=y +CONFIG_VIDEOBUF2_DMA_CONTIG=y +CONFIG_VIDEOBUF2_VMALLOC=y +CONFIG_VIDEOBUF2_DMA_SG=y +CONFIG_VIDEOBUF2_MSM_MEM=y +CONFIG_VIDEO_CAPTURE_DRIVERS=y +# CONFIG_VIDEO_ADV_DEBUG is not set +# CONFIG_VIDEO_FIXED_MINOR_RANGES is not set +# CONFIG_VIDEO_HELPER_CHIPS_AUTO is not set +CONFIG_VIDEO_IR_I2C=y + +# +# Encoders, decoders, sensors and other helper chips +# + +# +# Audio decoders, processors and mixers +# +# CONFIG_VIDEO_TVAUDIO is not set +# CONFIG_VIDEO_TDA7432 is not set +# CONFIG_VIDEO_TDA9840 is not set +# CONFIG_VIDEO_TEA6415C is not set +# CONFIG_VIDEO_TEA6420 is not set +# CONFIG_VIDEO_MSP3400 is not set +# CONFIG_VIDEO_CS5345 is not set +# CONFIG_VIDEO_CS53L32A is not set +# CONFIG_VIDEO_TLV320AIC23B is not set +# CONFIG_VIDEO_WM8775 is not set +# CONFIG_VIDEO_WM8739 is not set +# CONFIG_VIDEO_VP27SMPX is not set + +# +# RDS decoders +# +# CONFIG_VIDEO_SAA6588 is not set + +# +# Video decoders +# +# CONFIG_VIDEO_ADV7180 is not set +# CONFIG_VIDEO_ADV7183 is not set +# CONFIG_VIDEO_BT819 is not set +# CONFIG_VIDEO_BT856 is not set +# CONFIG_VIDEO_BT866 is not set +# CONFIG_VIDEO_KS0127 is not set +# CONFIG_VIDEO_SAA7110 is not set +# CONFIG_VIDEO_SAA711X is not set +# CONFIG_VIDEO_SAA7191 is not set +# CONFIG_VIDEO_TVP514X is not set +# CONFIG_VIDEO_TVP5150 is not set +# CONFIG_VIDEO_TVP7002 is not set +# CONFIG_VIDEO_VPX3220 is not set + +# +# Video and audio decoders +# +# CONFIG_VIDEO_SAA717X is not set +# CONFIG_VIDEO_CX25840 is not set + +# +# MPEG video encoders +# +# CONFIG_VIDEO_CX2341X is not set + +# +# Video encoders +# +# CONFIG_VIDEO_SAA7127 is not set +# CONFIG_VIDEO_SAA7185 is not set +# CONFIG_VIDEO_ADV7170 is not set +# CONFIG_VIDEO_ADV7175 is not set +# CONFIG_VIDEO_ADV7343 is not set +# CONFIG_VIDEO_AK881X is not set + +# +# Camera sensor devices +# +# CONFIG_VIDEO_OV7670 is not set +# CONFIG_VIDEO_VS6624 is not set +# CONFIG_VIDEO_MT9M032 is not set +# CONFIG_VIDEO_MT9P031 is not set +# CONFIG_VIDEO_MT9T001 is not set +# CONFIG_VIDEO_MT9V011 is not set +# CONFIG_VIDEO_MT9V032 is not set +# CONFIG_VIDEO_TCM825X is not set +# CONFIG_VIDEO_SR030PC30 is not set +# CONFIG_VIDEO_NOON010PC30 is not set +# CONFIG_VIDEO_M5MOLS is not set +# CONFIG_VIDEO_S5K6AA is not set + +# +# Flash devices +# +# CONFIG_VIDEO_ADP1653 is not set +# CONFIG_VIDEO_AS3645A is not set + +# +# Video improvement chips +# +# CONFIG_VIDEO_UPD64031A is not set +# CONFIG_VIDEO_UPD64083 is not set + +# +# Miscelaneous helper chips +# +# CONFIG_VIDEO_THS7303 is not set +# CONFIG_VIDEO_M52790 is not set +CONFIG_V4L_USB_DRIVERS=y +CONFIG_USB_VIDEO_CLASS=y +CONFIG_USB_VIDEO_CLASS_INPUT_EVDEV=y +# CONFIG_USB_GSPCA is not set +# CONFIG_VIDEO_PVRUSB2 is not set +# CONFIG_VIDEO_HDPVR is not set +# CONFIG_VIDEO_EM28XX is not set +# CONFIG_VIDEO_CX231XX is not set +# CONFIG_VIDEO_TM6000 is not set +# CONFIG_VIDEO_USBVISION is not set +# CONFIG_USB_ET61X251 is not set +# CONFIG_USB_SN9C102 is not set +# CONFIG_USB_PWC is not set +# CONFIG_VIDEO_CPIA2 is not set +# CONFIG_USB_ZR364XX is not set +# CONFIG_USB_STKWEBCAM is not set +# CONFIG_USB_S2255 is not set +CONFIG_V4L_PLATFORM_DRIVERS=y +# CONFIG_SOC_CAMERA is not set +# CONFIG_V4L_MEM2MEM_DRIVERS is not set +CONFIG_RADIO_ADAPTERS=y +# CONFIG_RADIO_SI470X is not set +# CONFIG_USB_MR800 is not set +# CONFIG_USB_DSBR is not set +# CONFIG_I2C_SI4713 is not set +# CONFIG_RADIO_SI4713 is not set +# CONFIG_USB_KEENE is not set +# CONFIG_RADIO_TEA5764 is not set +# CONFIG_RADIO_SAA7706H is not set +# CONFIG_RADIO_TEF6862 is not set +# CONFIG_RADIO_WL1273 is not set + +# +# Texas Instruments WL128x FM driver (ST based) +# +# CONFIG_RADIO_WL128X is not set +# CONFIG_RADIO_IRIS is not set + +# +# Graphics support +# +# CONFIG_DRM is not set +CONFIG_ION=y +CONFIG_ION_MSM=y +CONFIG_MSM_KGSL=y +# CONFIG_MSM_KGSL_CFF_DUMP is not set +# CONFIG_MSM_KGSL_PSTMRTMDMP_CP_STAT_NO_DETAIL is not set +# CONFIG_MSM_KGSL_PSTMRTMDMP_NO_IB_DUMP is not set +# CONFIG_MSM_KGSL_PSTMRTMDMP_RB_HEX is not set +CONFIG_MSM_KGSL_2D=y +CONFIG_KGSL_PER_PROCESS_PAGE_TABLE=y +CONFIG_MSM_KGSL_PAGE_TABLE_SIZE=0xFFF0000 +CONFIG_MSM_KGSL_PAGE_TABLE_COUNT=8 +CONFIG_MSM_KGSL_MMU_PAGE_FAULT=y +# CONFIG_MSM_KGSL_DISABLE_SHADOW_WRITES is not set +# CONFIG_VGASTATE is not set +# CONFIG_VIDEO_OUTPUT_CONTROL is not set +CONFIG_FB=y +# CONFIG_FIRMWARE_EDID is not set +# CONFIG_FB_DDC is not set +# CONFIG_FB_BOOT_VESA_SUPPORT is not set +CONFIG_FB_CFB_FILLRECT=y +CONFIG_FB_CFB_COPYAREA=y +CONFIG_FB_CFB_IMAGEBLIT=y +# CONFIG_FB_CFB_REV_PIXELS_IN_BYTE is not set +# CONFIG_FB_SYS_FILLRECT is not set +# CONFIG_FB_SYS_COPYAREA is not set +# CONFIG_FB_SYS_IMAGEBLIT is not set +# CONFIG_FB_FOREIGN_ENDIAN is not set +# CONFIG_FB_SYS_FOPS is not set +# CONFIG_FB_WMT_GE_ROPS is not set +# CONFIG_FB_SVGALIB is not set +# CONFIG_FB_MACMODES is not set +# CONFIG_FB_BACKLIGHT is not set +# CONFIG_FB_MODE_HELPERS is not set +# CONFIG_FB_TILEBLITTING is not set + +# +# Frame buffer hardware drivers +# +# CONFIG_FB_S1D13XXX is not set +# CONFIG_FB_TMIO is not set +# CONFIG_FB_SMSCUFX is not set +# CONFIG_FB_UDL is not set +# CONFIG_FB_VIRTUAL is not set +# CONFIG_FB_METRONOME is not set +# CONFIG_FB_BROADSHEET is not set +# CONFIG_MSM_VIDC_CONTENT_PROTECTION is not set +CONFIG_FB_MSM=y +# CONFIG_FB_MSM_BACKLIGHT is not set +# CONFIG_FB_MSM_LOGO is not set +# CONFIG_FB_MSM_LCDC_HW is not set +# CONFIG_FB_MSM_TRIPLE_BUFFER is not set +# CONFIG_FB_MSM_MDP_HW is not set +CONFIG_FB_MSM_MDSS_COMMON=y +# CONFIG_MDP_DEBUG_FS is not set +# CONFIG_FB_MSM_MDP22 is not set +# CONFIG_FB_MSM_MDP30 is not set +# CONFIG_FB_MSM_MDP31 is not set +# CONFIG_FB_MSM_MDP40 is not set +CONFIG_FB_MSM_MDSS=y +# CONFIG_FB_MSM_MDP_NONE is not set +# CONFIG_FB_MSM_EBI2 is not set +# CONFIG_FB_MSM_MDDI is not set +# CONFIG_FB_MSM_MIPI_DSI is not set +# CONFIG_FB_MSM_LCDC is not set +CONFIG_FB_MSM_LVDS=y +# CONFIG_FB_MSM_EXTMDDI is not set +# CONFIG_FB_MSM_TVOUT is not set +# CONFIG_FB_MSM_MDDI_TOSHIBA_COMMON is not set +# CONFIG_FB_MSM_MDDI_TOSHIBA_COMMON_VGA is not set +# CONFIG_FB_MSM_MDDI_ORISE is not set +# CONFIG_FB_MSM_MDDI_QUICKVX is not set +# CONFIG_FB_MSM_MDDI_AUTO_DETECT is not set +# CONFIG_FB_MSM_LCDC_AUTO_DETECT is not set +# CONFIG_FB_MSM_LCDC_PANEL is not set +# CONFIG_FB_MSM_MIPI_DSI_TOSHIBA is not set +# CONFIG_FB_MSM_MIPI_DSI_RENESAS is not set +# CONFIG_FB_MSM_MIPI_DSI_SIMULATOR is not set +# CONFIG_FB_MSM_MIPI_DSI_NOVATEK is not set +# CONFIG_FB_MSM_MIPI_DSI_ORISE is not set +# CONFIG_FB_MSM_LCDC_ST15_WXGA is not set +# CONFIG_FB_MSM_LCDC_PRISM_WVGA is not set +# CONFIG_FB_MSM_LCDC_SAMSUNG_WSVGA is not set +# CONFIG_FB_MSM_LCDC_CHIMEI_WXGA is not set +# CONFIG_FB_MSM_LCDC_GORDON_VGA is not set +# CONFIG_FB_MSM_LCDC_TOSHIBA_WVGA_PT is not set +# CONFIG_FB_MSM_LCDC_TOSHIBA_FWVGA_PT is not set +# CONFIG_FB_MSM_LCDC_SHARP_WVGA_PT is not set +# CONFIG_FB_MSM_LCDC_AUO_WVGA is not set +# CONFIG_FB_MSM_LCDC_TRULY_HVGA_IPS3P2335 is not set +# CONFIG_FB_MSM_LCDC_SAMSUNG_OLED_PT is not set +# CONFIG_FB_MSM_LCDC_NT35582_WVGA is not set +# CONFIG_FB_MSM_LCDC_WXGA is not set +CONFIG_FB_MSM_LVDS_CHIMEI_WXGA=y +# CONFIG_FB_MSM_LVDS_FRC_FHD is not set +# CONFIG_FB_MSM_MIPI_TOSHIBA_VIDEO_WVGA_PT is not set +# CONFIG_FB_MSM_MIPI_TOSHIBA_VIDEO_WSVGA_PT is not set +# CONFIG_FB_MSM_MIPI_TOSHIBA_VIDEO_WUXGA is not set +# CONFIG_FB_MSM_MIPI_NOVATEK_VIDEO_QHD_PT is not set +# CONFIG_FB_MSM_MIPI_NOVATEK_CMD_QHD_PT is not set +# CONFIG_FB_MSM_MIPI_ORISE_VIDEO_720P_PT is not set +# CONFIG_FB_MSM_MIPI_ORISE_CMD_720P_PT is not set +# CONFIG_FB_MSM_MIPI_RENESAS_VIDEO_FWVGA_PT is not set +# CONFIG_FB_MSM_MIPI_RENESAS_CMD_FWVGA_PT is not set +# CONFIG_FB_MSM_MIPI_NT35510_VIDEO_WVGA_PT is not set +# CONFIG_FB_MSM_MIPI_NT35510_CMD_WVGA_PT is not set +# CONFIG_FB_MSM_MIPI_NT35516_VIDEO_QHD_PT is not set +# CONFIG_FB_MSM_MIPI_NT35516_CMD_QHD_PT is not set +# CONFIG_FB_MSM_MIPI_CHIMEI_WXGA is not set +# CONFIG_FB_MSM_MIPI_CHIMEI_WUXGA is not set +# CONFIG_FB_MSM_MIPI_SIMULATOR_VIDEO is not set +CONFIG_FB_MSM_LVDS_CHIMEI_WXGA_PANEL=y +# CONFIG_FB_MSM_LVDS_FRC_FHD_PANEL is not set +# CONFIG_FB_MSM_MIPI_PANEL_DETECT is not set +# CONFIG_FB_MSM_MDDI_PANEL_AUTO_DETECT is not set +# CONFIG_FB_MSM_LCDC_PANEL_AUTO_DETECT is not set +# CONFIG_FB_MSM_LCDC_MIPI_PANEL_AUTO_DETECT is not set +# CONFIG_FB_MSM_LVDS_MIPI_PANEL_DETECT is not set +# CONFIG_FB_MSM_MDDI_PRISM_WVGA is not set +# CONFIG_FB_MSM_MDDI_TOSHIBA_WVGA_PORTRAIT is not set +# CONFIG_FB_MSM_MDDI_TOSHIBA_VGA is not set +# CONFIG_FB_MSM_MDDI_TOSHIBA_WVGA is not set +# CONFIG_FB_MSM_MDDI_SHARP_QVGA_128x128 is not set +# CONFIG_FB_MSM_MIPI_TOSHIBA_VIDEO_WVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_TOSHIBA_VIDEO_WSVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_TOSHIBA_VIDEO_WUXGA_PANEL is not set +# CONFIG_FB_MSM_MIPI_NOVATEK_VIDEO_QHD_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_NOVATEK_CMD_QHD_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_ORISE_VIDEO_720P_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_ORISE_CMD_720P_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_RENESAS_VIDEO_FWVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_RENESAS_CMD_FWVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_CHIMEI_WXGA_PANEL is not set +# CONFIG_FB_MSM_MIPI_CHIMEI_WUXGA_PANEL is not set +# CONFIG_FB_MSM_MIPI_TRULY_VIDEO_WVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_NT35510_VIDEO_WVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_NT35510_CMD_WVGA_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_NT35516_VIDEO_QHD_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_NT35516_CMD_QHD_PT_PANEL is not set +# CONFIG_FB_MSM_MIPI_SIMULATOR_VIDEO_PANEL is not set +# CONFIG_FB_MSM_EBI2_TMD_QVGA_EPSON_QCIF is not set +# CONFIG_FB_MSM_PANEL_NONE is not set +# CONFIG_FB_MSM_EXT_INTERFACE_COMMON is not set +# CONFIG_FB_MSM_HDMI_COMMON is not set +# CONFIG_FB_MSM_HDMI_3D is not set +# CONFIG_FB_MSM_EBI2_EPSON_S1D_QVGA_PANEL is not set +# CONFIG_FB_MSM_EBI2_PANEL_DETECT is not set +# CONFIG_FB_MSM_QPIC_ILI_QVGA_PANEL is not set +# CONFIG_FB_MSM_QPIC_PANEL_DETECT is not set +CONFIG_FB_MSM_MDSS_WRITEBACK=y +CONFIG_FB_MSM_MDSS_HDMI_PANEL=y +# CONFIG_FB_MSM_MDSS_HDMI_MHL_SII8334 is not set +# CONFIG_EXYNOS_VIDEO is not set +CONFIG_BACKLIGHT_LCD_SUPPORT=y +# CONFIG_LCD_CLASS_DEVICE is not set +CONFIG_BACKLIGHT_CLASS_DEVICE=y +# CONFIG_BACKLIGHT_GENERIC is not set +# CONFIG_BACKLIGHT_ADP8860 is not set +# CONFIG_BACKLIGHT_ADP8870 is not set +# CONFIG_BACKLIGHT_LP855X is not set +CONFIG_BACKLIGHT_LM3630=y + +# +# Console display driver support +# +CONFIG_DUMMY_CONSOLE=y +# CONFIG_FRAMEBUFFER_CONSOLE is not set +# CONFIG_LOGO is not set +CONFIG_SLIMPORT_ANX7808=y +CONFIG_SOUND=y +# CONFIG_SOUND_OSS_CORE is not set +CONFIG_SND=y +CONFIG_SND_TIMER=y +CONFIG_SND_PCM=y +CONFIG_SND_HWDEP=y +CONFIG_SND_RAWMIDI=y +CONFIG_SND_COMPRESS_OFFLOAD=y +CONFIG_SND_JACK=y +# CONFIG_SND_SEQUENCER is not set +# CONFIG_SND_MIXER_OSS is not set +# CONFIG_SND_PCM_OSS is not set +# CONFIG_SND_HRTIMER is not set +CONFIG_SND_DYNAMIC_MINORS=y +CONFIG_SND_SUPPORT_OLD_API=y +CONFIG_SND_VERBOSE_PROCFS=y +# CONFIG_SND_VERBOSE_PRINTK is not set +# CONFIG_SND_DEBUG is not set +# CONFIG_SND_RAWMIDI_SEQ is not set +# CONFIG_SND_OPL3_LIB_SEQ is not set +# CONFIG_SND_OPL4_LIB_SEQ is not set +# CONFIG_SND_SBAWE_SEQ is not set +# CONFIG_SND_EMU10K1_SEQ is not set +CONFIG_SND_DRIVERS=y +# CONFIG_SND_DUMMY is not set +# CONFIG_SND_ALOOP is not set +# CONFIG_SND_MTPAV is not set +# CONFIG_SND_SERIAL_U16550 is not set +# CONFIG_SND_MPU401 is not set +CONFIG_SND_ARM=y +CONFIG_SND_SPI=y +CONFIG_SND_USB=y +CONFIG_SND_USB_AUDIO=y +# CONFIG_SND_USB_UA101 is not set +# CONFIG_SND_USB_CAIAQ is not set +# CONFIG_SND_USB_6FIRE is not set +CONFIG_SND_SOC=y + +# +# MSM SoC Audio support +# +CONFIG_SND_SOC_MSM_HOSTLESS_PCM=y +CONFIG_SND_SOC_MSM_QDSP6V2_INTF=y +# CONFIG_SND_SOC_QDSP6 is not set +CONFIG_SND_SOC_QDSP6V2=y +CONFIG_AUDIO_OCMEM=y +# CONFIG_DOLBY_DAP is not set +CONFIG_SND_SOC_MSM8974=y +# CONFIG_SND_SOC_APQ8074 is not set +CONFIG_SND_SOC_I2C_AND_SPI=y +# CONFIG_SND_SOC_ALL_CODECS is not set +CONFIG_SND_SOC_WCD9320=y +CONFIG_SND_SOC_MSM_STUB=y +CONFIG_SND_SOC_MSM_HDMI_CODEC_RX=y +# CONFIG_SOUND_PRIME is not set +CONFIG_HID_SUPPORT=y +CONFIG_HID=y +# CONFIG_HID_BATTERY_STRENGTH is not set +# CONFIG_HIDRAW is not set +CONFIG_UHID=y + +# +# USB Input Devices +# +CONFIG_USB_HID=y +# CONFIG_HID_PID is not set +CONFIG_USB_HIDDEV=y + +# +# Special HID drivers +# +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +# CONFIG_HID_ACRUX_FF is not set +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +# CONFIG_DRAGONRISE_FF is not set +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_HOLTEK=y +# CONFIG_HOLTEK_FF is not set +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_UCLOGIC=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +# CONFIG_LOGITECH_FF is not set +# CONFIG_LOGIRUMBLEPAD2_FF is not set +# CONFIG_LOGIG940_FF is not set +# CONFIG_LOGIWHEELS_FF is not set +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_NTRIG=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +# CONFIG_PANTHERLORD_FF is not set +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +# CONFIG_HID_PICOLCD_FB is not set +# CONFIG_HID_PICOLCD_BACKLIGHT is not set +# CONFIG_HID_PICOLCD_LEDS is not set +CONFIG_HID_PRIMAX=y +CONFIG_HID_ROCCAT=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SONY=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +# CONFIG_GREENASIA_FF is not set +CONFIG_HID_SMARTJOYPLUS=y +# CONFIG_SMARTJOYPLUS_FF is not set +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +# CONFIG_THRUSTMASTER_FF is not set +CONFIG_HID_WACOM=y +CONFIG_HID_WACOM_POWER_SUPPLY=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_WIIMOTE_EXT=y +CONFIG_HID_ZEROPLUS=y +# CONFIG_ZEROPLUS_FF is not set +CONFIG_HID_ZYDACRON=y +# CONFIG_USB_ARCH_HAS_OHCI is not set +CONFIG_USB_ARCH_HAS_EHCI=y +CONFIG_USB_ARCH_HAS_XHCI=y +CONFIG_USB_SUPPORT=y +CONFIG_USB_COMMON=y +CONFIG_USB_ARCH_HAS_HCD=y +CONFIG_USB=y +# CONFIG_USB_DEBUG is not set +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y + +# +# Miscellaneous USB options +# +# CONFIG_USB_DEVICEFS is not set +CONFIG_USB_DEVICE_CLASS=y +# CONFIG_USB_DYNAMIC_MINORS is not set +CONFIG_USB_SUSPEND=y +# CONFIG_USB_OTG is not set +# CONFIG_USB_OTG_WHITELIST is not set +# CONFIG_USB_OTG_BLACKLIST_HUB is not set +CONFIG_USB_DWC3=y +# CONFIG_USB_DWC3_DEBUG is not set +# CONFIG_USB_MON is not set +# CONFIG_USB_WUSB_CBAF is not set + +# +# USB Host Controller Drivers +# +# CONFIG_USB_C67X00_HCD is not set +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +# CONFIG_USB_XHCI_HCD_DEBUGGING is not set +# CONFIG_USB_EHCI_HCD is not set +# CONFIG_USB_OXU210HP_HCD is not set +# CONFIG_USB_ISP116X_HCD is not set +# CONFIG_USB_ISP1760_HCD is not set +# CONFIG_USB_ISP1362_HCD is not set +# CONFIG_USB_SL811_HCD is not set +# CONFIG_USB_R8A66597_HCD is not set +# CONFIG_USB_PEHCI_HCD is not set +# CONFIG_USB_MUSB_HDRC is not set +# CONFIG_USB_RENESAS_USBHS is not set + +# +# USB Device Class drivers +# +CONFIG_USB_ACM=y +CONFIG_USB_PRINTER=y +CONFIG_USB_WDM=y +# CONFIG_USB_TMC is not set + +# +# NOTE: USB_STORAGE depends on SCSI but BLK_DEV_SD may +# + +# +# also be needed; see USB_STORAGE Help for more info +# +CONFIG_USB_STORAGE=y +# CONFIG_USB_STORAGE_DEBUG is not set +# CONFIG_USB_STORAGE_REALTEK is not set +CONFIG_USB_STORAGE_DATAFAB=y +CONFIG_USB_STORAGE_FREECOM=y +CONFIG_USB_STORAGE_ISD200=y +CONFIG_USB_STORAGE_USBAT=y +CONFIG_USB_STORAGE_SDDR09=y +CONFIG_USB_STORAGE_SDDR55=y +CONFIG_USB_STORAGE_JUMPSHOT=y +CONFIG_USB_STORAGE_ALAUDA=y +CONFIG_USB_STORAGE_ONETOUCH=y +CONFIG_USB_STORAGE_KARMA=y +CONFIG_USB_STORAGE_CYPRESS_ATACB=y +CONFIG_USB_STORAGE_ENE_UB6250=y +# CONFIG_USB_UAS is not set +# CONFIG_USB_LIBUSUAL is not set + +# +# USB Imaging devices +# +# CONFIG_USB_MDC800 is not set +# CONFIG_USB_MICROTEK is not set + +# +# USB port drivers +# +# CONFIG_USB_SERIAL is not set + +# +# USB Miscellaneous drivers +# +# CONFIG_USB_EMI62 is not set +# CONFIG_USB_EMI26 is not set +# CONFIG_USB_ADUTUX is not set +# CONFIG_USB_SEVSEG is not set +# CONFIG_USB_RIO500 is not set +# CONFIG_USB_LEGOTOWER is not set +# CONFIG_USB_LCD is not set +# CONFIG_USB_LED is not set +# CONFIG_USB_CYPRESS_CY7C63 is not set +# CONFIG_USB_CYTHERM is not set +# CONFIG_USB_IDMOUSE is not set +# CONFIG_USB_FTDI_ELAN is not set +# CONFIG_USB_APPLEDISPLAY is not set +# CONFIG_USB_LD is not set +CONFIG_USB_TRANCEVIBRATOR=y +# CONFIG_USB_IOWARRIOR is not set +# CONFIG_USB_TEST is not set +# CONFIG_USB_ISIGHTFW is not set +# CONFIG_USB_YUREX is not set +# CONFIG_USB_QCOM_DIAG_BRIDGE is not set +# CONFIG_USB_QCOM_MDM_BRIDGE is not set +# CONFIG_USB_QCOM_KS_BRIDGE is not set +CONFIG_USB_GADGET=y +# CONFIG_USB_GADGET_DEBUG is not set +CONFIG_USB_GADGET_DEBUG_FILES=y +# CONFIG_USB_GADGET_DEBUG_FS is not set +CONFIG_USB_GADGET_VBUS_DRAW=500 +CONFIG_USB_GADGET_STORAGE_NUM_BUFFERS=2 + +# +# USB Peripheral Controller +# +# CONFIG_USB_FUSB300 is not set +# CONFIG_USB_R8A66597 is not set +# CONFIG_USB_MV_UDC is not set +# CONFIG_USB_M66592 is not set +# CONFIG_USB_NET2272 is not set +# CONFIG_USB_CI13XXX_MSM is not set +# CONFIG_USB_CI13XXX_MSM_HSIC is not set +CONFIG_USB_DWC3_MSM=y +# CONFIG_USB_MSM_72K is not set +# CONFIG_USB_DUMMY_HCD is not set +# CONFIG_USB_ZERO is not set +# CONFIG_USB_AUDIO is not set +# CONFIG_USB_ETH is not set +# CONFIG_USB_G_NCM is not set +# CONFIG_USB_GADGETFS is not set +# CONFIG_USB_FUNCTIONFS is not set +# CONFIG_USB_FILE_STORAGE is not set +# CONFIG_USB_MASS_STORAGE is not set +# CONFIG_USB_G_SERIAL is not set +# CONFIG_USB_MIDI_GADGET is not set +# CONFIG_USB_G_PRINTER is not set +CONFIG_USB_G_ANDROID=y +# CONFIG_USB_CDC_COMPOSITE is not set +# CONFIG_USB_G_ACM_MS is not set +# CONFIG_USB_G_MULTI is not set +# CONFIG_USB_G_HID is not set +# CONFIG_USB_G_DBGP is not set +# CONFIG_USB_G_WEBCAM is not set +CONFIG_USB_CSW_HACK=y +# CONFIG_USB_MSC_PROFILING is not set +CONFIG_MODEM_SUPPORT=y +CONFIG_RMNET_SMD_CTL_CHANNEL="" +CONFIG_RMNET_SMD_DATA_CHANNEL="" + +# +# OTG and related infrastructure +# +CONFIG_USB_OTG_UTILS=y +# CONFIG_USB_OTG_WAKELOCK is not set +# CONFIG_USB_GPIO_VBUS is not set +# CONFIG_USB_ULPI is not set +# CONFIG_USB_MSM_OTG_72K is not set +# CONFIG_NOP_USB_XCEIV is not set +# CONFIG_USB_MSM_OTG is not set +CONFIG_MMC=y +# CONFIG_MMC_DEBUG is not set +CONFIG_MMC_PERF_PROFILING=y +CONFIG_MMC_UNSAFE_RESUME=y +CONFIG_MMC_CLKGATE=y +# CONFIG_MMC_EMBEDDED_SDIO is not set +CONFIG_MMC_PARANOID_SD_INIT=y + +# +# MMC/SD/SDIO Card Drivers +# +CONFIG_MMC_BLOCK=y +CONFIG_MMC_BLOCK_MINORS=32 +# CONFIG_MMC_BLOCK_BOUNCE is not set +# CONFIG_MMC_BLOCK_DEFERRED_RESUME is not set +# CONFIG_SDIO_UART is not set +# CONFIG_MMC_TEST is not set +CONFIG_MMC_BLOCK_TEST=y + +# +# MMC/SD/SDIO Host Controller Drivers +# +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y +# CONFIG_MMC_SDHCI_PXAV3 is not set +# CONFIG_MMC_SDHCI_PXAV2 is not set +CONFIG_MMC_SDHCI_MSM=y +CONFIG_MMC_SDHCI_MSM_DISABLE_HPI=y +CONFIG_MMC_MSM=y +CONFIG_MMC_MSM_SDC1_SUPPORT=y +# CONFIG_MMC_MSM_SDC1_8_BIT_SUPPORT is not set +CONFIG_MMC_MSM_SDC2_SUPPORT=y +# CONFIG_MMC_MSM_SDC2_8_BIT_SUPPORT is not set +# CONFIG_MMC_MSM_SDC3_SUPPORT is not set +# CONFIG_MMC_MSM_SDC3_POLLING is not set +# CONFIG_MMC_MSM_SDC4_SUPPORT is not set +# CONFIG_MMC_MSM_SDC5_SUPPORT is not set +CONFIG_MMC_MSM_SPS_SUPPORT=y +# CONFIG_MMC_DW is not set +# CONFIG_MMC_VUB300 is not set +# CONFIG_MMC_USHC is not set +# CONFIG_MEMSTICK is not set +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y + +# +# LED drivers +# +# CONFIG_LEDS_LM3530 is not set +# CONFIG_LEDS_PCA9532 is not set +CONFIG_LEDS_GPIO=y +# CONFIG_LEDS_MSM_PDM is not set +# CONFIG_LEDS_PMIC_MPP is not set +# CONFIG_LEDS_MSM_TRICOLOR is not set +# CONFIG_LEDS_LP3944 is not set +# CONFIG_LEDS_CPLD is not set +# CONFIG_LEDS_LP5521 is not set +# CONFIG_LEDS_LP5523 is not set +# CONFIG_LEDS_PCA955X is not set +# CONFIG_LEDS_PCA9633 is not set +CONFIG_LEDS_QPNP=y +# CONFIG_LEDS_DAC124S085 is not set +# CONFIG_LEDS_REGULATOR is not set +# CONFIG_LEDS_BD2802 is not set +# CONFIG_LEDS_MSM_PMIC is not set +# CONFIG_LEDS_LT3593 is not set +# CONFIG_LEDS_RENESAS_TPU is not set +# CONFIG_LEDS_TCA6507 is not set +# CONFIG_LEDS_OT200 is not set +CONFIG_LEDS_TRIGGERS=y + +# +# LED Triggers +# +# CONFIG_LEDS_TRIGGER_TIMER is not set +# CONFIG_LEDS_TRIGGER_HEARTBEAT is not set +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +# CONFIG_LEDS_TRIGGER_GPIO is not set +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y + +# +# iptables trigger is under Netfilter config (LED target) +# +CONFIG_SWITCH=y +# CONFIG_SWITCH_GPIO is not set +CONFIG_SWITCH_MAX1462X=y +# CONFIG_ACCESSIBILITY is not set +CONFIG_RTC_LIB=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_HCTOSYS=y +CONFIG_RTC_HCTOSYS_DEVICE="rtc0" +# CONFIG_RTC_DEBUG is not set + +# +# RTC interfaces +# +CONFIG_RTC_INTF_SYSFS=y +CONFIG_RTC_INTF_PROC=y +CONFIG_RTC_INTF_DEV=y +CONFIG_RTC_INTF_ALARM=y +CONFIG_RTC_INTF_ALARM_DEV=y +# CONFIG_RTC_INTF_DEV_UIE_EMUL is not set +# CONFIG_RTC_DRV_TEST is not set + +# +# I2C RTC drivers +# +# CONFIG_RTC_DRV_DS1307 is not set +# CONFIG_RTC_DRV_DS1374 is not set +# CONFIG_RTC_DRV_DS1672 is not set +# CONFIG_RTC_DRV_DS3232 is not set +# CONFIG_RTC_DRV_MAX6900 is not set +# CONFIG_RTC_DRV_RS5C372 is not set +# CONFIG_RTC_DRV_ISL1208 is not set +# CONFIG_RTC_DRV_ISL12022 is not set +# CONFIG_RTC_DRV_X1205 is not set +# CONFIG_RTC_DRV_PCF8563 is not set +# CONFIG_RTC_DRV_PCF8583 is not set +# CONFIG_RTC_DRV_M41T80 is not set +# CONFIG_RTC_DRV_BQ32K is not set +# CONFIG_RTC_DRV_S35390A is not set +# CONFIG_RTC_DRV_FM3130 is not set +# CONFIG_RTC_DRV_RX8581 is not set +# CONFIG_RTC_DRV_RX8025 is not set +# CONFIG_RTC_DRV_EM3027 is not set +# CONFIG_RTC_DRV_RV3029C2 is not set + +# +# SPI RTC drivers +# +# CONFIG_RTC_DRV_M41T93 is not set +# CONFIG_RTC_DRV_M41T94 is not set +# CONFIG_RTC_DRV_DS1305 is not set +# CONFIG_RTC_DRV_DS1390 is not set +# CONFIG_RTC_DRV_MAX6902 is not set +# CONFIG_RTC_DRV_R9701 is not set +# CONFIG_RTC_DRV_RS5C348 is not set +# CONFIG_RTC_DRV_DS3234 is not set +# CONFIG_RTC_DRV_PCF2123 is not set + +# +# Platform RTC drivers +# +# CONFIG_RTC_DRV_CMOS is not set +# CONFIG_RTC_DRV_DS1286 is not set +# CONFIG_RTC_DRV_DS1511 is not set +# CONFIG_RTC_DRV_DS1553 is not set +# CONFIG_RTC_DRV_DS1742 is not set +# CONFIG_RTC_DRV_STK17TA8 is not set +# CONFIG_RTC_DRV_M48T86 is not set +# CONFIG_RTC_DRV_M48T35 is not set +# CONFIG_RTC_DRV_M48T59 is not set +# CONFIG_RTC_DRV_MSM6242 is not set +# CONFIG_RTC_DRV_BQ4802 is not set +# CONFIG_RTC_DRV_RP5C01 is not set +# CONFIG_RTC_DRV_V3020 is not set + +# +# on-CPU RTC drivers +# +# CONFIG_RTC_DRV_MSM is not set +# CONFIG_RTC_DRV_MSM7X00A is not set +CONFIG_RTC_DRV_QPNP=y +# CONFIG_DMADEVICES is not set +# CONFIG_AUXDISPLAY is not set +CONFIG_UIO=y +# CONFIG_UIO_PDRV is not set +# CONFIG_UIO_PDRV_GENIRQ is not set +CONFIG_UIO_MSM_SHAREDMEM=y + +# +# Virtio drivers +# +# CONFIG_VIRTIO_BALLOON is not set +# CONFIG_VIRTIO_MMIO is not set + +# +# Microsoft Hyper-V guest support +# +CONFIG_STAGING=y +# CONFIG_USBIP_CORE is not set +# CONFIG_W35UND is not set +# CONFIG_PRISM2_USB is not set +# CONFIG_ECHO is not set +# CONFIG_ASUS_OLED is not set +# CONFIG_R8712U is not set +# CONFIG_RTS5139 is not set +# CONFIG_TRANZPORT is not set +# CONFIG_LINE6_USB is not set +# CONFIG_IIO is not set +# CONFIG_FB_SM7XX is not set +# CONFIG_BCM_WIMAX is not set +# CONFIG_FT1000 is not set + +# +# Speakup console speech +# +# CONFIG_SPEAKUP is not set +# CONFIG_TOUCHSCREEN_CLEARPAD_TM1217 is not set +# CONFIG_TOUCHSCREEN_SYNAPTICS_I2C_RMI4_STAGING is not set +# CONFIG_STAGING_MEDIA is not set + +# +# Android +# +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_ASHMEM=y +# CONFIG_ANDROID_LOGGER is not set +CONFIG_ANDROID_PERSISTENT_RAM=y +CONFIG_ANDROID_RAM_CONSOLE=y +CONFIG_PERSISTENT_TRACER=y +CONFIG_ANDROID_TIMED_OUTPUT=y +CONFIG_ANDROID_TIMED_GPIO=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_ANDROID_LOW_MEMORY_KILLER_AUTODETECT_OOM_ADJ_VALUES=y +# CONFIG_ANDROID_SWITCH is not set +# CONFIG_ANDROID_INTF_ALARM_DEV is not set +# CONFIG_PHONE is not set +# CONFIG_USB_WPAN_HCD is not set + +# +# Qualcomm MSM specific device drivers +# +# CONFIG_MSM_SSBI is not set +CONFIG_SPS=y +CONFIG_USB_BAM=y +CONFIG_SPS_SUPPORT_BAMDMA=y +CONFIG_SPS_SUPPORT_NDP_BAM=y +CONFIG_QPNP_PWM=y +CONFIG_QPNP_POWER_ON=y +CONFIG_QPNP_CLKDIV=y +# CONFIG_QPNP_VIBRATOR is not set +CONFIG_QPNP_REVID=y +CONFIG_QPNP_COINCELL=y +# CONFIG_IPA is not set +# CONFIG_SSM is not set +CONFIG_CLKDEV_LOOKUP=y +CONFIG_HAVE_CLK_PREPARE=y + +# +# Hardware Spinlock drivers +# +CONFIG_IOMMU_SUPPORT=y +CONFIG_MSM_IOMMU=y +CONFIG_MSM_IOMMU_V1=y +# CONFIG_MSM_IOMMU_PMON is not set +CONFIG_IOMMU_PGTABLES_L2=y + +# +# Remoteproc drivers (EXPERIMENTAL) +# + +# +# Rpmsg drivers (EXPERIMENTAL) +# +# CONFIG_VIRT_DRIVERS is not set +# CONFIG_PM_DEVFREQ is not set +# CONFIG_MOBICORE_SUPPORT is not set +# CONFIG_CORESIGHT is not set +# CONFIG_BIF is not set + +# +# File systems +# +CONFIG_EXT2_FS=y +CONFIG_EXT2_FS_XATTR=y +# CONFIG_EXT2_FS_POSIX_ACL is not set +# CONFIG_EXT2_FS_SECURITY is not set +# CONFIG_EXT2_FS_XIP is not set +CONFIG_EXT3_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_EXT3_FS_XATTR=y +# CONFIG_EXT3_FS_POSIX_ACL is not set +# CONFIG_EXT3_FS_SECURITY is not set +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_XATTR=y +# CONFIG_EXT4_FS_POSIX_ACL is not set +CONFIG_EXT4_FS_SECURITY=y +# CONFIG_EXT4_DEBUG is not set +CONFIG_JBD=y +# CONFIG_JBD_DEBUG is not set +CONFIG_JBD2=y +# CONFIG_JBD2_DEBUG is not set +CONFIG_FS_MBCACHE=y +# CONFIG_REISERFS_FS is not set +# CONFIG_JFS_FS is not set +# CONFIG_XFS_FS is not set +# CONFIG_GFS2_FS is not set +# CONFIG_BTRFS_FS is not set +# CONFIG_NILFS2_FS is not set +CONFIG_F2FS_FS=y +CONFIG_F2FS_STAT_FS=y +CONFIG_F2FS_FS_XATTR=y +CONFIG_F2FS_FS_POSIX_ACL=y +CONFIG_F2FS_FS_SECURITY=y +# CONFIG_F2FS_CHECK_FS is not set +# CONFIG_F2FS_FS_ENCRYPTION is not set +# CONFIG_F2FS_IO_TRACE is not set +CONFIG_FS_POSIX_ACL=y +CONFIG_FILE_LOCKING=y +CONFIG_FSNOTIFY=y +CONFIG_DNOTIFY=y +CONFIG_INOTIFY_USER=y +# CONFIG_FANOTIFY is not set +# CONFIG_QUOTA is not set +# CONFIG_QUOTACTL is not set +# CONFIG_AUTOFS4_FS is not set +CONFIG_FUSE_FS=y +# CONFIG_CUSE is not set + +# +# Caches +# +# CONFIG_FSCACHE is not set + +# +# CD-ROM/DVD Filesystems +# +# CONFIG_ISO9660_FS is not set +# CONFIG_UDF_FS is not set + +# +# DOS/FAT/NT Filesystems +# +CONFIG_FAT_FS=y +# CONFIG_MSDOS_FS is not set +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_CODEPAGE=437 +CONFIG_FAT_DEFAULT_IOCHARSET="iso8859-1" +# CONFIG_NTFS_FS is not set + +# +# Pseudo filesystems +# +CONFIG_PROC_FS=y +CONFIG_PROC_SYSCTL=y +CONFIG_PROC_PAGE_MONITOR=y +CONFIG_SYSFS=y +CONFIG_TMPFS=y +# CONFIG_TMPFS_POSIX_ACL is not set +# CONFIG_TMPFS_XATTR is not set +# CONFIG_HUGETLB_PAGE is not set +# CONFIG_CONFIGFS_FS is not set +CONFIG_MISC_FILESYSTEMS=y +# CONFIG_ADFS_FS is not set +# CONFIG_AFFS_FS is not set +# CONFIG_ECRYPT_FS is not set +# CONFIG_HFS_FS is not set +# CONFIG_HFSPLUS_FS is not set +# CONFIG_BEFS_FS is not set +# CONFIG_BFS_FS is not set +# CONFIG_EFS_FS is not set +# CONFIG_LOGFS is not set +# CONFIG_CRAMFS is not set +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_ZLIB=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +# CONFIG_SQUASHFS_4K_DEVBLK_SIZE is not set +# CONFIG_SQUASHFS_EMBEDDED is not set +CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE=3 +# CONFIG_VXFS_FS is not set +# CONFIG_MINIX_FS is not set +# CONFIG_OMFS_FS is not set +# CONFIG_HPFS_FS is not set +# CONFIG_QNX4FS_FS is not set +# CONFIG_QNX6FS_FS is not set +# CONFIG_ROMFS_FS is not set +# CONFIG_PSTORE is not set +# CONFIG_SYSV_FS is not set +# CONFIG_UFS_FS is not set +CONFIG_NETWORK_FILESYSTEMS=y +# CONFIG_NFS_FS is not set +# CONFIG_NFSD is not set +# CONFIG_CEPH_FS is not set +CONFIG_CIFS=y +# CONFIG_CIFS_STATS is not set +# CONFIG_CIFS_WEAK_PW_HASH is not set +# CONFIG_CIFS_UPCALL is not set +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_POSIX=y +# CONFIG_CIFS_DEBUG2 is not set +# CONFIG_CIFS_DFS_UPCALL is not set +# CONFIG_CIFS_ACL is not set +# CONFIG_NCP_FS is not set +# CONFIG_CODA_FS is not set +# CONFIG_AFS_FS is not set +CONFIG_NLS=y +CONFIG_NLS_DEFAULT="iso8859-1" +CONFIG_NLS_CODEPAGE_437=y +# CONFIG_NLS_CODEPAGE_737 is not set +# CONFIG_NLS_CODEPAGE_775 is not set +# CONFIG_NLS_CODEPAGE_850 is not set +# CONFIG_NLS_CODEPAGE_852 is not set +# CONFIG_NLS_CODEPAGE_855 is not set +# CONFIG_NLS_CODEPAGE_857 is not set +# CONFIG_NLS_CODEPAGE_860 is not set +# CONFIG_NLS_CODEPAGE_861 is not set +# CONFIG_NLS_CODEPAGE_862 is not set +# CONFIG_NLS_CODEPAGE_863 is not set +# CONFIG_NLS_CODEPAGE_864 is not set +# CONFIG_NLS_CODEPAGE_865 is not set +# CONFIG_NLS_CODEPAGE_866 is not set +# CONFIG_NLS_CODEPAGE_869 is not set +# CONFIG_NLS_CODEPAGE_936 is not set +# CONFIG_NLS_CODEPAGE_950 is not set +# CONFIG_NLS_CODEPAGE_932 is not set +# CONFIG_NLS_CODEPAGE_949 is not set +# CONFIG_NLS_CODEPAGE_874 is not set +# CONFIG_NLS_ISO8859_8 is not set +# CONFIG_NLS_CODEPAGE_1250 is not set +# CONFIG_NLS_CODEPAGE_1251 is not set +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +# CONFIG_NLS_ISO8859_2 is not set +# CONFIG_NLS_ISO8859_3 is not set +# CONFIG_NLS_ISO8859_4 is not set +# CONFIG_NLS_ISO8859_5 is not set +# CONFIG_NLS_ISO8859_6 is not set +# CONFIG_NLS_ISO8859_7 is not set +# CONFIG_NLS_ISO8859_9 is not set +# CONFIG_NLS_ISO8859_13 is not set +# CONFIG_NLS_ISO8859_14 is not set +# CONFIG_NLS_ISO8859_15 is not set +# CONFIG_NLS_KOI8_R is not set +# CONFIG_NLS_KOI8_U is not set +CONFIG_NLS_UTF8=y + +# +# Kernel hacking +# +CONFIG_PRINTK_TIME=y +CONFIG_DEFAULT_MESSAGE_LOGLEVEL=4 +CONFIG_ENABLE_WARN_DEPRECATED=y +CONFIG_ENABLE_MUST_CHECK=y +CONFIG_FRAME_WARN=1024 +CONFIG_MAGIC_SYSRQ=y +# CONFIG_STRIP_ASM_SYMS is not set +# CONFIG_UNUSED_SYMBOLS is not set +CONFIG_DEBUG_FS=y +# CONFIG_HEADERS_CHECK is not set +# CONFIG_DEBUG_SECTION_MISMATCH is not set +CONFIG_DEBUG_KERNEL=y +# CONFIG_DEBUG_SHIRQ is not set +# CONFIG_LOCKUP_DETECTOR is not set +# CONFIG_HARDLOCKUP_DETECTOR is not set +# CONFIG_DETECT_HUNG_TASK is not set +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +# CONFIG_DEBUG_OBJECTS is not set +# CONFIG_SLUB_DEBUG_ON is not set +# CONFIG_SLUB_STATS is not set +# CONFIG_DEBUG_KMEMLEAK is not set +# CONFIG_DEBUG_PREEMPT is not set +# CONFIG_DEBUG_RT_MUTEXES is not set +# CONFIG_RT_MUTEX_TESTER is not set +# CONFIG_DEBUG_SPINLOCK is not set +CONFIG_DEBUG_MUTEXES=y +# CONFIG_DEBUG_LOCK_ALLOC is not set +# CONFIG_PROVE_LOCKING is not set +# CONFIG_SPARSE_RCU_POINTER is not set +# CONFIG_LOCK_STAT is not set +# CONFIG_DEBUG_ATOMIC_SLEEP is not set +# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set +CONFIG_STACKTRACE=y +# CONFIG_DEBUG_STACK_USAGE is not set +# CONFIG_DEBUG_KOBJECT is not set +# CONFIG_DEBUG_HIGHMEM is not set +CONFIG_DEBUG_BUGVERBOSE=y +CONFIG_DEBUG_INFO=y +# CONFIG_DEBUG_INFO_REDUCED is not set +# CONFIG_DEBUG_VM is not set +# CONFIG_DEBUG_WRITECOUNT is not set +# CONFIG_DEBUG_MEMORY_INIT is not set +# CONFIG_DEBUG_LIST is not set +# CONFIG_TEST_LIST_SORT is not set +# CONFIG_DEBUG_SG is not set +# CONFIG_DEBUG_NOTIFIERS is not set +# CONFIG_DEBUG_CREDENTIALS is not set +CONFIG_FRAME_POINTER=y +# CONFIG_BOOT_PRINTK_DELAY is not set +# CONFIG_RCU_TORTURE_TEST is not set +CONFIG_RCU_CPU_STALL_TIMEOUT=60 +# CONFIG_RCU_CPU_STALL_VERBOSE is not set +# CONFIG_RCU_CPU_STALL_INFO is not set +# CONFIG_RCU_TRACE is not set +# CONFIG_BACKTRACE_SELF_TEST is not set +# CONFIG_DEBUG_BLOCK_EXT_DEVT is not set +# CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set +# CONFIG_DEBUG_PER_CPU_MAPS is not set +# CONFIG_LKDTM is not set +# CONFIG_CPU_NOTIFIER_ERROR_INJECT is not set +# CONFIG_FAULT_INJECTION is not set +# CONFIG_DEBUG_PAGEALLOC is not set +CONFIG_NOP_TRACER=y +CONFIG_HAVE_FUNCTION_TRACER=y +CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y +CONFIG_HAVE_DYNAMIC_FTRACE=y +CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y +CONFIG_HAVE_C_RECORDMCOUNT=y +CONFIG_RING_BUFFER=y +CONFIG_EVENT_TRACING=y +CONFIG_EVENT_POWER_TRACING_DEPRECATED=y +CONFIG_CONTEXT_SWITCH_TRACER=y +CONFIG_TRACING=y +CONFIG_GENERIC_TRACER=y +CONFIG_TRACING_SUPPORT=y +CONFIG_FTRACE=y +CONFIG_FUNCTION_TRACER=y +CONFIG_FUNCTION_GRAPH_TRACER=y +# CONFIG_IRQSOFF_TRACER is not set +# CONFIG_PREEMPT_TRACER is not set +# CONFIG_SCHED_TRACER is not set +CONFIG_BRANCH_PROFILE_NONE=y +# CONFIG_PROFILE_ANNOTATED_BRANCHES is not set +# CONFIG_PROFILE_ALL_BRANCHES is not set +# CONFIG_STACK_TRACER is not set +# CONFIG_BLK_DEV_IO_TRACE is not set +CONFIG_DYNAMIC_FTRACE=y +# CONFIG_FUNCTION_PROFILER is not set +# CONFIG_CPU_FREQ_SWITCH_PROFILER is not set +CONFIG_FTRACE_MCOUNT_RECORD=y +# CONFIG_FTRACE_STARTUP_TEST is not set +# CONFIG_RING_BUFFER_BENCHMARK is not set +CONFIG_DYNAMIC_DEBUG=y +# CONFIG_DMA_API_DEBUG is not set +# CONFIG_ATOMIC64_SELFTEST is not set +CONFIG_PANIC_ON_DATA_CORRUPTION=y +# CONFIG_SAMPLES is not set +CONFIG_HAVE_ARCH_KGDB=y +# CONFIG_KGDB is not set +# CONFIG_TEST_KSTRTOX is not set +# CONFIG_STRICT_DEVMEM is not set +CONFIG_ARM_UNWIND=y +CONFIG_OLD_MCOUNT=y +# CONFIG_DEBUG_USER is not set +# CONFIG_DEBUG_LL is not set +CONFIG_PID_IN_CONTEXTIDR=y + +# +# Security options +# +CONFIG_KEYS=y +# CONFIG_ENCRYPTED_KEYS is not set +# CONFIG_KEYS_DEBUG_PROC_KEYS is not set +# CONFIG_SECURITY_DMESG_RESTRICT is not set +CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y +CONFIG_SECURITY=y +# CONFIG_SECURITYFS is not set +CONFIG_SECURITY_NETWORK=y +# CONFIG_SECURITY_NETWORK_XFRM is not set +# CONFIG_SECURITY_PATH is not set +CONFIG_LSM_MMAP_MIN_ADDR=4096 +CONFIG_SECURITY_SELINUX=y +# CONFIG_SECURITY_SELINUX_BOOTPARAM is not set +# CONFIG_SECURITY_SELINUX_DISABLE is not set +CONFIG_SECURITY_SELINUX_DEVELOP=y +CONFIG_SECURITY_SELINUX_AVC_STATS=y +CONFIG_SECURITY_SELINUX_CHECKREQPROT_VALUE=1 +# CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX is not set +# CONFIG_SECURITY_TOMOYO is not set +# CONFIG_SECURITY_APPARMOR is not set +# CONFIG_SECURITY_YAMA is not set +# CONFIG_IMA is not set +# CONFIG_EVM is not set +CONFIG_DEFAULT_SECURITY_SELINUX=y +# CONFIG_DEFAULT_SECURITY_DAC is not set +CONFIG_DEFAULT_SECURITY="selinux" +CONFIG_CRYPTO=y + +# +# Crypto core or helper +# +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_ALGAPI2=y +CONFIG_CRYPTO_AEAD=y +CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_BLKCIPHER=y +CONFIG_CRYPTO_BLKCIPHER2=y +CONFIG_CRYPTO_HASH=y +CONFIG_CRYPTO_HASH2=y +CONFIG_CRYPTO_RNG2=y +CONFIG_CRYPTO_PCOMP2=y +CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_MANAGER2=y +# CONFIG_CRYPTO_USER is not set +CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y +# CONFIG_CRYPTO_GF128MUL is not set +CONFIG_CRYPTO_NULL=y +# CONFIG_CRYPTO_PCRYPT is not set +CONFIG_CRYPTO_WORKQUEUE=y +# CONFIG_CRYPTO_CRYPTD is not set +CONFIG_CRYPTO_AUTHENC=y + +# +# Authenticated Encryption with Associated Data +# +# CONFIG_CRYPTO_CCM is not set +# CONFIG_CRYPTO_GCM is not set +# CONFIG_CRYPTO_SEQIV is not set + +# +# Block modes +# +CONFIG_CRYPTO_CBC=y +# CONFIG_CRYPTO_CTR is not set +# CONFIG_CRYPTO_CTS is not set +CONFIG_CRYPTO_ECB=y +# CONFIG_CRYPTO_LRW is not set +# CONFIG_CRYPTO_PCBC is not set +# CONFIG_CRYPTO_XTS is not set + +# +# Hash modes +# +CONFIG_CRYPTO_HMAC=y +# CONFIG_CRYPTO_XCBC is not set +# CONFIG_CRYPTO_VMAC is not set + +# +# Digest +# +CONFIG_CRYPTO_CRC32C=y +# CONFIG_CRYPTO_GHASH is not set +CONFIG_CRYPTO_MD4=y +CONFIG_CRYPTO_MD5=y +# CONFIG_CRYPTO_MICHAEL_MIC is not set +# CONFIG_CRYPTO_RMD128 is not set +# CONFIG_CRYPTO_RMD160 is not set +# CONFIG_CRYPTO_RMD256 is not set +# CONFIG_CRYPTO_RMD320 is not set +CONFIG_CRYPTO_SHA1=y +# CONFIG_CRYPTO_SHA1_ARM is not set +CONFIG_CRYPTO_SHA256=y +# CONFIG_CRYPTO_SHA512 is not set +# CONFIG_CRYPTO_TGR192 is not set +# CONFIG_CRYPTO_WP512 is not set + +# +# Ciphers +# +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_AES_ARM=y +# CONFIG_CRYPTO_ANUBIS is not set +CONFIG_CRYPTO_ARC4=y +# CONFIG_CRYPTO_BLOWFISH is not set +# CONFIG_CRYPTO_CAMELLIA is not set +# CONFIG_CRYPTO_CAST5 is not set +# CONFIG_CRYPTO_CAST6 is not set +CONFIG_CRYPTO_DES=y +# CONFIG_CRYPTO_FCRYPT is not set +# CONFIG_CRYPTO_KHAZAD is not set +# CONFIG_CRYPTO_SALSA20 is not set +# CONFIG_CRYPTO_SEED is not set +# CONFIG_CRYPTO_SERPENT is not set +# CONFIG_CRYPTO_TEA is not set +CONFIG_CRYPTO_TWOFISH=y +CONFIG_CRYPTO_TWOFISH_COMMON=y + +# +# Compression +# +CONFIG_CRYPTO_DEFLATE=y +# CONFIG_CRYPTO_ZLIB is not set +# CONFIG_CRYPTO_LZO is not set + +# +# Random Number Generation +# +# CONFIG_CRYPTO_ANSI_CPRNG is not set +# CONFIG_CRYPTO_USER_API_HASH is not set +# CONFIG_CRYPTO_USER_API_SKCIPHER is not set +CONFIG_CRYPTO_HW=y +# CONFIG_CRYPTO_DEV_QCRYPTO is not set +# CONFIG_CRYPTO_DEV_QCE is not set +# CONFIG_CRYPTO_DEV_QCEDEV is not set +# CONFIG_CRYPTO_DEV_OTA_CRYPTO is not set +CONFIG_BINARY_PRINTF=y + +# +# Library routines +# +CONFIG_BITREVERSE=y +CONFIG_GENERIC_PCI_IOMAP=y +CONFIG_GENERIC_IO=y +CONFIG_CRC_CCITT=y +CONFIG_CRC16=y +# CONFIG_CRC_T10DIF is not set +# CONFIG_CRC_ITU_T is not set +CONFIG_CRC32=y +# CONFIG_CRC32_SELFTEST is not set +CONFIG_CRC32_SLICEBY8=y +# CONFIG_CRC32_SLICEBY4 is not set +# CONFIG_CRC32_SARWATE is not set +# CONFIG_CRC32_BIT is not set +# CONFIG_CRC7 is not set +CONFIG_LIBCRC32C=y +# CONFIG_CRC8 is not set +CONFIG_AUDIT_GENERIC=y +CONFIG_ZLIB_INFLATE=y +CONFIG_ZLIB_DEFLATE=y +CONFIG_LZO_COMPRESS=y +CONFIG_LZO_DECOMPRESS=y +CONFIG_XZ_DEC=y +CONFIG_XZ_DEC_X86=y +CONFIG_XZ_DEC_POWERPC=y +CONFIG_XZ_DEC_IA64=y +CONFIG_XZ_DEC_ARM=y +CONFIG_XZ_DEC_ARMTHUMB=y +CONFIG_XZ_DEC_SPARC=y +CONFIG_XZ_DEC_BCJ=y +# CONFIG_XZ_DEC_TEST is not set +CONFIG_DECOMPRESS_GZIP=y +CONFIG_DECOMPRESS_BZIP2=y +CONFIG_DECOMPRESS_LZMA=y +CONFIG_GENERIC_ALLOCATOR=y +CONFIG_REED_SOLOMON=y +CONFIG_REED_SOLOMON_ENC8=y +CONFIG_REED_SOLOMON_DEC8=y +CONFIG_TEXTSEARCH=y +CONFIG_TEXTSEARCH_KMP=y +CONFIG_TEXTSEARCH_BM=y +CONFIG_TEXTSEARCH_FSM=y +CONFIG_HAS_IOMEM=y +CONFIG_HAS_IOPORT=y +CONFIG_HAS_DMA=y +CONFIG_CPU_RMAP=y +CONFIG_DQL=y +CONFIG_NLATTR=y +CONFIG_AVERAGE=y +# CONFIG_CORDIC is not set +CONFIG_QMI_ENCDEC=y diff --git a/arch/arm/configs/hammerhead_defconfig b/arch/arm/configs/hammerhead_defconfig index f8f8b4f94ae..f94bb7ef199 100644 --- a/arch/arm/configs/hammerhead_defconfig +++ b/arch/arm/configs/hammerhead_defconfig @@ -1,7 +1,6 @@ # CONFIG_ARM_PATCH_PHYS_VIRT is not set CONFIG_EXPERIMENTAL=y # CONFIG_SWAP is not set -CONFIG_SYSVIPC=y CONFIG_AUDIT=y CONFIG_RCU_FAST_NO_HZ=y CONFIG_LOG_BUF_SHIFT=19 @@ -14,7 +13,6 @@ CONFIG_CGROUP_SCHED=y CONFIG_RT_GROUP_SCHED=y CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set -# CONFIG_IPC_NS is not set # CONFIG_USER_NS is not set # CONFIG_PID_NS is not set CONFIG_RELAY=y @@ -24,7 +22,6 @@ CONFIG_RD_LZMA=y CONFIG_PANIC_TIMEOUT=5 CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y -# CONFIG_SLUB_DEBUG is not set CONFIG_PROFILING=y CONFIG_PARTITION_ADVANCED=y CONFIG_EFI_PARTITION=y @@ -88,6 +85,7 @@ CONFIG_PREEMPT=y CONFIG_AEABI=y CONFIG_HIGHMEM=y CONFIG_COMPACTION=y +CONFIG_SECCOMP=y CONFIG_CC_STACKPROTECTOR=y CONFIG_USE_OF=y CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE=y @@ -106,6 +104,7 @@ CONFIG_PM_WAKELOCKS=y CONFIG_PM_WAKELOCKS_LIMIT=0 # CONFIG_PM_WAKELOCKS_GC is not set CONFIG_PM_RUNTIME=y +CONFIG_PARTIALRESUME=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -132,7 +131,6 @@ CONFIG_IPV6_MIP6=y CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_IPV6_SUBTREES=y CONFIG_NETFILTER=y -CONFIG_NETFILTER_NETLINK_LOG=y CONFIG_NF_CONNTRACK=y CONFIG_NF_CONNTRACK_SECMARK=y CONFIG_NF_CONNTRACK_EVENTS=y @@ -155,6 +153,7 @@ CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y CONFIG_NETFILTER_XT_TARGET_TCPMSS=y @@ -249,6 +248,7 @@ CONFIG_QPNP_MISC=y CONFIG_TI_DRV2667=y CONFIG_EARJACK_DEBUGGER=y CONFIG_FAN48632_BOOST=y +CONFIG_UID_CPUTIME=y CONFIG_SCSI=y CONFIG_SCSI_TGT=y CONFIG_BLK_DEV_SD=y @@ -346,6 +346,8 @@ CONFIG_INPUT_KEYCHORD=y CONFIG_INPUT_UINPUT=y # CONFIG_SERIO is not set # CONFIG_LEGACY_PTYS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set CONFIG_SERIAL_MSM_HS=y CONFIG_SERIAL_MSM_HSL=y CONFIG_SERIAL_MSM_HSL_CONSOLE=y @@ -596,10 +598,12 @@ CONFIG_DYNAMIC_DEBUG=y CONFIG_PANIC_ON_DATA_CORRUPTION=y CONFIG_PID_IN_CONTEXTIDR=y CONFIG_KEYS=y +CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y CONFIG_SECURITY=y CONFIG_SECURITY_NETWORK=y CONFIG_LSM_MMAP_MIN_ADDR=4096 CONFIG_SECURITY_SELINUX=y CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_AES_ARM=y CONFIG_CRYPTO_TWOFISH=y # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/arm/configs/lineageos_hammerhead_defconfig b/arch/arm/configs/lineageos_hammerhead_defconfig new file mode 100644 index 00000000000..c496e4b2be7 --- /dev/null +++ b/arch/arm/configs/lineageos_hammerhead_defconfig @@ -0,0 +1,612 @@ +# CONFIG_ARM_PATCH_PHYS_VIRT is not set +CONFIG_EXPERIMENTAL=y +# CONFIG_SWAP is not set +CONFIG_AUDIT=y +CONFIG_RCU_FAST_NO_HZ=y +CONFIG_LOG_BUF_SHIFT=19 +CONFIG_CGROUPS=y +CONFIG_CGROUP_DEBUG=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_RESOURCE_COUNTERS=y +CONFIG_CGROUP_SCHED=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_NAMESPACES=y +# CONFIG_UTS_NS is not set +# CONFIG_USER_NS is not set +# CONFIG_PID_NS is not set +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_RD_BZIP2=y +CONFIG_RD_LZMA=y +CONFIG_PANIC_TIMEOUT=5 +CONFIG_KALLSYMS_ALL=y +CONFIG_EMBEDDED=y +CONFIG_PROFILING=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_EFI_PARTITION=y +CONFIG_ARCH_MSM=y +CONFIG_ARCH_MSM8974=y +CONFIG_MSM_KRAIT_TBB_ABORT_HANDLER=y +CONFIG_MACH_MSM8974_HAMMERHEAD=y +CONFIG_LGE_HANDLE_PANIC=y +CONFIG_LGE_BLUETOOTH=y +# CONFIG_MSM_STACKED_MEMORY is not set +CONFIG_CPU_HAS_L2_PMU=y +# CONFIG_MSM_FIQ_SUPPORT is not set +# CONFIG_MSM_PROC_COMM is not set +CONFIG_MSM_SMD=y +CONFIG_MSM_SMD_PKG4=y +CONFIG_MSM_BAM_DMUX=y +CONFIG_MSM_SMP2P=y +CONFIG_MSM_SMP2P_TEST=y +# CONFIG_MSM_RESET_MODEM is not set +CONFIG_MSM_IPC_LOGGING=y +# CONFIG_MSM_SMD_TTY is not set +CONFIG_MSM_IPC_ROUTER=y +CONFIG_MSM_IPC_ROUTER_SMD_XPRT=y +CONFIG_MSM_IPC_ROUTER_SECURITY=y +CONFIG_MSM_QMI_INTERFACE=y +# CONFIG_MSM_DMA_TEST is not set +CONFIG_WIFI_CONTROL_FUNC=y +CONFIG_MSM_SUBSYSTEM_RESTART=y +CONFIG_MSM_SYSMON_COMM=y +CONFIG_MSM_PIL_LPASS_QDSP6V5=y +CONFIG_MSM_PIL_MSS_QDSP6V5=y +CONFIG_MSM_PIL_VENUS=y +# CONFIG_MSM_BUSPM_DEV is not set +CONFIG_MSM_TZ_LOG=y +CONFIG_MSM_RPM_RBCPR_STATS_V2_LOG=y +CONFIG_MSM_DIRECT_SCLK_ACCESS=y +CONFIG_MSM_EVENT_TIMER=y +CONFIG_MSM_BUS_SCALING=y +CONFIG_MSM_WATCHDOG_V2=y +CONFIG_MSM_MEMORY_DUMP=y +CONFIG_MSM_DLOAD_MODE=y +CONFIG_MSM_ADSP_LOADER=y +CONFIG_MSM_OCMEM=y +CONFIG_MSM_OCMEM_LOCAL_POWER_CTRL=y +CONFIG_MSM_OCMEM_DEBUG=y +CONFIG_SENSORS_ADSP=y +CONFIG_MSM_CACHE_ERP=y +CONFIG_MSM_L1_ERR_PANIC=y +CONFIG_MSM_L1_RECOV_ERR_PANIC=y +CONFIG_MSM_L1_ERR_LOG=y +CONFIG_MSM_ENABLE_WDOG_DEBUG_CONTROL=y +CONFIG_MSM_UARTDM_Core_v14=y +CONFIG_STRICT_MEMORY_RWX=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_SMP=y +# CONFIG_SMP_ON_UP is not set +CONFIG_SCHED_MC=y +CONFIG_ARM_ARCH_TIMER=y +CONFIG_PREEMPT=y +CONFIG_AEABI=y +CONFIG_HIGHMEM=y +CONFIG_COMPACTION=y +CONFIG_SECCOMP=y +CONFIG_CC_STACKPROTECTOR=y +CONFIG_USE_OF=y +CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE=y +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_INTERACTIVE=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_CPU_IDLE=y +CONFIG_VFP=y +CONFIG_NEON=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_WAKELOCKS_LIMIT=0 +# CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_PM_RUNTIME=y +CONFIG_PARTIALRESUME=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=y +CONFIG_NET_KEY=y +CONFIG_INET=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_INET_AH=y +CONFIG_INET_ESP=y +CONFIG_INET_IPCOMP=y +# CONFIG_INET_LRO is not set +CONFIG_IPV6_PRIVACY=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=y +CONFIG_INET6_ESP=y +CONFIG_INET6_IPCOMP=y +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPV6_SUBTREES=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_DCCP=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CONNTRACK_AMANDA=y +CONFIG_NF_CONNTRACK_FTP=y +CONFIG_NF_CONNTRACK_H323=y +CONFIG_NF_CONNTRACK_IRC=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=y +CONFIG_NF_CONNTRACK_PPTP=y +CONFIG_NF_CONNTRACK_SANE=y +CONFIG_NF_CONNTRACK_TFTP=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NETFILTER_TPROXY=y +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y +CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y +CONFIG_NETFILTER_XT_TARGET_LOG=y +CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y +CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_TCPMSS=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y +CONFIG_NETFILTER_XT_MATCH_CONNMARK=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y +CONFIG_NETFILTER_XT_MATCH_HELPER=y +CONFIG_NETFILTER_XT_MATCH_IPRANGE=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_MAC=y +CONFIG_NETFILTER_XT_MATCH_MARK=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +CONFIG_NETFILTER_XT_MATCH_POLICY=y +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y +CONFIG_NETFILTER_XT_MATCH_QTAGUID=y +CONFIG_NETFILTER_XT_MATCH_QUOTA=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y +CONFIG_NETFILTER_XT_MATCH_SOCKET=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_STATISTIC=y +CONFIG_NETFILTER_XT_MATCH_STRING=y +CONFIG_NETFILTER_XT_MATCH_TIME=y +CONFIG_NETFILTER_XT_MATCH_U32=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=y +CONFIG_IP_NF_MATCH_ECN=y +CONFIG_IP_NF_MATCH_TTL=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_TARGET_REJECT_SKERR=y +CONFIG_NF_NAT=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_TARGET_NETMAP=y +CONFIG_IP_NF_TARGET_REDIRECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_SECURITY=y +CONFIG_IP_NF_ARPTABLES=y +CONFIG_IP_NF_ARPFILTER=y +CONFIG_IP_NF_ARP_MANGLE=y +CONFIG_NF_CONNTRACK_IPV6=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_MATCH_RPFILTER=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_TARGET_REJECT_SKERR=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_RAW=y +CONFIG_BRIDGE_NF_EBTABLES=y +CONFIG_BRIDGE_EBT_BROUTE=y +CONFIG_L2TP=y +CONFIG_BRIDGE=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=y +CONFIG_NET_SCH_PRIO=y +CONFIG_NET_CLS_FW=y +CONFIG_NET_CLS_U32=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_FLOW=y +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=y +CONFIG_NET_EMATCH_NBYTE=y +CONFIG_NET_EMATCH_U32=y +CONFIG_NET_EMATCH_META=y +CONFIG_NET_EMATCH_TEXT=y +CONFIG_NET_CLS_ACT=y +CONFIG_BT=y +CONFIG_BT_RFCOMM=y +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=y +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=y +CONFIG_BT_MSM_SLEEP=y +# CONFIG_MSM_BT_POWER is not set +CONFIG_CFG80211=y +CONFIG_NL80211_TESTMODE=y +CONFIG_RFKILL=y +CONFIG_NFC_BCM2079X=y +CONFIG_CMA=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_UID_STAT=y +CONFIG_HAPTIC_ISA1200=y +CONFIG_MSM8974_PWM_VIBRATOR=y +CONFIG_QSEECOM=y +CONFIG_QPNP_MISC=y +CONFIG_TI_DRV2667=y +CONFIG_EARJACK_DEBUGGER=y +CONFIG_FAN48632_BOOST=y +CONFIG_UID_CPUTIME=y +CONFIG_SCSI=y +CONFIG_SCSI_TGT=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_MULTI_LUN=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_NETDEVICES=y +CONFIG_TUN=y +# CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_VENDOR_CHELSIO is not set +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_INTEL is not set +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MICROCHIP is not set +# CONFIG_MSM_RMNET is not set +CONFIG_MSM_RMNET_BAM=y +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +CONFIG_PPP=y +CONFIG_PPP_BSDCOMP=y +CONFIG_PPP_DEFLATE=y +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=y +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOE=y +CONFIG_PPPOL2TP=y +CONFIG_PPPOLAC=y +CONFIG_PPPOPNS=y +CONFIG_PPP_ASYNC=y +CONFIG_PPP_SYNC_TTY=y +CONFIG_SLIP=y +CONFIG_SLIP_COMPRESSED=y +CONFIG_SLIP_MODE_SLIP6=y +CONFIG_USB_CATC=y +CONFIG_USB_KAWETH=y +CONFIG_USB_PEGASUS=y +CONFIG_USB_RTL8150=y +CONFIG_USB_USBNET=y +CONFIG_USB_NET_CDC_EEM=y +CONFIG_USB_NET_DM9601=y +CONFIG_USB_NET_SMSC75XX=y +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_NET_GL620A=y +CONFIG_USB_NET_PLUSB=y +CONFIG_USB_NET_MCS7830=y +CONFIG_USB_NET_RNDIS_HOST=y +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_CX82310_ETH=y +CONFIG_USB_NET_KALMIA=y +CONFIG_USB_NET_QMI_WWAN=y +CONFIG_USB_HSO=y +CONFIG_USB_NET_INT51X1=y +CONFIG_USB_IPHETH=y +CONFIG_USB_SIERRA_NET=y +CONFIG_BCMDHD=y +CONFIG_BCM4339=y +CONFIG_BCMDHD_FW_PATH="/vendor/firmware/fw_bcmdhd.bin" +CONFIG_BCMDHD_NVRAM_PATH="/etc/wifi/bcmdhd.cal" +CONFIG_DHD_USE_SCHED_SCAN=y +# CONFIG_INPUT_MOUSEDEV is not set +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_KEYRESET=y +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_GPIO=y +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_XPAD=y +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=y +CONFIG_TABLET_USB_AIPTEK=y +CONFIG_TABLET_USB_GTCO=y +CONFIG_TABLET_USB_HANWANG=y +CONFIG_TABLET_USB_KBTAB=y +CONFIG_TABLET_USB_WACOM=y +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_LGE_SYNAPTICS=y +CONFIG_TOUCHSCREEN_CHARGER_NOTIFY=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_KEYCHORD=y +CONFIG_INPUT_UINPUT=y +# CONFIG_SERIO is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set +CONFIG_SERIAL_MSM_HS=y +CONFIG_SERIAL_MSM_HSL=y +CONFIG_SERIAL_MSM_HSL_CONSOLE=y +CONFIG_HW_RANDOM_MSM=y +CONFIG_I2C=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_QUP=y +CONFIG_SPI=y +CONFIG_SPI_QUP=y +CONFIG_SPMI=y +CONFIG_SPMI_MSM_PMIC_ARB=y +CONFIG_MSM_QPNP_INT=y +CONFIG_SLIMBUS_MSM_NGD=y +CONFIG_DEBUG_GPIO=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_QPNP_PIN=y +CONFIG_GPIO_QPNP_PIN_DEBUG=y +CONFIG_SMB349_CHARGER=y +CONFIG_QPNP_CHARGER=y +CONFIG_MAX17048_FUELGAUGE=y +CONFIG_BQ51013B_CHARGER=y +CONFIG_BQ24192_CHARGER=y +CONFIG_BATTERY_TEMP_CONTROL=y +CONFIG_SENSORS_EPM_ADC=y +CONFIG_SENSORS_QPNP_ADC_VOLTAGE=y +CONFIG_SENSORS_QPNP_ADC_CURRENT=y +CONFIG_THERMAL=y +CONFIG_THERMAL_TSENS8974=y +CONFIG_THERMAL_MONITOR=y +CONFIG_THERMAL_QPNP=y +CONFIG_THERMAL_QPNP_ADC_TM=y +CONFIG_WCD9320_CODEC=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_STUB=y +CONFIG_REGULATOR_QPNP=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_DEV=y +CONFIG_VIDEO_V4L2_SUBDEV_API=y +# CONFIG_MSM_CAMERA is not set +CONFIG_MSM_CAMERA_SENSOR=y +CONFIG_MSM_EEPROM=y +CONFIG_MSM_CPP=y +CONFIG_MSM_CCI=y +CONFIG_MSM_CSI30_HEADER=y +CONFIG_MSM_CSIPHY=y +CONFIG_MSM_CSID=y +CONFIG_MSM_ISPIF=y +CONFIG_MSMB_CAMERA=y +CONFIG_IMX179=y +CONFIG_OIS_ROHM_BU24205GWL=y +CONFIG_OIS_ONSEMI_LC898111A=y +CONFIG_MT9M114B=y +CONFIG_MSMB_JPEG=y +CONFIG_MSM_VIDC_V4L2=y +# CONFIG_MEDIA_TUNER_SIMPLE is not set +# CONFIG_MEDIA_TUNER_TDA8290 is not set +# CONFIG_MEDIA_TUNER_TDA827X is not set +# CONFIG_MEDIA_TUNER_TDA18271 is not set +# CONFIG_MEDIA_TUNER_TDA9887 is not set +# CONFIG_MEDIA_TUNER_TEA5761 is not set +# CONFIG_MEDIA_TUNER_TEA5767 is not set +# CONFIG_MEDIA_TUNER_MT20XX is not set +# CONFIG_MEDIA_TUNER_MT2060 is not set +# CONFIG_MEDIA_TUNER_MT2063 is not set +# CONFIG_MEDIA_TUNER_MT2266 is not set +# CONFIG_MEDIA_TUNER_MT2131 is not set +# CONFIG_MEDIA_TUNER_QT1010 is not set +# CONFIG_MEDIA_TUNER_XC2028 is not set +# CONFIG_MEDIA_TUNER_XC5000 is not set +# CONFIG_MEDIA_TUNER_XC4000 is not set +# CONFIG_MEDIA_TUNER_MXL5005S is not set +# CONFIG_MEDIA_TUNER_MXL5007T is not set +# CONFIG_MEDIA_TUNER_MC44S803 is not set +# CONFIG_MEDIA_TUNER_MAX2165 is not set +# CONFIG_MEDIA_TUNER_TDA18218 is not set +# CONFIG_MEDIA_TUNER_TDA18212 is not set +CONFIG_VIDEOBUF2_MSM_MEM=y +CONFIG_USB_VIDEO_CLASS=y +# CONFIG_USB_GSPCA is not set +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_ION=y +CONFIG_ION_MSM=y +CONFIG_MSM_KGSL=y +CONFIG_KGSL_PER_PROCESS_PAGE_TABLE=y +CONFIG_FB=y +CONFIG_FB_MSM=y +# CONFIG_FB_MSM_BACKLIGHT is not set +CONFIG_FB_MSM_MDSS=y +CONFIG_FB_MSM_MDSS_WRITEBACK=y +CONFIG_FB_MSM_MDSS_HDMI_PANEL=y +CONFIG_BACKLIGHT_LCD_SUPPORT=y +# CONFIG_LCD_CLASS_DEVICE is not set +# CONFIG_BACKLIGHT_GENERIC is not set +CONFIG_BACKLIGHT_LM3630=y +CONFIG_SLIMPORT_ANX7808=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_USB_AUDIO=y +CONFIG_SND_SOC=y +CONFIG_SND_SOC_MSM8974=y +CONFIG_SND_SOC_MSM_HDMI_CODEC_RX=y +CONFIG_UHID=y +CONFIG_USB_HIDDEV=y +CONFIG_HID_A4TECH=y +CONFIG_HID_ACRUX=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_PRODIKEYS=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_DRAGONRISE=y +CONFIG_HID_EMS_FF=y +CONFIG_HID_ELECOM=y +CONFIG_HID_EZKEY=y +CONFIG_HID_HOLTEK=y +CONFIG_HID_KEYTOUCH=y +CONFIG_HID_KYE=y +CONFIG_HID_UCLOGIC=y +CONFIG_HID_WALTOP=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_KENSINGTON=y +CONFIG_HID_LCPOWER=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_MULTITOUCH=y +CONFIG_HID_NTRIG=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_PICOLCD=y +CONFIG_HID_PRIMAX=y +CONFIG_HID_ROCCAT=y +CONFIG_HID_SAITEK=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SONY=y +CONFIG_HID_SPEEDLINK=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_HID_TIVO=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_WACOM=y +CONFIG_HID_WACOM_POWER_SUPPLY=y +CONFIG_HID_WIIMOTE=y +CONFIG_HID_ZEROPLUS=y +CONFIG_HID_ZYDACRON=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_SUSPEND=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_ACM=y +CONFIG_USB_PRINTER=y +CONFIG_USB_STORAGE=y +CONFIG_USB_STORAGE_DATAFAB=y +CONFIG_USB_STORAGE_FREECOM=y +CONFIG_USB_STORAGE_ISD200=y +CONFIG_USB_STORAGE_USBAT=y +CONFIG_USB_STORAGE_SDDR09=y +CONFIG_USB_STORAGE_SDDR55=y +CONFIG_USB_STORAGE_JUMPSHOT=y +CONFIG_USB_STORAGE_ALAUDA=y +CONFIG_USB_STORAGE_ONETOUCH=y +CONFIG_USB_STORAGE_KARMA=y +CONFIG_USB_STORAGE_CYPRESS_ATACB=y +CONFIG_USB_STORAGE_ENE_UB6250=y +CONFIG_USB_TRANCEVIBRATOR=y +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_DEBUG_FILES=y +CONFIG_USB_DWC3_MSM=y +CONFIG_USB_G_ANDROID=y +CONFIG_MMC=y +CONFIG_MMC_PERF_PROFILING=y +CONFIG_MMC_UNSAFE_RESUME=y +CONFIG_MMC_CLKGATE=y +CONFIG_MMC_PARANOID_SD_INIT=y +CONFIG_MMC_BLOCK_MINORS=32 +# CONFIG_MMC_BLOCK_BOUNCE is not set +CONFIG_MMC_BLOCK_TEST=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_SDHCI_MSM=y +CONFIG_MMC_SDHCI_MSM_DISABLE_HPI=y +CONFIG_MMC_MSM=y +CONFIG_MMC_MSM_SPS_SUPPORT=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_QPNP=y +CONFIG_LEDS_TRIGGERS=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_SWITCH=y +CONFIG_SWITCH_MAX1462X=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_DRV_MSM is not set +CONFIG_RTC_DRV_QPNP=y +CONFIG_UIO=y +CONFIG_UIO_MSM_SHAREDMEM=y +CONFIG_STAGING=y +CONFIG_ANDROID=y +CONFIG_ANDROID_BINDER_IPC=y +CONFIG_ASHMEM=y +CONFIG_ANDROID_RAM_CONSOLE=y +CONFIG_PERSISTENT_TRACER=y +CONFIG_ANDROID_TIMED_GPIO=y +CONFIG_ANDROID_LOW_MEMORY_KILLER=y +CONFIG_SPS=y +CONFIG_USB_BAM=y +CONFIG_SPS_SUPPORT_BAMDMA=y +CONFIG_SPS_SUPPORT_NDP_BAM=y +CONFIG_QPNP_PWM=y +CONFIG_QPNP_POWER_ON=y +CONFIG_QPNP_CLKDIV=y +CONFIG_QPNP_REVID=y +CONFIG_QPNP_COINCELL=y +CONFIG_MSM_IOMMU_V1=y +CONFIG_IOMMU_PGTABLES_L2=y +CONFIG_EXT2_FS=y +CONFIG_EXT2_FS_XATTR=y +CONFIG_EXT3_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_F2FS_FS=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_FUSE_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_CIFS=y +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_POSIX=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +CONFIG_PRINTK_TIME=y +CONFIG_MAGIC_SYSRQ=y +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +CONFIG_TIMER_STATS=y +# CONFIG_DEBUG_PREEMPT is not set +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_INFO=y +# CONFIG_RCU_CPU_STALL_VERBOSE is not set +CONFIG_DYNAMIC_DEBUG=y +CONFIG_PANIC_ON_DATA_CORRUPTION=y +CONFIG_PID_IN_CONTEXTIDR=y +CONFIG_KEYS=y +CONFIG_SECURITY_PERF_EVENTS_RESTRICT=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_LSM_MMAP_MIN_ADDR=4096 +CONFIG_SECURITY_SELINUX=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_AES_ARM=y +CONFIG_CRYPTO_TWOFISH=y +# CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/arm/crypto/.gitignore b/arch/arm/crypto/.gitignore new file mode 100644 index 00000000000..6231d36b363 --- /dev/null +++ b/arch/arm/crypto/.gitignore @@ -0,0 +1 @@ +aesbs-core.S diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile new file mode 100644 index 00000000000..b48fa341648 --- /dev/null +++ b/arch/arm/crypto/Makefile @@ -0,0 +1,23 @@ +# +# Arch-specific CryptoAPI modules. +# + +obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o +obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o +obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o +obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o +obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o + +aes-arm-y := aes-armv4.o aes_glue.o +aes-arm-bs-y := aesbs-core.o aesbs-glue.o +sha1-arm-y := sha1-armv4-large.o sha1_glue.o +sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o +sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o + +quiet_cmd_perl = PERL $@ + cmd_perl = $(PERL) $(<) > $(@) + +$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl + $(call cmd,perl) + +.PRECIOUS: $(obj)/aesbs-core.S diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S new file mode 100644 index 00000000000..3a14ea8fe97 --- /dev/null +++ b/arch/arm/crypto/aes-armv4.S @@ -0,0 +1,1088 @@ +#define __ARM_ARCH__ __LINUX_ARM_ARCH__ +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ ==================================================================== + +@ AES for ARMv4 + +@ January 2007. +@ +@ Code uses single 1K S-box and is >2 times faster than code generated +@ by gcc-3.4.1. This is thanks to unique feature of ARMv4 ISA, which +@ allows to merge logical or arithmetic operation with shift or rotate +@ in one instruction and emit combined result every cycle. The module +@ is endian-neutral. The performance is ~42 cycles/byte for 128-bit +@ key [on single-issue Xscale PXA250 core]. + +@ May 2007. +@ +@ AES_set_[en|de]crypt_key is added. + +@ July 2010. +@ +@ Rescheduling for dual-issue pipeline resulted in 12% improvement on +@ Cortex A8 core and ~25 cycles per byte processed with 128-bit key. + +@ February 2011. +@ +@ Profiler-assisted and platform-specific optimization resulted in 16% +@ improvement on Cortex A8 core and ~21.5 cycles per byte. + +@ A little glue here to select the correct code below for the ARM CPU +@ that is being targetted. + +#include + +.text + +.type AES_Te,%object +.align 5 +AES_Te: +.word 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d +.word 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554 +.word 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d +.word 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a +.word 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87 +.word 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b +.word 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea +.word 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b +.word 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a +.word 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f +.word 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108 +.word 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f +.word 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e +.word 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5 +.word 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d +.word 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f +.word 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e +.word 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb +.word 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce +.word 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497 +.word 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c +.word 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed +.word 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b +.word 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a +.word 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16 +.word 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594 +.word 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81 +.word 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3 +.word 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a +.word 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504 +.word 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163 +.word 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d +.word 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f +.word 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739 +.word 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47 +.word 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395 +.word 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f +.word 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883 +.word 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c +.word 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76 +.word 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e +.word 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4 +.word 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6 +.word 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b +.word 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7 +.word 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0 +.word 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25 +.word 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818 +.word 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72 +.word 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651 +.word 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21 +.word 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85 +.word 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa +.word 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12 +.word 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0 +.word 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9 +.word 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133 +.word 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7 +.word 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920 +.word 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a +.word 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17 +.word 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8 +.word 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11 +.word 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a +@ Te4[256] +.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 +.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76 +.byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0 +.byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0 +.byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc +.byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15 +.byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a +.byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75 +.byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0 +.byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84 +.byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b +.byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf +.byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85 +.byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8 +.byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5 +.byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2 +.byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17 +.byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73 +.byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88 +.byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb +.byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c +.byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79 +.byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9 +.byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08 +.byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6 +.byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a +.byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e +.byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e +.byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94 +.byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf +.byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68 +.byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 +@ rcon[] +.word 0x01000000, 0x02000000, 0x04000000, 0x08000000 +.word 0x10000000, 0x20000000, 0x40000000, 0x80000000 +.word 0x1B000000, 0x36000000, 0, 0, 0, 0, 0, 0 +.size AES_Te,.-AES_Te + +@ void AES_encrypt(const unsigned char *in, unsigned char *out, +@ const AES_KEY *key) { +.align 5 +ENTRY(AES_encrypt) + adr r3,AES_encrypt + stmdb sp!,{r1,r4-r12,lr} + mov r12,r0 @ inp + mov r11,r2 + sub r10,r3,#AES_encrypt-AES_Te @ Te +#if __ARM_ARCH__<7 + ldrb r0,[r12,#3] @ load input data in endian-neutral + ldrb r4,[r12,#2] @ manner... + ldrb r5,[r12,#1] + ldrb r6,[r12,#0] + orr r0,r0,r4,lsl#8 + ldrb r1,[r12,#7] + orr r0,r0,r5,lsl#16 + ldrb r4,[r12,#6] + orr r0,r0,r6,lsl#24 + ldrb r5,[r12,#5] + ldrb r6,[r12,#4] + orr r1,r1,r4,lsl#8 + ldrb r2,[r12,#11] + orr r1,r1,r5,lsl#16 + ldrb r4,[r12,#10] + orr r1,r1,r6,lsl#24 + ldrb r5,[r12,#9] + ldrb r6,[r12,#8] + orr r2,r2,r4,lsl#8 + ldrb r3,[r12,#15] + orr r2,r2,r5,lsl#16 + ldrb r4,[r12,#14] + orr r2,r2,r6,lsl#24 + ldrb r5,[r12,#13] + ldrb r6,[r12,#12] + orr r3,r3,r4,lsl#8 + orr r3,r3,r5,lsl#16 + orr r3,r3,r6,lsl#24 +#else + ldr r0,[r12,#0] + ldr r1,[r12,#4] + ldr r2,[r12,#8] + ldr r3,[r12,#12] +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif +#endif + bl _armv4_AES_encrypt + + ldr r12,[sp],#4 @ pop out +#if __ARM_ARCH__>=7 +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif + str r0,[r12,#0] + str r1,[r12,#4] + str r2,[r12,#8] + str r3,[r12,#12] +#else + mov r4,r0,lsr#24 @ write output in endian-neutral + mov r5,r0,lsr#16 @ manner... + mov r6,r0,lsr#8 + strb r4,[r12,#0] + strb r5,[r12,#1] + mov r4,r1,lsr#24 + strb r6,[r12,#2] + mov r5,r1,lsr#16 + strb r0,[r12,#3] + mov r6,r1,lsr#8 + strb r4,[r12,#4] + strb r5,[r12,#5] + mov r4,r2,lsr#24 + strb r6,[r12,#6] + mov r5,r2,lsr#16 + strb r1,[r12,#7] + mov r6,r2,lsr#8 + strb r4,[r12,#8] + strb r5,[r12,#9] + mov r4,r3,lsr#24 + strb r6,[r12,#10] + mov r5,r3,lsr#16 + strb r2,[r12,#11] + mov r6,r3,lsr#8 + strb r4,[r12,#12] + strb r5,[r12,#13] + strb r6,[r12,#14] + strb r3,[r12,#15] +#endif + ldmia sp!,{r4-r12,pc} +ENDPROC(AES_encrypt) + +.type _armv4_AES_encrypt,%function +.align 2 +_armv4_AES_encrypt: + str lr,[sp,#-4]! @ push lr + ldmia r11!,{r4-r7} + eor r0,r0,r4 + ldr r12,[r11,#240-16] + eor r1,r1,r5 + eor r2,r2,r6 + eor r3,r3,r7 + sub r12,r12,#1 + mov lr,#255 + + and r7,lr,r0 + and r8,lr,r0,lsr#8 + and r9,lr,r0,lsr#16 + mov r0,r0,lsr#24 +.Lenc_loop: + ldr r4,[r10,r7,lsl#2] @ Te3[s0>>0] + and r7,lr,r1,lsr#16 @ i0 + ldr r5,[r10,r8,lsl#2] @ Te2[s0>>8] + and r8,lr,r1 + ldr r6,[r10,r9,lsl#2] @ Te1[s0>>16] + and r9,lr,r1,lsr#8 + ldr r0,[r10,r0,lsl#2] @ Te0[s0>>24] + mov r1,r1,lsr#24 + + ldr r7,[r10,r7,lsl#2] @ Te1[s1>>16] + ldr r8,[r10,r8,lsl#2] @ Te3[s1>>0] + ldr r9,[r10,r9,lsl#2] @ Te2[s1>>8] + eor r0,r0,r7,ror#8 + ldr r1,[r10,r1,lsl#2] @ Te0[s1>>24] + and r7,lr,r2,lsr#8 @ i0 + eor r5,r5,r8,ror#8 + and r8,lr,r2,lsr#16 @ i1 + eor r6,r6,r9,ror#8 + and r9,lr,r2 + ldr r7,[r10,r7,lsl#2] @ Te2[s2>>8] + eor r1,r1,r4,ror#24 + ldr r8,[r10,r8,lsl#2] @ Te1[s2>>16] + mov r2,r2,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Te3[s2>>0] + eor r0,r0,r7,ror#16 + ldr r2,[r10,r2,lsl#2] @ Te0[s2>>24] + and r7,lr,r3 @ i0 + eor r1,r1,r8,ror#8 + and r8,lr,r3,lsr#8 @ i1 + eor r6,r6,r9,ror#16 + and r9,lr,r3,lsr#16 @ i2 + ldr r7,[r10,r7,lsl#2] @ Te3[s3>>0] + eor r2,r2,r5,ror#16 + ldr r8,[r10,r8,lsl#2] @ Te2[s3>>8] + mov r3,r3,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Te1[s3>>16] + eor r0,r0,r7,ror#24 + ldr r7,[r11],#16 + eor r1,r1,r8,ror#16 + ldr r3,[r10,r3,lsl#2] @ Te0[s3>>24] + eor r2,r2,r9,ror#8 + ldr r4,[r11,#-12] + eor r3,r3,r6,ror#8 + + ldr r5,[r11,#-8] + eor r0,r0,r7 + ldr r6,[r11,#-4] + and r7,lr,r0 + eor r1,r1,r4 + and r8,lr,r0,lsr#8 + eor r2,r2,r5 + and r9,lr,r0,lsr#16 + eor r3,r3,r6 + mov r0,r0,lsr#24 + + subs r12,r12,#1 + bne .Lenc_loop + + add r10,r10,#2 + + ldrb r4,[r10,r7,lsl#2] @ Te4[s0>>0] + and r7,lr,r1,lsr#16 @ i0 + ldrb r5,[r10,r8,lsl#2] @ Te4[s0>>8] + and r8,lr,r1 + ldrb r6,[r10,r9,lsl#2] @ Te4[s0>>16] + and r9,lr,r1,lsr#8 + ldrb r0,[r10,r0,lsl#2] @ Te4[s0>>24] + mov r1,r1,lsr#24 + + ldrb r7,[r10,r7,lsl#2] @ Te4[s1>>16] + ldrb r8,[r10,r8,lsl#2] @ Te4[s1>>0] + ldrb r9,[r10,r9,lsl#2] @ Te4[s1>>8] + eor r0,r7,r0,lsl#8 + ldrb r1,[r10,r1,lsl#2] @ Te4[s1>>24] + and r7,lr,r2,lsr#8 @ i0 + eor r5,r8,r5,lsl#8 + and r8,lr,r2,lsr#16 @ i1 + eor r6,r9,r6,lsl#8 + and r9,lr,r2 + ldrb r7,[r10,r7,lsl#2] @ Te4[s2>>8] + eor r1,r4,r1,lsl#24 + ldrb r8,[r10,r8,lsl#2] @ Te4[s2>>16] + mov r2,r2,lsr#24 + + ldrb r9,[r10,r9,lsl#2] @ Te4[s2>>0] + eor r0,r7,r0,lsl#8 + ldrb r2,[r10,r2,lsl#2] @ Te4[s2>>24] + and r7,lr,r3 @ i0 + eor r1,r1,r8,lsl#16 + and r8,lr,r3,lsr#8 @ i1 + eor r6,r9,r6,lsl#8 + and r9,lr,r3,lsr#16 @ i2 + ldrb r7,[r10,r7,lsl#2] @ Te4[s3>>0] + eor r2,r5,r2,lsl#24 + ldrb r8,[r10,r8,lsl#2] @ Te4[s3>>8] + mov r3,r3,lsr#24 + + ldrb r9,[r10,r9,lsl#2] @ Te4[s3>>16] + eor r0,r7,r0,lsl#8 + ldr r7,[r11,#0] + ldrb r3,[r10,r3,lsl#2] @ Te4[s3>>24] + eor r1,r1,r8,lsl#8 + ldr r4,[r11,#4] + eor r2,r2,r9,lsl#16 + ldr r5,[r11,#8] + eor r3,r6,r3,lsl#24 + ldr r6,[r11,#12] + + eor r0,r0,r7 + eor r1,r1,r4 + eor r2,r2,r5 + eor r3,r3,r6 + + sub r10,r10,#2 + ldr pc,[sp],#4 @ pop and return +.size _armv4_AES_encrypt,.-_armv4_AES_encrypt + +.align 5 +ENTRY(private_AES_set_encrypt_key) +_armv4_AES_set_encrypt_key: + adr r3,_armv4_AES_set_encrypt_key + teq r0,#0 + moveq r0,#-1 + beq .Labrt + teq r2,#0 + moveq r0,#-1 + beq .Labrt + + teq r1,#128 + beq .Lok + teq r1,#192 + beq .Lok + teq r1,#256 + movne r0,#-1 + bne .Labrt + +.Lok: stmdb sp!,{r4-r12,lr} + sub r10,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024 @ Te4 + + mov r12,r0 @ inp + mov lr,r1 @ bits + mov r11,r2 @ key + +#if __ARM_ARCH__<7 + ldrb r0,[r12,#3] @ load input data in endian-neutral + ldrb r4,[r12,#2] @ manner... + ldrb r5,[r12,#1] + ldrb r6,[r12,#0] + orr r0,r0,r4,lsl#8 + ldrb r1,[r12,#7] + orr r0,r0,r5,lsl#16 + ldrb r4,[r12,#6] + orr r0,r0,r6,lsl#24 + ldrb r5,[r12,#5] + ldrb r6,[r12,#4] + orr r1,r1,r4,lsl#8 + ldrb r2,[r12,#11] + orr r1,r1,r5,lsl#16 + ldrb r4,[r12,#10] + orr r1,r1,r6,lsl#24 + ldrb r5,[r12,#9] + ldrb r6,[r12,#8] + orr r2,r2,r4,lsl#8 + ldrb r3,[r12,#15] + orr r2,r2,r5,lsl#16 + ldrb r4,[r12,#14] + orr r2,r2,r6,lsl#24 + ldrb r5,[r12,#13] + ldrb r6,[r12,#12] + orr r3,r3,r4,lsl#8 + str r0,[r11],#16 + orr r3,r3,r5,lsl#16 + str r1,[r11,#-12] + orr r3,r3,r6,lsl#24 + str r2,[r11,#-8] + str r3,[r11,#-4] +#else + ldr r0,[r12,#0] + ldr r1,[r12,#4] + ldr r2,[r12,#8] + ldr r3,[r12,#12] +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif + str r0,[r11],#16 + str r1,[r11,#-12] + str r2,[r11,#-8] + str r3,[r11,#-4] +#endif + + teq lr,#128 + bne .Lnot128 + mov r12,#10 + str r12,[r11,#240-16] + add r6,r10,#256 @ rcon + mov lr,#255 + +.L128_loop: + and r5,lr,r3,lsr#24 + and r7,lr,r3,lsr#16 + ldrb r5,[r10,r5] + and r8,lr,r3,lsr#8 + ldrb r7,[r10,r7] + and r9,lr,r3 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#24 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r6],#4 @ rcon[i++] + orr r5,r5,r9,lsl#8 + eor r5,r5,r4 + eor r0,r0,r5 @ rk[4]=rk[0]^... + eor r1,r1,r0 @ rk[5]=rk[1]^rk[4] + str r0,[r11],#16 + eor r2,r2,r1 @ rk[6]=rk[2]^rk[5] + str r1,[r11,#-12] + eor r3,r3,r2 @ rk[7]=rk[3]^rk[6] + str r2,[r11,#-8] + subs r12,r12,#1 + str r3,[r11,#-4] + bne .L128_loop + sub r2,r11,#176 + b .Ldone + +.Lnot128: +#if __ARM_ARCH__<7 + ldrb r8,[r12,#19] + ldrb r4,[r12,#18] + ldrb r5,[r12,#17] + ldrb r6,[r12,#16] + orr r8,r8,r4,lsl#8 + ldrb r9,[r12,#23] + orr r8,r8,r5,lsl#16 + ldrb r4,[r12,#22] + orr r8,r8,r6,lsl#24 + ldrb r5,[r12,#21] + ldrb r6,[r12,#20] + orr r9,r9,r4,lsl#8 + orr r9,r9,r5,lsl#16 + str r8,[r11],#8 + orr r9,r9,r6,lsl#24 + str r9,[r11,#-4] +#else + ldr r8,[r12,#16] + ldr r9,[r12,#20] +#ifdef __ARMEL__ + rev r8,r8 + rev r9,r9 +#endif + str r8,[r11],#8 + str r9,[r11,#-4] +#endif + + teq lr,#192 + bne .Lnot192 + mov r12,#12 + str r12,[r11,#240-24] + add r6,r10,#256 @ rcon + mov lr,#255 + mov r12,#8 + +.L192_loop: + and r5,lr,r9,lsr#24 + and r7,lr,r9,lsr#16 + ldrb r5,[r10,r5] + and r8,lr,r9,lsr#8 + ldrb r7,[r10,r7] + and r9,lr,r9 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#24 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r6],#4 @ rcon[i++] + orr r5,r5,r9,lsl#8 + eor r9,r5,r4 + eor r0,r0,r9 @ rk[6]=rk[0]^... + eor r1,r1,r0 @ rk[7]=rk[1]^rk[6] + str r0,[r11],#24 + eor r2,r2,r1 @ rk[8]=rk[2]^rk[7] + str r1,[r11,#-20] + eor r3,r3,r2 @ rk[9]=rk[3]^rk[8] + str r2,[r11,#-16] + subs r12,r12,#1 + str r3,[r11,#-12] + subeq r2,r11,#216 + beq .Ldone + + ldr r7,[r11,#-32] + ldr r8,[r11,#-28] + eor r7,r7,r3 @ rk[10]=rk[4]^rk[9] + eor r9,r8,r7 @ rk[11]=rk[5]^rk[10] + str r7,[r11,#-8] + str r9,[r11,#-4] + b .L192_loop + +.Lnot192: +#if __ARM_ARCH__<7 + ldrb r8,[r12,#27] + ldrb r4,[r12,#26] + ldrb r5,[r12,#25] + ldrb r6,[r12,#24] + orr r8,r8,r4,lsl#8 + ldrb r9,[r12,#31] + orr r8,r8,r5,lsl#16 + ldrb r4,[r12,#30] + orr r8,r8,r6,lsl#24 + ldrb r5,[r12,#29] + ldrb r6,[r12,#28] + orr r9,r9,r4,lsl#8 + orr r9,r9,r5,lsl#16 + str r8,[r11],#8 + orr r9,r9,r6,lsl#24 + str r9,[r11,#-4] +#else + ldr r8,[r12,#24] + ldr r9,[r12,#28] +#ifdef __ARMEL__ + rev r8,r8 + rev r9,r9 +#endif + str r8,[r11],#8 + str r9,[r11,#-4] +#endif + + mov r12,#14 + str r12,[r11,#240-32] + add r6,r10,#256 @ rcon + mov lr,#255 + mov r12,#7 + +.L256_loop: + and r5,lr,r9,lsr#24 + and r7,lr,r9,lsr#16 + ldrb r5,[r10,r5] + and r8,lr,r9,lsr#8 + ldrb r7,[r10,r7] + and r9,lr,r9 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#24 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r6],#4 @ rcon[i++] + orr r5,r5,r9,lsl#8 + eor r9,r5,r4 + eor r0,r0,r9 @ rk[8]=rk[0]^... + eor r1,r1,r0 @ rk[9]=rk[1]^rk[8] + str r0,[r11],#32 + eor r2,r2,r1 @ rk[10]=rk[2]^rk[9] + str r1,[r11,#-28] + eor r3,r3,r2 @ rk[11]=rk[3]^rk[10] + str r2,[r11,#-24] + subs r12,r12,#1 + str r3,[r11,#-20] + subeq r2,r11,#256 + beq .Ldone + + and r5,lr,r3 + and r7,lr,r3,lsr#8 + ldrb r5,[r10,r5] + and r8,lr,r3,lsr#16 + ldrb r7,[r10,r7] + and r9,lr,r3,lsr#24 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#8 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r11,#-48] + orr r5,r5,r9,lsl#24 + + ldr r7,[r11,#-44] + ldr r8,[r11,#-40] + eor r4,r4,r5 @ rk[12]=rk[4]^... + ldr r9,[r11,#-36] + eor r7,r7,r4 @ rk[13]=rk[5]^rk[12] + str r4,[r11,#-16] + eor r8,r8,r7 @ rk[14]=rk[6]^rk[13] + str r7,[r11,#-12] + eor r9,r9,r8 @ rk[15]=rk[7]^rk[14] + str r8,[r11,#-8] + str r9,[r11,#-4] + b .L256_loop + +.Ldone: mov r0,#0 + ldmia sp!,{r4-r12,lr} +.Labrt: mov pc,lr +ENDPROC(private_AES_set_encrypt_key) + +.align 5 +ENTRY(private_AES_set_decrypt_key) + str lr,[sp,#-4]! @ push lr +#if 0 + @ kernel does both of these in setkey so optimise this bit out by + @ expecting the key to already have the enc_key work done (see aes_glue.c) + bl _armv4_AES_set_encrypt_key +#else + mov r0,#0 +#endif + teq r0,#0 + ldrne lr,[sp],#4 @ pop lr + bne .Labrt + + stmdb sp!,{r4-r12} + + ldr r12,[r2,#240] @ AES_set_encrypt_key preserves r2, + mov r11,r2 @ which is AES_KEY *key + mov r7,r2 + add r8,r2,r12,lsl#4 + +.Linv: ldr r0,[r7] + ldr r1,[r7,#4] + ldr r2,[r7,#8] + ldr r3,[r7,#12] + ldr r4,[r8] + ldr r5,[r8,#4] + ldr r6,[r8,#8] + ldr r9,[r8,#12] + str r0,[r8],#-16 + str r1,[r8,#16+4] + str r2,[r8,#16+8] + str r3,[r8,#16+12] + str r4,[r7],#16 + str r5,[r7,#-12] + str r6,[r7,#-8] + str r9,[r7,#-4] + teq r7,r8 + bne .Linv + ldr r0,[r11,#16]! @ prefetch tp1 + mov r7,#0x80 + mov r8,#0x1b + orr r7,r7,#0x8000 + orr r8,r8,#0x1b00 + orr r7,r7,r7,lsl#16 + orr r8,r8,r8,lsl#16 + sub r12,r12,#1 + mvn r9,r7 + mov r12,r12,lsl#2 @ (rounds-1)*4 + +.Lmix: and r4,r0,r7 + and r1,r0,r9 + sub r4,r4,r4,lsr#7 + and r4,r4,r8 + eor r1,r4,r1,lsl#1 @ tp2 + + and r4,r1,r7 + and r2,r1,r9 + sub r4,r4,r4,lsr#7 + and r4,r4,r8 + eor r2,r4,r2,lsl#1 @ tp4 + + and r4,r2,r7 + and r3,r2,r9 + sub r4,r4,r4,lsr#7 + and r4,r4,r8 + eor r3,r4,r3,lsl#1 @ tp8 + + eor r4,r1,r2 + eor r5,r0,r3 @ tp9 + eor r4,r4,r3 @ tpe + eor r4,r4,r1,ror#24 + eor r4,r4,r5,ror#24 @ ^= ROTATE(tpb=tp9^tp2,8) + eor r4,r4,r2,ror#16 + eor r4,r4,r5,ror#16 @ ^= ROTATE(tpd=tp9^tp4,16) + eor r4,r4,r5,ror#8 @ ^= ROTATE(tp9,24) + + ldr r0,[r11,#4] @ prefetch tp1 + str r4,[r11],#4 + subs r12,r12,#1 + bne .Lmix + + mov r0,#0 + ldmia sp!,{r4-r12,pc} +ENDPROC(private_AES_set_decrypt_key) + +.type AES_Td,%object +.align 5 +AES_Td: +.word 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96 +.word 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393 +.word 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25 +.word 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f +.word 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1 +.word 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6 +.word 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da +.word 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844 +.word 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd +.word 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4 +.word 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45 +.word 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94 +.word 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7 +.word 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a +.word 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5 +.word 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c +.word 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1 +.word 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a +.word 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75 +.word 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051 +.word 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46 +.word 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff +.word 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77 +.word 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb +.word 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000 +.word 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e +.word 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927 +.word 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a +.word 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e +.word 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16 +.word 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d +.word 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8 +.word 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd +.word 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34 +.word 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163 +.word 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120 +.word 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d +.word 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0 +.word 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422 +.word 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef +.word 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36 +.word 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4 +.word 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662 +.word 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5 +.word 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3 +.word 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b +.word 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8 +.word 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6 +.word 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6 +.word 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0 +.word 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815 +.word 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f +.word 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df +.word 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f +.word 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e +.word 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713 +.word 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89 +.word 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c +.word 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf +.word 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86 +.word 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f +.word 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541 +.word 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190 +.word 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742 +@ Td4[256] +.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38 +.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb +.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87 +.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb +.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d +.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e +.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2 +.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25 +.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16 +.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92 +.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda +.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84 +.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a +.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06 +.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02 +.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b +.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea +.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73 +.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85 +.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e +.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89 +.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b +.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20 +.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4 +.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31 +.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f +.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d +.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef +.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0 +.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 +.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 +.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d +.size AES_Td,.-AES_Td + +@ void AES_decrypt(const unsigned char *in, unsigned char *out, +@ const AES_KEY *key) { +.align 5 +ENTRY(AES_decrypt) + adr r3,AES_decrypt + stmdb sp!,{r1,r4-r12,lr} + mov r12,r0 @ inp + mov r11,r2 + sub r10,r3,#AES_decrypt-AES_Td @ Td +#if __ARM_ARCH__<7 + ldrb r0,[r12,#3] @ load input data in endian-neutral + ldrb r4,[r12,#2] @ manner... + ldrb r5,[r12,#1] + ldrb r6,[r12,#0] + orr r0,r0,r4,lsl#8 + ldrb r1,[r12,#7] + orr r0,r0,r5,lsl#16 + ldrb r4,[r12,#6] + orr r0,r0,r6,lsl#24 + ldrb r5,[r12,#5] + ldrb r6,[r12,#4] + orr r1,r1,r4,lsl#8 + ldrb r2,[r12,#11] + orr r1,r1,r5,lsl#16 + ldrb r4,[r12,#10] + orr r1,r1,r6,lsl#24 + ldrb r5,[r12,#9] + ldrb r6,[r12,#8] + orr r2,r2,r4,lsl#8 + ldrb r3,[r12,#15] + orr r2,r2,r5,lsl#16 + ldrb r4,[r12,#14] + orr r2,r2,r6,lsl#24 + ldrb r5,[r12,#13] + ldrb r6,[r12,#12] + orr r3,r3,r4,lsl#8 + orr r3,r3,r5,lsl#16 + orr r3,r3,r6,lsl#24 +#else + ldr r0,[r12,#0] + ldr r1,[r12,#4] + ldr r2,[r12,#8] + ldr r3,[r12,#12] +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif +#endif + bl _armv4_AES_decrypt + + ldr r12,[sp],#4 @ pop out +#if __ARM_ARCH__>=7 +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif + str r0,[r12,#0] + str r1,[r12,#4] + str r2,[r12,#8] + str r3,[r12,#12] +#else + mov r4,r0,lsr#24 @ write output in endian-neutral + mov r5,r0,lsr#16 @ manner... + mov r6,r0,lsr#8 + strb r4,[r12,#0] + strb r5,[r12,#1] + mov r4,r1,lsr#24 + strb r6,[r12,#2] + mov r5,r1,lsr#16 + strb r0,[r12,#3] + mov r6,r1,lsr#8 + strb r4,[r12,#4] + strb r5,[r12,#5] + mov r4,r2,lsr#24 + strb r6,[r12,#6] + mov r5,r2,lsr#16 + strb r1,[r12,#7] + mov r6,r2,lsr#8 + strb r4,[r12,#8] + strb r5,[r12,#9] + mov r4,r3,lsr#24 + strb r6,[r12,#10] + mov r5,r3,lsr#16 + strb r2,[r12,#11] + mov r6,r3,lsr#8 + strb r4,[r12,#12] + strb r5,[r12,#13] + strb r6,[r12,#14] + strb r3,[r12,#15] +#endif + ldmia sp!,{r4-r12,pc} +ENDPROC(AES_decrypt) + +.type _armv4_AES_decrypt,%function +.align 2 +_armv4_AES_decrypt: + str lr,[sp,#-4]! @ push lr + ldmia r11!,{r4-r7} + eor r0,r0,r4 + ldr r12,[r11,#240-16] + eor r1,r1,r5 + eor r2,r2,r6 + eor r3,r3,r7 + sub r12,r12,#1 + mov lr,#255 + + and r7,lr,r0,lsr#16 + and r8,lr,r0,lsr#8 + and r9,lr,r0 + mov r0,r0,lsr#24 +.Ldec_loop: + ldr r4,[r10,r7,lsl#2] @ Td1[s0>>16] + and r7,lr,r1 @ i0 + ldr r5,[r10,r8,lsl#2] @ Td2[s0>>8] + and r8,lr,r1,lsr#16 + ldr r6,[r10,r9,lsl#2] @ Td3[s0>>0] + and r9,lr,r1,lsr#8 + ldr r0,[r10,r0,lsl#2] @ Td0[s0>>24] + mov r1,r1,lsr#24 + + ldr r7,[r10,r7,lsl#2] @ Td3[s1>>0] + ldr r8,[r10,r8,lsl#2] @ Td1[s1>>16] + ldr r9,[r10,r9,lsl#2] @ Td2[s1>>8] + eor r0,r0,r7,ror#24 + ldr r1,[r10,r1,lsl#2] @ Td0[s1>>24] + and r7,lr,r2,lsr#8 @ i0 + eor r5,r8,r5,ror#8 + and r8,lr,r2 @ i1 + eor r6,r9,r6,ror#8 + and r9,lr,r2,lsr#16 + ldr r7,[r10,r7,lsl#2] @ Td2[s2>>8] + eor r1,r1,r4,ror#8 + ldr r8,[r10,r8,lsl#2] @ Td3[s2>>0] + mov r2,r2,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Td1[s2>>16] + eor r0,r0,r7,ror#16 + ldr r2,[r10,r2,lsl#2] @ Td0[s2>>24] + and r7,lr,r3,lsr#16 @ i0 + eor r1,r1,r8,ror#24 + and r8,lr,r3,lsr#8 @ i1 + eor r6,r9,r6,ror#8 + and r9,lr,r3 @ i2 + ldr r7,[r10,r7,lsl#2] @ Td1[s3>>16] + eor r2,r2,r5,ror#8 + ldr r8,[r10,r8,lsl#2] @ Td2[s3>>8] + mov r3,r3,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Td3[s3>>0] + eor r0,r0,r7,ror#8 + ldr r7,[r11],#16 + eor r1,r1,r8,ror#16 + ldr r3,[r10,r3,lsl#2] @ Td0[s3>>24] + eor r2,r2,r9,ror#24 + + ldr r4,[r11,#-12] + eor r0,r0,r7 + ldr r5,[r11,#-8] + eor r3,r3,r6,ror#8 + ldr r6,[r11,#-4] + and r7,lr,r0,lsr#16 + eor r1,r1,r4 + and r8,lr,r0,lsr#8 + eor r2,r2,r5 + and r9,lr,r0 + eor r3,r3,r6 + mov r0,r0,lsr#24 + + subs r12,r12,#1 + bne .Ldec_loop + + add r10,r10,#1024 + + ldr r5,[r10,#0] @ prefetch Td4 + ldr r6,[r10,#32] + ldr r4,[r10,#64] + ldr r5,[r10,#96] + ldr r6,[r10,#128] + ldr r4,[r10,#160] + ldr r5,[r10,#192] + ldr r6,[r10,#224] + + ldrb r0,[r10,r0] @ Td4[s0>>24] + ldrb r4,[r10,r7] @ Td4[s0>>16] + and r7,lr,r1 @ i0 + ldrb r5,[r10,r8] @ Td4[s0>>8] + and r8,lr,r1,lsr#16 + ldrb r6,[r10,r9] @ Td4[s0>>0] + and r9,lr,r1,lsr#8 + + ldrb r7,[r10,r7] @ Td4[s1>>0] + ARM( ldrb r1,[r10,r1,lsr#24] ) @ Td4[s1>>24] + THUMB( add r1,r10,r1,lsr#24 ) @ Td4[s1>>24] + THUMB( ldrb r1,[r1] ) + ldrb r8,[r10,r8] @ Td4[s1>>16] + eor r0,r7,r0,lsl#24 + ldrb r9,[r10,r9] @ Td4[s1>>8] + eor r1,r4,r1,lsl#8 + and r7,lr,r2,lsr#8 @ i0 + eor r5,r5,r8,lsl#8 + and r8,lr,r2 @ i1 + ldrb r7,[r10,r7] @ Td4[s2>>8] + eor r6,r6,r9,lsl#8 + ldrb r8,[r10,r8] @ Td4[s2>>0] + and r9,lr,r2,lsr#16 + + ARM( ldrb r2,[r10,r2,lsr#24] ) @ Td4[s2>>24] + THUMB( add r2,r10,r2,lsr#24 ) @ Td4[s2>>24] + THUMB( ldrb r2,[r2] ) + eor r0,r0,r7,lsl#8 + ldrb r9,[r10,r9] @ Td4[s2>>16] + eor r1,r8,r1,lsl#16 + and r7,lr,r3,lsr#16 @ i0 + eor r2,r5,r2,lsl#16 + and r8,lr,r3,lsr#8 @ i1 + ldrb r7,[r10,r7] @ Td4[s3>>16] + eor r6,r6,r9,lsl#16 + ldrb r8,[r10,r8] @ Td4[s3>>8] + and r9,lr,r3 @ i2 + + ldrb r9,[r10,r9] @ Td4[s3>>0] + ARM( ldrb r3,[r10,r3,lsr#24] ) @ Td4[s3>>24] + THUMB( add r3,r10,r3,lsr#24 ) @ Td4[s3>>24] + THUMB( ldrb r3,[r3] ) + eor r0,r0,r7,lsl#16 + ldr r7,[r11,#0] + eor r1,r1,r8,lsl#8 + ldr r4,[r11,#4] + eor r2,r9,r2,lsl#8 + ldr r5,[r11,#8] + eor r3,r6,r3,lsl#24 + ldr r6,[r11,#12] + + eor r0,r0,r7 + eor r1,r1,r4 + eor r2,r2,r5 + eor r3,r3,r6 + + sub r10,r10,#1024 + ldr pc,[sp],#4 @ pop and return +.size _armv4_AES_decrypt,.-_armv4_AES_decrypt +.asciz "AES for ARMv4, CRYPTOGAMS by " +.align 2 diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c new file mode 100644 index 00000000000..3003fa1f6fb --- /dev/null +++ b/arch/arm/crypto/aes_glue.c @@ -0,0 +1,98 @@ +/* + * Glue Code for the asm optimized version of the AES Cipher Algorithm + */ + +#include +#include +#include + +#include "aes_glue.h" + +EXPORT_SYMBOL(AES_encrypt); +EXPORT_SYMBOL(AES_decrypt); +EXPORT_SYMBOL(private_AES_set_encrypt_key); +EXPORT_SYMBOL(private_AES_set_decrypt_key); + +static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct AES_CTX *ctx = crypto_tfm_ctx(tfm); + AES_encrypt(src, dst, &ctx->enc_key); +} + +static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct AES_CTX *ctx = crypto_tfm_ctx(tfm); + AES_decrypt(src, dst, &ctx->dec_key); +} + +static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct AES_CTX *ctx = crypto_tfm_ctx(tfm); + + switch (key_len) { + case AES_KEYSIZE_128: + key_len = 128; + break; + case AES_KEYSIZE_192: + key_len = 192; + break; + case AES_KEYSIZE_256: + key_len = 256; + break; + default: + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + /* private_AES_set_decrypt_key expects an encryption key as input */ + ctx->dec_key = ctx->enc_key; + if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + return 0; +} + +static struct crypto_alg aes_alg = { + .cra_name = "aes", + .cra_driver_name = "aes-asm", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct AES_CTX), + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), + .cra_u = { + .cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = aes_set_key, + .cia_encrypt = aes_encrypt, + .cia_decrypt = aes_decrypt + } + } +}; + +static int __init aes_init(void) +{ + return crypto_register_alg(&aes_alg); +} + +static void __exit aes_fini(void) +{ + crypto_unregister_alg(&aes_alg); +} + +module_init(aes_init); +module_exit(aes_fini); + +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("aes"); +MODULE_ALIAS("aes-asm"); +MODULE_AUTHOR("David McCullough "); diff --git a/arch/arm/crypto/aes_glue.h b/arch/arm/crypto/aes_glue.h new file mode 100644 index 00000000000..cca3e51eb60 --- /dev/null +++ b/arch/arm/crypto/aes_glue.h @@ -0,0 +1,19 @@ + +#define AES_MAXNR 14 + +struct AES_KEY { + unsigned int rd_key[4 * (AES_MAXNR + 1)]; + int rounds; +}; + +struct AES_CTX { + struct AES_KEY enc_key; + struct AES_KEY dec_key; +}; + +asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct AES_KEY *ctx); +asmlinkage void AES_decrypt(const u8 *in, u8 *out, struct AES_KEY *ctx); +asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, + const int bits, struct AES_KEY *key); +asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, + const int bits, struct AES_KEY *key); diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped new file mode 100644 index 00000000000..71e5fc7cfb1 --- /dev/null +++ b/arch/arm/crypto/aesbs-core.S_shipped @@ -0,0 +1,2544 @@ + +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ +@ Specific modes and adaptation for Linux kernel by Ard Biesheuvel +@ . Permission to use under GPL terms is +@ granted. +@ ==================================================================== + +@ Bit-sliced AES for ARM NEON +@ +@ February 2012. +@ +@ This implementation is direct adaptation of bsaes-x86_64 module for +@ ARM NEON. Except that this module is endian-neutral [in sense that +@ it can be compiled for either endianness] by courtesy of vld1.8's +@ neutrality. Initial version doesn't implement interface to OpenSSL, +@ only low-level primitives and unsupported entry points, just enough +@ to collect performance results, which for Cortex-A8 core are: +@ +@ encrypt 19.5 cycles per byte processed with 128-bit key +@ decrypt 22.1 cycles per byte processed with 128-bit key +@ key conv. 440 cycles per 128-bit key/0.18 of 8x block +@ +@ Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +@ which is [much] worse than anticipated (for further details see +@ http://www.openssl.org/~appro/Snapdragon-S4.html). +@ +@ Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +@ manages in 20.0 cycles]. +@ +@ When comparing to x86_64 results keep in mind that NEON unit is +@ [mostly] single-issue and thus can't [fully] benefit from +@ instruction-level parallelism. And when comparing to aes-armv4 +@ results keep in mind key schedule conversion overhead (see +@ bsaes-x86_64.pl for further details)... +@ +@ + +@ April-August 2013 +@ +@ Add CBC, CTR and XTS subroutines, adapt for kernel use. +@ +@ + +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ 7 +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_ARCH__>=7 +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.fpu neon + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr r6,_bsaes_decrypt8 + vldmia r4!, {q9} @ round 0 key + add r6,r6,#.LM0ISR-_bsaes_decrypt8 + + vldmia r6!, {q8} @ .LM0ISR + veor q10, q0, q9 @ xor with round0 key + veor q11, q1, q9 + vtbl.8 d0, {q10}, d16 + vtbl.8 d1, {q10}, d17 + veor q12, q2, q9 + vtbl.8 d2, {q11}, d16 + vtbl.8 d3, {q11}, d17 + veor q13, q3, q9 + vtbl.8 d4, {q12}, d16 + vtbl.8 d5, {q12}, d17 + veor q14, q4, q9 + vtbl.8 d6, {q13}, d16 + vtbl.8 d7, {q13}, d17 + veor q15, q5, q9 + vtbl.8 d8, {q14}, d16 + vtbl.8 d9, {q14}, d17 + veor q10, q6, q9 + vtbl.8 d10, {q15}, d16 + vtbl.8 d11, {q15}, d17 + veor q11, q7, q9 + vtbl.8 d12, {q10}, d16 + vtbl.8 d13, {q10}, d17 + vtbl.8 d14, {q11}, d16 + vtbl.8 d15, {q11}, d17 + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q4, #1 + veor q10, q10, q7 + veor q11, q11, q5 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #1 + veor q5, q5, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q3 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q3, q3, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q5, #2 + vshr.u64 q11, q4, #2 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q7, q7, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q5, q5, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q3 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q3, q3, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q3, #4 + vshr.u64 q11, q2, #4 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q6, q6, q11 + vshl.u64 q11, q11, #4 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q5 + veor q11, q11, q4 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q4, q4, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + sub r5,r5,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: + vldmia r4!, {q8-q11} + veor q8, q8, q0 + veor q9, q9, q1 + vtbl.8 d0, {q8}, d24 + vtbl.8 d1, {q8}, d25 + vldmia r4!, {q8} + veor q10, q10, q2 + vtbl.8 d2, {q9}, d24 + vtbl.8 d3, {q9}, d25 + vldmia r4!, {q9} + veor q11, q11, q3 + vtbl.8 d4, {q10}, d24 + vtbl.8 d5, {q10}, d25 + vldmia r4!, {q10} + vtbl.8 d6, {q11}, d24 + vtbl.8 d7, {q11}, d25 + vldmia r4!, {q11} + veor q8, q8, q4 + veor q9, q9, q5 + vtbl.8 d8, {q8}, d24 + vtbl.8 d9, {q8}, d25 + veor q10, q10, q6 + vtbl.8 d10, {q9}, d24 + vtbl.8 d11, {q9}, d25 + veor q11, q11, q7 + vtbl.8 d12, {q10}, d24 + vtbl.8 d13, {q10}, d25 + vtbl.8 d14, {q11}, d24 + vtbl.8 d15, {q11}, d25 +.Ldec_sbox: + veor q1, q1, q4 + veor q3, q3, q4 + + veor q4, q4, q7 + veor q1, q1, q6 + veor q2, q2, q7 + veor q6, q6, q4 + + veor q0, q0, q1 + veor q2, q2, q5 + veor q7, q7, q6 + veor q3, q3, q0 + veor q5, q5, q0 + veor q1, q1, q3 + veor q11, q3, q0 + veor q10, q7, q4 + veor q9, q1, q6 + veor q13, q4, q0 + vmov q8, q10 + veor q12, q5, q2 + + vorr q10, q10, q9 + veor q15, q11, q8 + vand q14, q11, q12 + vorr q11, q11, q12 + veor q12, q12, q9 + vand q8, q8, q9 + veor q9, q6, q2 + vand q15, q15, q12 + vand q13, q13, q9 + veor q9, q3, q7 + veor q12, q1, q5 + veor q11, q11, q13 + veor q10, q10, q13 + vand q13, q9, q12 + vorr q9, q9, q12 + veor q11, q11, q15 + veor q8, q8, q13 + veor q10, q10, q14 + veor q9, q9, q15 + veor q8, q8, q14 + vand q12, q4, q6 + veor q9, q9, q14 + vand q13, q0, q2 + vand q14, q7, q1 + vorr q15, q3, q5 + veor q11, q11, q12 + veor q9, q9, q14 + veor q8, q8, q15 + veor q10, q10, q13 + + @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3 + + @ new smaller inversion + + vand q14, q11, q9 + vmov q12, q8 + + veor q13, q10, q14 + veor q15, q8, q14 + veor q14, q8, q14 @ q14=q15 + + vbsl q13, q9, q8 + vbsl q15, q11, q10 + veor q11, q11, q10 + + vbsl q12, q13, q14 + vbsl q8, q14, q13 + + vand q14, q12, q15 + veor q9, q9, q8 + + veor q14, q14, q11 + veor q12, q5, q2 + veor q8, q1, q6 + veor q10, q15, q14 + vand q10, q10, q5 + veor q5, q5, q1 + vand q11, q1, q15 + vand q5, q5, q14 + veor q1, q11, q10 + veor q5, q5, q11 + veor q15, q15, q13 + veor q14, q14, q9 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q2 + veor q12, q12, q8 + veor q2, q2, q6 + vand q8, q8, q15 + vand q6, q6, q13 + vand q12, q12, q14 + vand q2, q2, q9 + veor q8, q8, q12 + veor q2, q2, q6 + veor q12, q12, q11 + veor q6, q6, q10 + veor q5, q5, q12 + veor q2, q2, q12 + veor q1, q1, q8 + veor q6, q6, q8 + + veor q12, q3, q0 + veor q8, q7, q4 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q0 + veor q12, q12, q8 + veor q0, q0, q4 + vand q8, q8, q15 + vand q4, q4, q13 + vand q12, q12, q14 + vand q0, q0, q9 + veor q8, q8, q12 + veor q0, q0, q4 + veor q12, q12, q11 + veor q4, q4, q10 + veor q15, q15, q13 + veor q14, q14, q9 + veor q10, q15, q14 + vand q10, q10, q3 + veor q3, q3, q7 + vand q11, q7, q15 + vand q3, q3, q14 + veor q7, q11, q10 + veor q3, q3, q11 + veor q3, q3, q12 + veor q0, q0, q12 + veor q7, q7, q8 + veor q4, q4, q8 + veor q1, q1, q7 + veor q6, q6, q5 + + veor q4, q4, q1 + veor q2, q2, q7 + veor q5, q5, q7 + veor q4, q4, q2 + veor q7, q7, q0 + veor q4, q4, q5 + veor q3, q3, q6 + veor q6, q6, q1 + veor q3, q3, q4 + + veor q4, q4, q0 + veor q7, q7, q3 + subs r5,r5,#1 + bcc .Ldec_done + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 q8, q0, q0, #8 + vext.8 q14, q3, q3, #8 + vext.8 q15, q5, q5, #8 + veor q8, q8, q0 + vext.8 q9, q1, q1, #8 + veor q14, q14, q3 + vext.8 q10, q6, q6, #8 + veor q15, q15, q5 + vext.8 q11, q4, q4, #8 + veor q9, q9, q1 + vext.8 q12, q2, q2, #8 + veor q10, q10, q6 + vext.8 q13, q7, q7, #8 + veor q11, q11, q4 + veor q12, q12, q2 + veor q13, q13, q7 + + veor q0, q0, q14 + veor q1, q1, q14 + veor q6, q6, q8 + veor q2, q2, q10 + veor q4, q4, q9 + veor q1, q1, q15 + veor q6, q6, q15 + veor q2, q2, q14 + veor q7, q7, q11 + veor q4, q4, q14 + veor q3, q3, q12 + veor q2, q2, q15 + veor q7, q7, q15 + veor q5, q5, q13 + vext.8 q8, q0, q0, #12 @ x0 <<< 32 + vext.8 q9, q1, q1, #12 + veor q0, q0, q8 @ x0 ^ (x0 <<< 32) + vext.8 q10, q6, q6, #12 + veor q1, q1, q9 + vext.8 q11, q4, q4, #12 + veor q6, q6, q10 + vext.8 q12, q2, q2, #12 + veor q4, q4, q11 + vext.8 q13, q7, q7, #12 + veor q2, q2, q12 + vext.8 q14, q3, q3, #12 + veor q7, q7, q13 + vext.8 q15, q5, q5, #12 + veor q3, q3, q14 + + veor q9, q9, q0 + veor q5, q5, q15 + vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor q10, q10, q1 + veor q8, q8, q5 + veor q9, q9, q5 + vext.8 q1, q1, q1, #8 + veor q13, q13, q2 + veor q0, q0, q8 + veor q14, q14, q7 + veor q1, q1, q9 + vext.8 q8, q2, q2, #8 + veor q12, q12, q4 + vext.8 q9, q7, q7, #8 + veor q15, q15, q3 + vext.8 q2, q4, q4, #8 + veor q11, q11, q6 + vext.8 q7, q5, q5, #8 + veor q12, q12, q5 + vext.8 q4, q3, q3, #8 + veor q11, q11, q5 + vext.8 q3, q6, q6, #8 + veor q5, q9, q13 + veor q11, q11, q2 + veor q7, q7, q15 + veor q6, q4, q14 + veor q4, q8, q12 + veor q2, q3, q10 + vmov q3, q11 + @ vmov q5, q9 + vldmia r6, {q12} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq r6,r6,#0x10 + bne .Ldec_loop + vldmia r6, {q12} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q3, #1 + vshr.u64 q11, q2, #1 + veor q10, q10, q5 + veor q11, q11, q7 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #1 + veor q7, q7, q11 + vshl.u64 q11, q11, #1 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q4 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q4, q4, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q7, #2 + vshr.u64 q11, q2, #2 + veor q10, q10, q5 + veor q11, q11, q3 + vand q10, q10, q9 + vand q11, q11, q9 + veor q5, q5, q10 + vshl.u64 q10, q10, #2 + veor q3, q3, q11 + vshl.u64 q11, q11, #2 + veor q7, q7, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q4 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q4, q4, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q4, #4 + vshr.u64 q11, q6, #4 + veor q10, q10, q5 + veor q11, q11, q3 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q3, q3, q11 + vshl.u64 q11, q11, #4 + veor q4, q4, q10 + veor q6, q6, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q7 + veor q11, q11, q2 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q2, q2, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + vldmia r4, {q8} @ last round key + veor q6, q6, q8 + veor q4, q4, q8 + veor q2, q2, q8 + veor q7, q7, q8 + veor q3, q3, q8 + veor q5, q5, q8 + veor q0, q0, q8 + veor q1, q1, q8 + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr r6,_bsaes_encrypt8 + vldmia r4!, {q9} @ round 0 key + sub r6,r6,#_bsaes_encrypt8-.LM0SR + + vldmia r6!, {q8} @ .LM0SR +_bsaes_encrypt8_alt: + veor q10, q0, q9 @ xor with round0 key + veor q11, q1, q9 + vtbl.8 d0, {q10}, d16 + vtbl.8 d1, {q10}, d17 + veor q12, q2, q9 + vtbl.8 d2, {q11}, d16 + vtbl.8 d3, {q11}, d17 + veor q13, q3, q9 + vtbl.8 d4, {q12}, d16 + vtbl.8 d5, {q12}, d17 + veor q14, q4, q9 + vtbl.8 d6, {q13}, d16 + vtbl.8 d7, {q13}, d17 + veor q15, q5, q9 + vtbl.8 d8, {q14}, d16 + vtbl.8 d9, {q14}, d17 + veor q10, q6, q9 + vtbl.8 d10, {q15}, d16 + vtbl.8 d11, {q15}, d17 + veor q11, q7, q9 + vtbl.8 d12, {q10}, d16 + vtbl.8 d13, {q10}, d17 + vtbl.8 d14, {q11}, d16 + vtbl.8 d15, {q11}, d17 +_bsaes_encrypt8_bitslice: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q4, #1 + veor q10, q10, q7 + veor q11, q11, q5 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #1 + veor q5, q5, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q3 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q3, q3, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q5, #2 + vshr.u64 q11, q4, #2 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q7, q7, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q5, q5, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q3 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q3, q3, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q3, #4 + vshr.u64 q11, q2, #4 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q6, q6, q11 + vshl.u64 q11, q11, #4 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q5 + veor q11, q11, q4 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q4, q4, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + sub r5,r5,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: + vldmia r4!, {q8-q11} + veor q8, q8, q0 + veor q9, q9, q1 + vtbl.8 d0, {q8}, d24 + vtbl.8 d1, {q8}, d25 + vldmia r4!, {q8} + veor q10, q10, q2 + vtbl.8 d2, {q9}, d24 + vtbl.8 d3, {q9}, d25 + vldmia r4!, {q9} + veor q11, q11, q3 + vtbl.8 d4, {q10}, d24 + vtbl.8 d5, {q10}, d25 + vldmia r4!, {q10} + vtbl.8 d6, {q11}, d24 + vtbl.8 d7, {q11}, d25 + vldmia r4!, {q11} + veor q8, q8, q4 + veor q9, q9, q5 + vtbl.8 d8, {q8}, d24 + vtbl.8 d9, {q8}, d25 + veor q10, q10, q6 + vtbl.8 d10, {q9}, d24 + vtbl.8 d11, {q9}, d25 + veor q11, q11, q7 + vtbl.8 d12, {q10}, d24 + vtbl.8 d13, {q10}, d25 + vtbl.8 d14, {q11}, d24 + vtbl.8 d15, {q11}, d25 +.Lenc_sbox: + veor q2, q2, q1 + veor q5, q5, q6 + veor q3, q3, q0 + veor q6, q6, q2 + veor q5, q5, q0 + + veor q6, q6, q3 + veor q3, q3, q7 + veor q7, q7, q5 + veor q3, q3, q4 + veor q4, q4, q5 + + veor q2, q2, q7 + veor q3, q3, q1 + veor q1, q1, q5 + veor q11, q7, q4 + veor q10, q1, q2 + veor q9, q5, q3 + veor q13, q2, q4 + vmov q8, q10 + veor q12, q6, q0 + + vorr q10, q10, q9 + veor q15, q11, q8 + vand q14, q11, q12 + vorr q11, q11, q12 + veor q12, q12, q9 + vand q8, q8, q9 + veor q9, q3, q0 + vand q15, q15, q12 + vand q13, q13, q9 + veor q9, q7, q1 + veor q12, q5, q6 + veor q11, q11, q13 + veor q10, q10, q13 + vand q13, q9, q12 + vorr q9, q9, q12 + veor q11, q11, q15 + veor q8, q8, q13 + veor q10, q10, q14 + veor q9, q9, q15 + veor q8, q8, q14 + vand q12, q2, q3 + veor q9, q9, q14 + vand q13, q4, q0 + vand q14, q1, q5 + vorr q15, q7, q6 + veor q11, q11, q12 + veor q9, q9, q14 + veor q8, q8, q15 + veor q10, q10, q13 + + @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3 + + @ new smaller inversion + + vand q14, q11, q9 + vmov q12, q8 + + veor q13, q10, q14 + veor q15, q8, q14 + veor q14, q8, q14 @ q14=q15 + + vbsl q13, q9, q8 + vbsl q15, q11, q10 + veor q11, q11, q10 + + vbsl q12, q13, q14 + vbsl q8, q14, q13 + + vand q14, q12, q15 + veor q9, q9, q8 + + veor q14, q14, q11 + veor q12, q6, q0 + veor q8, q5, q3 + veor q10, q15, q14 + vand q10, q10, q6 + veor q6, q6, q5 + vand q11, q5, q15 + vand q6, q6, q14 + veor q5, q11, q10 + veor q6, q6, q11 + veor q15, q15, q13 + veor q14, q14, q9 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q0 + veor q12, q12, q8 + veor q0, q0, q3 + vand q8, q8, q15 + vand q3, q3, q13 + vand q12, q12, q14 + vand q0, q0, q9 + veor q8, q8, q12 + veor q0, q0, q3 + veor q12, q12, q11 + veor q3, q3, q10 + veor q6, q6, q12 + veor q0, q0, q12 + veor q5, q5, q8 + veor q3, q3, q8 + + veor q12, q7, q4 + veor q8, q1, q2 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q4 + veor q12, q12, q8 + veor q4, q4, q2 + vand q8, q8, q15 + vand q2, q2, q13 + vand q12, q12, q14 + vand q4, q4, q9 + veor q8, q8, q12 + veor q4, q4, q2 + veor q12, q12, q11 + veor q2, q2, q10 + veor q15, q15, q13 + veor q14, q14, q9 + veor q10, q15, q14 + vand q10, q10, q7 + veor q7, q7, q1 + vand q11, q1, q15 + vand q7, q7, q14 + veor q1, q11, q10 + veor q7, q7, q11 + veor q7, q7, q12 + veor q4, q4, q12 + veor q1, q1, q8 + veor q2, q2, q8 + veor q7, q7, q0 + veor q1, q1, q6 + veor q6, q6, q0 + veor q4, q4, q7 + veor q0, q0, q1 + + veor q1, q1, q5 + veor q5, q5, q2 + veor q2, q2, q3 + veor q3, q3, q5 + veor q4, q4, q5 + + veor q6, q6, q3 + subs r5,r5,#1 + bcc .Lenc_done + vext.8 q8, q0, q0, #12 @ x0 <<< 32 + vext.8 q9, q1, q1, #12 + veor q0, q0, q8 @ x0 ^ (x0 <<< 32) + vext.8 q10, q4, q4, #12 + veor q1, q1, q9 + vext.8 q11, q6, q6, #12 + veor q4, q4, q10 + vext.8 q12, q3, q3, #12 + veor q6, q6, q11 + vext.8 q13, q7, q7, #12 + veor q3, q3, q12 + vext.8 q14, q2, q2, #12 + veor q7, q7, q13 + vext.8 q15, q5, q5, #12 + veor q2, q2, q14 + + veor q9, q9, q0 + veor q5, q5, q15 + vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor q10, q10, q1 + veor q8, q8, q5 + veor q9, q9, q5 + vext.8 q1, q1, q1, #8 + veor q13, q13, q3 + veor q0, q0, q8 + veor q14, q14, q7 + veor q1, q1, q9 + vext.8 q8, q3, q3, #8 + veor q12, q12, q6 + vext.8 q9, q7, q7, #8 + veor q15, q15, q2 + vext.8 q3, q6, q6, #8 + veor q11, q11, q4 + vext.8 q7, q5, q5, #8 + veor q12, q12, q5 + vext.8 q6, q2, q2, #8 + veor q11, q11, q5 + vext.8 q2, q4, q4, #8 + veor q5, q9, q13 + veor q4, q8, q12 + veor q3, q3, q11 + veor q7, q7, q15 + veor q6, q6, q14 + @ vmov q4, q8 + veor q2, q2, q10 + @ vmov q5, q9 + vldmia r6, {q12} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq r6,r6,#0x10 + bne .Lenc_loop + vldmia r6, {q12} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q3, #1 + veor q10, q10, q5 + veor q11, q11, q7 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #1 + veor q7, q7, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q3, q3, q11 + vshr.u64 q10, q4, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q6 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q6, q6, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q4, q4, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q7, #2 + vshr.u64 q11, q3, #2 + veor q10, q10, q5 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q5, q5, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q7, q7, q10 + veor q3, q3, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q6 + veor q11, q11, q4 + vand q10, q10, q9 + vand q11, q11, q9 + veor q6, q6, q10 + vshl.u64 q10, q10, #2 + veor q4, q4, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q6, #4 + vshr.u64 q11, q4, #4 + veor q10, q10, q5 + veor q11, q11, q2 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q2, q2, q11 + vshl.u64 q11, q11, #4 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q7 + veor q11, q11, q3 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q3, q3, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + vldmia r4, {q8} @ last round key + veor q4, q4, q8 + veor q6, q6, q8 + veor q3, q3, q8 + veor q7, q7, q8 + veor q2, q2, q8 + veor q5, q5, q8 + veor q0, q0, q8 + veor q1, q1, q8 + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr r6,_bsaes_key_convert + vld1.8 {q7}, [r4]! @ load round 0 key + sub r6,r6,#_bsaes_key_convert-.LM0 + vld1.8 {q15}, [r4]! @ load round 1 key + + vmov.i8 q8, #0x01 @ bit masks + vmov.i8 q9, #0x02 + vmov.i8 q10, #0x04 + vmov.i8 q11, #0x08 + vmov.i8 q12, #0x10 + vmov.i8 q13, #0x20 + vldmia r6, {q14} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 q7, q7 + vrev32.8 q15, q15 +#endif + sub r5,r5,#1 + vstmia r12!, {q7} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 d14,{q15},d28 + vtbl.8 d15,{q15},d29 + vmov.i8 q6, #0x40 + vmov.i8 q15, #0x80 + + vtst.8 q0, q7, q8 + vtst.8 q1, q7, q9 + vtst.8 q2, q7, q10 + vtst.8 q3, q7, q11 + vtst.8 q4, q7, q12 + vtst.8 q5, q7, q13 + vtst.8 q6, q7, q6 + vtst.8 q7, q7, q15 + vld1.8 {q15}, [r4]! @ load next round key + vmvn q0, q0 @ "pnot" + vmvn q1, q1 + vmvn q5, q5 + vmvn q6, q6 +#ifdef __ARMEL__ + vrev32.8 q15, q15 +#endif + subs r5,r5,#1 + vstmia r12!,{q0-q7} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 q7,#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp r2, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr r8, [ip] @ IV is 1st arg on the stack + mov r2, r2, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov r9, sp @ save sp + + ldr r10, [r3, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key + add r12, #96 @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + mov sp, r12 @ sp is sp + bl _bsaes_key_convert + vldmia sp, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia sp, {q7} +#else + ldr r12, [r3, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [r3, #244] + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + add r12, r3, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, r3, #248 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} + +.align 2 +0: +#endif + + vld1.8 {q15}, [r8] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs r2, r2, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {q0-q1}, [r0]! @ load input + vld1.8 {q2-q3}, [r0]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, sp @ pass the key +#else + add r4, r3, #248 +#endif + vld1.8 {q4-q5}, [r0]! + mov r5, r10 + vld1.8 {q6-q7}, [r0] + sub r0, r0, #0x60 + vstmia r9, {q15} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q14-q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q3, q3, q13 + vst1.8 {q6}, [r1]! + veor q5, q5, q14 + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + vst1.8 {q3}, [r1]! + vst1.8 {q5}, [r1]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds r2, r2, #8 + beq .Lcbc_dec_done + + vld1.8 {q0}, [r0]! @ load input + cmp r2, #2 + blo .Lcbc_dec_one + vld1.8 {q1}, [r0]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, sp @ pass the key +#else + add r4, r3, #248 +#endif + mov r5, r10 + vstmia r9, {q15} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {q2}, [r0]! + cmp r2, #4 + blo .Lcbc_dec_three + vld1.8 {q3}, [r0]! + beq .Lcbc_dec_four + vld1.8 {q4}, [r0]! + cmp r2, #6 + blo .Lcbc_dec_five + vld1.8 {q5}, [r0]! + beq .Lcbc_dec_six + vld1.8 {q6}, [r0]! + sub r0, r0, #0x70 + + bl _bsaes_decrypt8 + + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q3, q3, q13 + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + vst1.8 {q3}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub r0, r0, #0x60 + bl _bsaes_decrypt8 + vldmia r9,{q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub r0, r0, #0x50 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q15}, [r0]! + veor q4, q4, q10 + vst1.8 {q0-q1}, [r1]! @ write output + veor q2, q2, q11 + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub r0, r0, #0x40 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q15}, [r0]! + veor q4, q4, q10 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub r0, r0, #0x30 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q15}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub r0, r0, #0x20 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q15}, [r0]! @ reload input + veor q1, q1, q8 + vst1.8 {q0-q1}, [r1]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub r0, r0, #0x10 + mov r10, r1 @ save original out pointer + mov r1, r9 @ use the iv scratch space as out buffer + mov r2, r3 + vmov q4,q15 @ just in case ensure that IV + vmov q5,q0 @ and input are preserved + bl AES_decrypt + vld1.8 {q0}, [r9,:64] @ load result + veor q0, q0, q4 @ ^= IV + vmov q15, q5 @ q5 holds input + vst1.8 {q0}, [r10] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r9 + bne .Lcbc_dec_bzero +#endif + + mov sp, r9 + add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb + vst1.8 {q15}, [r8] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp r2, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr r8, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov r9, sp @ save sp + + ldr r10, [r3, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key + add r12, #96 @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + mov sp, r12 @ sp is sp + bl _bsaes_key_convert + veor q7,q7,q15 @ fix up last round key + vstmia r12, {q7} @ save last round key + + vld1.8 {q0}, [r8] @ load counter + add r8, r6, #.LREVM0SR-.LM0 @ borrow r8 + vldmia sp, {q4} @ load round0 key +#else + ldr r12, [r3, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [r3, #244] + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + add r12, r3, #248 @ pass key schedule + bl _bsaes_key_convert + veor q7,q7,q15 @ fix up last round key + vstmia r12, {q7} @ save last round key + +.align 2 +0: add r12, r3, #248 + vld1.8 {q0}, [r8] @ load counter + adrl r8, .LREVM0SR @ borrow r8 + vldmia r12, {q4} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 q8,#1 @ compose 1<<96 + veor q9,q9,q9 + vrev32.8 q0,q0 + vext.8 q8,q9,q8,#4 + vrev32.8 q4,q4 + vadd.u32 q9,q8,q8 @ compose 2<<96 + vstmia sp, {q4} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 q10, q8, q9 @ compose 3<<96 + vadd.u32 q1, q0, q8 @ +1 + vadd.u32 q2, q0, q9 @ +2 + vadd.u32 q3, q0, q10 @ +3 + vadd.u32 q4, q1, q10 + vadd.u32 q5, q2, q10 + vadd.u32 q6, q3, q10 + vadd.u32 q7, q4, q10 + vadd.u32 q10, q5, q10 @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia sp, {q9} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x10 @ pass next round key +#else + add r4, r3, #264 +#endif + vldmia r8, {q8} @ .LREVM0SR + mov r5, r10 @ pass rounds + vstmia r9, {q10} @ save next counter + sub r6, r8, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs r2, r2, #8 + blo .Lctr_enc_loop_done + + vld1.8 {q8-q9}, [r0]! @ load input + vld1.8 {q10-q11}, [r0]! + veor q0, q8 + veor q1, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q10 + veor q6, q11 + vld1.8 {q14-q15}, [r0]! + veor q3, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q7, q13 + veor q2, q14 + vst1.8 {q4}, [r1]! + veor q5, q15 + vst1.8 {q6}, [r1]! + vmov.i32 q8, #1 @ compose 1<<96 + vst1.8 {q3}, [r1]! + veor q9, q9, q9 + vst1.8 {q7}, [r1]! + vext.8 q8, q9, q8, #4 + vst1.8 {q2}, [r1]! + vadd.u32 q9,q8,q8 @ compose 2<<96 + vst1.8 {q5}, [r1]! + vldmia r9, {q0} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add r2, r2, #8 + vld1.8 {q8}, [r0]! @ load input + veor q0, q8 + vst1.8 {q0}, [r1]! @ write output + cmp r2, #2 + blo .Lctr_enc_done + vld1.8 {q9}, [r0]! + veor q1, q9 + vst1.8 {q1}, [r1]! + beq .Lctr_enc_done + vld1.8 {q10}, [r0]! + veor q4, q10 + vst1.8 {q4}, [r1]! + cmp r2, #4 + blo .Lctr_enc_done + vld1.8 {q11}, [r0]! + veor q6, q11 + vst1.8 {q6}, [r1]! + beq .Lctr_enc_done + vld1.8 {q12}, [r0]! + veor q3, q12 + vst1.8 {q3}, [r1]! + cmp r2, #6 + blo .Lctr_enc_done + vld1.8 {q13}, [r0]! + veor q7, q13 + vst1.8 {q7}, [r1]! + beq .Lctr_enc_done + vld1.8 {q14}, [r0] + veor q2, q14 + vst1.8 {q2}, [r1]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r9 + bne .Lctr_enc_bzero +#else + vstmia sp, {q0-q1} +#endif + + mov sp, r9 + add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, r0 @ copy arguments + mov r5, r1 + mov r6, r2 + mov r7, r3 + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {q1}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {q1}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {q0}, [r4]! @ load input + vld1.8 {q1}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor q0,q0,q1 + vst1.8 {q0}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future r3 + + mov r7, r0 + mov r8, r1 + mov r9, r2 + mov r10, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr r1, [r10, #240] @ get # of rounds + mov r3, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key + @ add r12, #96 @ size of bit-sliced key schedule + sub r12, #48 @ place for tweak[9] + + @ populate the key schedule + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor q7, q7, q15 @ fix up last round key + vstmia r12, {q7} @ save last round key +#else + ldr r12, [r10, #244] + eors r12, #1 + beq 0f + + str r12, [r10, #244] + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + add r12, r10, #248 @ pass key schedule + bl _bsaes_key_convert + veor q7, q7, q15 @ fix up last round key + vstmia r12, {q7} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {q8}, [r0] @ initial tweak + adr r2, .Lxts_magic + + subs r9, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia r2, {q5} @ load XTS magic + vshr.s64 q6, q8, #63 + mov r0, sp + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q9, #63 + veor q9, q9, q6 + vand q7, q7, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q10, #63 + veor q10, q10, q7 + vand q6, q6, q5 + vld1.8 {q0}, [r7]! + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q11, #63 + veor q11, q11, q6 + vand q7, q7, q5 + vld1.8 {q1}, [r7]! + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q12, #63 + veor q12, q12, q7 + vand q6, q6, q5 + vld1.8 {q2}, [r7]! + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q13, #63 + veor q13, q13, q6 + vand q7, q7, q5 + vld1.8 {q3}, [r7]! + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q14, #63 + veor q14, q14, q7 + vand q6, q6, q5 + vld1.8 {q4}, [r7]! + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q15, #63 + veor q15, q15, q6 + vand q7, q7, q5 + vld1.8 {q5}, [r7]! + veor q4, q4, q12 + vadd.u64 q8, q15, q15 + vst1.64 {q15}, [r0,:128]! + vswp d15,d14 + veor q8, q8, q7 + vst1.64 {q8}, [r0,:128] @ next round tweak + + vld1.8 {q6-q7}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + veor q7, q7, q15 + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vld1.64 {q14-q15}, [r0,:128]! + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q2, q14 + vst1.8 {q10-q11}, [r8]! + veor q13, q5, q15 + vst1.8 {q12-q13}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + + subs r9, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds r9, #0x70 + bmi .Lxts_enc_done + + vldmia r2, {q5} @ load XTS magic + vshr.s64 q7, q8, #63 + mov r0, sp + vand q7, q7, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q9, #63 + veor q9, q9, q7 + vand q6, q6, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q10, #63 + veor q10, q10, q6 + vand q7, q7, q5 + vld1.8 {q0}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_1 + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q11, #63 + veor q11, q11, q7 + vand q6, q6, q5 + vld1.8 {q1}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_2 + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q12, #63 + veor q12, q12, q6 + vand q7, q7, q5 + vld1.8 {q2}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_3 + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q13, #63 + veor q13, q13, q7 + vand q6, q6, q5 + vld1.8 {q3}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_4 + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q14, #63 + veor q14, q14, q6 + vand q7, q7, q5 + vld1.8 {q4}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_5 + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q15, #63 + veor q15, q15, q7 + vand q6, q6, q5 + vld1.8 {q5}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_6 + veor q4, q4, q12 + sub r9, #0x10 + vst1.64 {q15}, [r0,:128] @ next round tweak + + vld1.8 {q6}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vld1.64 {q14}, [r0,:128]! + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q2, q14 + vst1.8 {q10-q11}, [r8]! + vst1.8 {q12}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {q14}, [r0,:128] @ next round tweak + + veor q4, q4, q12 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q5, q5, q13 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + vst1.8 {q10-q11}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {q13}, [r0,:128] @ next round tweak + + veor q3, q3, q11 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q4, q4, q12 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + vst1.8 {q10}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {q12}, [r0,:128] @ next round tweak + + veor q2, q2, q10 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q3, q3, q11 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vst1.8 {q8-q9}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {q11}, [r0,:128] @ next round tweak + + veor q1, q1, q9 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q2, q2, q10 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + vst1.8 {q8}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {q10}, [r0,:128] @ next round tweak + + veor q0, q0, q8 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q1, q1, q9 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + vst1.8 {q0-q1}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_encrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r8]! + mov r3, r4 + + vmov q8, q9 @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds r9, #0x10 + beq .Lxts_enc_ret + sub r6, r8, #0x10 + +.Lxts_enc_steal: + ldrb r0, [r7], #1 + ldrb r1, [r8, #-0x10] + strb r0, [r8, #-0x10] + strb r1, [r8], #1 + + subs r9, #1 + bhi .Lxts_enc_steal + + vld1.8 {q0}, [r6] + mov r0, sp + veor q0, q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_encrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r6] + mov r3, r4 +#endif + +.Lxts_enc_ret: + bic r0, r3, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, r3 +#ifdef XTS_CHAIN_TWEAK + vst1.8 {q8}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future r3 + + mov r7, r0 + mov r8, r1 + mov r9, r2 + mov r10, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr r1, [r10, #240] @ get # of rounds + mov r3, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key + @ add r12, #96 @ size of bit-sliced key schedule + sub r12, #48 @ place for tweak[9] + + @ populate the key schedule + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} +#else + ldr r12, [r10, #244] + eors r12, #1 + beq 0f + + str r12, [r10, #244] + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + add r12, r10, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, r10, #248 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {q8}, [r0] @ initial tweak + adr r2, .Lxts_magic + + tst r9, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne r9, #0x10 @ subtract another 16 bytes + subs r9, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia r2, {q5} @ load XTS magic + vshr.s64 q6, q8, #63 + mov r0, sp + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q9, #63 + veor q9, q9, q6 + vand q7, q7, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q10, #63 + veor q10, q10, q7 + vand q6, q6, q5 + vld1.8 {q0}, [r7]! + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q11, #63 + veor q11, q11, q6 + vand q7, q7, q5 + vld1.8 {q1}, [r7]! + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q12, #63 + veor q12, q12, q7 + vand q6, q6, q5 + vld1.8 {q2}, [r7]! + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q13, #63 + veor q13, q13, q6 + vand q7, q7, q5 + vld1.8 {q3}, [r7]! + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q14, #63 + veor q14, q14, q7 + vand q6, q6, q5 + vld1.8 {q4}, [r7]! + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q15, #63 + veor q15, q15, q6 + vand q7, q7, q5 + vld1.8 {q5}, [r7]! + veor q4, q4, q12 + vadd.u64 q8, q15, q15 + vst1.64 {q15}, [r0,:128]! + vswp d15,d14 + veor q8, q8, q7 + vst1.64 {q8}, [r0,:128] @ next round tweak + + vld1.8 {q6-q7}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + veor q7, q7, q15 + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vld1.64 {q14-q15}, [r0,:128]! + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q3, q14 + vst1.8 {q10-q11}, [r8]! + veor q13, q5, q15 + vst1.8 {q12-q13}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + + subs r9, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds r9, #0x70 + bmi .Lxts_dec_done + + vldmia r2, {q5} @ load XTS magic + vshr.s64 q7, q8, #63 + mov r0, sp + vand q7, q7, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q9, #63 + veor q9, q9, q7 + vand q6, q6, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q10, #63 + veor q10, q10, q6 + vand q7, q7, q5 + vld1.8 {q0}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_1 + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q11, #63 + veor q11, q11, q7 + vand q6, q6, q5 + vld1.8 {q1}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_2 + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q12, #63 + veor q12, q12, q6 + vand q7, q7, q5 + vld1.8 {q2}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_3 + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q13, #63 + veor q13, q13, q7 + vand q6, q6, q5 + vld1.8 {q3}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_4 + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q14, #63 + veor q14, q14, q6 + vand q7, q7, q5 + vld1.8 {q4}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_5 + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q15, #63 + veor q15, q15, q7 + vand q6, q6, q5 + vld1.8 {q5}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_6 + veor q4, q4, q12 + sub r9, #0x10 + vst1.64 {q15}, [r0,:128] @ next round tweak + + vld1.8 {q6}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vld1.64 {q14}, [r0,:128]! + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q3, q14 + vst1.8 {q10-q11}, [r8]! + vst1.8 {q12}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {q14}, [r0,:128] @ next round tweak + + veor q4, q4, q12 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q5, q5, q13 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + vst1.8 {q10-q11}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {q13}, [r0,:128] @ next round tweak + + veor q3, q3, q11 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q4, q4, q12 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + vst1.8 {q10}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {q12}, [r0,:128] @ next round tweak + + veor q2, q2, q10 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q3, q3, q11 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vst1.8 {q8-q9}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {q11}, [r0,:128] @ next round tweak + + veor q1, q1, q9 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q2, q2, q10 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + vst1.8 {q8}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {q10}, [r0,:128] @ next round tweak + + veor q0, q0, q8 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q1, q1, q9 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + vst1.8 {q0-q1}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + mov r5, r2 @ preserve magic + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r8]! + mov r3, r4 + mov r2, r5 + + vmov q8, q9 @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds r9, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia r2, {q5} + vshr.s64 q6, q8, #63 + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vswp d13,d12 + veor q9, q9, q6 + + @ perform the final decryption with the last tweak value + vld1.8 {q0}, [r7]! + mov r0, sp + veor q0, q0, q9 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q9 + vst1.8 {q0}, [r8] + + mov r6, r8 +.Lxts_dec_steal: + ldrb r1, [r8] + ldrb r0, [r7], #1 + strb r1, [r8, #0x10] + strb r0, [r8], #1 + + subs r9, #1 + bhi .Lxts_dec_steal + + vld1.8 {q0}, [r6] + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r6] + mov r3, r4 +#endif + +.Lxts_dec_ret: + bic r0, r3, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, r3 +#ifdef XTS_CHAIN_TWEAK + vst1.8 {q8}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +#endif diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c new file mode 100644 index 00000000000..15468fbbdea --- /dev/null +++ b/arch/arm/crypto/aesbs-glue.c @@ -0,0 +1,434 @@ +/* + * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES + * + * Copyright (C) 2013 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include + +#include "aes_glue.h" + +#define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE) + +struct BS_KEY { + struct AES_KEY rk; + int converted; + u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE]; +} __aligned(8); + +asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in); +asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in); + +asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 iv[]); + +asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks, + struct BS_KEY *key, u8 const iv[]); + +asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 tweak[]); + +asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 tweak[]); + +struct aesbs_cbc_ctx { + struct AES_KEY enc; + struct BS_KEY dec; +}; + +struct aesbs_ctr_ctx { + struct BS_KEY enc; +}; + +struct aesbs_xts_ctx { + struct BS_KEY enc; + struct BS_KEY dec; + struct AES_KEY twkey; +}; + +static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 8; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->dec.rk = ctx->enc; + private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk); + ctx->dec.converted = 0; + return 0; +} + +static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 8; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->enc.converted = 0; + return 0; +} + +static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 4; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->dec.rk = ctx->enc.rk; + private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk); + private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey); + ctx->enc.converted = ctx->dec.converted = 0; + return 0; +} + +static int aesbs_cbc_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while (walk.nbytes) { + u32 blocks = walk.nbytes / AES_BLOCK_SIZE; + u8 *src = walk.src.virt.addr; + + if (walk.dst.virt.addr == walk.src.virt.addr) { + u8 *iv = walk.iv; + + do { + crypto_xor(src, iv, AES_BLOCK_SIZE); + AES_encrypt(src, src, &ctx->enc); + iv = src; + src += AES_BLOCK_SIZE; + } while (--blocks); + memcpy(walk.iv, iv, AES_BLOCK_SIZE); + } else { + u8 *dst = walk.dst.virt.addr; + + do { + crypto_xor(walk.iv, src, AES_BLOCK_SIZE); + AES_encrypt(walk.iv, dst, &ctx->enc); + memcpy(walk.iv, dst, AES_BLOCK_SIZE); + src += AES_BLOCK_SIZE; + dst += AES_BLOCK_SIZE; + } while (--blocks); + } + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); + } + return err; +} + +static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) { + kernel_neon_begin(); + bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->dec, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); + } + while (walk.nbytes) { + u32 blocks = walk.nbytes / AES_BLOCK_SIZE; + u8 *dst = walk.dst.virt.addr; + u8 *src = walk.src.virt.addr; + u8 bk[2][AES_BLOCK_SIZE]; + u8 *iv = walk.iv; + + do { + if (walk.dst.virt.addr == walk.src.virt.addr) + memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE); + + AES_decrypt(src, dst, &ctx->dec.rk); + crypto_xor(dst, iv, AES_BLOCK_SIZE); + + if (walk.dst.virt.addr == walk.src.virt.addr) + iv = bk[blocks & 1]; + else + iv = src; + + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } while (--blocks); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); + } + return err; +} + +static void inc_be128_ctr(__be32 ctr[], u32 addend) +{ + int i; + + for (i = 3; i >= 0; i--, addend = 1) { + u32 n = be32_to_cpu(ctr[i]) + addend; + + ctr[i] = cpu_to_be32(n); + if (n >= addend) + break; + } +} + +static int aesbs_ctr_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + u32 blocks; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; + __be32 *ctr = (__be32 *)walk.iv; + u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]); + + /* avoid 32 bit counter overflow in the NEON code */ + if (unlikely(headroom < blocks)) { + blocks = headroom + 1; + tail = walk.nbytes - blocks * AES_BLOCK_SIZE; + } + kernel_neon_begin(); + bsaes_ctr32_encrypt_blocks(walk.src.virt.addr, + walk.dst.virt.addr, blocks, + &ctx->enc, walk.iv); + kernel_neon_end(); + inc_be128_ctr(ctr, blocks); + + nbytes -= blocks * AES_BLOCK_SIZE; + if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE) + break; + + err = blkcipher_walk_done(desc, &walk, tail); + } + if (walk.nbytes) { + u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; + u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; + u8 ks[AES_BLOCK_SIZE]; + + AES_encrypt(walk.iv, ks, &ctx->enc.rk); + if (tdst != tsrc) + memcpy(tdst, tsrc, nbytes); + crypto_xor(tdst, ks, nbytes); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_xts_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + /* generate the initial tweak */ + AES_encrypt(walk.iv, walk.iv, &ctx->twkey); + + while (walk.nbytes) { + kernel_neon_begin(); + bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->enc, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); + } + return err; +} + +static int aesbs_xts_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + /* generate the initial tweak */ + AES_encrypt(walk.iv, walk.iv, &ctx->twkey); + + while (walk.nbytes) { + kernel_neon_begin(); + bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->dec, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); + } + return err; +} + +static struct crypto_alg aesbs_algs[] = { { + .cra_name = "__cbc-aes-neonbs", + .cra_driver_name = "__driver-cbc-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aesbs_cbc_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_cbc_set_key, + .encrypt = aesbs_cbc_encrypt, + .decrypt = aesbs_cbc_decrypt, + }, +}, { + .cra_name = "__ctr-aes-neonbs", + .cra_driver_name = "__driver-ctr-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesbs_ctr_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_ctr_set_key, + .encrypt = aesbs_ctr_encrypt, + .decrypt = aesbs_ctr_encrypt, + }, +}, { + .cra_name = "__xts-aes-neonbs", + .cra_driver_name = "__driver-xts-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aesbs_xts_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_xts_set_key, + .encrypt = aesbs_xts_encrypt, + .decrypt = aesbs_xts_decrypt, + }, +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +} }; + +static int __init aesbs_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs)); +} + +static void __exit aesbs_mod_exit(void) +{ + crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs)); +} + +module_init(aesbs_mod_init); +module_exit(aesbs_mod_exit); + +MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON"); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL"); diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl new file mode 100644 index 00000000000..be068db960e --- /dev/null +++ b/arch/arm/crypto/bsaes-armv7.pl @@ -0,0 +1,2467 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# +# Specific modes and adaptation for Linux kernel by Ard Biesheuvel +# . Permission to use under GPL terms is +# granted. +# ==================================================================== + +# Bit-sliced AES for ARM NEON +# +# February 2012. +# +# This implementation is direct adaptation of bsaes-x86_64 module for +# ARM NEON. Except that this module is endian-neutral [in sense that +# it can be compiled for either endianness] by courtesy of vld1.8's +# neutrality. Initial version doesn't implement interface to OpenSSL, +# only low-level primitives and unsupported entry points, just enough +# to collect performance results, which for Cortex-A8 core are: +# +# encrypt 19.5 cycles per byte processed with 128-bit key +# decrypt 22.1 cycles per byte processed with 128-bit key +# key conv. 440 cycles per 128-bit key/0.18 of 8x block +# +# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +# which is [much] worse than anticipated (for further details see +# http://www.openssl.org/~appro/Snapdragon-S4.html). +# +# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +# manages in 20.0 cycles]. +# +# When comparing to x86_64 results keep in mind that NEON unit is +# [mostly] single-issue and thus can't [fully] benefit from +# instruction-level parallelism. And when comparing to aes-armv4 +# results keep in mind key schedule conversion overhead (see +# bsaes-x86_64.pl for further details)... +# +# + +# April-August 2013 +# +# Add CBC, CTR and XTS subroutines, adapt for kernel use. +# +# + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +my ($inp,$out,$len,$key)=("r0","r1","r2","r3"); +my @XMM=map("q$_",(0..15)); + +{ +my ($key,$rounds,$const)=("r4","r5","r6"); + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +sub Sbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InBasisChange (@b); + &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s); + &OutBasisChange (@b[7,1,4,2,6,5,0,3]); +} + +sub InBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[2], @b[2], @b[1] + veor @b[5], @b[5], @b[6] + veor @b[3], @b[3], @b[0] + veor @b[6], @b[6], @b[2] + veor @b[5], @b[5], @b[0] + + veor @b[6], @b[6], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[4], @b[4], @b[5] + + veor @b[2], @b[2], @b[7] + veor @b[3], @b[3], @b[1] + veor @b[1], @b[1], @b[5] +___ +} + +sub OutBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] + veor @b[4], @b[4], @b[6] + veor @b[2], @b[2], @b[0] + veor @b[6], @b[6], @b[1] + + veor @b[1], @b[1], @b[5] + veor @b[5], @b[5], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[2], @b[2], @b[5] + + veor @b[4], @b[4], @b[7] +___ +} + +sub InvSbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InvInBasisChange (@b); + &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s); + &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]); +} + +sub InvInBasisChange { # OutBasisChange in reverse (with twist) +my @b=@_[5,1,2,6,3,7,0,4]; +$code.=<<___ + veor @b[1], @b[1], @b[7] + veor @b[4], @b[4], @b[7] + + veor @b[7], @b[7], @b[5] + veor @b[1], @b[1], @b[3] + veor @b[2], @b[2], @b[5] + veor @b[3], @b[3], @b[7] + + veor @b[6], @b[6], @b[1] + veor @b[2], @b[2], @b[0] + veor @b[5], @b[5], @b[3] + veor @b[4], @b[4], @b[6] + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] +___ +} + +sub InvOutBasisChange { # InBasisChange in reverse +my @b=@_[2,5,7,3,6,1,0,4]; +$code.=<<___; + veor @b[1], @b[1], @b[5] + veor @b[2], @b[2], @b[7] + + veor @b[3], @b[3], @b[1] + veor @b[4], @b[4], @b[5] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[5], @b[5], @b[0] + veor @b[3], @b[3], @b[7] + veor @b[6], @b[6], @b[2] + veor @b[2], @b[2], @b[1] + veor @b[6], @b[6], @b[3] + + veor @b[3], @b[3], @b[0] + veor @b[5], @b[5], @b[6] +___ +} + +sub Mul_GF4 { +#;************************************************************* +#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) * +#;************************************************************* +my ($x0,$x1,$y0,$y1,$t0,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $t1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $t1, $t0 + veor $x0, $x0, $t1 +___ +} + +sub Mul_GF4_N { # not used, see next subroutine +# multiply and scale by N +my ($x0,$x1,$y0,$y1,$t0)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $x1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $x1, $x0 + veor $x0, $x0, $t0 +___ +} + +sub Mul_GF4_N_GF4 { +# interleaved Mul_GF4_N and Mul_GF4 +my ($x0,$x1,$y0,$y1,$t0, + $x2,$x3,$y2,$y3,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + veor $t1, $y2, $y3 + vand $t0, $t0, $x0 + vand $t1, $t1, $x2 + veor $x0, $x0, $x1 + veor $x2, $x2, $x3 + vand $x1, $x1, $y0 + vand $x3, $x3, $y2 + vand $x0, $x0, $y1 + vand $x2, $x2, $y3 + veor $x1, $x1, $x0 + veor $x2, $x2, $x3 + veor $x0, $x0, $t0 + veor $x3, $x3, $t1 +___ +} +sub Mul_GF16_2 { +my @x=@_[0..7]; +my @y=@_[8..11]; +my @t=@_[12..15]; +$code.=<<___; + veor @t[0], @x[0], @x[2] + veor @t[1], @x[1], @x[3] +___ + &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[2], @x[3], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @x[0], @x[0], @t[0] + veor @x[2], @x[2], @t[0] + veor @x[1], @x[1], @t[1] + veor @x[3], @x[3], @t[1] + + veor @t[0], @x[4], @x[6] + veor @t[1], @x[5], @x[7] +___ + &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[6], @x[7], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @x[4], @x[4], @t[0] + veor @x[6], @x[6], @t[0] + veor @x[5], @x[5], @t[1] + veor @x[7], @x[7], @t[1] +___ +} +sub Inv_GF256 { +#;******************************************************************** +#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) * +#;******************************************************************** +my @x=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; +# direct optimizations from hardware +$code.=<<___; + veor @t[3], @x[4], @x[6] + veor @t[2], @x[5], @x[7] + veor @t[1], @x[1], @x[3] + veor @s[1], @x[7], @x[6] + vmov @t[0], @t[2] + veor @s[0], @x[0], @x[2] + + vorr @t[2], @t[2], @t[1] + veor @s[3], @t[3], @t[0] + vand @s[2], @t[3], @s[0] + vorr @t[3], @t[3], @s[0] + veor @s[0], @s[0], @t[1] + vand @t[0], @t[0], @t[1] + veor @t[1], @x[3], @x[2] + vand @s[3], @s[3], @s[0] + vand @s[1], @s[1], @t[1] + veor @t[1], @x[4], @x[5] + veor @s[0], @x[1], @x[0] + veor @t[3], @t[3], @s[1] + veor @t[2], @t[2], @s[1] + vand @s[1], @t[1], @s[0] + vorr @t[1], @t[1], @s[0] + veor @t[3], @t[3], @s[3] + veor @t[0], @t[0], @s[1] + veor @t[2], @t[2], @s[2] + veor @t[1], @t[1], @s[3] + veor @t[0], @t[0], @s[2] + vand @s[0], @x[7], @x[3] + veor @t[1], @t[1], @s[2] + vand @s[1], @x[6], @x[2] + vand @s[2], @x[5], @x[1] + vorr @s[3], @x[4], @x[0] + veor @t[3], @t[3], @s[0] + veor @t[1], @t[1], @s[2] + veor @t[0], @t[0], @s[3] + veor @t[2], @t[2], @s[1] + + @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3 + + @ new smaller inversion + + vand @s[2], @t[3], @t[1] + vmov @s[0], @t[0] + + veor @s[1], @t[2], @s[2] + veor @s[3], @t[0], @s[2] + veor @s[2], @t[0], @s[2] @ @s[2]=@s[3] + + vbsl @s[1], @t[1], @t[0] + vbsl @s[3], @t[3], @t[2] + veor @t[3], @t[3], @t[2] + + vbsl @s[0], @s[1], @s[2] + vbsl @t[0], @s[2], @s[1] + + vand @s[2], @s[0], @s[3] + veor @t[1], @t[1], @t[0] + + veor @s[2], @s[2], @t[3] +___ +# output in s3, s2, s1, t1 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3 + &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]); + +### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb +} + +# AES linear components + +sub ShiftRows { +my @x=@_[0..7]; +my @t=@_[8..11]; +my $mask=pop; +$code.=<<___; + vldmia $key!, {@t[0]-@t[3]} + veor @t[0], @t[0], @x[0] + veor @t[1], @t[1], @x[1] + vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)` + vldmia $key!, {@t[0]} + veor @t[2], @t[2], @x[2] + vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)` + vldmia $key!, {@t[1]} + veor @t[3], @t[3], @x[3] + vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)` + vldmia $key!, {@t[2]} + vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)` + vldmia $key!, {@t[3]} + veor @t[0], @t[0], @x[4] + veor @t[1], @t[1], @x[5] + vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)` + veor @t[2], @t[2], @x[6] + vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)` + veor @t[3], @t[3], @x[7] + vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)` + vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)` +___ +} + +sub MixColumns { +# modified to emit output in order suitable for feeding back to aesenc[last] +my @x=@_[0..7]; +my @t=@_[8..15]; +my $inv=@_[16]; # optional +$code.=<<___; + vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32) + vext.8 @t[2], @x[2], @x[2], #12 + veor @x[1], @x[1], @t[1] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[2], @x[2], @t[2] + vext.8 @t[4], @x[4], @x[4], #12 + veor @x[3], @x[3], @t[3] + vext.8 @t[5], @x[5], @x[5], #12 + veor @x[4], @x[4], @t[4] + vext.8 @t[6], @x[6], @x[6], #12 + veor @x[5], @x[5], @t[5] + vext.8 @t[7], @x[7], @x[7], #12 + veor @x[6], @x[6], @t[6] + + veor @t[1], @t[1], @x[0] + veor @x[7], @x[7], @t[7] + vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor @t[2], @t[2], @x[1] + veor @t[0], @t[0], @x[7] + veor @t[1], @t[1], @x[7] + vext.8 @x[1], @x[1], @x[1], #8 + veor @t[5], @t[5], @x[4] + veor @x[0], @x[0], @t[0] + veor @t[6], @t[6], @x[5] + veor @x[1], @x[1], @t[1] + vext.8 @t[0], @x[4], @x[4], #8 + veor @t[4], @t[4], @x[3] + vext.8 @t[1], @x[5], @x[5], #8 + veor @t[7], @t[7], @x[6] + vext.8 @x[4], @x[3], @x[3], #8 + veor @t[3], @t[3], @x[2] + vext.8 @x[5], @x[7], @x[7], #8 + veor @t[4], @t[4], @x[7] + vext.8 @x[3], @x[6], @x[6], #8 + veor @t[3], @t[3], @x[7] + vext.8 @x[6], @x[2], @x[2], #8 + veor @x[7], @t[1], @t[5] +___ +$code.=<<___ if (!$inv); + veor @x[2], @t[0], @t[4] + veor @x[4], @x[4], @t[3] + veor @x[5], @x[5], @t[7] + veor @x[3], @x[3], @t[6] + @ vmov @x[2], @t[0] + veor @x[6], @x[6], @t[2] + @ vmov @x[7], @t[1] +___ +$code.=<<___ if ($inv); + veor @t[3], @t[3], @x[4] + veor @x[5], @x[5], @t[7] + veor @x[2], @x[3], @t[6] + veor @x[3], @t[0], @t[4] + veor @x[4], @x[6], @t[2] + vmov @x[6], @t[3] + @ vmov @x[7], @t[1] +___ +} + +sub InvMixColumns_orig { +my @x=@_[0..7]; +my @t=@_[8..15]; + +$code.=<<___; + @ multiplication by 0x0e + vext.8 @t[7], @x[7], @x[7], #12 + vmov @t[2], @x[2] + veor @x[2], @x[2], @x[5] @ 2 5 + veor @x[7], @x[7], @x[5] @ 7 5 + vext.8 @t[0], @x[0], @x[0], #12 + vmov @t[5], @x[5] + veor @x[5], @x[5], @x[0] @ 5 0 [1] + veor @x[0], @x[0], @x[1] @ 0 1 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[1], @x[1], @x[2] @ 1 25 + veor @x[0], @x[0], @x[6] @ 01 6 [2] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[1], @x[1], @x[3] @ 125 3 [4] + veor @x[2], @x[2], @x[0] @ 25 016 [3] + veor @x[3], @x[3], @x[7] @ 3 75 + veor @x[7], @x[7], @x[6] @ 75 6 [0] + vext.8 @t[6], @x[6], @x[6], #12 + vmov @t[4], @x[4] + veor @x[6], @x[6], @x[4] @ 6 4 + veor @x[4], @x[4], @x[3] @ 4 375 [6] + veor @x[3], @x[3], @x[7] @ 375 756=36 + veor @x[6], @x[6], @t[5] @ 64 5 [7] + veor @x[3], @x[3], @t[2] @ 36 2 + vext.8 @t[5], @t[5], @t[5], #12 + veor @x[3], @x[3], @t[4] @ 362 4 [5] +___ + my @y = @x[7,5,0,2,1,3,4,6]; +$code.=<<___; + @ multiplication by 0x0b + veor @y[1], @y[1], @y[0] + veor @y[0], @y[0], @t[0] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[5] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[1], @y[1], @t[6] + veor @y[0], @y[0], @t[7] + veor @t[7], @t[7], @t[6] @ clobber t[7] + + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @y[0] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[2], @y[2], @t[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + veor @y[2], @y[2], @t[2] + veor @y[3], @y[3], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[7] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[3], @y[3], @t[3] + veor @y[6], @y[6], @t[3] + veor @y[4], @y[4], @t[3] + veor @y[7], @y[7], @t[4] + vext.8 @t[3], @t[3], @t[3], #12 + veor @y[5], @y[5], @t[4] + veor @y[7], @y[7], @t[7] + veor @t[7], @t[7], @t[5] @ clobber t[7] even more + veor @y[3], @y[3], @t[5] + veor @y[4], @y[4], @t[4] + + veor @y[5], @y[5], @t[7] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[6], @y[6], @t[7] + veor @y[4], @y[4], @t[7] + + veor @t[7], @t[7], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + + @ multiplication by 0x0d + veor @y[4], @y[4], @y[7] + veor @t[7], @t[7], @t[6] @ restore t[7] + veor @y[7], @y[7], @t[4] + vext.8 @t[6], @t[6], @t[6], #12 + veor @y[2], @y[2], @t[0] + veor @y[7], @y[7], @t[5] + vext.8 @t[7], @t[7], @t[7], #12 + veor @y[2], @y[2], @t[2] + + veor @y[3], @y[3], @y[1] + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[0] + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @t[5] + veor @y[0], @y[0], @t[5] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[1], @y[1], @t[7] + veor @y[0], @y[0], @t[6] + veor @y[3], @y[3], @y[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + + veor @y[7], @y[7], @t[7] + veor @y[4], @y[4], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[6] + veor @t[6], @t[6], @t[3] @ clobber t[6] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[4], @y[4], @y[7] + veor @y[3], @y[3], @t[6] + + veor @y[6], @y[6], @t[6] + veor @y[5], @y[5], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + veor @y[6], @y[6], @t[4] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[5], @y[5], @t[6] + veor @y[6], @y[6], @t[7] + vext.8 @t[7], @t[7], @t[7], #12 + veor @t[6], @t[6], @t[3] @ restore t[6] + vext.8 @t[3], @t[3], @t[3], #12 + + @ multiplication by 0x09 + veor @y[4], @y[4], @y[1] + veor @t[1], @t[1], @y[1] @ t[1]=y[1] + veor @t[0], @t[0], @t[5] @ clobber t[0] + vext.8 @t[6], @t[6], @t[6], #12 + veor @t[1], @t[1], @t[5] + veor @y[3], @y[3], @t[0] + veor @t[0], @t[0], @y[0] @ t[0]=y[0] + veor @t[1], @t[1], @t[6] + veor @t[6], @t[6], @t[7] @ clobber t[6] + veor @y[4], @y[4], @t[1] + veor @y[7], @y[7], @t[4] + veor @y[6], @y[6], @t[3] + veor @y[5], @y[5], @t[2] + veor @t[4], @t[4], @y[4] @ t[4]=y[4] + veor @t[3], @t[3], @y[3] @ t[3]=y[3] + veor @t[5], @t[5], @y[5] @ t[5]=y[5] + veor @t[2], @t[2], @y[2] @ t[2]=y[2] + veor @t[3], @t[3], @t[7] + veor @XMM[5], @t[5], @t[6] + veor @XMM[6], @t[6], @y[6] @ t[6]=y[6] + veor @XMM[2], @t[2], @t[6] + veor @XMM[7], @t[7], @y[7] @ t[7]=y[7] + + vmov @XMM[0], @t[0] + vmov @XMM[1], @t[1] + @ vmov @XMM[2], @t[2] + vmov @XMM[3], @t[3] + vmov @XMM[4], @t[4] + @ vmov @XMM[5], @t[5] + @ vmov @XMM[6], @t[6] + @ vmov @XMM[7], @t[7] +___ +} + +sub InvMixColumns { +my @x=@_[0..7]; +my @t=@_[8..15]; + +# Thanks to Jussi Kivilinna for providing pointer to +# +# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 | +# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 | +# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 | +# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 | + +$code.=<<___; + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 @t[0], @x[0], @x[0], #8 + vext.8 @t[6], @x[6], @x[6], #8 + vext.8 @t[7], @x[7], @x[7], #8 + veor @t[0], @t[0], @x[0] + vext.8 @t[1], @x[1], @x[1], #8 + veor @t[6], @t[6], @x[6] + vext.8 @t[2], @x[2], @x[2], #8 + veor @t[7], @t[7], @x[7] + vext.8 @t[3], @x[3], @x[3], #8 + veor @t[1], @t[1], @x[1] + vext.8 @t[4], @x[4], @x[4], #8 + veor @t[2], @t[2], @x[2] + vext.8 @t[5], @x[5], @x[5], #8 + veor @t[3], @t[3], @x[3] + veor @t[4], @t[4], @x[4] + veor @t[5], @t[5], @x[5] + + veor @x[0], @x[0], @t[6] + veor @x[1], @x[1], @t[6] + veor @x[2], @x[2], @t[0] + veor @x[4], @x[4], @t[2] + veor @x[3], @x[3], @t[1] + veor @x[1], @x[1], @t[7] + veor @x[2], @x[2], @t[7] + veor @x[4], @x[4], @t[6] + veor @x[5], @x[5], @t[3] + veor @x[3], @x[3], @t[6] + veor @x[6], @x[6], @t[4] + veor @x[4], @x[4], @t[7] + veor @x[5], @x[5], @t[7] + veor @x[7], @x[7], @t[5] +___ + &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6 +} + +sub swapmove { +my ($a,$b,$n,$mask,$t)=@_; +$code.=<<___; + vshr.u64 $t, $b, #$n + veor $t, $t, $a + vand $t, $t, $mask + veor $a, $a, $t + vshl.u64 $t, $t, #$n + veor $b, $b, $t +___ +} +sub swapmove2x { +my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_; +$code.=<<___; + vshr.u64 $t0, $b0, #$n + vshr.u64 $t1, $b1, #$n + veor $t0, $t0, $a0 + veor $t1, $t1, $a1 + vand $t0, $t0, $mask + vand $t1, $t1, $mask + veor $a0, $a0, $t0 + vshl.u64 $t0, $t0, #$n + veor $a1, $a1, $t1 + vshl.u64 $t1, $t1, #$n + veor $b0, $b0, $t0 + veor $b1, $b1, $t1 +___ +} + +sub bitslice { +my @x=reverse(@_[0..7]); +my ($t0,$t1,$t2,$t3)=@_[8..11]; +$code.=<<___; + vmov.i8 $t0,#0x55 @ compose .LBS0 + vmov.i8 $t1,#0x33 @ compose .LBS1 +___ + &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3); + &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); +$code.=<<___; + vmov.i8 $t0,#0x0f @ compose .LBS2 +___ + &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3); + &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + + &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3); + &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ 7 +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_ARCH__>=7 +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.fpu neon + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr $const,_bsaes_decrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + add $const,$const,#.LM0ISR-_bsaes_decrypt8 + + vldmia $const!, {@XMM[8]} @ .LM0ISR + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Ldec_sbox:\n"; + &InvSbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Ldec_done +___ + &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq $const,$const,#0x10 + bne .Ldec_loop + vldmia $const, {@XMM[12]} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: +___ + &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr $const,_bsaes_encrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + sub $const,$const,#_bsaes_encrypt8-.LM0SR + + vldmia $const!, {@XMM[8]} @ .LM0SR +_bsaes_encrypt8_alt: + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +_bsaes_encrypt8_bitslice: +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Lenc_sbox:\n"; + &Sbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Lenc_done +___ + &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq $const,$const,#0x10 + bne .Lenc_loop + vldmia $const, {@XMM[12]} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: +___ + # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb + &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +___ +} +{ +my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6"); + +sub bitslice_key { +my @x=reverse(@_[0..7]); +my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12]; + + &swapmove (@x[0,1],1,$bs0,$t2,$t3); +$code.=<<___; + @ &swapmove(@x[2,3],1,$t0,$t2,$t3); + vmov @x[2], @x[0] + vmov @x[3], @x[1] +___ + #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); + + &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3); +$code.=<<___; + @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + vmov @x[4], @x[0] + vmov @x[6], @x[2] + vmov @x[5], @x[1] + vmov @x[7], @x[3] +___ + &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3); + &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3); +} + +$code.=<<___; +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr $const,_bsaes_key_convert + vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key + sub $const,$const,#_bsaes_key_convert-.LM0 + vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key + + vmov.i8 @XMM[8], #0x01 @ bit masks + vmov.i8 @XMM[9], #0x02 + vmov.i8 @XMM[10], #0x04 + vmov.i8 @XMM[11], #0x08 + vmov.i8 @XMM[12], #0x10 + vmov.i8 @XMM[13], #0x20 + vldmia $const, {@XMM[14]} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 @XMM[7], @XMM[7] + vrev32.8 @XMM[15], @XMM[15] +#endif + sub $rounds,$rounds,#1 + vstmia $out!, {@XMM[7]} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])` + vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])` + vmov.i8 @XMM[6], #0x40 + vmov.i8 @XMM[15], #0x80 + + vtst.8 @XMM[0], @XMM[7], @XMM[8] + vtst.8 @XMM[1], @XMM[7], @XMM[9] + vtst.8 @XMM[2], @XMM[7], @XMM[10] + vtst.8 @XMM[3], @XMM[7], @XMM[11] + vtst.8 @XMM[4], @XMM[7], @XMM[12] + vtst.8 @XMM[5], @XMM[7], @XMM[13] + vtst.8 @XMM[6], @XMM[7], @XMM[6] + vtst.8 @XMM[7], @XMM[7], @XMM[15] + vld1.8 {@XMM[15]}, [$inp]! @ load next round key + vmvn @XMM[0], @XMM[0] @ "pnot" + vmvn @XMM[1], @XMM[1] + vmvn @XMM[5], @XMM[5] + vmvn @XMM[6], @XMM[6] +#ifdef __ARMEL__ + vrev32.8 @XMM[15], @XMM[15] +#endif + subs $rounds,$rounds,#1 + vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 @XMM[7],#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +___ +} + +if (0) { # following four functions are unsupported interface + # used for benchmarking... +$code.=<<___; +.globl bsaes_enc_key_convert +.type bsaes_enc_key_convert,%function +.align 4 +bsaes_enc_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_enc_key_convert,.-bsaes_enc_key_convert + +.globl bsaes_encrypt_128 +.type bsaes_encrypt_128,%function +.align 4 +bsaes_encrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Lenc128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_encrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Lenc128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_encrypt_128,.-bsaes_encrypt_128 + +.globl bsaes_dec_key_convert +.type bsaes_dec_key_convert,%function +.align 4 +bsaes_dec_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + vldmia $out, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $out, {@XMM[7]} + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_dec_key_convert,.-bsaes_dec_key_convert + +.globl bsaes_decrypt_128 +.type bsaes_decrypt_128,%function +.align 4 +bsaes_decrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Ldec128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_decrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Ldec128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_decrypt_128,.-bsaes_decrypt_128 +___ +} +{ +my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10)); +my ($keysched)=("sp"); + +$code.=<<___; +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp $len, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ivp, [ip] @ IV is 1st arg on the stack + mov $len, $len, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + vldmia $keysched, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $keysched, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: +#endif + + vld1.8 {@XMM[15]}, [$ivp] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs $len, $len, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5, $rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp] + sub $inp, $inp, #0x60 + vstmia $fp, {@XMM[15]} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + veor @XMM[5], @XMM[5], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[5]}, [$out]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds $len, $len, #8 + beq .Lcbc_dec_done + + vld1.8 {@XMM[0]}, [$inp]! @ load input + cmp $len, #2 + blo .Lcbc_dec_one + vld1.8 {@XMM[1]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + mov r5, $rounds + vstmia $fp, {@XMM[15]} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {@XMM[2]}, [$inp]! + cmp $len, #4 + blo .Lcbc_dec_three + vld1.8 {@XMM[3]}, [$inp]! + beq .Lcbc_dec_four + vld1.8 {@XMM[4]}, [$inp]! + cmp $len, #6 + blo .Lcbc_dec_five + vld1.8 {@XMM[5]}, [$inp]! + beq .Lcbc_dec_six + vld1.8 {@XMM[6]}, [$inp]! + sub $inp, $inp, #0x70 + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub $inp, $inp, #0x60 + bl _bsaes_decrypt8 + vldmia $fp,{@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub $inp, $inp, #0x50 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[2], @XMM[2], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub $inp, $inp, #0x40 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub $inp, $inp, #0x30 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub $inp, $inp, #0x20 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! @ reload input + veor @XMM[1], @XMM[1], @XMM[8] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub $inp, $inp, #0x10 + mov $rounds, $out @ save original out pointer + mov $out, $fp @ use the iv scratch space as out buffer + mov r2, $key + vmov @XMM[4],@XMM[15] @ just in case ensure that IV + vmov @XMM[5],@XMM[0] @ and input are preserved + bl AES_decrypt + vld1.8 {@XMM[0]}, [$fp,:64] @ load result + veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV + vmov @XMM[15], @XMM[5] @ @XMM[5] holds input + vst1.8 {@XMM[0]}, [$rounds] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lcbc_dec_bzero +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + vst1.8 {@XMM[15]}, [$ivp] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +___ +} +{ +my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10))); +my $const = "r6"; # shared with _bsaes_encrypt8_alt +my $keysched = "sp"; + +$code.=<<___; +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp $len, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ctr, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vld1.8 {@XMM[0]}, [$ctr] @ load counter + add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr + vldmia $keysched, {@XMM[4]} @ load round0 key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + +.align 2 +0: add r12, $key, #248 + vld1.8 {@XMM[0]}, [$ctr] @ load counter + adrl $ctr, .LREVM0SR @ borrow $ctr + vldmia r12, {@XMM[4]} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 @XMM[8],#1 @ compose 1<<96 + veor @XMM[9],@XMM[9],@XMM[9] + vrev32.8 @XMM[0],@XMM[0] + vext.8 @XMM[8],@XMM[9],@XMM[8],#4 + vrev32.8 @XMM[4],@XMM[4] + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vstmia $keysched, {@XMM[4]} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96 + vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1 + vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2 + vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3 + vadd.u32 @XMM[4], @XMM[1], @XMM[10] + vadd.u32 @XMM[5], @XMM[2], @XMM[10] + vadd.u32 @XMM[6], @XMM[3], @XMM[10] + vadd.u32 @XMM[7], @XMM[4], @XMM[10] + vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia $keysched, {@XMM[9]} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, $keysched, #0x10 @ pass next round key +#else + add r4, $key, #`248+16` +#endif + vldmia $ctr, {@XMM[8]} @ .LREVM0SR + mov r5, $rounds @ pass rounds + vstmia $fp, {@XMM[10]} @ save next counter + sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs $len, $len, #8 + blo .Lctr_enc_loop_done + + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[0], @XMM[8] + veor @XMM[1], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[10] + veor @XMM[6], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[7], @XMM[13] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + veor @XMM[5], @XMM[15] + vst1.8 {@XMM[6]}, [$out]! + vmov.i32 @XMM[8], #1 @ compose 1<<96 + vst1.8 {@XMM[3]}, [$out]! + veor @XMM[9], @XMM[9], @XMM[9] + vst1.8 {@XMM[7]}, [$out]! + vext.8 @XMM[8], @XMM[9], @XMM[8], #4 + vst1.8 {@XMM[2]}, [$out]! + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vst1.8 {@XMM[5]}, [$out]! + vldmia $fp, {@XMM[0]} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add $len, $len, #8 + vld1.8 {@XMM[8]}, [$inp]! @ load input + veor @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! @ write output + cmp $len, #2 + blo .Lctr_enc_done + vld1.8 {@XMM[9]}, [$inp]! + veor @XMM[1], @XMM[9] + vst1.8 {@XMM[1]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[4], @XMM[10] + vst1.8 {@XMM[4]}, [$out]! + cmp $len, #4 + blo .Lctr_enc_done + vld1.8 {@XMM[11]}, [$inp]! + veor @XMM[6], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[3]}, [$out]! + cmp $len, #6 + blo .Lctr_enc_done + vld1.8 {@XMM[13]}, [$inp]! + veor @XMM[7], @XMM[13] + vst1.8 {@XMM[7]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[14]}, [$inp] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[2]}, [$out]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lctr_enc_bzero +#else + vstmia $keysched, {q0-q1} +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, $inp @ copy arguments + mov r5, $out + mov r6, $len + mov r7, $key + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {@XMM[1]}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [r4]! @ load input + vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor @XMM[0],@XMM[0],@XMM[1] + vst1.8 {@XMM[0]}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +___ +} +{ +###################################################################### +# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len, +# const AES_KEY *key1, const AES_KEY *key2, +# const unsigned char iv[16]); +# +my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3))); +my $const="r6"; # returned by _bsaes_key_convert +my $twmask=@XMM[5]; +my @T=@XMM[6..7]; + +$code.=<<___; +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + subs $len, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds $len, #0x70 + bmi .Lxts_enc_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_enc_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_enc_ret + sub r6, $out, #0x10 + +.Lxts_enc_steal: + ldrb r0, [$inp], #1 + ldrb r1, [$out, #-0x10] + strb r0, [$out, #-0x10] + strb r1, [$out], #1 + + subs $len, #1 + bhi .Lxts_enc_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_enc_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + tst $len, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne $len, #0x10 @ subtract another 16 bytes + subs $len, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds $len, #0x70 + bmi .Lxts_dec_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_dec_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + mov r5, $magic @ preserve magic + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + mov $magic, r5 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia $magic, {$twmask} + vshr.s64 @XMM[6], @XMM[8], #63 + vand @XMM[6], @XMM[6], $twmask + vadd.u64 @XMM[9], @XMM[8], @XMM[8] + vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")` + veor @XMM[9], @XMM[9], @XMM[6] + + @ perform the final decryption with the last tweak value + vld1.8 {@XMM[0]}, [$inp]! + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[9] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[9] + vst1.8 {@XMM[0]}, [$out] + + mov r6, $out +.Lxts_dec_steal: + ldrb r1, [$out] + ldrb r0, [$inp], #1 + strb r1, [$out, #0x10] + strb r0, [$out], #1 + + subs $len, #1 + bhi .Lxts_dec_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_dec_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +___ +} +$code.=<<___; +#endif +___ + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +print $code; + +close STDOUT; diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S new file mode 100644 index 00000000000..99207c45ec1 --- /dev/null +++ b/arch/arm/crypto/sha1-armv4-large.S @@ -0,0 +1,497 @@ +#define __ARM_ARCH__ __LINUX_ARM_ARCH__ +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ ==================================================================== + +@ sha1_block procedure for ARMv4. +@ +@ January 2007. + +@ Size/performance trade-off +@ ==================================================================== +@ impl size in bytes comp cycles[*] measured performance +@ ==================================================================== +@ thumb 304 3212 4420 +@ armv4-small 392/+29% 1958/+64% 2250/+96% +@ armv4-compact 740/+89% 1552/+26% 1840/+22% +@ armv4-large 1420/+92% 1307/+19% 1370/+34%[***] +@ full unroll ~5100/+260% ~1260/+4% ~1300/+5% +@ ==================================================================== +@ thumb = same as 'small' but in Thumb instructions[**] and +@ with recurring code in two private functions; +@ small = detached Xload/update, loops are folded; +@ compact = detached Xload/update, 5x unroll; +@ large = interleaved Xload/update, 5x unroll; +@ full unroll = interleaved Xload/update, full unroll, estimated[!]; +@ +@ [*] Manually counted instructions in "grand" loop body. Measured +@ performance is affected by prologue and epilogue overhead, +@ i-cache availability, branch penalties, etc. +@ [**] While each Thumb instruction is twice smaller, they are not as +@ diverse as ARM ones: e.g., there are only two arithmetic +@ instructions with 3 arguments, no [fixed] rotate, addressing +@ modes are limited. As result it takes more instructions to do +@ the same job in Thumb, therefore the code is never twice as +@ small and always slower. +@ [***] which is also ~35% better than compiler generated code. Dual- +@ issue Cortex A8 core was measured to process input block in +@ ~990 cycles. + +@ August 2010. +@ +@ Rescheduling for dual-issue pipeline resulted in 13% improvement on +@ Cortex A8 core and in absolute terms ~870 cycles per input block +@ [or 13.6 cycles per byte]. + +@ February 2011. +@ +@ Profiler-assisted and platform-specific optimization resulted in 10% +@ improvement on Cortex A8 core and 12.2 cycles per byte. + +#include + +.text + +.align 2 +ENTRY(sha1_block_data_order) + stmdb sp!,{r4-r12,lr} + add r2,r1,r2,lsl#6 @ r2 to point at the end of r1 + ldmia r0,{r3,r4,r5,r6,r7} +.Lloop: + ldr r8,.LK_00_19 + mov r14,sp + sub sp,sp,#15*4 + mov r5,r5,ror#30 + mov r6,r6,ror#30 + mov r7,r7,ror#30 @ [6] +.L_00_15: +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r7,r8,r7,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r5,r6 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r7,r8,r7,ror#2 @ E+=K_00_19 + eor r10,r5,r6 @ F_xx_xx + add r7,r7,r3,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r4,r10,ror#2 + add r7,r7,r9 @ E+=X[i] + eor r10,r10,r6,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r7,r7,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r6,r8,r6,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r4,r5 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r6,r8,r6,ror#2 @ E+=K_00_19 + eor r10,r4,r5 @ F_xx_xx + add r6,r6,r7,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r3,r10,ror#2 + add r6,r6,r9 @ E+=X[i] + eor r10,r10,r5,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r6,r6,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r5,r8,r5,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r3,r4 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r5,r8,r5,ror#2 @ E+=K_00_19 + eor r10,r3,r4 @ F_xx_xx + add r5,r5,r6,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r7,r10,ror#2 + add r5,r5,r9 @ E+=X[i] + eor r10,r10,r4,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r5,r5,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r4,r8,r4,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r7,r3 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r4,r8,r4,ror#2 @ E+=K_00_19 + eor r10,r7,r3 @ F_xx_xx + add r4,r4,r5,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r6,r10,ror#2 + add r4,r4,r9 @ E+=X[i] + eor r10,r10,r3,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r4,r4,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r3,r8,r3,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r6,r7 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r3,r8,r3,ror#2 @ E+=K_00_19 + eor r10,r6,r7 @ F_xx_xx + add r3,r3,r4,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r5,r10,ror#2 + add r3,r3,r9 @ E+=X[i] + eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r3,r3,r10 @ E+=F_00_19(B,C,D) + cmp r14,sp + bne .L_00_15 @ [((11+4)*5+2)*3] + sub sp,sp,#25*4 +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r7,r8,r7,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r5,r6 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r7,r8,r7,ror#2 @ E+=K_00_19 + eor r10,r5,r6 @ F_xx_xx + add r7,r7,r3,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r4,r10,ror#2 + add r7,r7,r9 @ E+=X[i] + eor r10,r10,r6,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r7,r7,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r6,r8,r6,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r4,r5 @ F_xx_xx + mov r9,r9,ror#31 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r3,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r6,r6,r9 @ E+=X[i] + eor r10,r10,r5,ror#2 @ F_00_19(B,C,D) + add r6,r6,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r5,r8,r5,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r3,r4 @ F_xx_xx + mov r9,r9,ror#31 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r7,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r5,r5,r9 @ E+=X[i] + eor r10,r10,r4,ror#2 @ F_00_19(B,C,D) + add r5,r5,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r4,r8,r4,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r7,r3 @ F_xx_xx + mov r9,r9,ror#31 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r6,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r4,r4,r9 @ E+=X[i] + eor r10,r10,r3,ror#2 @ F_00_19(B,C,D) + add r4,r4,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r3,r8,r3,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r6,r7 @ F_xx_xx + mov r9,r9,ror#31 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r5,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r3,r3,r9 @ E+=X[i] + eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) + add r3,r3,r10 @ E+=F_00_19(B,C,D) + + ldr r8,.LK_20_39 @ [+15+16*4] + cmn sp,#0 @ [+3], clear carry to denote 20_39 +.L_20_39_or_60_79: + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r7,r8,r7,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r5,r6 @ F_xx_xx + mov r9,r9,ror#31 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r4,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r7,r7,r9 @ E+=X[i] + add r7,r7,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r6,r8,r6,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r4,r5 @ F_xx_xx + mov r9,r9,ror#31 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r3,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r6,r6,r9 @ E+=X[i] + add r6,r6,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r5,r8,r5,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r3,r4 @ F_xx_xx + mov r9,r9,ror#31 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r7,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r5,r5,r9 @ E+=X[i] + add r5,r5,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r4,r8,r4,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r7,r3 @ F_xx_xx + mov r9,r9,ror#31 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r6,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r4,r4,r9 @ E+=X[i] + add r4,r4,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r3,r8,r3,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r6,r7 @ F_xx_xx + mov r9,r9,ror#31 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r5,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r3,r3,r9 @ E+=X[i] + add r3,r3,r10 @ E+=F_20_39(B,C,D) + ARM( teq r14,sp ) @ preserve carry + THUMB( mov r11,sp ) + THUMB( teq r14,r11 ) @ preserve carry + bne .L_20_39_or_60_79 @ [+((12+3)*5+2)*4] + bcs .L_done @ [+((12+3)*5+2)*4], spare 300 bytes + + ldr r8,.LK_40_59 + sub sp,sp,#20*4 @ [+2] +.L_40_59: + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r7,r8,r7,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r5,r6 @ F_xx_xx + mov r9,r9,ror#31 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r4,r10,ror#2 @ F_xx_xx + and r11,r5,r6 @ F_xx_xx + add r7,r7,r9 @ E+=X[i] + add r7,r7,r10 @ E+=F_40_59(B,C,D) + add r7,r7,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r6,r8,r6,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r4,r5 @ F_xx_xx + mov r9,r9,ror#31 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r3,r10,ror#2 @ F_xx_xx + and r11,r4,r5 @ F_xx_xx + add r6,r6,r9 @ E+=X[i] + add r6,r6,r10 @ E+=F_40_59(B,C,D) + add r6,r6,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r5,r8,r5,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r3,r4 @ F_xx_xx + mov r9,r9,ror#31 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r7,r10,ror#2 @ F_xx_xx + and r11,r3,r4 @ F_xx_xx + add r5,r5,r9 @ E+=X[i] + add r5,r5,r10 @ E+=F_40_59(B,C,D) + add r5,r5,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r4,r8,r4,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r7,r3 @ F_xx_xx + mov r9,r9,ror#31 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r6,r10,ror#2 @ F_xx_xx + and r11,r7,r3 @ F_xx_xx + add r4,r4,r9 @ E+=X[i] + add r4,r4,r10 @ E+=F_40_59(B,C,D) + add r4,r4,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r3,r8,r3,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r6,r7 @ F_xx_xx + mov r9,r9,ror#31 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r5,r10,ror#2 @ F_xx_xx + and r11,r6,r7 @ F_xx_xx + add r3,r3,r9 @ E+=X[i] + add r3,r3,r10 @ E+=F_40_59(B,C,D) + add r3,r3,r11,ror#2 + cmp r14,sp + bne .L_40_59 @ [+((12+5)*5+2)*4] + + ldr r8,.LK_60_79 + sub sp,sp,#20*4 + cmp sp,#0 @ set carry to denote 60_79 + b .L_20_39_or_60_79 @ [+4], spare 300 bytes +.L_done: + add sp,sp,#80*4 @ "deallocate" stack frame + ldmia r0,{r8,r9,r10,r11,r12} + add r3,r8,r3 + add r4,r9,r4 + add r5,r10,r5,ror#2 + add r6,r11,r6,ror#2 + add r7,r12,r7,ror#2 + stmia r0,{r3,r4,r5,r6,r7} + teq r1,r2 + bne .Lloop @ [+18], total 1307 + + ldmia sp!,{r4-r12,pc} +.align 2 +.LK_00_19: .word 0x5a827999 +.LK_20_39: .word 0x6ed9eba1 +.LK_40_59: .word 0x8f1bbcdc +.LK_60_79: .word 0xca62c1d6 +ENDPROC(sha1_block_data_order) +.asciz "SHA1 block transform for ARMv4, CRYPTOGAMS by " +.align 2 diff --git a/arch/arm/crypto/sha1-armv7-neon.S b/arch/arm/crypto/sha1-armv7-neon.S new file mode 100644 index 00000000000..50013c0e286 --- /dev/null +++ b/arch/arm/crypto/sha1-armv7-neon.S @@ -0,0 +1,634 @@ +/* sha1-armv7-neon.S - ARM/NEON accelerated SHA-1 transform function + * + * Copyright © 2013-2014 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include + + +.syntax unified +.code 32 +.fpu neon + +.text + + +/* Context structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 + + +/* Constants */ + +#define K1 0x5A827999 +#define K2 0x6ED9EBA1 +#define K3 0x8F1BBCDC +#define K4 0xCA62C1D6 +.align 4 +.LK_VEC: +.LK1: .long K1, K1, K1, K1 +.LK2: .long K2, K2, K2, K2 +.LK3: .long K3, K3, K3, K3 +.LK4: .long K4, K4, K4, K4 + + +/* Register macros */ + +#define RSTATE r0 +#define RDATA r1 +#define RNBLKS r2 +#define ROLDSTACK r3 +#define RWK lr + +#define _a r4 +#define _b r5 +#define _c r6 +#define _d r7 +#define _e r8 + +#define RT0 r9 +#define RT1 r10 +#define RT2 r11 +#define RT3 r12 + +#define W0 q0 +#define W1 q1 +#define W2 q2 +#define W3 q3 +#define W4 q4 +#define W5 q5 +#define W6 q6 +#define W7 q7 + +#define tmp0 q8 +#define tmp1 q9 +#define tmp2 q10 +#define tmp3 q11 + +#define qK1 q12 +#define qK2 q13 +#define qK3 q14 +#define qK4 q15 + + +/* Round function macros. */ + +#define WK_offs(i) (((i) & 15) * 4) + +#define _R_F1(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + bic RT0, d, b; \ + add e, e, a, ror #(32 - 5); \ + and RT1, c, b; \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add RT0, RT0, RT3; \ + add e, e, RT1; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT0; + +#define _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + eor RT0, d, b; \ + add e, e, a, ror #(32 - 5); \ + eor RT0, RT0, c; \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT3; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT0; \ + +#define _R_F3(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + eor RT0, b, c; \ + and RT1, b, c; \ + add e, e, a, ror #(32 - 5); \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + and RT0, RT0, d; \ + add RT1, RT1, RT3; \ + add e, e, RT0; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT1; + +#define _R_F4(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define _R(a,b,c,d,e,f,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + _R_##f(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define R(a,b,c,d,e,f,i) \ + _R_##f(a,b,c,d,e,i,dummy,dummy,dummy,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define dummy(...) + + +/* Input expansion macros. */ + +/********* Precalc macros for rounds 0-15 *************************************/ + +#define W_PRECALC_00_15() \ + add RWK, sp, #(WK_offs(0)); \ + \ + vld1.32 {tmp0, tmp1}, [RDATA]!; \ + vrev32.8 W0, tmp0; /* big => little */ \ + vld1.32 {tmp2, tmp3}, [RDATA]!; \ + vadd.u32 tmp0, W0, curK; \ + vrev32.8 W7, tmp1; /* big => little */ \ + vrev32.8 W6, tmp2; /* big => little */ \ + vadd.u32 tmp1, W7, curK; \ + vrev32.8 W5, tmp3; /* big => little */ \ + vadd.u32 tmp2, W6, curK; \ + vst1.32 {tmp0, tmp1}, [RWK]!; \ + vadd.u32 tmp3, W5, curK; \ + vst1.32 {tmp2, tmp3}, [RWK]; \ + +#define WPRECALC_00_15_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vld1.32 {tmp0, tmp1}, [RDATA]!; \ + +#define WPRECALC_00_15_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(0)); \ + +#define WPRECALC_00_15_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W0, tmp0; /* big => little */ \ + +#define WPRECALC_00_15_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vld1.32 {tmp2, tmp3}, [RDATA]!; \ + +#define WPRECALC_00_15_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W0, curK; \ + +#define WPRECALC_00_15_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W7, tmp1; /* big => little */ \ + +#define WPRECALC_00_15_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W6, tmp2; /* big => little */ \ + +#define WPRECALC_00_15_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp1, W7, curK; \ + +#define WPRECALC_00_15_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W5, tmp3; /* big => little */ \ + +#define WPRECALC_00_15_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp2, W6, curK; \ + +#define WPRECALC_00_15_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0, tmp1}, [RWK]!; \ + +#define WPRECALC_00_15_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp3, W5, curK; \ + +#define WPRECALC_00_15_12(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp2, tmp3}, [RWK]; \ + + +/********* Precalc macros for rounds 16-31 ************************************/ + +#define WPRECALC_16_31_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0; \ + vext.8 W, W_m16, W_m12, #8; \ + +#define WPRECALC_16_31_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(i)); \ + vext.8 tmp0, W_m04, tmp0, #4; \ + +#define WPRECALC_16_31_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0, W_m16; \ + veor.32 W, W, W_m08; \ + +#define WPRECALC_16_31_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp1, tmp1; \ + veor W, W, tmp0; \ + +#define WPRECALC_16_31_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp0, W, #1; \ + +#define WPRECALC_16_31_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vext.8 tmp1, tmp1, W, #(16-12); \ + vshr.u32 W, W, #31; \ + +#define WPRECALC_16_31_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vorr tmp0, tmp0, W; \ + vshr.u32 W, tmp1, #30; \ + +#define WPRECALC_16_31_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp1, tmp1, #2; \ + +#define WPRECALC_16_31_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0, W; \ + +#define WPRECALC_16_31_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, tmp0, tmp1; \ + +#define WPRECALC_16_31_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W, curK; \ + +#define WPRECALC_16_31_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0}, [RWK]; + + +/********* Precalc macros for rounds 32-79 ************************************/ + +#define WPRECALC_32_79_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, W_m28; \ + +#define WPRECALC_32_79_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vext.8 tmp0, W_m08, W_m04, #8; \ + +#define WPRECALC_32_79_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, W_m16; \ + +#define WPRECALC_32_79_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, tmp0; \ + +#define WPRECALC_32_79_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(i&~3)); \ + +#define WPRECALC_32_79_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp1, W, #2; \ + +#define WPRECALC_32_79_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshr.u32 tmp0, W, #30; \ + +#define WPRECALC_32_79_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vorr W, tmp0, tmp1; \ + +#define WPRECALC_32_79_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W, curK; \ + +#define WPRECALC_32_79_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0}, [RWK]; + + +/* + * Transform nblks*64 bytes (nblks*16 32-bit words) at DATA. + * + * unsigned int + * sha1_transform_neon (void *ctx, const unsigned char *data, + * unsigned int nblks) + */ +.align 3 +ENTRY(sha1_transform_neon) + /* input: + * r0: ctx, CTX + * r1: data (64*nblks bytes) + * r2: nblks + */ + + cmp RNBLKS, #0; + beq .Ldo_nothing; + + push {r4-r12, lr}; + /*vpush {q4-q7};*/ + + adr RT3, .LK_VEC; + + mov ROLDSTACK, sp; + + /* Align stack. */ + sub RT0, sp, #(16*4); + and RT0, #(~(16-1)); + mov sp, RT0; + + vld1.32 {qK1-qK2}, [RT3]!; /* Load K1,K2 */ + + /* Get the values of the chaining variables. */ + ldm RSTATE, {_a-_e}; + + vld1.32 {qK3-qK4}, [RT3]; /* Load K3,K4 */ + +#undef curK +#define curK qK1 + /* Precalc 0-15. */ + W_PRECALC_00_15(); + +.Loop: + /* Transform 0-15 + Precalc 16-31. */ + _R( _a, _b, _c, _d, _e, F1, 0, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 1, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _d, _e, _a, _b, _c, F1, 2, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _c, _d, _e, _a, _b, F1, 3, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,16, + W4, W5, W6, W7, W0, _, _, _ ); + +#undef curK +#define curK qK2 + _R( _b, _c, _d, _e, _a, F1, 4, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 5, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 6, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _d, _e, _a, _b, _c, F1, 7, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,20, + W3, W4, W5, W6, W7, _, _, _ ); + + _R( _c, _d, _e, _a, _b, F1, 8, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _b, _c, _d, _e, _a, F1, 9, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 10, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 11, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,24, + W2, W3, W4, W5, W6, _, _, _ ); + + _R( _d, _e, _a, _b, _c, F1, 12, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _c, _d, _e, _a, _b, F1, 13, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _b, _c, _d, _e, _a, F1, 14, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 15, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,28, + W1, W2, W3, W4, W5, _, _, _ ); + + /* Transform 16-63 + Precalc 32-79. */ + _R( _e, _a, _b, _c, _d, F1, 16, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _d, _e, _a, _b, _c, F1, 17, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _c, _d, _e, _a, _b, F1, 18, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _b, _c, _d, _e, _a, F1, 19, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + + _R( _a, _b, _c, _d, _e, F2, 20, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _e, _a, _b, _c, _d, F2, 21, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _d, _e, _a, _b, _c, F2, 22, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _c, _d, _e, _a, _b, F2, 23, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + +#undef curK +#define curK qK3 + _R( _b, _c, _d, _e, _a, F2, 24, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _a, _b, _c, _d, _e, F2, 25, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _e, _a, _b, _c, _d, F2, 26, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _d, _e, _a, _b, _c, F2, 27, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + + _R( _c, _d, _e, _a, _b, F2, 28, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _b, _c, _d, _e, _a, F2, 29, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _a, _b, _c, _d, _e, F2, 30, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _e, _a, _b, _c, _d, F2, 31, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + + _R( _d, _e, _a, _b, _c, F2, 32, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _c, _d, _e, _a, _b, F2, 33, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _b, _c, _d, _e, _a, F2, 34, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _a, _b, _c, _d, _e, F2, 35, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + + _R( _e, _a, _b, _c, _d, F2, 36, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _d, _e, _a, _b, _c, F2, 37, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _c, _d, _e, _a, _b, F2, 38, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _b, _c, _d, _e, _a, F2, 39, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + + _R( _a, _b, _c, _d, _e, F3, 40, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _e, _a, _b, _c, _d, F3, 41, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _d, _e, _a, _b, _c, F3, 42, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _c, _d, _e, _a, _b, F3, 43, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + +#undef curK +#define curK qK4 + _R( _b, _c, _d, _e, _a, F3, 44, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _a, _b, _c, _d, _e, F3, 45, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _e, _a, _b, _c, _d, F3, 46, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _d, _e, _a, _b, _c, F3, 47, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + + _R( _c, _d, _e, _a, _b, F3, 48, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _b, _c, _d, _e, _a, F3, 49, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _a, _b, _c, _d, _e, F3, 50, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _e, _a, _b, _c, _d, F3, 51, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + + _R( _d, _e, _a, _b, _c, F3, 52, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _c, _d, _e, _a, _b, F3, 53, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _b, _c, _d, _e, _a, F3, 54, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _a, _b, _c, _d, _e, F3, 55, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + + _R( _e, _a, _b, _c, _d, F3, 56, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _d, _e, _a, _b, _c, F3, 57, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _c, _d, _e, _a, _b, F3, 58, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _b, _c, _d, _e, _a, F3, 59, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + + subs RNBLKS, #1; + + _R( _a, _b, _c, _d, _e, F4, 60, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _e, _a, _b, _c, _d, F4, 61, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _d, _e, _a, _b, _c, F4, 62, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _c, _d, _e, _a, _b, F4, 63, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + + beq .Lend; + + /* Transform 64-79 + Precalc 0-15 of next block. */ +#undef curK +#define curK qK1 + _R( _b, _c, _d, _e, _a, F4, 64, + WPRECALC_00_15_0, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 65, + WPRECALC_00_15_1, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _e, _a, _b, _c, _d, F4, 66, + WPRECALC_00_15_2, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _d, _e, _a, _b, _c, F4, 67, + WPRECALC_00_15_3, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _c, _d, _e, _a, _b, F4, 68, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 69, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 70, + WPRECALC_00_15_4, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _e, _a, _b, _c, _d, F4, 71, + WPRECALC_00_15_5, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _d, _e, _a, _b, _c, F4, 72, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _c, _d, _e, _a, _b, F4, 73, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 74, + WPRECALC_00_15_6, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 75, + WPRECALC_00_15_7, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _e, _a, _b, _c, _d, F4, 76, + WPRECALC_00_15_8, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _d, _e, _a, _b, _c, F4, 77, + WPRECALC_00_15_9, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _c, _d, _e, _a, _b, F4, 78, + WPRECALC_00_15_10, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 79, + WPRECALC_00_15_11, dummy, WPRECALC_00_15_12, _, _, _, _, _, _, _, _, _ ); + + /* Update the chaining variables. */ + ldm RSTATE, {RT0-RT3}; + add _a, RT0; + ldr RT0, [RSTATE, #state_h4]; + add _b, RT1; + add _c, RT2; + add _d, RT3; + add _e, RT0; + stm RSTATE, {_a-_e}; + + b .Loop; + +.Lend: + /* Transform 64-79 */ + R( _b, _c, _d, _e, _a, F4, 64 ); + R( _a, _b, _c, _d, _e, F4, 65 ); + R( _e, _a, _b, _c, _d, F4, 66 ); + R( _d, _e, _a, _b, _c, F4, 67 ); + R( _c, _d, _e, _a, _b, F4, 68 ); + R( _b, _c, _d, _e, _a, F4, 69 ); + R( _a, _b, _c, _d, _e, F4, 70 ); + R( _e, _a, _b, _c, _d, F4, 71 ); + R( _d, _e, _a, _b, _c, F4, 72 ); + R( _c, _d, _e, _a, _b, F4, 73 ); + R( _b, _c, _d, _e, _a, F4, 74 ); + R( _a, _b, _c, _d, _e, F4, 75 ); + R( _e, _a, _b, _c, _d, F4, 76 ); + R( _d, _e, _a, _b, _c, F4, 77 ); + R( _c, _d, _e, _a, _b, F4, 78 ); + R( _b, _c, _d, _e, _a, F4, 79 ); + + mov sp, ROLDSTACK; + + /* Update the chaining variables. */ + ldm RSTATE, {RT0-RT3}; + add _a, RT0; + ldr RT0, [RSTATE, #state_h4]; + add _b, RT1; + add _c, RT2; + add _d, RT3; + /*vpop {q4-q7};*/ + add _e, RT0; + stm RSTATE, {_a-_e}; + + pop {r4-r12, pc}; + +.Ldo_nothing: + bx lr +ENDPROC(sha1_transform_neon) diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c new file mode 100644 index 00000000000..84f2a756588 --- /dev/null +++ b/arch/arm/crypto/sha1_glue.c @@ -0,0 +1,175 @@ +/* + * Cryptographic API. + * Glue code for the SHA1 Secure Hash Algorithm assembler implementation + * + * This file is based on sha1_generic.c and sha1_ssse3_glue.c + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright (c) Mathias Krause + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + + +asmlinkage void sha1_block_data_order(u32 *digest, + const unsigned char *data, unsigned int rounds); + + +static int sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + + +static int __sha1_update(struct sha1_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) +{ + unsigned int done = 0; + + sctx->count += len; + + if (partial) { + done = SHA1_BLOCK_SIZE - partial; + memcpy(sctx->buffer + partial, data, done); + sha1_block_data_order(sctx->state, sctx->buffer, 1); + } + + if (len - done >= SHA1_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; + sha1_block_data_order(sctx->state, data + done, rounds); + done += rounds * SHA1_BLOCK_SIZE; + } + + memcpy(sctx->buffer, data + done, len - done); + return 0; +} + + +int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA1_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buffer + partial, data, len); + return 0; + } + res = __sha1_update(sctx, data, len, partial); + return res; +} +EXPORT_SYMBOL_GPL(sha1_update_arm); + + +/* Add padding and return the message digest. */ +static int sha1_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA1_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); + /* We need to fill a whole block for __sha1_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buffer + index, padding, padlen); + } else { + __sha1_update(sctx, padding, padlen, index); + } + __sha1_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + return 0; +} + + +static int sha1_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + + +static int sha1_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = sha1_init, + .update = sha1_update_arm, + .final = sha1_final, + .export = sha1_export, + .import = sha1_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name= "sha1-asm", + .cra_priority = 150, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + + +static int __init sha1_mod_init(void) +{ + return crypto_register_shash(&alg); +} + + +static void __exit sha1_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + + +module_init(sha1_mod_init); +module_exit(sha1_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (ARM)"); +MODULE_ALIAS("sha1"); +MODULE_AUTHOR("David McCullough "); diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c new file mode 100644 index 00000000000..6f1b411b1d5 --- /dev/null +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -0,0 +1,197 @@ +/* + * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using + * ARM NEON instructions. + * + * Copyright © 2014 Jussi Kivilinna + * + * This file is based on sha1_generic.c and sha1_ssse3_glue.c: + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright (c) Mathias Krause + * Copyright (c) Chandramouli Narayanan + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +asmlinkage void sha1_transform_neon(void *state_h, const char *data, + unsigned int rounds); + + +static int sha1_neon_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int __sha1_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count += len; + + if (partial) { + done = SHA1_BLOCK_SIZE - partial; + memcpy(sctx->buffer + partial, data, done); + sha1_transform_neon(sctx->state, sctx->buffer, 1); + } + + if (len - done >= SHA1_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; + + sha1_transform_neon(sctx->state, data + done, rounds); + done += rounds * SHA1_BLOCK_SIZE; + } + + memcpy(sctx->buffer, data + done, len - done); + + return 0; +} + +static int sha1_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA1_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buffer + partial, data, len); + + return 0; + } + + if (!may_use_simd()) { + res = sha1_update_arm(desc, data, len); + } else { + kernel_neon_begin(); + res = __sha1_neon_update(desc, data, len, partial); + kernel_neon_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha1_neon_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA1_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); + if (!may_use_simd()) { + sha1_update_arm(desc, padding, padlen); + sha1_update_arm(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_neon_begin(); + /* We need to fill a whole block for __sha1_neon_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buffer + index, padding, padlen); + } else { + __sha1_neon_update(desc, padding, padlen, index); + } + __sha1_neon_update(desc, (const u8 *)&bits, sizeof(bits), 56); + kernel_neon_end(); + } + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha1_neon_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha1_neon_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = sha1_neon_init, + .update = sha1_neon_update, + .final = sha1_neon_final, + .export = sha1_neon_export, + .import = sha1_neon_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha1_neon_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_shash(&alg); +} + +static void __exit sha1_neon_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha1_neon_mod_init); +module_exit(sha1_neon_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, NEON accelerated"); +MODULE_ALIAS("sha1"); diff --git a/arch/arm/crypto/sha512-armv7-neon.S b/arch/arm/crypto/sha512-armv7-neon.S new file mode 100644 index 00000000000..fe99472e507 --- /dev/null +++ b/arch/arm/crypto/sha512-armv7-neon.S @@ -0,0 +1,455 @@ +/* sha512-armv7-neon.S - ARM/NEON assembly implementation of SHA-512 transform + * + * Copyright © 2013-2014 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include + + +.syntax unified +.code 32 +.fpu neon + +.text + +/* structure of SHA512_CONTEXT */ +#define hd_a 0 +#define hd_b ((hd_a) + 8) +#define hd_c ((hd_b) + 8) +#define hd_d ((hd_c) + 8) +#define hd_e ((hd_d) + 8) +#define hd_f ((hd_e) + 8) +#define hd_g ((hd_f) + 8) + +/* register macros */ +#define RK %r2 + +#define RA d0 +#define RB d1 +#define RC d2 +#define RD d3 +#define RE d4 +#define RF d5 +#define RG d6 +#define RH d7 + +#define RT0 d8 +#define RT1 d9 +#define RT2 d10 +#define RT3 d11 +#define RT4 d12 +#define RT5 d13 +#define RT6 d14 +#define RT7 d15 + +#define RT01q q4 +#define RT23q q5 +#define RT45q q6 +#define RT67q q7 + +#define RW0 d16 +#define RW1 d17 +#define RW2 d18 +#define RW3 d19 +#define RW4 d20 +#define RW5 d21 +#define RW6 d22 +#define RW7 d23 +#define RW8 d24 +#define RW9 d25 +#define RW10 d26 +#define RW11 d27 +#define RW12 d28 +#define RW13 d29 +#define RW14 d30 +#define RW15 d31 + +#define RW01q q8 +#define RW23q q9 +#define RW45q q10 +#define RW67q q11 +#define RW89q q12 +#define RW1011q q13 +#define RW1213q q14 +#define RW1415q q15 + +/*********************************************************************** + * ARM assembly implementation of sha512 transform + ***********************************************************************/ +#define rounds2_0_63(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, rw01q, rw2, \ + rw23q, rw1415q, rw9, rw10, interleave_op, arg1) \ + /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ + vshr.u64 RT2, re, #14; \ + vshl.u64 RT3, re, #64 - 14; \ + interleave_op(arg1); \ + vshr.u64 RT4, re, #18; \ + vshl.u64 RT5, re, #64 - 18; \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, re, #41; \ + vshl.u64 RT5, re, #64 - 41; \ + vadd.u64 RT0, RT0, rw0; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, re; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, rf, rg; \ + \ + vadd.u64 RT1, RT1, rh; \ + vshr.u64 RT2, ra, #28; \ + vshl.u64 RT3, ra, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, ra, #34; \ + vshl.u64 RT5, ra, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* h = Sum0 (a) + Maj (a, b, c); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, ra, #39; \ + vshl.u64 RT5, ra, #64 - 39; \ + veor.64 RT0, ra, rb; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rc, rb; \ + vadd.u64 rd, rd, RT1; /* d+=t1; */ \ + veor.64 rh, RT2, RT3; \ + \ + /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ + vshr.u64 RT2, rd, #14; \ + vshl.u64 RT3, rd, #64 - 14; \ + vadd.u64 rh, rh, RT0; \ + vshr.u64 RT4, rd, #18; \ + vshl.u64 RT5, rd, #64 - 18; \ + vadd.u64 rh, rh, RT1; /* h+=t1; */ \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rd, #41; \ + vshl.u64 RT5, rd, #64 - 41; \ + vadd.u64 RT0, RT0, rw1; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, rd; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, re, rf; \ + \ + vadd.u64 RT1, RT1, rg; \ + vshr.u64 RT2, rh, #28; \ + vshl.u64 RT3, rh, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, rh, #34; \ + vshl.u64 RT5, rh, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* g = Sum0 (h) + Maj (h, a, b); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rh, #39; \ + vshl.u64 RT5, rh, #64 - 39; \ + veor.64 RT0, rh, ra; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rb, ra; \ + vadd.u64 rc, rc, RT1; /* c+=t1; */ \ + veor.64 rg, RT2, RT3; \ + \ + /* w[0] += S1 (w[14]) + w[9] + S0 (w[1]); */ \ + /* w[1] += S1 (w[15]) + w[10] + S0 (w[2]); */ \ + \ + /**** S0(w[1:2]) */ \ + \ + /* w[0:1] += w[9:10] */ \ + /* RT23q = rw1:rw2 */ \ + vext.u64 RT23q, rw01q, rw23q, #1; \ + vadd.u64 rw0, rw9; \ + vadd.u64 rg, rg, RT0; \ + vadd.u64 rw1, rw10;\ + vadd.u64 rg, rg, RT1; /* g+=t1; */ \ + \ + vshr.u64 RT45q, RT23q, #1; \ + vshl.u64 RT67q, RT23q, #64 - 1; \ + vshr.u64 RT01q, RT23q, #8; \ + veor.u64 RT45q, RT45q, RT67q; \ + vshl.u64 RT67q, RT23q, #64 - 8; \ + veor.u64 RT45q, RT45q, RT01q; \ + vshr.u64 RT01q, RT23q, #7; \ + veor.u64 RT45q, RT45q, RT67q; \ + \ + /**** S1(w[14:15]) */ \ + vshr.u64 RT23q, rw1415q, #6; \ + veor.u64 RT01q, RT01q, RT45q; \ + vshr.u64 RT45q, rw1415q, #19; \ + vshl.u64 RT67q, rw1415q, #64 - 19; \ + veor.u64 RT23q, RT23q, RT45q; \ + vshr.u64 RT45q, rw1415q, #61; \ + veor.u64 RT23q, RT23q, RT67q; \ + vshl.u64 RT67q, rw1415q, #64 - 61; \ + veor.u64 RT23q, RT23q, RT45q; \ + vadd.u64 rw01q, RT01q; /* w[0:1] += S(w[1:2]) */ \ + veor.u64 RT01q, RT23q, RT67q; +#define vadd_RT01q(rw01q) \ + /* w[0:1] += S(w[14:15]) */ \ + vadd.u64 rw01q, RT01q; + +#define dummy(_) /*_*/ + +#define rounds2_64_79(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, \ + interleave_op1, arg1, interleave_op2, arg2) \ + /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ + vshr.u64 RT2, re, #14; \ + vshl.u64 RT3, re, #64 - 14; \ + interleave_op1(arg1); \ + vshr.u64 RT4, re, #18; \ + vshl.u64 RT5, re, #64 - 18; \ + interleave_op2(arg2); \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, re, #41; \ + vshl.u64 RT5, re, #64 - 41; \ + vadd.u64 RT0, RT0, rw0; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, re; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, rf, rg; \ + \ + vadd.u64 RT1, RT1, rh; \ + vshr.u64 RT2, ra, #28; \ + vshl.u64 RT3, ra, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, ra, #34; \ + vshl.u64 RT5, ra, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* h = Sum0 (a) + Maj (a, b, c); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, ra, #39; \ + vshl.u64 RT5, ra, #64 - 39; \ + veor.64 RT0, ra, rb; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rc, rb; \ + vadd.u64 rd, rd, RT1; /* d+=t1; */ \ + veor.64 rh, RT2, RT3; \ + \ + /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ + vshr.u64 RT2, rd, #14; \ + vshl.u64 RT3, rd, #64 - 14; \ + vadd.u64 rh, rh, RT0; \ + vshr.u64 RT4, rd, #18; \ + vshl.u64 RT5, rd, #64 - 18; \ + vadd.u64 rh, rh, RT1; /* h+=t1; */ \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rd, #41; \ + vshl.u64 RT5, rd, #64 - 41; \ + vadd.u64 RT0, RT0, rw1; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, rd; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, re, rf; \ + \ + vadd.u64 RT1, RT1, rg; \ + vshr.u64 RT2, rh, #28; \ + vshl.u64 RT3, rh, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, rh, #34; \ + vshl.u64 RT5, rh, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* g = Sum0 (h) + Maj (h, a, b); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rh, #39; \ + vshl.u64 RT5, rh, #64 - 39; \ + veor.64 RT0, rh, ra; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rb, ra; \ + vadd.u64 rc, rc, RT1; /* c+=t1; */ \ + veor.64 rg, RT2, RT3; +#define vadd_rg_RT0(rg) \ + vadd.u64 rg, rg, RT0; +#define vadd_rg_RT1(rg) \ + vadd.u64 rg, rg, RT1; /* g+=t1; */ + +.align 3 +ENTRY(sha512_transform_neon) + /* Input: + * %r0: SHA512_CONTEXT + * %r1: data + * %r2: u64 k[] constants + * %r3: nblks + */ + push {%lr}; + + mov %lr, #0; + + /* Load context to d0-d7 */ + vld1.64 {RA-RD}, [%r0]!; + vld1.64 {RE-RH}, [%r0]; + sub %r0, #(4*8); + + /* Load input to w[16], d16-d31 */ + /* NOTE: Assumes that on ARMv7 unaligned accesses are always allowed. */ + vld1.64 {RW0-RW3}, [%r1]!; + vld1.64 {RW4-RW7}, [%r1]!; + vld1.64 {RW8-RW11}, [%r1]!; + vld1.64 {RW12-RW15}, [%r1]!; +#ifdef __ARMEL__ + /* byteswap */ + vrev64.8 RW01q, RW01q; + vrev64.8 RW23q, RW23q; + vrev64.8 RW45q, RW45q; + vrev64.8 RW67q, RW67q; + vrev64.8 RW89q, RW89q; + vrev64.8 RW1011q, RW1011q; + vrev64.8 RW1213q, RW1213q; + vrev64.8 RW1415q, RW1415q; +#endif + + /* EABI says that d8-d15 must be preserved by callee. */ + /*vpush {RT0-RT7};*/ + +.Loop: + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, + RW23q, RW1415q, RW9, RW10, dummy, _); + b .Lenter_rounds; + +.Loop_rounds: + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, + RW23q, RW1415q, RW9, RW10, vadd_RT01q, RW1415q); +.Lenter_rounds: + rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, RW23q, RW4, + RW45q, RW01q, RW11, RW12, vadd_RT01q, RW01q); + rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, RW45q, RW6, + RW67q, RW23q, RW13, RW14, vadd_RT01q, RW23q); + rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, RW67q, RW8, + RW89q, RW45q, RW15, RW0, vadd_RT01q, RW45q); + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, RW89q, RW10, + RW1011q, RW67q, RW1, RW2, vadd_RT01q, RW67q); + rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, RW1011q, RW12, + RW1213q, RW89q, RW3, RW4, vadd_RT01q, RW89q); + add %lr, #16; + rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, RW1213q, RW14, + RW1415q, RW1011q, RW5, RW6, vadd_RT01q, RW1011q); + cmp %lr, #64; + rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, RW1415q, RW0, + RW01q, RW1213q, RW7, RW8, vadd_RT01q, RW1213q); + bne .Loop_rounds; + + subs %r3, #1; + + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, + vadd_RT01q, RW1415q, dummy, _); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); + beq .Lhandle_tail; + vld1.64 {RW0-RW3}, [%r1]!; + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); +#ifdef __ARMEL__ + vrev64.8 RW01q, RW01q; + vrev64.8 RW23q, RW23q; +#endif + vld1.64 {RW4-RW7}, [%r1]!; + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, + vadd_rg_RT0, RA, vadd_rg_RT1, RA); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); +#ifdef __ARMEL__ + vrev64.8 RW45q, RW45q; + vrev64.8 RW67q, RW67q; +#endif + vld1.64 {RW8-RW11}, [%r1]!; + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); +#ifdef __ARMEL__ + vrev64.8 RW89q, RW89q; + vrev64.8 RW1011q, RW1011q; +#endif + vld1.64 {RW12-RW15}, [%r1]!; + vadd_rg_RT0(RA); + vadd_rg_RT1(RA); + + /* Load context */ + vld1.64 {RT0-RT3}, [%r0]!; + vld1.64 {RT4-RT7}, [%r0]; + sub %r0, #(4*8); + +#ifdef __ARMEL__ + vrev64.8 RW1213q, RW1213q; + vrev64.8 RW1415q, RW1415q; +#endif + + vadd.u64 RA, RT0; + vadd.u64 RB, RT1; + vadd.u64 RC, RT2; + vadd.u64 RD, RT3; + vadd.u64 RE, RT4; + vadd.u64 RF, RT5; + vadd.u64 RG, RT6; + vadd.u64 RH, RT7; + + /* Store the first half of context */ + vst1.64 {RA-RD}, [%r0]!; + sub RK, $(8*80); + vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ + mov %lr, #0; + sub %r0, #(4*8); + + b .Loop; + +.Lhandle_tail: + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, + vadd_rg_RT0, RA, vadd_rg_RT1, RA); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); + + /* Load context to d16-d23 */ + vld1.64 {RW0-RW3}, [%r0]!; + vadd_rg_RT0(RA); + vld1.64 {RW4-RW7}, [%r0]; + vadd_rg_RT1(RA); + sub %r0, #(4*8); + + vadd.u64 RA, RW0; + vadd.u64 RB, RW1; + vadd.u64 RC, RW2; + vadd.u64 RD, RW3; + vadd.u64 RE, RW4; + vadd.u64 RF, RW5; + vadd.u64 RG, RW6; + vadd.u64 RH, RW7; + + /* Store the first half of context */ + vst1.64 {RA-RD}, [%r0]!; + + /* Clear used registers */ + /* d16-d31 */ + veor.u64 RW01q, RW01q; + veor.u64 RW23q, RW23q; + veor.u64 RW45q, RW45q; + veor.u64 RW67q, RW67q; + vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ + veor.u64 RW89q, RW89q; + veor.u64 RW1011q, RW1011q; + veor.u64 RW1213q, RW1213q; + veor.u64 RW1415q, RW1415q; + /* d8-d15 */ + /*vpop {RT0-RT7};*/ + /* d0-d7 (q0-q3) */ + veor.u64 %q0, %q0; + veor.u64 %q1, %q1; + veor.u64 %q2, %q2; + veor.u64 %q3, %q3; + + pop {%pc}; +ENDPROC(sha512_transform_neon) diff --git a/arch/arm/crypto/sha512_neon_glue.c b/arch/arm/crypto/sha512_neon_glue.c new file mode 100644 index 00000000000..0d2758ff5e1 --- /dev/null +++ b/arch/arm/crypto/sha512_neon_glue.c @@ -0,0 +1,305 @@ +/* + * Glue code for the SHA512 Secure Hash Algorithm assembly implementation + * using NEON instructions. + * + * Copyright © 2014 Jussi Kivilinna + * + * This file is based on sha512_ssse3_glue.c: + * Copyright (C) 2013 Intel Corporation + * Author: Tim Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static const u64 sha512_k[] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL +}; + + +asmlinkage void sha512_transform_neon(u64 *digest, const void *data, + const u64 k[], unsigned int num_blks); + + +static int sha512_neon_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA512_H0; + sctx->state[1] = SHA512_H1; + sctx->state[2] = SHA512_H2; + sctx->state[3] = SHA512_H3; + sctx->state[4] = SHA512_H4; + sctx->state[5] = SHA512_H5; + sctx->state[6] = SHA512_H6; + sctx->state[7] = SHA512_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int __sha512_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + + if (partial) { + done = SHA512_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha512_transform_neon(sctx->state, sctx->buf, sha512_k, 1); + } + + if (len - done >= SHA512_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; + + sha512_transform_neon(sctx->state, data + done, sha512_k, + rounds); + + done += rounds * SHA512_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); + + return 0; +} + +static int sha512_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA512_BLOCK_SIZE) { + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + memcpy(sctx->buf + partial, data, len); + + return 0; + } + + if (!may_use_simd()) { + res = crypto_sha512_update(desc, data, len); + } else { + kernel_neon_begin(); + res = __sha512_neon_update(desc, data, len, partial); + kernel_neon_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha512_neon_final(struct shash_desc *desc, u8 *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be64 *dst = (__be64 *)out; + __be64 bits[2]; + static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; + + /* save number of bits */ + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); + + /* Pad out to 112 mod 128 and append length */ + index = sctx->count[0] & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128+112) - index); + + if (!may_use_simd()) { + crypto_sha512_update(desc, padding, padlen); + crypto_sha512_update(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_neon_begin(); + /* We need to fill a whole block for __sha512_neon_update() */ + if (padlen <= 112) { + sctx->count[0] += padlen; + if (sctx->count[0] < padlen) + sctx->count[1]++; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha512_neon_update(desc, padding, padlen, index); + } + __sha512_neon_update(desc, (const u8 *)&bits, + sizeof(bits), 112); + kernel_neon_end(); + } + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be64(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha512_neon_export(struct shash_desc *desc, void *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha512_neon_import(struct shash_desc *desc, const void *in) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static int sha384_neon_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA384_H0; + sctx->state[1] = SHA384_H1; + sctx->state[2] = SHA384_H2; + sctx->state[3] = SHA384_H3; + sctx->state[4] = SHA384_H4; + sctx->state[5] = SHA384_H5; + sctx->state[6] = SHA384_H6; + sctx->state[7] = SHA384_H7; + + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int sha384_neon_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[SHA512_DIGEST_SIZE]; + + sha512_neon_final(desc, D); + + memcpy(hash, D, SHA384_DIGEST_SIZE); + memset(D, 0, SHA512_DIGEST_SIZE); + + return 0; +} + +static struct shash_alg algs[] = { { + .digestsize = SHA512_DIGEST_SIZE, + .init = sha512_neon_init, + .update = sha512_neon_update, + .final = sha512_neon_final, + .export = sha512_neon_export, + .import = sha512_neon_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA384_DIGEST_SIZE, + .init = sha384_neon_init, + .update = sha512_neon_update, + .final = sha384_neon_final, + .export = sha512_neon_export, + .import = sha512_neon_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init sha512_neon_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit sha512_neon_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_init(sha512_neon_mod_init); +module_exit(sha512_neon_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, NEON accelerated"); + +MODULE_ALIAS("sha512"); +MODULE_ALIAS("sha384"); diff --git a/arch/arm/include/asm/crypto/sha1.h b/arch/arm/include/asm/crypto/sha1.h new file mode 100644 index 00000000000..75e6a417416 --- /dev/null +++ b/arch/arm/include/asm/crypto/sha1.h @@ -0,0 +1,10 @@ +#ifndef ASM_ARM_CRYPTO_SHA1_H +#define ASM_ARM_CRYPTO_SHA1_H + +#include +#include + +extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len); + +#endif diff --git a/arch/arm/include/asm/syscall.h b/arch/arm/include/asm/syscall.h new file mode 100644 index 00000000000..4c123dbe281 --- /dev/null +++ b/arch/arm/include/asm/syscall.h @@ -0,0 +1,80 @@ +/* + * Access to user system call parameters and results + * + * Copyright (C) 2012 The Chromium OS Authors + * + * This copyrighted material is made available to anyone wishing to use, + * modify, copy, or redistribute it subject to the terms and conditions + * of the GNU General Public License v.2. + * + * See asm-generic/syscall.h for descriptions of what we must do here. + */ +#ifndef _ASM_ARM_SYSCALL_H +#define _ASM_ARM_SYSCALL_H + +#include /* for AUDIT_ARCH_* */ +#include /* for ELF_EM */ +#include +#include /* for task_thread_info */ +#include + +static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) +{ + return task_thread_info(task)->syscall; +} + +static inline void syscall_rollback(struct task_struct *task, + struct pt_regs *regs) +{ + regs->ARM_r0 = regs->ARM_ORIG_r0; +} + +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + unsigned long error = regs->ARM_r0; + return IS_ERR_VALUE(error) ? error : 0; +} + +static inline long syscall_get_return_value(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->ARM_r0; +} + +static inline void syscall_set_return_value(struct task_struct *task, + struct pt_regs *regs, + int error, long val) +{ + regs->ARM_r0 = (long) error ?: val; +} + +static inline void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned int i, unsigned int n, + unsigned long *args) +{ + BUG_ON(i + n > 6); + memcpy(args, ®s->ARM_r0 + i, n * sizeof(args[0])); +} + +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned int i, unsigned int n, + const unsigned long *args) +{ + BUG_ON(i + n > 6); + memcpy(®s->ARM_r0 + i, args, n * sizeof(args[0])); +} + +static inline int syscall_get_arch(struct task_struct *task, + struct pt_regs *regs) +{ + /* ARM tasks don't change audit architectures on the fly. */ +#ifdef __ARMEB__ + return AUDIT_ARCH_ARMEB; +#else + return AUDIT_ARCH_ARM; +#endif +} +#endif /* _ASM_ARM_SYSCALL_H */ diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 67d644334b0..2eb0c2c298e 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -58,7 +58,7 @@ struct thread_info { struct cpu_context_save cpu_context; /* cpu context */ __u32 syscall; /* syscall number */ __u8 used_cp[16]; /* thread used copro */ - unsigned long tp_value; + unsigned long tp_value[2]; /* TLS registers */ struct crunch_state crunchstate; union fp_state fpstate __attribute__((aligned(8))); union vfp_state vfpstate; diff --git a/arch/arm/include/asm/tls.h b/arch/arm/include/asm/tls.h index 73409e6c025..83259b87333 100644 --- a/arch/arm/include/asm/tls.h +++ b/arch/arm/include/asm/tls.h @@ -2,27 +2,30 @@ #define __ASMARM_TLS_H #ifdef __ASSEMBLY__ - .macro set_tls_none, tp, tmp1, tmp2 +#include + .macro switch_tls_none, base, tp, tpuser, tmp1, tmp2 .endm - .macro set_tls_v6k, tp, tmp1, tmp2 + .macro switch_tls_v6k, base, tp, tpuser, tmp1, tmp2 + mrc p15, 0, \tmp2, c13, c0, 2 @ get the user r/w register mcr p15, 0, \tp, c13, c0, 3 @ set TLS register - mov \tmp1, #0 - mcr p15, 0, \tmp1, c13, c0, 2 @ clear user r/w TLS register + mcr p15, 0, \tpuser, c13, c0, 2 @ and the user r/w register + str \tmp2, [\base, #TI_TP_VALUE + 4] @ save it .endm - .macro set_tls_v6, tp, tmp1, tmp2 + .macro switch_tls_v6, base, tp, tpuser, tmp1, tmp2 ldr \tmp1, =elf_hwcap ldr \tmp1, [\tmp1, #0] mov \tmp2, #0xffff0fff tst \tmp1, #HWCAP_TLS @ hardware TLS available? - mcrne p15, 0, \tp, c13, c0, 3 @ yes, set TLS register - movne \tmp1, #0 - mcrne p15, 0, \tmp1, c13, c0, 2 @ clear user r/w TLS register streq \tp, [\tmp2, #-15] @ set TLS value at 0xffff0ff0 + mrcne p15, 0, \tmp2, c13, c0, 2 @ get the user r/w register + mcrne p15, 0, \tp, c13, c0, 3 @ yes, set TLS register + mcrne p15, 0, \tpuser, c13, c0, 2 @ set user r/w register + strne \tmp2, [\base, #TI_TP_VALUE + 4] @ save it .endm - .macro set_tls_software, tp, tmp1, tmp2 + .macro switch_tls_software, base, tp, tpuser, tmp1, tmp2 mov \tmp1, #0xffff0fff str \tp, [\tmp1, #-15] @ set TLS value at 0xffff0ff0 .endm @@ -31,19 +34,30 @@ #ifdef CONFIG_TLS_REG_EMUL #define tls_emu 1 #define has_tls_reg 1 -#define set_tls set_tls_none +#define switch_tls switch_tls_none #elif defined(CONFIG_CPU_V6) #define tls_emu 0 #define has_tls_reg (elf_hwcap & HWCAP_TLS) -#define set_tls set_tls_v6 +#define switch_tls switch_tls_v6 #elif defined(CONFIG_CPU_32v6K) #define tls_emu 0 #define has_tls_reg 1 -#define set_tls set_tls_v6k +#define switch_tls switch_tls_v6k #else #define tls_emu 0 #define has_tls_reg 0 -#define set_tls set_tls_software +#define switch_tls switch_tls_software #endif +#ifndef __ASSEMBLY__ +static inline unsigned long get_tpuser(void) +{ + unsigned long reg = 0; + + if (has_tls_reg && !tls_emu) + __asm__("mrc p15, 0, %0, c13, c0, 2" : "=r" (reg)); + + return reg; +} +#endif #endif /* __ASMARM_TLS_H */ diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index 512cd147345..6ef9635a174 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -404,6 +404,7 @@ #define __NR_setns (__NR_SYSCALL_BASE+375) #define __NR_process_vm_readv (__NR_SYSCALL_BASE+376) #define __NR_process_vm_writev (__NR_SYSCALL_BASE+377) +#define __NR_seccomp (__NR_SYSCALL_BASE+383) /* * The following SWIs are ARM private. diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S index 463ff4a0ec8..d01eb013b0a 100644 --- a/arch/arm/kernel/calls.S +++ b/arch/arm/kernel/calls.S @@ -387,6 +387,12 @@ /* 375 */ CALL(sys_setns) CALL(sys_process_vm_readv) CALL(sys_process_vm_writev) + CALL(sys_ni_syscall) + CALL(sys_ni_syscall) +/* 380 */ CALL(sys_ni_syscall) + CALL(sys_ni_syscall) + CALL(sys_ni_syscall) + CALL(sys_seccomp) #ifndef syscalls_counted .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls #define syscalls_counted diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 9f5294027e5..a3d2004626e 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -698,15 +698,16 @@ ENTRY(__switch_to) UNWIND(.fnstart ) UNWIND(.cantunwind ) add ip, r1, #TI_CPU_SAVE - ldr r3, [r2, #TI_TP_VALUE] ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack THUMB( str sp, [ip], #4 ) THUMB( str lr, [ip], #4 ) + ldr r4, [r2, #TI_TP_VALUE] + ldr r5, [r2, #TI_TP_VALUE + 4] #ifdef CONFIG_CPU_USE_DOMAINS ldr r6, [r2, #TI_CPU_DOMAIN] #endif - set_tls r3, r4, r5 + switch_tls r1, r4, r5, r3, r7 #if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) ldr r7, [r2, #TI_TASK] ldr r8, =__stack_chk_guard diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 54ee265dd81..e4633311325 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -444,12 +444,7 @@ ENTRY(vector_swi) #ifdef CONFIG_SECCOMP tst r10, #_TIF_SECCOMP - beq 1f - mov r0, scno - bl __secure_computing - add r0, sp, #S_R0 + S_OFF @ pointer to regs - ldmia r0, {r0 - r3} @ have to reload r0 - r3 -1: + bne __sys_trace #endif tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls? diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index 8a6075cc0e4..b1777fa8405 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -311,6 +311,7 @@ armpmu_add(struct perf_event *event, int flags) pr_err("Event: %llx failed constraint check.\n", event->attr.config); event->state = PERF_EVENT_STATE_OFF; + err = -EPERM; goto out; } diff --git a/arch/arm/kernel/perf_event_msm_krait.c b/arch/arm/kernel/perf_event_msm_krait.c index 1fb5fd320b4..f514b689d8a 100644 --- a/arch/arm/kernel/perf_event_msm_krait.c +++ b/arch/arm/kernel/perf_event_msm_krait.c @@ -219,9 +219,6 @@ static unsigned int get_krait_evtinfo(unsigned int krait_evt_type, code = (krait_evt_type & 0x00FF0) >> 4; group = krait_evt_type & 0x0000F; - if ((group > 3) || (reg > krait_max_l1_reg)) - return -EINVAL; - if (prefix != KRAIT_EVT_PREFIX && prefix != KRAIT_VENUMEVT_PREFIX) return -EINVAL; @@ -232,6 +229,9 @@ static unsigned int get_krait_evtinfo(unsigned int krait_evt_type, reg += VENUM_BASE_OFFSET; } + if ((group > 3) || (reg > krait_max_l1_reg)) + return -EINVAL; + evtinfo->group_setval = 0x80000000 | (code << (group * 8)); evtinfo->groupcode = reg; evtinfo->armv7_evt_type = evt_type_base[evt_index][reg] | group; diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index fe97ff28cc0..ea3b4d8109a 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -38,6 +38,7 @@ #include #include #include +#include #ifdef CONFIG_CC_STACKPROTECTOR #include @@ -534,7 +535,8 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, clear_ptrace_hw_breakpoint(p); if (clone_flags & CLONE_SETTLS) - thread->tp_value = regs->ARM_r3; + thread->tp_value[0] = childregs->ARM_r3; + thread->tp_value[1] = get_tpuser(); thread_notify(THREAD_NOTIFY_COPY, thread); diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 9650c143afc..032919ebfdd 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -844,7 +844,7 @@ long arch_ptrace(struct task_struct *child, long request, #endif case PTRACE_GET_THREAD_AREA: - ret = put_user(task_thread_info(child)->tp_value, + ret = put_user(task_thread_info(child)->tp_value[0], datap); break; @@ -909,20 +909,22 @@ long arch_ptrace(struct task_struct *child, long request, asmlinkage int syscall_trace(int why, struct pt_regs *regs, int scno) { unsigned long ip; + current_thread_info()->syscall = scno; if (why) audit_syscall_exit(regs); - else + else { + if (secure_computing(scno) == -1) + return -1; audit_syscall_entry(AUDIT_ARCH_ARM, scno, regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3); + } if (!test_thread_flag(TIF_SYSCALL_TRACE)) return scno; if (!(current->ptrace & PT_PTRACED)) return scno; - current_thread_info()->syscall = scno; - /* * IP is used to denote syscall entry/exit: * IP = 0 -> entry, =1 -> exit diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index af0aaebf4de..f8f32da6f4c 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -193,52 +193,53 @@ struct oabi_flock64 { pid_t l_pid; } __attribute__ ((packed,aligned(4))); -asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, +static long do_locks(unsigned int fd, unsigned int cmd, unsigned long arg) { - struct oabi_flock64 user; struct flock64 kernel; - mm_segment_t fs = USER_DS; /* initialized to kill a warning */ - unsigned long local_arg = arg; - int ret; + struct oabi_flock64 user; + mm_segment_t fs; + long ret; - switch (cmd) { - case F_GETLK64: - case F_SETLK64: - case F_SETLKW64: - if (copy_from_user(&user, (struct oabi_flock64 __user *)arg, - sizeof(user))) - return -EFAULT; - kernel.l_type = user.l_type; - kernel.l_whence = user.l_whence; - kernel.l_start = user.l_start; - kernel.l_len = user.l_len; - kernel.l_pid = user.l_pid; - local_arg = (unsigned long)&kernel; - fs = get_fs(); - set_fs(KERNEL_DS); - } + if (copy_from_user(&user, (struct oabi_flock64 __user *)arg, + sizeof(user))) + return -EFAULT; + kernel.l_type = user.l_type; + kernel.l_whence = user.l_whence; + kernel.l_start = user.l_start; + kernel.l_len = user.l_len; + kernel.l_pid = user.l_pid; - ret = sys_fcntl64(fd, cmd, local_arg); + fs = get_fs(); + set_fs(KERNEL_DS); + ret = sys_fcntl64(fd, cmd, (unsigned long)&kernel); + set_fs(fs); + if (!ret && (cmd == F_GETLK64 || cmd == F_OFD_GETLK)) { + user.l_type = kernel.l_type; + user.l_whence = kernel.l_whence; + user.l_start = kernel.l_start; + user.l_len = kernel.l_len; + user.l_pid = kernel.l_pid; + if (copy_to_user((struct oabi_flock64 __user *)arg, + &user, sizeof(user))) + ret = -EFAULT; + } + return ret; +} + +asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, + unsigned long arg) +{ switch (cmd) { case F_GETLK64: - if (!ret) { - user.l_type = kernel.l_type; - user.l_whence = kernel.l_whence; - user.l_start = kernel.l_start; - user.l_len = kernel.l_len; - user.l_pid = kernel.l_pid; - if (copy_to_user((struct oabi_flock64 __user *)arg, - &user, sizeof(user))) - ret = -EFAULT; - } case F_SETLK64: case F_SETLKW64: - set_fs(fs); - } + return do_locks(fd, cmd, arg); - return ret; + default: + return sys_fcntl64(fd, cmd, arg); + } } struct oabi_epoll_event { @@ -275,8 +276,12 @@ asmlinkage long sys_oabi_epoll_wait(int epfd, mm_segment_t fs; long ret, err, i; - if (maxevents <= 0 || maxevents > (INT_MAX/sizeof(struct epoll_event))) + if (maxevents <= 0 || + maxevents > (INT_MAX/sizeof(*kbuf)) || + maxevents > (INT_MAX/sizeof(*events))) return -EINVAL; + if (!access_ok(VERIFY_WRITE, events, sizeof(*events) * maxevents)) + return -EFAULT; kbuf = kmalloc(sizeof(*kbuf) * maxevents, GFP_KERNEL); if (!kbuf) return -ENOMEM; @@ -313,6 +318,8 @@ asmlinkage long sys_oabi_semtimedop(int semid, if (nsops < 1 || nsops > SEMOPM) return -EINVAL; + if (!access_ok(VERIFY_READ, tsops, sizeof(*tsops) * nsops)) + return -EFAULT; sops = kmalloc(sizeof(*sops) * nsops, GFP_KERNEL); if (!sops) return -ENOMEM; diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 1e9504b1eb4..8a88773ca89 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -517,6 +517,10 @@ asmlinkage int arm_syscall(int no, struct pt_regs *regs) struct thread_info *thread = current_thread_info(); siginfo_t info; + /* Emulate/fallthrough. */ + if (no == -1) + return regs->ARM_r0; + if ((no >> 16) != (__ARM_NR_BASE>> 16)) return bad_syscall(no, regs); @@ -566,7 +570,7 @@ asmlinkage int arm_syscall(int no, struct pt_regs *regs) return regs->ARM_r0; case NR(set_tls): - thread->tp_value = regs->ARM_r0; + thread->tp_value[0] = regs->ARM_r0; if (tls_emu) return 0; if (has_tls_reg) { @@ -684,7 +688,7 @@ static int get_tp_trap(struct pt_regs *regs, unsigned int instr) int reg = (instr >> 12) & 15; if (reg == 15) return 1; - regs->uregs[reg] = current_thread_info()->tp_value; + regs->uregs[reg] = current_thread_info()->tp_value[0]; regs->ARM_pc += 4; return 0; } diff --git a/arch/arm/mach-msm/Kconfig b/arch/arm/mach-msm/Kconfig index ba5a33c5884..5066ff37903 100644 --- a/arch/arm/mach-msm/Kconfig +++ b/arch/arm/mach-msm/Kconfig @@ -281,7 +281,6 @@ config ARCH_MSM8974 select MSM_RPM_STATS_LOG select QMI_ENCDEC select DONT_MAP_HOLE_AFTER_MEMBANK0 - select MSM_ULTRASOUND_B select MSM_LPM_TEST select MSM_RPM_LOG select ARCH_WANT_KMAP_ATOMIC_FLUSH diff --git a/arch/arm/mach-msm/devices-8064.c b/arch/arm/mach-msm/devices-8064.c index 4daccb1fd4b..2fd1bf30bde 100644 --- a/arch/arm/mach-msm/devices-8064.c +++ b/arch/arm/mach-msm/devices-8064.c @@ -2634,11 +2634,12 @@ static char *master_names[] = { static struct msm_rpm_master_stats_platform_data msm_rpm_master_stat_pdata = { .masters = master_names, - .nomasters = ARRAY_SIZE(master_names), + .num_masters = ARRAY_SIZE(master_names), + .master_offset = 32, }; struct platform_device apq8064_rpm_master_stat_device = { - .name = "msm_rpm_master_stat", + .name = "msm_rpm_master_stats", .id = -1, .num_resources = ARRAY_SIZE(resources_rpm_master_stats), .resource = resources_rpm_master_stats, diff --git a/arch/arm/mach-msm/devices-8930.c b/arch/arm/mach-msm/devices-8930.c index e2a57f911fa..c4fe0df6837 100644 --- a/arch/arm/mach-msm/devices-8930.c +++ b/arch/arm/mach-msm/devices-8930.c @@ -610,11 +610,12 @@ static char *master_names[] = { static struct msm_rpm_master_stats_platform_data msm_rpm_master_stat_pdata = { .masters = master_names, - .nomasters = ARRAY_SIZE(master_names), + .num_masters = ARRAY_SIZE(master_names), + .master_offset = 32, }; struct platform_device msm8930_rpm_master_stat_device = { - .name = "msm_rpm_master_stat", + .name = "msm_rpm_master_stats", .id = -1, .num_resources = ARRAY_SIZE(resources_rpm_master_stats), .resource = resources_rpm_master_stats, diff --git a/arch/arm/mach-msm/devices-8960.c b/arch/arm/mach-msm/devices-8960.c index 71f58a67dc9..6664e17dadb 100644 --- a/arch/arm/mach-msm/devices-8960.c +++ b/arch/arm/mach-msm/devices-8960.c @@ -4018,11 +4018,12 @@ static char *master_names[] = { static struct msm_rpm_master_stats_platform_data msm_rpm_master_stat_pdata = { .masters = master_names, - .nomasters = ARRAY_SIZE(master_names), + .num_masters = ARRAY_SIZE(master_names), + .master_offset = 32, }; struct platform_device msm8960_rpm_master_stat_device = { - .name = "msm_rpm_master_stat", + .name = "msm_rpm_master_stats", .id = -1, .num_resources = ARRAY_SIZE(resources_rpm_master_stats), .resource = resources_rpm_master_stats, diff --git a/arch/arm/mach-msm/devices-9615.c b/arch/arm/mach-msm/devices-9615.c index 483d8b38734..6a486467131 100644 --- a/arch/arm/mach-msm/devices-9615.c +++ b/arch/arm/mach-msm/devices-9615.c @@ -1431,11 +1431,12 @@ static char *master_names[] = { static struct msm_rpm_master_stats_platform_data msm_rpm_master_stat_pdata = { .masters = master_names, - .nomasters = ARRAY_SIZE(master_names), + .num_masters = ARRAY_SIZE(master_names), + .master_offset = 32, }; struct platform_device msm9615_rpm_master_stat_device = { - .name = "msm_rpm_master_stat", + .name = "msm_rpm_master_stats", .id = -1, .num_resources = ARRAY_SIZE(resources_rpm_master_stats), .resource = resources_rpm_master_stats, diff --git a/arch/arm/mach-msm/include/mach/qdsp6v2/rtac.h b/arch/arm/mach-msm/include/mach/qdsp6v2/rtac.h index 523d5e58c36..42cf6f974da 100644 --- a/arch/arm/mach-msm/include/mach/qdsp6v2/rtac.h +++ b/arch/arm/mach-msm/include/mach/qdsp6v2/rtac.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011, 2013, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -19,6 +19,31 @@ #define RTAC_CVS 1 #define RTAC_VOICE_MODES 2 +enum { + ADM_RTAC_CAL, + ASM_RTAC_CAL, + VOICE_RTAC_CAL, + MAX_RTAC_BLOCKS +}; + +struct rtac_cal_mem_map_data { + uint32_t map_size; + uint32_t map_handle; + struct ion_client *ion_client; + struct ion_handle *ion_handle; +}; + +struct rtac_cal_data { + uint32_t size; + uint32_t kvaddr; + uint32_t paddr; +}; + +struct rtac_cal_block_data { + struct rtac_cal_mem_map_data map_data; + struct rtac_cal_data cal_data; +}; + void rtac_add_adm_device(u32 port_id, u32 copp_id, u32 path_id, u32 popp_id); void rtac_remove_adm_device(u32 port_id, u32 copp_id); void rtac_remove_popp_from_adm_devices(u32 popp_id); @@ -35,5 +60,6 @@ void rtac_copy_asm_payload_to_user(void *payload, u32 payload_size); void rtac_set_voice_handle(u32 mode, void *handle); bool rtac_make_voice_callback(u32 mode, uint32_t *payload, u32 payload_size); void rtac_copy_voice_payload_to_user(void *payload, u32 payload_size); +int rtac_clear_mapping(uint32_t cal_type); #endif diff --git a/arch/arm/mach-msm/ipc_router.c b/arch/arm/mach-msm/ipc_router.c index f2ffa5490fd..15e18b7d89c 100644 --- a/arch/arm/mach-msm/ipc_router.c +++ b/arch/arm/mach-msm/ipc_router.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -424,7 +424,7 @@ static void *msm_ipc_router_skb_to_buf(struct sk_buff_head *skb_head, unsigned int len) { struct sk_buff *temp; - int offset = 0, buf_len = 0, copy_len; + unsigned int offset = 0, buf_len = 0, copy_len; void *buf; if (!skb_head) { @@ -2492,7 +2492,7 @@ int msm_ipc_router_get_curr_pkt_size(struct msm_ipc_port *port_ptr) int msm_ipc_router_bind_control_port(struct msm_ipc_port *port_ptr) { - if (!port_ptr) + if (unlikely(!port_ptr || port_ptr->type != CLIENT_PORT)) return -EINVAL; down_write(&local_ports_lock_lha2); diff --git a/arch/arm/mach-msm/ipc_socket.c b/arch/arm/mach-msm/ipc_socket.c index 515dc92bb9b..d199c6a40fb 100644 --- a/arch/arm/mach-msm/ipc_socket.c +++ b/arch/arm/mach-msm/ipc_socket.c @@ -533,16 +533,20 @@ static int msm_ipc_router_ioctl(struct socket *sock, break; } server_arg.num_entries_found = ret; - ret = copy_to_user((void *)arg, &server_arg, sizeof(server_arg)); - if (srv_info_sz) { + + n = min(server_arg.num_entries_found, + server_arg.num_entries_in_array); + + if (ret == 0 && n) { ret = copy_to_user((void *)(arg + sizeof(server_arg)), - srv_info, srv_info_sz); - if (ret) - ret = -EFAULT; - kfree(srv_info); + srv_info, n * sizeof (*srv_info)); } + + if (ret) + ret = -EFAULT; + kfree(srv_info); break; case IPC_ROUTER_IOCTL_BIND_CONTROL_PORT: diff --git a/arch/arm/mach-msm/lge/board-wifi-bcm.c b/arch/arm/mach-msm/lge/board-wifi-bcm.c index bec95a61669..5acdb6b8e8a 100644 --- a/arch/arm/mach-msm/lge/board-wifi-bcm.c +++ b/arch/arm/mach-msm/lge/board-wifi-bcm.c @@ -13,6 +13,7 @@ * GNU General Public License for more details. */ +#include #include #include #include @@ -23,7 +24,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -660,21 +663,121 @@ static void *bcm_wifi_get_country_code(char *ccode) return &country_code; } +#ifdef CONFIG_PARTIALRESUME +static bool smd_partial_resume(struct partial_resume *pr) +{ + return true; +} + +#define PR_INIT_STATE 0 +#define PR_IN_RESUME_STATE 1 +#define PR_RESUME_OK_STATE 2 +#define PR_SUSPEND_OK_STATE 3 + +static DECLARE_COMPLETION(bcm_pk_comp); +static DECLARE_COMPLETION(bcm_wd_comp); +static int bcm_suspend = PR_INIT_STATE; +static spinlock_t bcm_lock; + +/* + * Partial Resume State Machine: + _______ + / else [INIT]________________________ + \______/ | notify_resume \ + [IN_RESUME] wait_for_ready + / \ vote_for_suspend / + vote_for_resume [SUSPEND_OK]________/ + \ / vote_for_resume / + [RESUME_OK] / + \________________/ + */ + +static bool bcm_wifi_process_partial_resume(int action) +{ + bool suspend = false; + int timeout = 0; + + if ((action != WIFI_PR_NOTIFY_RESUME) && (bcm_suspend == PR_INIT_STATE)) + return suspend; + + if (action == WIFI_PR_WAIT_FOR_READY) + timeout = wait_for_completion_timeout(&bcm_pk_comp, + msecs_to_jiffies(50)); + + spin_lock(&bcm_lock); + switch (action) { + case WIFI_PR_WAIT_FOR_READY: + suspend = (bcm_suspend == PR_SUSPEND_OK_STATE) && (timeout != 0); + if (suspend) { + spin_unlock(&bcm_lock); + timeout = wait_for_completion_timeout(&bcm_wd_comp, + msecs_to_jiffies(100)); + spin_lock(&bcm_lock); + suspend = (timeout != 0); + } + bcm_suspend = PR_INIT_STATE; + break; + case WIFI_PR_VOTE_FOR_RESUME: + bcm_suspend = PR_RESUME_OK_STATE; + complete(&bcm_pk_comp); + break; + case WIFI_PR_VOTE_FOR_SUSPEND: + if (bcm_suspend == PR_IN_RESUME_STATE) + bcm_suspend = PR_SUSPEND_OK_STATE; + complete(&bcm_pk_comp); + break; + case WIFI_PR_NOTIFY_RESUME: + INIT_COMPLETION(bcm_pk_comp); + bcm_suspend = PR_IN_RESUME_STATE; + break; + case WIFI_PR_INIT: + bcm_suspend = PR_INIT_STATE; + break; + case WIFI_PR_WD_INIT: + INIT_COMPLETION(bcm_wd_comp); + break; + case WIFI_PR_WD_COMPLETE: + complete(&bcm_wd_comp); + break; + } + spin_unlock(&bcm_lock); + return suspend; +} + +bool wlan_vote_for_suspend(void) +{ + return bcm_wifi_process_partial_resume(WIFI_PR_VOTE_FOR_SUSPEND); +} +EXPORT_SYMBOL(wlan_vote_for_suspend); + +static bool bcm_wifi_partial_resume(struct partial_resume *pr) +{ + bool suspend; + + suspend = bcm_wifi_process_partial_resume(WIFI_PR_WAIT_FOR_READY); + pr_info("%s: vote %d\n", __func__, suspend); + return suspend; +} +#endif + static struct wifi_platform_data bcm_wifi_control = { .mem_prealloc = bcm_wifi_mem_prealloc, .set_power = bcm_wifi_set_power, .set_reset = bcm_wifi_reset, .set_carddetect = bcm_wifi_carddetect, - .get_mac_addr = bcm_wifi_get_mac_addr, + .get_mac_addr = bcm_wifi_get_mac_addr, .get_country_code = bcm_wifi_get_country_code, +#ifdef CONFIG_PARTIALRESUME + .partial_resume = bcm_wifi_process_partial_resume, +#endif }; static struct resource wifi_resource[] = { [0] = { .name = "bcmdhd_wlan_irq", - .start = 0, //assigned later - .end = 0, //assigned later - .flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL | IORESOURCE_IRQ_SHAREABLE, // for HW_OOB + .start = 0, /* assigned later */ + .end = 0, /* assigned later */ + .flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL | IORESOURCE_IRQ_SHAREABLE, /* for HW_OOB */ }, }; @@ -696,3 +799,38 @@ void __init init_bcm_wifi(void) bcm_wifi_init_gpio_mem(&bcm_wifi_device); platform_device_register(&bcm_wifi_device); } + +#ifdef CONFIG_PARTIALRESUME +static struct partial_resume smd_pr = { + .irq = 200, + .partial_resume = smd_partial_resume, +}; + +static struct partial_resume mpm_pr = { + .irq = 203, + .partial_resume = smd_partial_resume, +}; + +static struct partial_resume wlan_pr = { + .partial_resume = bcm_wifi_partial_resume, +}; + +int __init wlan_partial_resume_init(void) +{ + int rc; + + spin_lock_init(&bcm_lock); /* Setup partial resume */ + complete(&bcm_wd_comp); /* Prepare for case when WD is not set */ + wlan_pr.irq = bcm_wifi_device.resource->start; + rc = register_partial_resume(&wlan_pr); + pr_debug("%s: after registering %pF: %d\n", __func__, + wlan_pr.partial_resume, rc); + rc = register_partial_resume(&mpm_pr); + rc = register_partial_resume(&smd_pr); + pr_debug("%s: after registering %pF: %d\n", __func__, + smd_pr.partial_resume, rc); + return rc; +} + +late_initcall(wlan_partial_resume_init); +#endif diff --git a/arch/arm/mach-msm/perf_event_msm_krait_l2.c b/arch/arm/mach-msm/perf_event_msm_krait_l2.c index c9e2b8f8faf..b8d0ee7f3bb 100644 --- a/arch/arm/mach-msm/perf_event_msm_krait_l2.c +++ b/arch/arm/mach-msm/perf_event_msm_krait_l2.c @@ -18,13 +18,15 @@ #include +#define PMU_CODES_SIZE 64 + /* * The L2 PMU is shared between all CPU's, so protect * its bitmap access. */ struct pmu_constraints { u64 pmu_bitmap; - u8 codes[64]; + u8 codes[PMU_CODES_SIZE]; raw_spinlock_t lock; } l2_pmu_constraints = { .pmu_bitmap = 0, @@ -419,7 +421,7 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) u8 group = evt_type & 0x0000F; u8 code = (evt_type & 0x00FF0) >> 4; unsigned long flags; - u32 err = 0; + int err = 0; u64 bitmap_t; u32 shift_idx; @@ -436,6 +438,11 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) shift_idx = ((reg * 4) + group); + if (shift_idx >= PMU_CODES_SIZE) { + err = -EINVAL; + goto out; + } + bitmap_t = 1 << shift_idx; if (!(l2_pmu_constraints.pmu_bitmap & bitmap_t)) { @@ -460,6 +467,7 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) if (!(event->cpu < 0)) { event->state = PERF_EVENT_STATE_OFF; event->attr.constraint_duplicate = 1; + err = -EPERM; } } out: @@ -476,12 +484,17 @@ static int msm_l2_clear_ev_constraint(struct perf_event *event) unsigned long flags; u64 bitmap_t; u32 shift_idx; + int err = 1; if (evt_prefix == L2_TRACECTR_PREFIX) return 1; raw_spin_lock_irqsave(&l2_pmu_constraints.lock, flags); shift_idx = ((reg * 4) + group); + if (shift_idx >= PMU_CODES_SIZE) { + err = -EINVAL; + goto out; + } bitmap_t = 1 << shift_idx; @@ -492,7 +505,8 @@ static int msm_l2_clear_ev_constraint(struct perf_event *event) l2_pmu_constraints.codes[shift_idx] = -1; raw_spin_unlock_irqrestore(&l2_pmu_constraints.lock, flags); - return 1; +out: + return err; } int get_num_events(void) diff --git a/arch/arm/mach-msm/perf_event_msm_l2.c b/arch/arm/mach-msm/perf_event_msm_l2.c index efd5e2148ba..0b5bc312224 100644 --- a/arch/arm/mach-msm/perf_event_msm_l2.c +++ b/arch/arm/mach-msm/perf_event_msm_l2.c @@ -836,8 +836,10 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) * This sets the event OFF on all but one * CPU. */ - if (!(event->cpu < 0)) + if (!(event->cpu < 0)) { event->state = PERF_EVENT_STATE_OFF; + err = -EPERM; + } } out: diff --git a/arch/arm/mach-msm/qdsp6v2/amrwb_in.c b/arch/arm/mach-msm/qdsp6v2/amrwb_in.c index ea3fe5b8406..b2253935eb8 100644 --- a/arch/arm/mach-msm/qdsp6v2/amrwb_in.c +++ b/arch/arm/mach-msm/qdsp6v2/amrwb_in.c @@ -195,7 +195,7 @@ static int amrwb_in_open(struct inode *inode, struct file *file) (void *)audio); if (!audio->ac) { - pr_err("%s:audio[%p]: Could not allocate memory for audio" + pr_err("%s:audio[%pK]: Could not allocate memory for audio" "client\n", __func__, audio); kfree(audio->enc_cfg); kfree(audio); diff --git a/arch/arm/mach-msm/qdsp6v2/apr.c b/arch/arm/mach-msm/qdsp6v2/apr.c index 8d9ad2993a7..d8ca8b551c0 100644 --- a/arch/arm/mach-msm/qdsp6v2/apr.c +++ b/arch/arm/mach-msm/qdsp6v2/apr.c @@ -340,7 +340,7 @@ void apr_cb_func(void *buf, int len, void *priv) pr_debug("\n*****************\n"); if (!buf || len <= APR_HDR_SIZE) { - pr_err("APR: Improper apr pkt received:%p %d\n", buf, len); + pr_err("APR: Improper apr pkt received:%pK %d\n", buf, len); return; } hdr = buf; @@ -424,7 +424,7 @@ void apr_cb_func(void *buf, int len, void *priv) return; } pr_debug("svc_idx = %d\n", i); - pr_debug("%x %x %x %p %p\n", c_svc->id, c_svc->dest_id, + pr_debug("%x %x %x %pK %pK\n", c_svc->id, c_svc->dest_id, c_svc->client_id, c_svc->fn, c_svc->priv); data.payload_size = hdr->pkt_size - hdr_size; data.opcode = hdr->opcode; @@ -488,7 +488,7 @@ static void apr_reset_deregister(struct work_struct *work) container_of(work, struct apr_reset_work, work); handle = apr_reset->handle; - pr_debug("%s:handle[%p]\n", __func__, handle); + pr_debug("%s:handle[%pK]\n", __func__, handle); apr_deregister(handle); kfree(apr_reset); } @@ -521,7 +521,7 @@ int apr_deregister(void *handle) client[dest_id][client_id].svc_cnt--; if (!client[dest_id][client_id].svc_cnt) { svc->need_reset = 0x0; - pr_debug("%s: service is reset %p\n", __func__, svc); + pr_debug("%s: service is reset %pK\n", __func__, svc); } } @@ -549,7 +549,7 @@ void apr_reset(void *handle) if (!handle) return; - pr_debug("%s: handle[%p]\n", __func__, handle); + pr_debug("%s: handle[%pK]\n", __func__, handle); if (apr_reset_workqueue == NULL) { pr_err("%s: apr_reset_workqueue is NULL\n", __func__); diff --git a/arch/arm/mach-msm/qdsp6v2/audio_aac.c b/arch/arm/mach-msm/qdsp6v2/audio_aac.c index caeb79d2288..262d7e57e3e 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_aac.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_aac.c @@ -182,10 +182,10 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); if (rc) - pr_err("%s[%p]:Failed in utils_ioctl: %d\n", + pr_err("%s[%pK]:Failed in utils_ioctl: %d\n", __func__, audio, rc); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_acdb.c b/arch/arm/mach-msm/qdsp6v2/audio_acdb.c index a13100be875..3329f00ff17 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_acdb.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_acdb.c @@ -630,12 +630,19 @@ static int acdb_open(struct inode *inode, struct file *f) static int deregister_memory(void) { + int i; + if (atomic64_read(&acdb_data.mem_len)) { mutex_lock(&acdb_data.acdb_mutex); atomic64_set(&acdb_data.mem_len, 0); atomic_set(&acdb_data.vocstrm_total_cal_size, 0); atomic_set(&acdb_data.vocproc_total_cal_size, 0); atomic_set(&acdb_data.vocvol_total_cal_size, 0); + + for (i = 0; i < MAX_VOCPROC_TYPES; i++) { + kfree(acdb_data.col_data[i]); + acdb_data.col_data[i] = NULL; + } ion_unmap_kernel(acdb_data.ion_client, acdb_data.ion_handle); ion_free(acdb_data.ion_client, acdb_data.ion_handle); ion_client_destroy(acdb_data.ion_client); @@ -647,12 +654,18 @@ static int deregister_memory(void) static int register_memory(void) { int result; + int i; unsigned long paddr; void *kvptr; unsigned long kvaddr; unsigned long mem_len; mutex_lock(&acdb_data.acdb_mutex); + for (i = 0; i < MAX_VOCPROC_TYPES; i++) { + acdb_data.col_data[i] = kmalloc(MAX_COL_SIZE, GFP_KERNEL); + atomic_set(&acdb_data.vocproc_col_cal[i].cal_kvaddr, + (uint32_t)acdb_data.col_data[i]); + } acdb_data.ion_client = msm_ion_client_create(UINT_MAX, "audio_acdb_client"); diff --git a/arch/arm/mach-msm/qdsp6v2/audio_amrnb.c b/arch/arm/mach-msm/qdsp6v2/audio_amrnb.c index fc023c1b825..2016e65d861 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_amrnb.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_amrnb.c @@ -30,7 +30,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case AUDIO_START: { - pr_debug("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -59,7 +59,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_amrwb.c b/arch/arm/mach-msm/qdsp6v2/audio_amrwb.c index 256da4d3791..5eac8ce5e75 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_amrwb.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_amrwb.c @@ -31,7 +31,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case AUDIO_START: { - pr_debug("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -62,7 +62,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_amrwbplus.c b/arch/arm/mach-msm/qdsp6v2/audio_amrwbplus.c index 282dd7c0d3d..e022f534635 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_amrwbplus.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_amrwbplus.c @@ -50,7 +50,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case AUDIO_START: { - pr_err("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_err("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -134,7 +134,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_evrc.c b/arch/arm/mach-msm/qdsp6v2/audio_evrc.c index 3498e6927a2..da9c81307bc 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_evrc.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_evrc.c @@ -33,7 +33,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case AUDIO_START: { - pr_debug("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -64,7 +64,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_lpa.c b/arch/arm/mach-msm/qdsp6v2/audio_lpa.c index f6dd9fab2c5..5d6b2c208cf 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_lpa.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_lpa.c @@ -441,9 +441,9 @@ static int audlpa_ion_check(struct audio *audio, list_for_each_entry(region_elt, &audio->ion_region_queue, list) { if (CONTAINS(region_elt, &t) || CONTAINS(&t, region_elt) || OVERLAPS(region_elt, &t)) { - pr_err("%s[%p]:region (vaddr %p len %ld)" + pr_err("%s[%pK]:region (vaddr %pK len %ld)" " clashes with registered region" - " (vaddr %p paddr %p len %ld)\n", + " (vaddr %pK paddr %pK len %ld)\n", __func__, audio, vaddr, len, region_elt->vaddr, (void *)region_elt->paddr, region_elt->len); @@ -465,7 +465,7 @@ static int audlpa_ion_add(struct audio *audio, unsigned long ionflag; void *temp_ptr; - pr_debug("%s[%p]:\n", __func__, audio); + pr_debug("%s[%pK]:\n", __func__, audio); region = kmalloc(sizeof(*region), GFP_KERNEL); if (!region) { @@ -512,14 +512,14 @@ static int audlpa_ion_add(struct audio *audio, region->kvaddr = kvaddr; region->len = len; region->ref_cnt = 0; - pr_debug("%s[%p]:add region paddr %lx vaddr %p, len %lu kvaddr %lx\n", + pr_debug("%s[%pK]:add region paddr %lx vaddr %pK, len %lu kvaddr %lx\n", __func__, audio, region->paddr, region->vaddr, region->len, region->kvaddr); list_add_tail(®ion->list, &audio->ion_region_queue); rc = q6asm_memory_map(audio->ac, (uint32_t)paddr, IN, (uint32_t)len, 1); if (rc < 0) { - pr_err("%s[%p]: memory map failed\n", __func__, audio); + pr_err("%s[%pK]: memory map failed\n", __func__, audio); goto ion_error; } else { goto end; @@ -549,7 +549,7 @@ static int audlpa_ion_remove(struct audio *audio, if (region != NULL && (region->fd == info->fd) && (region->vaddr == info->vaddr)) { if (region->ref_cnt) { - pr_debug("%s[%p]:region %p in use ref_cnt %d\n", + pr_debug("%s[%pK]:region %pK in use ref_cnt %d\n", __func__, audio, region, region->ref_cnt); break; @@ -557,7 +557,7 @@ static int audlpa_ion_remove(struct audio *audio, rc = q6asm_memory_unmap(audio->ac, (uint32_t) region->paddr, IN); if (rc < 0) - pr_err("%s[%p]: memory unmap failed\n", + pr_err("%s[%pK]: memory unmap failed\n", __func__, audio); list_del(®ion->list); @@ -595,14 +595,14 @@ static int audlpa_ion_lookup_vaddr(struct audio *audio, void *addr, } if (match_count > 1) { - pr_err("%s[%p]:multiple hits for vaddr %p, len %ld\n", + pr_err("%s[%pK]:multiple hits for vaddr %pK, len %ld\n", __func__, audio, addr, len); list_for_each_entry(region_elt, &audio->ion_region_queue, list) { if (addr >= region_elt->vaddr && addr < region_elt->vaddr + region_elt->len && addr + len <= region_elt->vaddr + region_elt->len) - pr_err("\t%s[%p]:%p, %ld --> %p\n", + pr_err("\t%s[%pK]:%pK, %ld --> %pK\n", __func__, audio, region_elt->vaddr, region_elt->len, @@ -620,7 +620,7 @@ static unsigned long audlpa_ion_fixup(struct audio *audio, void *addr, ret = audlpa_ion_lookup_vaddr(audio, addr, len, ®ion); if (ret) { - pr_err("%s[%p]:lookup (%p, %ld) failed\n", + pr_err("%s[%pK]:lookup (%pK, %ld) failed\n", __func__, audio, addr, len); return 0; } @@ -1100,10 +1100,10 @@ static void audlpa_unmap_ion_region(struct audio *audio) struct list_head *ptr, *next; int rc = -EINVAL; - pr_debug("%s[%p]:\n", __func__, audio); + pr_debug("%s[%pK]:\n", __func__, audio); list_for_each_safe(ptr, next, &audio->ion_region_queue) { region = list_entry(ptr, struct audlpa_ion_region, list); - pr_debug("%s[%p]: phy_address = 0x%lx\n", + pr_debug("%s[%pK]: phy_address = 0x%lx\n", __func__, audio, region->paddr); if (region != NULL) { rc = q6asm_memory_unmap(audio->ac, @@ -1385,7 +1385,7 @@ static int audio_open(struct inode *inode, struct file *file) pr_err("Unable to create ION client\n"); goto err; } - pr_debug("Allocating ION clinet in audio_open %p", audio->client); + pr_debug("Allocating ION clinet in audio_open %pK", audio->client); done: return rc; err: diff --git a/arch/arm/mach-msm/qdsp6v2/audio_mp3.c b/arch/arm/mach-msm/qdsp6v2/audio_mp3.c index d2f02702bb3..df1466f0b62 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_mp3.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_mp3.c @@ -30,7 +30,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) int rc = 0; switch (cmd) { case AUDIO_START: { - pr_debug("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -66,7 +66,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_qcelp.c b/arch/arm/mach-msm/qdsp6v2/audio_qcelp.c index 4993226d61b..bc26f7dc10b 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_qcelp.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_qcelp.c @@ -33,7 +33,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) switch (cmd) { case AUDIO_START: { - pr_debug("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -64,7 +64,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); } return rc; diff --git a/arch/arm/mach-msm/qdsp6v2/audio_utils.c b/arch/arm/mach-msm/qdsp6v2/audio_utils.c index 33bbac075ca..db2a69e8d45 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_utils.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_utils.c @@ -23,6 +23,7 @@ #include #include "audio_utils.h" +#define FRAME_SIZE (1 + ((1536+sizeof(struct meta_out_dsp)) * 5)) static int audio_in_pause(struct q6audio_in *audio) { int rc; @@ -257,6 +258,11 @@ long audio_in_ioctl(struct file *file, rc = -EINVAL; break; } + if ((cfg.buffer_size > FRAME_SIZE) || + (cfg.buffer_count != FRAME_NUM)) { + rc = -EINVAL; + break; + } audio->str_cfg.buffer_size = cfg.buffer_size; audio->str_cfg.buffer_count = cfg.buffer_count; rc = q6asm_audio_client_buf_alloc(OUT, audio->ac, @@ -480,7 +486,7 @@ ssize_t audio_in_read(struct file *file, count -= bytes_to_copy; buf += bytes_to_copy; } else { - pr_err("%s:session id %d: short read data[%p] bytesavail[%d]bytesrequest[%d]\n", + pr_err("%s:session id %d: short read data[%pK] bytesavail[%d]bytesrequest[%d]\n", __func__, audio->ac->session, data, size, count); diff --git a/arch/arm/mach-msm/qdsp6v2/audio_utils_aio.c b/arch/arm/mach-msm/qdsp6v2/audio_utils_aio.c index 06b4dcc8aa7..4398a75194f 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_utils_aio.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_utils_aio.c @@ -1,6 +1,6 @@ /* Copyright (C) 2008 Google, Inc. * Copyright (C) 2008 HTC Corporation - * Copyright (c) 2009-2013, The Linux Foundation. All rights reserved. + * Copyright (c) 2009-2016, The Linux Foundation. All rights reserved. * * This software is licensed under the terms of the GNU General Public * License version 2, as published by the Free Software Foundation, and @@ -75,7 +75,7 @@ int insert_eos_buf(struct q6audio_aio *audio, struct audio_aio_buffer_node *buf_node) { struct dec_meta_out *eos_buf = buf_node->kvaddr; - pr_debug("%s[%p]:insert_eos_buf\n", __func__, audio); + pr_debug("%s[%pK]:insert_eos_buf\n", __func__, audio); eos_buf->num_of_frames = 0xFFFFFFFF; eos_buf->meta_out_dsp[0].offset_to_frame = 0x0; eos_buf->meta_out_dsp[0].nflags = AUDIO_DEC_EOS_SET; @@ -123,14 +123,14 @@ static int audio_aio_ion_lookup_vaddr(struct q6audio_aio *audio, void *addr, } if (match_count > 1) { - pr_err("%s[%p]:multiple hits for vaddr %p, len %ld\n", + pr_err("%s[%pK]:multiple hits for vaddr %pK, len %ld\n", __func__, audio, addr, len); list_for_each_entry(region_elt, &audio->ion_region_queue, list) { if (addr >= region_elt->vaddr && addr < region_elt->vaddr + region_elt->len && addr + len <= region_elt->vaddr + region_elt->len) - pr_err("\t%s[%p]:%p, %ld --> %p\n", + pr_err("\t%s[%pK]:%pK, %ld --> %pK\n", __func__, audio, region_elt->vaddr, region_elt->len, @@ -150,7 +150,7 @@ static unsigned long audio_aio_ion_fixup(struct q6audio_aio *audio, void *addr, ret = audio_aio_ion_lookup_vaddr(audio, addr, len, ®ion); if (ret) { - pr_err("%s[%p]:lookup (%p, %ld) failed\n", + pr_err("%s[%pK]:lookup (%pK, %ld) failed\n", __func__, audio, addr, len); return 0; } @@ -158,7 +158,7 @@ static unsigned long audio_aio_ion_fixup(struct q6audio_aio *audio, void *addr, region->ref_cnt++; else region->ref_cnt--; - pr_debug("%s[%p]:found region %p ref_cnt %d\n", + pr_debug("%s[%pK]:found region %pK ref_cnt %d\n", __func__, audio, region, region->ref_cnt); paddr = region->paddr + (addr - region->vaddr); /* provide kernel virtual address for accessing meta information */ @@ -171,16 +171,16 @@ static int audio_aio_pause(struct q6audio_aio *audio) { int rc = -EINVAL; - pr_debug("%s[%p], enabled = %d\n", __func__, audio, + pr_debug("%s[%pK], enabled = %d\n", __func__, audio, audio->enabled); if (audio->enabled) { rc = q6asm_cmd(audio->ac, CMD_PAUSE); if (rc < 0) - pr_err("%s[%p]: pause cmd failed rc=%d\n", + pr_err("%s[%pK]: pause cmd failed rc=%d\n", __func__, audio, rc); } else - pr_err("%s[%p]: Driver not enabled\n", __func__, audio); + pr_err("%s[%pK]: Driver not enabled\n", __func__, audio); return rc; } @@ -194,7 +194,7 @@ static int audio_aio_flush(struct q6audio_aio *audio) if (!(audio->drv_status & ADRV_STATUS_PAUSE)) { rc = audio_aio_pause(audio); if (rc < 0) - pr_err("%s[%p}: pause cmd failed rc=%d\n", + pr_err("%s[%pK]: pause cmd failed rc=%d\n", __func__, audio, rc); else @@ -202,13 +202,13 @@ static int audio_aio_flush(struct q6audio_aio *audio) } rc = q6asm_cmd(audio->ac, CMD_FLUSH); if (rc < 0) - pr_err("%s[%p]: flush cmd failed rc=%d\n", + pr_err("%s[%pK]: flush cmd failed rc=%d\n", __func__, audio, rc); /* Not in stop state, reenable the stream */ if (audio->stopped == 0) { rc = audio_aio_enable(audio); if (rc) - pr_err("%s[%p]:audio re-enable failed\n", + pr_err("%s[%pK]:audio re-enable failed\n", __func__, audio); else { audio->enabled = 1; @@ -217,9 +217,9 @@ static int audio_aio_flush(struct q6audio_aio *audio) } } } - pr_debug("%s[%p]:in_bytes %d\n", + pr_debug("%s[%pK]:in_bytes %d\n", __func__, audio, atomic_read(&audio->in_bytes)); - pr_debug("%s[%p]:in_samples %d\n", + pr_debug("%s[%pK]:in_samples %d\n", __func__, audio, atomic_read(&audio->in_samples)); atomic_set(&audio->in_bytes, 0); atomic_set(&audio->in_samples, 0); @@ -232,7 +232,7 @@ static int audio_aio_outport_flush(struct q6audio_aio *audio) rc = q6asm_cmd(audio->ac, CMD_OUT_FLUSH); if (rc < 0) - pr_err("%s[%p}: output port flush cmd failed rc=%d\n", + pr_err("%s[%pK]: output port flush cmd failed rc=%d\n", __func__, audio, rc); return rc; } @@ -260,19 +260,19 @@ void audio_aio_async_write_ack(struct q6audio_aio *audio, uint32_t token, if (token == used_buf->token) { list_del(&used_buf->list); spin_unlock_irqrestore(&audio->dsp_lock, flags); - pr_debug("%s[%p]:consumed buffer\n", __func__, audio); + pr_debug("%s[%pK]:consumed buffer\n", __func__, audio); event_payload.aio_buf = used_buf->buf; audio_aio_post_event(audio, AUDIO_EVENT_WRITE_DONE, event_payload); kfree(used_buf); if (list_empty(&audio->out_queue) && (audio->drv_status & ADRV_STATUS_FSYNC)) { - pr_debug("%s[%p]: list is empty, reached EOS in Tunnel\n", + pr_debug("%s[%pK]: list is empty, reached EOS in Tunnel\n", __func__, audio); wake_up(&audio->write_wait); } } else { - pr_err("%s[%p]:expected=%lx ret=%x\n", + pr_err("%s[%pK]:expected=%lx ret=%x\n", __func__, audio, used_buf->token, token); spin_unlock_irqrestore(&audio->dsp_lock, flags); } @@ -286,13 +286,13 @@ void audio_aio_async_out_flush(struct q6audio_aio *audio) union msm_audio_event_payload payload; unsigned long flags; - pr_debug("%s[%p}\n", __func__, audio); + pr_debug("%s[%pK]\n", __func__, audio); /* EOS followed by flush, EOS response not guranteed, free EOS i/p buffer */ spin_lock_irqsave(&audio->dsp_lock, flags); if (audio->eos_flag && (audio->eos_write_payload.aio_buf.buf_addr)) { - pr_debug("%s[%p]: EOS followed by flush received,acknowledge"\ + pr_debug("%s[%pK]: EOS followed by flush received,acknowledge"\ " eos i/p buffer immediately\n", __func__, audio); audio_aio_post_event(audio, AUDIO_EVENT_WRITE_DONE, audio->eos_write_payload); @@ -306,7 +306,7 @@ void audio_aio_async_out_flush(struct q6audio_aio *audio) payload.aio_buf = buf_node->buf; audio_aio_post_event(audio, AUDIO_EVENT_WRITE_DONE, payload); kfree(buf_node); - pr_debug("%s[%p]: Propagate WRITE_DONE during flush\n", + pr_debug("%s[%pK]: Propagate WRITE_DONE during flush\n", __func__, audio); } } @@ -317,14 +317,14 @@ void audio_aio_async_in_flush(struct q6audio_aio *audio) struct list_head *ptr, *next; union msm_audio_event_payload payload; - pr_debug("%s[%p]\n", __func__, audio); + pr_debug("%s[%pK]\n", __func__, audio); list_for_each_safe(ptr, next, &audio->in_queue) { buf_node = list_entry(ptr, struct audio_aio_buffer_node, list); list_del(&buf_node->list); /* Forcefull send o/p eos buffer after flush, if no eos response * received by dsp even after sending eos command */ if ((audio->eos_rsp != 1) && audio->eos_flag) { - pr_debug("%s[%p]: send eos on o/p buffer during flush\n", + pr_debug("%s[%pK]: send eos on o/p buffer during flush\n", __func__, audio); payload.aio_buf = buf_node->buf; payload.aio_buf.data_len = @@ -337,7 +337,7 @@ void audio_aio_async_in_flush(struct q6audio_aio *audio) } audio_aio_post_event(audio, AUDIO_EVENT_READ_DONE, payload); kfree(buf_node); - pr_debug("%s[%p]: Propagate READ_DONE during flush\n", + pr_debug("%s[%pK]: Propagate READ_DONE during flush\n", __func__, audio); } } @@ -355,19 +355,19 @@ int audio_aio_disable(struct q6audio_aio *audio) if (audio->opened) { audio->enabled = 0; audio->opened = 0; - pr_debug("%s[%p]: inbytes[%d] insamples[%d]\n", __func__, + pr_debug("%s[%pK]: inbytes[%d] insamples[%d]\n", __func__, audio, atomic_read(&audio->in_bytes), atomic_read(&audio->in_samples)); /* Close the session */ rc = q6asm_cmd(audio->ac, CMD_CLOSE); if (rc < 0) - pr_err("%s[%p]:Failed to close the session rc=%d\n", + pr_err("%s[%pK]:Failed to close the session rc=%d\n", __func__, audio, rc); audio->stopped = 1; wake_up(&audio->write_wait); wake_up(&audio->cmd_wait); } - pr_debug("%s[%p]:enabled[%d]\n", __func__, audio, audio->enabled); + pr_debug("%s[%pK]:enabled[%d]\n", __func__, audio, audio->enabled); return rc; } @@ -416,16 +416,16 @@ static void audio_aio_unmap_ion_region(struct q6audio_aio *audio) struct list_head *ptr, *next; int rc = -EINVAL; - pr_debug("%s[%p]:\n", __func__, audio); + pr_debug("%s[%pK]:\n", __func__, audio); list_for_each_safe(ptr, next, &audio->ion_region_queue) { region = list_entry(ptr, struct audio_aio_ion_region, list); - pr_debug("%s[%p]: phy_address = 0x%lx\n", + pr_debug("%s[%pK]: phy_address = 0x%lx\n", __func__, audio, region->paddr); if (region != NULL) { rc = q6asm_memory_unmap(audio->ac, (uint32_t)region->paddr, IN); if (rc < 0) - pr_err("%s[%p]: memory unmap failed\n", + pr_err("%s[%pK]: memory unmap failed\n", __func__, audio); } } @@ -442,20 +442,20 @@ static void audio_aio_listner(u32 evt_id, union auddev_evt_data *evt_payload, switch (evt_id) { case AUDDEV_EVT_STREAM_VOL_CHG: audio->volume = evt_payload->session_vol; - pr_debug("%s[%p]: AUDDEV_EVT_STREAM_VOL_CHG, stream vol %d, enabled = %d\n", + pr_debug("%s[%pK]: AUDDEV_EVT_STREAM_VOL_CHG, stream vol %d, enabled = %d\n", __func__, audio, audio->volume, audio->enabled); if (audio->enabled == 1) { if (audio->ac) { rc = q6asm_set_volume(audio->ac, audio->volume); if (rc < 0) { - pr_err("%s[%p]: Send Volume command failed rc=%d\n", + pr_err("%s[%pK]: Send Volume command failed rc=%d\n", __func__, audio, rc); } } } break; default: - pr_err("%s[%p]:ERROR:wrong event\n", __func__, audio); + pr_err("%s[%pK]:ERROR:wrong event\n", __func__, audio); break; } } @@ -472,7 +472,7 @@ int register_volume_listener(struct q6audio_aio *audio) audio_aio_listner, (void *)audio); if (rc < 0) { - pr_err("%s[%p]: Event listener failed\n", __func__, audio); + pr_err("%s[%pK]: Event listener failed\n", __func__, audio); rc = -EACCES; } return rc; @@ -490,7 +490,7 @@ int enable_volume_ramp(struct q6audio_aio *audio) if (audio->ac == NULL) return -EINVAL; - pr_debug("%s[%p]\n", __func__, audio); + pr_debug("%s[%pK]\n", __func__, audio); softpause.enable = SOFT_PAUSE_ENABLE; softpause.period = SOFT_PAUSE_PERIOD; softpause.step = SOFT_PAUSE_STEP; @@ -550,8 +550,10 @@ int enable_volume_ramp(struct q6audio_aio *audio) int audio_aio_release(struct inode *inode, struct file *file) { struct q6audio_aio *audio = file->private_data; - pr_debug("%s[%p]\n", __func__, audio); + pr_debug("%s[%pK]\n", __func__, audio); mutex_lock(&audio->lock); + mutex_lock(&audio->read_lock); + mutex_lock(&audio->write_lock); audio->wflush = 1; if (audio->enabled) audio_aio_flush(audio); @@ -566,6 +568,8 @@ int audio_aio_release(struct inode *inode, struct file *file) wake_up(&audio->event_wait); audio_aio_reset_event_queue(audio); q6asm_audio_client_free(audio->ac); + mutex_unlock(&audio->write_lock); + mutex_unlock(&audio->read_lock); mutex_unlock(&audio->lock); mutex_destroy(&audio->lock); mutex_destroy(&audio->read_lock); @@ -595,56 +599,56 @@ int audio_aio_fsync(struct file *file, loff_t start, loff_t end, int datasync) audio->drv_status |= ADRV_STATUS_FSYNC; mutex_unlock(&audio->lock); - pr_debug("%s[%p]:\n", __func__, audio); + pr_debug("%s[%pK]:\n", __func__, audio); audio->eos_rsp = 0; - pr_debug("%s[%p]Wait for write done from DSP\n", __func__, audio); + pr_debug("%s[%pK]Wait for write done from DSP\n", __func__, audio); rc = wait_event_interruptible(audio->write_wait, (list_empty(&audio->out_queue)) || audio->wflush || audio->stopped); if (audio->stopped || audio->wflush) { - pr_debug("%s[%p]: Audio Flushed or Stopped,this is not EOS\n" + pr_debug("%s[%pK]: Audio Flushed or Stopped,this is not EOS\n" , __func__, audio); audio->wflush = 0; rc = -EBUSY; } if (rc < 0) { - pr_err("%s[%p]: wait event for list_empty failed, rc = %d\n", + pr_err("%s[%pK]: wait event for list_empty failed, rc = %d\n", __func__, audio, rc); goto done; } rc = q6asm_cmd(audio->ac, CMD_EOS); - pr_debug("%s[%p]: EOS cmd sent to DSP\n", __func__, audio); + pr_debug("%s[%pK]: EOS cmd sent to DSP\n", __func__, audio); if (rc < 0) - pr_err("%s[%p]: q6asm_cmd failed, rc = %d", + pr_err("%s[%pK]: q6asm_cmd failed, rc = %d", __func__, audio, rc); - pr_debug("%s[%p]: wait for RENDERED_EOS from DSP\n" + pr_debug("%s[%pK]: wait for RENDERED_EOS from DSP\n" , __func__, audio); rc = wait_event_interruptible(audio->write_wait, (audio->eos_rsp || audio->wflush || audio->stopped)); if (rc < 0) { - pr_err("%s[%p]: wait event for eos_rsp failed, rc = %d\n", + pr_err("%s[%pK]: wait event for eos_rsp failed, rc = %d\n", __func__, audio, rc); goto done; } if (audio->stopped || audio->wflush) { audio->wflush = 0; - pr_debug("%s[%p]: Audio Flushed or Stopped,this is not EOS\n" + pr_debug("%s[%pK]: Audio Flushed or Stopped,this is not EOS\n" , __func__, audio); rc = -EBUSY; } if (audio->eos_rsp == 1) - pr_debug("%s[%p]: EOS\n", __func__, audio); + pr_debug("%s[%pK]: EOS\n", __func__, audio); done: @@ -713,21 +717,21 @@ static long audio_aio_process_event_req(struct q6audio_aio *audio, usr_evt.event_payload = drv_evt->payload; list_add_tail(&drv_evt->list, &audio->free_event_queue); } else { - pr_err("%s[%p]:Unexpected path\n", __func__, audio); + pr_err("%s[%pK]:Unexpected path\n", __func__, audio); spin_unlock_irqrestore(&audio->event_queue_lock, flags); return -EPERM; } spin_unlock_irqrestore(&audio->event_queue_lock, flags); if (drv_evt->event_type == AUDIO_EVENT_WRITE_DONE) { - pr_debug("%s[%p]:posted AUDIO_EVENT_WRITE_DONE to user\n", + pr_debug("%s[%pK]:posted AUDIO_EVENT_WRITE_DONE to user\n", __func__, audio); mutex_lock(&audio->write_lock); audio_aio_ion_fixup(audio, drv_evt->payload.aio_buf.buf_addr, drv_evt->payload.aio_buf.buf_len, 0, 0); mutex_unlock(&audio->write_lock); } else if (drv_evt->event_type == AUDIO_EVENT_READ_DONE) { - pr_debug("%s[%p]:posted AUDIO_EVENT_READ_DONE to user\n", + pr_debug("%s[%pK]:posted AUDIO_EVENT_READ_DONE to user\n", __func__, audio); mutex_lock(&audio->read_lock); audio_aio_ion_fixup(audio, drv_evt->payload.aio_buf.buf_addr, @@ -739,7 +743,7 @@ static long audio_aio_process_event_req(struct q6audio_aio *audio, * Once EOS indicated */ if (audio->eos_rsp && !list_empty(&audio->in_queue)) { - pr_debug("%s[%p]:Send flush command to release read buffers"\ + pr_debug("%s[%pK]:Send flush command to release read buffers"\ " held up in DSP\n", __func__, audio); audio_aio_flush(audio); } @@ -759,7 +763,7 @@ static int audio_aio_ion_check(struct q6audio_aio *audio, list_for_each_entry(region_elt, &audio->ion_region_queue, list) { if (CONTAINS(region_elt, &t) || CONTAINS(&t, region_elt) || OVERLAPS(region_elt, &t)) { - pr_err("%s[%p]:region (vaddr %p len %ld) clashes with registered region (vaddr %p paddr %p len %ld)\n", + pr_err("%s[%pK]:region (vaddr %pK len %ld) clashes with registered region (vaddr %pK paddr %pK len %ld)\n", __func__, audio, vaddr, len, region_elt->vaddr, (void *)region_elt->paddr, region_elt->len); @@ -781,7 +785,7 @@ static int audio_aio_ion_add(struct q6audio_aio *audio, unsigned long ionflag; void *kvaddr = NULL; - pr_debug("%s[%p]:\n", __func__, audio); + pr_debug("%s[%pK]:\n", __func__, audio); region = kmalloc(sizeof(*region), GFP_KERNEL); if (!region) { @@ -810,14 +814,14 @@ static int audio_aio_ion_add(struct q6audio_aio *audio, region->kvaddr = (unsigned long)kvaddr; region->len = len; region->ref_cnt = 0; - pr_debug("%s[%p]:add region paddr %lx vaddr %p, len %lu kvaddr %lx\n", + pr_debug("%s[%pK]:add region paddr %lx vaddr %pK, len %lu kvaddr %lx\n", __func__, audio, region->paddr, region->vaddr, region->len, region->kvaddr); list_add_tail(®ion->list, &audio->ion_region_queue); rc = q6asm_memory_map(audio->ac, (uint32_t) paddr, IN, (uint32_t) len, 1); if (rc < 0) { - pr_err("%s[%p]: memory map failed\n", __func__, audio); + pr_err("%s[%pK]: memory map failed\n", __func__, audio); goto mmap_error; } else { goto end; @@ -839,7 +843,7 @@ static int audio_aio_ion_remove(struct q6audio_aio *audio, struct list_head *ptr, *next; int rc = -EINVAL; - pr_debug("%s[%p]:info fd %d vaddr %p\n", + pr_debug("%s[%pK]:info fd %d vaddr %pK\n", __func__, audio, info->fd, info->vaddr); list_for_each_safe(ptr, next, &audio->ion_region_queue) { @@ -848,17 +852,17 @@ static int audio_aio_ion_remove(struct q6audio_aio *audio, if ((region->fd == info->fd) && (region->vaddr == info->vaddr)) { if (region->ref_cnt) { - pr_debug("%s[%p]:region %p in use ref_cnt %d\n", + pr_debug("%s[%pK]:region %pK in use ref_cnt %d\n", __func__, audio, region, region->ref_cnt); break; } - pr_debug("%s[%p]:remove region fd %d vaddr %p\n", + pr_debug("%s[%pK]:remove region fd %d vaddr %pK\n", __func__, audio, info->fd, info->vaddr); rc = q6asm_memory_unmap(audio->ac, (uint32_t) region->paddr, IN); if (rc < 0) - pr_err("%s[%p]: memory unmap failed\n", + pr_err("%s[%pK]: memory unmap failed\n", __func__, audio); list_del(®ion->list); @@ -881,15 +885,15 @@ static void audio_aio_async_write(struct q6audio_aio *audio, struct audio_aio_write_param param; if (!audio || !buf_node) { - pr_err("%s NULL pointer audio=[0x%p], buf_node=[0x%p]\n", + pr_err("%s NULL pointer audio=[0x%pK], buf_node=[0x%pK]\n", __func__, audio, buf_node); return; } - pr_debug("%s[%p]: Send write buff %p phy %lx len %d meta_enable = %d\n", + pr_debug("%s[%pK]: Send write buff %pK phy %lx len %d meta_enable = %d\n", __func__, audio, buf_node, buf_node->paddr, buf_node->buf.data_len, audio->buf_cfg.meta_info_enable); - pr_debug("%s[%p]: flags = 0x%x\n", __func__, audio, + pr_debug("%s[%pK]: flags = 0x%x\n", __func__, audio, buf_node->meta_info.meta_in.nflags); ac = audio->ac; @@ -918,7 +922,7 @@ static void audio_aio_async_write(struct q6audio_aio *audio, buf_node->token = param.paddr; rc = q6asm_async_write(ac, ¶m); if (rc < 0) - pr_err("%s[%p]:failed\n", __func__, audio); + pr_err("%s[%pK]:failed\n", __func__, audio); } void audio_aio_post_event(struct q6audio_aio *audio, int type, @@ -936,7 +940,7 @@ void audio_aio_post_event(struct q6audio_aio *audio, int type, } else { e_node = kmalloc(sizeof(struct audio_aio_event), GFP_ATOMIC); if (!e_node) { - pr_err("%s[%p]:No mem to post event %d\n", + pr_err("%s[%pK]:No mem to post event %d\n", __func__, audio, type); spin_unlock_irqrestore(&audio->event_queue_lock, flags); return; @@ -958,7 +962,7 @@ static void audio_aio_async_read(struct q6audio_aio *audio, struct audio_aio_read_param param; int rc; - pr_debug("%s[%p]: Send read buff %p phy %lx len %d\n", + pr_debug("%s[%pK]: Send read buff %pK phy %lx len %d\n", __func__, audio, buf_node, buf_node->paddr, buf_node->buf.buf_len); ac = audio->ac; @@ -972,7 +976,7 @@ static void audio_aio_async_read(struct q6audio_aio *audio, buf_node->token = param.paddr; rc = q6asm_async_read(ac, ¶m); if (rc < 0) - pr_err("%s[%p]:failed\n", __func__, audio); + pr_err("%s[%pK]:failed\n", __func__, audio); } static int audio_aio_buf_add(struct q6audio_aio *audio, unsigned dir, @@ -992,7 +996,7 @@ static int audio_aio_buf_add(struct q6audio_aio *audio, unsigned dir, return -EFAULT; } - pr_debug("%s[%p]:node %p dir %x buf_addr %p buf_len %d data_len %d\n", + pr_debug("%s[%pK]:node %pK dir %x buf_addr %pK buf_len %d data_len %d\n", __func__, audio, buf_node, dir, buf_node->buf.buf_addr, buf_node->buf.buf_len, buf_node->buf.data_len); buf_node->paddr = audio_aio_ion_fixup(audio, buf_node->buf.buf_addr, @@ -1017,7 +1021,7 @@ static int audio_aio_buf_add(struct q6audio_aio *audio, unsigned dir, } else if (buf_node->meta_info.meta_in.nflags & AUDIO_DEC_EOS_SET) { if (!audio->wflush) { - pr_debug("%s[%p]:Send EOS cmd at i/p\n", + pr_debug("%s[%pK]:Send EOS cmd at i/p\n", __func__, audio); /* Driver will forcefully post writedone event * once eos ack recived from DSP @@ -1063,7 +1067,7 @@ static int audio_aio_buf_add(struct q6audio_aio *audio, unsigned dir, event_payload.aio_buf = buf_node->buf; event_payload.aio_buf.data_len = insert_eos_buf(audio, buf_node); - pr_debug("%s[%p]: propagate READ_DONE as EOS done\n",\ + pr_debug("%s[%pK]: propagate READ_DONE as EOS done\n",\ __func__, audio); audio_aio_post_event(audio, AUDIO_EVENT_READ_DONE, event_payload); @@ -1081,7 +1085,7 @@ void audio_aio_ioport_reset(struct q6audio_aio *audio) * abort due to flush */ if (audio->drv_status & ADRV_STATUS_FSYNC) { - pr_debug("%s[%p]:fsync in progress\n", __func__, audio); + pr_debug("%s[%pK]:fsync in progress\n", __func__, audio); audio->drv_ops.out_flush(audio); } else audio->drv_ops.out_flush(audio); @@ -1107,13 +1111,13 @@ int audio_aio_open(struct q6audio_aio *audio, struct file *file) /* Only AIO interface */ if (file->f_flags & O_NONBLOCK) { - pr_debug("%s[%p]:set to aio interface\n", __func__, audio); + pr_debug("%s[%pK]:set to aio interface\n", __func__, audio); audio->drv_status |= ADRV_STATUS_AIO_INTF; audio->drv_ops.out_flush = audio_aio_async_out_flush; audio->drv_ops.in_flush = audio_aio_async_in_flush; q6asm_set_io_mode(audio->ac, ASYNC_IO_MODE); } else { - pr_err("%s[%p]:SIO interface not supported\n", + pr_err("%s[%pK]:SIO interface not supported\n", __func__, audio); rc = -EACCES; goto fail; @@ -1145,7 +1149,7 @@ int audio_aio_open(struct q6audio_aio *audio, struct file *file) if (e_node) list_add_tail(&e_node->list, &audio->free_event_queue); else { - pr_err("%s[%p]:event pkt alloc failed\n", + pr_err("%s[%pK]:event pkt alloc failed\n", __func__, audio); break; } @@ -1157,7 +1161,7 @@ int audio_aio_open(struct q6audio_aio *audio, struct file *file) rc = -EACCES; goto fail; } - pr_debug("Ion client create in audio_aio_open %p", audio->client); + pr_debug("Ion client create in audio_aio_open %pK", audio->client); rc = register_volume_listener(audio); if (rc < 0) @@ -1191,7 +1195,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } case AUDIO_GET_EVENT: { - pr_debug("%s[%p]:AUDIO_GET_EVENT\n", __func__, audio); + pr_debug("%s[%pK]:AUDIO_GET_EVENT\n", __func__, audio); if (mutex_trylock(&audio->get_event_lock)) { rc = audio_aio_process_event_req(audio, (void __user *)arg); @@ -1230,11 +1234,11 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } case AUDIO_OUTPORT_FLUSH: { - pr_debug("%s[%p]:AUDIO_OUTPORT_FLUSH\n", __func__, audio); + pr_debug("%s[%pK]:AUDIO_OUTPORT_FLUSH\n", __func__, audio); mutex_lock(&audio->read_lock); rc = audio_aio_outport_flush(audio); if (rc < 0) { - pr_err("%s[%p]: AUDIO_OUTPORT_FLUSH failed\n", + pr_err("%s[%pK]: AUDIO_OUTPORT_FLUSH failed\n", __func__, audio); rc = -EINTR; } @@ -1242,13 +1246,13 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } case AUDIO_STOP: { - pr_debug("%s[%p]: AUDIO_STOP session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_STOP session_id[%d]\n", __func__, audio, audio->ac->session); mutex_lock(&audio->lock); audio->stopped = 1; rc = audio_aio_flush(audio); if (rc < 0) { - pr_err("%s[%p]:Audio Stop procedure failed rc=%d\n", + pr_err("%s[%pK]:Audio Stop procedure failed rc=%d\n", __func__, audio, rc); mutex_unlock(&audio->lock); break; @@ -1256,7 +1260,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) audio->enabled = 0; audio->drv_status &= ~ADRV_STATUS_PAUSE; if (audio->drv_status & ADRV_STATUS_FSYNC) { - pr_debug("%s[%p] Waking up the audio_aio_fsync\n", + pr_debug("%s[%pK] Waking up the audio_aio_fsync\n", __func__, audio); wake_up(&audio->write_wait); } @@ -1264,12 +1268,12 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } case AUDIO_PAUSE: { - pr_debug("%s[%p]:AUDIO_PAUSE %ld\n", __func__, audio, arg); + pr_debug("%s[%pK]:AUDIO_PAUSE %ld\n", __func__, audio, arg); mutex_lock(&audio->lock); if (arg == 1) { rc = audio_aio_pause(audio); if (rc < 0) { - pr_err("%s[%p]: pause FAILED rc=%d\n", + pr_err("%s[%pK]: pause FAILED rc=%d\n", __func__, audio, rc); mutex_unlock(&audio->lock); break; @@ -1279,7 +1283,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (audio->drv_status & ADRV_STATUS_PAUSE) { rc = audio_aio_enable(audio); if (rc) - pr_err("%s[%p]: audio enable failed\n", + pr_err("%s[%pK]: audio enable failed\n", __func__, audio); else { audio->drv_status &= ~ADRV_STATUS_PAUSE; @@ -1291,13 +1295,13 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } case AUDIO_FLUSH: { - pr_debug("%s[%p]: AUDIO_FLUSH sessionid[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_FLUSH sessionid[%d]\n", __func__, audio, audio->ac->session); mutex_lock(&audio->lock); audio->rflush = 1; audio->wflush = 1; if (audio->drv_status & ADRV_STATUS_FSYNC) { - pr_debug("%s[%p] Waking up the audio_aio_fsync\n", + pr_debug("%s[%pK] Waking up the audio_aio_fsync\n", __func__, audio); wake_up(&audio->write_wait); } @@ -1306,7 +1310,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) /* Flush input / Output buffer in software*/ audio_aio_ioport_reset(audio); if (rc < 0) { - pr_err("%s[%p]:AUDIO_FLUSH interrupted\n", + pr_err("%s[%pK]:AUDIO_FLUSH interrupted\n", __func__, audio); rc = -EINTR; } else { @@ -1324,23 +1328,33 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } case AUDIO_REGISTER_ION: { struct msm_audio_ion_info info; - pr_debug("%s[%p]:AUDIO_REGISTER_ION\n", __func__, audio); + pr_debug("%s[%pK]:AUDIO_REGISTER_ION\n", __func__, audio); mutex_lock(&audio->lock); if (copy_from_user(&info, (void *)arg, sizeof(info))) rc = -EFAULT; - else + else { + mutex_lock(&audio->read_lock); + mutex_lock(&audio->write_lock); rc = audio_aio_ion_add(audio, &info); + mutex_unlock(&audio->write_lock); + mutex_unlock(&audio->read_lock); + } mutex_unlock(&audio->lock); break; } case AUDIO_DEREGISTER_ION: { struct msm_audio_ion_info info; mutex_lock(&audio->lock); - pr_debug("%s[%p]:AUDIO_DEREGISTER_ION\n", __func__, audio); + pr_debug("%s[%pK]:AUDIO_DEREGISTER_ION\n", __func__, audio); if (copy_from_user(&info, (void *)arg, sizeof(info))) rc = -EFAULT; - else + else { + mutex_lock(&audio->read_lock); + mutex_lock(&audio->write_lock); rc = audio_aio_ion_remove(audio, &info); + mutex_unlock(&audio->write_lock); + mutex_unlock(&audio->read_lock); + } mutex_unlock(&audio->lock); break; } @@ -1350,7 +1364,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) memset(&cfg, 0, sizeof(cfg)); cfg.buffer_size = audio->str_cfg.buffer_size; cfg.buffer_count = audio->str_cfg.buffer_count; - pr_debug("%s[%p]:GET STREAM CFG %d %d\n", + pr_debug("%s[%pK]:GET STREAM CFG %d %d\n", __func__, audio, cfg.buffer_size, cfg.buffer_count); if (copy_to_user((void *)arg, &cfg, sizeof(cfg))) rc = -EFAULT; @@ -1359,7 +1373,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } case AUDIO_SET_STREAM_CONFIG: { struct msm_audio_stream_config cfg; - pr_debug("%s[%p]:SET STREAM CONFIG\n", __func__, audio); + pr_debug("%s[%pK]:SET STREAM CONFIG\n", __func__, audio); mutex_lock(&audio->lock); if (copy_from_user(&cfg, (void *)arg, sizeof(cfg))) { rc = -EFAULT; @@ -1382,7 +1396,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } case AUDIO_SET_CONFIG: { struct msm_audio_config config; - pr_err("%s[%p]:AUDIO_SET_CONFIG\n", __func__, audio); + pr_err("%s[%pK]:AUDIO_SET_CONFIG\n", __func__, audio); mutex_lock(&audio->lock); if (copy_from_user(&config, (void *)arg, sizeof(config))) { rc = -EFAULT; @@ -1390,7 +1404,7 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } if (audio->feedback != NON_TUNNEL_MODE) { - pr_err("%s[%p]:Not sufficient permission to change the playback mode\n", + pr_err("%s[%pK]:Not sufficient permission to change the playback mode\n", __func__, audio); rc = -EACCES; mutex_unlock(&audio->lock); @@ -1427,14 +1441,14 @@ long audio_aio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } audio->buf_cfg.meta_info_enable = cfg.meta_info_enable; - pr_debug("%s[%p]:session id %d: Set-buf-cfg: meta[%d]", + pr_debug("%s[%pK]:session id %d: Set-buf-cfg: meta[%d]", __func__, audio, audio->ac->session, cfg.meta_info_enable); mutex_unlock(&audio->lock); break; } case AUDIO_GET_BUF_CFG: { - pr_debug("%s[%p]:session id %d: Get-buf-cfg: meta[%d] framesperbuf[%d]\n", + pr_debug("%s[%pK]:session id %d: Get-buf-cfg: meta[%d] framesperbuf[%d]\n", __func__, audio, audio->ac->session, audio->buf_cfg.meta_info_enable, audio->buf_cfg.frames_per_buf); diff --git a/arch/arm/mach-msm/qdsp6v2/audio_wma.c b/arch/arm/mach-msm/qdsp6v2/audio_wma.c index 5e3de860980..20444671607 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_wma.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_wma.c @@ -35,7 +35,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case AUDIO_START: { struct asm_wma_cfg wma_cfg; struct msm_audio_wma_config_v2 *wma_config; - pr_debug("%s[%p]: AUDIO_START session_id[%d]\n", __func__, + pr_debug("%s[%pK]: AUDIO_START session_id[%d]\n", __func__, audio, audio->ac->session); if (audio->feedback == NON_TUNNEL_MODE) { /* Configure PCM output block */ @@ -95,7 +95,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); if (rc) pr_err("Failed in utils_ioctl: %d\n", rc); diff --git a/arch/arm/mach-msm/qdsp6v2/audio_wmapro.c b/arch/arm/mach-msm/qdsp6v2/audio_wmapro.c index ce49cac47fa..d318faffc9d 100644 --- a/arch/arm/mach-msm/qdsp6v2/audio_wmapro.c +++ b/arch/arm/mach-msm/qdsp6v2/audio_wmapro.c @@ -153,7 +153,7 @@ static long audio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; } default: - pr_debug("%s[%p]: Calling utils ioctl\n", __func__, audio); + pr_debug("%s[%pK]: Calling utils ioctl\n", __func__, audio); rc = audio->codec_ioctl(file, cmd, arg); if (rc) pr_err("Failed in utils_ioctl: %d\n", rc); diff --git a/arch/arm/mach-msm/qdsp6v2/dsp_debug.c b/arch/arm/mach-msm/qdsp6v2/dsp_debug.c index 26c8f75ac1f..5c1a65bece0 100644 --- a/arch/arm/mach-msm/qdsp6v2/dsp_debug.c +++ b/arch/arm/mach-msm/qdsp6v2/dsp_debug.c @@ -169,7 +169,7 @@ static ssize_t dsp_read(struct file *file, char __user *buf, } if (copy_to_user(buf, ptr, PAGE_SIZE)) { iounmap(mem_buffer); - pr_err("[%s:%s] copy error @ %p\n", __MM_FILE__, + pr_err("[%s:%s] copy error @ %pK\n", __MM_FILE__, __func__, buf); return -EFAULT; } diff --git a/arch/arm/mach-msm/qdsp6v2/msm_audio_ion.c b/arch/arm/mach-msm/qdsp6v2/msm_audio_ion.c index a7c1be2b3b2..2b94927850e 100644 --- a/arch/arm/mach-msm/qdsp6v2/msm_audio_ion.c +++ b/arch/arm/mach-msm/qdsp6v2/msm_audio_ion.c @@ -48,6 +48,12 @@ int msm_audio_ion_alloc(const char *name, struct ion_client **client, { int rc = 0; + if (!name || !client || !handle || !bufsz || !paddr + || !pa_len || !vaddr) { + pr_err("%s: Invalid params\n", __func__); + return -EINVAL; + } + *client = msm_audio_ion_client_create(UINT_MAX, name); if (IS_ERR_OR_NULL((void *)(*client))) { pr_err("%s: ION create client for AUDIO failed\n", __func__); @@ -85,10 +91,10 @@ int msm_audio_ion_alloc(const char *name, struct ion_client **client, pr_err("%s: ION memory mapping for AUDIO failed\n", __func__); goto err_ion_handle; } - pr_debug("%s: mapped address = %p, size=%d\n", __func__, *vaddr, bufsz); + pr_debug("%s: mapped address = %pK, size=%d\n", __func__, *vaddr, bufsz); if (bufsz != 0) { - pr_debug("%s: memset to 0 %p %d\n", __func__, *vaddr, bufsz); + pr_debug("%s: memset to 0 %pK %d\n", __func__, *vaddr, bufsz); memset((void *)*vaddr, 0, bufsz); } @@ -98,6 +104,8 @@ int msm_audio_ion_alloc(const char *name, struct ion_client **client, ion_free(*client, *handle); err_ion_client: msm_audio_ion_client_destroy(*client); + *client = NULL; + *handle = NULL; err: return -EINVAL; } @@ -108,10 +116,16 @@ int msm_audio_ion_import(const char *name, struct ion_client **client, ion_phys_addr_t *paddr, size_t *pa_len, void **vaddr) { int rc = 0; + if (!name || !client || !handle || !paddr || !vaddr || !pa_len) { + pr_err("%s: Invalid params\n", __func__); + rc = -EINVAL; + goto err; + } *client = msm_audio_ion_client_create(UINT_MAX, name); if (IS_ERR_OR_NULL((void *)(*client))) { pr_err("%s: ION create client for AUDIO failed\n", __func__); + rc = -EINVAL; goto err; } @@ -119,13 +133,14 @@ int msm_audio_ion_import(const char *name, struct ion_client **client, bufsz should be 0 and fd shouldn't be 0 as of now */ *handle = ion_import_dma_buf(*client, fd); - pr_err("%s: DMA Buf name=%s, fd=%d handle=%p\n", __func__, + pr_err("%s: DMA Buf name=%s, fd=%d handle=%pK\n", __func__, name, fd, *handle); if (IS_ERR_OR_NULL((void *) (*handle))) { pr_err("%s: ion import dma buffer failed\n", __func__); - goto err_ion_handle; - } + rc = -EINVAL; + goto err_destroy_client; + } if (ionflag != NULL) { rc = ion_handle_get_flags(*client, *handle, ionflag); @@ -146,24 +161,32 @@ int msm_audio_ion_import(const char *name, struct ion_client **client, *vaddr = ion_map_kernel(*client, *handle); if (IS_ERR_OR_NULL((void *)*vaddr)) { pr_err("%s: ION memory mapping for AUDIO failed\n", __func__); + rc = -ENOMEM; goto err_ion_handle; } - pr_debug("%s: mapped address = %p, size=%d\n", __func__, *vaddr, bufsz); + pr_debug("%s: mapped address = %pK, size=%d\n", __func__, *vaddr, bufsz); return 0; err_ion_handle: ion_free(*client, *handle); +err_destroy_client: msm_audio_ion_client_destroy(*client); + *client = NULL; + *handle = NULL; err: - return -EINVAL; + return rc; } int msm_audio_ion_free(struct ion_client *client, struct ion_handle *handle) { + if (!client || !handle) { + pr_err("%s Invalid params\n", __func__); + return -EINVAL; + } if (msm_audio_ion_data.smmu_enabled) { /* Need to populate book kept infomation */ - pr_debug("client=%p, domain=%p, domain_id=%d, group=%p", + pr_debug("client=%pK, domain=%pK, domain_id=%d, group=%pK", client, msm_audio_ion_data.domain, msm_audio_ion_data.domain_id, msm_audio_ion_data.group); @@ -228,7 +251,7 @@ int msm_audio_ion_mmap(struct audio_buffer *ab, offset = 0; } len = min(len, remainder); - pr_debug("vma=%p, addr=%x len=%ld vm_start=%x vm_end=%x vm_page_prot=%ld\n", + pr_debug("vma=%pK, addr=%x len=%ld vm_start=%x vm_end=%x vm_page_prot=%ld\n", vma, (unsigned int)addr, len, (unsigned int)vma->vm_start, (unsigned int)vma->vm_end, @@ -242,6 +265,7 @@ int msm_audio_ion_mmap(struct audio_buffer *ab, } else { ion_phys_addr_t phys_addr; size_t phys_len; + size_t va_len = 0; pr_debug("%s: page is NULL\n", __func__); ret = ion_phys(ab->client, ab->handle, &phys_addr, &phys_len); @@ -251,10 +275,16 @@ int msm_audio_ion_mmap(struct audio_buffer *ab, return ret; } pr_debug("phys=%x len=%d\n", (unsigned int)phys_addr, phys_len); - pr_debug("vma=%p, vm_start=%x vm_end=%x vm_pgoff=%ld vm_page_prot=%ld\n", + pr_debug("vma=%pK, vm_start=%x vm_end=%x vm_pgoff=%ld vm_page_prot=%ld\n", vma, (unsigned int)vma->vm_start, (unsigned int)vma->vm_end, vma->vm_pgoff, (unsigned long int)vma->vm_page_prot); + va_len = vma->vm_end - vma->vm_start; + if ((offset > phys_len) || (va_len > phys_len-offset)) { + pr_err("wrong offset size %ld, lens= %d, va_len=%d\n", + offset, phys_len, va_len); + return -EINVAL; + } ret = remap_pfn_range(vma, vma->vm_start, __phys_to_pfn(phys_addr) + vma->vm_pgoff, vma->vm_end - vma->vm_start, @@ -282,7 +312,7 @@ struct ion_client *msm_audio_ion_client_create(unsigned int heap_mask, void msm_audio_ion_client_destroy(struct ion_client *client) { - pr_debug("%s: client = %p smmu_enabled = %d\n", __func__, + pr_debug("%s: client = %pK smmu_enabled = %d\n", __func__, client, msm_audio_ion_data.smmu_enabled); ion_client_destroy(client); @@ -299,7 +329,7 @@ int msm_audio_ion_import_legacy(const char *name, struct ion_client *client, bufsz should be 0 and fd shouldn't be 0 as of now */ *handle = ion_import_dma_buf(client, fd); - pr_debug("%s: DMA Buf name=%s, fd=%d handle=%p\n", __func__, + pr_debug("%s: DMA Buf name=%s, fd=%d handle=%pK\n", __func__, name, fd, *handle); if (IS_ERR_OR_NULL((void *)(*handle))) { pr_err("%s: ion import dma buffer failed\n", @@ -359,7 +389,7 @@ int msm_audio_ion_cache_operations(struct audio_buffer *abuff, int cache_op) int msm_cache_ops = 0; if (!abuff) { - pr_err("Invalid params: %p, %p\n", __func__, abuff); + pr_err("Invalid params: %pK, %pK\n", __func__, abuff); return -EINVAL; } rc = ion_handle_get_flags(abuff->client, abuff->handle, @@ -405,7 +435,7 @@ static int msm_audio_ion_get_phys(struct ion_client *client, pr_err("%s: ION map iommu failed %d\n", __func__, rc); return rc; } - pr_debug("client=%p, domain=%p, domain_id=%d, group=%p", + pr_debug("client=%pK, domain=%pK, domain_id=%d, group=%pK", client, msm_audio_ion_data.domain, msm_audio_ion_data.domain_id, msm_audio_ion_data.group); } else { @@ -441,18 +471,18 @@ static int msm_audio_ion_probe(struct platform_device *pdev) msm_audio_ion_data.domain = iommu_group_get_iommudata(msm_audio_ion_data.group); if (IS_ERR_OR_NULL(msm_audio_ion_data.domain)) { - pr_err("Failed to get domain data for group %p", + pr_err("Failed to get domain data for group %pK", msm_audio_ion_data.group); goto fail_group; } msm_audio_ion_data.domain_id = msm_find_domain_no(msm_audio_ion_data.domain); if (msm_audio_ion_data.domain_id < 0) { - pr_err("Failed to get domain index for domain %p", + pr_err("Failed to get domain index for domain %pK", msm_audio_ion_data.domain); goto fail_group; } - pr_debug("domain=%p, domain_id=%d, group=%p", + pr_debug("domain=%pK, domain_id=%d, group=%pK", msm_audio_ion_data.domain, msm_audio_ion_data.domain_id, msm_audio_ion_data.group); @@ -476,7 +506,7 @@ static int msm_audio_ion_probe(struct platform_device *pdev) static int msm_audio_ion_remove(struct platform_device *pdev) { - pr_debug("%s: msm audio ion is unloaded, domain=%p, group=%p\n", + pr_debug("%s: msm audio ion is unloaded, domain=%pK, group=%pK\n", __func__, msm_audio_ion_data.domain, msm_audio_ion_data.group); iommu_detach_group(msm_audio_ion_data.domain, msm_audio_ion_data.group); diff --git a/arch/arm/mach-msm/qdsp6v2/pcm_in.c b/arch/arm/mach-msm/qdsp6v2/pcm_in.c index 0db489465a0..3b16d0d9b78 100644 --- a/arch/arm/mach-msm/qdsp6v2/pcm_in.c +++ b/arch/arm/mach-msm/qdsp6v2/pcm_in.c @@ -407,7 +407,7 @@ static ssize_t pcm_in_read(struct file *file, char __user *buf, len = size; else { len = count; - pr_err("%s: short read data[%p]bytesavail[%d]" + pr_err("%s: short read data[%pK]bytesavail[%d]" "bytesrequest[%d]" "bytesrejected%d]\n",\ __func__, data, size, diff --git a/arch/arm/mach-msm/qdsp6v2/q6audio_v1_aio.c b/arch/arm/mach-msm/qdsp6v2/q6audio_v1_aio.c index ffd14bd2c2e..dddafba47a2 100644 --- a/arch/arm/mach-msm/qdsp6v2/q6audio_v1_aio.c +++ b/arch/arm/mach-msm/qdsp6v2/q6audio_v1_aio.c @@ -56,18 +56,18 @@ void audio_aio_cb(uint32_t opcode, uint32_t token, switch (opcode) { case ASM_DATA_EVENT_WRITE_DONE: - pr_debug("%s[%p]:ASM_DATA_EVENT_WRITE_DONE token = 0x%x\n", + pr_debug("%s[%pK]:ASM_DATA_EVENT_WRITE_DONE token = 0x%x\n", __func__, audio, token); audio_aio_async_write_ack(audio, token, payload); break; case ASM_DATA_EVENT_READ_DONE: - pr_debug("%s[%p]:ASM_DATA_EVENT_READ_DONE token = 0x%x\n", + pr_debug("%s[%pK]:ASM_DATA_EVENT_READ_DONE token = 0x%x\n", __func__, audio, token); audio_aio_async_read_ack(audio, token, payload); break; case ASM_DATA_CMDRSP_EOS: /* EOS Handle */ - pr_debug("%s[%p]:ASM_DATA_CMDRSP_EOS\n", __func__, audio); + pr_debug("%s[%pK]:ASM_DATA_CMDRSP_EOS\n", __func__, audio); if (audio->feedback) { /* Non-Tunnel mode */ audio->eos_rsp = 1; /* propagate input EOS i/p buffer, @@ -89,18 +89,18 @@ void audio_aio_cb(uint32_t opcode, uint32_t token, break; case ASM_DATA_CMD_MEDIA_FORMAT_UPDATE: case ASM_STREAM_CMD_SET_ENCDEC_PARAM: - pr_debug("%s[%p]:payload0[%x] payloa1d[%x]opcode= 0x%x\n", + pr_debug("%s[%pK]:payload0[%x] payloa1d[%x]opcode= 0x%x\n", __func__, audio, payload[0], payload[1], opcode); break; case ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY: case ASM_DATA_EVENT_ENC_SR_CM_NOTIFY: - pr_debug("%s[%p]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, " + pr_debug("%s[%pK]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, " "payload[0]-sr = %d, payload[1]-chl = %d, " "payload[2] = %d, payload[3] = %d\n", __func__, audio, payload[0], payload[1], payload[2], payload[3]); - pr_debug("%s[%p]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, sr(prev) = %d, chl(prev) = %d,", + pr_debug("%s[%pK]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, sr(prev) = %d, chl(prev) = %d,", __func__, audio, audio->pcm_cfg.sample_rate, audio->pcm_cfg.channel_count); audio->pcm_cfg.sample_rate = payload[0]; @@ -145,7 +145,7 @@ void extract_meta_out_info(struct q6audio_aio *audio, else memset(&buf_node->meta_info.meta_in, 0, sizeof(struct dec_meta_in)); - pr_debug("%s[%p]:i/p: msw_ts 0x%lx lsw_ts 0x%lx nflags 0x%8x\n", + pr_debug("%s[%pK]:i/p: msw_ts 0x%lx lsw_ts 0x%lx nflags 0x%8x\n", __func__, audio, buf_node->meta_info.meta_in.ntimestamp.highpart, buf_node->meta_info.meta_in.ntimestamp.lowpart, @@ -155,7 +155,7 @@ void extract_meta_out_info(struct q6audio_aio *audio, &buf_node->meta_info.meta_out, sizeof(struct dec_meta_out)); meta_data->meta_out_dsp[0].nflags = 0x00000000; - pr_debug("%s[%p]:o/p: msw_ts 0x%8x lsw_ts 0x%8x nflags 0x%8x, num_frames = %d\n", + pr_debug("%s[%pK]:o/p: msw_ts 0x%8x lsw_ts 0x%8x nflags 0x%8x, num_frames = %d\n", __func__, audio, ((struct dec_meta_out *)buf_node->kvaddr)->\ meta_out_dsp[0].msw_ts, @@ -204,7 +204,7 @@ void audio_aio_async_read_ack(struct q6audio_aio *audio, uint32_t token, event_payload.aio_buf.data_len = payload[2] + \ payload[3] + \ sizeof(struct dec_meta_out); - pr_debug("%s[%p]:nr of frames 0x%8x len=%d\n", + pr_debug("%s[%pK]:nr of frames 0x%8x len=%d\n", __func__, audio, filled_buf->meta_info.meta_out.num_of_frames, event_payload.aio_buf.data_len); @@ -215,7 +215,7 @@ void audio_aio_async_read_ack(struct q6audio_aio *audio, uint32_t token, event_payload); kfree(filled_buf); } else { - pr_err("%s[%p]:expected=%lx ret=%x\n", + pr_err("%s[%pK]:expected=%lx ret=%x\n", __func__, audio, filled_buf->token, token); spin_unlock_irqrestore(&audio->dsp_lock, flags); } diff --git a/arch/arm/mach-msm/qdsp6v2/q6audio_v2_aio.c b/arch/arm/mach-msm/qdsp6v2/q6audio_v2_aio.c index 4681437db2e..be1b668b182 100644 --- a/arch/arm/mach-msm/qdsp6v2/q6audio_v2_aio.c +++ b/arch/arm/mach-msm/qdsp6v2/q6audio_v2_aio.c @@ -54,18 +54,18 @@ void audio_aio_cb(uint32_t opcode, uint32_t token, switch (opcode) { case ASM_DATA_EVENT_WRITE_DONE_V2: - pr_debug("%s[%p]:ASM_DATA_EVENT_WRITE_DONE token = 0x%x\n", + pr_debug("%s[%pK]:ASM_DATA_EVENT_WRITE_DONE token = 0x%x\n", __func__, audio, token); audio_aio_async_write_ack(audio, token, payload); break; case ASM_DATA_EVENT_READ_DONE_V2: - pr_debug("%s[%p]:ASM_DATA_EVENT_READ_DONE token = 0x%x\n", + pr_debug("%s[%pK]:ASM_DATA_EVENT_READ_DONE token = 0x%x\n", __func__, audio, token); audio_aio_async_read_ack(audio, token, payload); break; case ASM_DATA_EVENT_RENDERED_EOS: /* EOS Handle */ - pr_debug("%s[%p]:ASM_DATA_CMDRSP_EOS\n", __func__, audio); + pr_debug("%s[%pK]:ASM_DATA_CMDRSP_EOS\n", __func__, audio); if (audio->feedback) { /* Non-Tunnel mode */ audio->eos_rsp = 1; /* propagate input EOS i/p buffer, @@ -87,16 +87,16 @@ void audio_aio_cb(uint32_t opcode, uint32_t token, break; case ASM_DATA_CMD_MEDIA_FMT_UPDATE_V2: case ASM_STREAM_CMD_SET_ENCDEC_PARAM: - pr_debug("%s[%p]:payload0[%x] payloa1d[%x]opcode= 0x%x\n", + pr_debug("%s[%pK]:payload0[%x] payloa1d[%x]opcode= 0x%x\n", __func__, audio, payload[0], payload[1], opcode); break; case ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY: case ASM_DATA_EVENT_ENC_SR_CM_CHANGE_NOTIFY: - pr_debug("%s[%p]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, payload[0]-sr = %d, payload[1]-chl = %d, payload[2] = %d, payload[3] = %d\n", + pr_debug("%s[%pK]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, payload[0]-sr = %d, payload[1]-chl = %d, payload[2] = %d, payload[3] = %d\n", __func__, audio, payload[0], payload[1], payload[2], payload[3]); - pr_debug("%s[%p]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, sr(prev) = %d, chl(prev) = %d,", + pr_debug("%s[%pK]: ASM_DATA_EVENT_SR_CM_CHANGE_NOTIFY, sr(prev) = %d, chl(prev) = %d,", __func__, audio, audio->pcm_cfg.sample_rate, audio->pcm_cfg.channel_count); @@ -131,7 +131,7 @@ void extract_meta_out_info(struct q6audio_aio *audio, else memset(&buf_node->meta_info.meta_in, 0, sizeof(struct dec_meta_in)); - pr_debug("%s[%p]:i/p: msw_ts 0x%lx lsw_ts 0x%lx nflags 0x%8x\n", + pr_debug("%s[%pK]:i/p: msw_ts 0x%lx lsw_ts 0x%lx nflags 0x%8x\n", __func__, audio, buf_node->meta_info.meta_in.ntimestamp.highpart, buf_node->meta_info.meta_in.ntimestamp.lowpart, @@ -146,7 +146,7 @@ void extract_meta_out_info(struct q6audio_aio *audio, meta_data->meta_out_dsp[0].lsw_ts; meta_data->meta_out_dsp[0].lsw_ts = temp; - pr_debug("%s[%p]:o/p: msw_ts 0x%8x lsw_ts 0x%8x nflags 0x%8x, num_frames = %d\n", + pr_debug("%s[%pK]:o/p: msw_ts 0x%8x lsw_ts 0x%8x nflags 0x%8x, num_frames = %d\n", __func__, audio, ((struct dec_meta_out *)buf_node->kvaddr)->\ meta_out_dsp[0].msw_ts, @@ -202,7 +202,7 @@ void audio_aio_async_read_ack(struct q6audio_aio *audio, uint32_t token, = payload[9]; event_payload.aio_buf.data_len = payload[4]\ + payload[5] + sizeof(struct dec_meta_out); - pr_debug("%s[%p]:nr of frames 0x%8x len=%d\n", + pr_debug("%s[%pK]:nr of frames 0x%8x len=%d\n", __func__, audio, filled_buf->meta_info.meta_out.num_of_frames, event_payload.aio_buf.data_len); @@ -214,7 +214,7 @@ void audio_aio_async_read_ack(struct q6audio_aio *audio, uint32_t token, event_payload); kfree(filled_buf); } else { - pr_err("%s[%p]:expected=%lx ret=%x\n", + pr_err("%s[%pK]:expected=%lx ret=%x\n", __func__, audio, filled_buf->token, token); spin_unlock_irqrestore(&audio->dsp_lock, flags); } diff --git a/arch/arm/mach-msm/qdsp6v2/q6core.c b/arch/arm/mach-msm/qdsp6v2/q6core.c index fd699dfe5db..a497deb9c2d 100644 --- a/arch/arm/mach-msm/qdsp6v2/q6core.c +++ b/arch/arm/mach-msm/qdsp6v2/q6core.c @@ -160,7 +160,7 @@ void core_open(void) core_handle_q = apr_register("ADSP", "CORE", aprv2_core_fn_q, 0xFFFFFFFF, NULL); } - pr_debug("Open_q %p\n", core_handle_q); + pr_debug("Open_q %pK\n", core_handle_q); if (core_handle_q == NULL) pr_err("%s: Unable to register CORE\n", __func__); } @@ -265,11 +265,11 @@ static ssize_t apr_debug_write(struct file *file, const char __user *buf, if (!strncmp(l_buf + 20, "open_q", 64)) { apr_handle_q = apr_register("ADSP", "TEST", aprv2_debug_fn_q, 0xFFFFFFFF, NULL); - pr_info("Open_q %p\n", apr_handle_q); + pr_info("Open_q %pK\n", apr_handle_q); } else if (!strncmp(l_buf + 20, "open_m", 64)) { apr_handle_m = apr_register("MODEM", "TEST", aprv2_debug_fn_m, 0xFFFFFFFF, NULL); - pr_info("Open_m %p\n", apr_handle_m); + pr_info("Open_m %pK\n", apr_handle_m); } else if (!strncmp(l_buf + 20, "write_q", 64)) { struct apr_hdr *hdr; diff --git a/arch/arm/mach-msm/qdsp6v2/rtac.c b/arch/arm/mach-msm/qdsp6v2/rtac.c index 18816079a45..65543a44773 100644 --- a/arch/arm/mach-msm/qdsp6v2/rtac.c +++ b/arch/arm/mach-msm/qdsp6v2/rtac.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011, 2013 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -42,6 +42,7 @@ void rtac_remove_voice(u32 cvs_handle) {} void rtac_set_voice_handle(u32 mode, void *handle) {} bool rtac_make_voice_callback(u32 mode, uint32_t *payload, u32 payload_size) {return false; } +int rtac_clear_mapping(uint32_t cal_type) {return false; } #else @@ -123,6 +124,8 @@ struct mutex rtac_asm_apr_mutex; struct mutex rtac_voice_mutex; struct mutex rtac_voice_apr_mutex; +int rtac_clear_mapping(uint32_t cal_type) {return false; } + static int rtac_open(struct inode *inode, struct file *f) { pr_debug("%s\n", __func__); diff --git a/arch/arm/mach-msm/qdsp6v2/ultrasound/usf.c b/arch/arm/mach-msm/qdsp6v2/ultrasound/usf.c index d37a3255c55..bcba05199b1 100644 --- a/arch/arm/mach-msm/qdsp6v2/ultrasound/usf.c +++ b/arch/arm/mach-msm/qdsp6v2/ultrasound/usf.c @@ -51,6 +51,11 @@ #define Y_IND 1 #define Z_IND 2 +/* Shared memory limits */ +/* max_buf_size = (port_size(65535*2) * port_num(8) * group_size(3) */ +#define USF_MAX_BUF_SIZE 3145680 +#define USF_MAX_BUF_NUM 32 + /* Place for opreation result, received from QDSP6 */ #define APR_RESULT_IND 1 @@ -436,6 +441,15 @@ static int config_xx(struct usf_xx_type *usf_xx, struct us_xx_info_type *config) (config == NULL)) return -EINVAL; + if ((config->buf_size == 0) || + (config->buf_size > USF_MAX_BUF_SIZE) || + (config->buf_num == 0) || + (config->buf_num > USF_MAX_BUF_NUM)) { + pr_err("%s: wrong params: buf_size=%d; buf_num=%d\n", + __func__, config->buf_size, config->buf_num); + return -EINVAL; + } + data_map_size = sizeof(usf_xx->encdec_cfg.cfg_common.data_map); min_map_size = min(data_map_size, config->port_cnt); @@ -743,6 +757,7 @@ static int usf_set_us_detection(struct usf_type *usf, unsigned long arg) { uint32_t timeout = 0; struct us_detect_info_type detect_info; + struct usm_session_cmd_detect_info *p_allocated_memory = NULL; struct usm_session_cmd_detect_info usm_detect_info; struct usm_session_cmd_detect_info *p_usm_detect_info = &usm_detect_info; @@ -769,12 +784,13 @@ static int usf_set_us_detection(struct usf_type *usf, unsigned long arg) uint8_t *p_data = NULL; detect_info_size += detect_info.params_data_size; - p_usm_detect_info = kzalloc(detect_info_size, GFP_KERNEL); - if (p_usm_detect_info == NULL) { + p_allocated_memory = kzalloc(detect_info_size, GFP_KERNEL); + if (p_allocated_memory == NULL) { pr_err("%s: detect_info[%d] allocation failed\n", __func__, detect_info_size); return -ENOMEM; } + p_usm_detect_info = p_allocated_memory; p_data = (uint8_t *)p_usm_detect_info + sizeof(struct usm_session_cmd_detect_info); @@ -784,7 +800,7 @@ static int usf_set_us_detection(struct usf_type *usf, unsigned long arg) if (rc) { pr_err("%s: copy params from user; rc=%d\n", __func__, rc); - kfree(p_usm_detect_info); + kfree(p_allocated_memory); return -EFAULT; } p_usm_detect_info->algorithm_cfg_size = @@ -801,9 +817,7 @@ static int usf_set_us_detection(struct usf_type *usf, unsigned long arg) p_usm_detect_info, detect_info_size); if (rc || (detect_info.detect_timeout == USF_NO_WAIT_TIMEOUT)) { - if (detect_info_size > - sizeof(struct usm_session_cmd_detect_info)) - kfree(p_usm_detect_info); + kfree(p_allocated_memory); return rc; } @@ -826,22 +840,21 @@ static int usf_set_us_detection(struct usf_type *usf, unsigned long arg) if (rc < 0) { pr_err("%s: Getting US detection failed rc[%d]\n", __func__, rc); - return rc; - } - - usf->usf_rx.us_detect_type = usf->usf_tx.us_detect_type; - detect_info.is_us = (usf_xx->us_detect_type == USF_US_DETECT_YES); - rc = copy_to_user((void __user *)arg, - &detect_info, - sizeof(detect_info)); - if (rc) { - pr_err("%s: copy detect_info to user; rc=%d\n", - __func__, rc); - rc = -EFAULT; + } else { + usf->usf_rx.us_detect_type = usf->usf_tx.us_detect_type; + detect_info.is_us = + (usf_xx->us_detect_type == USF_US_DETECT_YES); + rc = copy_to_user((void __user *)arg, + &detect_info, + sizeof(detect_info)); + if (rc) { + pr_err("%s: copy detect_info to user; rc=%d\n", + __func__, rc); + rc = -EFAULT; + } } - if (detect_info_size > sizeof(struct usm_session_cmd_detect_info)) - kfree(p_usm_detect_info); + kfree(p_allocated_memory); return rc; } /* usf_set_us_detection */ @@ -942,16 +955,14 @@ static int usf_set_rx_info(struct usf_type *usf, unsigned long arg) if (rc) return rc; - if (usf_xx->buffer_size && usf_xx->buffer_count) { - rc = q6usm_us_client_buf_alloc( - IN, - usf_xx->usc, - usf_xx->buffer_size, - usf_xx->buffer_count); - if (rc) { - (void)q6usm_cmd(usf_xx->usc, CMD_CLOSE); - return rc; - } + rc = q6usm_us_client_buf_alloc( + IN, + usf_xx->usc, + usf_xx->buffer_size, + usf_xx->buffer_count); + if (rc) { + q6usm_cmd(usf_xx->usc, CMD_CLOSE); + return rc; } rc = q6usm_dec_cfg_blk(usf_xx->usc, @@ -1170,10 +1181,15 @@ static int usf_get_version(unsigned long arg) return -EFAULT; } - /* version_info.buf is pointer to place for the version string */ + if (version_info.buf_size < sizeof(DRV_VERSION)) { + pr_err("%s: buf_size (%d) < version string size (%d)\n", + __func__, version_info.buf_size, sizeof(DRV_VERSION)); + return -EINVAL; + } + rc = copy_to_user(version_info.pbuf, DRV_VERSION, - version_info.buf_size); + sizeof(DRV_VERSION)); if (rc) { pr_err("%s: copy to version_info.pbuf; rc=%d\n", __func__, rc); diff --git a/arch/arm/mach-msm/qdsp6v2/ultrasound/usfcdev.c b/arch/arm/mach-msm/qdsp6v2/ultrasound/usfcdev.c index 1299b96d437..65459b46651 100644 --- a/arch/arm/mach-msm/qdsp6v2/ultrasound/usfcdev.c +++ b/arch/arm/mach-msm/qdsp6v2/ultrasound/usfcdev.c @@ -221,7 +221,7 @@ bool usfcdev_register( bool rc = false; if ((event_type_ind >= MAX_EVENT_TYPE_NUM) || !match_cb) { - pr_err("%s: wrong input: event_type_ind=%d; match_cb=0x%p\n", + pr_err("%s: wrong input: event_type_ind=%d; match_cb=0x%pK\n", __func__, event_type_ind, match_cb); diff --git a/arch/arm/mach-msm/qdsp6v2/ultrasound/version_a/q6usm_a.c b/arch/arm/mach-msm/qdsp6v2/ultrasound/version_a/q6usm_a.c index 5d30eb1359b..9ae6f377c83 100644 --- a/arch/arm/mach-msm/qdsp6v2/ultrasound/version_a/q6usm_a.c +++ b/arch/arm/mach-msm/qdsp6v2/ultrasound/version_a/q6usm_a.c @@ -205,7 +205,7 @@ int q6usm_us_client_buf_free(unsigned int dir, if (rc) pr_err("%s: CMD Memory_unmap* failed\n", __func__); - pr_debug("%s: data[%p]phys[%p][%p]\n", __func__, + pr_debug("%s: data[%pK]phys[%pK][%pK]\n", __func__, (void *)port->data, (void *)port->phys, (void *)&port->phys); size = port->buf_size * port->buf_cnt; dma_free_coherent(NULL, size, port->data, port->phys); @@ -341,7 +341,7 @@ int q6usm_us_client_buf_alloc(unsigned int dir, port->buf_cnt = bufcnt; port->buf_size = bufsz; - pr_debug("%s: data[%p]; phys[%p]; [%p]\n", __func__, + pr_debug("%s: data[%pK]; phys[%pK]; [%pK]\n", __func__, (void *)port->data, (void *)port->phys, (void *)&port->phys); @@ -1119,7 +1119,7 @@ int q6usm_set_us_detection(struct us_client *usc, if ((usc == NULL) || (detect_info_size == 0) || (detect_info == NULL)) { - pr_err("%s: wrong input: usc=0x%p, inf_size=%d; info=0x%p", + pr_err("%s: wrong input: usc=0x%pK, inf_size=%d; info=0x%pK", __func__, usc, detect_info_size, diff --git a/arch/arm/mach-msm/qdsp6v2/ultrasound/version_b/q6usm_b.c b/arch/arm/mach-msm/qdsp6v2/ultrasound/version_b/q6usm_b.c index a3af3e783ab..f13d9fbaa93 100644 --- a/arch/arm/mach-msm/qdsp6v2/ultrasound/version_b/q6usm_b.c +++ b/arch/arm/mach-msm/qdsp6v2/ultrasound/version_b/q6usm_b.c @@ -210,7 +210,7 @@ int q6usm_us_client_buf_free(unsigned int dir, } rc = q6usm_memory_unmap(usc, port->phys, dir); - pr_debug("%s: data[%p]phys[%p][%p]\n", __func__, + pr_debug("%s: data[%pK]phys[%pK][%pK]\n", __func__, (void *)port->data, (void *)port->phys, (void *)&port->phys); /* 4K boundary is required by the API with QDSP6 */ size = (port->buf_size * port->buf_cnt + MEM_4K_OFFSET) & MEM_4K_MASK; @@ -320,7 +320,7 @@ struct us_client *q6usm_us_client_alloc( mutex_init(&usc->port[lcnt].lock); spin_lock_init(&usc->port[lcnt].dsp_lock); usc->port[lcnt].ext = (void *)p_mem_handle++; - pr_err("%s: usc->port[%d].ext=%p;\n", + pr_err("%s: usc->port[%d].ext=%pK;\n", __func__, lcnt, usc->port[lcnt].ext); } atomic_set(&usc->cmd_state, 0); @@ -366,7 +366,7 @@ int q6usm_us_client_buf_alloc(unsigned int dir, port->buf_cnt = bufcnt; port->buf_size = bufsz; - pr_debug("%s: data[%p]; phys[%p]; [%p]\n", __func__, + pr_debug("%s: data[%pK]; phys[%pK]; [%pK]\n", __func__, (void *)port->data, (void *)port->phys, (void *)&port->phys); @@ -1167,7 +1167,7 @@ int q6usm_set_us_detection(struct us_client *usc, if ((usc == NULL) || (detect_info_size == 0) || (detect_info == NULL)) { - pr_err("%s: wrong input: usc=0x%p, inf_size=%d; info=0x%p", + pr_err("%s: wrong input: usc=0x%pK, inf_size=%d; info=0x%pK", __func__, usc, detect_info_size, diff --git a/arch/arm/mach-msm/rpm_master_stat.c b/arch/arm/mach-msm/rpm_master_stat.c index 49a1039c5f0..3e1789fe8ff 100644 --- a/arch/arm/mach-msm/rpm_master_stat.c +++ b/arch/arm/mach-msm/rpm_master_stat.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -22,39 +22,57 @@ #include #include #include +#include #include #include #include "rpm_stats.h" -#define MSG_RAM_SIZE_PER_MASTER 32 - -enum { - NUMSHUTDOWNS, - ACTIVECORES, - MASTER_ID_MAX, -}; - -static char *msm_rpm_master_stats_id_labels[MASTER_ID_MAX] = { - [NUMSHUTDOWNS] = "num_shutdowns", - [ACTIVECORES] = "active_cores", -}; +#define RPM_MASTERS_BUF_LEN 400 + +#define SNPRINTF(buf, size, format, ...) \ + do { \ + if (size > 0) { \ + int ret; \ + ret = snprintf(buf, size, format, ## __VA_ARGS__); \ + if (ret > size) { \ + buf += size; \ + size = 0; \ + } else { \ + buf += ret; \ + size -= ret; \ + } \ + } \ + } while (0) + +#define GET_MASTER_NAME(a, prvdata) \ + ((a >= prvdata->num_masters) ? "Invalid Master Name" : \ + prvdata->master_names[a]) + +#define GET_FIELD(a) ((strnstr(#a, ".", 80) + 1)) struct msm_rpm_master_stats { - unsigned long numshutdowns; - unsigned long active_cores; + uint32_t active_cores; + uint32_t numshutdowns; + uint64_t shutdown_req; + uint64_t wakeup_ind; + uint64_t bringup_req; + uint64_t bringup_ack; + uint32_t wakeup_reason; /* 0 = rude wakeup, 1 = scheduled wakeup */ + uint32_t last_sleep_transition_duration; + uint32_t last_wake_transition_duration; }; struct msm_rpm_master_stats_private_data { void __iomem *reg_base; u32 len; char **master_names; - u32 nomasters; - char buf[256]; + u32 num_masters; + char buf[RPM_MASTERS_BUF_LEN]; struct msm_rpm_master_stats_platform_data *platform_data; }; -static int msm_rpm_master_stats_file_close(struct inode *inode, +int msm_rpm_master_stats_file_close(struct inode *inode, struct file *file) { struct msm_rpm_master_stats_private_data *private = file->private_data; @@ -67,53 +85,138 @@ static int msm_rpm_master_stats_file_close(struct inode *inode, } static int msm_rpm_master_copy_stats( - struct msm_rpm_master_stats_private_data *pdata) + struct msm_rpm_master_stats_private_data *prvdata) { struct msm_rpm_master_stats record; - static int nomasters; - int count; + struct msm_rpm_master_stats_platform_data *pdata; + static int master_cnt; + int count, j = 0; + char *buf; static DEFINE_MUTEX(msm_rpm_master_stats_mutex); - int j = 0; mutex_lock(&msm_rpm_master_stats_mutex); - /* - * iterrate possible nomasters times. - * 8960, 8064 have 5 masters. - * 8930 has 4 masters. - * 9x15 has 3 masters. - */ - if (nomasters > pdata->nomasters - 1) { - nomasters = 0; + + /* Iterate possible number of masters */ + if (master_cnt > prvdata->num_masters - 1) { + master_cnt = 0; mutex_unlock(&msm_rpm_master_stats_mutex); return 0; } - record.numshutdowns = readl_relaxed(pdata->reg_base + - (nomasters * MSG_RAM_SIZE_PER_MASTER)); - record.active_cores = readl_relaxed(pdata->reg_base + - (nomasters * MSG_RAM_SIZE_PER_MASTER + 4)); - - count = snprintf(pdata->buf, sizeof(pdata->buf), - "%s\n\t%s:%lu\n\t%s:%lu\n", - pdata->master_names[nomasters], - msm_rpm_master_stats_id_labels[0], - record.numshutdowns, - msm_rpm_master_stats_id_labels[1], - record.active_cores); + pdata = prvdata->platform_data; + count = RPM_MASTERS_BUF_LEN; + buf = prvdata->buf; + + if (prvdata->platform_data->version == 2) { + SNPRINTF(buf, count, "%s\n", + GET_MASTER_NAME(master_cnt, prvdata)); + + record.shutdown_req = readll_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, shutdown_req))); + + SNPRINTF(buf, count, "\t%s:0x%llX\n", + GET_FIELD(record.shutdown_req), + record.shutdown_req); + + record.wakeup_ind = readll_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, wakeup_ind))); + + SNPRINTF(buf, count, "\t%s:0x%llX\n", + GET_FIELD(record.wakeup_ind), + record.wakeup_ind); + + record.bringup_req = readll_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, bringup_req))); + + SNPRINTF(buf, count, "\t%s:0x%llX\n", + GET_FIELD(record.bringup_req), + record.bringup_req); + + record.bringup_ack = readll_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, bringup_ack))); + + SNPRINTF(buf, count, "\t%s:0x%llX\n", + GET_FIELD(record.bringup_ack), + record.bringup_ack); + + record.last_sleep_transition_duration = + readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, + last_sleep_transition_duration))); + + SNPRINTF(buf, count, "\t%s:0x%x\n", + GET_FIELD(record.last_sleep_transition_duration), + record.last_sleep_transition_duration); + + record.last_wake_transition_duration = + readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, + last_wake_transition_duration))); + + SNPRINTF(buf, count, "\t%s:0x%x\n", + GET_FIELD(record.last_wake_transition_duration), + record.last_wake_transition_duration); + + record.wakeup_reason = readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, + wakeup_reason))); + + SNPRINTF(buf, count, "\t%s:0x%x\n", + GET_FIELD(record.wakeup_reason), + record.wakeup_reason); + + record.numshutdowns = readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset + + offsetof(struct msm_rpm_master_stats, numshutdowns))); + + SNPRINTF(buf, count, "\t%s:0x%x\n", + GET_FIELD(record.numshutdowns), + record.numshutdowns); + + record.active_cores = readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset) + + offsetof(struct msm_rpm_master_stats, active_cores)); + + SNPRINTF(buf, count, "\t%s:0x%x\n", + GET_FIELD(record.active_cores), + record.active_cores); + } else { + SNPRINTF(buf, count, "%s\n", + GET_MASTER_NAME(master_cnt, prvdata)); + + record.numshutdowns = readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset) + 0x0); + + SNPRINTF(buf, count, "\t%s:0x%0x\n", + GET_FIELD(record.numshutdowns), + record.numshutdowns); + + record.active_cores = readl_relaxed(prvdata->reg_base + + (master_cnt * pdata->master_offset) + 0x4); + + SNPRINTF(buf, count, "\t%s:0x%0x\n", + GET_FIELD(record.active_cores), + record.active_cores); + } - j = find_first_bit(&record.active_cores, BITS_PER_LONG); + j = find_first_bit((unsigned long *)&record.active_cores, + BITS_PER_LONG); while (j < BITS_PER_LONG) { - count += snprintf(pdata->buf + count, - sizeof(pdata->buf) - count, - "\t\tcore%d\n", j); - j = find_next_bit(&record.active_cores, + SNPRINTF(buf, count, "\t\tcore%d\n", j); + j = find_next_bit((unsigned long *)&record.active_cores, BITS_PER_LONG, j + 1); } - - nomasters++; + master_cnt++; mutex_unlock(&msm_rpm_master_stats_mutex); - return count; + return RPM_MASTERS_BUF_LEN - count; } static int msm_rpm_master_stats_file_read(struct file *file, char __user *bufu, @@ -151,7 +254,7 @@ static int msm_rpm_master_stats_file_open(struct inode *inode, pdata = inode->i_private; file->private_data = - kmalloc(sizeof(struct msm_rpm_master_stats_private_data), + kzalloc(sizeof(struct msm_rpm_master_stats_private_data), GFP_KERNEL); if (!file->private_data) @@ -159,7 +262,7 @@ static int msm_rpm_master_stats_file_open(struct inode *inode, prvdata = file->private_data; prvdata->reg_base = ioremap(pdata->phys_addr_base, - pdata->phys_size); + pdata->phys_size); if (!prvdata->reg_base) { kfree(file->private_data); prvdata = NULL; @@ -170,7 +273,7 @@ static int msm_rpm_master_stats_file_open(struct inode *inode, } prvdata->len = 0; - prvdata->nomasters = pdata->nomasters; + prvdata->num_masters = pdata->num_masters; prvdata->master_names = pdata->masters; prvdata->platform_data = pdata; return 0; @@ -184,27 +287,108 @@ static const struct file_operations msm_rpm_master_stats_fops = { .llseek = no_llseek, }; +static struct msm_rpm_master_stats_platform_data + *msm_rpm_master_populate_pdata(struct device *dev) +{ + struct msm_rpm_master_stats_platform_data *pdata; + struct device_node *node = dev->of_node; + int rc = 0, i; + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(dev, "could not allocate memory for platform data\n"); + goto err; + } + + rc = of_property_read_u32(node, "qcom,master-stats-version", + &pdata->version); + if (rc) { + dev_err(dev, "master-stats-version missing rc=%d\n", rc); + goto err; + } + + rc = of_property_read_u32(node, "qcom,master-offset", + &pdata->master_offset); + if (rc) { + dev_err(dev, "master-offset missing rc=%d\n", rc); + goto err; + } + + pdata->num_masters = of_property_count_strings(node, "qcom,masters"); + if (pdata->num_masters < 0) { + dev_err(dev, "Failed to get number of masters =%d\n", + pdata->num_masters); + goto err; + } + + pdata->masters = devm_kzalloc(dev, sizeof(char *) * pdata->num_masters, + GFP_KERNEL); + if (!pdata->masters) { + dev_err(dev, "%s:Failed to allocated memory\n", __func__); + goto err; + } + + /* + * Read master names from DT + */ + for (i = 0; i < pdata->num_masters; i++) { + const char *master_name; + of_property_read_string_index(node, "qcom,masters", + i, &master_name); + pdata->masters[i] = devm_kzalloc(dev, sizeof(char) * + strlen(master_name) + 1, GFP_KERNEL); + if (!pdata->masters[i]) { + dev_err(dev, "%s:Failed to get memory\n", __func__); + goto err; + } + strlcpy(pdata->masters[i], master_name, + strlen(master_name) + 1); + } + return pdata; +err: + return NULL; +} + static int __devinit msm_rpm_master_stats_probe(struct platform_device *pdev) { struct dentry *dent; struct msm_rpm_master_stats_platform_data *pdata; - struct resource *res; + struct resource *res = NULL; - pdata = pdev->dev.platform_data; - if (!pdata) + if (!pdev) return -EINVAL; + if (pdev->dev.of_node) + pdata = msm_rpm_master_populate_pdata(&pdev->dev); + else + pdata = pdev->dev.platform_data; + + if (!pdata) { + dev_err(&pdev->dev, "%s: Unable to get pdata\n", __func__); + return -ENOMEM; + } + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + if (!res) { + dev_err(&pdev->dev, + "%s: Failed to get IO resource from platform device", + __func__); + return -ENXIO; + } + pdata->phys_addr_base = res->start; pdata->phys_size = resource_size(res); dent = debugfs_create_file("rpm_master_stats", S_IRUGO, NULL, - pdev->dev.platform_data, &msm_rpm_master_stats_fops); + pdata, &msm_rpm_master_stats_fops); if (!dent) { - pr_err("%s: ERROR debugfs_create_file failed\n", __func__); + dev_err(&pdev->dev, "%s: ERROR debugfs_create_file failed\n", + __func__); return -ENOMEM; } + platform_set_drvdata(pdev, dent); return 0; } @@ -219,12 +403,18 @@ static int __devexit msm_rpm_master_stats_remove(struct platform_device *pdev) return 0; } +static struct of_device_id rpm_master_table[] = { + {.compatible = "qcom,rpm-master-stats"}, + {}, +}; + static struct platform_driver msm_rpm_master_stats_driver = { .probe = msm_rpm_master_stats_probe, .remove = __devexit_p(msm_rpm_master_stats_remove), .driver = { - .name = "msm_rpm_master_stat", + .name = "msm_rpm_master_stats", .owner = THIS_MODULE, + .of_match_table = rpm_master_table, }, }; diff --git a/arch/arm/mach-msm/rpm_stats.h b/arch/arm/mach-msm/rpm_stats.h index c1dfe349245..34c1b99f126 100644 --- a/arch/arm/mach-msm/rpm_stats.h +++ b/arch/arm/mach-msm/rpm_stats.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -31,9 +31,11 @@ struct msm_rpm_master_stats_platform_data { * it allocates 256 bytes for this use. * No of masters differs for different targets. * Based on the number of masters, linux rpm stat - * driver reads (32 * nomasters) bytes to display + * driver reads (32 * num_masters) bytes to display * master stats. */ - u32 nomasters; + s32 num_masters; + u32 master_offset; + u32 version; }; #endif diff --git a/arch/arm/mach-msm/smp2p.c b/arch/arm/mach-msm/smp2p.c index 4b69cf057a5..a56d3a6db05 100644 --- a/arch/arm/mach-msm/smp2p.c +++ b/arch/arm/mach-msm/smp2p.c @@ -431,8 +431,8 @@ static void smp2p_find_entry_v1(struct smp2p_smem __iomem *item, struct smp2p_entry_v1 *pos; if (!item || !name || !entry_ptr) { - SMP2P_ERR("%s: invalid arguments %p, %p, %p\n", - __func__, item, name, entry_ptr); + SMP2P_ERR("%s: invalid arguments %d %d %d\n", + __func__, !item, !name, !entry_ptr); return; } diff --git a/arch/arm/mach-msm/smp2p_debug.c b/arch/arm/mach-msm/smp2p_debug.c index a493cbeac25..9abb17a7d39 100644 --- a/arch/arm/mach-msm/smp2p_debug.c +++ b/arch/arm/mach-msm/smp2p_debug.c @@ -41,7 +41,7 @@ static void smp2p_int_stats(struct seq_file *s) continue; seq_printf(s, - "| %5s (%d) | %11u | %10u | %10u | %p | %08x |\n", + "| %5s (%d) | %11u | %10u | %10u | %pK | %08x |\n", int_cfg[pid].name, pid, int_cfg[pid].in_int_id, int_cfg[pid].in_interrupt_count, diff --git a/arch/arm/mach-msm/smp2p_gpio.c b/arch/arm/mach-msm/smp2p_gpio.c index 2a85e5f15bc..55c69b4dfd2 100644 --- a/arch/arm/mach-msm/smp2p_gpio.c +++ b/arch/arm/mach-msm/smp2p_gpio.c @@ -337,7 +337,7 @@ static int smp2p_irq_map(struct irq_domain *domain_ptr, unsigned int virq, chip = domain_ptr->host_data; if (!chip) { - SMP2P_ERR("%s: invalid domain ptr %p\n", __func__, domain_ptr); + SMP2P_ERR("%s: invalid domain ptr\n", __func__); return -ENODEV; } diff --git a/arch/arm/mach-msm/smp2p_test_common.h b/arch/arm/mach-msm/smp2p_test_common.h index b9cddc45929..b5d37f02a62 100644 --- a/arch/arm/mach-msm/smp2p_test_common.h +++ b/arch/arm/mach-msm/smp2p_test_common.h @@ -51,7 +51,7 @@ void *b_tmp = (b); \ if (!((a_tmp)cmp(b_tmp))) { \ seq_printf(s, \ - "%s:%d Fail: " #a "(%p) " #cmp " " #b "(%p)\n", \ + "%s:%d Fail: " #a "(%pK) " #cmp " " #b "(%pK)\n", \ __func__, __LINE__, \ a_tmp, b_tmp); \ failed = 1; \ diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 1c8a25db24d..8420ccb15ed 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -474,6 +474,7 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page) if (pageno < pool->nr_pages) { bitmap_set(pool->bitmap, pageno, count); ptr = pool->vaddr + PAGE_SIZE * pageno; + memset(ptr, 0, size); *ret_page = pool->page + pageno; } spin_unlock_irqrestore(&pool->lock, flags); @@ -657,7 +658,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) { - pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel); + pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); void *memory; bool no_kernel_mapping = dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index ce8cb1970d7..a26960a562a 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -259,10 +259,9 @@ void arch_pick_mmap_layout(struct mm_struct *mm) { unsigned long random_factor = 0UL; - /* 8 bits of randomness in 20 address space bits */ if ((current->flags & PF_RANDOMIZE) && !(current->personality & ADDR_NO_RANDOMIZE)) - random_factor = (get_random_int() % (1 << 8)) << PAGE_SHIFT; + random_factor = (get_random_int() & ((1 << mmap_rnd_bits) - 1)) << PAGE_SHIFT; if (mmap_is_legacy()) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; diff --git a/arch/microblaze/kernel/ptrace.c b/arch/microblaze/kernel/ptrace.c index 6eb2aa927d8..ab1b9db661f 100644 --- a/arch/microblaze/kernel/ptrace.c +++ b/arch/microblaze/kernel/ptrace.c @@ -136,7 +136,7 @@ asmlinkage long do_syscall_trace_enter(struct pt_regs *regs) { long ret = 0; - secure_computing(regs->r12); + secure_computing_strict(regs->r12); if (test_thread_flag(TIF_SYSCALL_TRACE) && tracehook_report_syscall_entry(regs)) diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 7c24c2973c6..4812c6d916e 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -535,7 +535,7 @@ static inline int audit_arch(void) asmlinkage void syscall_trace_enter(struct pt_regs *regs) { /* do the secure computing check first */ - secure_computing(regs->regs[2]); + secure_computing_strict(regs->regs[2]); if (!(current->ptrace & PT_PTRACED)) goto out; diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index 8d8e028893b..dd5e214cdf2 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -1710,7 +1710,7 @@ long do_syscall_trace_enter(struct pt_regs *regs) { long ret = 0; - secure_computing(regs->gpr[0]); + secure_computing_strict(regs->gpr[0]); if (test_thread_flag(TIF_SYSCALL_TRACE) && tracehook_report_syscall_entry(regs)) diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 02f300fbf07..4993e689b2c 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -719,7 +719,7 @@ asmlinkage long do_syscall_trace_enter(struct pt_regs *regs) long ret = 0; /* Do the secure computing check first. */ - secure_computing(regs->gprs[2]); + secure_computing_strict(regs->gprs[2]); /* * The sysc_tracesys code in entry.S stored the system diff --git a/arch/sh/kernel/ptrace_32.c b/arch/sh/kernel/ptrace_32.c index 9698671444e..81f999a672f 100644 --- a/arch/sh/kernel/ptrace_32.c +++ b/arch/sh/kernel/ptrace_32.c @@ -503,7 +503,7 @@ asmlinkage long do_syscall_trace_enter(struct pt_regs *regs) { long ret = 0; - secure_computing(regs->regs[0]); + secure_computing_strict(regs->regs[0]); if (test_thread_flag(TIF_SYSCALL_TRACE) && tracehook_report_syscall_entry(regs)) diff --git a/arch/sh/kernel/ptrace_64.c b/arch/sh/kernel/ptrace_64.c index bc81e07dc09..af90339dadc 100644 --- a/arch/sh/kernel/ptrace_64.c +++ b/arch/sh/kernel/ptrace_64.c @@ -522,7 +522,7 @@ asmlinkage long long do_syscall_trace_enter(struct pt_regs *regs) { long long ret = 0; - secure_computing(regs->regs[9]); + secure_computing_strict(regs->regs[9]); if (test_thread_flag(TIF_SYSCALL_TRACE) && tracehook_report_syscall_entry(regs)) diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c index 6f97c076799..484dabac704 100644 --- a/arch/sparc/kernel/ptrace_64.c +++ b/arch/sparc/kernel/ptrace_64.c @@ -1062,7 +1062,7 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) int ret = 0; /* do the secure computing check first */ - secure_computing(regs->u_regs[UREG_G1]); + secure_computing_strict(regs->u_regs[UREG_G1]); if (test_thread_flag(TIF_SYSCALL_TRACE)) ret = tracehook_report_syscall_entry(regs); diff --git a/arch/um/os-Linux/start_up.c b/arch/um/os-Linux/start_up.c index 425162e22af..2f53b892fd8 100644 --- a/arch/um/os-Linux/start_up.c +++ b/arch/um/os-Linux/start_up.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include "init.h" #include "os.h" diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7cbdfdac3c7..993598ea088 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -82,7 +82,8 @@ config X86 select CLKEVT_I8253 select ARCH_HAVE_NMI_SAFE_CMPXCHG select GENERIC_IOMAP - select DCACHE_WORD_ACCESS + select DCACHE_WORD_ACCESS if !DEBUG_PAGEALLOC + select HAVE_ARCH_SECCOMP_FILTER config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS) diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index a69245ba27e..0b3f2354f6a 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -67,6 +67,10 @@ int copy_siginfo_to_user32(compat_siginfo_t __user *to, siginfo_t *from) switch (from->si_code >> 16) { case __SI_FAULT >> 16: break; + case __SI_SYS >> 16: + put_user_ex(from->si_syscall, &to->si_syscall); + put_user_ex(from->si_arch, &to->si_arch); + break; case __SI_CHLD >> 16: if (ia32) { put_user_ex(from->si_utime, &to->si_utime); diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h index ee52760549f..b04cbdb138c 100644 --- a/arch/x86/include/asm/ia32.h +++ b/arch/x86/include/asm/ia32.h @@ -144,6 +144,12 @@ typedef struct compat_siginfo { int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ int _fd; } _sigpoll; + + struct { + unsigned int _call_addr; /* calling insn */ + int _syscall; /* triggering system call number */ + unsigned int _arch; /* AUDIT_ARCH_* of syscall */ + } _sigsys; } _sifields; } compat_siginfo_t; diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h index 386b78686c4..1ace47b6259 100644 --- a/arch/x86/include/asm/syscall.h +++ b/arch/x86/include/asm/syscall.h @@ -13,9 +13,11 @@ #ifndef _ASM_X86_SYSCALL_H #define _ASM_X86_SYSCALL_H +#include #include #include #include /* For NR_syscalls */ +#include /* for TS_COMPAT */ #include extern const unsigned long sys_call_table[]; @@ -88,6 +90,12 @@ static inline void syscall_set_arguments(struct task_struct *task, memcpy(®s->bx + i, args, n * sizeof(args[0])); } +static inline int syscall_get_arch(struct task_struct *task, + struct pt_regs *regs) +{ + return AUDIT_ARCH_I386; +} + #else /* CONFIG_X86_64 */ static inline void syscall_get_arguments(struct task_struct *task, @@ -212,6 +220,25 @@ static inline void syscall_set_arguments(struct task_struct *task, } } +static inline int syscall_get_arch(struct task_struct *task, + struct pt_regs *regs) +{ +#ifdef CONFIG_IA32_EMULATION + /* + * TS_COMPAT is set for 32-bit syscall entry and then + * remains set until we return to user mode. + * + * TIF_IA32 tasks should always have TS_COMPAT set at + * system call time. + * + * x32 tasks should be considered AUDIT_ARCH_X86_64. + */ + if (task_thread_info(task)->status & TS_COMPAT) + return AUDIT_ARCH_I386; +#endif + /* Both x32 and x86_64 are considered "64-bit". */ + return AUDIT_ARCH_X86_64; +} #endif /* CONFIG_X86_32 */ #endif /* _ASM_X86_SYSCALL_H */ diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 685845cf16e..13b1990c7c5 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -1480,7 +1480,11 @@ long syscall_trace_enter(struct pt_regs *regs) regs->flags |= X86_EFLAGS_TF; /* do the secure computing check first */ - secure_computing(regs->orig_ax); + if (secure_computing(regs->orig_ax)) { + /* seccomp failures shouldn't expose any additional code. */ + ret = -1L; + goto out; + } if (unlikely(test_thread_flag(TIF_SYSCALL_EMU))) ret = -1L; @@ -1505,6 +1509,7 @@ long syscall_trace_enter(struct pt_regs *regs) regs->dx, regs->r10); #endif +out: return ret ?: regs->orig_ax; } diff --git a/arch/x86/syscalls/syscall_32.tbl b/arch/x86/syscalls/syscall_32.tbl index 29f9f0554f7..64e55260fbc 100644 --- a/arch/x86/syscalls/syscall_32.tbl +++ b/arch/x86/syscalls/syscall_32.tbl @@ -355,3 +355,4 @@ 346 i386 setns sys_setns 347 i386 process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv 348 i386 process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev +354 i386 seccomp sys_seccomp diff --git a/arch/x86/syscalls/syscall_64.tbl b/arch/x86/syscalls/syscall_64.tbl index dd29a9ea27c..9073c08114c 100644 --- a/arch/x86/syscalls/syscall_64.tbl +++ b/arch/x86/syscalls/syscall_64.tbl @@ -318,6 +318,8 @@ 309 common getcpu sys_getcpu 310 64 process_vm_readv sys_process_vm_readv 311 64 process_vm_writev sys_process_vm_writev +317 common seccomp sys_seccomp + # # x32-specific system call numbers start at 512 to avoid cache impact # for native 64-bit operation. diff --git a/block/bsg.c b/block/bsg.c index ff64ae3bace..b1c1d542aa2 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -675,6 +675,9 @@ bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) dprintk("%s: write %Zd bytes\n", bd->name, count); + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + return -EINVAL; + bsg_set_block(bd, file); bytes_written = 0; diff --git a/block/genhd.c b/block/genhd.c index 53c3f7e212a..ee048a19ce7 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -830,6 +830,7 @@ static void disk_seqf_stop(struct seq_file *seqf, void *v) if (iter) { class_dev_iter_exit(iter); kfree(iter); + seqf->private = NULL; } } diff --git a/build.config b/build.config new file mode 100644 index 00000000000..641eac3efce --- /dev/null +++ b/build.config @@ -0,0 +1,12 @@ +ARCH=arm +BRANCH=android-msm-hammerhead-3.4-mnc-mr1 +CROSS_COMPILE=arm-eabi- +DEFCONFIG=hammerhead_defconfig +EXTRA_CMDS='' +KERNEL_DIR=private/msm-lge +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/arm/arm-eabi-4.8/bin +FILES=" +arch/arm/boot/zImage-dtb +vmlinux +System.map +" diff --git a/crypto/Kconfig b/crypto/Kconfig index 8e84225c096..df3ed20338c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -423,6 +423,26 @@ config CRYPTO_SHA1_SSSE3 using Supplemental SSE3 (SSSE3) instructions or Advanced Vector Extensions (AVX), when available. +config CRYPTO_SHA1_ARM + tristate "SHA1 digest algorithm (ARM-asm)" + depends on ARM + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM assembler. + +config CRYPTO_SHA1_ARM_NEON + tristate "SHA1 digest algorithm (ARM NEON)" + depends on ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + select CRYPTO_SHA1_ARM + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM NEON assembly, when NEON instructions are + available. + config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH @@ -447,6 +467,21 @@ config CRYPTO_SHA512 This code also includes SHA-384, a 384 bit hash with 192 bits of security against collision attacks. +config CRYPTO_SHA512_ARM_NEON + tristate "SHA384 and SHA512 digest algorithm (ARM NEON)" + depends on ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using ARM NEON instructions, when available. + + This version of SHA implements a 512 bit hash with 256 bits of + security against collision attacks. + + This code also includes SHA-384, a 384 bit hash with 192 bits + of security against collision attacks. + config CRYPTO_TGR192 tristate "Tiger digest algorithms" select CRYPTO_HASH @@ -577,6 +612,46 @@ config CRYPTO_AES_NI_INTEL ECB, CBC, LRW, PCBC, XTS. The 64 bit version has additional acceleration for CTR. +config CRYPTO_AES_ARM + tristate "AES cipher algorithms (ARM-asm)" + depends on ARM + select CRYPTO_ALGAPI + select CRYPTO_AES + help + Use optimized AES assembler routines for ARM platforms. + + AES cipher algorithms (FIPS-197). AES uses the Rijndael + algorithm. + + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. + + The AES specifies three key sizes: 128, 192 and 256 bits + + See for more information. + +config CRYPTO_AES_ARM_BS + tristate "Bit sliced AES using NEON instructions" + depends on ARM && KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES_ARM + select CRYPTO_ABLK_HELPER + help + Use a faster and more secure NEON based implementation of AES in CBC, + CTR and XTS modes + + Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode + and for XTS mode encryption, CBC and XTS mode decryption speedup is + around 25%. (CBC encryption speed is not affected by this driver.) + This implementation does not rely on any lookup tables so it is + believed to be invulnerable to cache timing attacks. + config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" select CRYPTO_ALGAPI diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c index 7f159f0da00..7471595e2dc 100644 --- a/drivers/base/firmware_class.c +++ b/drivers/base/firmware_class.c @@ -298,6 +298,8 @@ static ssize_t firmware_loading_store(struct device *dev, static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store); +#ifndef CONFIG_UML + static int __firmware_data_rw(struct firmware_priv *fw_priv, char *buffer, loff_t *offset, size_t count, int read) { @@ -359,6 +361,8 @@ static ssize_t firmware_direct_read(struct file *filp, struct kobject *kobj, return ret_count; } +#endif + static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buffer, loff_t offset, size_t count) @@ -441,6 +445,8 @@ static int fw_realloc_buffer(struct firmware_priv *fw_priv, int min_size) return 0; } +#ifndef CONFIG_UML + static ssize_t firmware_direct_write(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buffer, loff_t offset, size_t count) @@ -470,6 +476,8 @@ static ssize_t firmware_direct_write(struct file *filp, struct kobject *kobj, return retval; } +#endif + /** * firmware_data_write - write method for firmware * @filp: open sysfs file @@ -538,8 +546,10 @@ static struct bin_attribute firmware_attr_data = { static struct bin_attribute firmware_direct_attr_data = { .attr = { .name = "data", .mode = 0644 }, .size = 0, +#ifndef CONFIG_UML .read = firmware_direct_read, .write = firmware_direct_write, +#endif }; static void firmware_class_timeout(u_long data) diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 10910244424..66ba4068ce1 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "../base.h" #include "power.h" @@ -893,6 +894,7 @@ static int device_suspend_noirq(struct device *dev, pm_message_t state) static int dpm_suspend_noirq(pm_message_t state) { ktime_t starttime = ktime_get(); + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; int error = 0; suspend_device_irqs(); @@ -919,6 +921,9 @@ static int dpm_suspend_noirq(pm_message_t state) put_device(dev); if (pm_wakeup_pending()) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); error = -EBUSY; break; } @@ -972,6 +977,7 @@ static int device_suspend_late(struct device *dev, pm_message_t state) static int dpm_suspend_late(pm_message_t state) { ktime_t starttime = ktime_get(); + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; int error = 0; mutex_lock(&dpm_list_mtx); @@ -997,6 +1003,9 @@ static int dpm_suspend_late(pm_message_t state) put_device(dev); if (pm_wakeup_pending()) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); error = -EBUSY; break; } @@ -1065,6 +1074,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) int error = 0; struct timer_list timer; struct dpm_drv_wd_data data; + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; dpm_wait_for_children(dev, async); @@ -1081,6 +1091,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) pm_wakeup_event(dev, 0); if (pm_wakeup_pending()) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); async_error = -EBUSY; goto Complete; } diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 8a0a9ca6ad6..22e2bfa7e83 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "power.h" @@ -649,6 +650,37 @@ void pm_wakeup_event(struct device *dev, unsigned int msec) } EXPORT_SYMBOL_GPL(pm_wakeup_event); +void pm_get_active_wakeup_sources(char *pending_wakeup_source, size_t max) +{ + struct wakeup_source *ws, *last_active_ws = NULL; + int len = 0; + bool active = false; + + rcu_read_lock(); + list_for_each_entry_rcu(ws, &wakeup_sources, entry) { + if (ws->active) { + if (!active) + len += scnprintf(pending_wakeup_source, max, + "Pending Wakeup Sources: "); + len += scnprintf(pending_wakeup_source + len, max - len, + "%s ", ws->name); + active = true; + } else if (!active && + (!last_active_ws || + ktime_to_ns(ws->last_time) > + ktime_to_ns(last_active_ws->last_time))) { + last_active_ws = ws; + } + } + if (!active && last_active_ws) { + scnprintf(pending_wakeup_source, max, + "Last active Wakeup Source: %s", + last_active_ws->name); + } + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(pm_get_active_wakeup_sources); + static void print_active_wakeup_sources(void) { struct wakeup_source *ws; diff --git a/drivers/base/syscore.c b/drivers/base/syscore.c index e8d11b6630e..e6f1e79b9e8 100644 --- a/drivers/base/syscore.c +++ b/drivers/base/syscore.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include static LIST_HEAD(syscore_ops_list); static DEFINE_MUTEX(syscore_ops_lock); @@ -73,6 +75,8 @@ int syscore_suspend(void) return 0; err_out: + log_suspend_abort_reason("System core suspend callback %pF failed", + ops->suspend); pr_err("PM: System core suspend callback %pF failed.\n", ops->suspend); list_for_each_entry_continue(ops, &syscore_ops_list, node) diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index a20a056b77b..b126ee97a39 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -2,6 +2,9 @@ # Makefile for the Linux Bluetooth HCI device drivers. # +KBUILD_CFLAGS += -w +KBUILD_CFLAGS += -Wno-error=unused-but-set-variable + obj-$(CONFIG_BT_HCISMD) += hci_smd.o obj-$(CONFIG_BT_HCIVHCI) += hci_vhci.o obj-$(CONFIG_BT_HCIUART) += hci_uart.o diff --git a/drivers/char/adsprpc.c b/drivers/char/adsprpc.c index f4dca5acb14..4284ec1bd09 100644 --- a/drivers/char/adsprpc.c +++ b/drivers/char/adsprpc.c @@ -198,7 +198,7 @@ static void free_mem(struct fastrpc_buf *buf) me->smmu.domain_id, 0); buf->phys = 0; } - if (buf->virt) { + if (!IS_ERR_OR_NULL(buf->virt)) { ion_unmap_kernel(me->iclient, buf->handle); buf->virt = 0; } @@ -211,7 +211,7 @@ static void free_map(struct fastrpc_mmap *map) { struct fastrpc_apps *me = &gfa; if (!IS_ERR_OR_NULL(map->handle)) { - if (map->virt) { + if (!IS_ERR_OR_NULL(map->virt)) { ion_unmap_kernel(me->iclient, map->handle); map->virt = 0; } @@ -230,13 +230,15 @@ static int alloc_mem(struct fastrpc_buf *buf) unsigned long len; buf->handle = 0; buf->virt = 0; + buf->phys = 0; heap = me->smmu.enabled ? ION_HEAP(ION_IOMMU_HEAP_ID) : ION_HEAP(ION_ADSP_HEAP_ID) | ION_HEAP(ION_AUDIO_HEAP_ID); buf->handle = ion_alloc(clnt, buf->size, SZ_4K, heap, 0); VERIFY(err, 0 == IS_ERR_OR_NULL(buf->handle)); if (err) goto bail; - VERIFY(err, 0 != (buf->virt = ion_map_kernel(clnt, buf->handle))); + buf->virt = ion_map_kernel(clnt, buf->handle); + VERIFY(err, 0 == IS_ERR_OR_NULL(buf->virt)); if (err) goto bail; if (me->smmu.enabled) { @@ -247,7 +249,8 @@ static int alloc_mem(struct fastrpc_buf *buf) if (err) goto bail; } else { - VERIFY(err, 0 != (sg = ion_sg_table(clnt, buf->handle))); + sg = ion_sg_table(clnt, buf->handle); + VERIFY(err, 0 == IS_ERR_OR_NULL(sg)); if (err) goto bail; VERIFY(err, 1 == sg->nents); @@ -355,6 +358,9 @@ static int get_page_list(uint32_t kernel, uint32_t sc, remote_arg_t *pra, list[i].num = 0; list[i].pgidx = 0; len = pra[i].buf.len; + VERIFY(err, len >= 0); + if (err) + goto bail; if (!len) continue; buf = pra[i].buf.pv; @@ -852,7 +858,7 @@ static int fastrpc_internal_invoke(struct fastrpc_apps *me, uint32_t kernel, context_free(ctx); if (me->smmu.enabled) { - bufs = REMOTE_SCALARS_LENGTH(sc); + bufs = REMOTE_SCALARS_INBUFS(sc) + REMOTE_SCALARS_OUTBUFS(sc); if (fds) { handles = (struct ion_handle **)(fds + bufs); for (i = 0; i < bufs; i++) @@ -1028,7 +1034,8 @@ static int fastrpc_internal_mmap(struct fastrpc_apps *me, VERIFY(err, 0 == IS_ERR_OR_NULL(map->handle)); if (err) goto bail; - VERIFY(err, 0 != (map->virt = ion_map_kernel(clnt, map->handle))); + map->virt = ion_map_kernel(clnt, map->handle); + VERIFY(err, 0 == IS_ERR_OR_NULL(map->virt)); if (err) goto bail; buf = (void *)mmap->vaddrin; diff --git a/drivers/char/diag/diag_dci.c b/drivers/char/diag/diag_dci.c index 3cf965c0d29..5e32a142881 100644 --- a/drivers/char/diag/diag_dci.c +++ b/drivers/char/diag/diag_dci.c @@ -95,7 +95,11 @@ void extract_dci_pkt_rsp(struct diag_smd_info *smd_info, unsigned char *buf) if (recv_pkt_cmd_code != DCI_PKT_RSP_CODE) cmd_code_len = 4; /* delayed response */ write_len = (int)(*(uint16_t *)(buf+2)) - cmd_code_len; - + if (write_len <= 0) { + pr_err("diag: Invalid length in %s, write_len: %d", + __func__, write_len); + return; + } pr_debug("diag: len = %d\n", write_len); /* look up DCI client with tag */ for (i = 0; i < dci_max_reg; i++) { @@ -243,17 +247,23 @@ void extract_dci_events(unsigned char *buf) void extract_dci_log(unsigned char *buf) { - uint16_t log_code, item_num; + uint16_t log_code, item_num, log_length; uint8_t equip_id, *log_mask_ptr, byte_mask; unsigned int i, byte_index, byte_offset = 0; struct diag_dci_client_tbl *entry; + log_length = *(uint16_t *)(buf + 2); log_code = *(uint16_t *)(buf + 6); equip_id = LOG_GET_EQUIP_ID(log_code); item_num = LOG_GET_ITEM_NUM(log_code); byte_index = item_num/8 + 2; byte_mask = 0x01 << (item_num % 8); + if (log_length > USHRT_MAX - 4) { + pr_err("diag: Integer overflow in %s, log_len:%d", + __func__, log_length); + return; + } byte_offset = (equip_id * 514) + byte_index; if (byte_offset >= DCI_LOG_MASK_SIZE) { pr_err("diag: Invalid byte_offset %d in dci log\n", @@ -287,8 +297,8 @@ void extract_dci_log(unsigned char *buf) *(int *)(entry->dci_data+entry->data_len) = DCI_LOG_TYPE; memcpy(entry->dci_data + entry->data_len + 4, - buf + 4, *(uint16_t *)(buf + 2)); - entry->data_len += 4 + *(uint16_t *)(buf + 2); + buf + 4, log_length); + entry->data_len += 4 + log_length; } mutex_unlock(&dci_health_mutex); } @@ -711,6 +721,22 @@ int diag_process_dci_transaction(unsigned char *buf, int len) return ret; } +int diag_dci_find_client_index_health(int client_id) +{ + int i, ret = DCI_CLIENT_INDEX_INVALID; + + for (i = 0; i < MAX_DCI_CLIENTS; i++) { + if (driver->dci_client_tbl[i].client != NULL) { + if (driver->dci_client_tbl[i].client_id == + client_id) { + ret = i; + break; + } + } + } + return ret; +} + int diag_dci_find_client_index(int client_id) { int i, ret = DCI_CLIENT_INDEX_INVALID; diff --git a/drivers/char/diag/diag_dci.h b/drivers/char/diag/diag_dci.h index d530de900f9..79e251a01c4 100644 --- a/drivers/char/diag/diag_dci.h +++ b/drivers/char/diag/diag_dci.h @@ -49,6 +49,7 @@ struct dci_pkt_req_tracking_tbl { }; struct diag_dci_client_tbl { + uint32_t client_id; struct task_struct *client; uint16_t list; /* bit mask */ int signal_type; @@ -65,6 +66,7 @@ struct diag_dci_client_tbl { /* This is used for DCI health stats */ struct diag_dci_health_stats { + int client_id; int dropped_logs; int dropped_events; int received_logs; @@ -99,6 +101,8 @@ int diag_process_dci_transaction(unsigned char *buf, int len); int diag_send_dci_pkt(struct diag_master_table entry, unsigned char *buf, int len, int index); void extract_dci_pkt_rsp(struct diag_smd_info *smd_info, unsigned char *buf); + +int diag_dci_find_client_index_health(int client_id); int diag_dci_find_client_index(int client_id); /* DCI Log streaming functions */ void create_dci_log_mask_tbl(unsigned char *tbl_buf); diff --git a/drivers/char/diag/diag_debugfs.c b/drivers/char/diag/diag_debugfs.c index 71eda68b912..14a9a268e83 100644 --- a/drivers/char/diag/diag_debugfs.c +++ b/drivers/char/diag/diag_debugfs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -30,13 +30,15 @@ static ssize_t diag_dbgfs_read_status(struct file *file, char __user *ubuf, char *buf; int ret; + unsigned int buf_size; buf = kzalloc(sizeof(char) * DEBUG_BUF_SIZE, GFP_KERNEL); if (!buf) { pr_err("diag: %s, Error allocating memory\n", __func__); return -ENOMEM; } - ret = scnprintf(buf, DEBUG_BUF_SIZE, + buf_size = ksize(buf); + ret = scnprintf(buf, buf_size, "modem ch: 0x%x\n" "lpass ch: 0x%x\n" "riva ch: 0x%x\n" @@ -99,7 +101,7 @@ static ssize_t diag_dbgfs_read_status(struct file *file, char __user *ubuf, driver->logging_mode); #ifdef CONFIG_DIAG_OVER_USB - ret += scnprintf(buf+ret, DEBUG_BUF_SIZE, + ret += scnprintf(buf+ret, buf_size-ret, "usb_connected: %d\n", driver->usb_connected); #endif @@ -114,6 +116,7 @@ static ssize_t diag_dbgfs_read_workpending(struct file *file, { char *buf; int ret; + unsigned int buf_size; buf = kzalloc(sizeof(char) * DEBUG_BUF_SIZE, GFP_KERNEL); if (!buf) { @@ -121,7 +124,8 @@ static ssize_t diag_dbgfs_read_workpending(struct file *file, return -ENOMEM; } - ret = scnprintf(buf, DEBUG_BUF_SIZE, + buf_size = ksize(buf); + ret = scnprintf(buf, buf_size, "Pending status for work_stucts:\n" "diag_drain_work: %d\n" "Modem data diag_read_smd_work: %d\n" @@ -169,7 +173,7 @@ static ssize_t diag_dbgfs_read_workpending(struct file *file, diag_notify_update_smd_work))); #ifdef CONFIG_DIAG_OVER_USB - ret += scnprintf(buf+ret, DEBUG_BUF_SIZE, + ret += scnprintf(buf+ret, buf_size-ret, "diag_proc_hdlc_work: %d\n" "diag_read_work: %d\n", work_pending(&(driver->diag_proc_hdlc_work)), @@ -187,10 +191,10 @@ static ssize_t diag_dbgfs_read_table(struct file *file, char __user *ubuf, char *buf; int ret = 0; int i; - int bytes_remaining; - int bytes_in_buffer = 0; - int bytes_written; - int buf_size = (DEBUG_BUF_SIZE < count) ? DEBUG_BUF_SIZE : count; + unsigned int bytes_remaining; + unsigned int bytes_in_buffer = 0; + unsigned int bytes_written; + unsigned int buf_size = (DEBUG_BUF_SIZE < count) ? DEBUG_BUF_SIZE : count; if (diag_dbgfs_table_index >= diag_max_reg) { /* Done. Reset to prepare for future requests */ @@ -203,7 +207,7 @@ static ssize_t diag_dbgfs_read_table(struct file *file, char __user *ubuf, pr_err("diag: %s, Error allocating memory\n", __func__); return -ENOMEM; } - + buf_size = ksize(buf); bytes_remaining = buf_size; if (diag_dbgfs_table_index == 0) { @@ -212,6 +216,7 @@ static ssize_t diag_dbgfs_read_table(struct file *file, char __user *ubuf, "WCNSS: %d, APPS: %d\n", MODEM_DATA, LPASS_DATA, WCNSS_DATA, APPS_DATA); bytes_in_buffer += bytes_written; + bytes_remaining -= bytes_written; } for (i = diag_dbgfs_table_index; i < diag_max_reg; i++) { @@ -254,10 +259,10 @@ static ssize_t diag_dbgfs_read_bridge(struct file *file, char __user *ubuf, char *buf; int ret; int i; - int bytes_remaining; - int bytes_in_buffer = 0; - int bytes_written; - int buf_size = (DEBUG_BUF_SIZE < count) ? DEBUG_BUF_SIZE : count; + unsigned int bytes_remaining; + unsigned int bytes_in_buffer = 0; + unsigned int bytes_written; + unsigned int buf_size = (DEBUG_BUF_SIZE < count) ? DEBUG_BUF_SIZE : count; int bytes_hsic_inited = 45; int bytes_hsic_not_inited = 410; @@ -272,6 +277,7 @@ static ssize_t diag_dbgfs_read_bridge(struct file *file, char __user *ubuf, return -ENOMEM; } + buf_size = ksize(buf); bytes_remaining = buf_size; /* Only one smux for now */ diff --git a/drivers/char/diag/diagchar_core.c b/drivers/char/diag/diagchar_core.c index 76d4d8ac96d..a0c583823b7 100644 --- a/drivers/char/diag/diagchar_core.c +++ b/drivers/char/diag/diagchar_core.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2008-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -47,6 +47,7 @@ MODULE_DESCRIPTION("Diag Char Driver"); MODULE_LICENSE("GPL v2"); MODULE_VERSION("1.0"); +#define MIN_SIZ_ALLOW 4 #define INIT 1 #define EXIT -1 struct diagchar_dev *driver; @@ -839,6 +840,7 @@ int diag_switch_logging(unsigned long ioarg) pr_err("socket process, status: %d\n", status); } + driver->socket_process = NULL; } } else if (driver->logging_mode == SOCKET_MODE) { driver->socket_process = current; @@ -961,6 +963,8 @@ long diagchar_ioctl(struct file *filp, for (i = 0; i < MAX_DCI_CLIENTS; i++) { if (driver->dci_client_tbl[i].client == NULL) { driver->dci_client_tbl[i].client = current; + driver->dci_client_tbl[i].client_id = + driver->dci_client_id; driver->dci_client_tbl[i].list = dci_params->list; driver->dci_client_tbl[i].signal_type = @@ -1041,7 +1045,7 @@ long diagchar_ioctl(struct file *filp, sizeof(struct diag_dci_health_stats))) return -EFAULT; mutex_lock(&dci_health_mutex); - i = diag_dci_find_client_index(current->tgid); + i = diag_dci_find_client_index_health(stats.client_id); if (i != DCI_CLIENT_INDEX_INVALID) { dci_params = &(driver->dci_client_tbl[i]); stats.dropped_logs = dci_params->dropped_logs; @@ -1407,6 +1411,10 @@ static int diagchar_write(struct file *file, const char __user *buf, index = 0; /* Get the packet type F3/log/event/Pkt response */ err = copy_from_user((&pkt_type), buf, 4); + if (err) { + pr_alert("diag: copy failed for pkt_type\n"); + return -EAGAIN; + } /* First 4 bytes indicate the type of payload - ignore these */ if (count < 4) { pr_err("diag: Client sending short data\n"); @@ -1445,8 +1453,9 @@ static int diagchar_write(struct file *file, const char __user *buf, return err; } if (pkt_type == CALLBACK_DATA_TYPE) { - if (payload_size > itemsize) { - pr_err("diag: Dropping packet, packet payload size crosses 4KB limit. Current payload size %d\n", + if (payload_size > driver->itemsize || + payload_size <= MIN_SIZ_ALLOW) { + pr_err("diag: Dropping packet, invalid packet size. Current payload size %d\n", payload_size); driver->dropped_count++; return -EBADMSG; @@ -1588,6 +1597,11 @@ static int diagchar_write(struct file *file, const char __user *buf, remote_proc = diag_get_remote(*(int *)user_space_data); if (remote_proc) { + if (payload_size <= MIN_SIZ_ALLOW) { + pr_err("diag: Integer underflow in %s, payload size: %d", + __func__, payload_size); + return -EBADMSG; + } token_offset = 4; payload_size -= 4; buf += 4; diff --git a/drivers/char/diag/diagchar_hdlc.c b/drivers/char/diag/diagchar_hdlc.c index 2369c4d1c8d..fddff3e8a42 100644 --- a/drivers/char/diag/diagchar_hdlc.c +++ b/drivers/char/diag/diagchar_hdlc.c @@ -176,8 +176,8 @@ int diag_hdlc_decode(struct diag_hdlc_decode_type *hdlc) int msg_start; if (hdlc && hdlc->src_ptr && hdlc->dest_ptr && - (hdlc->src_size - hdlc->src_idx > 0) && - (hdlc->dest_size - hdlc->dest_idx > 0)) { + (hdlc->src_size > hdlc->src_idx) && + (hdlc->dest_size > hdlc->dest_idx)) { msg_start = (hdlc->src_idx == 0) ? 1 : 0; diff --git a/drivers/char/diag/diagfwd.h b/drivers/char/diag/diagfwd.h index c6e1273dc58..9c514e629fb 100644 --- a/drivers/char/diag/diagfwd.h +++ b/drivers/char/diag/diagfwd.h @@ -20,7 +20,7 @@ #define RESET_AND_QUEUE 1 #define CHK_OVERFLOW(bufStart, start, end, length) \ - ((((bufStart) <= (start)) && ((end) - (start) >= (length))) ? 1 : 0) + ((((bufStart) <= (start)) && ((end) - (start) >= (length)) && ((length) > 0)) ? 1 : 0) void diagfwd_init(void); void diagfwd_exit(void); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 32ca70971b4..ef3ba27fabb 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -431,6 +431,9 @@ static ssize_t store_##file_name \ if (ret) \ return -EINVAL; \ \ + new_policy.min = new_policy.user_policy.min; \ + new_policy.max = new_policy.user_policy.max; \ + \ ret = sscanf(buf, "%u", &new_policy.object); \ if (ret != 1) \ return -EINVAL; \ @@ -439,7 +442,9 @@ static ssize_t store_##file_name \ if (ret) \ pr_err("cpufreq: Frequency verification failed\n"); \ \ - policy->user_policy.object = new_policy.object; \ + policy->user_policy.min = new_policy.min; \ + policy->user_policy.max = new_policy.max; \ + \ ret = __cpufreq_set_policy(policy, &new_policy); \ \ return ret ? ret : count; \ @@ -1114,10 +1119,10 @@ static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif #ifdef CONFIG_HOTPLUG_CPU strncpy(per_cpu(cpufreq_policy_save, cpu).gov, data->governor->name, CPUFREQ_NAME_LEN); - per_cpu(cpufreq_policy_save, cpu).min = data->min; - per_cpu(cpufreq_policy_save, cpu).max = data->max; - pr_debug("Saving CPU%d policy min %d and max %d\n", - cpu, data->min, data->max); + per_cpu(cpufreq_policy_save, cpu).min = data->user_policy.min; + per_cpu(cpufreq_policy_save, cpu).max = data->user_policy.max; + pr_debug("Saving CPU%d user policy min %d and max %d\n", + cpu, data->user_policy.min, data->user_policy.max); #endif /* if we have other CPUs still registered, we need to unlink them, @@ -1143,9 +1148,11 @@ static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif #ifdef CONFIG_HOTPLUG_CPU strncpy(per_cpu(cpufreq_policy_save, j).gov, data->governor->name, CPUFREQ_NAME_LEN); - per_cpu(cpufreq_policy_save, j).min = data->min; - per_cpu(cpufreq_policy_save, j).max = data->max; - pr_debug("Saving CPU%d policy min %d and max %d\n", + per_cpu(cpufreq_policy_save, j).min + = data->user_policy.min; + per_cpu(cpufreq_policy_save, j).max + = data->user_policy.max; + pr_debug("Saving CPU%d user policy min %d and max %d\n", j, data->min, data->max); #endif cpu_dev = get_cpu_device(j); diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index d60b5b4fef4..3aa1ec2c5e5 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include static spinlock_t cpufreq_stats_lock; @@ -52,6 +54,12 @@ struct all_cpufreq_stats { unsigned int *freq_table; }; +struct cpufreq_power_stats { + unsigned int state_num; + unsigned int *curr; + unsigned int *freq_table; +}; + struct all_freq_table { unsigned int *freq_table; unsigned int table_size; @@ -61,6 +69,7 @@ static struct all_freq_table *all_freq_table; static DEFINE_PER_CPU(struct all_cpufreq_stats *, all_cpufreq_stats); static DEFINE_PER_CPU(struct cpufreq_stats *, cpufreq_stats_table); +static DEFINE_PER_CPU(struct cpufreq_power_stats *, cpufreq_power_stats); struct cpufreq_stats_attribute { struct attribute attr; @@ -131,6 +140,48 @@ static int get_index_all_cpufreq_stat(struct all_cpufreq_stats *all_stat, return -1; } +void acct_update_power(struct task_struct *task, cputime_t cputime) { + struct cpufreq_power_stats *powerstats; + struct cpufreq_stats *stats; + unsigned int cpu_num, curr; + + if (!task) + return; + cpu_num = task_cpu(task); + powerstats = per_cpu(cpufreq_power_stats, cpu_num); + stats = per_cpu(cpufreq_stats_table, cpu_num); + if (!powerstats || !stats) + return; + + curr = powerstats->curr[stats->last_index]; + if (task->cpu_power != ULLONG_MAX) + task->cpu_power += curr * cputime_to_usecs(cputime); +} +EXPORT_SYMBOL_GPL(acct_update_power); + +static ssize_t show_current_in_state(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + ssize_t len = 0; + unsigned int i, cpu; + struct cpufreq_power_stats *powerstats; + + spin_lock(&cpufreq_stats_lock); + for_each_possible_cpu(cpu) { + powerstats = per_cpu(cpufreq_power_stats, cpu); + if (!powerstats) + continue; + len += scnprintf(buf + len, PAGE_SIZE - len, "CPU%d:", cpu); + for (i = 0; i < powerstats->state_num; i++) + len += scnprintf(buf + len, PAGE_SIZE - len, + "%d=%d ", powerstats->freq_table[i], + powerstats->curr[i]); + len += scnprintf(buf + len, PAGE_SIZE - len, "\n"); + } + spin_unlock(&cpufreq_stats_lock); + return len; +} + static ssize_t show_all_time_in_state(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -240,6 +291,9 @@ static struct attribute_group stats_attr_group = { static struct kobj_attribute _attr_all_time_in_state = __ATTR(all_time_in_state, 0444, show_all_time_in_state, NULL); +static struct kobj_attribute _attr_current_in_state = __ATTR(current_in_state, + 0444, show_current_in_state, NULL); + static int freq_table_get_index(struct cpufreq_stats *stat, unsigned int freq) { int index; @@ -276,19 +330,19 @@ static void cpufreq_stats_free_sysfs(unsigned int cpu) static void cpufreq_allstats_free(void) { - int i; + int cpu; struct all_cpufreq_stats *all_stat; sysfs_remove_file(cpufreq_global_kobject, &_attr_all_time_in_state.attr); - for (i = 0; i < total_cpus; i++) { - all_stat = per_cpu(all_cpufreq_stats, i); + for_each_possible_cpu(cpu) { + all_stat = per_cpu(all_cpufreq_stats, cpu); if (!all_stat) continue; kfree(all_stat->time_in_state); kfree(all_stat); - per_cpu(all_cpufreq_stats, i) = NULL; + per_cpu(all_cpufreq_stats, cpu) = NULL; } if (all_freq_table) { kfree(all_freq_table->freq_table); @@ -297,10 +351,27 @@ static void cpufreq_allstats_free(void) } } +static void cpufreq_powerstats_free(void) +{ + int cpu; + struct cpufreq_power_stats *powerstats; + + sysfs_remove_file(cpufreq_global_kobject, &_attr_current_in_state.attr); + + for_each_possible_cpu(cpu) { + powerstats = per_cpu(cpufreq_power_stats, cpu); + if (!powerstats) + continue; + kfree(powerstats->curr); + kfree(powerstats); + per_cpu(cpufreq_power_stats, cpu) = NULL; + } +} + static int cpufreq_stats_create_table(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table) + struct cpufreq_frequency_table *table, int count) { - unsigned int i, j, count = 0, ret = 0; + unsigned int i, j, ret = 0; struct cpufreq_stats *stat; struct cpufreq_policy *data; unsigned int alloc_size; @@ -324,12 +395,6 @@ static int cpufreq_stats_create_table(struct cpufreq_policy *policy, stat->cpu = cpu; per_cpu(cpufreq_stats_table, cpu) = stat; - for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - count++; - } alloc_size = count * sizeof(int) + count * sizeof(cputime64_t); @@ -370,6 +435,54 @@ static int cpufreq_stats_create_table(struct cpufreq_policy *policy, return ret; } +static void cpufreq_powerstats_create(unsigned int cpu, + struct cpufreq_frequency_table *table, int count) { + unsigned int alloc_size, i = 0, j = 0, ret = 0; + struct cpufreq_power_stats *powerstats; + struct device_node *cpu_node; + char device_path[16]; + + powerstats = kzalloc(sizeof(struct cpufreq_power_stats), + GFP_KERNEL); + if (!powerstats) + return; + + /* Allocate memory for freq table per cpu as well as clockticks per + * freq*/ + alloc_size = count * sizeof(unsigned int) + + count * sizeof(unsigned int); + powerstats->curr = kzalloc(alloc_size, GFP_KERNEL); + if (!powerstats->curr) { + kfree(powerstats); + return; + } + powerstats->freq_table = powerstats->curr + count; + + spin_lock(&cpufreq_stats_lock); + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END && j < count; i++) { + unsigned int freq = table[i].frequency; + + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + powerstats->freq_table[j++] = freq; + } + powerstats->state_num = j; + + snprintf(device_path, sizeof(device_path), "/cpus/cpu@%d", cpu); + cpu_node = of_find_node_by_path(device_path); + if (cpu_node) { + ret = of_property_read_u32_array(cpu_node, "current", + powerstats->curr, count); + if (ret) { + kfree(powerstats->curr); + kfree(powerstats); + powerstats = NULL; + } + } + per_cpu(cpufreq_power_stats, cpu) = powerstats; + spin_unlock(&cpufreq_stats_lock); +} + static int compare_for_sort(const void *lhs_ptr, const void *rhs_ptr) { unsigned int lhs = *(const unsigned int *)(lhs_ptr); @@ -410,24 +523,14 @@ static void add_all_freq_table(unsigned int freq) all_freq_table->freq_table[all_freq_table->table_size++] = freq; } -static void cpufreq_allstats_create(unsigned int cpu) +static void cpufreq_allstats_create(unsigned int cpu, + struct cpufreq_frequency_table *table, int count) { int i , j = 0; - unsigned int alloc_size, count = 0; - struct cpufreq_frequency_table *table = cpufreq_frequency_get_table(cpu); + unsigned int alloc_size; struct all_cpufreq_stats *all_stat; bool sort_needed = false; - if (!table) - return; - - for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - count++; - } - all_stat = kzalloc(sizeof(struct all_cpufreq_stats), GFP_KERNEL); if (!all_stat) { @@ -469,7 +572,7 @@ static void cpufreq_allstats_create(unsigned int cpu) static int cpufreq_stat_notifier_policy(struct notifier_block *nb, unsigned long val, void *data) { - int ret; + int ret, count = 0, i; struct cpufreq_policy *policy = data; struct cpufreq_frequency_table *table; unsigned int cpu = policy->cpu; @@ -479,10 +582,21 @@ static int cpufreq_stat_notifier_policy(struct notifier_block *nb, if (!table) return 0; + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned int freq = table[i].frequency; + + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + count++; + } + if (!per_cpu(all_cpufreq_stats, cpu)) - cpufreq_allstats_create(cpu); + cpufreq_allstats_create(cpu, table, count); - ret = cpufreq_stats_create_table(policy, table); + if (!per_cpu(cpufreq_power_stats, cpu)) + cpufreq_powerstats_create(cpu, table, count); + + ret = cpufreq_stats_create_table(policy, table, count); if (ret) return ret; return 0; @@ -528,7 +642,7 @@ static int cpufreq_stats_create_table_cpu(unsigned int cpu) { struct cpufreq_policy *policy; struct cpufreq_frequency_table *table; - int ret = -ENODEV; + int ret = -ENODEV, i, count = 0; policy = cpufreq_cpu_get(cpu); if (!policy) @@ -538,10 +652,21 @@ static int cpufreq_stats_create_table_cpu(unsigned int cpu) if (!table) goto out; + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned int freq = table[i].frequency; + + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + count++; + } + if (!per_cpu(all_cpufreq_stats, cpu)) - cpufreq_allstats_create(cpu); + cpufreq_allstats_create(cpu, table, count); + + if (!per_cpu(cpufreq_power_stats, cpu)) + cpufreq_powerstats_create(cpu, table, count); - ret = cpufreq_stats_create_table(policy, table); + ret = cpufreq_stats_create_table(policy, table, count); out: cpufreq_cpu_put(policy); @@ -617,7 +742,12 @@ static int __init cpufreq_stats_init(void) ret = sysfs_create_file(cpufreq_global_kobject, &_attr_all_time_in_state.attr); if (ret) - pr_warn("Error creating sysfs file for cpufreq stats\n"); + pr_warn("Cannot create sysfs file for cpufreq stats\n"); + + ret = sysfs_create_file(cpufreq_global_kobject, + &_attr_current_in_state.attr); + if (ret) + pr_warn("Cannot create sysfs file for cpufreq current stats\n"); return 0; } @@ -635,6 +765,7 @@ static void __exit cpufreq_stats_exit(void) cpufreq_stats_free_sysfs(cpu); } cpufreq_allstats_free(); + cpufreq_powerstats_free(); } MODULE_AUTHOR("Zou Nan hai "); diff --git a/drivers/crypto/msm/qce50.c b/drivers/crypto/msm/qce50.c index e237db5fcde..dec50d8216a 100644 --- a/drivers/crypto/msm/qce50.c +++ b/drivers/crypto/msm/qce50.c @@ -2944,7 +2944,6 @@ int qce_aead_req(void *handle, struct qce_req *q_req) ce_burst_size = pce_dev->ce_sps.ce_burst_size; if (q_req->dir == QCE_ENCRYPT) { q_req->cryptlen = areq->cryptlen; - totallen_in = q_req->cryptlen + areq->assoclen + ivsize; if (q_req->mode == QCE_MODE_CCM) { out_len = areq->cryptlen + authsize; hw_pad_out = ALIGN(authsize, ce_burst_size) - authsize; @@ -2953,14 +2952,18 @@ int qce_aead_req(void *handle, struct qce_req *q_req) } } else { q_req->cryptlen = areq->cryptlen - authsize; - if (q_req->mode == QCE_MODE_CCM) - totallen_in = areq->cryptlen + areq->assoclen; - else - totallen_in = q_req->cryptlen + areq->assoclen + ivsize; out_len = q_req->cryptlen; hw_pad_out = authsize; } + if ((q_req->cryptlen > UINT_MAX - areq->assoclen) || + (q_req->cryptlen + areq->assoclen > UINT_MAX - ivsize)) { + pr_err("Integer overflow on total aead req length.\n"); + return -EINVAL; + } + + totallen_in = q_req->cryptlen + areq->assoclen + ivsize; + pce_dev->assoc_nents = count_sg(areq->assoc, areq->assoclen); pce_dev->src_nents = count_sg(areq->src, areq->cryptlen); pce_dev->ivsize = q_req->ivsize; diff --git a/drivers/crypto/msm/qcedev.c b/drivers/crypto/msm/qcedev.c index 95803b4f453..dcdb34c0505 100644 --- a/drivers/crypto/msm/qcedev.c +++ b/drivers/crypto/msm/qcedev.c @@ -1421,44 +1421,6 @@ static int qcedev_vbuf_ablk_cipher(struct qcedev_async_req *areq, struct qcedev_cipher_op_req *saved_req; struct qcedev_cipher_op_req *creq = &areq->cipher_op_req; - /* Verify Source Address's */ - for (i = 0; i < areq->cipher_op_req.entries; i++) - if (!access_ok(VERIFY_READ, - (void __user *)areq->cipher_op_req.vbuf.src[i].vaddr, - areq->cipher_op_req.vbuf.src[i].len)) - return -EFAULT; - - /* Verify Destination Address's */ - if (creq->in_place_op != 1) { - for (i = 0, total = 0; i < QCEDEV_MAX_BUFFERS; i++) { - if ((areq->cipher_op_req.vbuf.dst[i].vaddr != 0) && - (total < creq->data_len)) { - if (!access_ok(VERIFY_WRITE, - (void __user *)creq->vbuf.dst[i].vaddr, - creq->vbuf.dst[i].len)) { - pr_err("%s:DST WR_VERIFY err %d=0x%x\n", - __func__, i, - (u32)creq->vbuf.dst[i].vaddr); - return -EFAULT; - } - total += creq->vbuf.dst[i].len; - } - } - } else { - for (i = 0, total = 0; i < creq->entries; i++) { - if (total < creq->data_len) { - if (!access_ok(VERIFY_WRITE, - (void __user *)creq->vbuf.src[i].vaddr, - creq->vbuf.src[i].len)) { - pr_err("%s:SRC WR_VERIFY err %d=0x%x\n", - __func__, i, - (u32)creq->vbuf.src[i].vaddr); - return -EFAULT; - } - total += creq->vbuf.src[i].len; - } - } - } total = 0; if (areq->cipher_op_req.mode == QCEDEV_AES_MODE_CTR) @@ -1730,6 +1692,36 @@ static int qcedev_check_cipher_params(struct qcedev_cipher_op_req *req, __func__, total, req->data_len); goto error; } + /* Verify Source Address's */ + for (i = 0, total = 0; i < req->entries; i++) { + if (total < req->data_len) { + if (!access_ok(VERIFY_READ, + (void __user *)req->vbuf.src[i].vaddr, + req->vbuf.src[i].len)) { + pr_err("%s:SRC RD_VERIFY err %d=0x%lx\n", + __func__, i, (uintptr_t) + req->vbuf.src[i].vaddr); + goto error; + } + total += req->vbuf.src[i].len; + } + } + + /* Verify Destination Address's */ + for (i = 0, total = 0; i < QCEDEV_MAX_BUFFERS; i++) { + if ((req->vbuf.dst[i].vaddr != 0) && + (total < req->data_len)) { + if (!access_ok(VERIFY_WRITE, + (void __user *)req->vbuf.dst[i].vaddr, + req->vbuf.dst[i].len)) { + pr_err("%s:DST WR_VERIFY err %d=0x%lx\n", + __func__, i, (uintptr_t) + req->vbuf.dst[i].vaddr); + goto error; + } + total += req->vbuf.dst[i].len; + } + } return 0; error: return -EINVAL; diff --git a/drivers/gpio/gpio-msm-common.c b/drivers/gpio/gpio-msm-common.c index 949d72d4398..09a94e518ff 100644 --- a/drivers/gpio/gpio-msm-common.c +++ b/drivers/gpio/gpio-msm-common.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -412,71 +411,6 @@ static struct irq_chip msm_gpio_irq_chip = { .irq_disable = msm_gpio_irq_disable, }; -#ifdef CONFIG_PM -static int msm_gpio_suspend(void) -{ - unsigned long irq_flags; - unsigned long i; - int ngpio = msm_gpio.gpio_chip.ngpio; - - spin_lock_irqsave(&tlmm_lock, irq_flags); - for_each_set_bit(i, msm_gpio.enabled_irqs, ngpio) - __msm_gpio_set_intr_cfg_enable(i, 0); - - for_each_set_bit(i, msm_gpio.wake_irqs, ngpio) - __msm_gpio_set_intr_cfg_enable(i, 1); - mb(); - spin_unlock_irqrestore(&tlmm_lock, irq_flags); - return 0; -} - -void msm_gpio_show_resume_irq(void) -{ - unsigned long irq_flags; - int i, irq, intstat; - int ngpio = msm_gpio.gpio_chip.ngpio; - - if (!msm_show_resume_irq_mask) - return; - - spin_lock_irqsave(&tlmm_lock, irq_flags); - for_each_set_bit(i, msm_gpio.wake_irqs, ngpio) { - intstat = __msm_gpio_get_intr_status(i); - if (intstat) { - irq = msm_gpio_to_irq(&msm_gpio.gpio_chip, i); - log_wakeup_reason(irq); - } - } - spin_unlock_irqrestore(&tlmm_lock, irq_flags); -} - -static void msm_gpio_resume(void) -{ - unsigned long irq_flags; - unsigned long i; - int ngpio = msm_gpio.gpio_chip.ngpio; - - msm_gpio_show_resume_irq(); - - spin_lock_irqsave(&tlmm_lock, irq_flags); - for_each_set_bit(i, msm_gpio.wake_irqs, ngpio) - __msm_gpio_set_intr_cfg_enable(i, 0); - - for_each_set_bit(i, msm_gpio.enabled_irqs, ngpio) - __msm_gpio_set_intr_cfg_enable(i, 1); - mb(); - spin_unlock_irqrestore(&tlmm_lock, irq_flags); -} -#else -#define msm_gpio_suspend NULL -#define msm_gpio_resume NULL -#endif - -static struct syscore_ops msm_gpio_syscore_ops = { - .suspend = msm_gpio_suspend, - .resume = msm_gpio_resume, -}; - static void msm_tlmm_set_field(const struct tlmm_field_cfg *configs, unsigned id, unsigned width, unsigned val) { @@ -633,7 +567,6 @@ static int __devinit msm_gpio_probe(struct platform_device *pdev) ret); return ret; } - register_syscore_ops(&msm_gpio_syscore_ops); return 0; } @@ -648,7 +581,6 @@ static int __devexit msm_gpio_remove(struct platform_device *pdev) { int ret; - unregister_syscore_ops(&msm_gpio_syscore_ops); ret = gpiochip_remove(&msm_gpio.gpio_chip); if (ret < 0) return ret; diff --git a/drivers/gpu/ion/ion.c b/drivers/gpu/ion/ion.c index 6777dae4cd2..f569e63b8f5 100644 --- a/drivers/gpu/ion/ion.c +++ b/drivers/gpu/ion/ion.c @@ -684,7 +684,7 @@ static int ion_debug_client_show(struct seq_file *s, void *unused) struct ion_handle *handle = rb_entry(n, struct ion_handle, node); - seq_printf(s, "%16.16s: %16x : %16d : %12p", + seq_printf(s, "%16.16s: %16x : %16d : %12pK", handle->buffer->heap->name, handle->buffer->size, atomic_read(&handle->ref.refcount), @@ -1002,7 +1002,7 @@ static void ion_vm_open(struct vm_area_struct *vma) mutex_lock(&buffer->lock); list_add(&vma_list->list, &buffer->vmas); mutex_unlock(&buffer->lock); - pr_debug("%s: adding %p\n", __func__, vma); + pr_debug("%s: adding %pK\n", __func__, vma); } static void ion_vm_close(struct vm_area_struct *vma) @@ -1017,7 +1017,7 @@ static void ion_vm_close(struct vm_area_struct *vma) continue; list_del(&vma_list->list); kfree(vma_list); - pr_debug("%s: deleting %p\n", __func__, vma); + pr_debug("%s: deleting %pK\n", __func__, vma); break; } mutex_unlock(&buffer->lock); diff --git a/drivers/gpu/ion/ion_heap.c b/drivers/gpu/ion/ion_heap.c index 3d37541978c..3832f231211 100644 --- a/drivers/gpu/ion/ion_heap.c +++ b/drivers/gpu/ion/ion_heap.c @@ -137,8 +137,9 @@ struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) switch (heap_data->type) { case ION_HEAP_TYPE_SYSTEM_CONTIG: - heap = ion_system_contig_heap_create(heap_data); - break; + pr_err("%s: Heap type is disabled: %d\n", __func__, + heap_data->type); + return ERR_PTR(-EINVAL); case ION_HEAP_TYPE_SYSTEM: heap = ion_system_heap_create(heap_data); break; @@ -174,7 +175,8 @@ void ion_heap_destroy(struct ion_heap *heap) switch (heap->type) { case ION_HEAP_TYPE_SYSTEM_CONTIG: - ion_system_contig_heap_destroy(heap); + pr_err("%s: Heap type is disabled: %d\n", __func__, + heap->type); break; case ION_HEAP_TYPE_SYSTEM: ion_system_heap_destroy(heap); diff --git a/drivers/gpu/ion/ion_iommu_heap.c b/drivers/gpu/ion/ion_iommu_heap.c index 16e2eea78c9..f9ca80ac7a8 100644 --- a/drivers/gpu/ion/ion_iommu_heap.c +++ b/drivers/gpu/ion/ion_iommu_heap.c @@ -52,7 +52,7 @@ struct ion_iommu_priv_data { #define MAX_VMAP_RETRIES 10 #define BAD_ORDER -1 -static const unsigned int orders[] = {8, 4, 0}; +static const unsigned int orders[] = {0}; static const int num_orders = ARRAY_SIZE(orders); static unsigned int low_gfp_flags = __GFP_HIGHMEM | GFP_KERNEL | __GFP_ZERO; static unsigned int high_gfp_flags = (__GFP_HIGHMEM | __GFP_NORETRY @@ -534,9 +534,9 @@ struct ion_heap *ion_iommu_heap_create(struct ion_platform_heap *heap_data) gfp_t gfp_flags; if (orders[i]) - gfp_flags = high_gfp_flags | __GFP_ZERO; + gfp_flags = high_gfp_flags; else - gfp_flags = low_gfp_flags | __GFP_ZERO; + gfp_flags = low_gfp_flags; pool = ion_page_pool_create(gfp_flags, orders[i]); if (!pool) goto err_create_cached_pool; @@ -548,9 +548,9 @@ struct ion_heap *ion_iommu_heap_create(struct ion_platform_heap *heap_data) gfp_t gfp_flags; if (orders[i]) - gfp_flags = high_gfp_flags | __GFP_ZERO; + gfp_flags = high_gfp_flags; else - gfp_flags = low_gfp_flags | __GFP_ZERO; + gfp_flags = low_gfp_flags; pool = ion_page_pool_create(gfp_flags, orders[i]); if (!pool) goto err_create_uncached_pool; diff --git a/drivers/gpu/msm/a3xx_reg.h b/drivers/gpu/msm/a3xx_reg.h index 676f46da39a..36b03e2b5ba 100644 --- a/drivers/gpu/msm/a3xx_reg.h +++ b/drivers/gpu/msm/a3xx_reg.h @@ -487,6 +487,11 @@ #define A3XX_VBIF_PERF_PWR_CNT2_LO 0x307b #define A3XX_VBIF_PERF_PWR_CNT2_HI 0x307c +#define A3XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A3XX_VBIF_XIN_HALT_CTRL0_MASK 0x3F +#define A3XX_VBIF_XIN_HALT_CTRL1 0x3081 + + /* Bit flags for RBBM_CTL */ #define RBBM_RBBM_CTL_RESET_PWR_CTR0 BIT(0) #define RBBM_RBBM_CTL_RESET_PWR_CTR1 BIT(1) diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index dd3f4072d1b..837e15c912c 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -1760,14 +1760,12 @@ static int adreno_start(struct kgsl_device *device) kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_ON); device->ftbl->irqctrl(device, 1); + adreno_perfcounter_start(adreno_dev); + status = adreno_ringbuffer_cold_start(&adreno_dev->ringbuffer); if (status) goto error_irq_off; - status = adreno_perfcounter_start(adreno_dev); - if (status) - goto error_rb_stop; - /* Start the dispatcher */ adreno_dispatcher_start(device); @@ -1777,8 +1775,6 @@ static int adreno_start(struct kgsl_device *device) return 0; -error_rb_stop: - adreno_ringbuffer_stop(&adreno_dev->ringbuffer); error_irq_off: kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); @@ -1793,19 +1789,44 @@ static int adreno_start(struct kgsl_device *device) return status; } +/** + * adreno_vbif_clear_pending_transactions() - Clear transactions in VBIF pipe + * @device: Pointer to the device whose VBIF pipe is to be cleared + */ +static void adreno_vbif_clear_pending_transactions(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + unsigned int mask = A3XX_VBIF_XIN_HALT_CTRL0_MASK; + unsigned int val; + unsigned long wait_for_vbif; + + adreno_writereg(adreno_dev, ADRENO_REG_VBIF_XIN_HALT_CTRL0, mask); + /* wait for the transactions to clear */ + wait_for_vbif = jiffies + msecs_to_jiffies(100); + while (1) { + adreno_readreg(adreno_dev, + ADRENO_REG_VBIF_XIN_HALT_CTRL1, &val); + if ((val & mask) == mask) + break; + if (time_after(jiffies, wait_for_vbif)) + break; + } + adreno_writereg(adreno_dev, ADRENO_REG_VBIF_XIN_HALT_CTRL0, 0); +} + + static int adreno_stop(struct kgsl_device *device) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - if (adreno_dev->drawctxt_active) - kgsl_context_put(&adreno_dev->drawctxt_active->base); + if (!test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)) + return 0; + + kgsl_pwrctrl_enable(device); adreno_dev->drawctxt_active = NULL; adreno_dispatcher_stop(adreno_dev); - adreno_ringbuffer_stop(&adreno_dev->ringbuffer); - - kgsl_mmu_stop(&device->mmu); device->ftbl->irqctrl(device, 0); kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); @@ -1813,6 +1834,8 @@ static int adreno_stop(struct kgsl_device *device) adreno_ocmem_gmem_free(adreno_dev); + kgsl_mmu_stop(&device->mmu); + /* Power down the device */ kgsl_pwrctrl_disable(device); @@ -1834,15 +1857,13 @@ static int adreno_stop(struct kgsl_device *device) int adreno_reset(struct kgsl_device *device) { int ret = -EINVAL; - struct kgsl_mmu *mmu = &device->mmu; int i = 0; - /* Try soft reset first, for non mmu fault case only */ - if (!atomic_read(&mmu->fault)) { - ret = adreno_soft_reset(device); - if (ret) - KGSL_DEV_ERR_ONCE(device, "Device soft reset failed\n"); - } + /* clear pending vbif transactions before reset */ + adreno_vbif_clear_pending_transactions(device); + + /* Try soft reset first */ + ret = adreno_soft_reset(device); if (ret) { /* If soft reset failed/skipped, then pull the power */ adreno_stop(device); @@ -2403,14 +2424,13 @@ int adreno_soft_reset(struct kgsl_device *device) return -EINVAL; } + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + if (adreno_dev->drawctxt_active) kgsl_context_put(&adreno_dev->drawctxt_active->base); adreno_dev->drawctxt_active = NULL; - /* Stop the ringbuffer */ - adreno_ringbuffer_stop(&adreno_dev->ringbuffer); - if (kgsl_pwrctrl_isenabled(device)) device->ftbl->irqctrl(device, 0); @@ -2446,8 +2466,11 @@ int adreno_soft_reset(struct kgsl_device *device) if (ret) return ret; + else { + device->reset_counter++; + set_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + } - device->reset_counter++; return 0; } diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 23309a12815..15084a08635 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -314,6 +314,8 @@ enum adreno_regs { ADRENO_REG_TP0_CHICKEN, ADRENO_REG_RBBM_RBBM_CTL, ADRENO_REG_UCHE_INVALIDATE0, + ADRENO_REG_VBIF_XIN_HALT_CTRL0, + ADRENO_REG_VBIF_XIN_HALT_CTRL1, ADRENO_REG_REGISTER_MAX, }; diff --git a/drivers/gpu/msm/adreno_a3xx.c b/drivers/gpu/msm/adreno_a3xx.c index 65d1d063cd2..a68635f36f8 100644 --- a/drivers/gpu/msm/adreno_a3xx.c +++ b/drivers/gpu/msm/adreno_a3xx.c @@ -4407,6 +4407,10 @@ static unsigned int a3xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { ADRENO_REG_DEFINE(ADRENO_REG_RBBM_RBBM_CTL, A3XX_RBBM_RBBM_CTL), ADRENO_REG_DEFINE(ADRENO_REG_UCHE_INVALIDATE0, A3XX_UCHE_CACHE_INVALIDATE0_REG), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL0, + A3XX_VBIF_XIN_HALT_CTRL0), + ADRENO_REG_DEFINE(ADRENO_REG_VBIF_XIN_HALT_CTRL1, + A3XX_VBIF_XIN_HALT_CTRL1), }; const struct adreno_reg_offsets a3xx_reg_offsets = { diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index ddf275c56cc..3f39264dd1d 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -965,6 +965,8 @@ static int dispatcher_do_fault(struct kgsl_device *device) /* Skip the PM dump for a timeout because it confuses people */ set_bit(KGSL_FT_SKIP_PMDUMP, &cmdbatch->fault_policy); } + /* Set pagefault if it occurred */ + kgsl_mmu_set_pagefault(&device->mmu); adreno_readreg(adreno_dev, ADRENO_REG_CP_IB1_BASE, &base); diff --git a/drivers/gpu/msm/adreno_drawctxt.c b/drivers/gpu/msm/adreno_drawctxt.c index fa03a06cf6c..16448610afe 100644 --- a/drivers/gpu/msm/adreno_drawctxt.c +++ b/drivers/gpu/msm/adreno_drawctxt.c @@ -545,9 +545,14 @@ int adreno_drawctxt_detach(struct kgsl_context *context) */ BUG_ON(!mutex_is_locked(&device->mutex)); - /* Wait for the last global timestamp to pass before continuing */ + /* Wait for the last global timestamp to pass before continuing. + * The maxumum wait time is 30s, some large IB's can take longer + * than 10s and if hang happens then the time for the context's + * commands to retire will be greater than 10s. 30s should be sufficient + * time to wait for the commands even if a hang happens. + */ ret = adreno_drawctxt_wait_global(adreno_dev, context, - drawctxt->internal_timestamp, 10 * 1000); + drawctxt->internal_timestamp, 30 * 1000); /* * If the wait for global fails then nothing after this point is likely diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index 6b0eb7389c2..372f7dd150d 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -436,9 +436,6 @@ int _ringbuffer_start_common(struct adreno_ringbuffer *rb) /* idle device to validate ME INIT */ status = adreno_idle(device); - if (status == 0) - rb->flags |= KGSL_FLAGS_STARTED; - return status; } @@ -455,9 +452,6 @@ int adreno_ringbuffer_warm_start(struct adreno_ringbuffer *rb) struct kgsl_device *device = rb->device; struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - if (rb->flags & KGSL_FLAGS_STARTED) - return 0; - _ringbuffer_setup_common(rb); /* If bootstrapping if supported to load jump tables */ @@ -499,8 +493,6 @@ int adreno_ringbuffer_cold_start(struct adreno_ringbuffer *rb) struct kgsl_device *device = rb->device; struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - if (rb->flags & KGSL_FLAGS_STARTED) - return 0; _ringbuffer_setup_common(rb); @@ -550,19 +542,6 @@ int adreno_ringbuffer_cold_start(struct adreno_ringbuffer *rb) return status; } -void adreno_ringbuffer_stop(struct adreno_ringbuffer *rb) -{ - struct kgsl_device *device = rb->device; - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - - if (rb->flags & KGSL_FLAGS_STARTED) { - if (adreno_is_a200(adreno_dev)) - kgsl_regwrite(rb->device, REG_CP_ME_CNTL, 0x10000000); - - rb->flags &= ~KGSL_FLAGS_STARTED; - } -} - int adreno_ringbuffer_init(struct kgsl_device *device) { int status; @@ -961,7 +940,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, level++; - KGSL_CMD_INFO(dev_priv->device, "ib: gpuaddr:0x%08x, wc:%d, hptr:%p\n", + KGSL_CMD_INFO(dev_priv->device, "ib: gpuaddr:0x%08x, wc:%d, hptr:%pK\n", gpuaddr, sizedwords, hostaddr); mb(); @@ -983,7 +962,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, break; default: KGSL_CMD_ERR(dev_priv->device, "unexpected type: " - "type:%d, word:0x%08x @ 0x%p, gpu:0x%08x\n", + "type:%d, word:0x%08x @ 0x%pK, gpu:0x%08x\n", *hostaddr >> 30, *hostaddr, hostaddr, gpuaddr+4*(sizedwords-dwords_left)); cur_ret = false; @@ -994,7 +973,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, if (!cur_ret) { KGSL_CMD_ERR(dev_priv->device, "bad sub-type: #:%d/%d, v:0x%08x" - " @ 0x%p[gb:0x%08x], level:%d\n", + " @ 0x%pK[gb:0x%08x], level:%d\n", sizedwords-dwords_left, sizedwords, *hostaddr, hostaddr, gpuaddr+4*(sizedwords-dwords_left), level); @@ -1014,7 +993,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, if (dwords_left < 0) { KGSL_CMD_ERR(dev_priv->device, "bad count: c:%d, #:%d/%d, " - "v:0x%08x @ 0x%p[gb:0x%08x], level:%d\n", + "v:0x%08x @ 0x%pK[gb:0x%08x], level:%d\n", count, sizedwords-(dwords_left+count), sizedwords, *(hostaddr-count), hostaddr-count, gpuaddr+4*(sizedwords-(dwords_left+count)), @@ -1034,7 +1013,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, if (!ret) KGSL_DRV_ERR(dev_priv->device, "parsing failed: gpuaddr:0x%08x, " - "host:0x%p, wc:%d\n", gpuaddr, hoststart, sizedwords); + "host:0x%pK, wc:%d\n", gpuaddr, hoststart, sizedwords); level--; diff --git a/drivers/gpu/msm/adreno_ringbuffer.h b/drivers/gpu/msm/adreno_ringbuffer.h index 697e113c576..59c60cebdfc 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.h +++ b/drivers/gpu/msm/adreno_ringbuffer.h @@ -75,8 +75,6 @@ int adreno_ringbuffer_warm_start(struct adreno_ringbuffer *rb); int adreno_ringbuffer_cold_start(struct adreno_ringbuffer *rb); -void adreno_ringbuffer_stop(struct adreno_ringbuffer *rb); - void adreno_ringbuffer_close(struct adreno_ringbuffer *rb); unsigned int adreno_ringbuffer_issuecmds(struct kgsl_device *device, diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index efc4c69210a..0b005af92dd 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2008-2013,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -246,18 +246,13 @@ kgsl_mem_entry_destroy(struct kref *kref) EXPORT_SYMBOL(kgsl_mem_entry_destroy); /** - * kgsl_mem_entry_track_gpuaddr - Insert a mem_entry in the address tree and - * assign it with a gpu address space before insertion + * kgsl_mem_entry_track_gpuaddr - Get the entry gpu address space before + * insertion to the process * @process: the process that owns the memory * @entry: the memory entry * - * @returns - 0 on succcess else error code + * @returns - 0 on success else error code * - * Insert the kgsl_mem_entry in to the rb_tree for searching by GPU address. - * The assignment of gpu address and insertion into list needs to - * happen with the memory lock held to avoid race conditions between - * gpu address being selected and some other thread looking through the - * rb list in search of memory based on gpuaddr * This function should be called with processes memory spinlock held */ static int @@ -265,8 +260,6 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process, struct kgsl_mem_entry *entry) { int ret = 0; - struct rb_node **node; - struct rb_node *parent = NULL; assert_spin_locked(&process->mem_lock); /* @@ -274,36 +267,17 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process, * gpu address */ if (kgsl_memdesc_use_cpu_map(&entry->memdesc)) { - if (!entry->memdesc.gpuaddr) - goto done; - } else if (entry->memdesc.gpuaddr) { - WARN_ONCE(1, "gpuaddr assigned w/o holding memory lock\n"); - ret = -EINVAL; - goto done; - } - if (!kgsl_memdesc_use_cpu_map(&entry->memdesc)) { - ret = kgsl_mmu_get_gpuaddr(process->pagetable, &entry->memdesc); - if (ret) + /* cpu map flag is enabled. do nothing */ + } else { + if (entry->memdesc.gpuaddr) { + WARN_ONCE(1, "gpuaddr assigned w/o holding memory lock\n"); + ret = -EINVAL; goto done; - } - - node = &process->mem_rb.rb_node; - - while (*node) { - struct kgsl_mem_entry *cur; - - parent = *node; - cur = rb_entry(parent, struct kgsl_mem_entry, node); + } - if (entry->memdesc.gpuaddr < cur->memdesc.gpuaddr) - node = &parent->rb_left; - else - node = &parent->rb_right; + ret = kgsl_mmu_get_gpuaddr(process->pagetable, &entry->memdesc); } - rb_link_node(&entry->node, parent, node); - rb_insert_color(&entry->node, &process->mem_rb); - done: return ret; } @@ -327,6 +301,47 @@ kgsl_mem_entry_untrack_gpuaddr(struct kgsl_process_private *process, } } +static void kgsl_mem_entry_commit_mem_list(struct kgsl_process_private *process, + struct kgsl_mem_entry *entry) +{ + struct rb_node **node; + struct rb_node *parent = NULL; + + if (!entry->memdesc.gpuaddr) + return; + + /* Insert mem entry in mem_rb tree */ + node = &process->mem_rb.rb_node; + while (*node) { + struct kgsl_mem_entry *cur; + + parent = *node; + cur = rb_entry(parent, struct kgsl_mem_entry, node); + + if (entry->memdesc.gpuaddr < cur->memdesc.gpuaddr) + node = &parent->rb_left; + else + node = &parent->rb_right; + } + + rb_link_node(&entry->node, parent, node); + rb_insert_color(&entry->node, &process->mem_rb); +} + +static void kgsl_mem_entry_commit_process(struct kgsl_process_private *process, + struct kgsl_mem_entry *entry) +{ + if (!entry) + return; + + spin_lock(&entry->priv->mem_lock); + /* Insert mem entry in mem_rb tree */ + kgsl_mem_entry_commit_mem_list(process, entry); + /* Replace mem entry in mem_idr using id */ + idr_replace(&entry->priv->mem_idr, entry, entry->id); + spin_unlock(&entry->priv->mem_lock); +} + /** * kgsl_mem_entry_attach_process - Attach a mem_entry to its owner process * @entry: the memory entry @@ -357,9 +372,11 @@ kgsl_mem_entry_attach_process(struct kgsl_mem_entry *entry, } spin_lock(&process->mem_lock); - ret = idr_get_new_above(&process->mem_idr, entry, 1, + /* Allocate the ID but don't attach the pointer just yet */ + ret = idr_get_new_above(&process->mem_idr, NULL, 1, &entry->id); spin_unlock(&process->mem_lock); + if (ret == 0) break; else if (ret != -EAGAIN) @@ -961,8 +978,6 @@ int kgsl_close_device(struct kgsl_device *device) /* Fail if the wait times out */ BUG_ON(atomic_read(&device->active_cnt) > 0); - /* Force power on to do the stop */ - kgsl_pwrctrl_enable(device); result = device->ftbl->stop(device); kgsl_pwrctrl_set_state(device, KGSL_STATE_INIT); } @@ -2896,6 +2911,7 @@ static long kgsl_ioctl_map_user_mem(struct kgsl_device_private *dev_priv, trace_kgsl_mem_map(entry, param->fd); + kgsl_mem_entry_commit_process(private, entry); return result; error_attach: @@ -3183,6 +3199,8 @@ kgsl_ioctl_gpumem_alloc(struct kgsl_device_private *dev_priv, param->gpuaddr = entry->memdesc.gpuaddr; param->size = entry->memdesc.size; param->flags = entry->memdesc.flags; + + kgsl_mem_entry_commit_process(private, entry); return result; err: kgsl_sharedmem_free(&entry->memdesc); @@ -3219,6 +3237,8 @@ kgsl_ioctl_gpumem_alloc_id(struct kgsl_device_private *dev_priv, param->size = entry->memdesc.size; param->mmapsize = kgsl_memdesc_mmapsize(&entry->memdesc); param->gpuaddr = entry->memdesc.gpuaddr; + + kgsl_mem_entry_commit_process(private, entry); return result; err: if (entry) @@ -3806,6 +3826,11 @@ kgsl_get_unmapped_area(struct file *file, unsigned long addr, kgsl_mem_entry_untrack_gpuaddr(private, entry); spin_unlock(&private->mem_lock); ret = ret_val; + } else { + /* Insert mem entry in mem_rb tree */ + spin_lock(&private->mem_lock); + kgsl_mem_entry_commit_mem_list(private, entry); + spin_unlock(&private->mem_lock); } break; } @@ -4108,9 +4133,8 @@ int kgsl_device_platform_probe(struct kgsl_device *device) disable_irq(device->pwrctrl.interrupt_num); KGSL_DRV_INFO(device, - "dev_id %d regs phys 0x%08lx size 0x%08x virt %p\n", - device->id, device->reg_phys, device->reg_len, - device->reg_virt); + "dev_id %d regs phys 0x%08lx size 0x%08x\n", + device->id, device->reg_phys, device->reg_len); rwlock_init(&device->context_lock); diff --git a/drivers/gpu/msm/kgsl_cffdump.c b/drivers/gpu/msm/kgsl_cffdump.c index ca2d1ee1e6d..ad01e74e819 100644 --- a/drivers/gpu/msm/kgsl_cffdump.c +++ b/drivers/gpu/msm/kgsl_cffdump.c @@ -426,7 +426,7 @@ void kgsl_cffdump_syncmem(struct kgsl_device *device, src = (uint *)kgsl_gpuaddr_to_vaddr(memdesc, gpuaddr); if (memdesc->hostptr == NULL) { KGSL_CORE_ERR( - "no kernel map for gpuaddr: 0x%08x, m->host: 0x%p, phys: %pa\n", + "no kernel map for gpuaddr: 0x%08x, m->host: 0x%pK, phys: %pa\n", gpuaddr, memdesc->hostptr, &memdesc->physaddr); return; } @@ -516,9 +516,6 @@ EXPORT_SYMBOL(kgsl_cffdump_waitirq); static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf, void *prev_subbuf, uint prev_padding) { - pr_debug("kgsl: cffdump: subbuf_start_handler(subbuf=%p, prev_subbuf" - "=%p, prev_padding=%08x)\n", subbuf, prev_subbuf, prev_padding); - if (relay_buf_full(buf)) { if (!suspended) { suspended = 1; @@ -575,9 +572,6 @@ static struct rchan *create_channel(unsigned subbuf_size, unsigned n_subbufs) { struct rchan *chan; - pr_info("kgsl: cffdump: relay: create_channel: subbuf_size %u, " - "n_subbufs %u, dir 0x%p\n", subbuf_size, n_subbufs, dir); - chan = relay_open("cpu", dir, subbuf_size, n_subbufs, &relay_callbacks, NULL); if (!chan) { diff --git a/drivers/gpu/msm/kgsl_gpummu.c b/drivers/gpu/msm/kgsl_gpummu.c index 2634e4f05a2..9d694375e1b 100644 --- a/drivers/gpu/msm/kgsl_gpummu.c +++ b/drivers/gpu/msm/kgsl_gpummu.c @@ -652,7 +652,7 @@ kgsl_gpummu_unmap(struct kgsl_pagetable *pt, /* check if PTE exists */ if (!kgsl_pt_map_get(gpummu_pt, pte)) KGSL_CORE_ERR("pt entry %x is already " - "unmapped for pagetable %p\n", pte, gpummu_pt); + "unmapped for pagetable %pK\n", pte, gpummu_pt); #endif kgsl_pt_map_set(gpummu_pt, pte, GSL_PT_PAGE_DIRTY); superpte = pte - (pte & (GSL_PT_SUPER_PTE - 1)); diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c index 6be9f852922..ea68a0289cb 100644 --- a/drivers/gpu/msm/kgsl_iommu.c +++ b/drivers/gpu/msm/kgsl_iommu.c @@ -42,6 +42,7 @@ static struct kgsl_iommu_register_list kgsl_iommuv0_reg[KGSL_IOMMU_REG_MAX] = { { 0x10, 1 }, /* TTBR0 */ { 0x14, 1 }, /* TTBR1 */ { 0x20, 1 }, /* FSR */ + { 0x28, 1 }, /* FAR */ { 0x800, 1 }, /* TLBIALL */ { 0x820, 1 }, /* RESUME */ { 0x03C, 1 }, /* TLBLKCR */ @@ -59,6 +60,7 @@ static struct kgsl_iommu_register_list kgsl_iommuv1_reg[KGSL_IOMMU_REG_MAX] = { { 0x20, 1 }, /* TTBR0 */ { 0x28, 1 }, /* TTBR1 */ { 0x58, 1 }, /* FSR */ + { 0x60, 1 }, /* FAR_0 */ { 0x618, 1 }, /* TLBIALL */ { 0x008, 1 }, /* RESUME */ { 0, 0 }, /* TLBLKCR not in V1 */ @@ -340,20 +342,32 @@ static int kgsl_iommu_fault_handler(struct iommu_domain *domain, device = mmu->device; adreno_dev = ADRENO_DEVICE(device); - if (atomic_read(&mmu->fault)) { - if (adreno_dev->ft_pf_policy & KGSL_FT_PAGEFAULT_GPUHALT_ENABLE) - ret = -EBUSY; + /* + * If mmu fault not set then set it and continue else + * exit this function since another thread has already set + * it and will execute rest of this function for the fault. + */ + if (1 == atomic_cmpxchg(&mmu->fault, 0, 1)) goto done; - } iommu_dev = get_iommu_device(iommu_unit, dev); if (!iommu_dev) { - KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev); + KGSL_CORE_ERR("Invalid IOMMU device %pK\n", dev); ret = -ENOSYS; goto done; } iommu = mmu->priv; + fsr = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, + iommu_dev->ctx_id, FSR); + /* + * If fsr is not set then it means that we cleared the fault while the + * bottom half called from IOMMU driver is running + */ + if (!fsr) { + atomic_set(&mmu->fault, 0); + goto done; + } /* * set the fault bits and stuff before any printks so that if fault * handler runs then it will know it's dealing with a pagefault @@ -376,7 +390,6 @@ static int kgsl_iommu_fault_handler(struct iommu_domain *domain, context = NULL; } - atomic_set(&mmu->fault, 1); iommu_dev->fault = 1; if (adreno_dev->ft_pf_policy & KGSL_FT_PAGEFAULT_GPUHALT_ENABLE) { @@ -386,11 +399,9 @@ static int kgsl_iommu_fault_handler(struct iommu_domain *domain, adreno_dispatcher_schedule(device); } - ptbase = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, + ptbase = KGSL_IOMMU_GET_CTX_REG_LL(iommu, iommu_unit, iommu_dev->ctx_id, TTBR0); - fsr = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, - iommu_dev->ctx_id, FSR); fsynr0 = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, iommu_dev->ctx_id, FSYNR0); fsynr1 = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, @@ -732,8 +743,8 @@ static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu) iommu_detach_device(iommu_pt->domain, iommu_unit->dev[j].dev); iommu_unit->dev[j].attached = false; - KGSL_MEM_INFO(mmu->device, "iommu %p detached " - "from user dev of MMU: %p\n", + KGSL_MEM_INFO(mmu->device, "iommu %pK detached " + "from user dev of MMU: %pK\n", iommu_pt->domain, mmu); } } @@ -772,9 +783,12 @@ static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) * If there is a 2nd default pagetable then priv domain * is attached to this pagetable */ - if (mmu->priv_bank_table && - (KGSL_IOMMU_CONTEXT_PRIV == j)) - iommu_pt = mmu->priv_bank_table->priv; + if (KGSL_IOMMU_CONTEXT_PRIV == j) { + if (mmu->priv_bank_table) + iommu_pt = mmu->priv_bank_table->priv; + else + continue; + } if (!iommu_unit->dev[j].attached) { ret = iommu_attach_device(iommu_pt->domain, iommu_unit->dev[j].dev); @@ -786,7 +800,7 @@ static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) } iommu_unit->dev[j].attached = true; KGSL_MEM_INFO(mmu->device, - "iommu pt %p attached to dev %p, ctx_id %d\n", + "iommu pt %pK attached to dev %pK, ctx_id %d\n", iommu_pt->domain, iommu_unit->dev[j].dev, iommu_unit->dev[j].ctx_id); /* Init IOMMU unit clks here */ @@ -857,7 +871,7 @@ static int _get_iommu_ctxs(struct kgsl_mmu *mmu, iommu_unit->dev[iommu_unit->dev_count].kgsldev = mmu->device; KGSL_DRV_INFO(mmu->device, - "Obtained dev handle %p for iommu context %s\n", + "Obtained dev handle %pK for iommu context %s\n", iommu_unit->dev[iommu_unit->dev_count].dev, data->iommu_ctxs[i].iommu_ctx_name); @@ -1245,19 +1259,17 @@ static int kgsl_iommu_setstate(struct kgsl_mmu *mmu, { int ret = 0; - if (mmu->flags & KGSL_FLAGS_STARTED) { - /* page table not current, then setup mmu to use new - * specified page table - */ - if (mmu->hwpagetable != pagetable) { - unsigned int flags = 0; - mmu->hwpagetable = pagetable; - flags |= kgsl_mmu_pt_get_flags(mmu->hwpagetable, - mmu->device->id) | - KGSL_MMUFLAGS_TLBFLUSH; - ret = kgsl_setstate(mmu, context_id, - KGSL_MMUFLAGS_PTUPDATE | flags); - } + /* page table not current, then setup mmu to use new + * specified page table + */ + if (mmu->hwpagetable != pagetable) { + unsigned int flags = 0; + mmu->hwpagetable = pagetable; + flags |= kgsl_mmu_pt_get_flags(mmu->hwpagetable, + mmu->device->id) | + KGSL_MMUFLAGS_TLBFLUSH; + ret = kgsl_setstate(mmu, context_id, + KGSL_MMUFLAGS_PTUPDATE | flags); } return ret; @@ -1551,6 +1563,8 @@ static void kgsl_iommu_lock_rb_in_tlb(struct kgsl_mmu *mmu) for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) { + if (!iommu_unit->dev[j].attached) + continue; tlblkcr = 0; if (cpu_is_msm8960()) tlblkcr |= ((num_tlb_entries & @@ -1576,6 +1590,8 @@ static void kgsl_iommu_lock_rb_in_tlb(struct kgsl_mmu *mmu) TLBLKCR, tlblkcr); } for (j = 0; j < iommu_unit->dev_count; j++) { + if (!iommu_unit->dev[j].attached) + continue; /* skip locking entries for private bank on 8960 */ if (cpu_is_msm8960() && KGSL_IOMMU_CONTEXT_PRIV == j) continue; @@ -1615,6 +1631,8 @@ static void kgsl_iommu_lock_rb_in_tlb(struct kgsl_mmu *mmu) } } for (j = 0; j < iommu_unit->dev_count; j++) { + if (!iommu_unit->dev[j].attached) + continue; tlblkcr = KGSL_IOMMU_GET_CTX_REG(iommu, iommu_unit, iommu_unit->dev[j].ctx_id, TLBLKCR); @@ -1636,9 +1654,6 @@ static int kgsl_iommu_start(struct kgsl_mmu *mmu) int sctlr_val = 0; struct adreno_device *adreno_dev = ADRENO_DEVICE(mmu->device); - if (mmu->flags & KGSL_FLAGS_STARTED) - return 0; - if (mmu->defaultpagetable == NULL) { status = kgsl_iommu_setup_defaultpagetable(mmu); if (status) @@ -1679,7 +1694,8 @@ static int kgsl_iommu_start(struct kgsl_mmu *mmu) for (i = 0; i < iommu->unit_count; i++) { struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) { - + if (!iommu_unit->dev[j].attached) + continue; /* * For IOMMU V1 do not halt IOMMU on pagefault if * FT pagefault policy is set accordingly @@ -1698,19 +1714,12 @@ static int kgsl_iommu_start(struct kgsl_mmu *mmu) iommu_unit->dev[j].ctx_id, SCTLR, sctlr_val); } - if (sizeof(phys_addr_t) > sizeof(unsigned long)) { - iommu_unit->dev[j].default_ttbr0 = - KGSL_IOMMU_GET_CTX_REG_LL(iommu, - iommu_unit, - iommu_unit->dev[j].ctx_id, - TTBR0); - } else { - iommu_unit->dev[j].default_ttbr0 = - KGSL_IOMMU_GET_CTX_REG(iommu, + + iommu_unit->dev[j].default_ttbr0 = + KGSL_IOMMU_GET_CTX_REG_LL(iommu, iommu_unit, iommu_unit->dev[j].ctx_id, TTBR0); - } } } kgsl_iommu_lock_rb_in_tlb(mmu); @@ -1722,7 +1731,6 @@ static int kgsl_iommu_start(struct kgsl_mmu *mmu) cp_nop_packet(1), sizeof(unsigned int)); kgsl_iommu_disable_clk(mmu, KGSL_IOMMU_MAX_UNITS); - mmu->flags |= KGSL_FLAGS_STARTED; done: return status; @@ -1753,7 +1761,7 @@ kgsl_iommu_unmap(struct kgsl_pagetable *pt, ret = iommu_unmap_range(iommu_pt->domain, gpuaddr, range); if (ret) { - KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed " + KGSL_CORE_ERR("iommu_unmap_range(%pK, %x, %d) failed " "with err: %d\n", iommu_pt->domain, gpuaddr, range, ret); return ret; @@ -1804,7 +1812,7 @@ kgsl_iommu_map(struct kgsl_pagetable *pt, ret = iommu_map_range(iommu_pt->domain, iommu_virt_addr, memdesc->sg, size, protflags); if (ret) { - KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %x) err: %d\n", + KGSL_CORE_ERR("iommu_map_range(%pK, %x, %pK, %d, %x) err: %d\n", iommu_pt->domain, iommu_virt_addr, memdesc->sg, size, protflags, ret); return ret; @@ -1814,7 +1822,7 @@ kgsl_iommu_map(struct kgsl_pagetable *pt, page_to_phys(kgsl_guard_page), PAGE_SIZE, protflags & ~IOMMU_WRITE); if (ret) { - KGSL_CORE_ERR("iommu_map(%p, %x, guard, %x) err: %d\n", + KGSL_CORE_ERR("iommu_map(%pK, %x, guard, %x) err: %d\n", iommu_pt->domain, iommu_virt_addr + size, protflags & ~IOMMU_WRITE, ret); @@ -1837,6 +1845,8 @@ void kgsl_iommu_pagefault_resume(struct kgsl_mmu *mmu) struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; for (j = 0; j < iommu_unit->dev_count; j++) { + if (!iommu_unit->dev[j].attached) + continue; if (iommu_unit->dev[j].fault) { _iommu_lock(); KGSL_IOMMU_SET_CTX_REG(iommu, @@ -1857,23 +1867,18 @@ void kgsl_iommu_pagefault_resume(struct kgsl_mmu *mmu) } } - static void kgsl_iommu_stop(struct kgsl_mmu *mmu) { /* * stop device mmu * * call this with the global lock held + * detach iommu attachment */ - if (mmu->flags & KGSL_FLAGS_STARTED) { - /* detach iommu attachment */ - kgsl_detach_pagetable_iommu_domain(mmu); - mmu->hwpagetable = NULL; - - mmu->flags &= ~KGSL_FLAGS_STARTED; + kgsl_detach_pagetable_iommu_domain(mmu); + mmu->hwpagetable = NULL; - kgsl_iommu_pagefault_resume(mmu); - } + kgsl_iommu_pagefault_resume(mmu); /* switch off MMU clocks and cancel any events it has queued */ kgsl_cancel_events(mmu->device, mmu); } @@ -1925,7 +1930,7 @@ kgsl_iommu_get_current_ptbase(struct kgsl_mmu *mmu) return 0; /* Return the current pt base by reading IOMMU pt_base register */ kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_MAX_UNITS); - pt_base = KGSL_IOMMU_GET_CTX_REG(iommu, (&iommu->iommu_units[0]), + pt_base = KGSL_IOMMU_GET_CTX_REG_LL(iommu, (&iommu->iommu_units[0]), KGSL_IOMMU_CONTEXT_USER, TTBR0); kgsl_iommu_disable_clk(mmu, KGSL_IOMMU_MAX_UNITS); @@ -1982,18 +1987,12 @@ static int kgsl_iommu_default_setstate(struct kgsl_mmu *mmu, pt_base &= KGSL_IOMMU_CTX_TTBR0_ADDR_MASK; pt_val &= ~KGSL_IOMMU_CTX_TTBR0_ADDR_MASK; pt_val |= pt_base; - if (sizeof(phys_addr_t) > sizeof(unsigned long)) { - KGSL_IOMMU_SET_CTX_REG_LL(iommu, + KGSL_IOMMU_SET_CTX_REG_LL(iommu, (&iommu->iommu_units[i]), KGSL_IOMMU_CONTEXT_USER, TTBR0, pt_val); - } else { - KGSL_IOMMU_SET_CTX_REG(iommu, - (&iommu->iommu_units[i]), - KGSL_IOMMU_CONTEXT_USER, TTBR0, pt_val); - } mb(); - temp = KGSL_IOMMU_GET_CTX_REG(iommu, + temp = KGSL_IOMMU_GET_CTX_REG_LL(iommu, (&iommu->iommu_units[i]), KGSL_IOMMU_CONTEXT_USER, TTBR0); } @@ -2087,6 +2086,103 @@ static int kgsl_iommu_get_num_iommu_units(struct kgsl_mmu *mmu) return iommu->unit_count; } +/* + * kgsl_iommu_set_pf_policy() - Set the pagefault policy for IOMMU + * @mmu: Pointer to mmu structure + * @pf_policy: The pagefault polict to set + * + * Check if the new policy indicated by pf_policy is same as current + * policy, if same then return else set the policy + */ +static int kgsl_iommu_set_pf_policy(struct kgsl_mmu *mmu, + unsigned int pf_policy) +{ + int i, j; + struct kgsl_iommu *iommu = mmu->priv; + struct adreno_device *adreno_dev = ADRENO_DEVICE(mmu->device); + int ret = 0; + unsigned int sctlr_val; + + if ((adreno_dev->ft_pf_policy & KGSL_FT_PAGEFAULT_GPUHALT_ENABLE) == + (pf_policy & KGSL_FT_PAGEFAULT_GPUHALT_ENABLE)) + return ret; + if (!msm_soc_version_supports_iommu_v0()) + return ret; + + kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_MAX_UNITS); + + /* Need to idle device before changing options */ + ret = mmu->device->ftbl->idle(mmu->device); + if (ret) { + kgsl_iommu_disable_clk(mmu, KGSL_IOMMU_MAX_UNITS); + return ret; + } + + for (i = 0; i < iommu->unit_count; i++) { + struct kgsl_iommu_unit *iommu_unit = &iommu->iommu_units[i]; + for (j = 0; j < iommu_unit->dev_count; j++) { + sctlr_val = KGSL_IOMMU_GET_CTX_REG(iommu, + iommu_unit, + iommu_unit->dev[j].ctx_id, + SCTLR); + if (pf_policy & KGSL_FT_PAGEFAULT_GPUHALT_ENABLE) + sctlr_val &= ~(0x1 << + KGSL_IOMMU_SCTLR_HUPCF_SHIFT); + else + sctlr_val |= (0x1 << + KGSL_IOMMU_SCTLR_HUPCF_SHIFT); + KGSL_IOMMU_SET_CTX_REG(iommu, + iommu_unit, + iommu_unit->dev[j].ctx_id, + SCTLR, sctlr_val); + } + } + kgsl_iommu_disable_clk(mmu, KGSL_IOMMU_MAX_UNITS); + return ret; +} + +/** + * kgsl_iommu_set_pagefault() - Checks if a IOMMU device has faulted + * @mmu: MMU pointer of the device + * + * This function is called to set the pagefault bits for the device so + * that recovery can run with pagefault in consideration + */ +static void kgsl_iommu_set_pagefault(struct kgsl_mmu *mmu) +{ + int i, j; + struct kgsl_iommu *iommu = mmu->priv; + unsigned int fsr; + + /* fault already detected then return early */ + if (atomic_read(&mmu->fault)) + return; + + kgsl_iommu_enable_clk(mmu, KGSL_IOMMU_MAX_UNITS); + /* Loop through all IOMMU devices to check for fault */ + for (i = 0; i < iommu->unit_count; i++) { + for (j = 0; j < iommu->iommu_units[i].dev_count; j++) { + if (!iommu->iommu_units[i].dev[j].attached) + continue; + fsr = KGSL_IOMMU_GET_CTX_REG(iommu, + (&(iommu->iommu_units[i])), + iommu->iommu_units[i].dev[j].ctx_id, FSR); + if (fsr) { + uint64_t far = + KGSL_IOMMU_GET_CTX_REG_LL(iommu, + (&(iommu->iommu_units[i])), + iommu->iommu_units[i].dev[j].ctx_id, + FAR); + kgsl_iommu_fault_handler(NULL, + iommu->iommu_units[i].dev[j].dev, far, 0, NULL); + break; + } + } + } + + kgsl_iommu_disable_clk(mmu, KGSL_IOMMU_MAX_UNITS); +} + struct kgsl_mmu_ops iommu_ops = { .mmu_init = kgsl_iommu_init, .mmu_close = kgsl_iommu_close, @@ -2112,6 +2208,8 @@ struct kgsl_mmu_ops iommu_ops = { .mmu_cleanup_pt = NULL, .mmu_sync_lock = kgsl_iommu_sync_lock, .mmu_sync_unlock = kgsl_iommu_sync_unlock, + .mmu_set_pf_policy = kgsl_iommu_set_pf_policy, + .mmu_set_pagefault = kgsl_iommu_set_pagefault }; struct kgsl_mmu_pt_ops iommu_pt_ops = { diff --git a/drivers/gpu/msm/kgsl_iommu.h b/drivers/gpu/msm/kgsl_iommu.h index 2ff665a2c97..8fd494c6dbd 100644 --- a/drivers/gpu/msm/kgsl_iommu.h +++ b/drivers/gpu/msm/kgsl_iommu.h @@ -70,6 +70,7 @@ enum kgsl_iommu_reg_map { KGSL_IOMMU_CTX_TTBR0, KGSL_IOMMU_CTX_TTBR1, KGSL_IOMMU_CTX_FSR, + KGSL_IOMMU_CTX_FAR, KGSL_IOMMU_CTX_TLBIALL, KGSL_IOMMU_CTX_RESUME, KGSL_IOMMU_CTX_TLBLKCR, @@ -111,7 +112,7 @@ enum kgsl_iommu_units { iommu->ctx_offset) #define KGSL_IOMMU_GET_CTX_REG_LL(iommu, iommu_unit, ctx, REG) \ - readl_relaxed( \ + readll_relaxed( \ iommu_unit->reg_map.hostptr + \ iommu->iommu_reg_list[KGSL_IOMMU_CTX_##REG].reg_offset +\ (ctx << KGSL_IOMMU_CTX_SHIFT) + \ diff --git a/drivers/gpu/msm/kgsl_mmu.c b/drivers/gpu/msm/kgsl_mmu.c index d987c3d1d6a..e48734adc71 100644 --- a/drivers/gpu/msm/kgsl_mmu.c +++ b/drivers/gpu/msm/kgsl_mmu.c @@ -892,7 +892,7 @@ int kgsl_mmu_map_global(struct kgsl_pagetable *pagetable, /*global mappings must have the same gpu address in all pagetables*/ if (gpuaddr && gpuaddr != memdesc->gpuaddr) { - KGSL_CORE_ERR("pt %p addr mismatch phys %pa gpu 0x%0x 0x%08x", + KGSL_CORE_ERR("pt %pK addr mismatch phys %pa gpu 0x%0x 0x%08x", pagetable, &memdesc->physaddr, gpuaddr, memdesc->gpuaddr); goto error_unmap; } diff --git a/drivers/gpu/msm/kgsl_mmu.h b/drivers/gpu/msm/kgsl_mmu.h index 5fcc6f4b190..7a6c7f2af6f 100644 --- a/drivers/gpu/msm/kgsl_mmu.h +++ b/drivers/gpu/msm/kgsl_mmu.h @@ -178,6 +178,8 @@ struct kgsl_mmu_ops { unsigned int (*mmu_sync_unlock) (struct kgsl_mmu *mmu, unsigned int *cmds); int (*mmu_hw_halt_supported)(struct kgsl_mmu *mmu, int iommu_unit_num); + int (*mmu_set_pf_policy)(struct kgsl_mmu *mmu, unsigned int pf_policy); + void (*mmu_set_pagefault)(struct kgsl_mmu *mmu); }; struct kgsl_mmu_pt_ops { @@ -481,4 +483,19 @@ static inline int kgsl_mmu_sync_unlock(struct kgsl_mmu *mmu, return 0; } +static inline int kgsl_mmu_set_pagefault_policy(struct kgsl_mmu *mmu, + unsigned int pf_policy) +{ + if (mmu->mmu_ops && mmu->mmu_ops->mmu_set_pf_policy) + return mmu->mmu_ops->mmu_set_pf_policy(mmu, pf_policy); + else + return 0; +} + +static inline void kgsl_mmu_set_pagefault(struct kgsl_mmu *mmu) +{ + if (mmu->mmu_ops && mmu->mmu_ops->mmu_set_pagefault) + return mmu->mmu_ops->mmu_set_pagefault(mmu); +} + #endif /* __KGSL_MMU_H */ diff --git a/drivers/gpu/msm/kgsl_pwrctrl.c b/drivers/gpu/msm/kgsl_pwrctrl.c index e7eaa93b12a..7ac9180b511 100644 --- a/drivers/gpu/msm/kgsl_pwrctrl.c +++ b/drivers/gpu/msm/kgsl_pwrctrl.c @@ -1098,7 +1098,7 @@ int kgsl_pwrctrl_init(struct kgsl_device *device) if (!pwr->pcl) { KGSL_PWR_ERR(device, "msm_bus_scale_register_client failed: " - "id %d table %p", device->id, + "id %d table %pK", device->id, pdata->bus_scale_table); result = -EINVAL; goto done; diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c index 8d8e9635b19..4aec22b4503 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.c +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -579,21 +579,17 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, struct kgsl_pagetable *pagetable, size_t size) { - int pcount = 0, order, ret = 0; - int j, len, page_size, sglen_alloc, sglen = 0; - struct page **pages = NULL; - pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL); + int order, ret = 0; + int sglen_alloc, sglen = 0; + size_t len; void *ptr; unsigned int align; - int step = ((VMALLOC_END - VMALLOC_START)/8) >> PAGE_SHIFT; - align = (memdesc->flags & KGSL_MEMALIGN_MASK) >> KGSL_MEMALIGN_SHIFT; + size = PAGE_ALIGN(size); + if (size == 0 || size > UINT_MAX) + return -EINVAL; - page_size = (align >= ilog2(SZ_64K) && size >= SZ_64K) - ? SZ_64K : PAGE_SIZE; - /* update align flags for what we actually use */ - if (page_size != PAGE_SIZE) - kgsl_memdesc_set_align(memdesc, ilog2(page_size)); + align = (memdesc->flags & KGSL_MEMALIGN_MASK) >> KGSL_MEMALIGN_SHIFT; /* * There needs to be enough room in the sg structure to be able to @@ -613,24 +609,6 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, goto done; } - /* - * Allocate space to store the list of pages to send to vmap. - * This is an array of pointers so we can t rack 1024 pages per page - * of allocation. Since allocations can be as large as the user dares, - * we have to use the kmalloc/vmalloc trick here to make sure we can - * get the memory we need. - */ - - if ((memdesc->sglen_alloc * sizeof(struct page *)) > PAGE_SIZE) - pages = vmalloc(memdesc->sglen_alloc * sizeof(struct page *)); - else - pages = kmalloc(PAGE_SIZE, GFP_KERNEL); - - if (pages == NULL) { - ret = -ENOMEM; - goto done; - } - kmemleak_not_leak(memdesc->sg); sg_init_table(memdesc->sg, memdesc->sglen_alloc); @@ -639,31 +617,10 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, while (len > 0) { struct page *page; - unsigned int gfp_mask = __GFP_HIGHMEM; - int j; - - /* don't waste space at the end of the allocation*/ - if (len < page_size) - page_size = PAGE_SIZE; - - /* - * Don't do some of the more aggressive memory recovery - * techniques for large order allocations - */ - if (page_size != PAGE_SIZE) - gfp_mask |= __GFP_COMP | __GFP_NORETRY | - __GFP_NO_KSWAPD | __GFP_NOWARN; - else - gfp_mask |= GFP_KERNEL; - page = alloc_pages(gfp_mask, get_order(page_size)); + page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); if (page == NULL) { - if (page_size != PAGE_SIZE) { - page_size = PAGE_SIZE; - continue; - } - /* * Update sglen and memdesc size,as requested allocation * not served fully. So that they can be correctly freed @@ -680,58 +637,17 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, goto done; } - for (j = 0; j < page_size >> PAGE_SHIFT; j++) - pages[pcount++] = nth_page(page, j); + ptr = kmap_atomic(page); + dmac_flush_range(ptr, ptr + PAGE_SIZE); + kunmap_atomic(ptr); - sg_set_page(&memdesc->sg[sglen++], page, page_size, 0); - len -= page_size; + sg_set_page(&memdesc->sg[sglen++], page, PAGE_SIZE, 0); + len -= PAGE_SIZE; } memdesc->sglen = sglen; memdesc->size = size; - /* - * All memory that goes to the user has to be zeroed out before it gets - * exposed to userspace. This means that the memory has to be mapped in - * the kernel, zeroed (memset) and then unmapped. This also means that - * the dcache has to be flushed to ensure coherency between the kernel - * and user pages. We used to pass __GFP_ZERO to alloc_page which mapped - * zeroed and unmaped each individual page, and then we had to turn - * around and call flush_dcache_page() on that page to clear the caches. - * This was killing us for performance. Instead, we found it is much - * faster to allocate the pages without GFP_ZERO, map a chunk of the - * range ('step' pages), memset it, flush it and then unmap - * - this results in a factor of 4 improvement for speed for large - * buffers. There is a small decrease in speed for small buffers, - * but only on the order of a few microseconds at best. The 'step' - * size is based on a guess at the amount of free vmalloc space, but - * will scale down if there's not enough free space. - */ - for (j = 0; j < pcount; j += step) { - step = min(step, pcount - j); - - ptr = vmap(&pages[j], step, VM_IOREMAP, page_prot); - - if (ptr != NULL) { - memset(ptr, 0, step * PAGE_SIZE); - dmac_flush_range(ptr, ptr + step * PAGE_SIZE); - vunmap(ptr); - } else { - int k; - /* Very, very, very slow path */ - - for (k = j; k < j + step; k++) { - ptr = kmap_atomic(pages[k]); - memset(ptr, 0, PAGE_SIZE); - dmac_flush_range(ptr, ptr + PAGE_SIZE); - kunmap_atomic(ptr); - } - /* scale down the step size to avoid this path */ - if (step > 1) - step >>= 1; - } - } - outer_cache_range_op_sg(memdesc->sg, memdesc->sglen, KGSL_CACHE_OP_FLUSH); @@ -744,11 +660,6 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, KGSL_STATS_ADD(memdesc->size, kgsl_driver.stats.page_alloc, kgsl_driver.stats.page_alloc_max); - if ((memdesc->sglen_alloc * sizeof(struct page *)) > PAGE_SIZE) - vfree(pages); - else - kfree(pages); - if (ret) kgsl_sharedmem_free(memdesc); diff --git a/drivers/gpu/msm/kgsl_snapshot.c b/drivers/gpu/msm/kgsl_snapshot.c index beda17f3169..9eff3ffcd5b 100644 --- a/drivers/gpu/msm/kgsl_snapshot.c +++ b/drivers/gpu/msm/kgsl_snapshot.c @@ -1076,7 +1076,7 @@ void kgsl_snapshot_save_frozen_objs(struct work_struct *work) goto done; KGSL_DRV_ERR(device, - "Allocated memory for snapshot objects at address %p, size %x\n", + "Allocated memory for snapshot objects at address %pK, size %x\n", device->snapshot_cur_ib_objs, remain); snapshot_dest = device->snapshot_cur_ib_objs; device->snapshot_cur_ib_objs_size = remain; diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 59f93d2721f..410e7c02bfd 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -983,6 +983,7 @@ static void hid_input_field(struct hid_device *hid, struct hid_field *field, /* Ignore report if ErrorRollOver */ if (!(field->flags & HID_MAIN_ITEM_VARIABLE) && value[n] >= min && value[n] <= max && + value[n] - min < field->maxusage && field->usage[value[n] - min].hid == HID_UP_KEYBOARD + 1) goto exit; } @@ -995,11 +996,13 @@ static void hid_input_field(struct hid_device *hid, struct hid_field *field, } if (field->value[n] >= min && field->value[n] <= max + && field->value[n] - min < field->maxusage && field->usage[field->value[n] - min].hid && search(value, field->value[n], count)) hid_process_event(hid, field, &field->usage[field->value[n] - min], 0, interrupt); if (value[n] >= min && value[n] <= max + && value[n] - min < field->maxusage && field->usage[value[n] - min].hid && search(field->value, value[n], count)) hid_process_event(hid, field, &field->usage[value[n] - min], 1, interrupt); diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c index f63341f20b9..e8c6a4842e9 100644 --- a/drivers/input/misc/ati_remote2.c +++ b/drivers/input/misc/ati_remote2.c @@ -817,26 +817,49 @@ static int ati_remote2_probe(struct usb_interface *interface, const struct usb_d ar2->udev = udev; + /* Sanity check, first interface must have an endpoint */ + if (alt->desc.bNumEndpoints < 1 || !alt->endpoint) { + dev_err(&interface->dev, + "%s(): interface 0 must have an endpoint\n", __func__); + r = -ENODEV; + goto fail1; + } ar2->intf[0] = interface; ar2->ep[0] = &alt->endpoint[0].desc; + /* Sanity check, the device must have two interfaces */ ar2->intf[1] = usb_ifnum_to_if(udev, 1); + if ((udev->actconfig->desc.bNumInterfaces < 2) || !ar2->intf[1]) { + dev_err(&interface->dev, "%s(): need 2 interfaces, found %d\n", + __func__, udev->actconfig->desc.bNumInterfaces); + r = -ENODEV; + goto fail1; + } + r = usb_driver_claim_interface(&ati_remote2_driver, ar2->intf[1], ar2); if (r) goto fail1; + + /* Sanity check, second interface must have an endpoint */ alt = ar2->intf[1]->cur_altsetting; + if (alt->desc.bNumEndpoints < 1 || !alt->endpoint) { + dev_err(&interface->dev, + "%s(): interface 1 must have an endpoint\n", __func__); + r = -ENODEV; + goto fail2; + } ar2->ep[1] = &alt->endpoint[0].desc; r = ati_remote2_urb_init(ar2); if (r) - goto fail2; + goto fail3; ar2->channel_mask = channel_mask; ar2->mode_mask = mode_mask; r = ati_remote2_setup(ar2, ar2->channel_mask); if (r) - goto fail2; + goto fail3; usb_make_path(udev, ar2->phys, sizeof(ar2->phys)); strlcat(ar2->phys, "/input0", sizeof(ar2->phys)); @@ -845,11 +868,11 @@ static int ati_remote2_probe(struct usb_interface *interface, const struct usb_d r = sysfs_create_group(&udev->dev.kobj, &ati_remote2_attr_group); if (r) - goto fail2; + goto fail3; r = ati_remote2_input_init(ar2); if (r) - goto fail3; + goto fail4; usb_set_intfdata(interface, ar2); @@ -857,10 +880,11 @@ static int ati_remote2_probe(struct usb_interface *interface, const struct usb_d return 0; - fail3: + fail4: sysfs_remove_group(&udev->dev.kobj, &ati_remote2_attr_group); - fail2: + fail3: ati_remote2_urb_cleanup(ar2); + fail2: usb_driver_release_interface(&ati_remote2_driver, ar2->intf[1]); fail1: kfree(ar2); diff --git a/drivers/input/misc/powermate.c b/drivers/input/misc/powermate.c index 538f7049ec6..3c3a8a09ec6 100644 --- a/drivers/input/misc/powermate.c +++ b/drivers/input/misc/powermate.c @@ -304,6 +304,9 @@ static int powermate_probe(struct usb_interface *intf, const struct usb_device_i int error = -ENOMEM; interface = intf->cur_altsetting; + if (interface->desc.bNumEndpoints < 1) + return -EINVAL; + endpoint = &interface->endpoint[0].desc; if (!usb_endpoint_is_int_in(endpoint)) return -EIO; diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c index 205d16aab44..2884f549c24 100644 --- a/drivers/input/tablet/aiptek.c +++ b/drivers/input/tablet/aiptek.c @@ -1814,6 +1814,14 @@ aiptek_probe(struct usb_interface *intf, const struct usb_device_id *id) input_set_abs_params(inputdev, ABS_TILT_Y, AIPTEK_TILT_MIN, AIPTEK_TILT_MAX, 0, 0); input_set_abs_params(inputdev, ABS_WHEEL, AIPTEK_WHEEL_MIN, AIPTEK_WHEEL_MAX - 1, 0, 0); + /* Verify that a device really has an endpoint */ + if (intf->altsetting[0].desc.bNumEndpoints < 1) { + dev_err(&intf->dev, + "interface has %d endpoints, but must have minimum 1\n", + intf->altsetting[0].desc.bNumEndpoints); + err = -EINVAL; + goto fail3; + } endpoint = &intf->altsetting[0].endpoint[0].desc; /* Go set up our URB, which is called when the tablet receives @@ -1856,6 +1864,7 @@ aiptek_probe(struct usb_interface *intf, const struct usb_device_id *id) if (i == ARRAY_SIZE(speeds)) { dev_info(&intf->dev, "Aiptek tried all speeds, no sane response\n"); + err = -EINVAL; goto fail2; } diff --git a/drivers/input/tablet/gtco.c b/drivers/input/tablet/gtco.c index 89a297801dc..371373cdcb7 100644 --- a/drivers/input/tablet/gtco.c +++ b/drivers/input/tablet/gtco.c @@ -866,6 +866,14 @@ static int gtco_probe(struct usb_interface *usbinterface, goto err_free_buf; } + /* Sanity check that a device has an endpoint */ + if (usbinterface->altsetting[0].desc.bNumEndpoints < 1) { + dev_err(&usbinterface->dev, + "Invalid number of endpoints\n"); + error = -EINVAL; + goto err_free_urb; + } + /* * The endpoint is always altsetting 0, we know this since we know * this device only has one interrupt endpoint @@ -887,7 +895,7 @@ static int gtco_probe(struct usb_interface *usbinterface, * HID report descriptor */ if (usb_get_extra_descriptor(usbinterface->cur_altsetting, - HID_DEVICE_TYPE, &hid_desc) != 0){ + HID_DEVICE_TYPE, &hid_desc) != 0) { err("Can't retrieve exta USB descriptor to get hid report descriptor length"); error = -EIO; goto err_free_urb; diff --git a/drivers/leds/leds-qpnp.c b/drivers/leds/leds-qpnp.c old mode 100644 new mode 100755 index 37df669d41f..a85b39305de --- a/drivers/leds/leds-qpnp.c +++ b/drivers/leds/leds-qpnp.c @@ -1167,6 +1167,20 @@ static int rgb_duration_config(struct qpnp_led_data *led) if (!on_ms) { return -EINVAL; + } else if (!off_ms) { + /* implement always on + * note: + * rgb_on_off_ms_store() bumps on_ms=0 up to RGB_LED_MIN_MS + * so setting ms on/off to 0/0 in /sys results in seeing + * 50/0 by the time we get here + */ + ramp_step_ms = 1000; + num_duty_pcts = 1; + pwm_cfg->duty_cycles->duty_pcts[0] = + (led->cdev.brightness * + led->rgb_cfg->calibrated_max * + 100) / + (RGB_MAX_LEVEL * RGB_MAX_LEVEL); } else { ramp_step_ms = on_ms / 20; ramp_step_ms = (ramp_step_ms < 5)? 5 : ramp_step_ms; @@ -3045,8 +3059,9 @@ static ssize_t rgb_start_store(struct device *dev, if (ret < 0) dev_err(led_array[i].cdev.dev, "RGB set rgb start failed (%d)\n", ret); + /* Checking lut flags is used to glean if the led really was started */ if (!(led_array[i].rgb_cfg->pwm_cfg->lut_params.flags & - PM_PWM_LUT_LOOP)) + PM_PWM_LUT_RAMP_UP)) led_array[i].rgb_cfg->start = 0; break; default: diff --git a/drivers/md/md.c b/drivers/md/md.c index 01233d855eb..fc5bc50cc30 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5413,9 +5413,9 @@ static int get_bitmap_file(struct mddev * mddev, void __user * arg) int err = -ENOMEM; if (md_allow_write(mddev)) - file = kmalloc(sizeof(*file), GFP_NOIO); + file = kzalloc(sizeof(*file), GFP_NOIO); else - file = kmalloc(sizeof(*file), GFP_KERNEL); + file = kzalloc(sizeof(*file), GFP_KERNEL); if (!file) goto out; diff --git a/drivers/media/media-device.c b/drivers/media/media-device.c index 6f9eb94e85b..4795a18599d 100644 --- a/drivers/media/media-device.c +++ b/drivers/media/media-device.c @@ -90,6 +90,7 @@ static long media_device_enum_entities(struct media_device *mdev, struct media_entity *ent; struct media_entity_desc u_ent; + memset(&u_ent, 0, sizeof(u_ent)); if (copy_from_user(&u_ent.id, &uent->id, sizeof(u_ent.id))) return -EFAULT; @@ -139,7 +140,7 @@ static long media_device_enum_links(struct media_device *mdev, unsigned int p; for (p = 0; p < entity->num_pads; p++) { - struct media_pad_desc pad; + struct media_pad_desc pad = {0}; media_device_kpad_to_upad(&entity->pads[p], &pad); if (copy_to_user(&links.pads[p], &pad, sizeof(pad))) return -EFAULT; diff --git a/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_hw.c b/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_hw.c index da6f69f9705..baea55e2866 100644 --- a/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_hw.c +++ b/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_hw.c @@ -491,7 +491,7 @@ void msm_gemini_io_dump(int size) int i; u32 *p = (u32 *) addr; u32 data; - pr_info("%s: %p %d reg_size %d\n", __func__, addr, size, + pr_info("%s: %pK %d reg_size %d\n", __func__, addr, size, gemini_region_size); line_str[0] = '\0'; p_str = line_str; diff --git a/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_sync.c b/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_sync.c index ebf8d4bc1ef..f54650f8a60 100644 --- a/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_sync.c +++ b/drivers/media/platform/msm/camera_v2/gemini/msm_gemini_sync.c @@ -187,7 +187,7 @@ int msm_gemini_framedone_irq(struct msm_gemini_device *pgmn_dev, { int rc = 0; - GMN_DBG("%s:%d] buf_in %p", __func__, __LINE__, buf_in); + GMN_DBG("%s:%d] buf_in %pK", __func__, __LINE__, buf_in); if (buf_in) { buf_in->vbuf.framedone_len = buf_in->framedone_len; @@ -711,7 +711,7 @@ int __msm_gemini_open(struct msm_gemini_device *pgmn_dev) return rc; } - GMN_DBG("%s:%d] platform resources - mem %p, base %p, irq %d\n", + GMN_DBG("%s:%d] platform resources - mem %pK, base %pK, irq %d\n", __func__, __LINE__, pgmn_dev->mem, pgmn_dev->base, pgmn_dev->irq); diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_buf_mgr.c b/drivers/media/platform/msm/camera_v2/isp/msm_buf_mgr.c index 739bd6863fb..86b044ad21a 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_buf_mgr.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_buf_mgr.c @@ -128,7 +128,7 @@ static int msm_isp_prepare_v4l2_buf(struct msm_isp_buf_mgr *buf_mgr, ion_import_dma_buf(buf_mgr->client, v4l2_buf->m.planes[i].m.userptr); if (IS_ERR_OR_NULL(mapped_info->handle)) { - pr_err("%s: buf has null/error ION handle %p\n", + pr_err("%s: buf has null/error ION handle %pK\n", __func__, mapped_info->handle); goto ion_map_error; } diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c b/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c index 7159a6119f7..477316fa9e4 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c @@ -329,7 +329,15 @@ int msm_isp_axi_check_stream_state( enum msm_vfe_axi_state valid_state = (stream_cfg_cmd->cmd == START_STREAM) ? INACTIVE : ACTIVE; + if (stream_cfg_cmd->num_streams > MAX_NUM_STREAM) { + return -EINVAL; + } + for (i = 0; i < stream_cfg_cmd->num_streams; i++) { + if (HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i]) >= + MAX_NUM_STREAM) { + return -EINVAL; + } stream_info = &axi_data->stream_info[ HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i])]; if (stream_info->state != valid_state) { @@ -525,8 +533,10 @@ int msm_isp_request_axi_stream(struct vfe_device *vfe_dev, void *arg) &vfe_dev->axi_data, stream_cfg_cmd); if (rc) { pr_err("%s: Request validation failed\n", __func__); - msm_isp_axi_destroy_stream(&vfe_dev->axi_data, - HANDLE_TO_IDX(stream_cfg_cmd->axi_stream_handle)); + if (HANDLE_TO_IDX(stream_cfg_cmd->axi_stream_handle) < + MAX_NUM_STREAM) + msm_isp_axi_destroy_stream(&vfe_dev->axi_data, + HANDLE_TO_IDX(stream_cfg_cmd->axi_stream_handle)); return rc; } @@ -566,11 +576,17 @@ int msm_isp_release_axi_stream(struct vfe_device *vfe_dev, void *arg) int rc = 0, i; struct msm_vfe_axi_stream_release_cmd *stream_release_cmd = arg; struct msm_vfe_axi_shared_data *axi_data = &vfe_dev->axi_data; - struct msm_vfe_axi_stream *stream_info = - &axi_data->stream_info[ - HANDLE_TO_IDX(stream_release_cmd->stream_handle)]; + struct msm_vfe_axi_stream *stream_info; struct msm_vfe_axi_stream_cfg_cmd stream_cfg; + + if (HANDLE_TO_IDX(stream_release_cmd->stream_handle) >= + MAX_NUM_STREAM) { + pr_err("%s: Invalid stream handle\n", __func__); + return -EINVAL; + } + stream_info = &axi_data->stream_info[ + HANDLE_TO_IDX(stream_release_cmd->stream_handle)]; if (stream_info->state == AVALIABLE) { pr_err("%s: Stream already released\n", __func__); return -EINVAL; @@ -772,9 +788,16 @@ static void msm_isp_process_done_buf(struct vfe_device *vfe_dev, int rc; struct msm_isp_event_data buf_event; uint32_t stream_idx = HANDLE_TO_IDX(stream_info->stream_handle); - uint32_t frame_id = vfe_dev->axi_data. - src_info[SRC_TO_INTF(stream_info->stream_src)].frame_id; uint32_t buf_src; + uint32_t frame_id; + + if (stream_idx >= MAX_NUM_STREAM) { + pr_err("%s: Invalid stream_idx", __func__); + return; + } + + frame_id = vfe_dev->axi_data. + src_info[SRC_TO_INTF(stream_info->stream_src)].frame_id; if (buf && ts) { rc = vfe_dev->buf_mgr->ops->get_buf_src(vfe_dev->buf_mgr, @@ -861,7 +884,16 @@ static void msm_isp_update_camif_output_count( int i; struct msm_vfe_axi_stream *stream_info; struct msm_vfe_axi_shared_data *axi_data = &vfe_dev->axi_data; + + if (stream_cfg_cmd->num_streams > MAX_NUM_STREAM) { + return; + } + for (i = 0; i < stream_cfg_cmd->num_streams; i++) { + if (HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i]) >= + MAX_NUM_STREAM) { + return; + } stream_info = &axi_data->stream_info[ HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i])]; @@ -893,7 +925,7 @@ void msm_camera_io_dump_2(void __iomem *addr, int size) int i; u32 *p = (u32 *) addr; u32 data; - ISP_DBG("%s: %p %d\n", __func__, addr, size); + ISP_DBG("%s: %pK %d\n", __func__, addr, size); line_str[0] = '\0'; p_str = line_str; for (i = 0; i < size/4; i++) { @@ -1041,7 +1073,16 @@ static int msm_isp_start_axi_stream(struct vfe_device *vfe_dev, uint32_t wm_reload_mask = 0x0; struct msm_vfe_axi_stream *stream_info; struct msm_vfe_axi_shared_data *axi_data = &vfe_dev->axi_data; + + if (stream_cfg_cmd->num_streams > MAX_NUM_STREAM) { + return -EINVAL; + } + for (i = 0; i < stream_cfg_cmd->num_streams; i++) { + if (HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i]) >= + MAX_NUM_STREAM) { + return -EINVAL; + } stream_info = &axi_data->stream_info[ HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i])]; src_state = axi_data->src_info[ @@ -1093,7 +1134,16 @@ static int msm_isp_stop_axi_stream(struct vfe_device *vfe_dev, uint8_t wait_for_complete = 0; struct msm_vfe_axi_stream *stream_info; struct msm_vfe_axi_shared_data *axi_data = &vfe_dev->axi_data; + + if (stream_cfg_cmd->num_streams > MAX_NUM_STREAM) { + return -EINVAL; + } + for (i = 0; i < stream_cfg_cmd->num_streams; i++) { + if (HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i]) >= + MAX_NUM_STREAM) { + return -EINVAL; + } stream_info = &axi_data->stream_info[ HANDLE_TO_IDX(stream_cfg_cmd->stream_handle[i])]; @@ -1174,6 +1224,12 @@ int msm_isp_update_axi_stream(struct vfe_device *vfe_dev, void *arg) struct msm_vfe_axi_stream *stream_info; struct msm_vfe_axi_shared_data *axi_data = &vfe_dev->axi_data; struct msm_vfe_axi_stream_update_cmd *update_cmd = arg; + + if (HANDLE_TO_IDX(update_cmd->stream_handle) >= + MAX_NUM_STREAM) { + return -EINVAL; + } + stream_info = &axi_data->stream_info[ HANDLE_TO_IDX(update_cmd->stream_handle)]; if (stream_info->state != ACTIVE && stream_info->state != INACTIVE) { @@ -1258,7 +1314,9 @@ void msm_isp_process_axi_irq(struct vfe_device *vfe_dev, for (i = 0; i < axi_data->hw_info->num_comp_mask; i++) { comp_info = &axi_data->composite_info[i]; if (comp_mask & (1 << i)) { - if (!comp_info->stream_handle) { + stream_idx = HANDLE_TO_IDX(comp_info->stream_handle); + if ((!comp_info->stream_handle) || + (stream_idx >= MAX_NUM_STREAM)) { pr_err("%s: Invalid handle for composite irq\n", __func__); continue; @@ -1314,7 +1372,9 @@ void msm_isp_process_axi_irq(struct vfe_device *vfe_dev, for (i = 0; i < axi_data->hw_info->num_wm; i++) { if (wm_mask & (1 << i)) { - if (!axi_data->free_wm[i]) { + stream_idx = HANDLE_TO_IDX(axi_data->free_wm[i]); + if ((!axi_data->free_wm[i]) || + (stream_idx >= MAX_NUM_STREAM)) { pr_err("%s: Invalid handle for wm irq\n", __func__); continue; diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_stats_util.c b/drivers/media/platform/msm/camera_v2/isp/msm_isp_stats_util.c index d857a14ad57..56b0da1290d 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_stats_util.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_stats_util.c @@ -23,8 +23,16 @@ static int msm_isp_stats_cfg_ping_pong_address(struct vfe_device *vfe_dev, struct msm_isp_buffer *buf; uint32_t pingpong_bit = 0; uint32_t bufq_handle = stream_info->bufq_handle; - uint32_t stats_pingpong_offset = - STATS_IDX(stream_info->stream_handle) + + uint32_t stats_pingpong_offset; + + if (STATS_IDX(stream_info->stream_handle) >= + vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, + STATS_IDX(stream_info->stream_handle)); + return -EINVAL; + } + + stats_pingpong_offset = STATS_IDX(stream_info->stream_handle) + vfe_dev->hw_info->stats_hw_info->stats_ping_pong_offset; pingpong_bit = (~(pingpong_status >> stats_pingpong_offset) & 0x1); @@ -150,6 +158,11 @@ int msm_isp_stats_create_stream(struct vfe_device *vfe_dev, stats_idx = vfe_dev->hw_info->vfe_ops.stats_ops. get_stats_idx(stream_req_cmd->stats_type); + if (stats_idx >= vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, stats_idx); + return -EINVAL; + } + stream_info = &stats_data->stream_info[stats_idx]; if (stream_info->state != STATS_AVALIABLE) { pr_err("%s: Stats already requested\n", __func__); @@ -188,7 +201,7 @@ int msm_isp_stats_create_stream(struct vfe_device *vfe_dev, int msm_isp_request_stats_stream(struct vfe_device *vfe_dev, void *arg) { - int rc = 0; + int rc = -1; struct msm_vfe_stats_stream_request_cmd *stream_req_cmd = arg; struct msm_vfe_stats_stream *stream_info = NULL; struct msm_vfe_stats_shared_data *stats_data = &vfe_dev->stats_data; @@ -202,6 +215,12 @@ int msm_isp_request_stats_stream(struct vfe_device *vfe_dev, void *arg) } stats_idx = STATS_IDX(stream_req_cmd->stream_handle); + + if (stats_idx >= vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, stats_idx); + return -EINVAL; + } + stream_info = &stats_data->stream_info[stats_idx]; framedrop_period = msm_isp_get_framedrop_period( @@ -228,9 +247,14 @@ int msm_isp_release_stats_stream(struct vfe_device *vfe_dev, void *arg) struct msm_vfe_stats_stream_release_cmd *stream_release_cmd = arg; struct msm_vfe_stats_shared_data *stats_data = &vfe_dev->stats_data; int stats_idx = STATS_IDX(stream_release_cmd->stream_handle); - struct msm_vfe_stats_stream *stream_info = - &stats_data->stream_info[stats_idx]; + struct msm_vfe_stats_stream *stream_info = NULL; + if (stats_idx >= vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, stats_idx); + return -EINVAL; + } + + stream_info = &stats_data->stream_info[stats_idx]; if (stream_info->state == STATS_AVALIABLE) { pr_err("%s: stream already release\n", __func__); return rc; @@ -362,6 +386,12 @@ static int msm_isp_start_stats_stream(struct vfe_device *vfe_dev, struct msm_vfe_stats_shared_data *stats_data = &vfe_dev->stats_data; for (i = 0; i < stream_cfg_cmd->num_streams; i++) { idx = STATS_IDX(stream_cfg_cmd->stream_handle[i]); + + if (idx >= vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, idx); + return -EINVAL; + } + stream_info = &stats_data->stream_info[idx]; if (stream_info->stream_handle != stream_cfg_cmd->stream_handle[i]) { @@ -406,6 +436,12 @@ static int msm_isp_stop_stats_stream(struct vfe_device *vfe_dev, struct msm_vfe_stats_shared_data *stats_data = &vfe_dev->stats_data; for (i = 0; i < stream_cfg_cmd->num_streams; i++) { idx = STATS_IDX(stream_cfg_cmd->stream_handle[i]); + + if (idx >= vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, idx); + return -EINVAL; + } + stream_info = &stats_data->stream_info[idx]; if (stream_info->stream_handle != stream_cfg_cmd->stream_handle[i]) { @@ -436,6 +472,12 @@ static int msm_isp_stop_stats_stream(struct vfe_device *vfe_dev, for (i = 0; i < stream_cfg_cmd->num_streams; i++) { idx = STATS_IDX(stream_cfg_cmd->stream_handle[i]); + + if (idx >= vfe_dev->hw_info->stats_hw_info->num_stats_type) { + pr_err("%s Invalid stats index %d", __func__, idx); + return -EINVAL; + } + stream_info = &stats_data->stream_info[idx]; msm_isp_deinit_stats_ping_pong_reg(vfe_dev, stream_info); } diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c index 205e57e3931..26db557afbb 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c @@ -366,7 +366,7 @@ long msm_isp_ioctl(struct v4l2_subdev *sd, break; case VIDIOC_MSM_ISP_SET_SRC_STATE: mutex_lock(&vfe_dev->core_mutex); - msm_isp_set_src_state(vfe_dev, arg); + rc = msm_isp_set_src_state(vfe_dev, arg); mutex_unlock(&vfe_dev->core_mutex); break; case VIDIOC_MSM_ISP_REQUEST_STATS_STREAM: @@ -410,14 +410,108 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, struct msm_vfe_reg_cfg_cmd *reg_cfg_cmd, uint32_t *cfg_data, uint32_t cmd_len) { + if (!vfe_dev || !reg_cfg_cmd) { + pr_err("%s:%d failed: vfe_dev %pK reg_cfg_cmd %pK\n", __func__, + __LINE__, vfe_dev, reg_cfg_cmd); + return -EINVAL; + } + if ((reg_cfg_cmd->cmd_type != VFE_CFG_MASK) && + (!cfg_data || !cmd_len)) { + pr_err("%s:%d failed: cmd type %d cfg_data %pK cmd_len %d\n", + __func__, __LINE__, reg_cfg_cmd->cmd_type, cfg_data, + cmd_len); + return -EINVAL; + } + + /* Validate input parameters */ switch (reg_cfg_cmd->cmd_type) { - case VFE_WRITE: { - if (resource_size(vfe_dev->vfe_mem) < - (reg_cfg_cmd->u.rw_info.reg_offset + - reg_cfg_cmd->u.rw_info.len)) { - pr_err("%s: Invalid length\n", __func__); + case VFE_WRITE: + case VFE_READ: + case VFE_WRITE_MB: { + if ((reg_cfg_cmd->u.rw_info.reg_offset > + (UINT_MAX - reg_cfg_cmd->u.rw_info.len)) || + ((reg_cfg_cmd->u.rw_info.reg_offset + + reg_cfg_cmd->u.rw_info.len) > + resource_size(vfe_dev->vfe_mem))) { + pr_err("%s:%d reg_offset %d len %d res %d\n", + __func__, __LINE__, + reg_cfg_cmd->u.rw_info.reg_offset, + reg_cfg_cmd->u.rw_info.len, + (uint32_t)resource_size(vfe_dev->vfe_mem)); + return -EINVAL; + } + + if ((reg_cfg_cmd->u.rw_info.cmd_data_offset > + (UINT_MAX - reg_cfg_cmd->u.rw_info.len)) || + ((reg_cfg_cmd->u.rw_info.cmd_data_offset + + reg_cfg_cmd->u.rw_info.len) > cmd_len)) { + pr_err("%s:%d cmd_data_offset %d len %d cmd_len %d\n", + __func__, __LINE__, + reg_cfg_cmd->u.rw_info.cmd_data_offset, + reg_cfg_cmd->u.rw_info.len, cmd_len); return -EINVAL; } + break; + } + + case VFE_WRITE_DMI_16BIT: + case VFE_WRITE_DMI_32BIT: + case VFE_WRITE_DMI_64BIT: + case VFE_READ_DMI_16BIT: + case VFE_READ_DMI_32BIT: + case VFE_READ_DMI_64BIT: { + if (reg_cfg_cmd->cmd_type == VFE_WRITE_DMI_64BIT || + reg_cfg_cmd->cmd_type == VFE_READ_DMI_64BIT) { + if ((reg_cfg_cmd->u.dmi_info.hi_tbl_offset <= + reg_cfg_cmd->u.dmi_info.lo_tbl_offset) || + (reg_cfg_cmd->u.dmi_info.hi_tbl_offset - + reg_cfg_cmd->u.dmi_info.lo_tbl_offset != + (sizeof(uint32_t)))) { + pr_err("%s:%d hi %d lo %d\n", + __func__, __LINE__, + reg_cfg_cmd->u.dmi_info.hi_tbl_offset, + reg_cfg_cmd->u.dmi_info.hi_tbl_offset); + return -EINVAL; + } + if (reg_cfg_cmd->u.dmi_info.len <= sizeof(uint32_t)) { + pr_err("%s:%d len %d\n", + __func__, __LINE__, + reg_cfg_cmd->u.dmi_info.len); + return -EINVAL; + } + if (((UINT_MAX - + reg_cfg_cmd->u.dmi_info.hi_tbl_offset) < + (reg_cfg_cmd->u.dmi_info.len - + sizeof(uint32_t))) || + ((reg_cfg_cmd->u.dmi_info.hi_tbl_offset + + reg_cfg_cmd->u.dmi_info.len - + sizeof(uint32_t)) > cmd_len)) { + pr_err("%s:%d hi_tbl_offset %d len %d cmd %d\n", + __func__, __LINE__, + reg_cfg_cmd->u.dmi_info.hi_tbl_offset, + reg_cfg_cmd->u.dmi_info.len, cmd_len); + return -EINVAL; + } + } + if ((reg_cfg_cmd->u.dmi_info.lo_tbl_offset > + (UINT_MAX - reg_cfg_cmd->u.dmi_info.len)) || + ((reg_cfg_cmd->u.dmi_info.lo_tbl_offset + + reg_cfg_cmd->u.dmi_info.len) > cmd_len)) { + pr_err("%s:%d lo_tbl_offset %d len %d cmd_len %d\n", + __func__, __LINE__, + reg_cfg_cmd->u.dmi_info.lo_tbl_offset, + reg_cfg_cmd->u.dmi_info.len, cmd_len); + return -EINVAL; + } + break; + } + + default: + break; + } + + switch (reg_cfg_cmd->cmd_type) { + case VFE_WRITE: { msm_camera_io_memcpy(vfe_dev->vfe_base + reg_cfg_cmd->u.rw_info.reg_offset, cfg_data + reg_cfg_cmd->u.rw_info.cmd_data_offset/4, @@ -425,16 +519,25 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, break; } case VFE_WRITE_MB: { - uint32_t *data_ptr = cfg_data + - reg_cfg_cmd->u.rw_info.cmd_data_offset/4; - msm_camera_io_w_mb(*data_ptr, vfe_dev->vfe_base + - reg_cfg_cmd->u.rw_info.reg_offset); + msm_camera_io_memcpy_mb(vfe_dev->vfe_base + + reg_cfg_cmd->u.rw_info.reg_offset, + cfg_data + reg_cfg_cmd->u.rw_info.cmd_data_offset/4, + reg_cfg_cmd->u.rw_info.len); break; } case VFE_CFG_MASK: { uint32_t temp; + if ((UINT_MAX - sizeof(temp) < + reg_cfg_cmd->u.mask_info.reg_offset) || + (resource_size(vfe_dev->vfe_mem) < + reg_cfg_cmd->u.mask_info.reg_offset + + sizeof(temp))) { + pr_err("%s: VFE_CFG_MASK: Invalid length\n", __func__); + return -EINVAL; + } temp = msm_camera_io_r(vfe_dev->vfe_base + reg_cfg_cmd->u.mask_info.reg_offset); + temp &= ~reg_cfg_cmd->u.mask_info.mask; temp |= reg_cfg_cmd->u.mask_info.val; msm_camera_io_w(temp, vfe_dev->vfe_base + @@ -448,20 +551,9 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, uint32_t *hi_tbl_ptr = NULL, *lo_tbl_ptr = NULL; uint32_t hi_val, lo_val, lo_val1; if (reg_cfg_cmd->cmd_type == VFE_WRITE_DMI_64BIT) { - if (reg_cfg_cmd->u.dmi_info.hi_tbl_offset + - reg_cfg_cmd->u.dmi_info.len > cmd_len) { - pr_err("Invalid Hi Table out of bounds\n"); - return -EINVAL; - } hi_tbl_ptr = cfg_data + reg_cfg_cmd->u.dmi_info.hi_tbl_offset/4; } - - if (reg_cfg_cmd->u.dmi_info.lo_tbl_offset + - reg_cfg_cmd->u.dmi_info.len > cmd_len) { - pr_err("Invalid Lo Table out of bounds\n"); - return -EINVAL; - } lo_tbl_ptr = cfg_data + reg_cfg_cmd->u.dmi_info.lo_tbl_offset/4; @@ -492,30 +584,18 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, uint32_t *hi_tbl_ptr = NULL, *lo_tbl_ptr = NULL; uint32_t hi_val, lo_val, lo_val1; if (reg_cfg_cmd->cmd_type == VFE_READ_DMI_64BIT) { - if (reg_cfg_cmd->u.dmi_info.hi_tbl_offset + - reg_cfg_cmd->u.dmi_info.len > cmd_len) { - pr_err("Invalid Hi Table out of bounds\n"); - return -EINVAL; - } hi_tbl_ptr = cfg_data + reg_cfg_cmd->u.dmi_info.hi_tbl_offset/4; } - if (reg_cfg_cmd->u.dmi_info.lo_tbl_offset + - reg_cfg_cmd->u.dmi_info.len > cmd_len) { - pr_err("Invalid Lo Table out of bounds\n"); - return -EINVAL; - } lo_tbl_ptr = cfg_data + reg_cfg_cmd->u.dmi_info.lo_tbl_offset/4; - for (i = 0; i < reg_cfg_cmd->u.dmi_info.len/4; i++) { - if (reg_cfg_cmd->cmd_type == VFE_READ_DMI_64BIT) { - hi_val = msm_camera_io_r(vfe_dev->vfe_base + - vfe_dev->hw_info->dmi_reg_offset); - *hi_tbl_ptr++ = hi_val; - } + if (reg_cfg_cmd->cmd_type == VFE_READ_DMI_64BIT) + reg_cfg_cmd->u.dmi_info.len = + reg_cfg_cmd->u.dmi_info.len / 2; + for (i = 0; i < reg_cfg_cmd->u.dmi_info.len/4; i++) { lo_val = msm_camera_io_r(vfe_dev->vfe_base + vfe_dev->hw_info->dmi_reg_offset + 0x4); @@ -525,6 +605,13 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, lo_val |= lo_val1 << 16; } *lo_tbl_ptr++ = lo_val; + if (reg_cfg_cmd->cmd_type == VFE_READ_DMI_64BIT) { + hi_val = msm_camera_io_r(vfe_dev->vfe_base + + vfe_dev->hw_info->dmi_reg_offset); + *hi_tbl_ptr = hi_val; + hi_tbl_ptr += 2; + lo_tbl_ptr++; + } } break; } @@ -533,6 +620,12 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, uint32_t *data_ptr = cfg_data + reg_cfg_cmd->u.rw_info.cmd_data_offset/4; for (i = 0; i < reg_cfg_cmd->u.rw_info.len/4; i++) { + if ((data_ptr < cfg_data) || + (UINT_MAX / sizeof(*data_ptr) < + (data_ptr - cfg_data)) || + (sizeof(*data_ptr) * (data_ptr - cfg_data) >= + cmd_len)) + return -EINVAL; *data_ptr++ = msm_camera_io_r(vfe_dev->vfe_base + reg_cfg_cmd->u.rw_info.reg_offset); reg_cfg_cmd->u.rw_info.reg_offset += 4; @@ -550,6 +643,11 @@ int msm_isp_proc_cmd(struct vfe_device *vfe_dev, void *arg) struct msm_vfe_reg_cfg_cmd *reg_cfg_cmd; uint32_t *cfg_data; + if (!proc_cmd->num_cfg) { + pr_err("%s: Passed num_cfg as 0\n", __func__); + return -EINVAL; + } + if (proc_cmd->num_cfg > ISP_REG_CFG_NUM_CFG_MAX) { pr_err("%s: Unsupported number of commands %d!\n", __func__, proc_cmd->num_cfg); @@ -734,6 +832,8 @@ void msm_isp_update_error_frame_count(struct vfe_device *vfe_dev) void msm_isp_process_error_info(struct vfe_device *vfe_dev) { int i; + uint8_t num_stats_type = + vfe_dev->hw_info->stats_hw_info->num_stats_type; struct msm_vfe_error_info *error_info = &vfe_dev->error_info; static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); @@ -757,7 +857,7 @@ void msm_isp_process_error_info(struct vfe_device *vfe_dev) error_info->stream_framedrop_count[i] = 0; } } - for (i = 0; i < MSM_ISP_STATS_MAX; i++) { + for (i = 0; i < num_stats_type; i++) { if (error_info->stats_framedrop_count[i] != 0 && __ratelimit(&rs_stats)) { pr_err("%s: Stats stream[%d]: dropped %d frames\n", @@ -868,11 +968,14 @@ void msm_isp_do_tasklet(unsigned long data) } } -void msm_isp_set_src_state(struct vfe_device *vfe_dev, void *arg) +int msm_isp_set_src_state(struct vfe_device *vfe_dev, void *arg) { struct msm_vfe_axi_src_state *src_state = arg; + if (src_state->input_src >= VFE_SRC_MAX) + return -EINVAL; vfe_dev->axi_data.src_info[src_state->input_src].active = src_state->src_active; + return 0; } int msm_isp_config_done(struct vfe_device *vfe_dev, void *arg) diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.h b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.h index 50b7a87be4f..2a7744e6971 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.h +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.h @@ -64,7 +64,7 @@ int msm_isp_cal_word_per_line(uint32_t output_format, uint32_t pixel_per_line); int msm_isp_get_bit_per_pixel(uint32_t output_format); irqreturn_t msm_isp_process_irq(int irq_num, void *data); -void msm_isp_set_src_state(struct vfe_device *vfe_dev, void *arg); +int msm_isp_set_src_state(struct vfe_device *vfe_dev, void *arg); int msm_isp_config_done(struct vfe_device *vfe_dev, void *arg); void msm_isp_do_tasklet(unsigned long data); void msm_isp_update_error_frame_count(struct vfe_device *vfe_dev); diff --git a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c index df4505abefe..83a35f0564a 100644 --- a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c +++ b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c @@ -55,8 +55,8 @@ static void msm_ispif_io_dump_reg(struct ispif_device *ispif) static inline int msm_ispif_is_intf_valid(uint32_t csid_version, uint8_t intf_type) { - return (csid_version <= CSID_VERSION_V22 && intf_type != VFE0) ? - false : true; + return ((csid_version <= CSID_VERSION_V22 && intf_type != VFE0) || + (intf_type >= VFE_MAX)) ? false : true; } static struct msm_cam_clk_info ispif_8974_ahb_clk_info[] = { @@ -778,6 +778,13 @@ static irqreturn_t msm_io_ispif_irq(int irq_num, void *data) static int msm_ispif_set_vfe_info(struct ispif_device *ispif, struct msm_ispif_vfe_info *vfe_info) { + if (!vfe_info || (vfe_info->num_vfe <= 0) || + ((uint32_t)(vfe_info->num_vfe) > VFE_MAX)) { + pr_err("Invalid VFE info: %pK %d\n", vfe_info, + (vfe_info ? vfe_info->num_vfe:0)); + return -EINVAL; + } + memcpy(&ispif->vfe_info, vfe_info, sizeof(struct msm_ispif_vfe_info)); return 0; @@ -808,7 +815,7 @@ static int msm_ispif_init(struct ispif_device *ispif, if (ispif->csid_version >= CSID_VERSION_V30) { if (!ispif->clk_mux_mem || !ispif->clk_mux_io) { - pr_err("%s csi clk mux mem %p io %p\n", __func__, + pr_err("%s csi clk mux mem %pK io %pK\n", __func__, ispif->clk_mux_mem, ispif->clk_mux_io); rc = -ENOMEM; return rc; diff --git a/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_hw.c b/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_hw.c index 0a0fa046d31..35270a1237e 100644 --- a/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_hw.c +++ b/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_hw.c @@ -388,7 +388,7 @@ void msm_jpeg_io_dump(void *base, int size) int i; u32 *p = (u32 *) addr; u32 data; - JPEG_DBG_HIGH("%s:%d] %p %d", __func__, __LINE__, addr, size); + JPEG_DBG_HIGH("%s:%d] %pK %d", __func__, __LINE__, addr, size); line_str[0] = '\0'; p_str = line_str; for (i = 0; i < size/4; i++) { diff --git a/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_sync.c b/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_sync.c index 7f5b8d32045..fa9ecfe2d4f 100644 --- a/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_sync.c +++ b/drivers/media/platform/msm/camera_v2/jpeg_10/msm_jpeg_sync.c @@ -620,7 +620,7 @@ int __msm_jpeg_open(struct msm_jpeg_device *pgmn_dev) return rc; } - JPEG_DBG("%s:%d] platform resources - mem %p, base %p, irq %d\n", + JPEG_DBG("%s:%d] platform resources - mem %pK, base %pK, irq %d\n", __func__, __LINE__, pgmn_dev->mem, pgmn_dev->base, pgmn_dev->irq); pgmn_dev->res_size = resource_size(pgmn_dev->mem); diff --git a/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c b/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c index cb0e9923ddf..6244cabec51 100644 --- a/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c +++ b/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c @@ -44,7 +44,8 @@ #define MSM_CPP_DRV_NAME "msm_cpp" #define MSM_CPP_MAX_BUFF_QUEUE 16 -#define MSM_CPP_MAX_FRAME_LENGTH 1024 +#define MSM_CPP_MIN_FRAME_LENGTH 13 +#define MSM_CPP_MAX_FRAME_LENGTH 2048 #define MSM_CPP_MAX_FW_NAME_LEN 32 #define CONFIG_MSM_CPP_DBG 0 @@ -54,6 +55,16 @@ #define MSM_CPP_NOMINAL_CLOCK 266670000 #define MSM_CPP_TURBO_CLOCK 320000000 +#define CPP_FW_VERSION_1_2_0 0x10020000 +#define CPP_FW_VERSION_1_4_0 0x10040000 +#define CPP_FW_VERSION_1_6_0 0x10060000 +#define CPP_FW_VERSION_1_8_0 0x10080000 + +/* stripe information offsets in frame command */ +#define STRIPE_BASE_FW_1_2_0 130 +#define STRIPE_BASE_FW_1_4_0 140 +#define STRIPE_BASE_FW_1_6_0 464 + struct msm_cpp_timer_data_t { struct cpp_device *cpp_dev; struct msm_cpp_frame_info_t *processed_frame; @@ -260,8 +271,8 @@ static unsigned long msm_cpp_queue_buffer_info(struct cpp_device *cpp_dev, QUEUE_BUFF_ERROR2: ion_unmap_iommu(cpp_dev->client, buff->map_info.ion_handle, cpp_dev->domain_num, 0); -QUEUE_BUFF_ERROR1: ion_free(cpp_dev->client, buff->map_info.ion_handle); +QUEUE_BUFF_ERROR1: buff->map_info.ion_handle = NULL; kzfree(buff); @@ -770,7 +781,7 @@ static void cpp_load_fw(struct cpp_device *cpp_dev, char *fw_name_bin) rc = request_firmware(&fw, fw_name_bin, dev); if (rc) { dev_err(dev, - "Fail to loc blob %s from dev %p, Error: %d\n", + "Fail to loc blob %s from dev %pK, Error: %d\n", fw_name_bin, dev, rc); } if (NULL != fw) @@ -823,7 +834,8 @@ static void cpp_load_fw(struct cpp_device *cpp_dev, char *fw_name_bin) msm_cpp_poll(cpp_dev->base, MSM_CPP_MSG_ID_CMD); msm_cpp_poll(cpp_dev->base, 0x2); msm_cpp_poll(cpp_dev->base, MSM_CPP_MSG_ID_FW_VER); - pr_info("CPP FW Version: 0x%x\n", msm_cpp_read(cpp_dev->base)); + cpp_dev->fw_version = msm_cpp_read(cpp_dev->base); + pr_info("CPP FW Version: 0x%08x\n", cpp_dev->fw_version); msm_cpp_poll(cpp_dev->base, MSM_CPP_MSG_ID_TRAILER); /*Disable MC clock*/ @@ -857,7 +869,7 @@ static int cpp_open_node(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) return -ENODEV; } - CPP_DBG("open %d %p\n", i, &fh->vfh); + CPP_DBG("open %d %pK\n", i, &fh->vfh); cpp_dev->cpp_open_cnt++; if (cpp_dev->cpp_open_cnt == 1) { cpp_init_hardware(cpp_dev); @@ -1202,9 +1214,9 @@ static int msm_cpp_cfg(struct cpp_device *cpp_dev, struct msm_cpp_frame_info_t *u_frame_info = (struct msm_cpp_frame_info_t *)ioctl_ptr->ioctl_ptr; int32_t status = 0; - uint8_t fw_version_1_2_x = 0; int in_fd; + int32_t stripe_base = 0; int i = 0; if (!new_frame) { pr_err("Insufficient memory. return\n"); @@ -1249,7 +1261,16 @@ static int msm_cpp_cfg(struct cpp_device *cpp_dev, } new_frame->cpp_cmd_msg = cpp_frame_msg; - + if (cpp_frame_msg == NULL || + (new_frame->msg_len < MSM_CPP_MIN_FRAME_LENGTH)) { + pr_err("%s %d Length is not correct or frame message is missing\n", + __func__, __LINE__); + return -EINVAL; + } + if (cpp_frame_msg[new_frame->msg_len - 1] != MSM_CPP_MSG_ID_TRAILER) { + pr_err("%s %d Invalid frame message\n", __func__, __LINE__); + return -EINVAL; + } in_phyaddr = msm_cpp_fetch_buffer_info(cpp_dev, &new_frame->input_buffer_info, ((new_frame->identity >> 16) & 0xFFFF), @@ -1320,20 +1341,36 @@ static int msm_cpp_cfg(struct cpp_device *cpp_dev, ((cpp_frame_msg[12] >> 10) & 0x3FF) + (cpp_frame_msg[12] & 0x3FF); - fw_version_1_2_x = 0; - if (cpp_dev->hw_info.cpp_hw_version == 0x10010000) { - fw_version_1_2_x = 2; + if ((cpp_dev->fw_version & 0xffff0000) == + CPP_FW_VERSION_1_2_0) { + stripe_base = STRIPE_BASE_FW_1_2_0; + } else if ((cpp_dev->fw_version & 0xffff0000) == + CPP_FW_VERSION_1_4_0) { + stripe_base = STRIPE_BASE_FW_1_4_0; + } else if ((cpp_dev->fw_version & 0xffff0000) == + CPP_FW_VERSION_1_6_0) { + stripe_base = STRIPE_BASE_FW_1_6_0; + } else { + pr_err("invalid fw version %08x", cpp_dev->fw_version); + } + + if ((stripe_base + num_stripes*27 + 1) != new_frame->msg_len) { + pr_err("Invalid frame message\n"); + rc = -EINVAL; + goto ERROR3; } + + for (i = 0; i < num_stripes; i++) { - cpp_frame_msg[(133 + fw_version_1_2_x) + i * 27] += + cpp_frame_msg[stripe_base + 5 + i*27] += (uint32_t) in_phyaddr; - cpp_frame_msg[(139 + fw_version_1_2_x) + i * 27] += + cpp_frame_msg[stripe_base + 11 + i * 27] += (uint32_t) out_phyaddr0; - cpp_frame_msg[(140 + fw_version_1_2_x) + i * 27] += + cpp_frame_msg[stripe_base + 12 + i * 27] += (uint32_t) out_phyaddr1; - cpp_frame_msg[(141 + fw_version_1_2_x) + i * 27] += + cpp_frame_msg[stripe_base + 13 + i * 27] += (uint32_t) out_phyaddr0; - cpp_frame_msg[(142 + fw_version_1_2_x) + i * 27] += + cpp_frame_msg[stripe_base + 14 + i * 27] += (uint32_t) out_phyaddr1; } diff --git a/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.h b/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.h index 0a70d37bd7a..032091bebbf 100644 --- a/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.h +++ b/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.h @@ -173,7 +173,7 @@ struct cpp_device { char *fw_name_bin; struct workqueue_struct *timer_wq; struct msm_cpp_work_t *work; - + uint32_t fw_version; int domain_num; struct iommu_domain *domain; struct device *iommu_ctx; diff --git a/drivers/media/platform/msm/camera_v2/pproc/vpe/msm_vpe.c b/drivers/media/platform/msm/camera_v2/pproc/vpe/msm_vpe.c index e0266d60c3f..aeae2bc83e6 100644 --- a/drivers/media/platform/msm/camera_v2/pproc/vpe/msm_vpe.c +++ b/drivers/media/platform/msm/camera_v2/pproc/vpe/msm_vpe.c @@ -52,7 +52,7 @@ static void vpe_mem_dump(const char * const name, const void * const addr, int i; u32 *p = (u32 *) addr; u32 data; - VPE_DBG("%s: (%s) %p %d\n", __func__, name, addr, size); + VPE_DBG("%s: (%s) %pK %d\n", __func__, name, addr, size); line_str[0] = '\0'; p_str = line_str; for (i = 0; i < size/4; i++) { @@ -595,7 +595,7 @@ static int vpe_open_node(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) goto err_mutex_unlock; } - VPE_DBG("open %d %p\n", i, &fh->vfh); + VPE_DBG("open %d %pK\n", i, &fh->vfh); vpe_dev->vpe_open_cnt++; if (vpe_dev->vpe_open_cnt == 1) { rc = vpe_init_hardware(vpe_dev); @@ -651,7 +651,7 @@ static int vpe_close_node(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) return -ENODEV; } - VPE_DBG("close %d %p\n", i, &fh->vfh); + VPE_DBG("close %d %pK\n", i, &fh->vfh); vpe_dev->vpe_open_cnt--; if (vpe_dev->vpe_open_cnt == 0) { vpe_deinit_mem(vpe_dev); diff --git a/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c b/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c index 08f2f448f8e..c6d5d5fefe5 100755 --- a/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c +++ b/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c @@ -92,6 +92,11 @@ static void msm_actuator_parse_i2c_params(struct msm_actuator_ctrl_t *a_ctrl, i2c_byte2 = value & 0xFF; CDBG("byte1:0x%x, byte2:0x%x\n", i2c_byte1, i2c_byte2); + if (a_ctrl->i2c_tbl_index > + a_ctrl->total_steps) { + pr_err("failed:i2c table index out of bound\n"); + break; + } i2c_tbl[a_ctrl->i2c_tbl_index]. reg_addr = i2c_byte1; i2c_tbl[a_ctrl->i2c_tbl_index]. @@ -112,6 +117,10 @@ static void msm_actuator_parse_i2c_params(struct msm_actuator_ctrl_t *a_ctrl, i2c_byte2 = (hw_dword & write_arr[i].hw_mask) >> write_arr[i].hw_shift; } + if (a_ctrl->i2c_tbl_index > a_ctrl->total_steps) { + pr_err("failed: i2c table index out of bound\n"); + break; + } CDBG("i2c_byte1:0x%x, i2c_byte2:0x%x\n", i2c_byte1, i2c_byte2); i2c_tbl[a_ctrl->i2c_tbl_index].reg_addr = i2c_byte1; i2c_tbl[a_ctrl->i2c_tbl_index].reg_data = i2c_byte2; @@ -196,11 +205,19 @@ static int32_t msm_actuator_piezo_move_focus( struct msm_actuator_move_params_t *move_params) { int32_t dest_step_position = move_params->dest_step_pos; + struct damping_params_t ringing_params_kernel; int32_t rc = 0; int32_t num_steps = move_params->num_steps; struct msm_camera_i2c_reg_setting reg_setting; CDBG("Enter\n"); + if (copy_from_user(&ringing_params_kernel, + &(move_params->ringing_params[0]), + sizeof(struct damping_params_t))) { + pr_err("copy_from_user failed\n"); + return -EFAULT; + } + if (num_steps == 0) return rc; @@ -208,7 +225,7 @@ static int32_t msm_actuator_piezo_move_focus( a_ctrl->func_tbl->actuator_parse_i2c_params(a_ctrl, (num_steps * a_ctrl->region_params[0].code_per_step), - move_params->ringing_params[0].hw_params, 0); + ringing_params_kernel.hw_params, 0); reg_setting.reg_setting = a_ctrl->i2c_reg_tbl; reg_setting.data_type = a_ctrl->i2c_data_type; @@ -230,6 +247,7 @@ static int32_t msm_actuator_move_focus( struct msm_actuator_move_params_t *move_params) { int32_t rc = 0; + struct damping_params_t ringing_params_kernel; int8_t sign_dir = move_params->sign_dir; uint16_t step_boundary = 0; uint16_t target_step_pos = 0; @@ -240,11 +258,28 @@ static int32_t msm_actuator_move_focus( int32_t num_steps = move_params->num_steps; struct msm_camera_i2c_reg_setting reg_setting; + if (copy_from_user(&ringing_params_kernel, + &(move_params->ringing_params[a_ctrl->curr_region_index]), + sizeof(struct damping_params_t))) { + pr_err("copy_from_user failed\n"); + return -EFAULT; + } + + CDBG("called, dir %d, num_steps %d\n", dir, num_steps); if (dest_step_pos == a_ctrl->curr_step_pos) return rc; + if ((sign_dir > MSM_ACTUATOR_MOVE_SIGNED_NEAR) || + (sign_dir < MSM_ACTUATOR_MOVE_SIGNED_FAR)) { + pr_err("Invalid sign_dir = %d\n", sign_dir); + return -EFAULT; + } + if ((dir > MOVE_FAR) || (dir < MOVE_NEAR)) { + pr_err("Invalid direction = %d\n", dir); + return -EFAULT; + } if (dest_step_pos >= a_ctrl->total_steps) { CDBG("Step pos greater than total steps = %d\n", dest_step_pos); return -EFAULT; @@ -267,9 +302,7 @@ static int32_t msm_actuator_move_focus( a_ctrl->step_position_table[target_step_pos]; a_ctrl->func_tbl->actuator_write_focus(a_ctrl, curr_lens_pos, - &(move_params-> - ringing_params[a_ctrl-> - curr_region_index]), + &ringing_params_kernel, sign_dir, target_lens_pos); curr_lens_pos = target_lens_pos; @@ -280,8 +313,7 @@ static int32_t msm_actuator_move_focus( a_ctrl->step_position_table[target_step_pos]; a_ctrl->func_tbl->actuator_write_focus(a_ctrl, curr_lens_pos, - &(move_params->ringing_params[a_ctrl-> - curr_region_index]), + &ringing_params_kernel, sign_dir, target_lens_pos); curr_lens_pos = target_lens_pos; @@ -323,6 +355,12 @@ static int32_t msm_actuator_init_step_table(struct msm_actuator_ctrl_t *a_ctrl, kfree(a_ctrl->step_position_table); a_ctrl->step_position_table = NULL; + if (set_info->af_tuning_params.total_steps + > MAX_ACTUATOR_AF_TOTAL_STEPS) { + pr_err("Max actuator totalsteps exceeded = %d\n", + set_info->af_tuning_params.total_steps); + return -EFAULT; + } /* Fill step position table */ a_ctrl->step_position_table = kmalloc(sizeof(uint16_t) * @@ -426,12 +464,19 @@ static int32_t msm_actuator_init(struct msm_actuator_ctrl_t *a_ctrl, pr_err("Actuator function table not found\n"); return rc; } - - a_ctrl->region_size = set_info->af_tuning_params.region_size; - if (a_ctrl->region_size > MAX_ACTUATOR_REGION) { + if (set_info->af_tuning_params.total_steps + > MAX_ACTUATOR_AF_TOTAL_STEPS) { + pr_err("Max actuator totalsteps exceeded = %d\n", + set_info->af_tuning_params.total_steps); + return -EFAULT; + } + if (set_info->af_tuning_params.region_size + > MAX_ACTUATOR_REGION) { pr_err("MAX_ACTUATOR_REGION is exceeded.\n"); return -EFAULT; } + + a_ctrl->region_size = set_info->af_tuning_params.region_size; a_ctrl->pwd_step = set_info->af_tuning_params.pwd_step; a_ctrl->total_steps = set_info->af_tuning_params.total_steps; @@ -454,8 +499,11 @@ static int32_t msm_actuator_init(struct msm_actuator_ctrl_t *a_ctrl, a_ctrl->i2c_data_type = set_info->actuator_params.i2c_data_type; a_ctrl->i2c_client.addr_type = set_info->actuator_params.i2c_addr_type; - a_ctrl->reg_tbl_size = set_info->actuator_params.reg_tbl_size; - if (a_ctrl->reg_tbl_size > MAX_ACTUATOR_REG_TBL_SIZE) { + if (set_info->actuator_params.reg_tbl_size <= + MAX_ACTUATOR_REG_TBL_SIZE) { + a_ctrl->reg_tbl_size = set_info->actuator_params.reg_tbl_size; + } else { + a_ctrl->reg_tbl_size = 0; pr_err("MAX_ACTUATOR_REG_TBL_SIZE is exceeded.\n"); return -EFAULT; } @@ -478,7 +526,9 @@ static int32_t msm_actuator_init(struct msm_actuator_ctrl_t *a_ctrl, return -EFAULT; } - if (set_info->actuator_params.init_setting_size) { + if (set_info->actuator_params.init_setting_size && + set_info->actuator_params.init_setting_size + <= MAX_ACTUATOR_REG_TBL_SIZE) { if (a_ctrl->func_tbl->actuator_init_focus) { init_settings = kmalloc(sizeof(struct reg_settings_t) * (set_info->actuator_params.init_setting_size), @@ -657,7 +707,7 @@ static long msm_actuator_subdev_ioctl(struct v4l2_subdev *sd, struct msm_actuator_ctrl_t *a_ctrl = v4l2_get_subdevdata(sd); void __user *argp = (void __user *)arg; CDBG("Enter\n"); - CDBG("%s:%d a_ctrl %p argp %p\n", __func__, __LINE__, a_ctrl, argp); + CDBG("%s:%d a_ctrl %pK argp %pK\n", __func__, __LINE__, a_ctrl, argp); switch (cmd) { case VIDIOC_MSM_SENSOR_GET_SUBDEV_ID: return msm_actuator_get_subdev_id(a_ctrl, argp); @@ -810,6 +860,7 @@ static int32_t msm_actuator_platform_probe(struct platform_device *pdev) &pdev->id); CDBG("cell-index %d, rc %d\n", pdev->id, rc); if (rc < 0) { + kfree(msm_actuator_t); pr_err("failed rc %d\n", rc); return rc; } @@ -818,6 +869,7 @@ static int32_t msm_actuator_platform_probe(struct platform_device *pdev) &msm_actuator_t->cci_master); CDBG("qcom,cci-master %d, rc %d\n", msm_actuator_t->cci_master, rc); if (rc < 0) { + kfree(msm_actuator_t); pr_err("failed rc %d\n", rc); return rc; } @@ -834,6 +886,7 @@ static int32_t msm_actuator_platform_probe(struct platform_device *pdev) msm_actuator_t->i2c_client.cci_client = kzalloc(sizeof( struct msm_camera_cci_client), GFP_KERNEL); if (!msm_actuator_t->i2c_client.cci_client) { + kfree(msm_actuator_t); pr_err("failed no memory\n"); return -ENOMEM; } diff --git a/drivers/media/platform/msm/camera_v2/sensor/cci/msm_cci.c b/drivers/media/platform/msm/camera_v2/sensor/cci/msm_cci.c index c99d6185162..73ec7ecb5e5 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/cci/msm_cci.c +++ b/drivers/media/platform/msm/camera_v2/sensor/cci/msm_cci.c @@ -184,7 +184,7 @@ static int32_t msm_cci_data_queue(struct cci_device *cci_dev, uint16_t i = 0, j = 0, k = 0, h = 0, len = 0; int32_t rc = 0; uint32_t cmd = 0, delay = 0; - uint8_t data[10]; + uint8_t data[11]; uint16_t reg_addr = 0; struct msm_camera_i2c_reg_setting *i2c_msg = &c_ctrl->cfg.cci_i2c_write_cfg; @@ -471,7 +471,7 @@ static int32_t msm_cci_i2c_read_bytes(struct v4l2_subdev *sd, uint16_t read_bytes = 0; if (!sd || !c_ctrl) { - pr_err("%s:%d sd %p c_ctrl %p\n", __func__, + pr_err("%s:%d sd %pK c_ctrl %pK\n", __func__, __LINE__, sd, c_ctrl); return -EINVAL; } @@ -626,7 +626,7 @@ static int32_t msm_cci_i2c_write(struct v4l2_subdev *sd, msm_cci_flush_queue(cci_dev, master); goto ERROR; } else { - rc = 0; + rc = cci_dev->cci_master_info[master].status; } CDBG("%s:%d X wait_for_completion_interruptible\n", __func__, __LINE__); @@ -661,7 +661,7 @@ static int32_t msm_cci_init(struct v4l2_subdev *sd, cci_dev = v4l2_get_subdevdata(sd); if (!cci_dev || !c_ctrl) { - pr_err("%s:%d failed: invalid params %p %p\n", __func__, + pr_err("%s:%d failed: invalid params %pK %pK\n", __func__, __LINE__, cci_dev, c_ctrl); rc = -ENOMEM; return rc; @@ -1085,7 +1085,7 @@ static int __devinit msm_cci_probe(struct platform_device *pdev) { struct cci_device *new_cci_dev; int rc = 0; - pr_err("%s: pdev %p device id = %d\n", __func__, pdev, pdev->id); + pr_err("%s: pdev %pK device id = %d\n", __func__, pdev, pdev->id); new_cci_dev = kzalloc(sizeof(struct cci_device), GFP_KERNEL); if (!new_cci_dev) { CDBG("%s: no enough memory\n", __func__); @@ -1097,7 +1097,7 @@ static int __devinit msm_cci_probe(struct platform_device *pdev) ARRAY_SIZE(new_cci_dev->msm_sd.sd.name), "msm_cci"); v4l2_set_subdevdata(&new_cci_dev->msm_sd.sd, new_cci_dev); platform_set_drvdata(pdev, &new_cci_dev->msm_sd.sd); - CDBG("%s sd %p\n", __func__, &new_cci_dev->msm_sd.sd); + CDBG("%s sd %pK\n", __func__, &new_cci_dev->msm_sd.sd); if (pdev->dev.of_node) of_property_read_u32((&pdev->dev)->of_node, "cell-index", &pdev->id); @@ -1153,7 +1153,7 @@ static int __devinit msm_cci_probe(struct platform_device *pdev) pr_err("%s: failed to add child nodes, rc=%d\n", __func__, rc); new_cci_dev->cci_state = CCI_STATE_DISABLED; g_cci_subdev = &new_cci_dev->msm_sd.sd; - CDBG("%s cci subdev %p\n", __func__, &new_cci_dev->msm_sd.sd); + CDBG("%s cci subdev %pK\n", __func__, &new_cci_dev->msm_sd.sd); CDBG("%s line %d\n", __func__, __LINE__); return 0; diff --git a/drivers/media/platform/msm/camera_v2/sensor/csid/msm_csid.c b/drivers/media/platform/msm/camera_v2/sensor/csid/msm_csid.c index 9f19e51a362..33b25635aca 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/csid/msm_csid.c +++ b/drivers/media/platform/msm/camera_v2/sensor/csid/msm_csid.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -49,6 +49,13 @@ static int msm_csid_cid_lut( return -EINVAL; } for (i = 0; i < csid_lut_params->num_cid && i < 16; i++) { + if (csid_lut_params->vc_cfg[i]->cid >= + csid_lut_params->num_cid || + csid_lut_params->vc_cfg[i]->cid < 0) { + pr_err("%s: cid outside range %d\n", + __func__, csid_lut_params->vc_cfg[i]->cid); + return -EINVAL; + } CDBG("%s lut params num_cid = %d, cid = %d, dt = %x, df = %d\n", __func__, csid_lut_params->num_cid, @@ -106,7 +113,7 @@ static int msm_csid_config(struct csid_device *csid_dev, void __iomem *csidbase; csidbase = csid_dev->base; if (!csidbase || !csid_params) { - pr_err("%s:%d csidbase %p, csid params %p\n", __func__, + pr_err("%s:%d csidbase %pK, csid params %pK\n", __func__, __LINE__, csidbase, csid_params); return -EINVAL; } @@ -426,7 +433,7 @@ static long msm_csid_cmd(struct csid_device *csid_dev, void *arg) struct csid_cfg_data *cdata = (struct csid_cfg_data *)arg; if (!csid_dev || !cdata) { - pr_err("%s:%d csid_dev %p, cdata %p\n", __func__, __LINE__, + pr_err("%s:%d csid_dev %pK, cdata %pK\n", __func__, __LINE__, csid_dev, cdata); return -EINVAL; } @@ -441,7 +448,7 @@ static long msm_csid_cmd(struct csid_device *csid_dev, void *arg) struct msm_sensor_csid_cfg_params *u_csid_cfg_params; struct msm_camera_csid_params csid_params; struct msm_camera_csid_vc_cfg *vc_cfg = NULL; - int32_t i = 0; + int8_t i = 0; u_csid_cfg_params = &cdata->cfg.csid_cfg_params; if (u_csid_cfg_params->csid_params_size != sizeof(csid_params)) { @@ -459,10 +466,10 @@ static long msm_csid_cmd(struct csid_device *csid_dev, void *arg) rc = -EFAULT; break; } - if (csid_params.lut_params.num_cid > MAX_CID) { - pr_err("%s:%d invalid num_cid %d max %d", __func__, - __LINE__, csid_params.lut_params.num_cid, - MAX_CID); + if (csid_params.lut_params.num_cid < 1 || + csid_params.lut_params.num_cid > MAX_CID) { + pr_err("%s: %d num_cid outside range\n", + __func__, __LINE__); rc = -EINVAL; break; } @@ -498,6 +505,10 @@ static long msm_csid_cmd(struct csid_device *csid_dev, void *arg) } csid_params.lut_params.vc_cfg[i] = vc_cfg; } + if (rc < 0) { + pr_err("%s:%d failed\n", __func__, __LINE__); + break; + } rc = msm_csid_config(csid_dev, &csid_params); for (i = 0; i < csid_params.lut_params.num_cid; i++) kfree(csid_params.lut_params.vc_cfg[i]); diff --git a/drivers/media/platform/msm/camera_v2/sensor/csiphy/msm_csiphy.c b/drivers/media/platform/msm/camera_v2/sensor/csiphy/msm_csiphy.c index 239dace98a0..1480367567e 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/csiphy/msm_csiphy.c +++ b/drivers/media/platform/msm/camera_v2/sensor/csiphy/msm_csiphy.c @@ -79,7 +79,7 @@ static int msm_csiphy_lane_config(struct csiphy_device *csiphy_dev, val |= csiphy_params->csid_core; } msm_camera_io_w(val, csiphy_dev->clk_mux_base); - CDBG("%s clk mux addr %p val 0x%x\n", __func__, + CDBG("%s clk mux addr %pK val 0x%x\n", __func__, csiphy_dev->clk_mux_base, val); mb(); } @@ -234,7 +234,7 @@ static int msm_csiphy_init(struct csiphy_device *csiphy_dev) ARRAY_SIZE(csiphy_8610_clk_info), 1); } else if (CSIPHY_VERSION == CSIPHY_VERSION_V30) { if (!csiphy_dev->clk_mux_mem || !csiphy_dev->clk_mux_io) { - pr_err("%s clk mux mem %p io %p\n", __func__, + pr_err("%s clk mux mem %pK io %pK\n", __func__, csiphy_dev->clk_mux_mem, csiphy_dev->clk_mux_io); rc = -ENOMEM; @@ -337,7 +337,7 @@ static int msm_csiphy_init(struct csiphy_device *csiphy_dev) ARRAY_SIZE(csiphy_8610_clk_info), 1); } else if (CSIPHY_VERSION == CSIPHY_VERSION_V30) { if (!csiphy_dev->clk_mux_mem || !csiphy_dev->clk_mux_io) { - pr_err("%s clk mux mem %p io %p\n", __func__, + pr_err("%s clk mux mem %pK io %pK\n", __func__, csiphy_dev->clk_mux_mem, csiphy_dev->clk_mux_io); rc = -ENOMEM; @@ -399,7 +399,7 @@ static int msm_csiphy_release(struct csiphy_device *csiphy_dev, void *arg) struct msm_camera_csi_lane_params *csi_lane_params; uint16_t csi_lane_mask; csi_lane_params = (struct msm_camera_csi_lane_params *)arg; - csi_lane_mask = csi_lane_params->csi_lane_mask; + csi_lane_mask = (csi_lane_params->csi_lane_mask & 0x1F); if (!csiphy_dev || !csiphy_dev->ref_count) { pr_err("%s csiphy dev NULL / ref_count ZERO\n", __func__); @@ -426,7 +426,7 @@ static int msm_csiphy_release(struct csiphy_device *csiphy_dev, void *arg) csiphy_dev->lane_mask[csiphy_dev->pdev->id] &= ~(csi_lane_params->csi_lane_mask); i = 0; - while (csi_lane_mask & 0x1F) { + while (csi_lane_mask) { if (csi_lane_mask & 0x1) { msm_camera_io_w(0x0, csiphy_dev->base + MIPI_CSIPHY_LNn_CFG2_ADDR + 0x40*i); @@ -474,7 +474,7 @@ static int msm_csiphy_release(struct csiphy_device *csiphy_dev, void *arg) struct msm_camera_csi_lane_params *csi_lane_params; uint16_t csi_lane_mask; csi_lane_params = (struct msm_camera_csi_lane_params *)arg; - csi_lane_mask = csi_lane_params->csi_lane_mask; + csi_lane_mask = (csi_lane_params->csi_lane_mask & 0x1F); if (!csiphy_dev || !csiphy_dev->ref_count) { pr_err("%s csiphy dev NULL / ref_count ZERO\n", __func__); @@ -501,7 +501,7 @@ static int msm_csiphy_release(struct csiphy_device *csiphy_dev, void *arg) csiphy_dev->lane_mask[csiphy_dev->pdev->id] &= ~(csi_lane_params->csi_lane_mask); i = 0; - while (csi_lane_mask & 0x1F) { + while (csi_lane_mask) { if (csi_lane_mask & 0x1) { msm_camera_io_w(0x0, csiphy_dev->base + MIPI_CSIPHY_LNn_CFG2_ADDR + 0x40*i); diff --git a/drivers/media/platform/msm/camera_v2/sensor/eeprom/msm_eeprom.c b/drivers/media/platform/msm/camera_v2/sensor/eeprom/msm_eeprom.c index 84222daf98e..f3d32598691 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/eeprom/msm_eeprom.c +++ b/drivers/media/platform/msm/camera_v2/sensor/eeprom/msm_eeprom.c @@ -83,7 +83,7 @@ static long msm_eeprom_subdev_ioctl(struct v4l2_subdev *sd, struct msm_eeprom_ctrl_t *e_ctrl = v4l2_get_subdevdata(sd); void __user *argp = (void __user *)arg; CDBG("%s E\n", __func__); - CDBG("%s:%d a_ctrl %p argp %p\n", __func__, __LINE__, e_ctrl, argp); + CDBG("%s:%d a_ctrl %pK argp %pK\n", __func__, __LINE__, e_ctrl, argp); switch (cmd) { case VIDIOC_MSM_SENSOR_GET_SUBDEV_ID: return msm_eeprom_get_subdev_id(e_ctrl, argp); diff --git a/drivers/media/platform/msm/camera_v2/sensor/flash/msm_led_trigger.c b/drivers/media/platform/msm/camera_v2/sensor/flash/msm_led_trigger.c index 377e309878d..51a451b5c46 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/flash/msm_led_trigger.c +++ b/drivers/media/platform/msm/camera_v2/sensor/flash/msm_led_trigger.c @@ -50,7 +50,7 @@ static int32_t msm_led_trigger_config(struct msm_led_flash_ctrl_t *fctrl, CDBG("called led_state %d\n", cfg->cfgtype); if (!fctrl) { - pr_err("failed: fctrl %p\n", fctrl); + pr_err("failed: fctrl %pK\n", fctrl); return -EINVAL; } diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c index 0c2aa0fd87d..8bbb02a9510 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -32,7 +32,7 @@ int32_t msm_camera_cci_i2c_read(struct msm_camera_i2c_client *client, enum msm_camera_i2c_data_type data_type) { int32_t rc = -EFAULT; - unsigned char buf[client->addr_type+data_type]; + unsigned char *buf = NULL; struct msm_camera_cci_ctrl cci_ctrl; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR @@ -41,6 +41,17 @@ int32_t msm_camera_cci_i2c_read(struct msm_camera_i2c_client *client, && data_type != MSM_CAMERA_I2C_WORD_DATA)) return rc; + if (client->addr_type > UINT_MAX - data_type) { + pr_err("%s: integer overflow prevented\n", __func__); + return rc; + } + + buf = kzalloc(client->addr_type+data_type, GFP_KERNEL); + if (!buf) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } + cci_ctrl.cmd = MSM_CCI_I2C_READ; cci_ctrl.cci_info = client->cci_client; cci_ctrl.cfg.cci_i2c_read_cfg.addr = addr; @@ -51,6 +62,8 @@ int32_t msm_camera_cci_i2c_read(struct msm_camera_i2c_client *client, core, ioctl, VIDIOC_MSM_CCI_CFG, &cci_ctrl); if (rc < 0) { pr_err("%s: line %d rc = %d\n", __func__, __LINE__, rc); + kfree(buf); + buf = NULL; return rc; } rc = cci_ctrl.status; @@ -60,6 +73,8 @@ int32_t msm_camera_cci_i2c_read(struct msm_camera_i2c_client *client, *data = buf[0] << 8 | buf[1]; S_I2C_DBG("%s addr = 0x%x data: 0x%x\n", __func__, addr, *data); + kfree(buf); + buf = NULL; return rc; } @@ -76,6 +91,12 @@ int32_t msm_camera_cci_i2c_read_seq(struct msm_camera_i2c_client *client, || num_byte == 0) return rc; + if (num_byte > I2C_REG_DATA_MAX) { + pr_err("%s: Error num_byte:0x%x exceeds 8K max supported:0x%x\n", + __func__, num_byte, I2C_REG_DATA_MAX); + return rc; + } + buf = kzalloc(num_byte, GFP_KERNEL); if (!buf) { pr_err("%s:%d no memory\n", __func__, __LINE__); @@ -142,7 +163,7 @@ int32_t msm_camera_cci_i2c_write_seq(struct msm_camera_i2c_client *client, int32_t rc = -EFAULT; uint8_t i = 0; struct msm_camera_cci_ctrl cci_ctrl; - struct msm_camera_i2c_reg_array reg_conf_tbl[num_byte]; + struct msm_camera_i2c_reg_array *reg_conf_tbl = NULL; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR && client->addr_type != MSM_CAMERA_I2C_WORD_ADDR) @@ -151,8 +172,14 @@ int32_t msm_camera_cci_i2c_write_seq(struct msm_camera_i2c_client *client, S_I2C_DBG("%s reg addr = 0x%x num bytes: %d\n", __func__, addr, num_byte); - memset(reg_conf_tbl, 0, - num_byte * sizeof(struct msm_camera_i2c_reg_array)); + + reg_conf_tbl = kzalloc(num_byte * + (sizeof(struct msm_camera_i2c_reg_array)), GFP_KERNEL); + if (!reg_conf_tbl) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } + reg_conf_tbl[0].reg_addr = addr; for (i = 0; i < num_byte; i++) { reg_conf_tbl[i].reg_data = data[i]; @@ -168,6 +195,8 @@ int32_t msm_camera_cci_i2c_write_seq(struct msm_camera_i2c_client *client, core, ioctl, VIDIOC_MSM_CCI_CFG, &cci_ctrl); CDBG("%s line %d rc = %d\n", __func__, __LINE__, rc); rc = cci_ctrl.status; + kfree(reg_conf_tbl); + reg_conf_tbl = NULL; return rc; } @@ -233,6 +262,12 @@ int32_t msm_camera_cci_i2c_write_seq_table( client_addr_type = client->addr_type; client->addr_type = write_setting->addr_type; + if (reg_setting->reg_data_size > I2C_SEQ_REG_DATA_MAX) { + pr_err("%s: number of bytes %u exceeding the max supported %d\n", + __func__, reg_setting->reg_data_size, I2C_SEQ_REG_DATA_MAX); + return rc; + } + for (i = 0; i < write_setting->size; i++) { rc = msm_camera_cci_i2c_write_seq(client, reg_setting->reg_addr, reg_setting->reg_data, reg_setting->reg_data_size); diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_dt_util.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_dt_util.c index 2511651ca1e..08341cc3bf1 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_dt_util.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_dt_util.c @@ -427,7 +427,7 @@ int32_t msm_camera_power_up(struct msm_camera_power_ctrl_t *ctrl, CDBG("%s:%d\n", __func__, __LINE__); if (!ctrl || !sensor_i2c_client) { - pr_err("failed ctrl %p sensor_i2c_client %p\n", ctrl, + pr_err("failed ctrl %pK sensor_i2c_client %pK\n", ctrl, sensor_i2c_client); return -EINVAL; } @@ -593,7 +593,7 @@ int32_t msm_camera_power_down(struct msm_camera_power_ctrl_t *ctrl, CDBG("%s:%d\n", __func__, __LINE__); if (!ctrl || !sensor_i2c_client) { - pr_err("failed ctrl %p sensor_i2c_client %p\n", ctrl, + pr_err("failed ctrl %pK sensor_i2c_client %pK\n", ctrl, sensor_i2c_client); return -EINVAL; } diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.c index c2a5cad308e..d95081ee3ba 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.c @@ -78,7 +78,7 @@ void msm_camera_io_dump(void __iomem *addr, int size) int i; u32 *p = (u32 *) addr; u32 data; - CDBG("%s: %p %d\n", __func__, addr, size); + CDBG("%s: %pK %d\n", __func__, addr, size); line_str[0] = '\0'; p_str = line_str; for (i = 0; i < size/4; i++) { @@ -102,11 +102,22 @@ void msm_camera_io_dump(void __iomem *addr, int size) void msm_camera_io_memcpy(void __iomem *dest_addr, void __iomem *src_addr, u32 len) { - CDBG("%s: %p %p %d\n", __func__, dest_addr, src_addr, len); + CDBG("%s: %pK %pK %d\n", __func__, dest_addr, src_addr, len); msm_camera_io_memcpy_toio(dest_addr, src_addr, len / 4); msm_camera_io_dump(dest_addr, len); } +void msm_camera_io_memcpy_mb(void __iomem *dest_addr, + void __iomem *src_addr, u32 len) +{ + int i; + u32 *d = (u32 *) dest_addr; + u32 *s = (u32 *) src_addr; + + for (i = 0; i < (len / 4); i++) + msm_camera_io_w_mb(*s++, d++); +} + int msm_cam_clk_sel_src(struct device *dev, struct msm_cam_clk_info *clk_info, struct msm_cam_clk_info *clk_src_info, int num_clk) { @@ -528,7 +539,7 @@ int msm_camera_request_gpio_table(struct gpio *gpio_tbl, uint8_t size, int rc = 0, i = 0; if (!gpio_tbl || !size) { - pr_err("%s:%d invalid gpio_tbl %p / size %d\n", __func__, + pr_err("%s:%d invalid gpio_tbl %pK / size %d\n", __func__, __LINE__, gpio_tbl, size); return -EINVAL; } diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.h b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.h index 73376e2c072..40bc7ce34bc 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.h +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_io_util.h @@ -25,6 +25,8 @@ u32 msm_camera_io_r_mb(void __iomem *addr); void msm_camera_io_dump(void __iomem *addr, int size); void msm_camera_io_memcpy(void __iomem *dest_addr, void __iomem *src_addr, u32 len); +void msm_camera_io_memcpy_mb(void __iomem *dest_addr, + void __iomem *src_addr, u32 len); int msm_cam_clk_sel_src(struct device *dev, struct msm_cam_clk_info *clk_info, struct msm_cam_clk_info *clk_src_info, int num_clk); int msm_cam_clk_enable(struct device *dev, struct msm_cam_clk_info *clk_info, diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c index a5b9da0c252..89f1ef2b866 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011, 2013-2014,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -73,7 +73,7 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, enum msm_camera_i2c_data_type data_type) { int32_t rc = -EFAULT; - unsigned char buf[client->addr_type+data_type]; + unsigned char *buf = NULL; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR && client->addr_type != MSM_CAMERA_I2C_WORD_ADDR) @@ -81,6 +81,17 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, && data_type != MSM_CAMERA_I2C_WORD_DATA)) return rc; + if (client->addr_type > UINT_MAX - data_type) { + pr_err("%s: integer overflow prevented\n", __func__); + return rc; + } + + buf = kzalloc(client->addr_type+data_type, GFP_KERNEL); + if (!buf) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } + if (client->addr_type == MSM_CAMERA_I2C_BYTE_ADDR) { buf[0] = addr; } else if (client->addr_type == MSM_CAMERA_I2C_WORD_ADDR) { @@ -90,6 +101,8 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, rc = msm_camera_qup_i2c_rxdata(client, buf, data_type); if (rc < 0) { S_I2C_DBG("%s fail\n", __func__); + kfree(buf); + buf = NULL; return rc; } @@ -99,6 +112,8 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, *data = buf[0] << 8 | buf[1]; S_I2C_DBG("%s addr = 0x%x data: 0x%x\n", __func__, addr, *data); + kfree(buf); + buf = NULL; return rc; } @@ -106,7 +121,7 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, uint32_t addr, uint8_t *data, uint16_t num_byte) { int32_t rc = -EFAULT; - unsigned char buf[client->addr_type+num_byte]; + unsigned char *buf = NULL; int i; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR @@ -114,6 +129,22 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, || num_byte == 0) return rc; + if (num_byte > I2C_REG_DATA_MAX) { + pr_err("%s: Error num_byte:0x%x exceeds 8K max supported:0x%x\n", + __func__, num_byte, I2C_REG_DATA_MAX); + return rc; + } + if (client->addr_type > UINT_MAX - num_byte) { + pr_err("%s: integer overflow prevented\n", __func__); + return rc; + } + + buf = kzalloc(client->addr_type+num_byte, GFP_KERNEL); + if (!buf) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } + if (client->addr_type == MSM_CAMERA_I2C_BYTE_ADDR) { buf[0] = addr; } else if (client->addr_type == MSM_CAMERA_I2C_WORD_ADDR) { @@ -123,6 +154,8 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, rc = msm_camera_qup_i2c_rxdata(client, buf, num_byte); if (rc < 0) { S_I2C_DBG("%s fail\n", __func__); + kfree(buf); + buf = NULL; return rc; } @@ -132,6 +165,8 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, S_I2C_DBG("Byte %d: 0x%x\n", i, buf[i]); S_I2C_DBG("Data: 0x%x\n", data[i]); } + kfree(buf); + buf = NULL; return rc; } @@ -285,6 +320,12 @@ int32_t msm_camera_qup_i2c_write_seq_table(struct msm_camera_i2c_client *client, client_addr_type = client->addr_type; client->addr_type = write_setting->addr_type; + if (reg_setting->reg_data_size > I2C_SEQ_REG_DATA_MAX) { + pr_err("%s: number of bytes %u exceeding the max supported %d\n", + __func__, reg_setting->reg_data_size, I2C_SEQ_REG_DATA_MAX); + return rc; + } + for (i = 0; i < write_setting->size; i++) { rc = msm_camera_qup_i2c_write_seq(client, reg_setting->reg_addr, reg_setting->reg_data, reg_setting->reg_data_size); diff --git a/drivers/media/platform/msm/camera_v2/sensor/msm_sensor.c b/drivers/media/platform/msm/camera_v2/sensor/msm_sensor.c index 5b7a5eb19ac..7eb046fc574 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/msm_sensor.c +++ b/drivers/media/platform/msm/camera_v2/sensor/msm_sensor.c @@ -1250,6 +1250,10 @@ int32_t msm_sensor_match_id(struct msm_sensor_ctrl_t *s_ctrl) CDBG("%s: read id: %x expected id %x:\n", __func__, chipid, s_ctrl->sensordata->slave_info->sensor_id); +#ifdef CONFIG_IMX179 + s_ctrl->sensordata->slave_info->sensor_id &= 0xfff; + chipid &= 0xfff; +#endif if (chipid != s_ctrl->sensordata->slave_info->sensor_id) { pr_err("msm_sensor_match_id chip id doesnot match\n"); return -ENODEV; @@ -1961,7 +1965,7 @@ int32_t msm_sensor_platform_probe(struct platform_device *pdev, void *data) uint32_t session_id; s_ctrl->pdev = pdev; s_ctrl->dev = &pdev->dev; - CDBG("%s called data %p\n", __func__, data); + CDBG("%s called data %pK\n", __func__, data); CDBG("%s pdev name %s\n", __func__, pdev->id_entry->name); if (pdev->dev.of_node) { rc = msm_sensor_get_dt_data(pdev->dev.of_node, s_ctrl); diff --git a/drivers/media/platform/msm/camera_v2/sensor/ois/msm_ois.c b/drivers/media/platform/msm/camera_v2/sensor/ois/msm_ois.c index 27470fc166f..794e73e146c 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/ois/msm_ois.c +++ b/drivers/media/platform/msm/camera_v2/sensor/ois/msm_ois.c @@ -419,7 +419,7 @@ static long msm_ois_subdev_ioctl(struct v4l2_subdev *sd, CDBG("Enter\n"); - CDBG("%s:%d a_ctrl %p argp %p\n", __func__, __LINE__, a_ctrl, argp); + CDBG("%s:%d a_ctrl %pK argp %pK\n", __func__, __LINE__, a_ctrl, argp); return -ENOIOCTLCMD; } diff --git a/drivers/media/platform/msm/vidc/hfi_packetization.c b/drivers/media/platform/msm/vidc/hfi_packetization.c index f8d7084fac7..794fac2daa9 100644 --- a/drivers/media/platform/msm/vidc/hfi_packetization.c +++ b/drivers/media/platform/msm/vidc/hfi_packetization.c @@ -753,7 +753,7 @@ int create_pkt_cmd_session_set_property( struct hfi_enable *hfi; pkt->rg_property_data[0] = HFI_PROPERTY_CONFIG_REALTIME; hfi = (struct hfi_enable *) &pkt->rg_property_data[1]; - hfi->enable = ((struct hfi_enable *) pdata)->enable; + hfi->enable = !(((struct hfi_enable *) pdata)->enable); pkt->size += sizeof(u32) * 2; break; } @@ -1373,43 +1373,6 @@ int create_pkt_cmd_session_set_property( pkt->size += sizeof(u32) + sizeof(struct hfi_enable); break; } - case HAL_PARAM_VENC_LTRMODE: - { - struct hfi_ltrmode *hfi; - struct hal_ltrmode *hal = pdata; - pkt->rg_property_data[0] = - HFI_PROPERTY_PARAM_VENC_H264_LTRMODE; - hfi = (struct hfi_ltrmode *) &pkt->rg_property_data[1]; - hfi->ltrmode = get_hfi_ltr_mode(hal->ltrmode); - hfi->ltrcount = hal->ltrcount; - hfi->trustmode = hal->trustmode; - pkt->size += sizeof(u32) + sizeof(struct hfi_ltrmode); - break; - } - case HAL_CONFIG_VENC_USELTRFRAME: - { - struct hfi_ltruse *hfi; - struct hal_ltruse *hal = pdata; - pkt->rg_property_data[0] = - HFI_PROPERTY_CONFIG_VENC_H264_USELTRFRAME; - hfi = (struct hfi_ltruse *) &pkt->rg_property_data[1]; - hfi->frames = hal->frames; - hfi->refltr = hal->refltr; - hfi->useconstrnt = hal->useconstrnt; - pkt->size += sizeof(u32) + sizeof(struct hfi_ltruse); - break; - } - case HAL_CONFIG_VENC_MARKLTRFRAME: - { - struct hfi_ltrmark *hfi; - struct hal_ltrmark *hal = pdata; - pkt->rg_property_data[0] = - HFI_PROPERTY_CONFIG_VENC_H264_MARKLTRFRAME; - hfi = (struct hfi_ltrmark *) &pkt->rg_property_data[1]; - hfi->markframe = hal->markframe; - pkt->size += sizeof(u32) * 2; - break; - } case HAL_PARAM_VENC_H264_VUI_BITSTREAM_RESTRC: { struct hfi_enable *hfi; @@ -1434,6 +1397,43 @@ int create_pkt_cmd_session_set_property( pkt->size += sizeof(u32) + sizeof(struct hfi_enable); break; } + case HAL_PARAM_VENC_LTRMODE: + { + struct hfi_ltr_mode *hfi; + struct hal_ltr_mode *hal = pdata; + pkt->rg_property_data[0] = + HFI_PROPERTY_PARAM_VENC_LTRMODE; + hfi = (struct hfi_ltr_mode *) &pkt->rg_property_data[1]; + hfi->ltr_mode = get_hfi_ltr_mode(hal->mode); + hfi->ltr_count = hal->count; + hfi->trust_mode = hal->trust_mode; + pkt->size += sizeof(u32) + sizeof(struct hfi_ltr_mode); + break; + } + case HAL_CONFIG_VENC_USELTRFRAME: + { + struct hfi_ltr_use *hfi; + struct hal_ltr_use *hal = pdata; + pkt->rg_property_data[0] = + HFI_PROPERTY_CONFIG_VENC_USELTRFRAME; + hfi = (struct hfi_ltr_use *) &pkt->rg_property_data[1]; + hfi->frames = hal->frames; + hfi->ref_ltr = hal->ref_ltr; + hfi->use_constrnt = hal->use_constraint; + pkt->size += sizeof(u32) + sizeof(struct hfi_ltr_use); + break; + } + case HAL_CONFIG_VENC_MARKLTRFRAME: + { + struct hfi_ltr_mark *hfi; + struct hal_ltr_mark *hal = pdata; + pkt->rg_property_data[0] = + HFI_PROPERTY_CONFIG_VENC_MARKLTRFRAME; + hfi = (struct hfi_ltr_mark *) &pkt->rg_property_data[1]; + hfi->mark_frame = hal->mark_frame; + pkt->size += sizeof(u32) + sizeof(struct hfi_ltr_mark); + break; + } case HAL_PARAM_VENC_HIER_P_MAX_ENH_LAYERS: { pkt->rg_property_data[0] = @@ -1476,6 +1476,33 @@ int create_pkt_cmd_session_set_property( sizeof(struct hfi_initial_quantization); break; } + case HAL_PARAM_VENC_H264_NAL_SVC_EXT: + { + struct hfi_enable *hfi; + struct hal_enable *svc_nal = pdata; + pkt->rg_property_data[0] = + HFI_PROPERTY_PARAM_VENC_H264_NAL_SVC_EXT; + hfi = (struct hfi_enable *)&pkt->rg_property_data[1]; + hfi->enable = svc_nal->enable; + pkt->size += sizeof(u32) + sizeof(struct hfi_enable); + break; + } + case HAL_CONFIG_VENC_PERF_MODE: + { + pkt->rg_property_data[0] = + HFI_PROPERTY_CONFIG_VENC_PERF_MODE; + pkt->rg_property_data[1] = *(u32 *)pdata; + pkt->size += sizeof(u32) * 2; + break; + } + case HAL_PARAM_VENC_HIER_B_MAX_ENH_LAYERS: + { + pkt->rg_property_data[0] = + HFI_PROPERTY_PARAM_VENC_HIER_B_MAX_NUM_ENH_LAYER; + pkt->rg_property_data[1] = *(u32 *)pdata; + pkt->size += sizeof(u32) * 2; + break; + } /* FOLLOWING PROPERTIES ARE NOT IMPLEMENTED IN CORE YET */ case HAL_CONFIG_BUFFER_REQUIREMENTS: case HAL_CONFIG_PRIORITY: @@ -1534,7 +1561,7 @@ int create_pkt_ssr_cmd(enum hal_ssr_trigger_type type, struct hfi_cmd_sys_test_ssr_packet *pkt) { if (!pkt) { - dprintk(VIDC_ERR, "Invalid params, device: %p\n", pkt); + dprintk(VIDC_ERR, "Invalid params, device: %pK\n", pkt); return -EINVAL; } pkt->size = sizeof(struct hfi_cmd_sys_test_ssr_packet); @@ -1547,7 +1574,7 @@ int create_pkt_cmd_sys_image_version( struct hfi_cmd_sys_get_property_packet *pkt) { if (!pkt) { - dprintk(VIDC_ERR, "%s invalid param :%p\n", __func__, pkt); + dprintk(VIDC_ERR, "%s invalid param :%pK\n", __func__, pkt); return -EINVAL; } pkt->size = sizeof(struct hfi_cmd_sys_get_property_packet); diff --git a/drivers/media/platform/msm/vidc/hfi_response_handler.c b/drivers/media/platform/msm/vidc/hfi_response_handler.c index f41e68efeef..840c173dfb4 100644 --- a/drivers/media/platform/msm/vidc/hfi_response_handler.c +++ b/drivers/media/platform/msm/vidc/hfi_response_handler.c @@ -78,6 +78,26 @@ static enum vidc_status hfi_map_err_status(int hfi_err) return vidc_err; } +static int validate_session_pkt(struct list_head *sessions, + struct hal_session *sess, struct mutex *session_lock) +{ + struct hal_session *session; + int invalid = 1; + if (session_lock) { + mutex_lock(session_lock); + list_for_each_entry(session, sessions, list) { + if (session == sess) { + invalid = 0; + break; + } + } + mutex_unlock(session_lock); + } + if (invalid) + dprintk(VIDC_WARN, "Invalid session from FW: %pK\n", sess); + return invalid; +} + static void hfi_process_sess_evt_seq_changed( msm_vidc_callback callback, u32 device_id, struct hfi_msg_event_notify_packet *pkt) @@ -197,8 +217,10 @@ static void hfi_process_session_error( } static void hfi_process_event_notify( msm_vidc_callback callback, u32 device_id, - struct hfi_msg_event_notify_packet *pkt) + struct hfi_msg_event_notify_packet *pkt, + struct list_head *sessions, struct mutex *session_lock) { + struct hal_session *sess = NULL; dprintk(VIDC_DBG, "RECVD:EVENT_NOTIFY"); if (!callback || !pkt || @@ -206,6 +228,7 @@ static void hfi_process_event_notify( dprintk(VIDC_ERR, "Invalid Params"); return; } + sess = (struct hal_session *)pkt->session_id; switch (pkt->event_id) { case HFI_EVENT_SYS_ERROR: @@ -215,11 +238,14 @@ static void hfi_process_event_notify( break; case HFI_EVENT_SESSION_ERROR: dprintk(VIDC_ERR, "HFI_EVENT_SESSION_ERROR"); - hfi_process_session_error(callback, device_id, pkt); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_error(callback, device_id, pkt); break; case HFI_EVENT_SESSION_SEQUENCE_CHANGED: dprintk(VIDC_INFO, "HFI_EVENT_SESSION_SEQUENCE_CHANGED"); - hfi_process_sess_evt_seq_changed(callback, device_id, pkt); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_sess_evt_seq_changed(callback, + device_id, pkt); break; case HFI_EVENT_SESSION_PROPERTY_CHANGED: dprintk(VIDC_INFO, "HFI_EVENT_SESSION_PROPERTY_CHANGED"); @@ -594,7 +620,7 @@ static void hfi_process_sess_get_prop_buf_req( dprintk(VIDC_DBG, "Entered "); if (!prop) { dprintk(VIDC_ERR, - "hal_process_sess_get_prop_buf_req:bad_prop: %p", + "hal_process_sess_get_prop_buf_req:bad_prop: %pK", prop); return; } @@ -770,7 +796,7 @@ static void hfi_process_session_init_done( sess_close = (struct hal_session *)pkt->session_id; if (sess_close) { dprintk(VIDC_INFO, - "Sess init failed: Deleting session: 0x%x 0x%p", + "Sess init failed: Deleting session: 0x%x 0x%pK", sess_close->session_id, sess_close); list_del(&sess_close->list); kfree(sess_close); @@ -918,7 +944,7 @@ static void hfi_process_session_ftb_done( data_done.output_done.packet_buffer1 = pkt->packet_buffer; data_done.output_done.extra_data_buffer = pkt->extra_data_buffer; - dprintk(VIDC_DBG, "FBD: Received buf: %p, of len: %d\n", + dprintk(VIDC_DBG, "FBD: Received buf: %pK, of len: %d\n", pkt->packet_buffer, pkt->filled_len); } else if (is_decoder == 1) { struct hfi_msg_session_fbd_uncompressed_plane0_packet *pkt = @@ -1138,7 +1164,7 @@ static void hfi_process_session_get_seq_hdr_done( data_done.status = hfi_map_err_status((u32)pkt->error_type); data_done.output_done.packet_buffer1 = pkt->sequence_header; data_done.output_done.filled_len1 = pkt->header_len; - dprintk(VIDC_INFO, "seq_hdr: %p, Length: %d", + dprintk(VIDC_INFO, "seq_hdr: %pK, Length: %d", pkt->sequence_header, pkt->header_len); callback(SESSION_GET_SEQ_HDR_DONE, &data_done); } @@ -1183,9 +1209,11 @@ void hfi_process_sys_property_info( u32 hfi_process_msg_packet( msm_vidc_callback callback, u32 device_id, - struct vidc_hal_msg_pkt_hdr *msg_hdr) + struct vidc_hal_msg_pkt_hdr *msg_hdr, + struct list_head *sessions, struct mutex *session_lock) { u32 rc = 0; + struct hal_session *sess = NULL; if (!callback || !msg_hdr || msg_hdr->size < VIDC_IFACEQ_MIN_PKT_SIZE) { dprintk(VIDC_ERR, "hal_process_msg_packet:bad" @@ -1196,10 +1224,14 @@ u32 hfi_process_msg_packet( dprintk(VIDC_INFO, "Received: 0x%x in ", msg_hdr->packet); rc = (u32) msg_hdr->packet; + sess = (struct hal_session *)((struct + vidc_hal_session_cmd_pkt*) msg_hdr)->session_id; + switch (msg_hdr->packet) { case HFI_MSG_EVENT_NOTIFY: hfi_process_event_notify(callback, device_id, - (struct hfi_msg_event_notify_packet *) msg_hdr); + (struct hfi_msg_event_notify_packet *) msg_hdr, + sessions, session_lock); break; case HFI_MSG_SYS_INIT_DONE: hfi_process_sys_init_done(callback, device_id, @@ -1209,9 +1241,10 @@ u32 hfi_process_msg_packet( case HFI_MSG_SYS_IDLE: break; case HFI_MSG_SYS_SESSION_INIT_DONE: - hfi_process_session_init_done(callback, device_id, - (struct hfi_msg_sys_session_init_done_packet *) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_init_done(callback, device_id, + (struct hfi_msg_sys_session_init_done_packet *) + msg_hdr); break; case HFI_MSG_SYS_PROPERTY_INFO: hfi_process_sys_property_info( @@ -1219,68 +1252,81 @@ u32 hfi_process_msg_packet( msg_hdr); break; case HFI_MSG_SYS_SESSION_END_DONE: - hfi_process_session_end_done(callback, device_id, - (struct hfi_msg_sys_session_end_done_packet *) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_end_done(callback, device_id, + (struct hfi_msg_sys_session_end_done_packet *) + msg_hdr); break; case HFI_MSG_SESSION_LOAD_RESOURCES_DONE: - hfi_process_session_load_res_done(callback, device_id, + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_load_res_done(callback, device_id, (struct hfi_msg_session_load_resources_done_packet *) - msg_hdr); + msg_hdr); break; case HFI_MSG_SESSION_START_DONE: - hfi_process_session_start_done(callback, device_id, - (struct hfi_msg_session_start_done_packet *) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_start_done(callback, device_id, + (struct hfi_msg_session_start_done_packet *) + msg_hdr); break; case HFI_MSG_SESSION_STOP_DONE: - hfi_process_session_stop_done(callback, device_id, - (struct hfi_msg_session_stop_done_packet *) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_stop_done(callback, device_id, + (struct hfi_msg_session_stop_done_packet *) + msg_hdr); break; case HFI_MSG_SESSION_EMPTY_BUFFER_DONE: - hfi_process_session_etb_done(callback, device_id, + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_etb_done(callback, device_id, (struct hfi_msg_session_empty_buffer_done_packet *) - msg_hdr); + msg_hdr); break; case HFI_MSG_SESSION_FILL_BUFFER_DONE: - hfi_process_session_ftb_done(callback, device_id, msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_ftb_done(callback, device_id, + msg_hdr); break; case HFI_MSG_SESSION_FLUSH_DONE: - hfi_process_session_flush_done(callback, device_id, - (struct hfi_msg_session_flush_done_packet *) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_flush_done(callback, device_id, + (struct hfi_msg_session_flush_done_packet *) + msg_hdr); break; case HFI_MSG_SESSION_PROPERTY_INFO: - hfi_process_session_prop_info(callback, device_id, - (struct hfi_msg_session_property_info_packet *) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_prop_info(callback, device_id, + (struct hfi_msg_session_property_info_packet *) + msg_hdr); break; case HFI_MSG_SESSION_RELEASE_RESOURCES_DONE: - hfi_process_session_rel_res_done(callback, device_id, + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_rel_res_done(callback, device_id, (struct hfi_msg_session_release_resources_done_packet *) - msg_hdr); + msg_hdr); break; case HFI_MSG_SYS_RELEASE_RESOURCE: hfi_process_sys_rel_resource_done(callback, device_id, (struct hfi_msg_sys_release_resource_done_packet *) - msg_hdr); + msg_hdr); break; case HFI_MSG_SESSION_GET_SEQUENCE_HEADER_DONE: - hfi_process_session_get_seq_hdr_done( + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_get_seq_hdr_done( callback, device_id, (struct hfi_msg_session_get_sequence_header_done_packet*) - msg_hdr); + msg_hdr); break; case HFI_MSG_SESSION_RELEASE_BUFFERS_DONE: - hfi_process_session_rel_buf_done( - callback, device_id, (struct - hfi_msg_session_release_buffers_done_packet*) - msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_rel_buf_done(callback, device_id, + (struct hfi_msg_session_release_buffers_done_packet *) + msg_hdr); break; case HFI_MSG_SYS_SESSION_ABORT_DONE: - hfi_process_session_abort_done(callback, device_id, (struct - hfi_msg_sys_session_abort_done_packet*) msg_hdr); + if (!validate_session_pkt(sessions, sess, session_lock)) + hfi_process_session_abort_done(callback, device_id, + (struct hfi_msg_sys_session_abort_done_packet *) + msg_hdr); break; default: dprintk(VIDC_DBG, "UNKNOWN_MSG_TYPE : %d", msg_hdr->packet); diff --git a/drivers/media/platform/msm/vidc/msm_smem.c b/drivers/media/platform/msm/vidc/msm_smem.c index 79a492e4fae..bbf50693f75 100644 --- a/drivers/media/platform/msm/vidc/msm_smem.c +++ b/drivers/media/platform/msm/vidc/msm_smem.c @@ -54,7 +54,7 @@ static int get_device_address(struct smem_client *smem_client, struct ion_client *clnt = NULL; if (!iova || !buffer_size || !hndl || !smem_client) { - dprintk(VIDC_ERR, "Invalid params: %p, %p, %p, %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK, %pK, %pK, %pK\n", smem_client, hndl, iova, buffer_size); return -EINVAL; } @@ -109,7 +109,7 @@ static void put_device_address(struct smem_client *smem_client, struct ion_client *clnt = NULL; if (!hndl || !smem_client) { - dprintk(VIDC_WARN, "Invalid params: %p, %p\n", + dprintk(VIDC_WARN, "Invalid params: %pK, %pK\n", smem_client, hndl); return; } @@ -143,7 +143,7 @@ static int ion_user_to_kernel(struct smem_client *client, int fd, u32 offset, hndl = ion_import_dma_buf(client->clnt, fd); if (IS_ERR_OR_NULL(hndl)) { - dprintk(VIDC_ERR, "Failed to get handle: %p, %d, %d, %p\n", + dprintk(VIDC_ERR, "Failed to get handle: %pK, %d, %d, %pK\n", client, fd, offset, hndl); rc = -ENOMEM; goto fail_import_fd; @@ -213,7 +213,7 @@ static int alloc_ion_mem(struct smem_client *client, size_t size, u32 align, hndl = ion_alloc(client->clnt, size, align, heap_mask, flags); if (IS_ERR_OR_NULL(hndl)) { dprintk(VIDC_ERR, - "Failed to allocate shared memory = %p, %d, %d, 0x%x\n", + "Failed to allocate shared memory = %pK, %d, %d, 0x%x\n", client, size, align, flags); rc = -ENOMEM; goto fail_shared_mem_alloc; @@ -242,7 +242,7 @@ static int alloc_ion_mem(struct smem_client *client, size_t size, u32 align, } mem->device_addr = iova; dprintk(VIDC_DBG, - "device_address = 0x%lx, kvaddr = 0x%p, size = %d\n", + "device_address = 0x%lx, kvaddr = 0x%pK, size = %d\n", mem->device_addr, mem->kvaddr, size); mem->size = size; return rc; @@ -327,7 +327,7 @@ static int ion_cache_operations(struct smem_client *client, int rc = 0; int msm_cache_ops = 0; if (!mem || !client) { - dprintk(VIDC_ERR, "Invalid params: %p, %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK, %pK\n", mem, client); return -EINVAL; } @@ -374,7 +374,7 @@ int msm_smem_cache_operations(void *clt, struct msm_smem *mem, struct smem_client *client = clt; int rc = 0; if (!client) { - dprintk(VIDC_ERR, "Invalid params: %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK\n", client); return -EINVAL; } diff --git a/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c b/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c index a3458f4ca58..5aedc397141 100644 --- a/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c @@ -283,7 +283,7 @@ static int read_platform_resources(struct msm_vidc_core *core, struct platform_device *pdev) { if (!core || !pdev) { - dprintk(VIDC_ERR, "%s: Invalid params %p %p\n", + dprintk(VIDC_ERR, "%s: Invalid params %pK %pK\n", __func__, core, pdev); return -EINVAL; } @@ -473,7 +473,7 @@ static int __devexit msm_vidc_remove(struct platform_device *pdev) struct msm_vidc_core *core; if (!pdev) { - dprintk(VIDC_ERR, "%s invalid input %p", __func__, pdev); + dprintk(VIDC_ERR, "%s invalid input %pK", __func__, pdev); return -EINVAL; } core = pdev->dev.platform_data; diff --git a/drivers/media/platform/msm/vidc/msm_vdec.c b/drivers/media/platform/msm/vidc/msm_vdec.c index 66cb21fedad..513dde186c5 100644 --- a/drivers/media/platform/msm/vidc/msm_vdec.c +++ b/drivers/media/platform/msm/vidc/msm_vdec.c @@ -24,6 +24,7 @@ #define MIN_NUM_OUTPUT_BUFFERS 4 #define MAX_NUM_OUTPUT_BUFFERS VIDEO_MAX_FRAME #define DEFAULT_VIDEO_CONCEAL_COLOR_BLACK 0x8080 +#define MB_SIZE_IN_PIXEL (16 * 16) #define TZ_INFO_GET_FEATURE_VERSION_ID 0x3 #define TZ_DYNAMIC_BUFFER_FEATURE_ID 12 @@ -302,6 +303,26 @@ static struct msm_vidc_ctrl msm_vdec_ctrls[] = { .default_value = DEFAULT_VIDEO_CONCEAL_COLOR_BLACK, .step = 1, }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY, + .name = "Session Priority", + .type = V4L2_CTRL_TYPE_INTEGER, + .minimum = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_ENABLE, + .maximum = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE, + .default_value = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE, + .step = 1, + .qmenu = NULL, + }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE, + .name = "Set Decoder Operating rate", + .type = V4L2_CTRL_TYPE_INTEGER, + .minimum = 0, + .maximum = 300 << 16, /* 300 fps in Q16 format*/ + .default_value = 0, + .step = 1, + .qmenu = NULL, + }, }; #define NUM_CTRLS ARRAY_SIZE(msm_vdec_ctrls) @@ -313,9 +334,9 @@ static u32 get_frame_size_nv12(int plane, } static u32 get_frame_size_compressed(int plane, - u32 height, u32 width) + u32 max_mbs_per_frame, u32 size_per_mb) { - return (width * height * 3/2)/2; + return (max_mbs_per_frame * size_per_mb * 3/2)/2; } struct msm_vidc_format vdec_formats[] = { @@ -534,7 +555,7 @@ int msm_vdec_release_buf(struct msm_vidc_inst *inst, if (inst->state == MSM_VIDC_CORE_INVALID || core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p in bad state, ignoring release output buf\n", + "Core %pK in bad state, ignoring release output buf\n", core); goto exit; } @@ -542,7 +563,7 @@ int msm_vdec_release_buf(struct msm_vidc_inst *inst, rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to relase res done\n", + "Failed to move inst: %pK to relase res done\n", inst); goto exit; } @@ -634,7 +655,7 @@ int msm_vdec_reqbufs(struct msm_vidc_inst *inst, struct v4l2_requestbuffers *b) int rc = 0; if (!inst || !b) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, buffer = %p\n", inst, b); + "Invalid input, inst = %pK, buffer = %pK\n", inst, b); return -EINVAL; } q = msm_comm_get_vb2q(inst, b->type); @@ -664,7 +685,7 @@ int msm_vdec_g_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) struct hal_buffer_requirements *buff_req_buffer; if (!inst || !f || !inst->core || !inst->core->device) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + "Invalid input, inst = %pK, format = %pK\n", inst, f); return -EINVAL; } hdev = inst->core->device; @@ -705,8 +726,9 @@ int msm_vdec_g_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) for (i = 0; i < fmt->num_planes; ++i) { f->fmt.pix_mp.plane_fmt[i].sizeimage = fmt->get_frame_size(i, - inst->capability.height.max, - inst->capability.width.max); + inst->capability. + mbs_per_frame.max, + MB_SIZE_IN_PIXEL); inst->bufq[OUTPUT_PORT]. vb2_bufq.plane_sizes[i] = f->fmt.pix_mp.plane_fmt[i].sizeimage; @@ -806,7 +828,7 @@ int msm_vdec_s_parm(struct msm_vidc_inst *inst, struct v4l2_streamparm *a) fps = fps - 1; if (inst->prop.fps != fps) { - dprintk(VIDC_PROF, "reported fps changed for %p: %d->%d\n", + dprintk(VIDC_PROF, "reported fps changed for %pK: %d->%d\n", inst, inst->prop.fps, fps); inst->prop.fps = fps; @@ -824,11 +846,15 @@ int msm_vdec_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) int ret = 0; int i; struct hal_buffer_requirements *buff_req_buffer; - if (!inst || !f) { - dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + struct hfi_device *hdev; + + if (!inst || !f || !inst->core || !inst->core->device) { + dprintk(VIDC_ERR, "%s invalid parameters\n", __func__); return -EINVAL; } + + hdev = inst->core->device; + if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { fmt = msm_comm_get_pixel_fmt_fourcc(vdec_formats, ARRAY_SIZE(vdec_formats), f->fmt.pix_mp.pixelformat, @@ -906,8 +932,8 @@ int msm_vdec_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) frame_sz.height = inst->prop.height; msm_comm_try_set_prop(inst, HAL_PARAM_FRAME_SIZE, &frame_sz); f->fmt.pix_mp.plane_fmt[0].sizeimage = - fmt->get_frame_size(0, inst->capability.height.max, - inst->capability.width.max); + fmt->get_frame_size(0, inst->capability. + mbs_per_frame.max, MB_SIZE_IN_PIXEL); f->fmt.pix_mp.num_planes = fmt->num_planes; for (i = 0; i < fmt->num_planes; ++i) { inst->bufq[OUTPUT_PORT].vb2_bufq.plane_sizes[i] = @@ -922,7 +948,7 @@ int msm_vdec_querycap(struct msm_vidc_inst *inst, struct v4l2_capability *cap) { if (!inst || !cap) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, cap = %p\n", inst, cap); + "Invalid input, inst = %pK, cap = %pK\n", inst, cap); return -EINVAL; } strlcpy(cap->driver, MSM_VIDC_DRV_NAME, sizeof(cap->driver)); @@ -942,7 +968,7 @@ int msm_vdec_enum_fmt(struct msm_vidc_inst *inst, struct v4l2_fmtdesc *f) int rc = 0; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, f = %p\n", inst, f); + "Invalid input, inst = %pK, f = %pK\n", inst, f); return -EINVAL; } if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { @@ -981,7 +1007,7 @@ static int msm_vdec_queue_setup(struct vb2_queue *q, enum hal_property property_id; if (!q || !num_buffers || !num_planes || !sizes || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p, %p, %p\n", + dprintk(VIDC_ERR, "Invalid input, q = %pK, %pK, %pK\n", q, num_buffers, num_planes); return -EINVAL; } @@ -1002,8 +1028,8 @@ static int msm_vdec_queue_setup(struct vb2_queue *q, *num_buffers = MIN_NUM_OUTPUT_BUFFERS; for (i = 0; i < *num_planes; i++) { sizes[i] = inst->fmts[OUTPUT_PORT]->get_frame_size( - i, inst->capability.height.max, - inst->capability.width.max); + i, inst->capability.mbs_per_frame.max, + MB_SIZE_IN_PIXEL); } property_id = HAL_PARAM_BUFFER_COUNT_ACTUAL; new_buf_count.buffer_type = HAL_BUFFER_INPUT; @@ -1097,7 +1123,7 @@ static inline int start_streaming(struct msm_vidc_inst *inst) rc = msm_comm_try_state(inst, MSM_VIDC_START_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto fail_start; } @@ -1127,7 +1153,7 @@ static inline int stop_streaming(struct msm_vidc_inst *inst) rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); if (rc) dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); return rc; } @@ -1137,7 +1163,7 @@ static int msm_vdec_start_streaming(struct vb2_queue *q, unsigned int count) int rc = 0; struct hfi_device *hdev; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1170,7 +1196,7 @@ static int msm_vdec_stop_streaming(struct vb2_queue *q) struct msm_vidc_inst *inst; int rc = 0; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1195,7 +1221,7 @@ static int msm_vdec_stop_streaming(struct vb2_queue *q) if (rc) dprintk(VIDC_ERR, - "Failed to move inst: %p, cap = %d to state: %d\n", + "Failed to move inst: %pK, cap = %d to state: %d\n", inst, q->type, MSM_VIDC_RELEASE_RESOURCES_DONE); return rc; } @@ -1240,7 +1266,7 @@ int msm_vdec_cmd(struct msm_vidc_inst *inst, struct v4l2_decoder_cmd *dec) if (inst->state == MSM_VIDC_CORE_INVALID || core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p in bad state, Sending CLOSE event\n", + "Core %pK in bad state, Sending CLOSE event\n", core); msm_vidc_queue_v4l2_event(inst, V4L2_EVENT_MSM_VIDC_CLOSE_DONE); @@ -1282,7 +1308,7 @@ int msm_vdec_inst_init(struct msm_vidc_inst *inst) { int rc = 0; if (!inst) { - dprintk(VIDC_ERR, "Invalid input = %p\n", inst); + dprintk(VIDC_ERR, "Invalid input = %pK\n", inst); return -EINVAL; } inst->fmts[OUTPUT_PORT] = &vdec_formats[1]; @@ -1465,6 +1491,14 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) property_val = ctrl->val; pdata = &property_val; break; + case V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY: + property_id = HAL_CONFIG_REALTIME; + hal_property.enable = ctrl->val; + pdata = &hal_property; + break; + case V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE: + property_id = 0; + break; default: break; } @@ -1488,7 +1522,7 @@ static int msm_vdec_op_s_ctrl(struct v4l2_ctrl *ctrl) rc = msm_comm_try_state(inst, MSM_VIDC_OPEN_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto failed_open_done; } diff --git a/drivers/media/platform/msm/vidc/msm_venc.c b/drivers/media/platform/msm/vidc/msm_venc.c index 30853e263bc..5d81ac1d1e4 100644 --- a/drivers/media/platform/msm/vidc/msm_venc.c +++ b/drivers/media/platform/msm/vidc/msm_venc.c @@ -92,6 +92,41 @@ static const char *const h263_profile[] = { "High Latency", }; +static const char *const hevc_tier_level[] = { + "Main Tier Level 1", + "Main Tier Level 2", + "Main Tier Level 2.1", + "Main Tier Level 3", + "Main Tier Level 3.1", + "Main Tier Level 4", + "Main Tier Level 4.1", + "Main Tier Level 5", + "Main Tier Level 5.1", + "Main Tier Level 5.2", + "Main Tier Level 6", + "Main Tier Level 6.1", + "Main Tier Level 6.2", + "High Tier Level 1", + "High Tier Level 2", + "High Tier Level 2.1", + "High Tier Level 3", + "High Tier Level 3.1", + "High Tier Level 4", + "High Tier Level 4.1", + "High Tier Level 5", + "High Tier Level 5.1", + "High Tier Level 5.2", + "High Tier Level 6", + "High Tier Level 6.1", + "High Tier Level 6.2", +}; + +static const char *const hevc_profile[] = { + "Main", + "Main10", + "Main Still Pic", +}; + static const char *const vp8_profile_level[] = { "Unused", "0.0", @@ -367,6 +402,52 @@ static struct msm_vidc_ctrl msm_venc_ctrls[] = { ), .qmenu = vp8_profile_level, }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE, + .name = "HEVC Profile", + .type = V4L2_CTRL_TYPE_MENU, + .minimum = V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN, + .maximum = V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN_STILL_PIC, + .default_value = V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN, + .step = 0, + .menu_skip_mask = ~( + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN10) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN_STILL_PIC) + ), + .qmenu = hevc_profile, + }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL, + .name = "HEVC Tier and Level", + .type = V4L2_CTRL_TYPE_MENU, + .minimum = V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_1, + .maximum = V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_6_2, + .default_value = + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_1, + .step = 0, + .menu_skip_mask = ~( + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_2) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_2_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_3) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_3_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_4) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_4_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_2) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_2_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_3) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_3_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_4) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_4_1) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5) | + (1 << V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5_1) + ), + .qmenu = hevc_tier_level, + }, { .id = V4L2_CID_MPEG_VIDC_VIDEO_ROTATION, .name = "Rotation", @@ -823,7 +904,55 @@ static struct msm_vidc_ctrl msm_venc_ctrls[] = { .step = 1, .qmenu = NULL, }, - + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC, + .name = "Enable H264 SVC NAL", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .minimum = V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC_DISABLED, + .maximum = V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC_ENABLED, + .default_value = V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC_DISABLED, + .step = 1, + }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_PERF_MODE, + .name = "Set Encoder performance mode", + .type = V4L2_CTRL_TYPE_INTEGER, + .minimum = V4L2_MPEG_VIDC_VIDEO_PERF_MAX_QUALITY, + .maximum = V4L2_MPEG_VIDC_VIDEO_PERF_POWER_SAVE, + .default_value = V4L2_MPEG_VIDC_VIDEO_PERF_MAX_QUALITY, + .step = 1, + .qmenu = NULL, + }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_HIER_B_NUM_LAYERS, + .name = "Set Hier B num layers", + .type = V4L2_CTRL_TYPE_INTEGER, + .minimum = 0, + .maximum = 3, + .default_value = 0, + .step = 1, + .qmenu = NULL, + }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY, + .name = "Session Priority", + .type = V4L2_CTRL_TYPE_INTEGER, + .minimum = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_ENABLE, + .maximum = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE, + .default_value = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE, + .step = 1, + .qmenu = NULL, + }, + { + .id = V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE, + .name = "Set Encoder Operating rate", + .type = V4L2_CTRL_TYPE_INTEGER, + .minimum = 0, + .maximum = 300 << 16, /* 300 fps in Q16 format*/ + .default_value = 0, + .step = 1, + .qmenu = NULL, + }, }; #define NUM_CTRLS ARRAY_SIZE(msm_venc_ctrls) @@ -998,7 +1127,7 @@ static int msm_venc_queue_setup(struct vb2_queue *q, return rc; } -static int msm_venc_toggle_hier_p(struct msm_vidc_inst *inst, bool enable) +static int msm_venc_toggle_hier_p(struct msm_vidc_inst *inst, int layers) { int num_enh_layers = 0; u32 property_id = 0; @@ -1010,17 +1139,16 @@ static int msm_venc_toggle_hier_p(struct msm_vidc_inst *inst, bool enable) return -EINVAL; } - if (inst->fmts[CAPTURE_PORT]->fourcc != V4L2_PIX_FMT_VP8) + if ((inst->fmts[CAPTURE_PORT]->fourcc != V4L2_PIX_FMT_VP8) && + (inst->fmts[CAPTURE_PORT]->fourcc != V4L2_PIX_FMT_H264)) return 0; - num_enh_layers = enable ? inst->capability.hier_p.max - 1 : 0; - + num_enh_layers = layers ? : 0; dprintk(VIDC_DBG, "%s Hier-P in firmware\n", num_enh_layers ? "Enable" : "Disable"); hdev = inst->core->device; property_id = HAL_PARAM_VENC_HIER_P_MAX_ENH_LAYERS; - rc = call_hfi_op(hdev, session_set_property, (void *)inst->session, property_id, (void *)&num_enh_layers); @@ -1031,6 +1159,42 @@ static int msm_venc_toggle_hier_p(struct msm_vidc_inst *inst, bool enable) return rc; } +static int set_bitrate_for_each_layer(struct msm_vidc_inst *inst, + u32 num_enh_layers, u32 total_bitrate) +{ + u32 property_id = 0; + int i = 0; + struct hfi_device *hdev = NULL; + struct hal_bitrate bitrate; + int rc = 0; + int bitrate_table[3][4] = { + {50, 50, 0, 0}, + {34, 33, 33, 0}, + {25, 25, 25, 25} }; + + if (!inst || !inst->core || !inst->core->device) { + dprintk(VIDC_ERR, "%s - invalid input\n", __func__); + return -EINVAL; + } + + if (!num_enh_layers || num_enh_layers > ARRAY_SIZE(bitrate_table)) { + dprintk(VIDC_ERR, "%s - invalid number of enh layers: %d\n", + __func__, num_enh_layers); + return -EINVAL; + } + hdev = inst->core->device; + + for (i = 0; !rc && (i <= num_enh_layers); i++) { + property_id = HAL_CONFIG_VENC_TARGET_BITRATE; + bitrate.bit_rate = (u32)((total_bitrate * + bitrate_table[num_enh_layers - 1][i]) / 100); + bitrate.layer_id = i; + rc = call_hfi_op(hdev, session_set_property, + (void *)inst->session, property_id, &bitrate); + } + return rc; +} + static inline int start_streaming(struct msm_vidc_inst *inst) { int rc = 0; @@ -1064,7 +1228,7 @@ static inline int start_streaming(struct msm_vidc_inst *inst) rc = msm_comm_try_state(inst, MSM_VIDC_START_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto fail_start; } mutex_lock(&inst->sync_lock); @@ -1092,7 +1256,7 @@ static int msm_venc_start_streaming(struct vb2_queue *q, unsigned int count) struct msm_vidc_inst *inst; int rc = 0; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1119,7 +1283,7 @@ static int msm_venc_stop_streaming(struct vb2_queue *q) struct msm_vidc_inst *inst; int rc = 0; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1140,7 +1304,7 @@ static int msm_venc_stop_streaming(struct vb2_queue *q) if (rc) dprintk(VIDC_ERR, - "Failed to move inst: %p, cap = %d to state: %d\n", + "Failed to move inst: %pK, cap = %d to state: %d\n", inst, q->type, MSM_VIDC_CLOSE_DONE); return rc; } @@ -1350,6 +1514,72 @@ static inline int venc_v4l2_to_hal(int id, int value) default: goto unknown_value; } + case V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE: + switch (value) { + case V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN: + return HAL_HEVC_PROFILE_MAIN; + case V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN10: + return HAL_HEVC_PROFILE_MAIN10; + case V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN_STILL_PIC: + return HAL_HEVC_PROFILE_MAIN_STILL_PIC; + default: + goto unknown_value; + } + case V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL: + switch (value) { + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_1: + return HAL_HEVC_MAIN_TIER_LEVEL_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_2: + return HAL_HEVC_MAIN_TIER_LEVEL_2; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_2_1: + return HAL_HEVC_MAIN_TIER_LEVEL_2_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_3: + return HAL_HEVC_MAIN_TIER_LEVEL_3; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_3_1: + return HAL_HEVC_MAIN_TIER_LEVEL_3_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_4: + return HAL_HEVC_MAIN_TIER_LEVEL_4; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_4_1: + return HAL_HEVC_MAIN_TIER_LEVEL_4_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5: + return HAL_HEVC_MAIN_TIER_LEVEL_5; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5_1: + return HAL_HEVC_MAIN_TIER_LEVEL_5_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5_2: + return HAL_HEVC_MAIN_TIER_LEVEL_5_2; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_6: + return HAL_HEVC_MAIN_TIER_LEVEL_6; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_6_1: + return HAL_HEVC_MAIN_TIER_LEVEL_6_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_6_2: + return HAL_HEVC_MAIN_TIER_LEVEL_6_2; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_1: + return HAL_HEVC_HIGH_TIER_LEVEL_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_2: + return HAL_HEVC_HIGH_TIER_LEVEL_2; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_2_1: + return HAL_HEVC_HIGH_TIER_LEVEL_2_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_3: + return HAL_HEVC_HIGH_TIER_LEVEL_3; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_3_1: + return HAL_HEVC_HIGH_TIER_LEVEL_3_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_4: + return HAL_HEVC_HIGH_TIER_LEVEL_4; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_4_1: + return HAL_HEVC_HIGH_TIER_LEVEL_4_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5: + return HAL_HEVC_HIGH_TIER_LEVEL_5; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5_1: + return HAL_HEVC_HIGH_TIER_LEVEL_5_1; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5_2: + return HAL_HEVC_HIGH_TIER_LEVEL_5_2; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_6: + return HAL_HEVC_HIGH_TIER_LEVEL_6; + case V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_6_1: + return HAL_HEVC_HIGH_TIER_LEVEL_6_1; + default: + goto unknown_value; + } case V4L2_CID_MPEG_VIDC_VIDEO_ROTATION: switch (value) { case V4L2_CID_MPEG_VIDC_VIDEO_ROTATION_NONE: @@ -1406,9 +1636,10 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) struct v4l2_ctrl *temp_ctrl = NULL; struct hfi_device *hdev; struct hal_extradata_enable extra; - struct hal_ltruse useltr; - struct hal_ltrmark markltr; - u32 hier_p_layers; + struct hal_ltr_use use_ltr; + struct hal_ltr_mark mark_ltr; + u32 hier_p_layers = 0, hier_b_layers = 0; + struct hal_venc_perf_mode venc_mode; if (!inst || !inst->core || !inst->core->device) { dprintk(VIDC_ERR, "%s invalid parameters", __func__); @@ -1430,6 +1661,12 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) } \ __temp; \ }) + /* + * Unlock the control prior to setting to the hardware. Otherwise + * lower level code that attempts to do a get_ctrl() will end up + * deadlocking. + */ + v4l2_ctrl_unlock(ctrl); switch (ctrl->id) { case V4L2_CID_MPEG_VIDC_VIDEO_FRAME_RATE: @@ -1563,12 +1800,29 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) pdata = &property_val; break; case V4L2_CID_MPEG_VIDEO_BITRATE: - property_id = - HAL_CONFIG_VENC_TARGET_BITRATE; - bitrate.bit_rate = ctrl->val; + { + struct v4l2_ctrl *hier_p = TRY_GET_CTRL( + V4L2_CID_MPEG_VIDC_VIDEO_HIER_P_NUM_LAYERS); + bitrate.layer_id = 0; - pdata = &bitrate; + if (hier_p->val && + inst->fmts[CAPTURE_PORT]->fourcc == + V4L2_PIX_FMT_H264) { + rc = set_bitrate_for_each_layer(inst, + hier_p->val, ctrl->val); + if (rc) { + dprintk(VIDC_ERR, + "failed to set bitrate for multiple layers\n"); + rc = -EINVAL; + } + } else { + property_id = HAL_CONFIG_VENC_TARGET_BITRATE; + bitrate.bit_rate = ctrl->val; + bitrate.layer_id = 0; + pdata = &bitrate; + } break; + } case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: { struct v4l2_ctrl *avg_bitrate = TRY_GET_CTRL( @@ -1702,6 +1956,29 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) profile_level.level = HAL_VPX_PROFILE_UNUSED; pdata = &profile_level; break; + case V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE: + temp_ctrl = + TRY_GET_CTRL(V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL); + + property_id = HAL_PARAM_PROFILE_LEVEL_CURRENT; + profile_level.profile = venc_v4l2_to_hal(ctrl->id, + ctrl->val); + profile_level.level = venc_v4l2_to_hal( + V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL, + temp_ctrl->val); + pdata = &profile_level; + break; + case V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL: + temp_ctrl = TRY_GET_CTRL(V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE); + + property_id = HAL_PARAM_PROFILE_LEVEL_CURRENT; + profile_level.level = venc_v4l2_to_hal(ctrl->id, + ctrl->val); + profile_level.profile = venc_v4l2_to_hal( + V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE, + temp_ctrl->val); + pdata = &profile_level; + break; case V4L2_CID_MPEG_VIDC_VIDEO_ROTATION: property_id = HAL_CONFIG_VPE_OPERATIONS; @@ -2103,7 +2380,7 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) rc = -ENOTSUPP; break; } - + msm_comm_scale_clocks_and_bus(inst); break; case V4L2_CID_MPEG_VIDC_VIDEO_H264_VUI_BITSTREAM_RESTRICT: property_id = HAL_PARAM_VENC_H264_VUI_BITSTREAM_RESTRC; @@ -2120,27 +2397,27 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) break; case V4L2_CID_MPEG_VIDC_VIDEO_USELTRFRAME: property_id = HAL_CONFIG_VENC_USELTRFRAME; - useltr.refltr = 0x1; - useltr.useconstrnt = false; - useltr.frames = 0; - pdata = &useltr; + use_ltr.ref_ltr = (1 << ctrl->val); + use_ltr.use_constraint = false; + use_ltr.frames = 0; + pdata = &use_ltr; break; case V4L2_CID_MPEG_VIDC_VIDEO_MARKLTRFRAME: property_id = HAL_CONFIG_VENC_MARKLTRFRAME; - markltr.markframe = 0x1; - pdata = &markltr; + mark_ltr.mark_frame = ctrl->val; + pdata = &mark_ltr; break; case V4L2_CID_MPEG_VIDC_VIDEO_HIER_P_NUM_LAYERS: property_id = HAL_CONFIG_VENC_HIER_P_NUM_FRAMES; hier_p_layers = ctrl->val; - rc = msm_venc_toggle_hier_p(inst, hier_p_layers ? true : false); + rc = msm_venc_toggle_hier_p(inst, ctrl->val); if (rc) break; - if (hier_p_layers > (inst->capability.hier_p.max - 1)) { + if (hier_p_layers > inst->capability.hier_p.max) { dprintk(VIDC_ERR, "Error setting hier p num layers = %d max supported by f/w = %d\n", hier_p_layers, - inst->capability.hier_p.max - 1); + inst->capability.hier_p.max); rc = -ENOTSUPP; break; } @@ -2151,11 +2428,40 @@ static int try_set_ctrl(struct msm_vidc_inst *inst, struct v4l2_ctrl *ctrl) enable.enable = 1; pdata = &enable; break; + case V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC: + property_id = HAL_PARAM_VENC_H264_NAL_SVC_EXT; + enable.enable = ctrl->val; + pdata = &enable; + break; + case V4L2_CID_MPEG_VIDC_VIDEO_PERF_MODE: + property_id = HAL_CONFIG_VENC_PERF_MODE; + venc_mode.mode = ctrl->val; + pdata = &venc_mode; + break; + case V4L2_CID_MPEG_VIDC_VIDEO_HIER_B_NUM_LAYERS: + if (inst->fmts[CAPTURE_PORT]->fourcc != V4L2_PIX_FMT_HEVC) { + dprintk(VIDC_ERR, "Hier B supported for HEVC only\n"); + rc = -ENOTSUPP; + break; + } + property_id = HAL_PARAM_VENC_HIER_B_MAX_ENH_LAYERS; + hier_b_layers = ctrl->val; + pdata = &hier_b_layers; + break; + case V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY: + property_id = HAL_CONFIG_REALTIME; + enable.enable = ctrl->val; + pdata = &enable; + break; + case V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE: + property_id = 0; + break; default: dprintk(VIDC_ERR, "Unsupported index: %x\n", ctrl->id); rc = -ENOTSUPP; break; } + v4l2_ctrl_lock(ctrl); #undef TRY_GET_CTRL if (!rc && property_id) { @@ -2175,7 +2481,7 @@ static int try_set_ext_ctrl(struct msm_vidc_inst *inst, int rc = 0, i; struct v4l2_ext_control *control; struct hfi_device *hdev; - struct hal_ltrmode ltrmode; + struct hal_ltr_mode ltr_mode; u32 property_id = 0; void *pdata = NULL; struct hal_initial_quantization quant; @@ -2199,16 +2505,16 @@ static int try_set_ext_ctrl(struct msm_vidc_inst *inst, if (rc) break; } - ltrmode.ltrmode = control[i].value; - ltrmode.trustmode = 1; + ltr_mode.mode = control[i].value; + ltr_mode.trust_mode = 1; property_id = HAL_PARAM_VENC_LTRMODE; - pdata = <rmode; + pdata = <r_mode; break; case V4L2_CID_MPEG_VIDC_VIDEO_LTRCOUNT: - ltrmode.ltrcount = control[i].value; - ltrmode.trustmode = 1; + ltr_mode.count = control[i].value; + ltr_mode.trust_mode = 1; property_id = HAL_PARAM_VENC_LTRMODE; - pdata = <rmode; + pdata = <r_mode; break; case V4L2_CID_MPEG_VIDC_VIDEO_ENABLE_INITIAL_QP: property_id = HAL_PARAM_VENC_ENABLE_INITIAL_QP; @@ -2258,7 +2564,7 @@ static int msm_venc_op_s_ctrl(struct v4l2_ctrl *ctrl) if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto failed_open_done; } @@ -2301,7 +2607,7 @@ int msm_venc_inst_init(struct msm_vidc_inst *inst) { int rc = 0; if (!inst) { - dprintk(VIDC_ERR, "Invalid input = %p\n", inst); + dprintk(VIDC_ERR, "Invalid input = %pK\n", inst); return -EINVAL; } inst->fmts[CAPTURE_PORT] = &venc_formats[1]; @@ -2380,7 +2686,7 @@ int msm_venc_querycap(struct msm_vidc_inst *inst, struct v4l2_capability *cap) { if (!inst || !cap) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, cap = %p\n", inst, cap); + "Invalid input, inst = %pK, cap = %pK\n", inst, cap); return -EINVAL; } strlcpy(cap->driver, MSM_VIDC_DRV_NAME, sizeof(cap->driver)); @@ -2400,7 +2706,7 @@ int msm_venc_enum_fmt(struct msm_vidc_inst *inst, struct v4l2_fmtdesc *f) int rc = 0; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, f = %p\n", inst, f); + "Invalid input, inst = %pK, f = %pK\n", inst, f); return -EINVAL; } if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { @@ -2473,7 +2779,7 @@ int msm_venc_s_parm(struct msm_vidc_inst *inst, struct v4l2_streamparm *a) fps = fps - 1; if (inst->prop.fps != fps) { - dprintk(VIDC_PROF, "reported fps changed for %p: %d->%d\n", + dprintk(VIDC_PROF, "reported fps changed for %pK: %d->%d\n", inst, inst->prop.fps, fps); inst->prop.fps = fps; frame_rate.frame_rate = inst->prop.fps * (0x1<<16); @@ -2497,9 +2803,10 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) int rc = 0; int i; struct hfi_device *hdev; + if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + "Invalid input, inst = %pK, format = %pK\n", inst, f); return -EINVAL; } @@ -2619,7 +2926,7 @@ int msm_venc_g_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) int extra_idx = 0; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + "Invalid input, inst = %pK, format = %pK\n", inst, f); return -EINVAL; } if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) @@ -2661,7 +2968,7 @@ int msm_venc_reqbufs(struct msm_vidc_inst *inst, struct v4l2_requestbuffers *b) int rc = 0; if (!inst || !b) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, buffer = %p\n", inst, b); + "Invalid input, inst = %pK, buffer = %pK\n", inst, b); return -EINVAL; } q = msm_comm_get_vb2q(inst, b->type); @@ -2760,7 +3067,7 @@ int msm_venc_release_buf(struct msm_vidc_inst *inst, rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to release res done state\n", + "Failed to move inst: %pK to release res done state\n", inst); goto exit; } diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index 09e01e194f6..f5a5c51122a 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -457,7 +457,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) goto exit; } - dprintk(VIDC_DBG, "[MAP] Create binfo = %p fd = %d type = %d\n", + dprintk(VIDC_DBG, "[MAP] Create binfo = %pK fd = %d type = %d\n", binfo, b->m.planes[0].reserved[0], b->type); for (i = 0; i < b->length; ++i) { @@ -518,7 +518,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) goto exit; } dprintk(VIDC_DBG, - "[MAP] - mapped handle[%d] = %p fd[%d] = %d", + "[MAP] - mapped handle[%d] = %pK fd[%d] = %d", i, binfo->handle[i], i, binfo->fd[i]); binfo->mapped[i] = true; binfo->device_addr[i] = @@ -542,7 +542,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) return rc; } } - dprintk(VIDC_DBG, "[MAP] Adding binfo = %p to list\n", binfo); + dprintk(VIDC_DBG, "[MAP] Adding binfo = %pK to list\n", binfo); mutex_lock(&inst->lock); list_add_tail(&binfo->list, &inst->registered_bufs); mutex_unlock(&inst->lock); @@ -561,7 +561,7 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, bool found = false, keep_node = false; if (!inst || !binfo) { - dprintk(VIDC_ERR, "%s invalid param: %p %p\n", + dprintk(VIDC_ERR, "%s invalid param: %pK %pK\n", __func__, inst, binfo); return -EINVAL; } @@ -603,7 +603,7 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, if (temp->handle[i] && temp->mapped[i] && !temp->same_fd_ref[i]) { dprintk(VIDC_DBG, - "[UNMAP] - handle[%d] = %p fd[%d] = %d", + "[UNMAP] - handle[%d] = %pK fd[%d] = %d", i, temp->handle[i], i, temp->fd[i]); msm_smem_free(inst->mem_client, temp->handle[i]); @@ -619,12 +619,12 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, } } if (!keep_node) { - dprintk(VIDC_DBG, "[UNMAP] AND-FREED binfo: %p\n", temp); + dprintk(VIDC_DBG, "[UNMAP] AND-FREED binfo: %pK\n", temp); list_del(&temp->list); kfree(temp); } else { temp->inactive = true; - dprintk(VIDC_DBG, "[UNMAP] NOT-FREED binfo: %p\n", temp); + dprintk(VIDC_DBG, "[UNMAP] NOT-FREED binfo: %pK\n", temp); } exit: mutex_unlock(&inst->lock); @@ -639,7 +639,7 @@ int qbuf_dynamic_buf(struct msm_vidc_inst *inst, struct v4l2_plane plane[VIDEO_MAX_PLANES] = { {0} }; if (!binfo) { - dprintk(VIDC_ERR, "%s invalid param: %p\n", __func__, binfo); + dprintk(VIDC_ERR, "%s invalid param: %pK\n", __func__, binfo); return -EINVAL; } dprintk(VIDC_DBG, "%s fd[0] = %d\n", __func__, binfo->fd[0]); @@ -662,7 +662,7 @@ int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, int rc = 0; if (!inst) { - dprintk(VIDC_ERR, "%s: invalid inst: %p\n", __func__, inst); + dprintk(VIDC_ERR, "%s: invalid inst: %pK\n", __func__, inst); return -EINVAL; } @@ -670,7 +670,7 @@ int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, return 0; if (!binfo) { - dprintk(VIDC_ERR, "%s: invalid buffer info: %p\n", + dprintk(VIDC_ERR, "%s: invalid buffer info: %pK\n", __func__, inst); return -EINVAL; } @@ -694,11 +694,23 @@ int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, return 0; } +static bool valid_v4l2_buffer(struct v4l2_buffer *b, + struct msm_vidc_inst *inst) { + enum vidc_ports port = + !V4L2_TYPE_IS_MULTIPLANAR(b->type) ? MAX_PORT_NUM : + b->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE ? CAPTURE_PORT : + b->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE ? OUTPUT_PORT : + MAX_PORT_NUM; + + return port != MAX_PORT_NUM && + inst->fmts[port]->num_planes == b->length; +} + int msm_vidc_prepare_buf(void *instance, struct v4l2_buffer *b) { struct msm_vidc_inst *inst = instance; - if (!inst || !b) + if (!inst || !b || !valid_v4l2_buffer(b, inst)) return -EINVAL; if (is_dynamic_output_buffer_mode(b, inst)) { @@ -733,6 +745,25 @@ int msm_vidc_release_buffers(void *instance, int buffer_type) if (!inst) return -EINVAL; + if (!inst->in_reconfig) { + rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); + if (rc) { + dprintk(VIDC_ERR, + "Failed to move inst: %pK to release res done\n", + inst); + } + } + /* + * In dynamic buffer mode, driver needs to release resources, + * but not call release buffers on firmware, as the buffers + * were never registered with firmware. + */ + if ((buffer_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) && + (inst->output_alloc_mode == + HAL_BUFFER_MODE_DYNAMIC)) { + goto free_and_unmap; + } + list_for_each_safe(ptr, next, &inst->registered_bufs) { bool release_buf = false; mutex_lock(&inst->lock); @@ -773,15 +804,22 @@ int msm_vidc_release_buffers(void *instance, int buffer_type) buffer_info.m.planes[0].length); } +free_and_unmap: mutex_lock(&inst->lock); list_for_each_safe(ptr, next, &inst->registered_bufs) { bi = list_entry(ptr, struct buffer_info, list); if (bi->type == buffer_type) { list_del(&bi->list); for (i = 0; i < bi->num_planes; i++) { - if (bi->handle[i]) + if (bi->handle[i] && bi->mapped[i]) { + dprintk(VIDC_DBG, + "%s: [UNMAP] binfo = %pK, handle[%d] = %pK, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", + __func__, bi, i, bi->handle[i], + bi->device_addr[i], bi->fd[i], + bi->buff_off[i], bi->mapped[i]); msm_smem_free(inst->mem_client, bi->handle[i]); + } } kfree(bi); } @@ -814,14 +852,8 @@ int msm_vidc_qbuf(void *instance, struct v4l2_buffer *b) int rc = 0; int i; - if (!inst || !b) - return -EINVAL; - - if (b->length > VIDEO_MAX_PLANES) { - dprintk(VIDC_ERR, "num planes exceeds max: %d\n", - b->length); + if (!inst || !b || !valid_v4l2_buffer(b, inst)) return -EINVAL; - } if (is_dynamic_output_buffer_mode(b, inst)) { if (b->m.planes[0].reserved[0]) @@ -887,14 +919,8 @@ int msm_vidc_dqbuf(void *instance, struct v4l2_buffer *b) struct buffer_info *buffer_info = NULL; int i = 0, rc = 0; - if (!inst || !b) - return -EINVAL; - - if (b->length > VIDEO_MAX_PLANES) { - dprintk(VIDC_ERR, "num planes exceed maximum: %d\n", - b->length); + if (!inst || !b || !valid_v4l2_buffer(b, inst)) return -EINVAL; - } if (inst->session_type == MSM_VIDC_DECODER) rc = msm_vdec_dqbuf(instance, b); @@ -980,7 +1006,7 @@ int msm_vidc_enum_framesizes(void *instance, struct v4l2_frmsizeenum *fsize) struct msm_vidc_core_capability *capability = NULL; if (!inst || !fsize) { - dprintk(VIDC_ERR, "%s: invalid parameter: %p %p\n", + dprintk(VIDC_ERR, "%s: invalid parameter: %pK %pK\n", __func__, inst, fsize); return -EINVAL; } @@ -1111,7 +1137,7 @@ void *msm_vidc_open(int core_id, int session_type) goto err_invalid_core; } - pr_info(VIDC_DBG_TAG "Opening video instance: %p, %d\n", + pr_info(VIDC_DBG_TAG "Opening video instance: %pK, %d\n", VIDC_INFO, inst, session_type); mutex_init(&inst->sync_lock); mutex_init(&inst->bufq[CAPTURE_PORT].lock); @@ -1293,7 +1319,7 @@ int msm_vidc_close(void *instance) for (i = 0; i < MAX_PORT_NUM; i++) vb2_queue_release(&inst->bufq[i].vb2_bufq); - pr_info(VIDC_DBG_TAG "Closed video instance: %p\n", VIDC_INFO, inst); + pr_info(VIDC_DBG_TAG "Closed video instance: %pK\n", VIDC_INFO, inst); kfree(inst); return 0; } diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index e5ca2408d4d..e542cbb29b3 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -86,13 +86,27 @@ static bool is_thumbnail_session(struct msm_vidc_inst *inst) return false; } +static inline bool is_non_realtime_session(struct msm_vidc_inst *inst) +{ + int rc = 0; + struct v4l2_control ctrl = { + .id = V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY }; + rc = v4l2_g_ctrl(&inst->ctrl_handler, &ctrl); + if (!rc && ctrl.value) + return true; + return false; +} + static int msm_comm_get_load(struct msm_vidc_core *core, - enum session_type type) + enum session_type type, enum vidc_calculation calc) { struct msm_vidc_inst *inst = NULL; - int num_mbs_per_sec = 0; + int num_mbs_per_sec = 0, rc = 0, op_rate = 0; + struct v4l2_control ctrl = { + .id = V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE }; + if (!core) { - dprintk(VIDC_ERR, "Invalid args: %p\n", core); + dprintk(VIDC_ERR, "Invalid args: %pK\n", core); return -EINVAL; } mutex_lock(&core->sync_lock); @@ -101,10 +115,26 @@ static int msm_comm_get_load(struct msm_vidc_core *core, if (inst->session_type == type && inst->state >= MSM_VIDC_OPEN_DONE && inst->state < MSM_VIDC_STOP_DONE) { - if (!is_thumbnail_session(inst)) + rc = v4l2_g_ctrl(&inst->ctrl_handler, &ctrl); + if (!rc && ctrl.value) + op_rate = ctrl.value >> 16; + if (is_non_realtime_session(inst) && calc == LOAD) { + // 1 fps load for non-realtime num_mbs_per_sec += NUM_MBS_PER_SEC( inst->prop.height, - inst->prop.width, inst->prop.fps); + inst->prop.width, inst->prop.fps) / + inst->prop.fps; + } else if (!is_thumbnail_session(inst)) { + if (op_rate) { + num_mbs_per_sec += NUM_MBS_PER_SEC( + inst->prop.height, + inst->prop.width, op_rate); + } else { + num_mbs_per_sec += NUM_MBS_PER_SEC( + inst->prop.height, + inst->prop.width, inst->prop.fps); + } + } } mutex_unlock(&inst->lock); } @@ -118,22 +148,22 @@ static int msm_comm_scale_bus(struct msm_vidc_core *core, int load; int rc = 0; struct hfi_device *hdev; + enum vidc_calculation calc = CLOCKS; if (!core || type >= MSM_VIDC_MAX_DEVICES) { - dprintk(VIDC_ERR, "Invalid args: %p, %d\n", core, type); + dprintk(VIDC_ERR, "Invalid args: %pK, %d\n", core, type); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device handle %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device handle %pK\n", hdev); return -EINVAL; } - if (is_turbo_requested(core, type)) load = core->resources.max_load; else - load = msm_comm_get_load(core, type); + load = msm_comm_get_load(core, type, calc); rc = call_hfi_op(hdev, scale_bus, hdev->hfi_device_data, load, type, mtype); @@ -197,7 +227,7 @@ const struct msm_vidc_format *msm_comm_get_pixel_fmt_index( { int i, k = 0; if (!fmt || index < 0) { - dprintk(VIDC_ERR, "Invalid inputs, fmt = %p, index = %d\n", + dprintk(VIDC_ERR, "Invalid inputs, fmt = %pK, index = %d\n", fmt, index); return NULL; } @@ -219,7 +249,7 @@ struct msm_vidc_format *msm_comm_get_pixel_fmt_fourcc( { int i; if (!fmt) { - dprintk(VIDC_ERR, "Invalid inputs, fmt = %p\n", fmt); + dprintk(VIDC_ERR, "Invalid inputs, fmt = %pK\n", fmt); return NULL; } for (i = 0; i < size; i++) { @@ -260,7 +290,7 @@ static void handle_sys_init_done(enum command_response cmd, void *data) return; } dprintk(VIDC_DBG, "index = %d\n", index); - dprintk(VIDC_DBG, "ptr = %p\n", &(core->completions[index])); + dprintk(VIDC_DBG, "ptr = %pK\n", &(core->completions[index])); complete(&(core->completions[index])); sys_init_msg = response->data; if (!sys_init_msg) { @@ -339,11 +369,11 @@ static void change_inst_state(struct msm_vidc_inst *inst, mutex_lock(&inst->lock); if (inst->state == MSM_VIDC_CORE_INVALID) { dprintk(VIDC_DBG, - "Inst: %p is in bad state can't change state", + "Inst: %pK is in bad state can't change state", inst); goto exit; } - dprintk(VIDC_DBG, "Moved inst: %p from state: %d to state: %d\n", + dprintk(VIDC_DBG, "Moved inst: %pK from state: %d to state: %d\n", inst, inst->state, state); inst->state = state; exit: @@ -354,7 +384,7 @@ static int signal_session_msg_receipt(enum command_response cmd, struct msm_vidc_inst *inst) { if (!inst) { - dprintk(VIDC_ERR, "Invalid(%p) instance id\n", inst); + dprintk(VIDC_ERR, "Invalid(%pK) instance id\n", inst); return -EINVAL; } complete(&inst->completions[SESSION_MSG_INDEX(cmd)]); @@ -385,7 +415,7 @@ static int wait_for_state(struct msm_vidc_inst *inst, { int rc = 0; if (IS_ALREADY_IN_STATE(flipped_state, desired_state)) { - dprintk(VIDC_INFO, "inst: %p is already in state: %d\n", + dprintk(VIDC_INFO, "inst: %pK is already in state: %d\n", inst, inst->state); goto err_same_state; } @@ -437,6 +467,8 @@ static void handle_session_init_done(enum command_response cmd, void *data) inst->capability.frame_rate = session_init_done->frame_rate; inst->capability.hier_p = session_init_done->hier_p; + inst->capability.mbs_per_frame = + session_init_done->mbs_per_frame; inst->capability.capability_set = true; inst->output_alloc_mode_supported = @@ -487,7 +519,7 @@ static void handle_event_change(enum command_response cmd, void *data) u32 *ptr = NULL; dprintk(VIDC_DBG, - "%s - inst: %p buffer: %p extra: %p\n", + "%s - inst: %pK buffer: %pK extra: %pK\n", __func__, inst, event_notify->packet_buffer, event_notify->exra_data_buffer); @@ -669,7 +701,7 @@ static void handle_session_error(enum command_response cmd, void *data) inst = (struct msm_vidc_inst *)response->session_id; if (inst) { dprintk(VIDC_WARN, - "Session error receivd for session %p\n", inst); + "Session error receivd for session %pK\n", inst); mutex_lock(&inst->sync_lock); inst->state = MSM_VIDC_CORE_INVALID; mutex_unlock(&inst->sync_lock); @@ -692,7 +724,7 @@ static void handle_sys_error(enum command_response cmd, void *data) subsystem_crashed("venus"); if (response) { core = get_vidc_core(response->device_id); - dprintk(VIDC_WARN, "SYS_ERROR received for core %p\n", core); + dprintk(VIDC_WARN, "SYS_ERROR received for core %pK\n", core); if (core) { mutex_lock(&core->lock); core->state = VIDC_CORE_INVALID; @@ -707,12 +739,12 @@ static void handle_sys_error(enum command_response cmd, void *data) hdev = inst->core->device; if (hdev && inst->session) { dprintk(VIDC_DBG, - "cleaning up inst: 0x%p", inst); + "cleaning up inst: 0x%pK", inst); rc = call_hfi_op(hdev, session_clean, (void *) inst->session); if (rc) dprintk(VIDC_ERR, - "Sess clean failed :%p", + "Sess clean failed :%pK", inst); } inst->session = NULL; @@ -763,7 +795,7 @@ static void handle_sys_watchdog_timeout(enum command_response cmd, void *data) (void *) inst->session); if (rc) dprintk(VIDC_ERR, - "Sess clean failed :%p", + "Sess clean failed :%pK", inst); } @@ -789,7 +821,7 @@ static void handle_session_close(enum command_response cmd, void *data) hdev = inst->core->device; mutex_lock(&inst->lock); if (inst->session) { - dprintk(VIDC_DBG, "cleaning up inst: 0x%p", inst); + dprintk(VIDC_DBG, "cleaning up inst: 0x%pK", inst); call_hfi_op(hdev, session_clean, (void *) inst->session); } @@ -1005,6 +1037,8 @@ static void handle_fbd(enum command_response cmd, void *data) vb->v4l2_planes[0].reserved[3] = fill_buf_done->start_y_coord; vb->v4l2_planes[0].reserved[4] = fill_buf_done->frame_width; vb->v4l2_planes[0].reserved[5] = fill_buf_done->frame_height; + vb->v4l2_planes[0].reserved[6] = inst->prop.width; + vb->v4l2_planes[0].reserved[7] = inst->prop.height; if (vb->v4l2_planes[0].data_offset > vb->v4l2_planes[0].length) dprintk(VIDC_INFO, "fbd:Overflow data_offset = %d; length = %d\n", @@ -1218,14 +1252,16 @@ static int msm_comm_scale_clocks(struct msm_vidc_core *core) int num_mbs_per_sec; int rc = 0; struct hfi_device *hdev; + enum vidc_calculation calc = CLOCKS; + if (!core) { - dprintk(VIDC_ERR, "%s Invalid args: %p\n", __func__, core); + dprintk(VIDC_ERR, "%s Invalid args: %pK\n", __func__, core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "%s Invalid device handle: %p\n", + dprintk(VIDC_ERR, "%s Invalid device handle: %pK\n", __func__, hdev); return -EINVAL; } @@ -1234,10 +1270,9 @@ static int msm_comm_scale_clocks(struct msm_vidc_core *core) is_turbo_requested(core, MSM_VIDC_DECODER)) { num_mbs_per_sec = core->resources.max_load; } else { - num_mbs_per_sec = msm_comm_get_load(core, MSM_VIDC_ENCODER); - num_mbs_per_sec += msm_comm_get_load(core, MSM_VIDC_DECODER); + num_mbs_per_sec = msm_comm_get_load(core, MSM_VIDC_ENCODER, calc); + num_mbs_per_sec += msm_comm_get_load(core, MSM_VIDC_DECODER, calc); } - dprintk(VIDC_INFO, "num_mbs_per_sec = %d\n", num_mbs_per_sec); rc = call_hfi_op(hdev, scale_clocks, hdev->hfi_device_data, num_mbs_per_sec); @@ -1542,7 +1577,7 @@ static int msm_comm_session_init(int flipped_state, hdev = inst->core->device; if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_OPEN)) { - dprintk(VIDC_INFO, "inst: %p is already in state: %d\n", + dprintk(VIDC_INFO, "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1603,6 +1638,7 @@ static int msm_vidc_load_resources(int flipped_state, u32 ocmem_sz = 0; struct hfi_device *hdev; int num_mbs_per_sec = 0; + enum vidc_calculation calc = LOAD; if (!inst || !inst->core || !inst->core->device) { dprintk(VIDC_ERR, "%s invalid parameters", __func__); @@ -1614,9 +1650,8 @@ static int msm_vidc_load_resources(int flipped_state, "Core is in bad state can't do load res"); return -EINVAL; } - - num_mbs_per_sec = msm_comm_get_load(inst->core, MSM_VIDC_DECODER); - num_mbs_per_sec += msm_comm_get_load(inst->core, MSM_VIDC_ENCODER); + num_mbs_per_sec = msm_comm_get_load(inst->core, MSM_VIDC_DECODER, calc); + num_mbs_per_sec += msm_comm_get_load(inst->core, MSM_VIDC_ENCODER, calc); if (num_mbs_per_sec > inst->core->resources.max_load) { dprintk(VIDC_ERR, "HW is overloaded, needed: %d max: %d\n", num_mbs_per_sec, inst->core->resources.max_load); @@ -1628,7 +1663,7 @@ static int msm_vidc_load_resources(int flipped_state, hdev = inst->core->device; if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_LOAD_RESOURCES)) { - dprintk(VIDC_INFO, "inst: %p is already in state: %d\n", + dprintk(VIDC_INFO, "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1684,7 +1719,7 @@ static int msm_vidc_start(int flipped_state, struct msm_vidc_inst *inst) if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_START)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1714,7 +1749,7 @@ static int msm_vidc_stop(int flipped_state, struct msm_vidc_inst *inst) if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_STOP)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1744,7 +1779,7 @@ static int msm_vidc_release_res(int flipped_state, struct msm_vidc_inst *inst) if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_RELEASE_RESOURCES)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1776,7 +1811,7 @@ static int msm_comm_session_close(int flipped_state, hdev = inst->core->device; if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_CLOSE)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -2001,16 +2036,16 @@ int msm_comm_try_state(struct msm_vidc_inst *inst, int state) struct msm_vidc_core *core; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } dprintk(VIDC_DBG, - "Trying to move inst: %p from: 0x%x to 0x%x\n", + "Trying to move inst: %pK from: 0x%x to 0x%x\n", inst, inst->state, state); core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", inst); + "Invalid core pointer = %pK\n", inst); return -EINVAL; } mutex_lock(&inst->sync_lock); @@ -2118,18 +2153,18 @@ int msm_comm_qbuf(struct vb2_buffer *vb) q = vb->vb2_queue; inst = q->drv_priv; if (!inst || !vb) { - dprintk(VIDC_ERR, "Invalid input: %p, %p\n", inst, vb); + dprintk(VIDC_ERR, "Invalid input: %pK, %pK\n", inst, vb); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid input: %p, %p, %p\n", inst, core, vb); + "Invalid input: %pK, %pK, %pK\n", inst, core, vb); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid input: %p", hdev); + dprintk(VIDC_ERR, "Invalid input: %pK", hdev); return -EINVAL; } @@ -2240,7 +2275,7 @@ int msm_comm_qbuf(struct vb2_buffer *vb) (void *) inst->session, &seq_hdr); if (!rc) { inst->vb2_seq_hdr = vb; - dprintk(VIDC_DBG, "Seq_hdr: %p\n", + dprintk(VIDC_DBG, "Seq_hdr: %pK\n", inst->vb2_seq_hdr); } atomic_dec(&inst->get_seq_hdr_cnt); @@ -2325,18 +2360,18 @@ int msm_comm_release_scratch_buffers(struct msm_vidc_inst *inst) struct hfi_device *hdev; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK\n", hdev); return -EINVAL; } mutex_lock(&inst->lock); @@ -2396,18 +2431,18 @@ int msm_comm_release_persist_buffers(struct msm_vidc_inst *inst) struct hfi_device *hdev; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK\n", hdev); return -EINVAL; } mutex_lock(&inst->lock); @@ -2462,7 +2497,7 @@ int msm_comm_try_set_prop(struct msm_vidc_inst *inst, int rc = 0; struct hfi_device *hdev; if (!inst) { - dprintk(VIDC_ERR, "Invalid input: %p\n", inst); + dprintk(VIDC_ERR, "Invalid input: %pK\n", inst); return -EINVAL; } @@ -2669,7 +2704,7 @@ void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) if (binfo && binfo->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { dprintk(VIDC_DBG, - "%s: binfo = %p device_addr = 0x%pa\n", + "%s: binfo = %pK device_addr = 0x%pa\n", __func__, binfo, &binfo->device_addr[0]); buf_ref_put(inst, binfo); } @@ -2689,18 +2724,18 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK", hdev); return -EINVAL; } @@ -2717,7 +2752,7 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) if (inst->state == MSM_VIDC_CORE_INVALID || core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p and inst %p are in bad state\n", + "Core %pK and inst %pK are in bad state\n", core, inst); msm_comm_flush_in_invalid_state(inst); return 0; @@ -2879,7 +2914,7 @@ int msm_vidc_trigger_ssr(struct msm_vidc_core *core, int rc = 0; struct hfi_device *hdev; if (!core && !core->device) { - dprintk(VIDC_WARN, "Invalid parameters: %p\n", core); + dprintk(VIDC_WARN, "Invalid parameters: %pK\n", core); return -EINVAL; } hdev = core->device; @@ -2892,12 +2927,13 @@ int msm_vidc_trigger_ssr(struct msm_vidc_core *core, static int msm_vidc_load_supported(struct msm_vidc_inst *inst) { int num_mbs_per_sec = 0; + enum vidc_calculation calc = LOAD; if (inst->state == MSM_VIDC_OPEN_DONE) { num_mbs_per_sec = msm_comm_get_load(inst->core, - MSM_VIDC_DECODER); + MSM_VIDC_DECODER, calc); num_mbs_per_sec += msm_comm_get_load(inst->core, - MSM_VIDC_ENCODER); + MSM_VIDC_ENCODER, calc); if (num_mbs_per_sec > inst->core->resources.max_load) { dprintk(VIDC_ERR, "H/w is overloaded. needed: %d max: %d\n", diff --git a/drivers/media/platform/msm/vidc/msm_vidc_debug.c b/drivers/media/platform/msm/vidc/msm_vidc_debug.c index e93f7741b12..3012d637d49 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_debug.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_debug.c @@ -20,7 +20,6 @@ int msm_vidc_debug_out = VIDC_OUT_PRINTK; int msm_fw_debug = 0x18; int msm_fw_debug_mode = 0x1; int msm_fw_low_power_mode = 0x1; -int msm_vp8_low_tier = 0x1; struct debug_buffer { char ptr[MAX_DBG_BUF_SIZE]; @@ -64,13 +63,13 @@ static ssize_t core_info_read(struct file *file, char __user *buf, struct hfi_device *hdev; int i = 0; if (!core || !core->device) { - dprintk(VIDC_ERR, "Invalid params, core: %p\n", core); + dprintk(VIDC_ERR, "Invalid params, core: %pK\n", core); return 0; } hdev = core->device; INIT_DBG_BUF(dbg_buf); write_str(&dbg_buf, "===============================\n"); - write_str(&dbg_buf, "CORE %d: 0x%p\n", core->id, core); + write_str(&dbg_buf, "CORE %d: 0x%pK\n", core->id, core); write_str(&dbg_buf, "===============================\n"); write_str(&dbg_buf, "state: %d\n", core->state); write_str(&dbg_buf, "base addr: 0x%x\n", @@ -132,7 +131,7 @@ struct dentry *msm_vidc_debugfs_init_core(struct msm_vidc_core *core, struct dentry *dir = NULL; char debugfs_name[MAX_DEBUGFS_NAME]; if (!core) { - dprintk(VIDC_ERR, "Invalid params, core: %p\n", core); + dprintk(VIDC_ERR, "Invalid params, core: %pK\n", core); goto failed_create_dir; } @@ -171,11 +170,6 @@ struct dentry *msm_vidc_debugfs_init_core(struct msm_vidc_core *core, dprintk(VIDC_DBG, "debugfs_create_file: fail\n"); goto failed_create_dir; } - if (!debugfs_create_u32("vp8_low_tier", S_IRUGO | S_IWUSR, - parent, &msm_vp8_low_tier)) { - dprintk(VIDC_DBG, "debugfs_create_file: fail\n"); - goto failed_create_dir; - } if (!debugfs_create_u32("debug_output", S_IRUGO | S_IWUSR, parent, &msm_vidc_debug_out)) { dprintk(VIDC_ERR, "debugfs_create_file: fail\n"); @@ -231,15 +225,15 @@ static ssize_t inst_info_read(struct file *file, char __user *buf, struct msm_vidc_inst *inst = file->private_data; int i, j; if (!inst) { - dprintk(VIDC_ERR, "Invalid params, core: %p\n", inst); + dprintk(VIDC_ERR, "Invalid params, core: %pK\n", inst); return 0; } INIT_DBG_BUF(dbg_buf); write_str(&dbg_buf, "===============================\n"); - write_str(&dbg_buf, "INSTANCE: 0x%p (%s)\n", inst, + write_str(&dbg_buf, "INSTANCE: 0x%pK (%s)\n", inst, inst->session_type == MSM_VIDC_ENCODER ? "Encoder" : "Decoder"); write_str(&dbg_buf, "===============================\n"); - write_str(&dbg_buf, "core: 0x%p\n", inst->core); + write_str(&dbg_buf, "core: 0x%pK\n", inst->core); write_str(&dbg_buf, "height: %d\n", inst->prop.height); write_str(&dbg_buf, "width: %d\n", inst->prop.width); write_str(&dbg_buf, "fps: %d\n", inst->prop.fps); @@ -298,10 +292,10 @@ struct dentry *msm_vidc_debugfs_init_inst(struct msm_vidc_inst *inst, struct dentry *dir = NULL; char debugfs_name[MAX_DEBUGFS_NAME]; if (!inst) { - dprintk(VIDC_ERR, "Invalid params, inst: %p\n", inst); + dprintk(VIDC_ERR, "Invalid params, inst: %pK\n", inst); goto failed_create_dir; } - snprintf(debugfs_name, MAX_DEBUGFS_NAME, "inst_%p", inst); + snprintf(debugfs_name, MAX_DEBUGFS_NAME, "inst_%pK", inst); dir = debugfs_create_dir(debugfs_name, parent); if (!dir) { dprintk(VIDC_ERR, "Failed to create debugfs for msm_vidc\n"); diff --git a/drivers/media/platform/msm/vidc/msm_vidc_debug.h b/drivers/media/platform/msm/vidc/msm_vidc_debug.h index 5b572c95205..329a225cab0 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_debug.h +++ b/drivers/media/platform/msm/vidc/msm_vidc_debug.h @@ -52,7 +52,6 @@ extern int msm_vidc_debug_out; extern int msm_fw_debug; extern int msm_fw_debug_mode; extern int msm_fw_low_power_mode; -extern int msm_vp8_low_tier; #define dprintk(__level, __fmt, arg...) \ do { \ diff --git a/drivers/media/platform/msm/vidc/msm_vidc_internal.h b/drivers/media/platform/msm/vidc/msm_vidc_internal.h index 6460082985d..1b172349eb9 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_internal.h +++ b/drivers/media/platform/msm/vidc/msm_vidc_internal.h @@ -73,6 +73,11 @@ enum vidc_core_state { VIDC_CORE_INVALID }; +enum vidc_calculation { + CLOCKS = 0, + LOAD +}; + /*Donot change the enum values unless * you know what you are doing*/ enum instance_state { @@ -181,6 +186,7 @@ struct msm_vidc_core_capability { struct hal_capability_supported height; struct hal_capability_supported frame_rate; struct hal_capability_supported hier_p; + struct hal_capability_supported mbs_per_frame; u32 capability_set; }; diff --git a/drivers/media/platform/msm/vidc/q6_hfi.c b/drivers/media/platform/msm/vidc/q6_hfi.c index bc3b93d2448..4bed4c0b3aa 100644 --- a/drivers/media/platform/msm/vidc/q6_hfi.c +++ b/drivers/media/platform/msm/vidc/q6_hfi.c @@ -184,7 +184,8 @@ static void q6_hfi_core_work_handler(struct work_struct *work) if (!rc) hfi_process_msg_packet(device->callback, device->device_id, - (struct vidc_hal_msg_pkt_hdr *) packet); + (struct vidc_hal_msg_pkt_hdr *) packet, + &device->sess_head, &device->session_lock); } while (!rc); if (rc != -ENODATA) @@ -199,7 +200,7 @@ static int q6_hfi_register_iommu_domains(struct q6_hfi_device *device) struct iommu_info *iommu_map; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return -EINVAL; } @@ -216,14 +217,14 @@ static int q6_hfi_register_iommu_domains(struct q6_hfi_device *device) domain = iommu_group_get_iommudata(iommu_map->group); if (IS_ERR_OR_NULL(domain)) { dprintk(VIDC_ERR, - "Failed to get domain data for group %p", + "Failed to get domain data for group %pK", iommu_map->group); goto fail_group; } iommu_map->domain = msm_find_domain_no(domain); if (iommu_map->domain < 0) { dprintk(VIDC_ERR, - "Failed to get domain index for domain %p", + "Failed to get domain index for domain %pK", domain); goto fail_group; } @@ -248,7 +249,7 @@ static void q6_hfi_deregister_iommu_domains(struct q6_hfi_device *device) int i = 0; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return; } @@ -337,7 +338,7 @@ static void *q6_hfi_get_device(u32 device_id, int rc = 0; if (!callback) { - dprintk(VIDC_ERR, "%s Invalid params: %p\n", + dprintk(VIDC_ERR, "%s Invalid params: %pK\n", __func__, callback); return NULL; } @@ -483,6 +484,7 @@ static int q6_hfi_core_init(void *device) } INIT_LIST_HEAD(&dev->sess_head); + mutex_init(&dev->session_lock); if (!dev->event_queue.buffer) { rc = q6_init_event_queue(dev); @@ -583,7 +585,9 @@ static void *q6_hfi_session_init(void *device, u32 session_id, rc = -EBADE; goto err_session_init; } + mutex_lock(&dev->session_lock); list_add_tail(&new_session->list, &dev->sess_head); + mutex_unlock(&dev->session_lock); return new_session; err_session_init: @@ -646,7 +650,11 @@ static int q6_hfi_session_clean(void *session) sess_close = session; dprintk(VIDC_DBG, "deleted the session: 0x%x", sess_close->session_id); + mutex_lock(&((struct q6_hfi_device *) + sess_close->device)->session_lock); list_del(&sess_close->list); + mutex_unlock(&((struct q6_hfi_device *) + sess_close->device)->session_lock); kfree(sess_close); return 0; } @@ -1244,7 +1252,7 @@ static int q6_hfi_iommu_attach(struct q6_hfi_device *device) struct iommu_info *iommu_map; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return -EINVAL; } @@ -1259,7 +1267,7 @@ static int q6_hfi_iommu_attach(struct q6_hfi_device *device) rc = IS_ERR(domain) ? PTR_ERR(domain) : -EINVAL; break; } - dprintk(VIDC_DBG, "Attaching domain(id:%d) %p to group %p", + dprintk(VIDC_DBG, "Attaching domain(id:%d) %pK to group %pK", iommu_map->domain, domain, group); rc = iommu_attach_group(domain, group); if (rc) { @@ -1290,7 +1298,7 @@ static void q6_hfi_iommu_detach(struct q6_hfi_device *device) int i; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return; } @@ -1434,7 +1442,7 @@ int q6_hfi_initialize(struct hfi_device *hdev, u32 device_id, int rc = 0; if (!hdev || !res || !callback) { - dprintk(VIDC_ERR, "Invalid params: %p %p %p", + dprintk(VIDC_ERR, "Invalid params: %pK %pK %pK", hdev, res, callback); rc = -EINVAL; goto err_hfi_init; diff --git a/drivers/media/platform/msm/vidc/q6_hfi.h b/drivers/media/platform/msm/vidc/q6_hfi.h index 3dc4607130a..c2ed9183df9 100644 --- a/drivers/media/platform/msm/vidc/q6_hfi.h +++ b/drivers/media/platform/msm/vidc/q6_hfi.h @@ -43,6 +43,7 @@ struct q6_hfi_device { struct q6_resources resources; struct msm_vidc_platform_resources *res; void *apr; + struct mutex session_lock; }; struct q6_apr_cmd_sys_init_packet { diff --git a/drivers/media/platform/msm/vidc/venus_hfi.c b/drivers/media/platform/msm/vidc/venus_hfi.c index 47b88db5491..ab1493f3247 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.c +++ b/drivers/media/platform/msm/vidc/venus_hfi.c @@ -198,7 +198,7 @@ static int venus_hfi_write_queue(void *info, u8 *packet, u32 *rx_req_is_set) venus_hfi_sim_modify_cmd_packet(packet); if (msm_vidc_debug & VIDC_PKT) { - dprintk(VIDC_PKT, "%s: %p\n", __func__, qinfo); + dprintk(VIDC_PKT, "%s: %pK\n", __func__, qinfo); venus_hfi_dump_packet(packet); } @@ -382,7 +382,7 @@ static int venus_hfi_read_queue(void *info, u8 *packet, u32 *pb_tx_req_is_set) *pb_tx_req_is_set = (1 == queue->qhdr_tx_req) ? 1 : 0; venus_hfi_hal_sim_modify_msg_packet(packet); if (msm_vidc_debug & VIDC_PKT) { - dprintk(VIDC_PKT, "%s: %p\n", __func__, qinfo); + dprintk(VIDC_PKT, "%s: %pK\n", __func__, qinfo); venus_hfi_dump_packet(packet); } dprintk(VIDC_DBG, "Out : "); @@ -410,7 +410,7 @@ static int venus_hfi_alloc(void *mem, void *clnt, u32 size, u32 align, rc = -ENOMEM; goto fail_smem_alloc; } - dprintk(VIDC_DBG, "venus_hfi_alloc:ptr=%p,size=%d", + dprintk(VIDC_DBG, "venus_hfi_alloc:ptr=%pK,size=%d", alloc->kvaddr, size); rc = msm_smem_cache_operations(clnt, alloc, SMEM_CACHE_CLEAN); @@ -438,7 +438,7 @@ static void venus_hfi_write_register(struct venus_hfi_device *device, u32 reg, u32 hwiosymaddr = reg; u8 *base_addr; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return; } @@ -478,7 +478,7 @@ static int venus_hfi_read_register(struct venus_hfi_device *device, u32 reg) int rc ; u8 *base_addr; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } @@ -497,7 +497,7 @@ static inline void venus_hfi_clk_gating_on(struct venus_hfi_device *device) int i; struct venus_core_clock *cl; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return; } if (!device->clocks_enabled) { @@ -517,7 +517,7 @@ static inline int venus_hfi_clk_gating_off(struct venus_hfi_device *device) struct venus_core_clock *cl; int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } if (device->clocks_enabled) { @@ -566,7 +566,7 @@ static int venus_hfi_scale_clocks(void *dev, int load) int rc = 0; struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "Invalid args: %p\n", device); + dprintk(VIDC_ERR, "Invalid args: %pK\n", device); return -EINVAL; } device->load = load; @@ -1045,6 +1045,7 @@ static int venus_hfi_core_init(void *device) INIT_LIST_HEAD(&dev->sess_head); mutex_init(&dev->read_lock); mutex_init(&dev->write_lock); + mutex_init(&dev->session_lock); venus_hfi_set_registers(dev); if (!dev->hal_client) { @@ -1484,7 +1485,10 @@ static void *venus_hfi_session_init(void *device, u32 session_id, else if (session_type == 2) new_session->is_decoder = 1; new_session->device = dev; + + mutex_lock(&dev->session_lock); list_add_tail(&new_session->list, &dev->sess_head); + mutex_unlock(&dev->session_lock); if (create_pkt_cmd_sys_session_init(&pkt, (u32)new_session, session_type, codec_type)) { @@ -1552,9 +1556,13 @@ static int venus_hfi_session_clean(void *session) return -EINVAL; } sess_close = session; - dprintk(VIDC_DBG, "deleted the session: 0x%p", + dprintk(VIDC_DBG, "deleted the session: 0x%pK", sess_close); + mutex_lock(&((struct venus_hfi_device *) + sess_close->device)->session_lock); list_del(&sess_close->list); + mutex_unlock(&((struct venus_hfi_device *) + sess_close->device)->session_lock); kfree(sess_close); return 0; } @@ -1985,7 +1993,8 @@ static void venus_hfi_response_handler(struct venus_hfi_device *device) while (!venus_hfi_iface_msgq_read(device, packet)) { rc = hfi_process_msg_packet(device->callback, device->device_id, - (struct vidc_hal_msg_pkt_hdr *) packet); + (struct vidc_hal_msg_pkt_hdr *) packet, + &device->sess_head, &device->session_lock); if (rc == HFI_MSG_EVENT_NOTIFY) venus_hfi_process_msg_event_notify( device, (void *)packet); @@ -2009,7 +2018,7 @@ static void venus_hfi_core_work_handler(struct work_struct *work) dprintk(VIDC_INFO, " GOT INTERRUPT () "); if (!device->callback) { - dprintk(VIDC_ERR, "No interrupt callback function: %p\n", + dprintk(VIDC_ERR, "No interrupt callback function: %pK\n", device); return; } @@ -2095,7 +2104,7 @@ static inline int venus_hfi_init_clocks(struct msm_vidc_platform_resources *res, int rc = 0; struct venus_core_clock *clock; if (!res || !device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } clock = device->resources.clock; @@ -2174,7 +2183,7 @@ static inline void venus_hfi_disable_clks(struct venus_hfi_device *device) int i; struct venus_core_clock *cl; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return; } if (device->clocks_enabled) { @@ -2203,7 +2212,7 @@ static inline int venus_hfi_enable_clks(struct venus_hfi_device *device) struct venus_core_clock *cl; int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } @@ -2252,14 +2261,14 @@ static int venus_hfi_register_iommu_domains(struct venus_hfi_device *device, domain = iommu_group_get_iommudata(iommu_map->group); if (!domain) { dprintk(VIDC_ERR, - "Failed to get domain data for group %p", + "Failed to get domain data for group %pK", iommu_map->group); goto fail_group; } iommu_map->domain = msm_find_domain_no(domain); if (iommu_map->domain < 0) { dprintk(VIDC_ERR, - "Failed to get domain index for domain %p", + "Failed to get domain index for domain %pK", domain); goto fail_group; } @@ -2423,7 +2432,7 @@ static int venus_hfi_scale_bus(void *dev, int load, int bus_vector = 0; if (!device) { - dprintk(VIDC_ERR, "%s invalid device handle %p", + dprintk(VIDC_ERR, "%s invalid device handle %pK", __func__, device); return -EINVAL; } @@ -2460,7 +2469,7 @@ static int venus_hfi_unvote_bus(void *dev, struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s invalid device handle %p", + dprintk(VIDC_ERR, "%s invalid device handle %pK", __func__, device); return -EINVAL; } @@ -2489,7 +2498,7 @@ static int venus_hfi_set_ocmem(void *dev, struct ocmem_buf *ocmem) struct venus_hfi_device *device = dev; int rc = 0; if (!device || !ocmem) { - dprintk(VIDC_ERR, "Invalid params, core:%p, ocmem: %p\n", + dprintk(VIDC_ERR, "Invalid params, core:%pK, ocmem: %pK\n", device, ocmem); return -EINVAL; } @@ -2514,7 +2523,7 @@ static int venus_hfi_unset_ocmem(void *dev) int rc = 0; if (!device) { - dprintk(VIDC_ERR, "%s Invalid params, device:%p\n", + dprintk(VIDC_ERR, "%s Invalid params, device:%pK\n", __func__, device); rc = -EINVAL; goto ocmem_unset_failed; @@ -2585,7 +2594,7 @@ static int venus_hfi_alloc_ocmem(void *dev, unsigned long size) struct venus_hfi_device *device = dev; if (!device || !size) { - dprintk(VIDC_ERR, "%s Invalid param, core: %p, size: %lu\n", + dprintk(VIDC_ERR, "%s Invalid param, core: %pK, size: %lu\n", __func__, device, size); return -EINVAL; } @@ -2620,7 +2629,7 @@ static int venus_hfi_free_ocmem(void *dev) int rc = 0; if (!device) { - dprintk(VIDC_ERR, "%s invalid device handle %p", + dprintk(VIDC_ERR, "%s invalid device handle %pK", __func__, device); return -EINVAL; } @@ -2741,7 +2750,7 @@ static void venus_hfi_iommu_detach(struct venus_hfi_device *device) int i; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid paramter: %p\n", device); + dprintk(VIDC_ERR, "Invalid paramter: %pK\n", device); return; } @@ -2761,7 +2770,7 @@ static int venus_hfi_iommu_get_domain_partition(void *dev, u32 flags, struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s: Invalid param device: %p\n", + dprintk(VIDC_ERR, "%s: Invalid param device: %pK\n", __func__, device); return -EINVAL; } @@ -2785,7 +2794,7 @@ static int protect_cp_mem(struct venus_hfi_device *device) iommu_group_set = &device->res->iommu_group_set; if (!iommu_group_set) { - dprintk(VIDC_ERR, "invalid params: %p\n", iommu_group_set); + dprintk(VIDC_ERR, "invalid params: %pK\n", iommu_group_set); return -EINVAL; } @@ -2822,7 +2831,7 @@ static int venus_hfi_load_fw(void *dev) struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s Invalid paramter: %p\n", + dprintk(VIDC_ERR, "%s Invalid paramter: %pK\n", __func__, device); return -EINVAL; } @@ -2872,7 +2881,7 @@ static void venus_hfi_unload_fw(void *dev) { struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s Invalid paramter: %p\n", + dprintk(VIDC_ERR, "%s Invalid paramter: %pK\n", __func__, device); return; } @@ -2892,7 +2901,7 @@ static int venus_hfi_get_fw_info(void *dev, enum fw_info info) struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s Invalid paramter: %p\n", + dprintk(VIDC_ERR, "%s Invalid paramter: %pK\n", __func__, device); return -EINVAL; } @@ -2936,10 +2945,6 @@ int venus_hfi_capability_check(u32 fourcc, u32 width, return -EINVAL; } - if (msm_vp8_low_tier && fourcc == V4L2_PIX_FMT_VP8) { - *max_width = DEFAULT_WIDTH; - *max_height = DEFAULT_HEIGHT; - } if (width > *max_width) { dprintk(VIDC_ERR, "Unsupported width = %u supported max width = %u", @@ -3007,7 +3012,7 @@ static void *venus_hfi_get_device(u32 device_id, int rc = 0; if (!res || !callback) { - dprintk(VIDC_ERR, "Invalid params: %p %p\n", res, callback); + dprintk(VIDC_ERR, "Invalid params: %pK %pK\n", res, callback); return NULL; } @@ -3099,7 +3104,7 @@ int venus_hfi_initialize(struct hfi_device *hdev, u32 device_id, int rc = 0; if (!hdev || !res || !callback) { - dprintk(VIDC_ERR, "Invalid params: %p %p %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK %pK %pK\n", hdev, res, callback); rc = -EINVAL; goto err_venus_hfi_init; diff --git a/drivers/media/platform/msm/vidc/venus_hfi.h b/drivers/media/platform/msm/vidc/venus_hfi.h index a59a053f94c..44cdf313a14 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.h +++ b/drivers/media/platform/msm/vidc/venus_hfi.h @@ -188,6 +188,7 @@ struct venus_hfi_device { struct mutex read_lock; struct mutex write_lock; struct mutex clock_lock; + struct mutex session_lock; msm_vidc_callback callback; struct vidc_mem_addr iface_q_table; struct vidc_mem_addr qdss; diff --git a/drivers/media/platform/msm/vidc/vidc_hfi.c b/drivers/media/platform/msm/vidc/vidc_hfi.c index 46293a6e651..d82681b4519 100644 --- a/drivers/media/platform/msm/vidc/vidc_hfi.c +++ b/drivers/media/platform/msm/vidc/vidc_hfi.c @@ -60,7 +60,7 @@ void vidc_hfi_deinitialize(enum msm_vidc_hfi_type hfi_type, struct hfi_device *hdev) { if (!hdev) { - dprintk(VIDC_ERR, "%s invalid device %p", __func__, hdev); + dprintk(VIDC_ERR, "%s invalid device %pK", __func__, hdev); return; } diff --git a/drivers/media/platform/msm/vidc/vidc_hfi.h b/drivers/media/platform/msm/vidc/vidc_hfi.h index 1e38ae87226..a6646220de4 100644 --- a/drivers/media/platform/msm/vidc/vidc_hfi.h +++ b/drivers/media/platform/msm/vidc/vidc_hfi.h @@ -309,7 +309,7 @@ struct hfi_multi_view_select { }; #define HFI_PRIORITY_LOW 10 -#define HFI_PRIOIRTY_MEDIUM 20 +#define HFI_PRIORITY_MEDIUM 20 #define HFI_PRIORITY_HIGH 30 #define HFI_OUTPUT_ORDER_DISPLAY (HFI_OX_BASE + 0x1) @@ -848,6 +848,7 @@ struct msm_vidc_fw { }; u32 hfi_process_msg_packet(msm_vidc_callback callback, - u32 device_id, struct vidc_hal_msg_pkt_hdr *msg_hdr); + u32 device_id, struct vidc_hal_msg_pkt_hdr *msg_hdr, + struct list_head *sessions, struct mutex *session_lock); #endif diff --git a/drivers/media/platform/msm/vidc/vidc_hfi_api.h b/drivers/media/platform/msm/vidc/vidc_hfi_api.h index 9289b47d5bc..b5b7d0bef8b 100644 --- a/drivers/media/platform/msm/vidc/vidc_hfi_api.h +++ b/drivers/media/platform/msm/vidc/vidc_hfi_api.h @@ -189,6 +189,9 @@ enum hal_property { HAL_PARAM_VENC_VPX_ERROR_RESILIENCE_MODE, HAL_PARAM_VENC_ENABLE_INITIAL_QP, HAL_PARAM_VDEC_CONCEAL_COLOR, + HAL_PARAM_VENC_H264_NAL_SVC_EXT, + HAL_CONFIG_VENC_PERF_MODE, + HAL_PARAM_VENC_HIER_B_MAX_ENH_LAYERS, }; enum hal_domain { @@ -331,6 +334,49 @@ enum hal_h264_level { HAL_UNUSED_H264_LEVEL = 0x10000000, }; +enum hal_hevc_profile { + HAL_HEVC_PROFILE_MAIN = 0x00000001, + HAL_HEVC_PROFILE_MAIN10 = 0x00000002, + HAL_HEVC_PROFILE_MAIN_STILL_PIC = 0x00000004, + HAL_UNUSED_HEVC_PROFILE = 0x10000000, +}; + +enum hal_hevc_level { + HAL_HEVC_MAIN_TIER_LEVEL_1 = 0x10000001, + HAL_HEVC_MAIN_TIER_LEVEL_2 = 0x10000002, + HAL_HEVC_MAIN_TIER_LEVEL_2_1 = 0x10000004, + HAL_HEVC_MAIN_TIER_LEVEL_3 = 0x10000008, + HAL_HEVC_MAIN_TIER_LEVEL_3_1 = 0x10000010, + HAL_HEVC_MAIN_TIER_LEVEL_4 = 0x10000020, + HAL_HEVC_MAIN_TIER_LEVEL_4_1 = 0x10000040, + HAL_HEVC_MAIN_TIER_LEVEL_5 = 0x10000080, + HAL_HEVC_MAIN_TIER_LEVEL_5_1 = 0x10000100, + HAL_HEVC_MAIN_TIER_LEVEL_5_2 = 0x10000200, + HAL_HEVC_MAIN_TIER_LEVEL_6 = 0x10000400, + HAL_HEVC_MAIN_TIER_LEVEL_6_1 = 0x10000800, + HAL_HEVC_MAIN_TIER_LEVEL_6_2 = 0x10001000, + HAL_HEVC_HIGH_TIER_LEVEL_1 = 0x20000001, + HAL_HEVC_HIGH_TIER_LEVEL_2 = 0x20000002, + HAL_HEVC_HIGH_TIER_LEVEL_2_1 = 0x20000004, + HAL_HEVC_HIGH_TIER_LEVEL_3 = 0x20000008, + HAL_HEVC_HIGH_TIER_LEVEL_3_1 = 0x20000010, + HAL_HEVC_HIGH_TIER_LEVEL_4 = 0x20000020, + HAL_HEVC_HIGH_TIER_LEVEL_4_1 = 0x20000040, + HAL_HEVC_HIGH_TIER_LEVEL_5 = 0x20000080, + HAL_HEVC_HIGH_TIER_LEVEL_5_1 = 0x20000100, + HAL_HEVC_HIGH_TIER_LEVEL_5_2 = 0x20000200, + HAL_HEVC_HIGH_TIER_LEVEL_6 = 0x20000400, + HAL_HEVC_HIGH_TIER_LEVEL_6_1 = 0x20000800, + HAL_HEVC_HIGH_TIER_LEVEL_6_2 = 0x20001000, + HAL_UNUSED_HEVC_TIER_LEVEL = 0x80000000, +}; + +enum hal_hevc_tier { + HAL_HEVC_TIER_MAIN = 0x00000001, + HAL_HEVC_TIER_HIGH = 0x00000002, + HAL_UNUSED_HEVC_TIER = 0x10000000, +}; + enum hal_vpx_profile { HAL_VPX_PROFILE_SIMPLE = 0x00000001, HAL_VPX_PROFILE_ADVANCED = 0x00000002, @@ -704,7 +750,7 @@ struct hal_buffer_requirements { enum hal_priority {/* Priority increases with number */ HAL_PRIORITY_LOW = 10, - HAL_PRIOIRTY_MEDIUM = 20, + HAL_PRIORITY_MEDIUM = 20, HAL_PRIORITY_HIGH = 30, HAL_UNUSED_PRIORITY = 0x10000000, }; @@ -911,21 +957,26 @@ enum ltr_mode { HAL_LTR_MODE_PERIODIC, }; -struct hal_ltrmode { - enum ltr_mode ltrmode; - u32 ltrcount; - u32 trustmode; +struct hal_ltr_mode { + enum ltr_mode mode; + u32 count; + u32 trust_mode; }; -struct hal_ltruse { - u32 refltr; - u32 useconstrnt; +struct hal_ltr_use { + u32 ref_ltr; + u32 use_constraint; u32 frames; }; -struct hal_ltrmark { - u32 markframe; +struct hal_ltr_mark { + u32 mark_frame; }; + +struct hal_venc_perf_mode { + u32 mode; +}; + /* HAL Response */ enum command_response { diff --git a/drivers/media/platform/msm/vidc/vidc_hfi_helper.h b/drivers/media/platform/msm/vidc/vidc_hfi_helper.h index b3bee31f0f4..f1bcf5b98d3 100644 --- a/drivers/media/platform/msm/vidc/vidc_hfi_helper.h +++ b/drivers/media/platform/msm/vidc/vidc_hfi_helper.h @@ -177,12 +177,36 @@ #define HFI_DIVX_PROFILE_HT 0x00000008 #define HFI_DIVX_PROFILE_HD 0x00000010 +#define HFI_HEVC_PROFILE_MAIN 0x00000001 +#define HFI_HEVC_PROFILE_MAIN10 0x00000002 +#define HFI_HEVC_PROFILE_MAIN_STILL_PIC 0x00000004 + +#define HFI_HEVC_LEVEL_1 0x00000001 +#define HFI_HEVC_LEVEL_2 0x00000002 +#define HFI_HEVC_LEVEL_21 0x00000004 +#define HFI_HEVC_LEVEL_3 0x00000008 +#define HFI_HEVC_LEVEL_31 0x00000010 +#define HFI_HEVC_LEVEL_4 0x00000020 +#define HFI_HEVC_LEVEL_41 0x00000040 +#define HFI_HEVC_LEVEL_5 0x00000080 +#define HFI_HEVC_LEVEL_51 0x00000100 +#define HFI_HEVC_LEVEL_52 0x00000200 +#define HFI_HEVC_LEVEL_6 0x00000400 +#define HFI_HEVC_LEVEL_61 0x00000800 +#define HFI_HEVC_LEVEL_62 0x00001000 + +#define HFI_HEVC_TIER_MAIN 0x1 +#define HFI_HEVC_TIER_HIGH0 0x2 + #define HFI_BUFFER_INPUT (HFI_COMMON_BASE + 0x1) #define HFI_BUFFER_OUTPUT (HFI_COMMON_BASE + 0x2) #define HFI_BUFFER_OUTPUT2 (HFI_COMMON_BASE + 0x3) #define HFI_BUFFER_INTERNAL_PERSIST (HFI_COMMON_BASE + 0x4) #define HFI_BUFFER_INTERNAL_PERSIST_1 (HFI_COMMON_BASE + 0x5) +#define HFI_VENC_PERFMODE_MAX_QUALITY 0x1 +#define HFI_VENC_PERFMODE_POWER_SAVE 0x2 + struct hfi_buffer_info { u32 buffer_addr; u32 extra_data_addr; @@ -301,7 +325,7 @@ struct hfi_buffer_info { (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x019) #define HFI_PROPERTY_PARAM_VENC_H264_NAL_SVC_EXT \ (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01B) -#define HFI_PROPERTY_PARAM_VENC_H264_LTRMODE \ +#define HFI_PROPERTY_PARAM_VENC_LTRMODE \ (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01C) #define HFI_PROPERTY_PARAM_VENC_VIDEO_FULL_RANGE \ (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01D) @@ -321,6 +345,10 @@ struct hfi_buffer_info { (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x028) #define HFI_PROPERTY_PARAM_VENC_VPX_ERROR_RESILIENCE_MODE \ (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x029) +#define HFI_PROPERTY_PARAM_VENC_HIER_B_MAX_NUM_ENH_LAYER \ + (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x02C) + + #define HFI_PROPERTY_CONFIG_VENC_COMMON_START \ (HFI_DOMAIN_BASE_VENC + HFI_ARCH_COMMON_OFFSET + 0x6000) #define HFI_PROPERTY_CONFIG_VENC_TARGET_BITRATE \ @@ -340,14 +368,17 @@ struct hfi_buffer_info { (HFI_DOMAIN_BASE_VPE + HFI_ARCH_COMMON_OFFSET + 0x7000) #define HFI_PROPERTY_CONFIG_VENC_SYNC_FRAME_SEQUENCE_HEADER \ (HFI_PROPERTY_CONFIG_VENC_COMMON_START + 0x008) -#define HFI_PROPERTY_CONFIG_VENC_H264_MARKLTRFRAME \ +#define HFI_PROPERTY_CONFIG_VENC_MARKLTRFRAME \ (HFI_PROPERTY_CONFIG_VENC_COMMON_START + 0x009) -#define HFI_PROPERTY_CONFIG_VENC_H264_USELTRFRAME \ +#define HFI_PROPERTY_CONFIG_VENC_USELTRFRAME \ (HFI_PROPERTY_CONFIG_VENC_COMMON_START + 0x00A) #define HFI_PROPERTY_CONFIG_VENC_HIER_P_ENH_LAYER \ (HFI_PROPERTY_CONFIG_VENC_COMMON_START + 0x00B) #define HFI_PROPERTY_CONFIG_VENC_LTRPERIOD \ (HFI_PROPERTY_CONFIG_VENC_COMMON_START + 0x00C) +#define HFI_PROPERTY_CONFIG_VENC_PERF_MODE \ + (HFI_PROPERTY_CONFIG_VENC_COMMON_START + 0x00E) + #define HFI_PROPERTY_CONFIG_VPE_COMMON_START \ (HFI_DOMAIN_BASE_VPE + HFI_ARCH_COMMON_OFFSET + 0x8000) #define HFI_PROPERTY_CONFIG_VPE_DEINTERLACE \ @@ -547,20 +578,20 @@ struct hfi_quantization_range { #define HFI_LTR_MODE_MANUAL 0x1 #define HFI_LTR_MODE_PERIODIC 0x2 -struct hfi_ltrmode { - u32 ltrmode; - u32 ltrcount; - u32 trustmode; +struct hfi_ltr_mode { + u32 ltr_mode; + u32 ltr_count; + u32 trust_mode; }; -struct hfi_ltruse { - u32 refltr; - u32 useconstrnt; +struct hfi_ltr_use { + u32 ref_ltr; + u32 use_constrnt; u32 frames; }; -struct hfi_ltrmark { - u32 markframe; +struct hfi_ltr_mark { + u32 mark_frame; }; struct hfi_frame_size { diff --git a/drivers/media/video/videobuf2-core.c b/drivers/media/video/videobuf2-core.c index 668cc733ee2..c58b354af37 100644 --- a/drivers/media/video/videobuf2-core.c +++ b/drivers/media/video/videobuf2-core.c @@ -513,13 +513,6 @@ int vb2_reqbufs(struct vb2_queue *q, struct v4l2_requestbuffers *req) return -EINVAL; } - /* - * If the same number of buffers and memory access method is requested - * then return immediately. - */ - if (q->memory == req->memory && req->count == q->num_buffers) - return 0; - if (req->count == 0 || q->num_buffers != 0 || q->memory != req->memory) { /* * We already have buffers allocated, so first check if they diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index d7f2105d358..9bff185b4bd 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -700,6 +700,12 @@ config FAN48632_BOOST output voltage from a single cell Li-Ion battery, even when the battery voltage is below system cutoff. +config UID_CPUTIME + tristate "Per-UID cpu time statistics" + depends on PROFILING + help + Per UID based cpu time statistics exported to /proc/uid_cputime + source "drivers/misc/c2port/Kconfig" source "drivers/misc/eeprom/Kconfig" source "drivers/misc/cb710/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index e2b77515e2c..34955ab3dcd 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -74,3 +74,4 @@ obj-$(CONFIG_TI_DRV2667) += ti_drv2667.o obj-$(CONFIG_QPNP_MISC) += qpnp-misc.o obj-$(CONFIG_EARJACK_DEBUGGER) += earjack_debugger.o obj-$(CONFIG_FAN48632_BOOST) += fan48632_boost.o +obj-$(CONFIG_UID_CPUTIME) += uid_cputime.o \ No newline at end of file diff --git a/drivers/misc/msm8974_pwm_vibrator.c b/drivers/misc/msm8974_pwm_vibrator.c index 3c67b3e74df..f722e36e5ea 100644 --- a/drivers/misc/msm8974_pwm_vibrator.c +++ b/drivers/misc/msm8974_pwm_vibrator.c @@ -85,7 +85,10 @@ struct timed_vibrator_data { int min_timeout; int ms_time; /* vibrator duration */ int status; /* vibe status */ - int gain; /* default max gain(amp) */ + int vtg_default; /* default gain */ + int vtg_min; /* min gain */ + int vtg_max; /* max gain */ + int vtg_level; /* current gain */ int pwm; /* n-value */ int braking_gain; int braking_ms; @@ -331,11 +334,11 @@ static void msm8974_pwm_vibrator_on(struct work_struct *work) struct timed_vibrator_data *vib = container_of(delayed_work, struct timed_vibrator_data, work_vibrator_on); - int gain = vib->gain; + int vtg_level = vib->vtg_level; int pwm = vib->pwm; - pr_debug("%s: gain = %d pwm = %d\n", __func__, gain, pwm); - msm8974_pwm_vibrator_force_set(vib, gain, pwm); + pr_debug("%s: vtg_level = %d pwm = %d\n", __func__, vtg_level, pwm); + msm8974_pwm_vibrator_force_set(vib, vtg_level, pwm); } static void msm8974_pwm_vibrator_off(struct work_struct *work) @@ -470,9 +473,22 @@ static int vibrator_parse_dt(struct device *dev, } vib->motor_pwm_gpio = ret; - ret = of_property_read_u32(np, "motor-amp", &vib->gain); + ret = of_property_read_u32(np, "vtg_default", &vib->vtg_default); if (ret < 0) { - pr_err("%s: motor-amp failed\n", __func__); + pr_err("%s: vtg_default failed\n", __func__); + return ret; + } + vib->vtg_level = vib->vtg_default; + + ret = of_property_read_u32(np, "vtg_min", &vib->vtg_min); + if (ret < 0) { + pr_err("%s: vtg_min failed\n", __func__); + return ret; + } + + ret = of_property_read_u32(np, "vtg_max", &vib->vtg_max); + if (ret < 0) { + pr_err("%s: vtg_max failed\n", __func__); return ret; } @@ -485,10 +501,10 @@ static int vibrator_parse_dt(struct device *dev, vib->use_vdd_supply = of_property_read_bool(np, "use-vdd-supply"); pr_debug("%s: motor_gpio_en %d, motor_gpio_pwm %d, " - "amp(gain) %d, n_value(pwm) %d vdd %d\n", __func__, + "vtg_level %d, n_value(pwm) %d vdd %d\n", __func__, vib->haptic_en_gpio, vib->motor_pwm_gpio, - vib->gain, vib->pwm, + vib->vtg_level, vib->pwm, vib->use_vdd_supply); ret = of_property_read_u32(np, "vibe-warmup-delay", @@ -518,38 +534,66 @@ static int vibrator_parse_dt(struct device *dev, return 0; } -static ssize_t vibrator_amp_show(struct device *dev, +static ssize_t vibrator_vtg_default_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct timed_output_dev *_dev = dev_get_drvdata(dev); + struct timed_vibrator_data *vib = + container_of(_dev, struct timed_vibrator_data, dev); + + return sprintf(buf, "%d\n", vib->vtg_default); +} + +static ssize_t vibrator_vtg_min_show(struct device *dev, struct device_attribute *attr, char *buf) { struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - int gain = vib->gain; - return sprintf(buf, "%d\n", gain); + return sprintf(buf, "%d\n", vib->vtg_min); } -static ssize_t vibrator_amp_store(struct device *dev, +static ssize_t vibrator_vtg_max_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct timed_output_dev *_dev = dev_get_drvdata(dev); + struct timed_vibrator_data *vib = + container_of(_dev, struct timed_vibrator_data, dev); + + return sprintf(buf, "%d\n", vib->vtg_max); +} + +static ssize_t vibrator_vtg_level_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct timed_output_dev *_dev = dev_get_drvdata(dev); + struct timed_vibrator_data *vib = + container_of(_dev, struct timed_vibrator_data, dev); + + return sprintf(buf, "%d\n", vib->vtg_level); +} + +static ssize_t vibrator_vtg_level_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t size) { struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - long r; - int ret; + int ret, r; - ret = kstrtol(buf, 10, &r); + ret = kstrtoint(buf, 10, &r); if (ret < 0) { pr_err("%s: failed to store value\n", __func__); return ret; } - if (r < 0 || r > 100) { - pr_err("%s: out of range\n", __func__); + if (r < vib->vtg_min || r > vib->vtg_max) { + pr_err("%s: vtg_level %d out of range\n", __func__, r); return -EINVAL; } - vib->gain = r; + vib->vtg_level = r; return size; } @@ -568,17 +612,16 @@ static ssize_t vibrator_pwm_store(struct device *dev, struct device_attribute *a struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - long r; - int ret; + int ret, r; - ret = kstrtol(buf, 10, &r); + ret = kstrtoint(buf, 10, &r); if (ret < 0) { pr_err("%s: failed to store value\n", __func__); return ret; } if (r < 0) { - pr_err("%s: out of range\n", __func__); + pr_err("%s: pwm %d out of range\n", __func__, r); return -EINVAL; } @@ -603,17 +646,16 @@ static ssize_t vibrator_braking_gain_store(struct device *dev, struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - long r; - int ret; + int ret, r; - ret = kstrtol(buf, 10, &r); + ret = kstrtoint(buf, 10, &r); if (ret < 0) { pr_err("%s: failed to store value\n", __func__); return ret; } if (r < 0 || r > 100) { - pr_err("%s: out of range\n", __func__); + pr_err("%s: braking_gain %d out of range\n", __func__, r); return -EINVAL; } @@ -638,17 +680,16 @@ static ssize_t vibrator_braking_ms_store(struct device *dev, struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - long r; - int ret; + int ret, r; - ret = kstrtol(buf, 10, &r); + ret = kstrtoint(buf, 10, &r); if (ret < 0) { pr_err("%s: failed to store value\n", __func__); return ret; } if (r < 0 || r > vib->max_timeout) { - pr_err("%s: out of range\n", __func__); + pr_err("%s: braking_ms %d out of range\n", __func__, r); return -EINVAL; } @@ -673,17 +714,16 @@ static ssize_t vibrator_driving_ms_store(struct device *dev, struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - long r; - int ret; + int ret, r; - ret = kstrtol(buf, 10, &r); + ret = kstrtoint(buf, 10, &r); if (ret < 0) { pr_err("%s: failed to store value\n", __func__); return ret; } if (r < 0 || r > vib->max_timeout) { - pr_err("%s: out of range\n", __func__); + pr_err("%s: driving_ms %d out of range\n", __func__, r); return -EINVAL; } @@ -708,17 +748,16 @@ static ssize_t vibrator_warmup_ms_store(struct device *dev, struct timed_output_dev *_dev = dev_get_drvdata(dev); struct timed_vibrator_data *vib = container_of(_dev, struct timed_vibrator_data, dev); - long r; - int ret; + int ret, r; - ret = kstrtol(buf, 10, &r); + ret = kstrtoint(buf, 10, &r); if (ret < 0) { pr_err("%s: failed to store value\n", __func__); return ret; } if (r < 0 || r > vib->max_timeout) { - pr_err("%s: out of range\n", __func__); + pr_err("%s: warmup_ms %d out of range\n", __func__, r); return -EINVAL; } @@ -728,7 +767,11 @@ static ssize_t vibrator_warmup_ms_store(struct device *dev, } static struct device_attribute vibrator_device_attrs[] = { - __ATTR(amp, S_IRUGO | S_IWUSR, vibrator_amp_show, vibrator_amp_store), + __ATTR(vtg_default, S_IRUGO, vibrator_vtg_default_show, NULL), + __ATTR(vtg_min, S_IRUGO, vibrator_vtg_min_show, NULL), + __ATTR(vtg_max, S_IRUGO, vibrator_vtg_max_show, NULL), + __ATTR(vtg_level, S_IRUGO | S_IWUSR, vibrator_vtg_level_show, + vibrator_vtg_level_store), __ATTR(n_val, S_IRUGO | S_IWUSR, vibrator_pwm_show, vibrator_pwm_store), __ATTR(braking_gain, S_IRUGO | S_IWUSR, vibrator_braking_gain_show, vibrator_braking_gain_store), diff --git a/drivers/misc/qseecom.c b/drivers/misc/qseecom.c index 317219704f3..aee25871e99 100644 --- a/drivers/misc/qseecom.c +++ b/drivers/misc/qseecom.c @@ -81,6 +81,7 @@ enum qseecom_client_handle_type { QSEECOM_LISTENER_SERVICE, QSEECOM_SECURE_SERVICE, QSEECOM_GENERIC, + QSEECOM_UNAVAILABLE_CLIENT_APP, }; enum qseecom_ce_hw_instance { @@ -99,7 +100,7 @@ static DEFINE_MUTEX(clk_access_lock); struct qseecom_registered_listener_list { struct list_head list; struct qseecom_register_listener_req svc; - u8 *sb_reg_req; + uint32_t user_virt_sb_base; u8 *sb_virt; s32 sb_phys; size_t sb_length; @@ -312,6 +313,10 @@ static int qseecom_register_listener(struct qseecom_dev_handle *data, pr_err("copy_from_user failed\n"); return ret; } + if (!access_ok(VERIFY_WRITE, (void __user *)rcvd_lstnr.virt_sb_base, + rcvd_lstnr.sb_size)) + return -EFAULT; + data->listener.id = 0; data->type = QSEECOM_LISTENER_SERVICE; if (!__qseecom_is_svc_unique(data, &rcvd_lstnr)) { @@ -330,6 +335,7 @@ static int qseecom_register_listener(struct qseecom_dev_handle *data, new_entry->svc.listener_id = rcvd_lstnr.listener_id; new_entry->sb_length = rcvd_lstnr.sb_size; + new_entry->user_virt_sb_base = rcvd_lstnr.virt_sb_base; if (__qseecom_set_sb_memory(new_entry, data, &rcvd_lstnr)) { pr_err("qseecom_set_sb_memoryfailed\n"); kzfree(new_entry); @@ -433,7 +439,16 @@ static int qseecom_set_client_mem_param(struct qseecom_dev_handle *data, /* Copy the relevant information needed for loading the image */ if (copy_from_user(&req, (void __user *)argp, sizeof(req))) return -EFAULT; + if (!access_ok(VERIFY_WRITE, (void __user *)req.virt_sb_base, + req.sb_len)) + return -EFAULT; + if ((req.ifd_data_fd <= 0) || (req.virt_sb_base == 0) || + (req.sb_len == 0)) { + pr_err("Inavlid input(s)ion_fd(%d), sb_len(%d), vaddr(0x%x)\n", + req.ifd_data_fd, req.sb_len, req.virt_sb_base); + return -EFAULT; + } /* Get the handle of the shared fd */ data->client.ihandle = ion_import_dma_buf(qseecom.ion_clnt, req.ifd_data_fd); @@ -602,6 +617,7 @@ static int qseecom_load_app(struct qseecom_dev_handle *data, void __user *argp) struct qseecom_command_scm_resp resp; struct qseecom_check_app_ireq req; struct qseecom_load_app_ireq load_req; + bool first_time = false; /* Copy the relevant information needed for loading the image */ if (copy_from_user(&load_img_req, @@ -639,6 +655,7 @@ static int qseecom_load_app(struct qseecom_dev_handle *data, void __user *argp) spin_unlock_irqrestore( &qseecom.registered_app_list_lock, flags); } else { + first_time = true; pr_warn("App (%s) does'nt exist, loading apps for first time\n", (char *)(load_img_req.img_name)); /* Get the handle of the shared fd */ @@ -731,9 +748,16 @@ static int qseecom_load_app(struct qseecom_dev_handle *data, void __user *argp) load_img_req.app_id = app_id; if (copy_to_user(argp, &load_img_req, sizeof(load_img_req))) { pr_err("copy_to_user failed\n"); - kzfree(entry); qsee_disable_clock_vote(data, CLK_SFPB); return -EFAULT; + if (first_time == true) { + spin_lock_irqsave( + &qseecom.registered_app_list_lock, flags); + list_del(&entry->list); + spin_unlock_irqrestore( + &qseecom.registered_app_list_lock, flags); + kzfree(entry); + } } qsee_disable_clock_vote(data, CLK_SFPB); return 0; @@ -844,19 +868,49 @@ static uint32_t __qseecom_uvirt_to_kphys(struct qseecom_dev_handle *data, return data->client.sb_phys + (virt - data->client.user_virt_sb_base); } +static uint32_t __qseecom_uvirt_to_kvirt(struct qseecom_dev_handle *data, + uint32_t virt) +{ + return (uint32_t)data->client.sb_virt + + (virt - data->client.user_virt_sb_base); +} + int __qseecom_process_rpmb_svc_cmd(struct qseecom_dev_handle *data_ptr, struct qseecom_send_svc_cmd_req *req_ptr, struct qseecom_client_send_service_ireq *send_svc_ireq_ptr) { int ret = 0; + void *req_buf = NULL; + if ((req_ptr == NULL) || (send_svc_ireq_ptr == NULL)) { - pr_err("Error with pointer: req_ptr = %p, send_svc_ptr = %p\n", + pr_err("Error with pointer: req_ptr = %pK, send_svc_ptr = %pK\n", req_ptr, send_svc_ireq_ptr); return -EINVAL; } + + if (((uint32_t)req_ptr->cmd_req_buf < + data_ptr->client.user_virt_sb_base) + || ((uint32_t)req_ptr->cmd_req_buf >= + (data_ptr->client.user_virt_sb_base + + data_ptr->client.sb_length))) { + pr_err("cmd buffer address not within shared bufffer\n"); + return -EINVAL; + } + + + if (((uint32_t)req_ptr->resp_buf < data_ptr->client.user_virt_sb_base) + || ((uint32_t)req_ptr->resp_buf >= + (data_ptr->client.user_virt_sb_base + + data_ptr->client.sb_length))){ + pr_err("response buffer address not within shared bufffer\n"); + return -EINVAL; + } + + req_buf = data_ptr->client.sb_virt; + send_svc_ireq_ptr->qsee_cmd_id = req_ptr->cmd_id; send_svc_ireq_ptr->key_type = - ((struct qseecom_rpmb_provision_key *)req_ptr->cmd_req_buf)->key_type; + ((struct qseecom_rpmb_provision_key *)req_buf)->key_type; send_svc_ireq_ptr->req_len = req_ptr->cmd_req_len; send_svc_ireq_ptr->rsp_ptr = (void *)(__qseecom_uvirt_to_kphys(data_ptr, (uint32_t)req_ptr->resp_buf)); @@ -951,9 +1005,22 @@ static int __qseecom_send_cmd(struct qseecom_dev_handle *data, pr_err("cmd buffer or response buffer is null\n"); return -EINVAL; } + if (((uint32_t)req->cmd_req_buf < data->client.user_virt_sb_base) || + ((uint32_t)req->cmd_req_buf >= (data->client.user_virt_sb_base + + data->client.sb_length))) { + pr_err("cmd buffer address not within shared bufffer\n"); + return -EINVAL; + } - if (req->cmd_req_len <= 0 || - req->resp_len <= 0 || + + if (((uint32_t)req->resp_buf < data->client.user_virt_sb_base) || + ((uint32_t)req->resp_buf >= (data->client.user_virt_sb_base + + data->client.sb_length))){ + pr_err("response buffer address not within shared bufffer\n"); + return -EINVAL; + } + + if ((req->cmd_req_len == 0) || (req->resp_len == 0) || req->cmd_req_len > data->client.sb_length || req->resp_len > data->client.sb_length) { pr_err("cmd buffer length or " @@ -1032,7 +1099,7 @@ static int qseecom_send_cmd(struct qseecom_dev_handle *data, void __user *argp) if (ret) return ret; - pr_debug("sending cmd_req->rsp size: %u, ptr: 0x%p\n", + pr_debug("sending cmd_req->rsp size: %u, ptr: 0x%pK\n", req.resp_len, req.resp_buf); return ret; } @@ -1128,6 +1195,7 @@ static int qseecom_send_modfd_cmd(struct qseecom_dev_handle *data, void __user *argp) { int ret = 0; + int i; struct qseecom_send_modfd_cmd_req req; struct qseecom_send_cmd_req send_cmd_req; @@ -1136,11 +1204,47 @@ static int qseecom_send_modfd_cmd(struct qseecom_dev_handle *data, pr_err("copy_from_user failed\n"); return ret; } + if (req.cmd_req_buf == NULL || req.resp_buf == NULL) { + pr_err("cmd buffer or response buffer is null\n"); + return -EINVAL; + } + if (((uint32_t)req.cmd_req_buf < data->client.user_virt_sb_base) || + ((uint32_t)req.cmd_req_buf >= (data->client.user_virt_sb_base + + data->client.sb_length))) { + pr_err("cmd buffer address not within shared bufffer\n"); + return -EINVAL; + } + + if (((uint32_t)req.resp_buf < data->client.user_virt_sb_base) || + ((uint32_t)req.resp_buf >= (data->client.user_virt_sb_base + + data->client.sb_length))){ + pr_err("response buffer address not within shared bufffer\n"); + return -EINVAL; + } + if (req.cmd_req_len == 0 || req.cmd_req_len > data->client.sb_length || + req.resp_len > data->client.sb_length) { + pr_err("cmd or response buffer length not valid\n"); + return -EINVAL; + } + send_cmd_req.cmd_req_buf = req.cmd_req_buf; send_cmd_req.cmd_req_len = req.cmd_req_len; send_cmd_req.resp_buf = req.resp_buf; send_cmd_req.resp_len = req.resp_len; + /* validate offsets */ + for (i = 0; i < MAX_ION_FD; i++) { + if (req.ifd_data[i].cmd_buf_offset >= req.cmd_req_len) { + pr_err("Invalid offset %d = 0x%x\n", + i, req.ifd_data[i].cmd_buf_offset); + return -EINVAL; + } + } + req.cmd_req_buf = (void *)__qseecom_uvirt_to_kvirt(data, + (uint32_t)req.cmd_req_buf); + req.resp_buf = (void *)__qseecom_uvirt_to_kvirt(data, + (uint32_t)req.resp_buf); + ret = __qseecom_update_cmd_buf(&req, false); if (ret) return ret; @@ -1150,7 +1254,7 @@ static int qseecom_send_modfd_cmd(struct qseecom_dev_handle *data, ret = __qseecom_update_cmd_buf(&req, true); if (ret) return ret; - pr_debug("sending cmd_req->rsp size: %u, ptr: 0x%p\n", + pr_debug("sending cmd_req->rsp size: %u, ptr: 0x%pK\n", req.resp_len, req.resp_buf); return ret; } @@ -1689,7 +1793,7 @@ int qseecom_send_command(struct qseecom_handle *handle, void *send_buf, if (ret) return ret; - pr_debug("sending cmd_req->rsp size: %u, ptr: 0x%p\n", + pr_debug("sending cmd_req->rsp size: %u, ptr: 0x%pK\n", req.resp_len, req.resp_buf); return ret; } @@ -2097,6 +2201,9 @@ static int qseecom_unload_external_elf(struct qseecom_dev_handle *data) struct qseecom_unload_app_ireq req; struct cpumask mask; + /* unavailable client app */ + data->type = QSEECOM_UNAVAILABLE_CLIENT_APP; + /* Populate the structure for sending scm call to unload image */ req.qsee_cmd_id = QSEOS_UNLOAD_EXTERNAL_ELF_COMMAND; @@ -2581,8 +2688,15 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, switch (cmd) { case QSEECOM_IOCTL_REGISTER_LISTENER_REQ: { + if (data->type != QSEECOM_GENERIC) { + pr_err("reg lstnr req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } pr_debug("ioctl register_listener_req()\n"); atomic_inc(&data->ioctl_count); + data->type = QSEECOM_LISTENER_SERVICE; ret = qseecom_register_listener(data, argp); atomic_dec(&data->ioctl_count); wake_up_all(&data->abort_wq); @@ -2591,6 +2705,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_UNREGISTER_LISTENER_REQ: { + if ((data->listener.id == 0) || + (data->type != QSEECOM_LISTENER_SERVICE)) { + pr_err("unreg lstnr req: invalid handle (%d) lid(%d)\n", + data->type, data->listener.id); + ret = -EINVAL; + break; + } pr_debug("ioctl unregister_listener_req()\n"); atomic_inc(&data->ioctl_count); ret = qseecom_unregister_listener(data); @@ -2601,6 +2722,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_SEND_CMD_REQ: { + if ((data->client.app_id == 0) || + (data->type != QSEECOM_CLIENT_APP)) { + pr_err("send cmd req: invalid handle (%d) app_id(%d)\n", + data->type, data->client.app_id); + ret = -EINVAL; + break; + } /* Only one client allowed here at a time */ mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); @@ -2613,6 +2741,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_SEND_MODFD_CMD_REQ: { + if ((data->client.app_id == 0) || + (data->type != QSEECOM_CLIENT_APP)) { + pr_err("send mdfd cmd: invalid handle (%d) appid(%d)\n", + data->type, data->client.app_id); + ret = -EINVAL; + break; + } /* Only one client allowed here at a time */ mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); @@ -2625,6 +2760,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_RECEIVE_REQ: { + if ((data->listener.id == 0) || + (data->type != QSEECOM_LISTENER_SERVICE)) { + pr_err("receive req: invalid handle (%d), lid(%d)\n", + data->type, data->listener.id); + ret = -EINVAL; + break; + } atomic_inc(&data->ioctl_count); ret = qseecom_receive_req(data); atomic_dec(&data->ioctl_count); @@ -2634,6 +2776,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_SEND_RESP_REQ: { + if ((data->listener.id == 0) || + (data->type != QSEECOM_LISTENER_SERVICE)) { + pr_err("send resp req: invalid handle (%d), lid(%d)\n", + data->type, data->listener.id); + ret = -EINVAL; + break; + } atomic_inc(&data->ioctl_count); ret = qseecom_send_resp(); atomic_dec(&data->ioctl_count); @@ -2643,6 +2792,14 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_SET_MEM_PARAM_REQ: { + if ((data->type != QSEECOM_CLIENT_APP) && + (data->type != QSEECOM_GENERIC) && + (data->type != QSEECOM_SECURE_SERVICE)) { + pr_err("set mem param req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } ret = qseecom_set_client_mem_param(data, argp); if (ret) pr_err("failed Qqseecom_set_mem_param request: %d\n", @@ -2650,6 +2807,14 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_LOAD_APP_REQ: { + if ((data->type != QSEECOM_GENERIC) && + (data->type != QSEECOM_CLIENT_APP)) { + pr_err("load app req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } + data->type = QSEECOM_CLIENT_APP; mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); if (qseecom.qsee_version > QSEEE_VERSION_00) { @@ -2668,6 +2833,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_UNLOAD_APP_REQ: { + if ((data->client.app_id == 0) || + (data->type != QSEECOM_CLIENT_APP)) { + pr_err("unload app req:invalid handle(%d) app_id(%d)\n", + data->type, data->client.app_id); + ret = -EINVAL; + break; + } mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); ret = qseecom_unload_app(data); @@ -2686,6 +2858,20 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_PERF_ENABLE_REQ:{ + if ((data->type != QSEECOM_GENERIC) && + (data->type != QSEECOM_CLIENT_APP)) { + pr_err("perf enable req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } + if ((data->type == QSEECOM_CLIENT_APP) && + (data->client.app_id == 0)) { + pr_err("perf enable req:invalid handle(%d) appid(%d)\n", + data->type, data->client.app_id); + ret = -EINVAL; + break; + } atomic_inc(&data->ioctl_count); ret = qsee_vote_for_clock(data, CLK_DFAB); if (ret) @@ -2697,6 +2883,20 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_PERF_DISABLE_REQ:{ + if ((data->type != QSEECOM_SECURE_SERVICE) && + (data->type != QSEECOM_CLIENT_APP)) { + pr_err("perf disable req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } + if ((data->type == QSEECOM_CLIENT_APP) && + (data->client.app_id == 0)) { + pr_err("perf disable: invalid handle (%d)app_id(%d)\n", + data->type, data->client.app_id); + ret = -EINVAL; + break; + } atomic_inc(&data->ioctl_count); qsee_disable_clock_vote(data, CLK_DFAB); qsee_disable_clock_vote(data, CLK_SFPB); @@ -2704,6 +2904,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_LOAD_EXTERNAL_ELF_REQ: { + if (data->type != QSEECOM_UNAVAILABLE_CLIENT_APP) { + pr_err("unload ext elf req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } + data->type = QSEECOM_UNAVAILABLE_CLIENT_APP; data->released = true; mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); @@ -2715,6 +2922,12 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_UNLOAD_EXTERNAL_ELF_REQ: { + if (data->type != QSEECOM_UNAVAILABLE_CLIENT_APP) { + pr_err("unload ext elf req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } data->released = true; mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); @@ -2726,6 +2939,7 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_APP_LOADED_QUERY_REQ: { + data->type = QSEECOM_CLIENT_APP; mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); ret = qseecom_query_app_loaded(data, argp); @@ -2739,6 +2953,13 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, qseecom.qsee_version); return -EINVAL; } + data->type = QSEECOM_SECURE_SERVICE; + if (data->type != QSEECOM_GENERIC) { + pr_err("send cmd svc req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); ret = qseecom_send_service_cmd(data, argp); @@ -2747,8 +2968,14 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_CREATE_KEY_REQ: { + if (data->type != QSEECOM_GENERIC) { + pr_err("create key req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } if (qseecom.qsee_version < QSEE_VERSION_05) { - pr_err("Create Key feature not supported in qsee version %u\n", + pr_err("Create Key feature unsupported: qsee ver %u\n", qseecom.qsee_version); return -EINVAL; } @@ -2764,8 +2991,14 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_WIPE_KEY_REQ: { + if (data->type != QSEECOM_GENERIC) { + pr_err("wipe key req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } if (qseecom.qsee_version < QSEE_VERSION_05) { - pr_err("Wipe Key feature not supported in qsee version %u\n", + pr_err("Wipe Key feature unsupported in qsee ver %u\n", qseecom.qsee_version); return -EINVAL; } @@ -2780,6 +3013,12 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_SAVE_PARTITION_HASH_REQ: { + if (data->type != QSEECOM_GENERIC) { + pr_err("save part hash req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } data->released = true; mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); @@ -2789,6 +3028,12 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } case QSEECOM_IOCTL_IS_ES_ACTIVATED_REQ: { + if (data->type != QSEECOM_GENERIC) { + pr_err("ES activated req: invalid handle (%d)\n", + data->type); + ret = -EINVAL; + break; + } data->released = true; mutex_lock(&app_access_lock); atomic_inc(&data->ioctl_count); @@ -2798,6 +3043,7 @@ static long qseecom_ioctl(struct file *file, unsigned cmd, break; } default: + pr_err("Invalid IOCTL: %d\n", cmd); return -EINVAL; } return ret; @@ -2845,6 +3091,8 @@ static int qseecom_release(struct inode *inode, struct file *file) return ret; } break; + case QSEECOM_UNAVAILABLE_CLIENT_APP: + break; default: pr_err("Unsupported clnt_handle_type %d", data->type); diff --git a/drivers/misc/uid_cputime.c b/drivers/misc/uid_cputime.c new file mode 100644 index 00000000000..31682c20f59 --- /dev/null +++ b/drivers/misc/uid_cputime.c @@ -0,0 +1,253 @@ +/* drivers/misc/uid_cputime.c + * + * Copyright (C) 2014 - 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UID_HASH_BITS 10 +DECLARE_HASHTABLE(hash_table, UID_HASH_BITS); + +static DEFINE_MUTEX(uid_lock); +static struct proc_dir_entry *parent; + +struct uid_entry { + uid_t uid; + cputime_t utime; + cputime_t stime; + cputime_t active_utime; + cputime_t active_stime; + unsigned long long active_power; + unsigned long long power; + struct hlist_node hash; +}; + +static struct uid_entry *find_uid_entry(uid_t uid) +{ + struct uid_entry *uid_entry; + struct hlist_node *node; + + hash_for_each_possible(hash_table, uid_entry, node, hash, uid) { + if (uid_entry->uid == uid) + return uid_entry; + } + return NULL; +} + +static struct uid_entry *find_or_register_uid(uid_t uid) +{ + struct uid_entry *uid_entry; + + uid_entry = find_uid_entry(uid); + if (uid_entry) + return uid_entry; + + uid_entry = kzalloc(sizeof(struct uid_entry), GFP_ATOMIC); + if (!uid_entry) + return NULL; + + uid_entry->uid = uid; + + hash_add(hash_table, &uid_entry->hash, uid); + + return uid_entry; +} + +static int uid_stat_show(struct seq_file *m, void *v) +{ + struct uid_entry *uid_entry; + struct task_struct *task, *temp; + struct hlist_node *node; + cputime_t utime; + cputime_t stime; + unsigned long bkt; + + mutex_lock(&uid_lock); + + hash_for_each(hash_table, bkt, node, uid_entry, hash) { + uid_entry->active_stime = 0; + uid_entry->active_utime = 0; + uid_entry->active_power = 0; + } + + read_lock(&tasklist_lock); + do_each_thread(temp, task) { + uid_entry = find_or_register_uid(task_uid(task)); + if (!uid_entry) { + read_unlock(&tasklist_lock); + mutex_unlock(&uid_lock); + pr_err("%s: failed to find the uid_entry for uid %d\n", + __func__, task_uid(task)); + return -ENOMEM; + } + /* if this task is exiting, we have already accounted for the + * time and power. */ + if (task->cpu_power == ULLONG_MAX) + continue; + task_times(task, &utime, &stime); + uid_entry->active_utime += utime; + uid_entry->active_stime += stime; + uid_entry->active_power += task->cpu_power; + } while_each_thread(temp, task); + read_unlock(&tasklist_lock); + + hash_for_each(hash_table, bkt, node, uid_entry, hash) { + cputime_t total_utime = uid_entry->utime + + uid_entry->active_utime; + cputime_t total_stime = uid_entry->stime + + uid_entry->active_stime; + unsigned long long total_power = uid_entry->power + + uid_entry->active_power; + seq_printf(m, "%d: %llu %llu %llu\n", uid_entry->uid, + (unsigned long long)jiffies_to_msecs( + cputime_to_jiffies(total_utime)) * USEC_PER_MSEC, + (unsigned long long)jiffies_to_msecs( + cputime_to_jiffies(total_stime)) * USEC_PER_MSEC, + total_power); + } + + mutex_unlock(&uid_lock); + return 0; +} + +static int uid_stat_open(struct inode *inode, struct file *file) +{ + return single_open(file, uid_stat_show, PDE(inode)->data); +} + +static const struct file_operations uid_stat_fops = { + .open = uid_stat_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int uid_remove_open(struct inode *inode, struct file *file) +{ + return single_open(file, NULL, NULL); +} + +static ssize_t uid_remove_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + struct uid_entry *uid_entry; + struct hlist_node *node, *tmp; + char uids[128]; + char *start_uid, *end_uid = NULL; + long int uid_start = 0, uid_end = 0; + + if (count >= sizeof(uids)) + count = sizeof(uids) - 1; + + if (copy_from_user(uids, buffer, count)) + return -EFAULT; + + uids[count] = '\0'; + end_uid = uids; + start_uid = strsep(&end_uid, "-"); + + if (!start_uid || !end_uid) + return -EINVAL; + + if (kstrtol(start_uid, 10, &uid_start) != 0 || + kstrtol(end_uid, 10, &uid_end) != 0) { + return -EINVAL; + } + + mutex_lock(&uid_lock); + + for (; uid_start <= uid_end; uid_start++) { + hash_for_each_possible_safe(hash_table, uid_entry, node, tmp, + hash, uid_start) { + hash_del(&uid_entry->hash); + kfree(uid_entry); + } + } + + mutex_unlock(&uid_lock); + return count; +} + +static const struct file_operations uid_remove_fops = { + .open = uid_remove_open, + .release = single_release, + .write = uid_remove_write, +}; + +static int process_notifier(struct notifier_block *self, + unsigned long cmd, void *v) +{ + struct task_struct *task = v; + struct uid_entry *uid_entry; + cputime_t utime, stime; + uid_t uid; + + if (!task) + return NOTIFY_OK; + + mutex_lock(&uid_lock); + uid = task_uid(task); + uid_entry = find_or_register_uid(uid); + if (!uid_entry) { + pr_err("%s: failed to find uid %d\n", __func__, uid); + goto exit; + } + + task_times(task, &utime, &stime); + uid_entry->utime += utime; + uid_entry->stime += stime; + uid_entry->power += task->cpu_power; + task->cpu_power = ULLONG_MAX; + +exit: + mutex_unlock(&uid_lock); + return NOTIFY_OK; +} + +static struct notifier_block process_notifier_block = { + .notifier_call = process_notifier, +}; + +static int __init proc_uid_cputime_init(void) +{ + hash_init(hash_table); + + parent = proc_mkdir("uid_cputime", NULL); + if (!parent) { + pr_err("%s: failed to create proc entry\n", __func__); + return -ENOMEM; + } + + proc_create_data("remove_uid_range", S_IWUGO, parent, &uid_remove_fops, + NULL); + + proc_create_data("show_uid_stat", S_IRUGO, parent, &uid_stat_fops, + NULL); + + profile_event_register(PROFILE_TASK_EXIT, &process_notifier_block); + + return 0; +} + +early_initcall(proc_uid_cputime_init); diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c index 8431fed4f06..73fff0d8b36 100644 --- a/drivers/mmc/card/block.c +++ b/drivers/mmc/card/block.c @@ -147,7 +147,9 @@ module_param(perdev_minors, int, 0444); MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device"); static inline int mmc_blk_part_switch(struct mmc_card *card, - struct mmc_blk_data *md); + struct mmc_blk_data *md); +static inline int __mmc_blk_part_switch(struct mmc_card *card, + struct mmc_blk_data *md, unsigned int part_type); static int get_card_status(struct mmc_card *card, u32 *status, int retries); static inline void mmc_blk_clear_packed(struct mmc_queue_req *mqrq) @@ -751,6 +753,74 @@ static struct mmc_blk_ioc_rpmb_data *mmc_blk_ioctl_rpmb_copy_from_user( return ERR_PTR(err); } +/* + * Prepare Dummy mmc request. + */ +static void mmc_dummy_prepare_mrq(struct mmc_card *card, + struct mmc_request *mrq, struct scatterlist *sg, unsigned sg_len, + unsigned dev_addr, unsigned blocks, unsigned blksz, int write) +{ + BUG_ON(!mrq || !mrq->cmd || !mrq->data || !mrq->stop); + if (blocks > 1) { + mrq->cmd->opcode = write ? + MMC_WRITE_MULTIPLE_BLOCK : MMC_READ_MULTIPLE_BLOCK; + } else { + mrq->cmd->opcode = write ? + MMC_WRITE_BLOCK : MMC_READ_SINGLE_BLOCK; + } + + mrq->cmd->arg = dev_addr; + if (!mmc_card_blockaddr(card)) + mrq->cmd->arg <<= 9; + + mrq->cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC; + + if (blocks == 1) + mrq->stop = NULL; + else { + mrq->stop->opcode = MMC_STOP_TRANSMISSION; + mrq->stop->arg = 0; + mrq->stop->flags = MMC_RSP_R1B | MMC_CMD_AC; + } + + mrq->data->blksz = blksz; + mrq->data->blocks = blocks; + mrq->data->flags = write ? MMC_DATA_WRITE : MMC_DATA_READ; + mrq->data->sg = sg; + mrq->data->sg_len = sg_len; + + mmc_set_data_timeout(mrq->data, card); +} + +/* + * Dummy Read Sequence + */ +static int mmc_dummy_read_sequence(struct mmc_card *card, + struct mmc_blk_data *md) +{ + struct mmc_request mrq = {0}; + struct mmc_command cmd = {0}; + struct mmc_command stop = {0}; + struct mmc_data data = {0}; + u8 buffer[512] = {0,}; + int addr; + struct scatterlist sg; + + for (addr = 0; addr <= 1; addr++) { + mrq.cmd = &cmd; + mrq.data = &data; + mrq.stop = &stop; + sg_init_one(&sg, buffer, 512); + mmc_dummy_prepare_mrq(card, &mrq, &sg, 1, addr*0x20, 1, 512, 0); + mmc_wait_for_req(card->host, &mrq); + if (cmd.error) + return cmd.error; + if (data.error) + return data.error; + } + return 0; +} + static int mmc_blk_ioctl_rpmb_cmd(struct block_device *bdev, struct mmc_ioc_rpmb __user *ic_ptr) { @@ -882,6 +952,22 @@ static int mmc_blk_ioctl_rpmb_cmd(struct block_device *bdev, __func__, status, err); } + /* Some eMMC cards requires workaround */ + if (card->quirks & MMC_QUIRK_BLK_NEED_DUMMY_READ) { + /* + * Switch Part to Main area + */ + err = __mmc_blk_part_switch(card, md, 0); + if (err) + goto cmd_rel_host; + /* + * Dummy Read Sequence + */ + err = mmc_dummy_read_sequence(card, md); + if (err) + goto cmd_rel_host; + } + cmd_rel_host: mmc_release_host(card->host); mmc_rpm_release(card->host, &card->dev); @@ -929,21 +1015,21 @@ static const struct block_device_operations mmc_bdops = { #endif }; -static inline int mmc_blk_part_switch(struct mmc_card *card, - struct mmc_blk_data *md) +static inline int __mmc_blk_part_switch(struct mmc_card *card, + struct mmc_blk_data *md, unsigned int part_type) { int ret; struct mmc_blk_data *main_md = mmc_get_drvdata(card); - if ((main_md->part_curr == md->part_type) && - (card->part_curr == md->part_type)) + if ((main_md->part_curr == part_type) && + (card->part_curr == part_type)) return 0; if (mmc_card_mmc(card)) { u8 part_config = card->ext_csd.part_config; part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK; - part_config |= md->part_type; + part_config |= part_type; ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_PART_CONFIG, part_config, @@ -952,13 +1038,19 @@ static inline int mmc_blk_part_switch(struct mmc_card *card, return ret; card->ext_csd.part_config = part_config; - card->part_curr = md->part_type; + card->part_curr = part_type; } main_md->part_curr = md->part_type; return 0; } +static inline int mmc_blk_part_switch(struct mmc_card *card, + struct mmc_blk_data *md) +{ + return __mmc_blk_part_switch(card, md, md->part_type); +} + static u32 mmc_sd_num_wr_blocks(struct mmc_card *card) { int err; @@ -2668,10 +2760,24 @@ static int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req) if (card->host->areq) mmc_blk_issue_rw_rq(mq, NULL); if (req->cmd_flags & REQ_SECURE && - !(card->quirks & MMC_QUIRK_SEC_ERASE_TRIM_BROKEN)) + !(card->quirks & MMC_QUIRK_SEC_ERASE_TRIM_BROKEN)) { ret = mmc_blk_issue_secdiscard_rq(mq, req); - else + if (card->quirks & MMC_QUIRK_BLK_NEED_DUMMY_READ) { + /* + * Dummy Read Sequence + */ + int err; + + err = mmc_dummy_read_sequence(card, md); + if (err) { + pr_warn("%s: failed in mmc_dummy_read_sequence " + "after mmc_blk_issue_secdiscard_rq %d\n", + md->disk->disk_name, err); + } + } + } else { ret = mmc_blk_issue_discard_rq(mq, req); + } } else if (req && req->cmd_flags & REQ_FLUSH) { /* complete ongoing async transfer before issuing flush */ if (card->host->areq) @@ -3039,6 +3145,7 @@ static int mmc_add_disk(struct mmc_blk_data *md) #define CID_MANFID_TOSHIBA 0x11 #define CID_MANFID_MICRON 0x13 #define CID_MANFID_SAMSUNG 0x15 +#define CID_MANFID_SANDISK_SEM 0x45 static const struct mmc_fixup blk_fixups[] = { @@ -3101,6 +3208,14 @@ static const struct mmc_fixup blk_fixups[] = MMC_FIXUP("VZL00M", CID_MANFID_SAMSUNG, CID_OEMID_ANY, add_quirk_mmc, MMC_QUIRK_SEC_ERASE_TRIM_BROKEN), + /* + * Some devices have issues that requires dummy read + */ + MMC_FIXUP("SEM16G", CID_MANFID_SANDISK_SEM, CID_OEMID_ANY, add_quirk_mmc, + MMC_QUIRK_BLK_NEED_DUMMY_READ), + MMC_FIXUP("SEM32G", CID_MANFID_SANDISK_SEM, CID_OEMID_ANY, add_quirk_mmc, + MMC_QUIRK_BLK_NEED_DUMMY_READ), + END_FIXUP }; diff --git a/drivers/mmc/card/mmc_block_test.c b/drivers/mmc/card/mmc_block_test.c index 7a4d19e37bc..e68ece0d9c1 100644 --- a/drivers/mmc/card/mmc_block_test.c +++ b/drivers/mmc/card/mmc_block_test.c @@ -2286,6 +2286,9 @@ static ssize_t send_write_packing_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -2384,6 +2387,9 @@ static ssize_t err_check_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -2492,6 +2498,9 @@ static ssize_t send_invalid_packed_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -2606,6 +2615,9 @@ static ssize_t write_packing_control_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -2731,6 +2743,9 @@ static ssize_t bkops_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -2819,6 +2834,9 @@ static ssize_t long_sequential_read_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -2978,6 +2996,9 @@ static ssize_t long_sequential_write_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, @@ -3051,6 +3072,9 @@ static ssize_t new_req_notification_test_read(struct file *file, size_t count, loff_t *offset) { + if (!access_ok(VERIFY_WRITE, buffer, count)) + return count; + memset((void *)buffer, 0, count); snprintf(buffer, count, diff --git a/drivers/mmc/core/debugfs.c b/drivers/mmc/core/debugfs.c index 903decf30d6..4ec89410d2d 100644 --- a/drivers/mmc/core/debugfs.c +++ b/drivers/mmc/core/debugfs.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -392,6 +393,9 @@ static ssize_t mmc_wr_pack_stats_read(struct file *filp, char __user *ubuf, if (!card) return cnt; + if (!access_ok(VERIFY_WRITE, ubuf, cnt)) + return cnt; + if (!card->wr_pack_stats.print_in_read) return 0; @@ -532,6 +536,9 @@ static ssize_t mmc_wr_pack_stats_write(struct file *filp, if (!card) return cnt; + if (!access_ok(VERIFY_READ, ubuf, cnt)) + return cnt; + sscanf(ubuf, "%d", &value); if (value) { mmc_blk_init_packed_statistics(card); @@ -571,6 +578,9 @@ static ssize_t mmc_bkops_stats_read(struct file *filp, char __user *ubuf, if (!card) return cnt; + if (!access_ok(VERIFY_WRITE, ubuf, cnt)) + return cnt; + bkops_stats = &card->bkops_info.bkops_stats; if (!bkops_stats->print_stats) @@ -637,6 +647,9 @@ static ssize_t mmc_bkops_stats_write(struct file *filp, if (!card) return cnt; + if (!access_ok(VERIFY_READ, ubuf, cnt)) + return cnt; + bkops_stats = &card->bkops_info.bkops_stats; sscanf(ubuf, "%d", &value); diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index 21d7151fb0a..e0890534ae3 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -2177,7 +2177,7 @@ int ppp_register_net_channel(struct net *net, struct ppp_channel *chan) pch->ppp = NULL; pch->chan = chan; - pch->chan_net = net; + pch->chan_net = get_net(net); chan->ppp = pch; init_ppp_file(&pch->file, CHANNEL); pch->file.hdrlen = chan->hdrlen; @@ -2274,6 +2274,8 @@ ppp_unregister_channel(struct ppp_channel *chan) spin_lock_bh(&pn->all_channels_lock); list_del(&pch->list); spin_unlock_bh(&pn->all_channels_lock); + put_net(pch->chan_net); + pch->chan_net = NULL; pch->file.dead = 1; wake_up_interruptible(&pch->file.rwait); diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index b42050c7633..a63b756886d 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -1514,6 +1514,13 @@ usbnet_probe (struct usb_interface *udev, const struct usb_device_id *prod) if (info->unbind) info->unbind (dev, udev); out1: + /* subdrivers must undo all they did in bind() if they + * fail it, but we may fail later and a deferred kevent + * may trigger an error resubmitting itself and, worse, + * schedule a timer. So we kill it all just in case. + */ + cancel_work_sync(&dev->kevent); + del_timer_sync(&dev->delay); free_netdev(net); out: usb_put_dev(xdev); diff --git a/drivers/net/wireless/bcmdhd/Kconfig b/drivers/net/wireless/bcmdhd/Kconfig index 9c633e6b87a..2744dc5c343 100644 --- a/drivers/net/wireless/bcmdhd/Kconfig +++ b/drivers/net/wireless/bcmdhd/Kconfig @@ -31,14 +31,6 @@ config BCMDHD_NVRAM_PATH ---help--- Path to the calibration file. -config BCMDHD_WEXT - bool "Enable WEXT support" - depends on BCMDHD && CFG80211 = n - select WIRELESS_EXT - select WEXT_PRIV - help - Enables WEXT support - config DHD_USE_STATIC_BUF bool "Enable memory preallocation" depends on BCMDHD diff --git a/drivers/net/wireless/bcmdhd/Makefile b/drivers/net/wireless/bcmdhd/Makefile index 7b209f92d5d..484262d6300 100644 --- a/drivers/net/wireless/bcmdhd/Makefile +++ b/drivers/net/wireless/bcmdhd/Makefile @@ -37,7 +37,7 @@ endif ifneq ($(CONFIG_CFG80211),) bcmdhd-objs += wl_cfg80211.o wl_cfgp2p.o wl_linux_mon.o dhd_cfg80211.o wl_cfgvendor.o DHDCFLAGS += -DWL_CFG80211 -DWL_CFG80211_STA_EVENT -DWL_ENABLE_P2P_IF -DHDCFLAGS += -DGSCAN_SUPPORT -DWL_VENDOR_EXT_SUPPORT +DHDCFLAGS += -DWL_VENDOR_EXT_SUPPORT DHDCFLAGS += -DWL_IFACE_COMB_NUM_CHANNELS DHDCFLAGS += -DCUSTOM_ROAM_TRIGGER_SETTING=-75 DHDCFLAGS += -DCUSTOM_ROAM_DELTA_SETTING=10 @@ -52,10 +52,17 @@ ifeq ($(CONFIG_BCMDHD),m) EXTRA_LDFLAGS += --strip-debug endif +#Gscan +DHDCFLAGS += -DGSCAN_SUPPORT + DHDCFLAGS += -DCUSTOM_TDLS_RSSI_THRESHOLD_HIGH=-60 DHDCFLAGS += -DCUSTOM_TDLS_RSSI_THRESHOLD_LOW=-70 DHDCFLAGS += -DCUSTOM_TDLS_IDLE_MODE_SETTING=40000 +# debug info + DHDCFLAGS += -DDHD_WAKE_STATUS -DDHD_WAKE_RX_STATUS + DHDCFLAGS += -DDHD_WAKE_EVENT_STATUS + ######################### # Chip dependent feature ######################### diff --git a/drivers/net/wireless/bcmdhd/bcmevent.c b/drivers/net/wireless/bcmdhd/bcmevent.c index 71995f770da..77238037b24 100644 --- a/drivers/net/wireless/bcmdhd/bcmevent.c +++ b/drivers/net/wireless/bcmdhd/bcmevent.c @@ -2,13 +2,13 @@ * bcmevent read-only data shared by kernel or app layers * * Copyright (C) 1999-2013, Broadcom Corporation - * + * * Unless you and Broadcom execute a separate written software license * agreement governing use of this software, this software is licensed to you * under the terms of the GNU General Public License version 2 (the "GPL"), * available at http://www.broadcom.com/licenses/GPLv2.php, with the * following added to such license: - * + * * As a special exception, the copyright holders of this software give you * permission to link this software with independent modules, and to copy and * distribute the resulting executable under terms of your choice, provided that @@ -16,7 +16,7 @@ * the license of that module. An independent module is a module which is not * derived from this software. The special exception does not apply to any * modifications of the software. - * + * * Notwithstanding the above, under no circumstances may you combine this * software in any way with any other Broadcom software provided under a license * other than the GPL, without Broadcom's express prior written consent. @@ -25,11 +25,14 @@ #include #include +#include #include #include #include +#include +#include -#if WLC_E_LAST != 139 +#if WLC_E_LAST != 144 #error "You need to add an entry to bcmevent_names[] for the new event" #endif @@ -155,8 +158,146 @@ const bcmevent_name_t bcmevent_names[] = { { WLC_E_CCA_CHAN_QUAL, "CCA_BASED_CHANNEL_QUALITY" }, #ifdef GSCAN_SUPPORT { WLC_E_PFN_GSCAN_FULL_RESULT, "PFN_GSCAN_FULL_RESULT"}, - { WLC_E_PFN_SWC, "PFN_SIGNIFICANT_WIFI_CHANGE"} + { WLC_E_PFN_SWC, "PFN_SIGNIFICANT_WIFI_CHANGE"}, + { WLC_E_PFN_SSID_EXT, "PFN_SSID_EXT"}, + { WLC_E_ROAM_EXP_EVENT, "ROAM_EXP_EVENT"}, #endif /* GSCAN_SUPPORT */ }; const int bcmevent_names_size = ARRAYSIZE(bcmevent_names); + +const char *bcmevent_get_name(uint event_type) +{ + const char *event_name = NULL; + + uint idx; + for (idx = 0; idx < bcmevent_names_size; idx++) { + + if (bcmevent_names[idx].event == event_type) { + event_name = bcmevent_names[idx].name; + break; + } + } + + /* if we find an event name in the array, return it. + * otherwise return unknown string. + */ + return ((event_name) ? event_name : "Unknown Event"); +} + +/* + * Validate if the event is proper and if valid copy event header to event. + * If proper event pointer is passed, to just validate, pass NULL to event. + * + * Return values are + * BCME_OK - It is a BRCM event or BRCM dongle event + * BCME_NOTFOUND - Not BRCM, not an event, may be okay + * BCME_BADLEN - Bad length, should not process, just drop + */ +int +is_wlc_event_frame(void *pktdata, uint pktlen, uint16 exp_usr_subtype, + bcm_event_msg_u_t *out_event) +{ + uint16 len; + uint16 subtype; + uint16 usr_subtype; + bcm_event_t *bcm_event; + uint8 *pktend; + int err = BCME_OK; + + pktend = (uint8 *)pktdata + pktlen; + bcm_event = (bcm_event_t *)pktdata; + + /* only care about 16-bit subtype / length versions */ + if ((uint8 *)&bcm_event->bcm_hdr < pktend) { + uint8 short_subtype = *(uint8 *)&bcm_event->bcm_hdr; + if (!(short_subtype & 0x80)) { + err = BCME_NOTFOUND; + goto done; + } + } + + /* must have both ether_header and bcmeth_hdr */ + if (pktlen < OFFSETOF(bcm_event_t, event)) { + err = BCME_BADLEN; + goto done; + } + + /* check length in bcmeth_hdr */ + len = ntoh16_ua((void *)&bcm_event->bcm_hdr.length); + if (((uint8 *)&bcm_event->bcm_hdr.version + len) > pktend) { + err = BCME_BADLEN; + goto done; + } + + /* match on subtype, oui and usr subtype for BRCM events */ + subtype = ntoh16_ua((void *)&bcm_event->bcm_hdr.subtype); + if (subtype != BCMILCP_SUBTYPE_VENDOR_LONG) { + err = BCME_NOTFOUND; + goto done; + } + + if (bcmp(BRCM_OUI, &bcm_event->bcm_hdr.oui[0], DOT11_OUI_LEN)) { + err = BCME_NOTFOUND; + goto done; + } + + /* if it is a bcm_event or bcm_dngl_event_t, validate it */ + usr_subtype = ntoh16_ua((void *)&bcm_event->bcm_hdr.usr_subtype); + switch (usr_subtype) { + case BCMILCP_BCM_SUBTYPE_EVENT: + if (pktlen < sizeof(bcm_event_t)) { + err = BCME_BADLEN; + goto done; + } + + len = sizeof(bcm_event_t) + ntoh32_ua((void *)&bcm_event->event.datalen); + if ((uint8 *)pktdata + len > pktend) { + err = BCME_BADLEN; + goto done; + } + + if (exp_usr_subtype && (exp_usr_subtype != usr_subtype)) { + err = BCME_NOTFOUND; + goto done; + } + + if (out_event) { + /* ensure BRCM event pkt aligned */ + memcpy(&out_event->event, &bcm_event->event, sizeof(wl_event_msg_t)); + } + + break; + case BCMILCP_BCM_SUBTYPE_DNGLEVENT: + if (pktlen < sizeof(bcm_dngl_event_t)) { + err = BCME_BADLEN; + goto done; + } + + len = sizeof(bcm_dngl_event_t) + + ntoh16_ua((void *)&((bcm_dngl_event_t *)pktdata)->dngl_event.datalen); + if ((uint8 *)pktdata + len > pktend) { + err = BCME_BADLEN; + goto done; + } + + if (exp_usr_subtype && (exp_usr_subtype != usr_subtype)) { + err = BCME_NOTFOUND; + goto done; + } + + if (out_event) { + /* ensure BRCM dngl event pkt aligned */ + memcpy(&out_event->dngl_event, &((bcm_dngl_event_t *)pktdata)->dngl_event, + sizeof(bcm_dngl_event_msg_t)); + } + + break; + default: + err = BCME_NOTFOUND; + goto done; + } + +done: + return err; +} diff --git a/drivers/net/wireless/bcmdhd/bcmsdh.c b/drivers/net/wireless/bcmdhd/bcmsdh.c index b2ffb18d64e..bb264e89943 100644 --- a/drivers/net/wireless/bcmdhd/bcmsdh.c +++ b/drivers/net/wireless/bcmdhd/bcmsdh.c @@ -48,18 +48,6 @@ #define SDIOH_API_ACCESS_RETRY_LIMIT 2 const uint bcmsdh_msglevel = BCMSDH_ERROR_VAL; -/** - * BCMSDH API context - */ -struct bcmsdh_info -{ - bool init_success; /* underlying driver successfully attached */ - void *sdioh; /* handler for sdioh */ - uint32 vendevid; /* Target Vendor and Device ID on SD bus */ - osl_t *osh; - bool regfail; /* Save status of last reg_read/reg_write call */ - uint32 sbwad; /* Save backplane window address */ -}; /* local copy of bcm sd handler */ bcmsdh_info_t * l_bcmsdh = NULL; diff --git a/drivers/net/wireless/bcmdhd/bcmsdh_linux.c b/drivers/net/wireless/bcmdhd/bcmsdh_linux.c index 4b0b5f371f5..0cd2430d2bb 100644 --- a/drivers/net/wireless/bcmdhd/bcmsdh_linux.c +++ b/drivers/net/wireless/bcmdhd/bcmsdh_linux.c @@ -2,13 +2,13 @@ * SDIO access interface for drivers - linux specific (pci only) * * Copyright (C) 1999-2013, Broadcom Corporation - * + * * Unless you and Broadcom execute a separate written software license * agreement governing use of this software, this software is licensed to you * under the terms of the GNU General Public License version 2 (the "GPL"), * available at http://www.broadcom.com/licenses/GPLv2.php, with the * following added to such license: - * + * * As a special exception, the copyright holders of this software give you * permission to link this software with independent modules, and to copy and * distribute the resulting executable under terms of your choice, provided that @@ -16,7 +16,7 @@ * the license of that module. An independent module is a module which is not * derived from this software. The special exception does not apply to any * modifications of the software. - * + * * Notwithstanding the above, under no circumstances may you combine this * software in any way with any other Broadcom software provided under a license * other than the GPL, without Broadcom's express prior written consent. @@ -35,6 +35,9 @@ #include #include +#ifdef DHD_WAKE_STATUS +#include +#endif #include #include @@ -47,7 +50,7 @@ extern void dhdsdio_isr(void * args); #include #include #include -#endif +#endif /** * SDIO Host Controller info @@ -71,7 +74,7 @@ struct bcmsdh_hc { bool oob_irq_enable_flag; #if defined(OOB_INTR_ONLY) spinlock_t irq_lock; -#endif +#endif }; static bcmsdh_hc_t *sdhcinfo = NULL; @@ -146,11 +149,11 @@ EXPORT_SYMBOL(bcmsdh_remove); /* forward declarations */ static int __devinit bcmsdh_probe(struct device *dev); static int __devexit bcmsdh_remove(struct device *dev); -#endif +#endif #if !defined(BCMLXSDMMC) static -#endif +#endif int bcmsdh_probe(struct device *dev) { osl_t *osh = NULL; @@ -160,7 +163,7 @@ int bcmsdh_probe(struct device *dev) #if !defined(BCMLXSDMMC) && defined(BCMPLATFORM_BUS) struct platform_device *pdev; struct resource *r; -#endif +#endif int irq = 0; uint32 vendevid; unsigned long irq_flags = 0; @@ -171,7 +174,7 @@ int bcmsdh_probe(struct device *dev) irq = platform_get_irq(pdev, 0); if (!r || irq < 0) return -ENXIO; -#endif +#endif #if defined(OOB_INTR_ONLY) #ifdef HW_OOB @@ -187,7 +190,7 @@ int bcmsdh_probe(struct device *dev) SDLX_MSG(("%s: Host irq is not defined\n", __FUNCTION__)); goto err; } -#endif +#endif /* allocate SDIO Host Controller state info */ if (!(osh = osl_attach(dev, PCI_BUS, FALSE))) { SDLX_MSG(("%s: osl_attach failed\n", __FUNCTION__)); @@ -216,7 +219,7 @@ int bcmsdh_probe(struct device *dev) SDLX_MSG(("%s: bcmsdh_attach failed\n", __FUNCTION__)); goto err; } -#endif +#endif sdhc->sdh = sdh; sdhc->oob_irq = irq; sdhc->oob_flags = irq_flags; @@ -224,7 +227,7 @@ int bcmsdh_probe(struct device *dev) sdhc->oob_irq_enable_flag = FALSE; #if defined(OOB_INTR_ONLY) spin_lock_init(&sdhc->irq_lock); -#endif +#endif /* chain SDIO Host Controller info together */ sdhc->next = sdhcinfo; @@ -260,9 +263,34 @@ int bcmsdh_probe(struct device *dev) return -ENODEV; } +#ifdef DHD_WAKE_STATUS +int bcmsdh_get_total_wake(bcmsdh_info_t *bcmsdh) +{ + return bcmsdh->total_wake_count; +} + +int bcmsdh_set_get_wake(int flag) +{ + unsigned long flags; + int ret = 0; + + if (sdhcinfo) { + bcmsdh_info_t *bcmsdh = sdhcinfo->sdh; + spin_lock_irqsave(&sdhcinfo->irq_lock, flags); + + ret = bcmsdh->pkt_wake; + bcmsdh->total_wake_count += flag; + bcmsdh->pkt_wake = flag; + + spin_unlock_irqrestore(&sdhcinfo->irq_lock, flags); + } + return ret; +} +#endif + #if !defined(BCMLXSDMMC) static -#endif +#endif int bcmsdh_remove(struct device *dev) { bcmsdh_hc_t *sdhc, *prev; @@ -311,7 +339,7 @@ int bcmsdh_remove(struct device *dev) #if !defined(BCMLXSDMMC) || defined(OOB_INTR_ONLY) dev_set_drvdata(dev, NULL); -#endif +#endif return 0; } @@ -698,7 +726,7 @@ bool bcmsdh_is_oob_intr_registered(void) else return FALSE; } -#endif +#endif #if defined(BCMLXSDMMC) void *bcmsdh_get_drvdata(void) @@ -707,6 +735,13 @@ void *bcmsdh_get_drvdata(void) return NULL; return dev_get_drvdata(sdhcinfo->dev); } + +int bcmsdh_get_irq(void) +{ + if (!sdhcinfo) + return -1; + return sdhcinfo->oob_irq; +} #endif /* Module parameters specific to each host-controller driver */ diff --git a/drivers/net/wireless/bcmdhd/bcmsdh_sdmmc_linux.c b/drivers/net/wireless/bcmdhd/bcmsdh_sdmmc_linux.c index 17e4177eb2c..dc95c86ad02 100644 --- a/drivers/net/wireless/bcmdhd/bcmsdh_sdmmc_linux.c +++ b/drivers/net/wireless/bcmdhd/bcmsdh_sdmmc_linux.c @@ -2,13 +2,13 @@ * BCMSDH Function Driver for the native SDIO/MMC driver in the Linux Kernel * * Copyright (C) 1999-2013, Broadcom Corporation - * + * * Unless you and Broadcom execute a separate written software license * agreement governing use of this software, this software is licensed to you * under the terms of the GNU General Public License version 2 (the "GPL"), * available at http://www.broadcom.com/licenses/GPLv2.php, with the * following added to such license: - * + * * As a special exception, the copyright holders of this software give you * permission to link this software with independent modules, and to copy and * distribute the resulting executable under terms of your choice, provided that @@ -16,7 +16,7 @@ * the license of that module. An independent module is a module which is not * derived from this software. The special exception does not apply to any * modifications of the software. - * + * * Notwithstanding the above, under no circumstances may you combine this * software in any way with any other Broadcom software provided under a license * other than the GPL, without Broadcom's express prior written consent. @@ -37,6 +37,11 @@ #include #include +#ifdef CONFIG_PARTIALRESUME +#include +#include "wl_android.h" +#endif + #if !defined(SDIO_VENDOR_ID_BROADCOM) #define SDIO_VENDOR_ID_BROADCOM 0x02d0 #endif /* !defined(SDIO_VENDOR_ID_BROADCOM) */ @@ -76,7 +81,6 @@ #ifdef WL_CFG80211 extern void wl_cfg80211_set_parent_dev(void *dev); #endif - extern void sdioh_sdmmc_devintr_off(sdioh_info_t *sd); extern void sdioh_sdmmc_devintr_on(sdioh_info_t *sd); extern int dhd_os_check_wakelock(void *dhdp); @@ -190,6 +194,9 @@ static const struct sdio_device_id bcmsdh_sdmmc_ids[] = { MODULE_DEVICE_TABLE(sdio, bcmsdh_sdmmc_ids); #if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39)) && defined(CONFIG_PM) + +int bcmsdh_get_irq(void); + static int bcmsdh_sdmmc_suspend(struct device *pdev) { struct sdio_func *func = dev_to_sdio_func(pdev); @@ -217,7 +224,10 @@ static int bcmsdh_sdmmc_suspend(struct device *pdev) } #if defined(OOB_INTR_ONLY) bcmsdh_oob_intr_set(0); -#endif +#ifdef CONFIG_PARTIALRESUME + wifi_process_partial_resume(WIFI_PR_INIT); +#endif +#endif dhd_mmc_suspend = TRUE; smp_mb(); @@ -228,14 +238,26 @@ static int bcmsdh_sdmmc_resume(struct device *pdev) { #if defined(OOB_INTR_ONLY) struct sdio_func *func = dev_to_sdio_func(pdev); -#endif +#endif + int wakeup = 0; +#if defined(CONFIG_PARTIALRESUME) || defined(DHD_WAKE_STATUS) + wakeup = check_wakeup_reason(bcmsdh_get_irq()); +#endif /* CONFIG_PARTIALRESUME || DHD_WAKE_STATUS */ sd_trace(("%s Enter\n", __FUNCTION__)); dhd_mmc_suspend = FALSE; #if defined(OOB_INTR_ONLY) - if ((func->num == 2) && dhd_os_check_if_up(bcmsdh_get_drvdata())) + if ((func->num == 2) && dhd_os_check_if_up(bcmsdh_get_drvdata())) { + if (wakeup) { +#ifdef CONFIG_PARTIALRESUME + wifi_process_partial_resume(WIFI_PR_NOTIFY_RESUME); +#endif +#ifdef DHD_WAKE_STATUS + bcmsdh_set_get_wake(1); +#endif + } bcmsdh_oob_intr_set(1); -#endif - + } +#endif smp_mb(); return 0; } diff --git a/drivers/net/wireless/bcmdhd/common/include/proto/bcmeth.h b/drivers/net/wireless/bcmdhd/common/include/proto/bcmeth.h index 94ba2469c14..ce255c1c503 100644 --- a/drivers/net/wireless/bcmdhd/common/include/proto/bcmeth.h +++ b/drivers/net/wireless/bcmdhd/common/include/proto/bcmeth.h @@ -90,6 +90,7 @@ */ /* #define BCMILCP_BCM_SUBTYPE_EAPOL 3 */ #define BCMILCP_BCM_SUBTYPE_DPT 4 +#define BCMILCP_BCM_SUBTYPE_DNGLEVENT 5 #define BCMILCP_BCM_SUBTYPEHDR_MINLENGTH 8 #define BCMILCP_BCM_SUBTYPEHDR_VERSION 0 diff --git a/drivers/net/wireless/bcmdhd/common/include/proto/bcmevent.h b/drivers/net/wireless/bcmdhd/common/include/proto/bcmevent.h index 1e5b1603ac3..bae3320e431 100644 --- a/drivers/net/wireless/bcmdhd/common/include/proto/bcmevent.h +++ b/drivers/net/wireless/bcmdhd/common/include/proto/bcmevent.h @@ -40,6 +40,7 @@ #endif /* #include -- TODO: req., excluded to overwhelming coupling (break up ethernet.h) */ #include +#include /* This marks the start of a packed structure section. */ #include @@ -94,6 +95,17 @@ typedef BWL_PRE_PACKED_STRUCT struct bcm_event { /* data portion follows */ } BWL_POST_PACKED_STRUCT bcm_event_t; +/* + * used by host event + * Note: If additional event types are added, it should come on is_wlc_event_frame() as well. + */ +typedef union bcm_event_msg_u { + wl_event_msg_t event; + bcm_dngl_event_msg_t dngl_event; + + /* add new event here */ +} bcm_event_msg_u_t; + #define BCM_MSG_LEN (sizeof(bcm_event_t) - sizeof(bcmeth_hdr_t) - sizeof(struct ether_header)) /* Event messages */ @@ -230,8 +242,15 @@ typedef BWL_PRE_PACKED_STRUCT struct bcm_event { #define WLC_E_PFN_GSCAN_FULL_RESULT 134 /* Full probe/beacon (IEs etc) results */ #define WLC_E_PFN_SWC 135 /* Significant change in rssi of bssids being tracked */ #define WLC_E_PFN_SCAN_COMPLETE 138 /* PFN completed scan of network list */ -#define WLC_E_LAST 139 /* highest val + 1 for range checking */ +#define WLC_E_PFN_SSID_EXT 142 /* SSID EXT event */ +#define WLC_E_ROAM_EXP_EVENT 143 /* Expanded roam event */ +#define WLC_E_LAST 144 /* highest val + 1 for range checking */ + +extern const char *bcmevent_get_name(uint event_type); +/* validate if the event is proper and if valid copy event header to event */ +extern int is_wlc_event_frame(void *pktdata, uint pktlen, uint16 exp_usr_subtype, + bcm_event_msg_u_t *out_event); /* Table of event name strings for UIs and debugging dumps */ typedef struct { diff --git a/drivers/net/wireless/bcmdhd/common/include/proto/dnglevent.h b/drivers/net/wireless/bcmdhd/common/include/proto/dnglevent.h new file mode 100755 index 00000000000..584e9d29013 --- /dev/null +++ b/drivers/net/wireless/bcmdhd/common/include/proto/dnglevent.h @@ -0,0 +1,80 @@ +/* + * Broadcom Event protocol definitions + * + * $Copyright Open Broadcom Corporation$ + * + * Dependencies: proto/bcmeth.h + * + * $Id: dnglevent.h $ + * + */ + +/* + * Broadcom dngl Ethernet Events protocol defines + * + */ + +#ifndef _DNGLEVENT_H_ +#define _DNGLEVENT_H_ + +#ifndef _TYPEDEFS_H_ +#include +#endif +#include + +/* This marks the start of a packed structure section. */ +#include +#define BCM_DNGL_EVENT_MSG_VERSION 1 +#define DNGL_E_SOCRAM_IND 0x2 +typedef BWL_PRE_PACKED_STRUCT struct +{ + uint16 version; /* Current version is 1 */ + uint16 reserved; /* reserved for any future extension */ + uint16 event_type; /* DNGL_E_SOCRAM_IND */ + uint16 datalen; /* Length of the event payload */ +} BWL_POST_PACKED_STRUCT bcm_dngl_event_msg_t; + +typedef BWL_PRE_PACKED_STRUCT struct bcm_dngl_event { + struct ether_header eth; + bcmeth_hdr_t bcm_hdr; + bcm_dngl_event_msg_t dngl_event; + /* data portion follows */ +} BWL_POST_PACKED_STRUCT bcm_dngl_event_t; + + +/* SOCRAM_IND type tags */ +#define SOCRAM_IND_ASSRT_TAG 0x1 +#define SOCRAM_IND_TAG_HEALTH_CHECK 0x2 +typedef BWL_PRE_PACKED_STRUCT struct bcm_dngl_socramind { + uint16 tag; /* data tag */ + uint16 length; /* data length */ + uint8 value[1]; /* data value with variable length specified by length */ +} BWL_POST_PACKED_STRUCT bcm_dngl_socramind_t; + +/* Health check top level module tags */ +#define HEALTH_CHECK_TOP_LEVEL_MODULE_PCIEDEV_RTE 1 +typedef BWL_PRE_PACKED_STRUCT struct bcm_dngl_healthcheck { + uint16 top_module_tag; /* top level module tag */ + uint16 top_module_len; /* Type of PCIE issue indication */ + uint8 value[1]; /* data value with variable length specified by length */ +} BWL_POST_PACKED_STRUCT bcm_dngl_healthcheck_t; + +#define HEALTH_CHECK_PCIEDEV_VERSION 1 +#define HEALTH_CHECK_PCIEDEV_FLAG_IN_D3_SHIFT 0 +#define HEALTH_CHECK_PCIEDEV_FLAG_IN_D3_FLAG 1 << HEALTH_CHECK_PCIEDEV_FLAG_IN_D3_SHIFT +/* PCIE Module TAGs */ +#define HEALTH_CHECK_PCIEDEV_INDUCED_IND 0x1 +#define HEALTH_CHECK_PCIEDEV_H2D_DMA_IND 0x2 +#define HEALTH_CHECK_PCIEDEV_D2H_DMA_IND 0x3 +typedef BWL_PRE_PACKED_STRUCT struct bcm_dngl_pcie_hc { + uint16 version; /* HEALTH_CHECK_PCIEDEV_VERSION */ + uint16 reserved; + uint16 pcie_err_ind_type; /* PCIE Module TAGs */ + uint16 pcie_flag; + uint32 pcie_control_reg; +} BWL_POST_PACKED_STRUCT bcm_dngl_pcie_hc_t; + +/* This marks the end of a packed structure section. */ +#include + +#endif /* _DNGLEVENT_H_ */ diff --git a/drivers/net/wireless/bcmdhd/dhd.h b/drivers/net/wireless/bcmdhd/dhd.h index 069d49ed63e..ac0140b4480 100644 --- a/drivers/net/wireless/bcmdhd/dhd.h +++ b/drivers/net/wireless/bcmdhd/dhd.h @@ -346,7 +346,32 @@ typedef struct dhd_pub { tcp_ack_info_t tcp_ack_info_tbl[MAXTCPSTREAMS]; #endif /* DHDTCPACK_SUPPRESS */ uint32 arp_version; +#ifdef GSCAN_SUPPORT + bool lazy_roam_enable; +#endif /* GSCAN_SUPPORT */ } dhd_pub_t; + +typedef struct { + uint rxwake; + uint rcwake; +#ifdef DHD_WAKE_RX_STATUS + uint rx_bcast; + uint rx_arp; + uint rx_mcast; + uint rx_multi_ipv6; + uint rx_icmpv6; + uint rx_icmpv6_ra; + uint rx_icmpv6_na; + uint rx_icmpv6_ns; + uint rx_multi_ipv4; + uint rx_multi_other; + uint rx_ucast; +#endif +#ifdef DHD_WAKE_EVENT_STATUS + uint rc_event[WLC_E_LAST]; +#endif +} wake_counts_t; + typedef struct dhd_cmn { osl_t *osh; /* OSL handle */ dhd_pub_t *dhd; @@ -524,7 +549,8 @@ extern void dhd_store_conn_status(uint32 event, uint32 status, uint32 reason); extern bool dhd_prec_enq(dhd_pub_t *dhdp, struct pktq *q, void *pkt, int prec); /* Receive frame for delivery to OS. Callee disposes of rxp. */ -extern void dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *rxp, int numpkt, uint8 chan); +extern void dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *rxp, int numpkt, + uint8 chan, int pkt_wake, wake_counts_t *wcp); /* Return pointer to interface name */ extern char *dhd_ifname(dhd_pub_t *dhdp, int idx); @@ -551,12 +577,26 @@ extern void dhd_txcomplete(dhd_pub_t *dhdp, void *txp, bool success); #define WIFI_FEATURE_TDLS_OFFCHANNEL 0x2000 /* Support for TDLS off channel */ #define WIFI_FEATURE_EPR 0x4000 /* Enhanced power reporting */ #define WIFI_FEATURE_AP_STA 0x8000 /* Support for AP STA Concurrency */ +#define WIFI_FEATURE_LINKSTAT 0x10000 /* Support for Linkstats */ +#define WIFI_FEATURE_HAL_EPNO 0x40000 /* WiFi PNO enhanced */ #define MAX_FEATURE_SET_CONCURRRENT_GROUPS 3 extern int dhd_dev_get_feature_set(struct net_device *dev); extern int *dhd_dev_get_feature_set_matrix(struct net_device *dev, int *num); +#ifdef GSCAN_SUPPORT +extern int dhd_dev_set_lazy_roam_cfg(struct net_device *dev, + wlc_roam_exp_params_t *roam_param); +extern int dhd_dev_lazy_roam_enable(struct net_device *dev, uint32 enable); +extern int dhd_dev_set_lazy_roam_bssid_pref(struct net_device *dev, + wl_bssid_pref_cfg_t *bssid_pref, uint32 flush); +extern int dhd_dev_set_blacklist_bssid(struct net_device *dev, maclist_t *blacklist, + uint32 len, uint32 flush); +extern int dhd_dev_set_whitelist_ssid(struct net_device *dev, wl_ssid_whitelist_t *whitelist, + uint32 len, uint32 flush); +#endif /* GSCAN_SUPPORT */ + /* OS independent layer functions */ extern int dhd_os_proto_block(dhd_pub_t * pub); extern int dhd_os_proto_unblock(dhd_pub_t * pub); @@ -633,9 +673,10 @@ extern int dhd_ifname2idx(struct dhd_info *dhd, char *name); extern int dhd_net2idx(struct dhd_info *dhd, struct net_device *net); extern struct net_device * dhd_idx2net(void *pub, int ifidx); extern int net_os_send_hang_message(struct net_device *dev); -extern int wl_host_event(dhd_pub_t *dhd_pub, int *idx, void *pktdata, +extern int wl_host_event(dhd_pub_t *dhd_pub, int *idx, void *pktdata, size_t pktlen, wl_event_msg_t *, void **data_ptr); extern void wl_event_to_host_order(wl_event_msg_t * evt); +extern int wl_host_event_get_data(void *pktdata, uint pktlen, bcm_event_msg_u_t *evu); extern int dhd_wl_ioctl(dhd_pub_t *dhd_pub, int ifindex, wl_ioctl_t *ioc, void *buf, int len); extern int dhd_wl_ioctl_cmd(dhd_pub_t *dhd_pub, int cmd, void *arg, int len, uint8 set, @@ -787,7 +828,7 @@ extern uint dhd_force_tx_queueing; #define CUSTOM_LISTEN_INTERVAL LISTEN_INTERVAL #endif /* CUSTOM_LISTEN_INTERVAL */ -#define DEFAULT_SUSPEND_BCN_LI_DTIM 3 +#define DEFAULT_SUSPEND_BCN_LI_DTIM 5 #ifndef CUSTOM_SUSPEND_BCN_LI_DTIM #define CUSTOM_SUSPEND_BCN_LI_DTIM DEFAULT_SUSPEND_BCN_LI_DTIM #endif @@ -815,7 +856,9 @@ extern uint dhd_force_tx_queueing; #define MAX_DTIM_SKIP_BEACON_INTERVAL 100 /* max allowed associated AP beacon for DTIM skip */ -#define MAX_DTIM_ALLOWED_INTERVAL 600 /* max allowed total beacon interval for DTIM skip */ +#ifndef MAX_DTIM_ALLOWED_INTERVAL +#define MAX_DTIM_ALLOWED_INTERVAL 900 /* max allowed total beacon interval for DTIM skip */ +#endif #define NO_DTIM_SKIP 1 #ifdef SDTEST /* Echo packet generator (SDIO), pkts/s */ diff --git a/drivers/net/wireless/bcmdhd/dhd_common.c b/drivers/net/wireless/bcmdhd/dhd_common.c index 08f78ac46d2..6dc0b8e4a54 100644 --- a/drivers/net/wireless/bcmdhd/dhd_common.c +++ b/drivers/net/wireless/bcmdhd/dhd_common.c @@ -71,9 +71,6 @@ extern void htsf_update(struct dhd_info *dhd, void *data); #endif int dhd_msg_level = DHD_ERROR_VAL; - -#include - char fw_path[MOD_PARAM_PATHLEN]; char nv_path[MOD_PARAM_PATHLEN]; @@ -1094,8 +1091,23 @@ wl_show_host_event(wl_event_msg_t *event, void *event_data) } #endif /* SHOW_EVENTS */ +/* Check whether packet is a BRCM event pkt. If it is, record event data. */ int -wl_host_event(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata, +wl_host_event_get_data(void *pktdata, uint pktlen, bcm_event_msg_u_t *evu) +{ + int ret; + + ret = is_wlc_event_frame(pktdata, pktlen, 0, evu); + if (ret != BCME_OK) { + DHD_ERROR(("%s: Invalid event frame, err = %d\n", + __FUNCTION__, ret)); + } + + return ret; +} + +int +wl_host_event(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata, size_t pktlen, wl_event_msg_t *event, void **data_ptr) { /* check whether packet is a BRCM event pkt */ @@ -1104,19 +1116,30 @@ wl_host_event(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata, uint32 type, status, datalen; uint16 flags; int evlen; + int ret; + uint16 usr_subtype; + bcm_event_msg_u_t evu; - if (bcmp(BRCM_OUI, &pvt_data->bcm_hdr.oui[0], DOT11_OUI_LEN)) { - DHD_ERROR(("%s: mismatched OUI, bailing\n", __FUNCTION__)); - return (BCME_ERROR); + ret = wl_host_event_get_data(pktdata, pktlen, &evu); + if (ret != BCME_OK) { + return ret; } - /* BRCM event pkt may be unaligned - use xxx_ua to load user_subtype. */ - if (ntoh16_ua((void *)&pvt_data->bcm_hdr.usr_subtype) != BCMILCP_BCM_SUBTYPE_EVENT) { - DHD_ERROR(("%s: mismatched subtype, bailing\n", __FUNCTION__)); - return (BCME_ERROR); + usr_subtype = ntoh16_ua((void *)&pvt_data->bcm_hdr.usr_subtype); + switch (usr_subtype) { + case BCMILCP_BCM_SUBTYPE_EVENT: + memcpy(event, &evu.event, sizeof(wl_event_msg_t)); + *data_ptr = &pvt_data[1]; + break; + + case BCMILCP_BCM_SUBTYPE_DNGLEVENT: + return BCME_NOTFOUND; + + default: + return BCME_NOTFOUND; } - *data_ptr = &pvt_data[1]; + /* start wl_event_msg process */ event_data = *data_ptr; /* memcpy since BRCM event pkt may be unaligned. */ @@ -1125,6 +1148,7 @@ wl_host_event(dhd_pub_t *dhd_pub, int *ifidx, void *pktdata, type = ntoh32_ua((void *)&event->event_type); flags = ntoh16_ua((void *)&event->flags); status = ntoh32_ua((void *)&event->status); + datalen = ntoh32_ua((void *)&event->datalen); evlen = datalen + sizeof(bcm_event_t); @@ -2030,204 +2054,3 @@ wl_iw_parse_data_tlv(char** list_str, void *dst, int dst_size, const char token, } return 1; } - -/* - * channel list parsing from cscan tlv list -*/ -int -wl_iw_parse_channel_list_tlv(char** list_str, uint16* channel_list, - int channel_num, int *bytes_left) -{ - char* str; - int idx = 0; - - if ((list_str == NULL) || (*list_str == NULL) ||(bytes_left == NULL) || (*bytes_left < 0)) { - DHD_ERROR(("%s error paramters\n", __FUNCTION__)); - return -1; - } - str = *list_str; - - while (*bytes_left > 0) { - - if (str[0] != CSCAN_TLV_TYPE_CHANNEL_IE) { - *list_str = str; - DHD_TRACE(("End channel=%d left_parse=%d %d\n", idx, *bytes_left, str[0])); - return idx; - } - /* Get proper CSCAN_TLV_TYPE_CHANNEL_IE */ - *bytes_left -= 1; - str += 1; - - if (str[0] == 0) { - /* All channels */ - channel_list[idx] = 0x0; - } - else { - channel_list[idx] = (uint16)str[0]; - DHD_TRACE(("%s channel=%d \n", __FUNCTION__, channel_list[idx])); - } - *bytes_left -= 1; - str += 1; - - if (idx++ > 255) { - DHD_ERROR(("%s Too many channels \n", __FUNCTION__)); - return -1; - } - } - - *list_str = str; - return idx; -} - -/* - * SSIDs list parsing from cscan tlv list - */ -int -wl_iw_parse_ssid_list_tlv(char** list_str, wlc_ssid_ext_t* ssid, int max, int *bytes_left) -{ - char* str; - int idx = 0; - - if ((list_str == NULL) || (*list_str == NULL) || (*bytes_left < 0)) { - DHD_ERROR(("%s error paramters\n", __FUNCTION__)); - return -1; - } - str = *list_str; - while (*bytes_left > 0) { - - if (str[0] != CSCAN_TLV_TYPE_SSID_IE) { - *list_str = str; - DHD_TRACE(("nssid=%d left_parse=%d %d\n", idx, *bytes_left, str[0])); - return idx; - } - - /* Get proper CSCAN_TLV_TYPE_SSID_IE */ - *bytes_left -= 1; - str += 1; - - if (str[0] == 0) { - /* Broadcast SSID */ - ssid[idx].SSID_len = 0; - memset((char*)ssid[idx].SSID, 0x0, DOT11_MAX_SSID_LEN); - *bytes_left -= 1; - str += 1; - - DHD_TRACE(("BROADCAST SCAN left=%d\n", *bytes_left)); - } - else if (str[0] <= DOT11_MAX_SSID_LEN) { - /* Get proper SSID size */ - ssid[idx].SSID_len = str[0]; - *bytes_left -= 1; - str += 1; - - /* Get SSID */ - if (ssid[idx].SSID_len > *bytes_left) { - DHD_ERROR(("%s out of memory range len=%d but left=%d\n", - __FUNCTION__, ssid[idx].SSID_len, *bytes_left)); - return -1; - } - - memcpy((char*)ssid[idx].SSID, str, ssid[idx].SSID_len); - - *bytes_left -= ssid[idx].SSID_len; - str += ssid[idx].SSID_len; - ssid[idx].hidden = TRUE; - - DHD_TRACE(("%s :size=%d left=%d\n", - (char*)ssid[idx].SSID, ssid[idx].SSID_len, *bytes_left)); - } - else { - DHD_ERROR(("### SSID size more that %d\n", str[0])); - return -1; - } - - if (idx++ > max) { - DHD_ERROR(("%s number of SSIDs more that %d\n", __FUNCTION__, idx)); - return -1; - } - } - - *list_str = str; - return idx; -} - -/* Parse a comma-separated list from list_str into ssid array, starting - * at index idx. Max specifies size of the ssid array. Parses ssids - * and returns updated idx; if idx >= max not all fit, the excess have - * not been copied. Returns -1 on empty string, or on ssid too long. - */ -int -wl_iw_parse_ssid_list(char** list_str, wlc_ssid_t* ssid, int idx, int max) -{ - char* str, *ptr; - - if ((list_str == NULL) || (*list_str == NULL)) - return -1; - - for (str = *list_str; str != NULL; str = ptr) { - - /* check for next TAG */ - if (!strncmp(str, GET_CHANNEL, strlen(GET_CHANNEL))) { - *list_str = str + strlen(GET_CHANNEL); - return idx; - } - - if ((ptr = strchr(str, ',')) != NULL) { - *ptr++ = '\0'; - } - - if (strlen(str) > DOT11_MAX_SSID_LEN) { - DHD_ERROR(("ssid <%s> exceeds %d\n", str, DOT11_MAX_SSID_LEN)); - return -1; - } - - if (strlen(str) == 0) - ssid[idx].SSID_len = 0; - - if (idx < max) { - bzero(ssid[idx].SSID, sizeof(ssid[idx].SSID)); - strncpy((char*)ssid[idx].SSID, str, sizeof(ssid[idx].SSID) - 1); - ssid[idx].SSID_len = strlen(str); - } - idx++; - } - return idx; -} - -/* - * Parse channel list from iwpriv CSCAN - */ -int -wl_iw_parse_channel_list(char** list_str, uint16* channel_list, int channel_num) -{ - int num; - int val; - char* str; - char* endptr = NULL; - - if ((list_str == NULL)||(*list_str == NULL)) - return -1; - - str = *list_str; - num = 0; - while (strncmp(str, GET_NPROBE, strlen(GET_NPROBE))) { - val = (int)strtoul(str, &endptr, 0); - if (endptr == str) { - printf("could not parse channel number starting at" - " substring \"%s\" in list:\n%s\n", - str, *list_str); - return -1; - } - str = endptr + strspn(endptr, " ,"); - - if (num == channel_num) { - DHD_ERROR(("too many channels (more than %d) in channel list:\n%s\n", - channel_num, *list_str)); - return -1; - } - - channel_list[num++] = (uint16)val; - } - *list_str = str; - return num; -} diff --git a/drivers/net/wireless/bcmdhd/dhd_custom_gpio.c b/drivers/net/wireless/bcmdhd/dhd_custom_gpio.c index 075c84980a4..8e364cb221d 100644 --- a/drivers/net/wireless/bcmdhd/dhd_custom_gpio.c +++ b/drivers/net/wireless/bcmdhd/dhd_custom_gpio.c @@ -32,7 +32,6 @@ #include #include -#include #define WL_ERROR(x) printf x #define WL_TRACE(x) @@ -197,6 +196,12 @@ dhd_custom_get_mac_address(unsigned char *buf) } #endif /* GET_CUSTOM_MAC_ENABLE */ +struct cntry_locales_custom { + char iso_abbrev[WLC_CNTRY_BUF_SZ]; + char custom_locale[WLC_CNTRY_BUF_SZ]; + int32 custom_locale_rev; +}; + /* Customized Locale table : OPTIONAL feature */ const struct cntry_locales_custom translate_custom_table[] = { /* Table should be filled out based on custom platform regulatory requirement */ diff --git a/drivers/net/wireless/bcmdhd/dhd_linux.c b/drivers/net/wireless/bcmdhd/dhd_linux.c index e6b43afc586..8915af2175c 100644 --- a/drivers/net/wireless/bcmdhd/dhd_linux.c +++ b/drivers/net/wireless/bcmdhd/dhd_linux.c @@ -3,13 +3,13 @@ * Basically selected code segments from usb-cdc.c and usb-rndis.c * * Copyright (C) 1999-2013, Broadcom Corporation - * + * * Unless you and Broadcom execute a separate written software license * agreement governing use of this software, this software is licensed to you * under the terms of the GNU General Public License version 2 (the "GPL"), * available at http://www.broadcom.com/licenses/GPLv2.php, with the * following added to such license: - * + * * As a special exception, the copyright holders of this software give you * permission to link this software with independent modules, and to copy and * distribute the resulting executable under terms of your choice, provided that @@ -17,7 +17,7 @@ * the license of that module. An independent module is a module which is not * derived from this software. The special exception does not apply to any * modifications of the software. - * + * * Notwithstanding the above, under no circumstances may you combine this * software in any way with any other Broadcom software provided under a license * other than the GPL, without Broadcom's express prior written consent. @@ -145,10 +145,10 @@ DECLARE_WAIT_QUEUE_HEAD(dhd_dpc_wait); #if defined(OOB_INTR_ONLY) extern void dhd_enable_oob_intr(struct dhd_bus *bus, bool enable); -#endif +#endif #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && (1) static void dhd_hang_process(struct work_struct *work); -#endif +#endif #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0)) MODULE_LICENSE("GPL v2"); #endif /* LinuxVer */ @@ -179,12 +179,6 @@ print_tainted() } #endif /* LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 15) */ -/* Linux wireless extension support */ -#if defined(WL_WIRELESS_EXT) -#include -extern wl_iw_extra_params_t g_wl_iw_params; -#endif /* defined(WL_WIRELESS_EXT) */ - #if defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND) #include #endif /* defined(CONFIG_HAS_EARLYSUSPEND) && defined(DHD_USE_EARLYSUSPEND) */ @@ -264,9 +258,6 @@ static uint32 maxdelay = 0, tspktcnt = 0, maxdelaypktno = 0; /* Local private structure (extension of pub) */ typedef struct dhd_info { -#if defined(WL_WIRELESS_EXT) - wl_iw_t iw; /* wireless extensions state (must be first) */ -#endif /* defined(WL_WIRELESS_EXT) */ dhd_pub_t pub; @@ -551,10 +542,6 @@ int dhd_monitor_uninit(void); -#if defined(WL_WIRELESS_EXT) -struct iw_statistics *dhd_get_wireless_stats(struct net_device *dev); -#endif /* defined(WL_WIRELESS_EXT) */ - static void dhd_dpc(ulong data); /* forward decl */ extern int dhd_wait_pend8021x(struct net_device *dev); @@ -568,7 +555,7 @@ static int dhd_toe_get(dhd_info_t *dhd, int idx, uint32 *toe_ol); static int dhd_toe_set(dhd_info_t *dhd, int idx, uint32 toe_ol); #endif /* TOE */ -static int dhd_wl_host_event(dhd_info_t *dhd, int *ifidx, void *pktdata, +static int dhd_wl_host_event(dhd_info_t *dhd, int *ifidx, void *pktdata, size_t pktlen, wl_event_msg_t *event_ptr, void **data_ptr); #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && (LINUX_VERSION_CODE <= \ @@ -794,7 +781,6 @@ static int dhd_set_suspend(int value, dhd_pub_t *dhd) if (dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, iovbuf, sizeof(iovbuf), TRUE, 0) < 0) DHD_ERROR(("%s: set dtim failed\n", __FUNCTION__)); - #ifndef ENABLE_FW_ROAM_SUSPEND /* Disable firmware roaming during suspend */ bcm_mkiovar("roam_off", (char *)&roamvar, 4, @@ -1540,7 +1526,7 @@ dhd_sendpkt(dhd_pub_t *dhdp, int ifidx, void *pktbuf) /* Look into the packet and update the packet priority */ #ifndef PKTPRIO_OVERRIDE if (PKTPRIO(pktbuf) == 0) -#endif +#endif pktsetprio(pktbuf, FALSE); #ifdef PROP_TXSTATUS @@ -1758,6 +1744,8 @@ static const PKTTYPE_INFO packet_type_info[] = { ETHER_TYPE_BRCM, "BRCM" }, { ETHER_TYPE_802_1X, "802.1X" }, { ETHER_TYPE_WAI, "WAPI" }, + { ETHER_TYPE_IPV6, "IPv6" }, + { ETHER_TYPE_8021Q, "1Q" }, { 0, ""} }; @@ -1775,8 +1763,59 @@ static const char *_get_packet_type_str(uint16 type) } #endif /* DHD_RX_DUMP */ +#ifdef CONFIG_PARTIALRESUME +static unsigned int dhd_get_ipv6_stat(u8 type) +{ + static unsigned int ra = 0; + static unsigned int na = 0; + static unsigned int other = 0; + + switch (type) { + case NDISC_ROUTER_ADVERTISEMENT: + ra++; + return ra; + case NDISC_NEIGHBOUR_ADVERTISEMENT: + na++; + return na; + default: + other++; + break; + } + return other; +} + +static int dhd_rx_suspend_again(struct sk_buff *skb) +{ + u8 *pptr = skb_mac_header(skb); + + if (pptr && + (memcmp(pptr, "\x33\x33\x00\x00\x00\x01", ETHER_ADDR_LEN) == 0) && + (ntoh16(skb->protocol) == ETHER_TYPE_IPV6)) { + u8 type = 0; +#define ETHER_ICMP6_TYPE 54 +#define ETHER_ICMP6_DADDR 38 + if (skb->len > ETHER_ICMP6_TYPE) + type = pptr[ETHER_ICMP6_TYPE]; + if ((type == NDISC_NEIGHBOUR_ADVERTISEMENT) && + (ipv6_addr_equal(&in6addr_linklocal_allnodes, + (const struct in6_addr *)(pptr + ETHER_ICMP6_DADDR)))) { + pr_debug("%s: Suspend, type = %d [%u]\n", __func__, + type, dhd_get_ipv6_stat(type)); + return 0; + } else { + pr_debug("%s: Resume, type = %d [%u]\n", __func__, + type, dhd_get_ipv6_stat(type)); + } +#undef ETHER_ICMP6_TYPE +#undef ETHER_ICMP6_DADDR + } + return DHD_PACKET_TIMEOUT_MS; +} +#endif + void -dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan) +dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan, + int pkt_wake, wake_counts_t *wcp) { dhd_info_t *dhd = (dhd_info_t *)dhdp->info; struct sk_buff *skb; @@ -1792,11 +1831,13 @@ dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan) void *skbhead = NULL; void *skbprev = NULL; #endif /* defined(DHDTHREAD) && defined(RXFRAME_THREAD) */ +#if defined(DHD_RX_DUMP) || defined(DHD_WAKE_STATUS) + char *dump_data; +#endif /* DHD_RX_DUMP || DHD_WAKE_STATUS */ #ifdef DHD_RX_DUMP #ifdef DHD_RX_FULL_DUMP int k; #endif /* DHD_RX_FULL_DUMP */ - char *dump_data; uint16 protocol; #endif /* DHD_RX_DUMP */ @@ -1855,8 +1896,10 @@ dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan) eth = skb->data; len = skb->len; -#ifdef DHD_RX_DUMP +#if defined(DHD_RX_DUMP) || defined(DHD_WAKE_STATUS) dump_data = skb->data; +#endif /* DHD_RX_DUMP || DHD_WAKE_STATUS */ +#ifdef DHD_RX_DUMP protocol = (dump_data[12] << 8) | dump_data[13]; DHD_ERROR(("RX DUMP - %s\n", _get_packet_type_str(protocol))); @@ -1919,19 +1962,38 @@ dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan) /* Process special event packets and then discard them */ memset(&event, 0, sizeof(event)); if (ntoh16(skb->protocol) == ETHER_TYPE_BRCM) { - dhd_wl_host_event(dhd, &ifidx, + int ret_event; + + ret_event = dhd_wl_host_event(dhd, &ifidx, #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22) skb_mac_header(skb), #else skb->mac.raw, #endif + len, &event, &data); + if (ret_event != BCME_OK) { + PKTFREE(dhdp->osh, pktbuf, FALSE); + continue; + } + wl_event_to_host_order(&event); if (!tout_ctrl) tout_ctrl = DHD_PACKET_TIMEOUT_MS; - +#ifdef DHD_WAKE_STATUS + if (unlikely(pkt_wake)) { + wcp->rcwake++; + DHD_ERROR(("DHD_WAKE_DBG: %s - %d\n", bcmevent_get_name(event.event_type), + event.event_type)); +#ifdef DHD_WAKE_EVENT_STATUS + if (event.event_type < WLC_E_LAST) + wcp->rc_event[event.event_type]++; +#endif + pkt_wake = 0; + } +#endif #if defined(PNO_SUPPORT) if (event.event_type == WLC_E_PFN_NET_FOUND) { /* enforce custom wake lock to garantee that Kernel not suspended */ @@ -1944,9 +2006,72 @@ dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan) continue; #endif } else { +#ifdef CONFIG_PARTIALRESUME + tout_rx |= dhd_rx_suspend_again(skb); +#else tout_rx = DHD_PACKET_TIMEOUT_MS; +#endif +#ifdef DHD_WAKE_STATUS + if (unlikely(pkt_wake)) { + wcp->rxwake++; +#ifdef DHD_WAKE_RX_STATUS +#define ETHER_ICMP6_HEADER 20 +#define ETHER_IPV6_SADDR (ETHER_ICMP6_HEADER + 2) +#define ETHER_IPV6_DAADR (ETHER_IPV6_SADDR + IPV6_ADDR_LEN) +#define ETHER_ICMPV6_TYPE (ETHER_IPV6_DAADR + IPV6_ADDR_LEN) + if (ntoh16(skb->protocol) == ETHER_TYPE_ARP) { /* ARP */ + wcp->rx_arp++; + DHD_ERROR(("DHD_WAKE_DBG: ARP\n")); + } + if (dump_data[0] == 0xFF) { /* Broadcast */ + wcp->rx_bcast++; + DHD_ERROR(("DHD_WAKE_DBG: BCAST\n")); + } else if (dump_data[0] & 0x01) { /* Multicast */ + wcp->rx_mcast++; + DHD_ERROR(("DHD_WAKE_DBG: Multicast ")); + if (ntoh16(skb->protocol) == ETHER_TYPE_IPV6) { + wcp->rx_multi_ipv6++; + if ((skb->len > ETHER_ICMP6_HEADER) && + (dump_data[ETHER_ICMP6_HEADER] == IPPROTO_ICMPV6)) { + wcp->rx_icmpv6++; + if (skb->len > ETHER_ICMPV6_TYPE) { + switch (dump_data[ETHER_ICMPV6_TYPE]) { + case NDISC_ROUTER_ADVERTISEMENT: + wcp->rx_icmpv6_ra++; + DHD_ERROR(("RA")); + break; + case NDISC_NEIGHBOUR_ADVERTISEMENT: + wcp->rx_icmpv6_na++; + DHD_ERROR(("NA")); + break; + case NDISC_NEIGHBOUR_SOLICITATION: + wcp->rx_icmpv6_ns++; + DHD_ERROR(("NS")); + break; + } + } + } + DHD_ERROR(("\n")); + } else if (dump_data[2] == 0x5E) { + wcp->rx_multi_ipv4++; + DHD_ERROR(("ipv4\n")); + } else { + wcp->rx_multi_other++; + DHD_ERROR(("other\n")); + } + } else { /* Unicast */ + wcp->rx_ucast++; + DHD_ERROR(("DHD_WAKE_DBG: Unicast\n")); + } +#undef ETHER_ICMP6_HEADER +#undef ETHER_IPV6_SADDR +#undef ETHER_IPV6_DAADR +#undef ETHER_ICMPV6_TYPE +#endif + pkt_wake = 0; + } +#endif } - ASSERT(ifidx < DHD_MAX_IFS && dhd->iflist[ifidx]); if (dhd->iflist[ifidx] && !dhd->iflist[ifidx]->state) ifp = dhd->iflist[ifidx]; @@ -1993,6 +2118,11 @@ dhd_rx_frame(dhd_pub_t *dhdp, int ifidx, void *pktbuf, int numpkt, uint8 chan) #endif DHD_OS_WAKE_LOCK_RX_TIMEOUT_ENABLE(dhdp, tout_rx); DHD_OS_WAKE_LOCK_CTRL_TIMEOUT_ENABLE(dhdp, tout_ctrl); + +#ifdef CONFIG_PARTIALRESUME + if (tout_rx || tout_ctrl) + wifi_process_partial_resume(WIFI_PR_VOTE_FOR_RESUME); +#endif } void @@ -2737,16 +2867,6 @@ dhd_ioctl_entry(struct net_device *net, struct ifreq *ifr, int cmd) return -1; } -#if defined(WL_WIRELESS_EXT) - /* linux wireless extensions */ - if ((cmd >= SIOCIWFIRST) && (cmd <= SIOCIWLAST)) { - /* may recurse, do NOT lock */ - ret = wl_iw_ioctl(net, ifr, cmd); - DHD_OS_WAKE_UNLOCK(&dhd->pub); - return ret; - } -#endif /* defined(WL_WIRELESS_EXT) */ - #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 4, 2) if (cmd == SIOCETHTOOL) { ret = dhd_ethtool(dhd, (void*)ifr->ifr_data); @@ -2887,7 +3007,7 @@ dhd_stop(struct net_device *net) #if defined(WL_CFG80211) if (ifidx == 0 && !dhd_download_fw_on_driverload) wl_android_wifi_off(net); -#endif +#endif dhd->pub.rxcnt_timeout = 0; dhd->pub.txcnt_timeout = 0; @@ -2939,7 +3059,7 @@ dhd_open(struct net_device *net) goto exit; } -#endif +#endif ifidx = dhd_net2idx(dhd, net); DHD_TRACE(("%s: ifidx %d\n", __FUNCTION__, ifidx)); @@ -2969,7 +3089,7 @@ dhd_open(struct net_device *net) goto exit; } } -#endif +#endif if (dhd->pub.busstate != DHD_BUS_DATA) { @@ -3068,7 +3188,7 @@ dhd_osl_detach(osl_t *osh) #if defined(BCMLXSDMMC) up(&dhd_chipup_sem); #endif -#endif +#endif } int @@ -3320,17 +3440,6 @@ dhd_attach(osl_t *osh, struct dhd_bus *bus, uint bus_hdrlen) dhd_monitor_init(&dhd->pub); dhd_state |= DHD_ATTACH_STATE_CFG80211; #endif -#if defined(WL_WIRELESS_EXT) - /* Attach and link in the iw */ - if (!(dhd_state & DHD_ATTACH_STATE_CFG80211)) { - if (wl_iw_attach(net, (void *)&dhd->pub) != 0) { - DHD_ERROR(("wl_iw_attach failed\n")); - goto fail; - } - dhd_state |= DHD_ATTACH_STATE_WL_ATTACH; - } -#endif /* defined(WL_WIRELESS_EXT) */ - /* Set up the watchdog timer */ init_timer(&dhd->timer); @@ -3521,7 +3630,7 @@ dhd_bus_start(dhd_pub_t *dhdp) /* Enable oob at firmware */ dhd_enable_oob_intr(dhd->pub.bus, TRUE); -#endif +#endif /* If bus is not ready, can't come up */ if (dhd->pub.busstate != DHD_BUS_DATA) { @@ -3690,13 +3799,13 @@ dhd_get_concurrent_capabilites(dhd_pub_t *dhd) return ret; #else return 0; -#endif +#endif } } } return 0; } -#endif +#endif int dhd_preinit_ioctls(dhd_pub_t *dhd) { @@ -3912,7 +4021,7 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) } #else (void)concurrent_mode; -#endif +#endif } DHD_ERROR(("Firmware up: op_mode=0x%04x, " @@ -4001,7 +4110,7 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) if (ap_fw_loaded == TRUE) { dhd_wl_ioctl_cmd(dhd, WLC_SET_DTIMPRD, (char *)&dtim, sizeof(dtim), TRUE, 0); } -#endif +#endif #if defined(KEEP_ALIVE) { @@ -4010,7 +4119,7 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) #if defined(SOFTAP) if (ap_fw_loaded == FALSE) -#endif +#endif if (!(dhd->op_mode & DHD_FLAG_HOSTAP_MODE)) { if ((res = dhd_keep_alive_onoff(dhd)) < 0) DHD_ERROR(("%s set keeplive failed %d\n", @@ -4139,6 +4248,8 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) setbit(eventmask_msg->mask, WLC_E_PFN_GSCAN_FULL_RESULT); setbit(eventmask_msg->mask, WLC_E_PFN_SCAN_COMPLETE); setbit(eventmask_msg->mask, WLC_E_PFN_SWC); + setbit(eventmask_msg->mask, WLC_E_PFN_SSID_EXT); + setbit(eventmask_msg->mask, WLC_E_ROAM_EXP_EVENT); #endif /* GSCAN_SUPPORT */ #ifdef BCMCCX_S69 setbit(eventmask_msg->mask, WLC_E_CCX_S69_RESP_RX); @@ -4179,7 +4290,7 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) if (arpoe && !ap_fw_loaded) { #else if (arpoe) { -#endif +#endif dhd_arp_offload_enable(dhd, TRUE); dhd_arp_offload_set(dhd, dhd_arp_mode); } else { @@ -4197,8 +4308,7 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) dhd->pktfilter[DHD_BROADCAST_FILTER_NUM] = NULL; dhd->pktfilter[DHD_MULTICAST4_FILTER_NUM] = NULL; dhd->pktfilter[DHD_MULTICAST6_FILTER_NUM] = NULL; - /* Add filter to pass multicastDNS packet and NOT filter out as Broadcast */ - dhd->pktfilter[DHD_MDNS_FILTER_NUM] = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB"; + dhd->pktfilter[DHD_MDNS_FILTER_NUM] = NULL; /* apply APP pktfilter */ dhd->pktfilter[DHD_ARP_FILTER_NUM] = "105 0 0 12 0xFFFF 0x0806"; @@ -4218,7 +4328,7 @@ dhd_preinit_ioctls(dhd_pub_t *dhd) #if defined(PROP_TXSTATUS) && defined(PROP_TXSTATUS_VSDB) bcm_mkiovar("ampdu_hostreorder", (char *)&hostreorder, 4, buf, sizeof(buf)); dhd_wl_ioctl_cmd(dhd, WLC_SET_VAR, buf, sizeof(buf), TRUE, 0); -#endif +#endif #endif /* DISABLE_11N */ @@ -4591,15 +4701,6 @@ dhd_net_attach(dhd_pub_t *dhdp, int ifidx) net->ethtool_ops = &dhd_ethtool_ops; #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24) */ -#if defined(WL_WIRELESS_EXT) -#if WIRELESS_EXT < 19 - net->get_wireless_stats = dhd_get_wireless_stats; -#endif /* WIRELESS_EXT < 19 */ -#if WIRELESS_EXT > 12 - net->wireless_handlers = (struct iw_handler_def *)&wl_iw_handler_def; -#endif /* WIRELESS_EXT > 12 */ -#endif /* defined(WL_WIRELESS_EXT) */ - dhd->pub.rxsz = DBUS_RX_BUFFER_SIZE_DHD(net); memcpy(net->dev_addr, temp_addr, ETHER_ADDR_LEN); @@ -4614,16 +4715,12 @@ dhd_net_attach(dhd_pub_t *dhdp, int ifidx) net->name, MAC2STRDBG(net->dev_addr)); -#if defined(SOFTAP) && defined(WL_WIRELESS_EXT) && !defined(WL_CFG80211) - wl_iw_iscan_set_scan_broadcast_prep(net, 1); -#endif - #if 1 && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) if (ifidx == 0) { dhd_registration_check = TRUE; up(&dhd_registration_sem); } -#endif +#endif return 0; fail: @@ -4660,7 +4757,7 @@ dhd_bus_detach(dhd_pub_t *dhdp) #if defined(OOB_INTR_ONLY) bcmsdh_unregister_oob_intr(); -#endif +#endif } } } @@ -4710,13 +4807,6 @@ void dhd_detach(dhd_pub_t *dhdp) cancel_work_sync(&dhd->work_hang); #endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) */ -#if defined(WL_WIRELESS_EXT) - if (dhd->dhd_state & DHD_ATTACH_STATE_WL_ATTACH) { - /* Detatch and unlink in the iw */ - wl_iw_detach(); - } -#endif /* defined(WL_WIRELESS_EXT) */ - if (dhd->thr_sysioc_ctl.thr_pid >= 0) { PROC_STOP(&dhd->thr_sysioc_ctl); } @@ -4885,7 +4975,7 @@ dhd_module_init(void) #if 1 && defined(BCMLXSDMMC) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) int retry = POWERUP_MAX_RETRY; int chip_up = 0; -#endif +#endif DHD_TRACE(("%s: Enter\n", __FUNCTION__)); @@ -4905,7 +4995,7 @@ dhd_module_init(void) DHD_ERROR(("Invalid module parameters.\n")); error = -EINVAL; } while (0); -#endif +#endif if (error) goto fail_0; @@ -4947,7 +5037,7 @@ dhd_module_init(void) goto fail_1; #endif /* defined(CONFIG_WIFI_CONTROL_FUNC) */ -#endif +#endif #if defined(CONFIG_WIFI_CONTROL_FUNC) && defined(BCMLXSDMMC) /* If the wifi_set_power() is failed, @@ -4962,7 +5052,7 @@ dhd_module_init(void) #if 1 && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) sema_init(&dhd_registration_sem, 0); -#endif +#endif error = dhd_bus_register(); @@ -5003,7 +5093,7 @@ dhd_module_init(void) #if defined(CONFIG_WIFI_CONTROL_FUNC) wl_android_wifictrl_func_del(); -#endif +#endif /* Call customer gpio to turn off power with WL_REG_ON signal */ dhd_customer_gpio_wlan_ctrl(WLAN_POWER_OFF); @@ -5322,51 +5412,19 @@ void dhd_os_prefree(void *osh, void *addr, uint size) } #endif /* defined(CONFIG_WIFI_CONTROL_FUNC) */ -#if defined(WL_WIRELESS_EXT) -struct iw_statistics * -dhd_get_wireless_stats(struct net_device *dev) -{ - int res = 0; - dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); - - if (!dhd->pub.up) { - return NULL; - } - - res = wl_iw_get_wireless_stats(dev, &dhd->iw.wstats); - - if (res == 0) - return &dhd->iw.wstats; - else - return NULL; -} -#endif /* defined(WL_WIRELESS_EXT) */ - static int -dhd_wl_host_event(dhd_info_t *dhd, int *ifidx, void *pktdata, +dhd_wl_host_event(dhd_info_t *dhd, int *ifidx, void *pktdata, size_t pktlen, wl_event_msg_t *event, void **data) { int bcmerror = 0; ASSERT(dhd != NULL); - bcmerror = wl_host_event(&dhd->pub, ifidx, pktdata, event, data); + bcmerror = wl_host_event(&dhd->pub, ifidx, pktdata, pktlen, event, data); if (bcmerror != BCME_OK) return (bcmerror); -#if defined(WL_WIRELESS_EXT) - if (event->bsscfgidx == 0) { - /* - * Wireless ext is on primary interface only - */ - - ASSERT(dhd->iflist[*ifidx] != NULL); - ASSERT(dhd->iflist[*ifidx]->net != NULL); - - if (dhd->iflist[*ifidx]->net) { - wl_iw_event(dhd->iflist[*ifidx]->net, event, *data); - } - } -#endif /* defined(WL_WIRELESS_EXT) */ + if ((dhd->iflist[*ifidx] == NULL) || (dhd->iflist[*ifidx]->net == NULL)) + return BCME_ERROR; #ifdef WL_CFG80211 if ((ntoh32(event->event_type) == WLC_E_IF) && @@ -5387,6 +5445,7 @@ dhd_wl_host_event(dhd_info_t *dhd, int *ifidx, void *pktdata, ASSERT(dhd->iflist[*ifidx] != NULL); ASSERT(dhd->iflist[*ifidx]->net != NULL); + if (dhd->iflist[*ifidx]->event2cfg80211 && dhd->iflist[*ifidx]->net) { wl_cfg80211_event(dhd->iflist[*ifidx]->net, event, *data); } @@ -5420,7 +5479,7 @@ void dhd_wait_for_event(dhd_pub_t *dhd, bool *lockvar) dhd_os_sdunlock(dhd); wait_event_timeout(dhdinfo->ctrl_wait, (*lockvar == FALSE), timeout); dhd_os_sdlock(dhd); -#endif +#endif return; } @@ -5514,8 +5573,7 @@ int net_os_rxfilter_add_remove(struct net_device *dev, int add_remove, int num) int filter_id = 0; int ret = 0; - if (!dhd || (num == DHD_UNICAST_FILTER_NUM) || - (num == DHD_MDNS_FILTER_NUM)) + if (!dhd || (num == DHD_UNICAST_FILTER_NUM)) return ret; if (num >= dhd->pub.pktfilter_count) return -EINVAL; @@ -5532,6 +5590,10 @@ int net_os_rxfilter_add_remove(struct net_device *dev, int add_remove, int num) filterp = "103 0 0 0 0xFFFF 0x3333"; filter_id = 103; break; + case DHD_MDNS_FILTER_NUM: + filterp = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB"; + filter_id = 104; + break; default: return -EINVAL; } @@ -5617,9 +5679,12 @@ int dhd_dev_get_feature_set(struct net_device *dev) if (FW_SUPPORTED(dhd, rttd2d)) feature_set |= WIFI_FEATURE_D2D_RTT; } - if (FW_SUPPORTED(dhd, proxd)) - feature_set |= WIFI_FEATURE_D2AP_RTT; - +#ifdef RTT_SUPPORT + feature_set |= WIFI_FEATURE_D2AP_RTT; +#endif /* RTT_SUPPORT */ +#ifdef LINKSTAT_SUPPORT + feature_set |= WIFI_FEATURE_LINKSTAT; +#endif /* LINKSTAT_SUPPORT */ /* Supports STA + STA always */ feature_set |= WIFI_FEATURE_ADDITIONAL_STA; #ifdef PNO_SUPPORT @@ -5628,6 +5693,7 @@ int dhd_dev_get_feature_set(struct net_device *dev) feature_set |= WIFI_FEATURE_BATCH_SCAN; #ifdef GSCAN_SUPPORT feature_set |= WIFI_FEATURE_GSCAN; + feature_set |= WIFI_FEATURE_HAL_EPNO; #endif /* GSCAN_SUPPORT */ } #endif /* PNO_SUPPORT */ @@ -5660,6 +5726,7 @@ int *dhd_dev_get_feature_set_matrix(struct net_device *dev, int *num) (feature_set_full & WIFI_FEATURE_D2D_RTT) | (feature_set_full & WIFI_FEATURE_D2AP_RTT) | (feature_set_full & WIFI_FEATURE_PNO) | + (feature_set_full & WIFI_FEATURE_HAL_EPNO) | (feature_set_full & WIFI_FEATURE_BATCH_SCAN) | (feature_set_full & WIFI_FEATURE_GSCAN) | (feature_set_full & WIFI_FEATURE_HOTSPOT) | @@ -5746,6 +5813,12 @@ dhd_dev_pno_get_for_batch(struct net_device *dev, char *buf, int bufsize) dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); return (dhd_pno_get_for_batch(&dhd->pub, buf, bufsize, PNO_STATUS_NORMAL)); } +/* Linux wrapper to call common dhd_pno_set_mac_oui */ +int dhd_dev_pno_set_mac_oui(struct net_device *dev, uint8 *oui) +{ + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + return (dhd_pno_set_mac_oui(&dhd->pub, oui)); +} #endif /* PNO_SUPPORT */ #ifdef GSCAN_SUPPORT @@ -5770,7 +5843,7 @@ dhd_dev_pno_get_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type, } /* Linux wrapper to call common dhd_wait_batch_results_complete */ -void dhd_dev_wait_batch_results_complete(struct net_device *dev) +int dhd_dev_wait_batch_results_complete(struct net_device *dev) { dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); @@ -5778,7 +5851,7 @@ void dhd_dev_wait_batch_results_complete(struct net_device *dev) } /* Linux wrapper to call common dhd_pno_lock_batch_results */ -void +int dhd_dev_pno_lock_access_batch_results(struct net_device *dev) { dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); @@ -5859,9 +5932,144 @@ int dhd_dev_retrieve_batch_scan(struct net_device *dev) return (dhd_retreive_batch_scan_results(&dhd->pub)); } +/* Linux wrapper to call common dhd_pno_process_epno_result */ +void * dhd_dev_process_epno_result(struct net_device *dev, + const void *data, uint32 event, int *send_evt_bytes) +{ + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + + return (dhd_pno_process_epno_result(&dhd->pub, data, event, send_evt_bytes)); +} + +int dhd_dev_set_lazy_roam_cfg(struct net_device *dev, + wlc_roam_exp_params_t *roam_param) +{ + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + wl_roam_exp_cfg_t roam_exp_cfg; + int err; + + if (!roam_param) { + return BCME_BADARG; + } + + DHD_ERROR(("a_band_boost_thr %d a_band_penalty_thr %d\n", + roam_param->a_band_boost_threshold, roam_param->a_band_penalty_threshold)); + DHD_ERROR(("a_band_boost_factor %d a_band_penalty_factor %d cur_bssid_boost %d\n", + roam_param->a_band_boost_factor, roam_param->a_band_penalty_factor, + roam_param->cur_bssid_boost)); + DHD_ERROR(("alert_roam_trigger_thr %d a_band_max_boost %d\n", + roam_param->alert_roam_trigger_threshold, roam_param->a_band_max_boost)); + + memcpy(&roam_exp_cfg.params, roam_param, sizeof(*roam_param)); + roam_exp_cfg.version = ROAM_EXP_CFG_VERSION; + roam_exp_cfg.flags = ROAM_EXP_CFG_PRESENT; + if (dhd->pub.lazy_roam_enable) { + roam_exp_cfg.flags |= ROAM_EXP_ENABLE_FLAG; + } + err = dhd_iovar(&(dhd->pub), 0, "roam_exp_params", (char *)&roam_exp_cfg, sizeof(roam_exp_cfg), 1); + if (err < 0) { + DHD_ERROR(("%s : Failed to execute roam_exp_params %d\n", __FUNCTION__, err)); + } + return err; +} + +int dhd_dev_lazy_roam_enable(struct net_device *dev, uint32 enable) +{ + int err; + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + wl_roam_exp_cfg_t roam_exp_cfg; + + memset(&roam_exp_cfg, 0, sizeof(roam_exp_cfg)); + roam_exp_cfg.version = ROAM_EXP_CFG_VERSION; + if (enable) { + roam_exp_cfg.flags = ROAM_EXP_ENABLE_FLAG; + } + + err = dhd_iovar(&(dhd->pub), 0, "roam_exp_params", (char *)&roam_exp_cfg, sizeof(roam_exp_cfg), 1); + if (err < 0) { + DHD_ERROR(("%s : Failed to execute roam_exp_params %d\n", __FUNCTION__, err)); + } else { + dhd->pub.lazy_roam_enable = (enable != 0); + } + return err; +} +int dhd_dev_set_lazy_roam_bssid_pref(struct net_device *dev, + wl_bssid_pref_cfg_t *bssid_pref, uint32 flush) +{ + int err; + int len; + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + bssid_pref->version = BSSID_PREF_LIST_VERSION; + /* By default programming bssid pref flushes out old values */ + bssid_pref->flags = (flush && !bssid_pref->count) ? ROAM_EXP_CLEAR_BSSID_PREF: 0; + len = sizeof(wl_bssid_pref_cfg_t); + len += (bssid_pref->count - 1) * sizeof(wl_bssid_pref_list_t); + err = dhd_iovar(&(dhd->pub), 0, "roam_exp_bssid_pref", (char *)bssid_pref, + len, 1); + if (err != BCME_OK) { + DHD_ERROR(("%s : Failed to execute roam_exp_bssid_pref %d\n", __FUNCTION__, err)); + } + return err; +} +int dhd_dev_set_blacklist_bssid(struct net_device *dev, maclist_t *blacklist, + uint32 len, uint32 flush) +{ + int err; + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + int macmode; + + if (blacklist) { + err = dhd_wl_ioctl_cmd(&(dhd->pub), WLC_SET_MACLIST, (char *)blacklist, + len, TRUE, 0); + if (err != BCME_OK) { + DHD_ERROR(("%s : WLC_SET_MACLIST failed %d\n", __FUNCTION__, err)); + return err; + } + } + /* By default programming blacklist flushes out old values */ + macmode = (flush && !blacklist) ? WLC_MACMODE_DISABLED : WLC_MACMODE_DENY; + err = dhd_wl_ioctl_cmd(&(dhd->pub), WLC_SET_MACMODE, (char *)&macmode, + sizeof(macmode), TRUE, 0); + if (err != BCME_OK) { + DHD_ERROR(("%s : WLC_SET_MACMODE failed %d\n", __FUNCTION__, err)); + } + return err; +} +int dhd_dev_set_whitelist_ssid(struct net_device *dev, wl_ssid_whitelist_t *ssid_whitelist, + uint32 len, uint32 flush) +{ + int err; + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + wl_ssid_whitelist_t whitelist_ssid_flush; + + if (!ssid_whitelist) { + if (flush) { + ssid_whitelist = &whitelist_ssid_flush; + ssid_whitelist->ssid_count = 0; + } else { + DHD_ERROR(("%s : Nothing to do here\n", __FUNCTION__)); + return BCME_BADARG; + } + } + ssid_whitelist->version = SSID_WHITELIST_VERSION; + ssid_whitelist->flags = flush ? ROAM_EXP_CLEAR_SSID_WHITELIST : 0; + err = dhd_iovar(&(dhd->pub), 0, "roam_exp_ssid_whitelist", (char *)ssid_whitelist, + len, 1); + if (err != BCME_OK) { + DHD_ERROR(("%s : Failed to execute roam_exp_bssid_pref %d\n", __FUNCTION__, err)); + } + return err; +} #endif /* GSCAN_SUPPORT */ +bool dhd_dev_is_legacy_pno_enabled(struct net_device *dev) +{ + dhd_info_t *dhd = *(dhd_info_t **)netdev_priv(dev); + + return (dhd_is_legacy_pno_enabled(&dhd->pub)); +} + #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && (1) static void dhd_hang_process(struct work_struct *work) { @@ -5875,9 +6083,6 @@ static void dhd_hang_process(struct work_struct *work) rtnl_lock(); dev_close(dev); rtnl_unlock(); -#if defined(WL_WIRELESS_EXT) - wl_iw_send_priv_event(dev, "HANG"); -#endif #if defined(WL_CFG80211) wl_cfg80211_hang(dev, WLAN_REASON_UNSPECIFIED); #endif @@ -6278,6 +6483,10 @@ int dhd_os_wd_wake_lock(dhd_pub_t *pub) /* if wakelock_wd_counter was never used : lock it at once */ if (!dhd->wakelock_wd_counter) wake_lock(&dhd->wl_wdwake); +#endif +#ifdef CONFIG_PARTIALRESUME + if (!dhd->wakelock_wd_counter) + wifi_process_partial_resume(WIFI_PR_WD_INIT); #endif dhd->wakelock_wd_counter++; ret = dhd->wakelock_wd_counter; @@ -6298,6 +6507,9 @@ int dhd_os_wd_wake_unlock(dhd_pub_t *pub) dhd->wakelock_wd_counter = 0; #ifdef CONFIG_HAS_WAKELOCK wake_unlock(&dhd->wl_wdwake); +#endif +#ifdef CONFIG_PARTIALRESUME + wifi_process_partial_resume(WIFI_PR_WD_COMPLETE); #endif } spin_unlock_irqrestore(&dhd->wakelock_spinlock, flags); diff --git a/drivers/net/wireless/bcmdhd/dhd_pno.c b/drivers/net/wireless/bcmdhd/dhd_pno.c index ac5d8dd8788..9b097228eaf 100644 --- a/drivers/net/wireless/bcmdhd/dhd_pno.c +++ b/drivers/net/wireless/bcmdhd/dhd_pno.c @@ -43,6 +43,9 @@ #include #include #include +#ifdef GSCAN_SUPPORT +#include +#endif /* GSCAN_SUPPORT */ #ifdef __BIG_ENDIAN #include @@ -70,7 +73,7 @@ } \ } while (0) #define PNO_GET_PNOSTATE(dhd) ((dhd_pno_status_info_t *)dhd->pno_state) -#define PNO_BESTNET_LEN 1024 +#define PNO_BESTNET_LEN 2048 #define PNO_ON 1 #define PNO_OFF 0 #define CHANNEL_2G_MAX 14 @@ -89,7 +92,7 @@ static wlc_ssid_ext_t * dhd_pno_get_legacy_pno_ssid(dhd_pub_t *dhd, dhd_pno_status_info_t *pno_state); #ifdef GSCAN_SUPPORT -static wl_pfn_gscan_channel_bucket_t * +static wl_pfn_gscan_ch_bucket_cfg_t * dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd, dhd_pno_status_info_t *pno_state, uint16 *chan_list, uint32 *num_buckets, uint32 *num_buckets_to_fw); #endif /* GSCAN_SUPPORT */ @@ -144,6 +147,42 @@ bool dhd_is_pno_supported(dhd_pub_t *dhd) return WLS_SUPPORTED(_pno_state); } +bool dhd_is_legacy_pno_enabled(dhd_pub_t *dhd) +{ + dhd_pno_status_info_t *_pno_state; + + if (!dhd || !dhd->pno_state) { + DHD_ERROR(("NULL POINTER : %s\n", + __FUNCTION__)); + return FALSE; + } + _pno_state = PNO_GET_PNOSTATE(dhd); + return ((_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) != 0); +} + +int dhd_pno_set_mac_oui(dhd_pub_t *dhd, uint8 *oui) +{ + int err = BCME_OK; + dhd_pno_status_info_t *_pno_state; + + if (!dhd || !dhd->pno_state) { + DHD_ERROR(("NULL POINTER : %s\n", + __FUNCTION__)); + return BCME_ERROR; + } + _pno_state = PNO_GET_PNOSTATE(dhd); + if (ETHER_ISMULTI(oui)) { + DHD_ERROR(("Expected unicast OUI\n")); + err = BCME_ERROR; + } else { + memcpy(_pno_state->pno_oui, oui, DOT11_OUI_LEN); + DHD_PNO(("PNO mac oui to be used - %02x:%02x:%02x\n", _pno_state->pno_oui[0], + _pno_state->pno_oui[1], _pno_state->pno_oui[2])); + } + + return err; +} + #ifdef GSCAN_SUPPORT static uint64 convert_fw_rel_time_to_systime(uint32 fw_ts_ms) { @@ -153,6 +192,48 @@ static uint64 convert_fw_rel_time_to_systime(uint32 fw_ts_ms) return ((uint64)(TIMESPEC_TO_US(ts)) - (uint64)(fw_ts_ms * 1000)); } +static void +dhd_pno_idx_to_ssid(struct dhd_pno_gscan_params *gscan_params, + dhd_epno_results_t *res, uint32 idx) +{ + dhd_epno_params_t *iter, *next; + + if (gscan_params->num_epno_ssid > 0) { + list_for_each_entry_safe(iter, next, + &gscan_params->epno_ssid_list, list) { + if (iter->index == idx) { + memcpy(res->ssid, iter->ssid, iter->ssid_len); + res->ssid_len = iter->ssid_len; + return; + } + } + } + /* If we are here then there was no match */ + res->ssid[0] = '\0'; + res->ssid_len = 0; + return; +} + +/* Cleanup all results */ +static void +dhd_gscan_clear_all_batch_results(dhd_pub_t *dhd) +{ + struct dhd_pno_gscan_params *gscan_params; + dhd_pno_status_info_t *_pno_state; + gscan_results_cache_t *iter; + + _pno_state = PNO_GET_PNOSTATE(dhd); + gscan_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan; + iter = gscan_params->gscan_batch_cache; + /* Mark everything as consumed */ + while (iter) { + iter->tot_consumed = iter->tot_count; + iter = iter->next; + } + dhd_gscan_batch_cache_cleanup(dhd); + return; +} + static int _dhd_pno_gscan_cfg(dhd_pub_t *dhd, wl_pfn_gscan_cfg_t *pfncfg_gscan_param, int size) { @@ -171,7 +252,7 @@ _dhd_pno_gscan_cfg(dhd_pub_t *dhd, wl_pfn_gscan_cfg_t *pfncfg_gscan_param, int s } static bool -is_batch_retreival_complete(struct dhd_pno_gscan_params *gscan_params) +is_batch_retrieval_complete(struct dhd_pno_gscan_params *gscan_params) { smp_rmb(); return (gscan_params->get_batch_flag == GSCAN_BATCH_RETRIEVAL_COMPLETE); @@ -367,19 +448,25 @@ _dhd_pno_set(dhd_pub_t *dhd, const dhd_pno_params_t *pno_params, dhd_pno_mode_t } /* RSSI margin of 30 dBm */ pfn_param.rssi_margin = htod16(PNO_RSSI_MARGIN_DBM); - /* ADAPTIVE turned off */ - pfn_param.flags &= ~(htod16(ENABLE << ENABLE_ADAPTSCAN_BIT)); pfn_param.repeat = 0; pfn_param.exp = 0; pfn_param.slow_freq = 0; + pfn_param.flags |= htod16(ENABLE << ENABLE_ADAPTSCAN_BIT); if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) { dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd); dhd_pno_params_t *_params; _params = &(_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]); - pfn_param.scan_freq = htod32(MIN(pno_params->params_gscan.scan_fr, - _params->params_legacy.scan_fr)); + + pfn_param.scan_freq = gcd(pno_params->params_gscan.scan_fr, + _params->params_legacy.scan_fr); + + if ((_params->params_legacy.pno_repeat != 0) || + (_params->params_legacy.pno_freq_expo_max != 0)) { + pfn_param.repeat = (uchar) (_params->params_legacy.pno_repeat); + pfn_param.exp = (uchar) (_params->params_legacy.pno_freq_expo_max); + } } lost_network_timeout = (pno_params->params_gscan.max_ch_bucket_freq * @@ -422,8 +509,9 @@ _dhd_pno_set(dhd_pub_t *dhd, const dhd_pno_params_t *pno_params, dhd_pno_mode_t DHD_ERROR(("%s : failed to get pfnmem\n", __FUNCTION__)); goto exit; } - DHD_PNO((" returned mscan : %d, set bestn : %d\n", _tmp, pfn_param.bestn)); pfn_param.mscan = MIN(pfn_param.mscan, _tmp); + DHD_PNO((" returned mscan : %d, set bestn : %d mscan %d\n", _tmp, pfn_param.bestn, + pfn_param.mscan)); } err = dhd_iovar(dhd, 0, "pfn_set", (char *)&pfn_param, sizeof(pfn_param), 1); if (err < 0) { @@ -449,8 +537,9 @@ _dhd_pno_add_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssids_list, int nssid) { int j; for (j = 0; j < nssid; j++) { - DHD_PNO(("%d: scan for %s size = %d hidden = %d\n", j, - ssids_list[j].SSID, ssids_list[j].SSID_len, ssids_list[j].hidden)); + DHD_PNO(("%s size = %d hidden = %d flags = %x\n", + ssids_list[j].SSID, ssids_list[j].SSID_len, ssids_list[j].hidden, + ssids_list[j].flags)); } } /* Check for broadcast ssid */ @@ -472,6 +561,7 @@ _dhd_pno_add_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssids_list, int nssid) pfn_element.flags = htod32(ENABLE << WL_PFN_HIDDEN_BIT); else pfn_element.flags = 0; + pfn_element.flags |= htod32(ssids_list[i].flags); memcpy((char *)pfn_element.ssid.SSID, ssids_list[i].SSID, ssids_list[i].SSID_len); pfn_element.ssid.SSID_len = ssids_list[i].SSID_len; @@ -755,13 +845,16 @@ _dhd_pno_cfg(dhd_pub_t *dhd, uint16 *channel_list, int nchan) if (nchan) { NULL_CHECK(channel_list, "nchan is NULL", err); } + if (nchan > WL_NUMCHANNELS) { + return BCME_RANGE; + } DHD_PNO(("%s enter : nchan : %d\n", __FUNCTION__, nchan)); memset(&pfncfg_param, 0, sizeof(wl_pfn_cfg_t)); /* Setup default values */ pfncfg_param.reporttype = htod32(WL_PFN_REPORT_ALLNET); pfncfg_param.channel_num = htod32(0); - for (i = 0; i < nchan && nchan < WL_NUMCHANNELS; i++) + for (i = 0; i < nchan; i++) pfncfg_param.channel_list[i] = channel_list[i]; pfncfg_param.channel_num = htod32(nchan); @@ -918,19 +1011,25 @@ dhd_pno_stop_for_ssid(dhd_pub_t *dhd) _params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; gscan_params = &_params->params_gscan; - if (gscan_params->mscan) - dhd_pno_get_for_batch(dhd, NULL, 0, PNO_STATUS_DISABLE); - + if (gscan_params->mscan) { + /* retrieve the batching data from firmware into host */ + err = dhd_wait_batch_results_complete(dhd); + if (err != BCME_OK) + goto exit; + } /* save current pno_mode before calling dhd_pno_clean */ + mutex_lock(&_pno_state->pno_mutex); mode = _pno_state->pno_mode; err = dhd_pno_clean(dhd); if (err < 0) { DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n", __FUNCTION__, err)); + mutex_unlock(&_pno_state->pno_mutex); goto exit; } /* restore previous pno_mode */ _pno_state->pno_mode = mode; + mutex_unlock(&_pno_state->pno_mutex); /* Restart gscan */ err = dhd_pno_initiate_gscan_request(dhd, 1, 0); goto exit; @@ -1037,6 +1136,102 @@ static wlc_ssid_ext_t * dhd_pno_get_legacy_pno_ssid(dhd_pub_t *dhd, return p_ssid_list; } +#ifdef GSCAN_SUPPORT +static int dhd_epno_set_ssid(dhd_pub_t *dhd, + dhd_pno_status_info_t *pno_state) +{ + int err = BCME_OK; + dhd_epno_params_t *iter, *next; + dhd_pno_params_t *_params1 = &pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; + struct dhd_pno_gscan_params *gscan_params; + wlc_ssid_ext_t ssid_elem; + wl_pfn_ext_list_t *p_ssid_ext_elem = NULL; + uint32 mem_needed = 0, i = 0; + uint16 num_visible_epno_ssid; + uint8 flags; + + gscan_params = &_params1->params_gscan; + num_visible_epno_ssid = gscan_params->num_visible_epno_ssid; + + if (num_visible_epno_ssid) { + mem_needed = sizeof(wl_pfn_ext_list_t) + (sizeof(wl_pfn_ext_t) * + (num_visible_epno_ssid - 1)); + p_ssid_ext_elem = kzalloc(mem_needed, GFP_KERNEL); + if (p_ssid_ext_elem == NULL) { + DHD_ERROR(("%s : failed to allocate memory %zd\n", + __FUNCTION__, mem_needed)); + err = BCME_NOMEM; + goto exit; + } + p_ssid_ext_elem->version = PFN_SSID_EXT_VERSION; + p_ssid_ext_elem->count = num_visible_epno_ssid; + } + + DHD_PNO(("Total ssids %d, visible SSIDs %d\n", gscan_params->num_epno_ssid, + num_visible_epno_ssid)); + + /* convert dhd_pno_ssid to wlc_ssid_ext_t */ + list_for_each_entry_safe(iter, next, &gscan_params->epno_ssid_list, list) { + if (iter->flags & DHD_PNO_USE_SSID) { + memset(&ssid_elem, 0, sizeof(ssid_elem)); + ssid_elem.SSID_len = iter->ssid_len; + ssid_elem.hidden = TRUE; + flags = (iter->flags & DHD_EPNO_A_BAND_TRIG) ? + WL_PFN_SSID_A_BAND_TRIG: 0; + flags |= (iter->flags & DHD_EPNO_BG_BAND_TRIG) ? + WL_PFN_SSID_BG_BAND_TRIG: 0; + ssid_elem.flags = flags; + memcpy(ssid_elem.SSID, iter->ssid, iter->ssid_len); + if ((err = _dhd_pno_add_ssid(dhd, &ssid_elem, 1)) < 0) { + DHD_ERROR(("failed to add ssid list (err %d) in firmware\n", err)); + goto exit; + } + } else if (i < num_visible_epno_ssid) { + p_ssid_ext_elem->pfn_ext[i].rssi_thresh = iter->rssi_thresh; + switch (iter->auth) { + case DHD_PNO_AUTH_CODE_OPEN: + p_ssid_ext_elem->pfn_ext[i].wpa_auth = WPA_AUTH_DISABLED; + break; + case DHD_PNO_AUTH_CODE_PSK: + p_ssid_ext_elem->pfn_ext[i].wpa_auth = + (WPA2_AUTH_PSK | WPA_AUTH_PSK); + break; + case DHD_PNO_AUTH_CODE_EAPOL: + p_ssid_ext_elem->pfn_ext[i].wpa_auth = + (uint16)WPA_AUTH_PFN_ANY; + break; + default: + p_ssid_ext_elem->pfn_ext[i].wpa_auth = + (uint16)WPA_AUTH_PFN_ANY; + break; + } + memcpy(p_ssid_ext_elem->pfn_ext[i].ssid, iter->ssid, iter->ssid_len); + p_ssid_ext_elem->pfn_ext[i].ssid_len = iter->ssid_len; + iter->index = gscan_params->ssid_ext_last_used_index++; + flags = (iter->flags & DHD_EPNO_A_BAND_TRIG) ? + WL_PFN_SSID_A_BAND_TRIG: 0; + flags |= (iter->flags & DHD_EPNO_BG_BAND_TRIG) ? + WL_PFN_SSID_BG_BAND_TRIG: 0; + p_ssid_ext_elem->pfn_ext[i].flags = flags; + DHD_PNO(("SSID %s idx %d rssi thresh %d flags %x\n", iter->ssid, + iter->index, iter->rssi_thresh, flags)); + i++; + } + } + if (num_visible_epno_ssid) { + err = dhd_iovar(dhd, 0, "pfn_add_ssid_ext", (char *)p_ssid_ext_elem, + mem_needed, 1); + if (err < 0) { + DHD_ERROR(("%s : failed to execute pfn_add_pno_ext_ssid %d\n", __FUNCTION__, + err)); + } + } +exit: + kfree(p_ssid_ext_elem); + return err; +} +#endif /* GSCAN_SUPPORT */ + static int dhd_pno_add_to_ssid_list(dhd_pno_params_t *params, wlc_ssid_ext_t *ssid_list, int nssid) @@ -1063,6 +1258,7 @@ dhd_pno_add_to_ssid_list(dhd_pno_params_t *params, wlc_ssid_ext_t *ssid_list, _pno_ssid->hidden = ssid_list[i].hidden; memcpy(_pno_ssid->SSID, ssid_list[i].SSID, _pno_ssid->SSID_len); list_add_tail(&_pno_ssid->list, ¶ms->params_legacy.ssid_list); + params->params_legacy.nssid++; } exit: @@ -1127,7 +1323,7 @@ dhd_pno_set_for_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssid_list, int nssid, else { tot_nchan = WL_NUMCHANNELS; err = _dhd_pno_get_channels(dhd, _chan_list, &tot_nchan, - (WLC_BAND_2G | WLC_BAND_5G), TRUE); + (WLC_BAND_2G | WLC_BAND_5G), FALSE); if (err < 0) { tot_nchan = 0; DHD_PNO(("Could not get channel list for PNO SSID\n")); @@ -1189,7 +1385,7 @@ dhd_pno_set_for_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssid_list, int nssid, _params->params_legacy.pno_repeat = pno_repeat; _params->params_legacy.pno_freq_expo_max = pno_freq_expo_max; _params->params_legacy.nchan = tot_nchan; - _params->params_legacy.nssid = nssid; + _params->params_legacy.nssid = 0; INIT_LIST_HEAD(&_params->params_legacy.ssid_list); #ifdef GSCAN_SUPPORT /* dhd_pno_initiate_gscan_request will handle simultaneous Legacy PNO and GSCAN */ @@ -1231,8 +1427,16 @@ dhd_pno_set_for_ssid(dhd_pub_t *dhd, wlc_ssid_ext_t* ssid_list, int nssid, _dhd_pno_reinitialize_prof(dhd, _params, DHD_PNO_LEGACY_MODE); exit_no_clear: /* clear mode in case of error */ - if (err < 0) - _pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE; + if (err < 0) { + int ret = dhd_pno_clean(dhd); + + if (ret < 0) { + DHD_ERROR(("%s : dhd_pno_clean failure (err: %d)\n", + __FUNCTION__, ret)); + } else { + _pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE; + } + } return err; } int @@ -1403,8 +1607,7 @@ static void dhd_pno_reset_cfg_gscan(dhd_pno_params_t *_params, _params->params_gscan.nchannel_buckets = 0; DHD_PNO(("Flush Scan config\n")); } - if (flags & GSCAN_FLUSH_HOTLIST_CFG) - { + if (flags & GSCAN_FLUSH_HOTLIST_CFG) { struct dhd_pno_bssid *iter, *next; if (_params->params_gscan.nbssid_hotlist > 0) { list_for_each_entry_safe(iter, next, @@ -1416,8 +1619,7 @@ static void dhd_pno_reset_cfg_gscan(dhd_pno_params_t *_params, _params->params_gscan.nbssid_hotlist = 0; DHD_PNO(("Flush Hotlist Config\n")); } - if (flags & GSCAN_FLUSH_SIGNIFICANT_CFG) - { + if (flags & GSCAN_FLUSH_SIGNIFICANT_CFG) { dhd_pno_significant_bssid_t *iter, *next; if (_params->params_gscan.nbssid_significant_change > 0) { @@ -1430,16 +1632,35 @@ static void dhd_pno_reset_cfg_gscan(dhd_pno_params_t *_params, _params->params_gscan.nbssid_significant_change = 0; DHD_PNO(("Flush Significant Change Config\n")); } + if (flags & GSCAN_FLUSH_EPNO_CFG) { + dhd_epno_params_t *iter, *next; + + if (_params->params_gscan.num_epno_ssid > 0) { + list_for_each_entry_safe(iter, next, + &_params->params_gscan.epno_ssid_list, list) { + list_del(&iter->list); + kfree(iter); + } + } + _params->params_gscan.num_epno_ssid = 0; + _params->params_gscan.num_visible_epno_ssid = 0; + _params->params_gscan.ssid_ext_last_used_index = 0; + DHD_PNO(("Flushed ePNO Config\n")); + } return; } -void dhd_pno_lock_batch_results(dhd_pub_t *dhd) +int dhd_pno_lock_batch_results(dhd_pub_t *dhd) { dhd_pno_status_info_t *_pno_state; + int err = BCME_OK; + + NULL_CHECK(dhd, "dhd is NULL", err); + NULL_CHECK(dhd->pno_state, "pno_state is NULL", err); _pno_state = PNO_GET_PNOSTATE(dhd); mutex_lock(&_pno_state->pno_mutex); - return; + return err; } void dhd_pno_unlock_batch_results(dhd_pub_t *dhd) @@ -1450,11 +1671,14 @@ void dhd_pno_unlock_batch_results(dhd_pub_t *dhd) return; } -void dhd_wait_batch_results_complete(dhd_pub_t *dhd) +int dhd_wait_batch_results_complete(dhd_pub_t *dhd) { dhd_pno_status_info_t *_pno_state; dhd_pno_params_t *_params; + int err = BCME_OK; + NULL_CHECK(dhd, "dhd is NULL", err); + NULL_CHECK(dhd->pno_state, "pno_state is NULL", err); _pno_state = PNO_GET_PNOSTATE(dhd); _params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; @@ -1462,12 +1686,11 @@ void dhd_wait_batch_results_complete(dhd_pub_t *dhd) if (_params->params_gscan.get_batch_flag == GSCAN_BATCH_RETRIEVAL_IN_PROGRESS) { DHD_PNO(("%s: Waiting to complete retrieval..\n", __FUNCTION__)); wait_event_interruptible_timeout(_pno_state->batch_get_wait, - is_batch_retreival_complete(&_params->params_gscan), + is_batch_retrieval_complete(&_params->params_gscan), msecs_to_jiffies(GSCAN_BATCH_GET_MAX_WAIT)); } else { /* GSCAN_BATCH_RETRIEVAL_COMPLETE */ gscan_results_cache_t *iter; uint16 num_results = 0; - int err; mutex_lock(&_pno_state->pno_mutex); iter = _params->params_gscan.gscan_batch_cache; @@ -1480,19 +1703,18 @@ void dhd_wait_batch_results_complete(dhd_pub_t *dhd) /* All results consumed/No results cached?? * Get fresh results from FW */ - if (!num_results) { + if ((_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) && !num_results) { DHD_PNO(("%s: No results cached, getting from FW..\n", __FUNCTION__)); err = dhd_retreive_batch_scan_results(dhd); if (err == BCME_OK) { wait_event_interruptible_timeout(_pno_state->batch_get_wait, - is_batch_retreival_complete(&_params->params_gscan), + is_batch_retrieval_complete(&_params->params_gscan), msecs_to_jiffies(GSCAN_BATCH_GET_MAX_WAIT)); } } } DHD_PNO(("%s: Wait complete\n", __FUNCTION__)); - - return; + return err; } static void *dhd_get_gscan_batch_results(dhd_pub_t *dhd, uint32 *len) @@ -1521,10 +1743,19 @@ void * dhd_pno_get_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, { void *ret = NULL; dhd_pno_gscan_capabilities_t *ptr; + dhd_epno_params_t *epno_params; + dhd_pno_params_t *_params; + dhd_pno_status_info_t *_pno_state; + if (!dhd || !dhd->pno_state) { + DHD_ERROR(("NULL POINTER : %s\n", __FUNCTION__)); + return NULL; + } + _pno_state = PNO_GET_PNOSTATE(dhd); + _params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; if (!len) { DHD_ERROR(("%s: len is NULL\n", __FUNCTION__)); - return ret; + return NULL; } switch (type) { @@ -1536,13 +1767,16 @@ void * dhd_pno_get_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, /* Hardcoding these values for now, need to get * these values from FW, will change in a later check-in */ - ptr->max_scan_cache_size = 12; + ptr->max_scan_cache_size = GSCAN_MAX_AP_CACHE; ptr->max_scan_buckets = GSCAN_MAX_CH_BUCKETS; - ptr->max_ap_cache_per_scan = 16; + ptr->max_ap_cache_per_scan = GSCAN_MAX_AP_CACHE_PER_SCAN; ptr->max_rssi_sample_size = PFN_SWC_RSSI_WINDOW_MAX; ptr->max_scan_reporting_threshold = 100; - ptr->max_hotlist_aps = 64; + ptr->max_hotlist_aps = PFN_HOTLIST_MAX_NUM_APS; ptr->max_significant_wifi_change_aps = PFN_SWC_MAX_NUM_APS; + ptr->max_epno_ssid_crc32 = MAX_EPNO_SSID_NUM; + ptr->max_epno_hidden_ssid = MAX_EPNO_HIDDEN_SSID; + ptr->max_white_list_ssid = MAX_WHITELIST_SSID; ret = (void *)ptr; *len = sizeof(dhd_pno_gscan_capabilities_t); break; @@ -1602,7 +1836,27 @@ void * dhd_pno_get_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, DHD_ERROR(("%s: info buffer is NULL\n", __FUNCTION__)); } break; + case DHD_PNO_GET_EPNO_SSID_ELEM: + if (_params->params_gscan.num_epno_ssid >= + (MAX_EPNO_SSID_NUM + MAX_EPNO_HIDDEN_SSID)) { + DHD_ERROR(("Excessive number of ePNO SSIDs programmed %d\n", + _params->params_gscan.num_epno_ssid)); + return NULL; + } + + if (!_params->params_gscan.num_epno_ssid) + INIT_LIST_HEAD(&_params->params_gscan.epno_ssid_list); + epno_params = kzalloc(sizeof(dhd_epno_params_t), GFP_KERNEL); + if (!epno_params) { + DHD_ERROR(("EPNO ssid: cannot alloc %zd bytes", + sizeof(dhd_epno_params_t))); + return NULL; + } + _params->params_gscan.num_epno_ssid++; + epno_params->index = DHD_EPNO_DEFAULT_INDEX; + list_add_tail(&epno_params->list, &_params->params_gscan.epno_ssid_list); + ret = epno_params; default: break; } @@ -1620,6 +1874,7 @@ int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, dhd_pno_status_info_t *_pno_state; NULL_CHECK(dhd, "dhd is NULL", err); + NULL_CHECK(dhd->pno_state, "pno_state is NULL", err); DHD_PNO(("%s enter\n", __FUNCTION__)); @@ -1741,8 +1996,8 @@ int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, break; case DHD_PNO_SCAN_CFG_ID: { - int i, k, valid = 0; - uint16 band, min; + int i, k; + uint16 band; gscan_scan_params_t *ptr = (gscan_scan_params_t *)buf; struct dhd_pno_gscan_channel_bucket *ch_bucket; @@ -1752,7 +2007,6 @@ int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, memcpy(_params->params_gscan.channel_bucket, ptr->channel_bucket, _params->params_gscan.nchannel_buckets * sizeof(struct dhd_pno_gscan_channel_bucket)); - min = ptr->channel_bucket[0].bucket_freq_multiple; ch_bucket = _params->params_gscan.channel_bucket; for (i = 0; i < ptr->nchannel_buckets; i++) { @@ -1773,21 +2027,17 @@ int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, if (band & GSCAN_DFS_MASK) ch_bucket[i].band |= GSCAN_DFS_MASK; - if (ptr->scan_fr == - ptr->channel_bucket[i].bucket_freq_multiple) { - valid = 1; - } - if (ptr->channel_bucket[i].bucket_freq_multiple < min) - min = ptr->channel_bucket[i].bucket_freq_multiple; DHD_PNO(("band %d report_flag %d\n", ch_bucket[i].band, ch_bucket[i].report_flag)); } - if (!valid) - ptr->scan_fr = min; for (i = 0; i < ptr->nchannel_buckets; i++) { ch_bucket[i].bucket_freq_multiple = ch_bucket[i].bucket_freq_multiple/ptr->scan_fr; + ch_bucket[i].bucket_max_multiple = + ch_bucket[i].bucket_max_multiple/ptr->scan_fr; + DHD_PNO(("mult %d max_mult %d repeat %d\n", ch_bucket[i].bucket_freq_multiple, + ch_bucket[i].bucket_max_multiple, ch_bucket[i].repeat)); } _params->params_gscan.scan_fr = ptr->scan_fr; @@ -1798,6 +2048,14 @@ int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, } } break; + case DHD_PNO_EPNO_CFG_ID: + if (flush) { + dhd_pno_reset_cfg_gscan(_params, _pno_state, + GSCAN_FLUSH_EPNO_CFG); + } else { + _params->params_gscan.num_visible_epno_ssid += *((uint16 *)buf); + } + break; default: err = BCME_BADARG; break; @@ -1844,7 +2102,7 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) int tot_nchan = 0; int num_buckets_to_fw, tot_num_buckets, gscan_param_size; dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd); - wl_pfn_gscan_channel_bucket_t *ch_bucket = NULL; + wl_pfn_gscan_ch_bucket_cfg_t *ch_bucket = NULL; wl_pfn_gscan_cfg_t *pfn_gscan_cfg_t = NULL; wl_pfn_significant_bssid_t *p_pfn_significant_bssid = NULL; wl_pfn_bssid_t *p_pfn_bssid = NULL; @@ -1880,19 +2138,25 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) _chan_list, &tot_num_buckets, &num_buckets_to_fw))) goto exit; + mutex_lock(&_pno_state->pno_mutex); + /* Clear any pre-existing results in our cache + * not consumed by framework + */ + dhd_gscan_clear_all_batch_results(dhd); if (_pno_state->pno_mode & (DHD_PNO_GSCAN_MODE | DHD_PNO_LEGACY_MODE)) { /* store current pno_mode before disabling pno */ mode = _pno_state->pno_mode; err = dhd_pno_clean(dhd); if (err < 0) { DHD_ERROR(("%s : failed to disable PNO\n", __FUNCTION__)); + mutex_unlock(&_pno_state->pno_mutex); goto exit; } /* restore the previous mode */ _pno_state->pno_mode = mode; } - _pno_state->pno_mode |= DHD_PNO_GSCAN_MODE; + mutex_unlock(&_pno_state->pno_mutex); if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) { pssid_list = dhd_pno_get_legacy_pno_ssid(dhd, _pno_state); @@ -1916,7 +2180,7 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) } gscan_param_size = sizeof(wl_pfn_gscan_cfg_t) + - (num_buckets_to_fw - 1) * sizeof(wl_pfn_gscan_channel_bucket_t); + (num_buckets_to_fw - 1) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t); pfn_gscan_cfg_t = (wl_pfn_gscan_cfg_t *) MALLOC(dhd->osh, gscan_param_size); if (!pfn_gscan_cfg_t) { @@ -1926,6 +2190,7 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) goto exit; } + pfn_gscan_cfg_t->version = WL_GSCAN_CFG_VERSION; if (gscan_params->mscan) pfn_gscan_cfg_t->buffer_threshold = gscan_params->buffer_threshold; else @@ -1940,9 +2205,11 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) pfn_gscan_cfg_t->swc_rssi_window_size = 0; pfn_gscan_cfg_t->lost_ap_window = 0; } + pfn_gscan_cfg_t->flags = (gscan_params->send_all_results_flag & GSCAN_SEND_ALL_RESULTS_MASK); pfn_gscan_cfg_t->count_of_channel_buckets = num_buckets_to_fw; + pfn_gscan_cfg_t->retry_threshold = GSCAN_RETRY_THRESHOLD; for (i = 0, k = 0; i < tot_num_buckets; i++) { if (ch_bucket[i].bucket_end_index != CHANNEL_BUCKET_EMPTY_INDEX) { @@ -1950,8 +2217,12 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) ch_bucket[i].bucket_end_index; pfn_gscan_cfg_t->channel_bucket[k].bucket_freq_multiple = ch_bucket[i].bucket_freq_multiple; - pfn_gscan_cfg_t->channel_bucket[k].report_flag = - ch_bucket[i].report_flag; + pfn_gscan_cfg_t->channel_bucket[k].max_freq_multiple = + ch_bucket[i].max_freq_multiple; + pfn_gscan_cfg_t->channel_bucket[k].repeat = + ch_bucket[i].repeat; + pfn_gscan_cfg_t->channel_bucket[k].flag = + ch_bucket[i].flag; k++; } } @@ -2036,13 +2307,30 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) } } + if (gscan_params->num_epno_ssid > 0) { + DHD_PNO(("num_epno_ssid %d\n", gscan_params->num_epno_ssid)); + err = dhd_epno_set_ssid(dhd, _pno_state); + if (err < 0) { + DHD_ERROR(("failed to add ssid list (err %d) in firmware\n", err)); + goto exit; + } + } + if ((err = _dhd_pno_enable(dhd, PNO_ON)) < 0) DHD_ERROR(("%s : failed to enable PNO err %d\n", __FUNCTION__, err)); exit: /* clear mode in case of error */ - if (err < 0) - _pno_state->pno_mode &= ~DHD_PNO_GSCAN_MODE; + if (err < 0) { + int ret = dhd_pno_clean(dhd); + + if (ret < 0) { + DHD_ERROR(("%s : failed to call dhd_pno_clean (err: %d)\n", + __FUNCTION__, ret)); + } else { + _pno_state->pno_mode &= ~DHD_PNO_GSCAN_MODE; + } + } kfree(pssid_list); kfree(p_pfn_significant_bssid); kfree(p_pfn_bssid); @@ -2050,151 +2338,21 @@ dhd_pno_set_for_gscan(dhd_pub_t *dhd, struct dhd_pno_gscan_params *gscan_params) MFREE(dhd->osh, pfn_gscan_cfg_t, gscan_param_size); if (ch_bucket) MFREE(dhd->osh, ch_bucket, - (tot_num_buckets * sizeof(wl_pfn_gscan_channel_bucket_t))); + (tot_num_buckets * sizeof(wl_pfn_gscan_ch_bucket_cfg_t))); return err; } -static void -dhd_pno_merge_gscan_pno_channels(dhd_pno_status_info_t *pno_state, - uint16 *chan_list, - uint8 *ch_scratch_pad, - wl_pfn_gscan_channel_bucket_t *ch_bucket, - uint32 *num_buckets_to_fw, - int num_channels) -{ - uint16 chan_buf[WL_NUMCHANNELS]; - int i, j = 0, ch_bucket_idx = 0; - dhd_pno_params_t *_params = &pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; - dhd_pno_params_t *_params1 = &pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]; - uint16 *legacy_chan_list = _params1->params_legacy.chan_list; - bool is_legacy_scan_freq_higher; - uint8 report_flag = CH_BUCKET_REPORT_REGULAR; - - if (!_params1->params_legacy.scan_fr) - _params1->params_legacy.scan_fr = PNO_SCAN_MIN_FW_SEC; - - is_legacy_scan_freq_higher = - _params->params_gscan.scan_fr < _params1->params_legacy.scan_fr; - - /* Calculate new Legacy scan multiple of base scan_freq - * The legacy PNO channel bucket is added at the end of the - * channel bucket list. - */ - if (is_legacy_scan_freq_higher) { - ch_bucket[_params->params_gscan.nchannel_buckets].bucket_freq_multiple = - _params1->params_legacy.scan_fr/_params->params_gscan.scan_fr; - - } else { - uint16 max = 0; - - /* Calculate new multiple of base scan_freq for gscan buckets */ - ch_bucket[_params->params_gscan.nchannel_buckets].bucket_freq_multiple = 1; - for (i = 0; i < _params->params_gscan.nchannel_buckets; i++) { - ch_bucket[i].bucket_freq_multiple *= _params->params_gscan.scan_fr; - ch_bucket[i].bucket_freq_multiple /= _params1->params_legacy.scan_fr; - if (max < ch_bucket[i].bucket_freq_multiple) - max = ch_bucket[i].bucket_freq_multiple; - } - _params->params_gscan.max_ch_bucket_freq = max; - } - - /* Off to remove duplicates!! - * Find channels that are already being serviced by gscan before legacy bucket - * These have to be removed from legacy bucket. - * !!Assuming chan_list channels are validated list of channels!! - * ch_scratch_pad is 1 at gscan bucket locations see dhd_pno_gscan_create_channel_list() - */ - for (i = 0; i < _params1->params_legacy.nchan; i++) - ch_scratch_pad[legacy_chan_list[i]] += 2; - - ch_bucket_idx = 0; - memcpy(chan_buf, chan_list, num_channels * sizeof(uint16)); - - /* Finally create channel list and bucket - * At this point ch_scratch_pad can have 4 values: - * 0 - Channel not present in either Gscan or Legacy PNO bucket - * 1 - Channel present only in Gscan bucket - * 2 - Channel present only in Legacy PNO bucket - * 3 - Channel present in both Gscan and Legacy PNO buckets - * Thus Gscan buckets can have values 1 or 3 and Legacy 2 or 3 - * For channel buckets with scan_freq < legacy accept all - * channels i.e. ch_scratch_pad = 1 and 3 - * else accept only ch_scratch_pad = 1 and mark rejects as - * ch_scratch_pad = 4 so that they go in legacy - */ - for (i = 0; i < _params->params_gscan.nchannel_buckets; i++) { - if (ch_bucket[i].bucket_freq_multiple <= - ch_bucket[_params->params_gscan.nchannel_buckets].bucket_freq_multiple) { - for (; ch_bucket_idx <= ch_bucket[i].bucket_end_index; ch_bucket_idx++, j++) - chan_list[j] = chan_buf[ch_bucket_idx]; - - ch_bucket[i].bucket_end_index = j - 1; - } else { - num_channels = 0; - for (; ch_bucket_idx <= ch_bucket[i].bucket_end_index; ch_bucket_idx++) { - if (ch_scratch_pad[chan_buf[ch_bucket_idx]] == 1) { - chan_list[j] = chan_buf[ch_bucket_idx]; - j++; - num_channels++; - } else { - ch_scratch_pad[chan_buf[ch_bucket_idx]] = 4; - /* If Gscan channel is merged off to legacy bucket and - * if the gscan channel bucket has a report flag > 0 - * use the same for legacy - */ - if (report_flag < ch_bucket[i].report_flag) - report_flag = ch_bucket[i].report_flag; - } - } - - if (num_channels) { - ch_bucket[i].bucket_end_index = j - 1; - } else { - ch_bucket[i].bucket_end_index = CHANNEL_BUCKET_EMPTY_INDEX; - *num_buckets_to_fw = *num_buckets_to_fw - 1; - } - } - - } - - num_channels = 0; - ch_bucket[_params->params_gscan.nchannel_buckets].report_flag = report_flag; - /* Now add channels to the legacy scan bucket - * ch_scratch_pad = 0 to 4 at this point, for legacy -> 2,3,4. 2 means exclusively - * Legacy so add to bucket. 4 means it is a reject of gscan bucket and must - * be added to Legacy bucket,reject 3 - */ - for (i = 0; i < _params1->params_legacy.nchan; i++) { - if (ch_scratch_pad[legacy_chan_list[i]] != 3) { - chan_list[j] = legacy_chan_list[i]; - j++; - num_channels++; - } - } - if (num_channels) { - ch_bucket[_params->params_gscan.nchannel_buckets].bucket_end_index = j - 1; - } - else { - ch_bucket[_params->params_gscan.nchannel_buckets].bucket_end_index = - CHANNEL_BUCKET_EMPTY_INDEX; - *num_buckets_to_fw = *num_buckets_to_fw - 1; - } - - return; -} - -static wl_pfn_gscan_channel_bucket_t * +static wl_pfn_gscan_ch_bucket_cfg_t * dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd, dhd_pno_status_info_t *_pno_state, uint16 *chan_list, uint32 *num_buckets, uint32 *num_buckets_to_fw) { - int i, num_channels, err, nchan = WL_NUMCHANNELS; + int i, num_channels, err, nchan = WL_NUMCHANNELS, ch_cnt; uint16 *ptr = chan_list, max; - uint8 *ch_scratch_pad; - wl_pfn_gscan_channel_bucket_t *ch_bucket; + wl_pfn_gscan_ch_bucket_cfg_t *ch_bucket; dhd_pno_params_t *_params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; bool is_pno_legacy_running = _pno_state->pno_mode & DHD_PNO_LEGACY_MODE; dhd_pno_gscan_channel_bucket_t *gscan_buckets = _params->params_gscan.channel_bucket; @@ -2204,26 +2362,30 @@ dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd, else *num_buckets = _params->params_gscan.nchannel_buckets; - *num_buckets_to_fw = *num_buckets; + *num_buckets_to_fw = 0; - ch_bucket = (wl_pfn_gscan_channel_bucket_t *) MALLOC(dhd->osh, - ((*num_buckets) * sizeof(wl_pfn_gscan_channel_bucket_t))); + ch_bucket = (wl_pfn_gscan_ch_bucket_cfg_t *) MALLOC(dhd->osh, + ((*num_buckets) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t))); if (!ch_bucket) { DHD_ERROR(("%s: failed to malloc memory of size %zd\n", - __FUNCTION__, (*num_buckets) * sizeof(wl_pfn_gscan_channel_bucket_t))); + __FUNCTION__, (*num_buckets) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t))); *num_buckets_to_fw = *num_buckets = 0; return NULL; } max = gscan_buckets[0].bucket_freq_multiple; num_channels = 0; - for (i = 0; i < _params->params_gscan.nchannel_buckets; i++) { + /* nchan is the remaining space left in chan_list buffer + * So any overflow list of channels is ignored + */ + for (i = 0; i < _params->params_gscan.nchannel_buckets && nchan; i++) { if (!gscan_buckets[i].band) { - num_channels += gscan_buckets[i].num_channels; + ch_cnt = MIN(gscan_buckets[i].num_channels, (uint8)nchan); + num_channels += ch_cnt; memcpy(ptr, gscan_buckets[i].chan_list, - gscan_buckets[i].num_channels * sizeof(uint16)); - ptr = ptr + gscan_buckets[i].num_channels; + ch_cnt * sizeof(uint16)); + ptr = ptr + ch_cnt; } else { /* get a valid channel list based on band B or A */ err = _dhd_pno_get_channels(dhd, ptr, @@ -2234,7 +2396,7 @@ dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd, DHD_ERROR(("%s: failed to get valid channel list(band : %d)\n", __FUNCTION__, gscan_buckets[i].band)); MFREE(dhd->osh, ch_bucket, - ((*num_buckets) * sizeof(wl_pfn_gscan_channel_bucket_t))); + ((*num_buckets) * sizeof(wl_pfn_gscan_ch_bucket_cfg_t))); *num_buckets_to_fw = *num_buckets = 0; return NULL; } @@ -2245,57 +2407,60 @@ dhd_pno_gscan_create_channel_list(dhd_pub_t *dhd, ch_bucket[i].bucket_end_index = num_channels - 1; ch_bucket[i].bucket_freq_multiple = gscan_buckets[i].bucket_freq_multiple; - ch_bucket[i].report_flag = gscan_buckets[i].report_flag; + ch_bucket[i].repeat = gscan_buckets[i].repeat; + ch_bucket[i].max_freq_multiple = gscan_buckets[i].bucket_max_multiple; + ch_bucket[i].flag = gscan_buckets[i].report_flag; + /* HAL and FW interpretations are opposite for this bit */ + ch_bucket[i].flag ^= DHD_PNO_REPORT_NO_BATCH; if (max < gscan_buckets[i].bucket_freq_multiple) max = gscan_buckets[i].bucket_freq_multiple; nchan = WL_NUMCHANNELS - num_channels; + *num_buckets_to_fw = *num_buckets_to_fw + 1; DHD_PNO(("end_idx %d freq_mult - %d\n", ch_bucket[i].bucket_end_index, ch_bucket[i].bucket_freq_multiple)); } - ch_scratch_pad = (uint8 *) kzalloc(CHANNEL_5G_MAX, GFP_KERNEL); - if (!ch_scratch_pad) { - DHD_ERROR(("%s: failed to malloc memory of size %d\n", - __FUNCTION__, CHANNEL_5G_MAX)); - MFREE(dhd->osh, ch_bucket, - ((*num_buckets) * sizeof(wl_pfn_gscan_channel_bucket_t))); - *num_buckets_to_fw = *num_buckets = 0; - return NULL; - } - - /* Need to look for duplicates in gscan buckets if the framework programmed - * the gscan buckets badly, for now return error if there are duplicates. - * Plus as an added bonus, we get all channels in Gscan bucket - * set to 1 for dhd_pno_merge_gscan_pno_channels() - */ - for (i = 0; i < num_channels; i++) { - if (!ch_scratch_pad[chan_list[i]]) { - ch_scratch_pad[chan_list[i]] = 1; - } else { - DHD_ERROR(("%s: Duplicate channel - %d programmed in channel bucket\n", - __FUNCTION__, chan_list[i])); - MFREE(dhd->osh, ch_bucket, ((*num_buckets) * - sizeof(wl_pfn_gscan_channel_bucket_t))); - *num_buckets_to_fw = *num_buckets = 0; - kfree(ch_scratch_pad); - return NULL; - } - - } - _params->params_gscan.max_ch_bucket_freq = max; /* Legacy PNO maybe running, which means we need to create a legacy PNO bucket - * Plus need to remove duplicates as the legacy PNO chan_list may have common channels - * If channel is to be scanned more frequently as per gscan requirements - * remove from legacy PNO ch_bucket. Similarly, if legacy wants a channel scanned - * more often, it is removed from the Gscan channel bucket. - * In the end both are satisfied. + * Get GCF of Legacy PNO and Gscan scanfreq */ - if (is_pno_legacy_running) - dhd_pno_merge_gscan_pno_channels(_pno_state, chan_list, - ch_scratch_pad, ch_bucket, num_buckets_to_fw, num_channels); - - kfree(ch_scratch_pad); + if (is_pno_legacy_running) { + dhd_pno_params_t *_params1 = &_pno_state->pno_params_arr[INDEX_OF_LEGACY_PARAMS]; + uint16 *legacy_chan_list = _params1->params_legacy.chan_list; + uint16 common_freq; + uint32 legacy_bucket_idx = _params->params_gscan.nchannel_buckets; + /* If no space is left then only gscan buckets will be sent to FW */ + if (nchan) { + common_freq = gcd(_params->params_gscan.scan_fr, + _params1->params_legacy.scan_fr); + max = gscan_buckets[0].bucket_freq_multiple; + /* GSCAN buckets */ + for (i = 0; i < _params->params_gscan.nchannel_buckets; i++) { + ch_bucket[i].bucket_freq_multiple *= _params->params_gscan.scan_fr; + ch_bucket[i].bucket_freq_multiple /= common_freq; + if (max < gscan_buckets[i].bucket_freq_multiple) + max = gscan_buckets[i].bucket_freq_multiple; + } + /* Legacy PNO bucket */ + ch_bucket[legacy_bucket_idx].bucket_freq_multiple = + _params1->params_legacy.scan_fr; + ch_bucket[legacy_bucket_idx].bucket_freq_multiple /= + common_freq; + _params->params_gscan.max_ch_bucket_freq = MAX(max, + ch_bucket[legacy_bucket_idx].bucket_freq_multiple); + ch_bucket[legacy_bucket_idx].flag = CH_BUCKET_REPORT_REGULAR; + /* Now add channels to the legacy scan bucket */ + for (i = 0; i < _params1->params_legacy.nchan && nchan; i++, nchan--) { + ptr[i] = legacy_chan_list[i]; + num_channels++; + } + ch_bucket[legacy_bucket_idx].bucket_end_index = num_channels - 1; + *num_buckets_to_fw = *num_buckets_to_fw + 1; + DHD_PNO(("end_idx %d freq_mult - %d\n", + ch_bucket[legacy_bucket_idx].bucket_end_index, + ch_bucket[legacy_bucket_idx].bucket_freq_multiple)); + } + } return ch_bucket; } @@ -2324,6 +2489,12 @@ static int dhd_pno_stop_for_gscan(dhd_pub_t *dhd) DHD_ERROR(("%s : GSCAN is not enabled\n", __FUNCTION__)); goto exit; } + if (_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan.mscan) { + /* retrieve the batching data from firmware into host */ + err = dhd_wait_batch_results_complete(dhd); + if (err != BCME_OK) + goto exit; + } mutex_lock(&_pno_state->pno_mutex); mode = _pno_state->pno_mode & ~DHD_PNO_GSCAN_MODE; err = dhd_pno_clean(dhd); @@ -2335,6 +2506,7 @@ static int dhd_pno_stop_for_gscan(dhd_pub_t *dhd) } _pno_state->pno_mode = mode; mutex_unlock(&_pno_state->pno_mutex); + _pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan.ssid_ext_last_used_index = 0; /* Reprogram Legacy PNO if it was running */ if (_pno_state->pno_mode & DHD_PNO_LEGACY_MODE) { @@ -2358,7 +2530,6 @@ static int dhd_pno_stop_for_gscan(dhd_pub_t *dhd) params_legacy->pno_freq_expo_max, chan_list, params_legacy->nchan); if (err < 0) { - _pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE; DHD_ERROR(("%s : failed to restart legacy PNO scan(err: %d)\n", __FUNCTION__, err)); goto exit; @@ -2383,9 +2554,7 @@ dhd_pno_initiate_gscan_request(dhd_pub_t *dhd, bool run, bool flush) NULL_CHECK(dhd->pno_state, "pno_state is NULL", err); _pno_state = PNO_GET_PNOSTATE(dhd); - DHD_PNO(("%s enter\n", __FUNCTION__)); - /* !!Temp!! Will remove after debug */ - printk("%s enter run - %d flush %d\n", __FUNCTION__, run, flush); + DHD_ERROR(("%s enter - run %d flush %d\n", __FUNCTION__, run, flush)); params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; gscan_params = ¶ms->params_gscan; @@ -2436,6 +2605,7 @@ int dhd_pno_enable_full_scan_result(dhd_pub_t *dhd, bool real_time_flag) if (old_flag != gscan_params->send_all_results_flag) { wl_pfn_gscan_cfg_t gscan_cfg; + gscan_cfg.version = WL_GSCAN_CFG_VERSION; gscan_cfg.flags = (gscan_params->send_all_results_flag & GSCAN_SEND_ALL_RESULTS_MASK); gscan_cfg.flags |= GSCAN_CFG_FLAGS_ONLY_MASK; @@ -2458,6 +2628,7 @@ int dhd_pno_enable_full_scan_result(dhd_pub_t *dhd, bool real_time_flag) return err; } +/* Cleanup any consumed results */ int dhd_gscan_batch_cache_cleanup(dhd_pub_t *dhd) { int ret = 0; @@ -2511,7 +2682,10 @@ static int _dhd_pno_get_gscan_batch_from_fw(dhd_pub_t *dhd) err = BCME_UNSUPPORTED; goto exit; } - + if (!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE)) { + DHD_ERROR(("%s: GSCAN is not enabled\n", __FUNCTION__)); + goto exit; + } gscan_params = ¶ms->params_gscan; nAPs_per_scan = (uint8 *) MALLOC(dhd->osh, gscan_params->mscan); @@ -2526,13 +2700,7 @@ static int _dhd_pno_get_gscan_batch_from_fw(dhd_pub_t *dhd) mutex_lock(&_pno_state->pno_mutex); - iter = gscan_params->gscan_batch_cache; - /* If a cache has not been consumed , just delete it */ - while (iter) { - iter->tot_consumed = iter->tot_count; - iter = iter->next; - } - dhd_gscan_batch_cache_cleanup(dhd); + dhd_gscan_clear_all_batch_results(dhd); if (!(_pno_state->pno_mode & DHD_PNO_GSCAN_MODE)) { DHD_ERROR(("%s : GSCAN is not enabled\n", __FUNCTION__)); @@ -2610,6 +2778,7 @@ static int _dhd_pno_get_gscan_batch_from_fw(dhd_pub_t *dhd) DHD_PNO(("scan_id %d tot_count %d\n", scan_id, nAPs_per_scan[i])); iter->tot_count = nAPs_per_scan[i]; iter->tot_consumed = 0; + iter->flag = 0; if (plnetinfo->flags & PFN_PARTIAL_SCAN_MASK) { DHD_PNO(("This scan is aborted\n")); iter->flag = (ENABLE << PNO_STATUS_ABORT); @@ -2711,11 +2880,8 @@ _dhd_pno_get_for_batch(dhd_pub_t *dhd, char *buf, int bufsize, int reason) err = BCME_UNSUPPORTED; goto exit_no_unlock; } -#ifdef GSCAN_SUPPORT - if (!(_pno_state->pno_mode & (DHD_PNO_BATCH_MODE | DHD_PNO_GSCAN_MODE))) { -#else + if (!(_pno_state->pno_mode & DHD_PNO_BATCH_MODE)) { -#endif /* GSCAN_SUPPORT */ DHD_ERROR(("%s: Batching SCAN mode is not enabled\n", __FUNCTION__)); goto exit_no_unlock; } @@ -2896,9 +3062,10 @@ _dhd_pno_get_for_batch(dhd_pub_t *dhd, char *buf, int bufsize, int reason) list_del(&pscan_results->list); MFREE(dhd->osh, pscan_results, SCAN_RESULTS_SIZE); _params->params_batch.get_batch.top_node_cnt--; + } else { + /* increase total scan count using current scan count */ + _params->params_batch.get_batch.tot_scan_cnt += pscan_results->cnt_header; } - /* increase total scan count using current scan count */ - _params->params_batch.get_batch.tot_scan_cnt += pscan_results->cnt_header; if (buf && bufsize) { /* This is a first try to get batching results */ @@ -2949,20 +3116,15 @@ _dhd_pno_get_batch_handler(struct work_struct *work) DHD_ERROR(("%s : dhd is NULL\n", __FUNCTION__)); return; } - #ifdef GSCAN_SUPPORT - if (_pno_state->pno_mode & DHD_PNO_GSCAN_MODE) { - _dhd_pno_get_gscan_batch_from_fw(dhd); - return; - } else + _dhd_pno_get_gscan_batch_from_fw(dhd); #endif /* GSCAN_SUPPORT */ - { + if (_pno_state->pno_mode & DHD_PNO_BATCH_MODE) { params_batch = &_pno_state->pno_params_arr[INDEX_OF_BATCH_PARAMS].params_batch; _dhd_pno_get_for_batch(dhd, params_batch->get_batch.buf, params_batch->get_batch.bufsize, params_batch->get_batch.reason); } - } int @@ -2995,7 +3157,7 @@ dhd_pno_get_for_batch(dhd_pub_t *dhd, char *buf, int bufsize, int reason) err = dhd_retreive_batch_scan_results(dhd); if (err == BCME_OK) { wait_event_interruptible_timeout(_pno_state->batch_get_wait, - is_batch_retreival_complete(gscan_params), + is_batch_retrieval_complete(gscan_params), msecs_to_jiffies(GSCAN_BATCH_GET_MAX_WAIT)); } } else @@ -3082,7 +3244,6 @@ dhd_pno_stop_for_batch(dhd_pub_t *dhd) _params_legacy->pno_freq_expo_max, _params_legacy->chan_list, _params_legacy->nchan); if (err < 0) { - _pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE; DHD_ERROR(("%s : failed to restart legacy PNO scan(err: %d)\n", __FUNCTION__, err)); goto exit; @@ -3336,7 +3497,6 @@ dhd_pno_stop_for_hotlist(dhd_pub_t *dhd) _params_legacy->pno_freq_expo_max, _params_legacy->chan_list, _params_legacy->nchan); if (err < 0) { - _pno_state->pno_mode &= ~DHD_PNO_LEGACY_MODE; DHD_ERROR(("%s : failed to restart legacy PNO scan(err: %d)\n", __FUNCTION__, err)); goto exit; @@ -3373,6 +3533,9 @@ int dhd_retreive_batch_scan_results(dhd_pub_t *dhd) dhd_pno_status_info_t *_pno_state; dhd_pno_params_t *_params; struct dhd_pno_batch_params *params_batch; + + NULL_CHECK(dhd, "dhd is NULL", err); + NULL_CHECK(dhd->pno_state, "pno_state is NULL", err); _pno_state = PNO_GET_PNOSTATE(dhd); _params = &_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS]; @@ -3383,6 +3546,7 @@ int dhd_retreive_batch_scan_results(dhd_pub_t *dhd) params_batch->get_batch.bufsize = 0; params_batch->get_batch.reason = PNO_STATUS_EVENT; _params->params_gscan.get_batch_flag = GSCAN_BATCH_RETRIEVAL_IN_PROGRESS; + smp_wmb(); schedule_work(&_pno_state->work); } else { DHD_PNO(("%s : WLC_E_PFN_BEST_BATCHING retrieval" @@ -3560,6 +3724,104 @@ dhd_process_full_gscan_result(dhd_pub_t *dhd, const void *data, int *size) return result; } +void * +dhd_pno_process_epno_result(dhd_pub_t *dhd, const void *data, uint32 event, int *size) +{ + dhd_epno_results_t *results = NULL; + dhd_pno_status_info_t *_pno_state = PNO_GET_PNOSTATE(dhd); + struct dhd_pno_gscan_params *gscan_params; + uint32 count, mem_needed = 0, i; + uint8 ssid[DOT11_MAX_SSID_LEN + 1]; + struct ether_addr *bssid; + + *size = 0; + if (!_pno_state) + return NULL; + gscan_params = &(_pno_state->pno_params_arr[INDEX_OF_GSCAN_PARAMS].params_gscan); + + if (event == WLC_E_PFN_SSID_EXT) { + wl_pfn_ssid_ext_result_t *evt_data; + evt_data = (wl_pfn_ssid_ext_result_t *) data; + + if (evt_data->version != PFN_SSID_EXT_VERSION) { + DHD_PNO(("ePNO event: Incorrect version %d %d\n", evt_data->version, + PFN_SSID_EXT_VERSION)); + return NULL; + } + count = evt_data->count; + mem_needed = sizeof(dhd_epno_results_t) * count; + results = (dhd_epno_results_t *) kmalloc(mem_needed, GFP_KERNEL); + if (!results) { + DHD_ERROR(("%s: Can't malloc %d bytes for results\n", __FUNCTION__, + mem_needed)); + return NULL; + } + DHD_ERROR(("Rx'ed WLC_E_PFN_SSID_EXT event: %d results\n", count)); + for (i = 0; i < count; i++) { + results[i].rssi = evt_data->net[i].rssi; + results[i].channel = wf_channel2mhz(evt_data->net[i].channel, + (evt_data->net[i].channel <= CH_MAX_2G_CHANNEL ? + WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G)); + results[i].flags = evt_data->net[i].flags; + dhd_pno_idx_to_ssid(gscan_params, &results[i], + evt_data->net[i].index); + memcpy(ssid, results[i].ssid, results[i].ssid_len); + bssid = &results[i].bssid; + memcpy(bssid, &evt_data->net[i].bssid, ETHER_ADDR_LEN); + ssid[results[i].ssid_len] = '\0'; + DHD_PNO(("ssid - %s bssid %02x:%02x:%02x:%02x:%02x:%02x " + "idx %d ch %d rssi %d flags %d\n", ssid, + bssid->octet[0], bssid->octet[1], + bssid->octet[2], bssid->octet[3], + bssid->octet[4], bssid->octet[5], + evt_data->net[i].index, results[i].channel, + results[i].rssi, results[i].flags)); + } + } else if (event == WLC_E_PFN_NET_FOUND || event == WLC_E_PFN_NET_LOST) { + wl_pfn_scanresults_t *pfn_result = (wl_pfn_scanresults_t *)data; + wl_pfn_net_info_t *net; + + if (pfn_result->version != PFN_SCANRESULT_VERSION) { + DHD_ERROR(("%s event %d: Incorrect version %d %d\n", __FUNCTION__, event, + pfn_result->version, PFN_SCANRESULT_VERSION)); + return NULL; + } + count = pfn_result->count; + mem_needed = sizeof(dhd_epno_results_t) * count; + results = (dhd_epno_results_t *) kmalloc(mem_needed, GFP_KERNEL); + if (!results) { + DHD_ERROR(("%s: Can't malloc %d bytes for results\n", __FUNCTION__, + mem_needed)); + return NULL; + } + for (i = 0; i < count; i++) { + net = &pfn_result->netinfo[i]; + results[i].rssi = net->RSSI; + results[i].channel = wf_channel2mhz(net->pfnsubnet.channel, + (net->pfnsubnet.channel <= CH_MAX_2G_CHANNEL ? + WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G)); + results[i].flags = (event == WLC_E_PFN_NET_FOUND) ? + WL_PFN_SSID_EXT_FOUND: WL_PFN_SSID_EXT_LOST; + results[i].ssid_len = min(net->pfnsubnet.SSID_len, + (uint8)DOT11_MAX_SSID_LEN); + bssid = &results[i].bssid; + memcpy(bssid, &net->pfnsubnet.BSSID, ETHER_ADDR_LEN); + memcpy(results[i].ssid, net->pfnsubnet.SSID, results[i].ssid_len); + memcpy(ssid, results[i].ssid, results[i].ssid_len); + ssid[results[i].ssid_len] = '\0'; + DHD_PNO(("ssid - %s bssid %02x:%02x:%02x:%02x:%02x:%02x " + "ch %d rssi %d flags %d\n", ssid, + bssid->octet[0], bssid->octet[1], + bssid->octet[2], bssid->octet[3], + bssid->octet[4], bssid->octet[5], + results[i].channel, results[i].rssi, results[i].flags)); + } + } + *size = mem_needed; + return results; +} + + void *dhd_handle_hotlist_scan_evt(dhd_pub_t *dhd, const void *event_data, int *send_evt_bytes, hotlist_type_t type) { diff --git a/drivers/net/wireless/bcmdhd/dhd_pno.h b/drivers/net/wireless/bcmdhd/dhd_pno.h index 63b6c033c42..4b8c286c081 100644 --- a/drivers/net/wireless/bcmdhd/dhd_pno.h +++ b/drivers/net/wireless/bcmdhd/dhd_pno.h @@ -63,6 +63,9 @@ #ifdef GSCAN_SUPPORT #define GSCAN_MAX_CH_BUCKETS 8 +#define GSCAN_MAX_CHANNELS_IN_BUCKET 32 +#define GSCAN_MAX_AP_CACHE_PER_SCAN 16 +#define GSCAN_MAX_AP_CACHE 160 #define GSCAN_BG_BAND_MASK (1 << 0) #define GSCAN_A_BAND_MASK (1 << 1) #define GSCAN_DFS_MASK (1 << 2) @@ -72,9 +75,17 @@ #define GSCAN_FLUSH_HOTLIST_CFG (1 << 0) #define GSCAN_FLUSH_SIGNIFICANT_CFG (1 << 1) #define GSCAN_FLUSH_SCAN_CFG (1 << 2) +#define GSCAN_FLUSH_EPNO_CFG (1 << 3) #define GSCAN_FLUSH_ALL_CFG (GSCAN_FLUSH_SCAN_CFG | \ GSCAN_FLUSH_SIGNIFICANT_CFG | \ - GSCAN_FLUSH_HOTLIST_CFG) + GSCAN_FLUSH_HOTLIST_CFG | \ + GSCAN_FLUSH_EPNO_CFG) +#define DHD_EPNO_HIDDEN_SSID (1 << 0) +#define DHD_EPNO_A_BAND_TRIG (1 << 1) +#define DHD_EPNO_BG_BAND_TRIG (1 << 2) +#define DHD_EPNO_STRICT_MATCH (1 << 3) +#define DHD_PNO_USE_SSID (DHD_EPNO_HIDDEN_SSID | DHD_EPNO_STRICT_MATCH) + /* Do not change GSCAN_BATCH_RETRIEVAL_COMPLETE */ #define GSCAN_BATCH_RETRIEVAL_COMPLETE 0 #define GSCAN_BATCH_RETRIEVAL_IN_PROGRESS 1 @@ -84,6 +95,9 @@ #define GSCAN_BATCH_GET_MAX_WAIT 500 #define CHANNEL_BUCKET_EMPTY_INDEX 0xFFFF +#define GSCAN_RETRY_THRESHOLD 3 +#define MAX_EPNO_SSID_NUM 32 + #endif /* GSCAN_SUPPORT */ enum scan_status { @@ -150,7 +164,10 @@ typedef enum dhd_pno_gscan_cmd_cfg { DHD_PNO_SCAN_CFG_ID, DHD_PNO_GET_CAPABILITIES, DHD_PNO_GET_BATCH_RESULTS, - DHD_PNO_GET_CHANNEL_LIST + DHD_PNO_GET_CHANNEL_LIST, + DHD_PNO_GET_EPNO_SSID_ELEM, + DHD_PNO_EPNO_CFG_ID, + DHD_PNO_GET_AUTOJOIN_CAPABILITIES } dhd_pno_gscan_cmd_cfg_t; typedef enum dhd_pno_mode { @@ -258,6 +275,8 @@ struct dhd_pno_hotlist_params { struct list_head bssid_list; }; #ifdef GSCAN_SUPPORT +#define DHD_PNO_REPORT_NO_BATCH (1 << 2) + typedef struct dhd_pno_gscan_channel_bucket { uint16 bucket_freq_multiple; /* band = 1 All bg band channels, @@ -267,9 +286,38 @@ typedef struct dhd_pno_gscan_channel_bucket { uint16 band; uint8 report_flag; uint8 num_channels; - uint16 chan_list[GSCAN_MAX_CH_BUCKETS]; + uint16 repeat; + uint16 bucket_max_multiple; + uint16 chan_list[GSCAN_MAX_CHANNELS_IN_BUCKET]; } dhd_pno_gscan_channel_bucket_t; + +#define DHD_PNO_AUTH_CODE_OPEN 1 /* Open */ +#define DHD_PNO_AUTH_CODE_PSK 2 /* WPA_PSK or WPA2PSK */ +#define DHD_PNO_AUTH_CODE_EAPOL 4 /* any EAPOL */ + +#define DHD_EPNO_DEFAULT_INDEX 0xFFFFFFFF + +typedef struct dhd_epno_params { + uint8 ssid[DOT11_MAX_SSID_LEN]; + uint8 ssid_len; + int8 rssi_thresh; + uint8 flags; + uint8 auth; + /* index required only for visble ssid */ + uint32 index; + struct list_head list; +} dhd_epno_params_t; + +typedef struct dhd_epno_results { + uint8 ssid[DOT11_MAX_SSID_LEN]; + uint8 ssid_len; + int8 rssi; + uint16 channel; + uint16 flags; + struct ether_addr bssid; +} dhd_epno_results_t; + struct dhd_pno_swc_evt_param { uint16 results_rxed_so_far; wl_pfn_significant_net_t *change_array; @@ -306,6 +354,9 @@ typedef struct dhd_pno_gscan_capabilities { int max_scan_reporting_threshold; int max_hotlist_aps; int max_significant_wifi_change_aps; + int max_epno_ssid_crc32; + int max_epno_hidden_ssid; + int max_white_list_ssid; } dhd_pno_gscan_capabilities_t; struct dhd_pno_gscan_params { @@ -326,10 +377,18 @@ struct dhd_pno_gscan_params { gscan_results_cache_t *gscan_hotlist_lost; uint16 nbssid_significant_change; uint16 nbssid_hotlist; + uint16 num_epno_ssid; + uint8 num_visible_epno_ssid; + /* To keep track of visble ssid index + * across multiple FW configs i.e. config + * w/o clear in between + */ + uint8 ssid_ext_last_used_index; struct dhd_pno_swc_evt_param param_significant; struct dhd_pno_gscan_channel_bucket channel_bucket[GSCAN_MAX_CH_BUCKETS]; struct list_head hotlist_bssid_list; struct list_head significant_bssid_list; + struct list_head epno_ssid_list; }; typedef struct gscan_scan_params { @@ -385,6 +444,7 @@ typedef union dhd_pno_params { #endif /* GSCAN_SUPPORT */ } dhd_pno_params_t; typedef struct dhd_pno_status_info { + uint8 pno_oui[DOT11_OUI_LEN]; dhd_pub_t *dhd; struct work_struct work; struct mutex pno_mutex; @@ -423,6 +483,8 @@ dhd_dev_pno_stop_for_batch(struct net_device *dev); extern int dhd_dev_pno_set_for_hotlist(struct net_device *dev, wl_pfn_bssid_t *p_pfn_bssid, struct dhd_pno_hotlist_params *hotlist_params); +extern int dhd_dev_pno_set_mac_oui(struct net_device *dev, uint8 *oui); +extern bool dhd_dev_is_legacy_pno_enabled(struct net_device *dev); #ifdef GSCAN_SUPPORT extern int dhd_dev_pno_set_cfg_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type, @@ -430,7 +492,7 @@ dhd_dev_pno_set_cfg_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type, extern void * dhd_dev_pno_get_gscan(struct net_device *dev, dhd_pno_gscan_cmd_cfg_t type, void *info, uint32 *len); -void dhd_dev_pno_lock_access_batch_results(struct net_device *dev); +int dhd_dev_pno_lock_access_batch_results(struct net_device *dev); void dhd_dev_pno_unlock_access_batch_results(struct net_device *dev); extern int dhd_dev_pno_run_gscan(struct net_device *dev, bool run, bool flush); extern int dhd_dev_pno_enable_full_scan_result(struct net_device *dev, bool real_time); @@ -443,7 +505,9 @@ void * dhd_dev_process_full_gscan_result(struct net_device *dev, const void *data, int *send_evt_bytes); extern int dhd_dev_gscan_batch_cache_cleanup(struct net_device *dev); extern void dhd_dev_gscan_hotlist_cache_cleanup(struct net_device *dev, hotlist_type_t type); -extern void dhd_dev_wait_batch_results_complete(struct net_device *dev); +extern int dhd_dev_wait_batch_results_complete(struct net_device *dev); +extern void * dhd_dev_process_epno_result(struct net_device *dev, + const void *data, uint32 event, int *send_evt_bytes); #endif /* GSCAN_SUPPORT */ /* dhd pno fuctions */ extern int dhd_pno_stop_for_ssid(dhd_pub_t *dhd); @@ -467,13 +531,14 @@ extern int dhd_pno_event_handler(dhd_pub_t *dhd, wl_event_msg_t *event, void *ev extern int dhd_pno_init(dhd_pub_t *dhd); extern int dhd_pno_deinit(dhd_pub_t *dhd); extern bool dhd_is_pno_supported(dhd_pub_t *dhd); - +extern int dhd_pno_set_mac_oui(dhd_pub_t *dhd, uint8 *oui); +extern bool dhd_is_legacy_pno_enabled(dhd_pub_t *dhd); #ifdef GSCAN_SUPPORT extern int dhd_pno_set_cfg_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, void *buf, uint8 flush); extern void * dhd_pno_get_gscan(dhd_pub_t *dhd, dhd_pno_gscan_cmd_cfg_t type, void *info, uint32 *len); -extern void dhd_pno_lock_batch_results(dhd_pub_t *dhd); +extern int dhd_pno_lock_batch_results(dhd_pub_t *dhd); extern void dhd_pno_unlock_batch_results(dhd_pub_t *dhd); extern int dhd_pno_initiate_gscan_request(dhd_pub_t *dhd, bool run, bool flush); extern int dhd_pno_enable_full_scan_result(dhd_pub_t *dhd, bool real_time_flag); @@ -486,6 +551,8 @@ extern void *dhd_process_full_gscan_result(dhd_pub_t *dhd, const void *event_dat int *send_evt_bytes); extern int dhd_gscan_batch_cache_cleanup(dhd_pub_t *dhd); extern void dhd_gscan_hotlist_cache_cleanup(dhd_pub_t *dhd, hotlist_type_t type); -extern void dhd_wait_batch_results_complete(dhd_pub_t *dhd); +extern int dhd_wait_batch_results_complete(dhd_pub_t *dhd); +extern void * dhd_pno_process_epno_result(dhd_pub_t *dhd, const void *data, + uint32 event, int *size); #endif /* GSCAN_SUPPORT */ #endif /* __DHD_PNO_H__ */ diff --git a/drivers/net/wireless/bcmdhd/dhd_sdio.c b/drivers/net/wireless/bcmdhd/dhd_sdio.c index 5e4fa1bd1d0..6fb3b9279ca 100755 --- a/drivers/net/wireless/bcmdhd/dhd_sdio.c +++ b/drivers/net/wireless/bcmdhd/dhd_sdio.c @@ -370,6 +370,7 @@ typedef struct dhd_bus { uint8 glom_mode; /* Glom mode - 0-copy mode, 1 - Multi-descriptor mode */ uint32 glomsize; /* Glom size limitation */ #endif + wake_counts_t wake_counts; } dhd_bus_t; /* clkstate */ @@ -2833,7 +2834,9 @@ void dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf) { dhd_bus_t *bus = dhdp->bus; - +#if defined(DHD_WAKE_STATUS) && defined(DHD_WAKE_EVENT_STATUS) + int i; +#endif bcm_bprintf(strbuf, "Bus SDIO structure:\n"); bcm_bprintf(strbuf, "hostintmask 0x%08x intstatus 0x%08x sdpcm_ver %d\n", bus->hostintmask, bus->intstatus, bus->sdpcm_ver); @@ -2842,6 +2845,29 @@ dhd_bus_dump(dhd_pub_t *dhdp, struct bcmstrbuf *strbuf) bus->rxlen, bus->rx_seq); bcm_bprintf(strbuf, "intr %d intrcount %u lastintrs %u spurious %u\n", bus->intr, bus->intrcount, bus->lastintrs, bus->spurious); +#ifdef DHD_WAKE_STATUS + bcm_bprintf(strbuf, "wake %u rxwake %u readctrlwake %u\n", + bcmsdh_get_total_wake(bus->sdh), bus->wake_counts.rxwake, + bus->wake_counts.rcwake); +#ifdef DHD_WAKE_RX_STATUS + bcm_bprintf(strbuf, " unicast %u multicast %u broadcast %u arp %u\n", + bus->wake_counts.rx_ucast, bus->wake_counts.rx_mcast, + bus->wake_counts.rx_bcast, bus->wake_counts.rx_arp); + bcm_bprintf(strbuf, " multi4 %u multi6 %u icmp6 %u multiother %u\n", + bus->wake_counts.rx_multi_ipv4, bus->wake_counts.rx_multi_ipv6, + bus->wake_counts.rx_icmpv6, bus->wake_counts.rx_multi_other); + bcm_bprintf(strbuf, " icmp6_ra %u, icmp6_na %u, icmp6_ns %u\n", + bus->wake_counts.rx_icmpv6_ra, bus->wake_counts.rx_icmpv6_na, + bus->wake_counts.rx_icmpv6_ns); +#endif +#ifdef DHD_WAKE_EVENT_STATUS + for (i = 0; i < WLC_E_LAST; i++) + if (bus->wake_counts.rc_event[i] != 0) + bcm_bprintf(strbuf, " %s = %u\n", bcmevent_get_name(i), + bus->wake_counts.rc_event[i]); + bcm_bprintf(strbuf, "\n"); +#endif +#endif bcm_bprintf(strbuf, "pollrate %u pollcnt %u regfails %u\n", bus->pollrate, bus->pollcnt, bus->regfails); @@ -4897,7 +4923,7 @@ dhd_process_pkt_reorder_info(dhd_pub_t *dhd, uchar *reorder_info_buf, uint reord void **pkt, uint32 *pkt_count); static uint8 -dhdsdio_rxglom(dhd_bus_t *bus, uint8 rxseq) +dhdsdio_rxglom(dhd_bus_t *bus, uint8 rxseq, int pkt_wake) { uint16 dlen, totlen; uint8 *dptr, num = 0; @@ -5310,7 +5336,8 @@ dhdsdio_rxglom(dhd_bus_t *bus, uint8 rxseq) } while (temp); if (cnt) { dhd_os_sdunlock(bus->dhd); - dhd_rx_frame(bus->dhd, idx, list_head[idx], cnt, 0); + dhd_rx_frame(bus->dhd, idx, list_head[idx], cnt, 0, pkt_wake, &bus->wake_counts); + pkt_wake = 0; dhd_os_sdlock(bus->dhd); } } @@ -5348,13 +5375,16 @@ dhdsdio_readframes(dhd_bus_t *bus, uint maxframes, bool *finished) uchar reorder_info_buf[WLHOST_REORDERDATA_TOTLEN]; uint reorder_info_len; uint pkt_count; - + int pkt_wake = 0; #if defined(DHD_DEBUG) || defined(SDTEST) bool sdtest = FALSE; /* To limit message spew from test mode */ #endif DHD_TRACE(("%s: Enter\n", __FUNCTION__)); +#ifdef DHD_WAKE_STATUS + pkt_wake = bcmsdh_set_get_wake(0); +#endif bus->readframes = TRUE; if (!KSO_ENAB(bus)) { @@ -5396,7 +5426,8 @@ dhdsdio_readframes(dhd_bus_t *bus, uint maxframes, bool *finished) uint8 cnt; DHD_GLOM(("%s: calling rxglom: glomd %p, glom %p\n", __FUNCTION__, bus->glomd, bus->glom)); - cnt = dhdsdio_rxglom(bus, rxseq); + cnt = dhdsdio_rxglom(bus, rxseq, pkt_wake); + pkt_wake = 0; DHD_GLOM(("%s: rxglom returned %d\n", __FUNCTION__, cnt)); rxseq += cnt - 1; rxleft = (rxleft > cnt) ? (rxleft - cnt) : 1; @@ -5913,7 +5944,8 @@ dhdsdio_readframes(dhd_bus_t *bus, uint maxframes, bool *finished) /* Unlock during rx call */ dhd_os_sdunlock(bus->dhd); - dhd_rx_frame(bus->dhd, ifidx, pkt, pkt_count, chan); + dhd_rx_frame(bus->dhd, ifidx, pkt, pkt_count, chan, pkt_wake, &bus->wake_counts); + pkt_wake = 0; dhd_os_sdlock(bus->dhd); } rxcount = maxframes - rxleft; diff --git a/drivers/net/wireless/bcmdhd/dngl_stats.h b/drivers/net/wireless/bcmdhd/dngl_stats.h index 59bc4431c95..47d8ebe7eb1 100644 --- a/drivers/net/wireless/bcmdhd/dngl_stats.h +++ b/drivers/net/wireless/bcmdhd/dngl_stats.h @@ -208,6 +208,15 @@ typedef struct { wifi_interface_handle iface; // wifi interface wifi_interface_info info; // current state of the interface u32 beacon_rx; // access point beacon received count from connected AP + u64 average_tsf_offset; // average beacon offset encountered (beacon_TSF - TBTT) + // The average_tsf_offset field is used so as to calculate the + // typical beacon contention time on the channel as well may be + // used to debug beacon synchronization and related power consumption issue + u32 leaky_ap_detected; // indicate that this AP typically leaks packets beyond the driver guard time. + u32 leaky_ap_avg_num_frames_leaked; // average number of frame leaked by AP after frame with PM bit set was ACK'ed by AP + u32 leaky_ap_guard_time; // guard time currently in force (when implementing IEEE power management based on + // frame control PM bit), How long driver waits before shutting down the radio and + // after receiving an ACK for a data frame with PM bit set) u32 mgmt_rx; // access point mgmt frames received count from connected AP (including Beacon) u32 mgmt_action_rx; // action frames received count u32 mgmt_action_tx; // action frames transmit count diff --git a/drivers/net/wireless/bcmdhd/include/bcmsdh.h b/drivers/net/wireless/bcmdhd/include/bcmsdh.h index 4937daaa094..bf4fcb65692 100644 --- a/drivers/net/wireless/bcmdhd/include/bcmsdh.h +++ b/drivers/net/wireless/bcmdhd/include/bcmsdh.h @@ -4,13 +4,13 @@ * abstract OS and BUS specific details of SDIO * * Copyright (C) 1999-2013, Broadcom Corporation - * + * * Unless you and Broadcom execute a separate written software license * agreement governing use of this software, this software is licensed to you * under the terms of the GNU General Public License version 2 (the "GPL"), * available at http://www.broadcom.com/licenses/GPLv2.php, with the * following added to such license: - * + * * As a special exception, the copyright holders of this software give you * permission to link this software with independent modules, and to copy and * distribute the resulting executable under terms of your choice, provided that @@ -18,7 +18,7 @@ * the license of that module. An independent module is a module which is not * derived from this software. The special exception does not apply to any * modifications of the software. - * + * * Notwithstanding the above, under no circumstances may you combine this * software in any way with any other Broadcom software provided under a license * other than the GPL, without Broadcom's express prior written consent. @@ -44,8 +44,25 @@ extern const uint bcmsdh_msglevel; #define BCMSDH_ADAPTER #endif /* BCMSDIO && (BCMSDIOH_STD || BCMSDIOH_BCM || BCMSDIOH_SPI) */ +/** + * BCMSDH API context + */ +typedef struct bcmsdh_info +{ + bool init_success; /* underlying driver successfully attached */ + void *sdioh; /* handler for sdioh */ + uint32 vendevid; /* Target Vendor and Device ID on SD bus */ + osl_t *osh; + bool regfail; /* Save status of last reg_read/reg_write call */ + uint32 sbwad; /* Save backplane window address */ +#ifdef DHD_WAKE_STATUS + unsigned int total_wake_count; + int pkt_wake; + int wake_irq; +#endif +} bcmsdh_info_t; + /* forward declarations */ -typedef struct bcmsdh_info bcmsdh_info_t; typedef void (*bcmsdh_cb_fn_t)(void *); extern struct device *pm_dev; @@ -80,6 +97,11 @@ extern void bcmsdh_intr_forward(void *sdh, bool pass); extern bool bcmsdh_intr_pending(void *sdh); #endif +#ifdef DHD_WAKE_STATUS +int bcmsdh_get_total_wake(bcmsdh_info_t *bcmsdh); +int bcmsdh_set_get_wake(int flag); +#endif + /* Register a callback to be called if and when bcmsdh detects * device removal. No-op in the case of non-removable/hardwired devices. */ @@ -216,7 +238,7 @@ extern int bcmsdh_register_oob_intr(void * dhdp); extern void bcmsdh_unregister_oob_intr(void); extern void bcmsdh_oob_intr_set(bool enable); extern bool bcmsdh_is_oob_intr_registered(void); -#endif +#endif /* Function to pass device-status bits to DHD. */ extern uint32 bcmsdh_get_dstatus(void *sdh); diff --git a/drivers/net/wireless/bcmdhd/include/wlioctl.h b/drivers/net/wireless/bcmdhd/include/wlioctl.h index 9602199b477..9288462c179 100644 --- a/drivers/net/wireless/bcmdhd/include/wlioctl.h +++ b/drivers/net/wireless/bcmdhd/include/wlioctl.h @@ -5,13 +5,13 @@ * Definitions subject to change without notice. * * Copyright (C) 1999-2013, Broadcom Corporation - * + * * Unless you and Broadcom execute a separate written software license * agreement governing use of this software, this software is licensed to you * under the terms of the GNU General Public License version 2 (the "GPL"), * available at http://www.broadcom.com/licenses/GPLv2.php, with the * following added to such license: - * + * * As a special exception, the copyright holders of this software give you * permission to link this software with independent modules, and to copy and * distribute the resulting executable under terms of your choice, provided that @@ -19,7 +19,7 @@ * the license of that module. An independent module is a module which is not * derived from this software. The special exception does not apply to any * modifications of the software. - * + * * Notwithstanding the above, under no circumstances may you combine this * software in any way with any other Broadcom software provided under a license * other than the GPL, without Broadcom's express prior written consent. @@ -324,7 +324,8 @@ typedef struct wlc_ssid { typedef struct wlc_ssid_ext { bool hidden; - uint32 SSID_len; + uint16 flags; + uint16 SSID_len; uchar SSID[DOT11_MAX_SSID_LEN]; } wlc_ssid_ext_t; @@ -562,6 +563,62 @@ typedef struct wl_join_params { */ } wl_join_params_t; +typedef struct wlc_roam_exp_params { + int8 a_band_boost_threshold; + int8 a_band_penalty_threshold; + uint8 a_band_boost_factor; + uint8 a_band_penalty_factor; + uint8 cur_bssid_boost; + int8 alert_roam_trigger_threshold; + uint16 a_band_max_boost; +} wlc_roam_exp_params_t; + +#define ROAM_EXP_CFG_VERSION 1 +#define ROAM_EXP_ENABLE_FLAG (1 << 0) +#define ROAM_EXP_CFG_PRESENT (1 << 1) +typedef struct wl_roam_exp_cfg { + uint8 version; + uint8 flags; + uint16 reserved; + wlc_roam_exp_params_t params; +} wl_roam_exp_cfg_t; + +typedef struct wl_bssid_pref_list { + struct ether_addr bssid; + /* Add this to modify rssi */ + int8 rssi_factor; + int8 flags; +} wl_bssid_pref_list_t; + +#define BSSID_PREF_LIST_VERSION 1 +#define ROAM_EXP_CLEAR_BSSID_PREF (1 << 0) +typedef struct wl_bssid_pref_cfg { + uint8 version; + uint8 flags; + uint16 count; + wl_bssid_pref_list_t bssids[1]; +} wl_bssid_pref_cfg_t; + +#define SSID_WHITELIST_VERSION 1 +#define ROAM_EXP_CLEAR_SSID_WHITELIST (1 << 0) +/* Roam SSID whitelist, ssids in this list are ok to */ +/* be considered as targets to join when considering a roam */ +typedef struct wl_ssid_whitelist { + uint8 version; + uint8 flags; + uint8 ssid_count; + uint8 reserved; + wlc_ssid_t ssids[1]; +} wl_ssid_whitelist_t; + +#define ROAM_EXP_EVENT_VERSION 1 +typedef struct wl_roam_exp_event { + uint8 version; + uint8 flags; + uint16 reserved; + wlc_ssid_t cur_ssid; +} wl_roam_exp_event_t; + #ifndef LINUX_POSTMOGRIFY_REMOVAL #define WL_JOIN_PARAMS_FIXED_SIZE (OFFSETOF(wl_join_params_t, params) + \ WL_ASSOC_PARAMS_FIXED_SIZE) @@ -887,7 +944,7 @@ typedef enum sup_auth_status { #define CRYPTO_ALGO_CKIP_MMH 8 #define CRYPTO_ALGO_WEP_MMH 9 #define CRYPTO_ALGO_NALG 10 -#endif +#endif #define CRYPTO_ALGO_PMK 12 /* for 802.1x supp to set PMK before 4-way */ #define CRYPTO_ALGO_BIP 13 /* 802.11w BIP (aes cmac) */ @@ -906,7 +963,7 @@ typedef enum sup_auth_status { #else #define WL_KF_RES_4 (1 << 4) /* Reserved for backward compat */ #define WL_KF_RES_5 (1 << 5) /* Reserved for backward compat */ -#endif +#endif #define WL_IBSS_PEER_GROUP_KEY (1 << 6) /* Indicates a group key for a IBSS PEER */ typedef struct wl_wsec_key { @@ -969,7 +1026,7 @@ typedef struct { #if defined(BCMEXTCCX) #define WPA_AUTH_CCKM 0x0008 /* CCKM */ #define WPA2_AUTH_CCKM 0x0010 /* CCKM2 */ -#endif +#endif /* #define WPA_AUTH_8021X 0x0020 */ /* 802.1x, reserved */ #define WPA2_AUTH_UNSPECIFIED 0x0040 /* over 802.1x */ #define WPA2_AUTH_PSK 0x0080 /* Pre-shared key */ @@ -1193,10 +1250,10 @@ typedef struct channel_info { } channel_info_t; /* For ioctls that take a list of MAC addresses */ -struct maclist { +typedef struct maclist { uint count; /* number of MAC addresses */ struct ether_addr ea[1]; /* variable length array of MAC addresses */ -}; +} maclist_t; #ifndef LINUX_POSTMOGRIFY_REMOVAL /* get pkt count struct passed through ioctl */ @@ -3927,6 +3984,10 @@ enum { #define PFN_SWC_RSSI_WINDOW_MAX 8 #define PFN_SWC_MAX_NUM_APS 16 +#define PFN_HOTLIST_MAX_NUM_APS 64 + +#define MAX_EPNO_HIDDEN_SSID 8 +#define MAX_WHITELIST_SSID 2 /* PFN network info structure */ typedef struct wl_pfn_subnet_info { @@ -4028,17 +4089,22 @@ typedef struct wl_pfn_cfg { #define CH_BUCKET_REPORT_REGULAR 0 #define CH_BUCKET_REPORT_FULL_RESULT 2 +#define CH_BUCKET_GSCAN 4 -typedef struct wl_pfn_gscan_channel_bucket { - uint16 bucket_end_index; +typedef struct wl_pfn_gscan_ch_bucket_cfg { + uint8 bucket_end_index; uint8 bucket_freq_multiple; - uint8 report_flag; -} wl_pfn_gscan_channel_bucket_t; - -#define GSCAN_SEND_ALL_RESULTS_MASK (1 << 0) -#define GSCAN_CFG_FLAGS_ONLY_MASK (1 << 7) - + uint8 flag; + uint8 reserved; + uint16 repeat; + uint16 max_freq_multiple; +} wl_pfn_gscan_ch_bucket_cfg_t; + +#define GSCAN_SEND_ALL_RESULTS_MASK (1 << 0) +#define GSCAN_CFG_FLAGS_ONLY_MASK (1 << 7) +#define WL_GSCAN_CFG_VERSION 2 typedef struct wl_pfn_gscan_cfg { + uint16 version; /* BIT0 1 = send probes/beacons to HOST * BIT1 Reserved * BIT2 Reserved @@ -4054,9 +4120,10 @@ typedef struct wl_pfn_gscan_cfg { uint8 swc_nbssid_threshold; /* Max=8 (for now) Size of rssi cache buffer */ uint8 swc_rssi_window_size; - uint16 count_of_channel_buckets; + uint8 count_of_channel_buckets; + uint8 retry_threshold; uint16 lost_ap_window; - wl_pfn_gscan_channel_bucket_t channel_bucket[1]; + wl_pfn_gscan_ch_bucket_cfg_t channel_bucket[1]; } wl_pfn_gscan_cfg_t; #define WL_PFN_REPORT_ALLNET 0 @@ -4065,7 +4132,8 @@ typedef struct wl_pfn_gscan_cfg { #define WL_PFN_CFG_FLAGS_PROHIBITED 0x00000001 /* Accept and use prohibited channels */ #define WL_PFN_CFG_FLAGS_RESERVED 0xfffffffe /* Remaining reserved for future use */ - +#define WL_PFN_SSID_A_BAND_TRIG 0x20 +#define WL_PFN_SSID_BG_BAND_TRIG 0x40 typedef struct wl_pfn { wlc_ssid_t ssid; /* ssid name and its length */ int32 flags; /* bit2: hidden */ @@ -4080,11 +4148,55 @@ typedef struct wl_pfn_list { uint32 count; wl_pfn_t pfn[1]; } wl_pfn_list_t; + +#define PFN_SSID_EXT_VERSION 2 + +typedef struct wl_pfn_ext { + uint8 flags; + int8 rssi_thresh; /* RSSI threshold, track only if RSSI > threshold */ + uint16 wpa_auth; /* Match the wpa auth type defined in wlioctl_defs.h */ + uint8 ssid[DOT11_MAX_SSID_LEN]; + uint8 ssid_len; + uint8 pad; +} wl_pfn_ext_t; + +typedef struct wl_pfn_ext_list { + uint16 version; + uint16 count; + wl_pfn_ext_t pfn_ext[1]; +} wl_pfn_ext_list_t; + +#define WL_PFN_SSID_EXT_FOUND 0x1 +#define WL_PFN_SSID_EXT_LOST 0x2 +typedef struct wl_pfn_result_ssid { + uint8 flags; + int8 rssi; + /* channel number */ + uint16 channel; + /* Assume idx in order of cfg */ + uint16 index; + struct ether_addr bssid; +} wl_pfn_result_ssid_crc32_t; + +typedef struct wl_pfn_ssid_ext_result { + uint16 version; + uint16 count; + wl_pfn_result_ssid_crc32_t net[1]; +} wl_pfn_ssid_ext_result_t; + +#define PFN_EXT_AUTH_CODE_OPEN 1 /* open */ +#define PFN_EXT_AUTH_CODE_PSK 2 /* WPA_PSK or WPA2PSK */ +#define PFN_EXT_AUTH_CODE_EAPOL 4 /* any EAPOL */ + #define WL_PFN_HIDDEN_BIT 2 #define PNO_SCAN_MAX_FW 508*1000 /* max time scan time in msec */ #define PNO_SCAN_MAX_FW_SEC PNO_SCAN_MAX_FW/1000 /* max time scan time in SEC */ #define PNO_SCAN_MIN_FW_SEC 10 /* min time scan time in SEC */ #define WL_PFN_HIDDEN_MASK 0x4 +#define MAX_SSID_WHITELIST_NUM 4 +#define MAX_BSSID_PREF_LIST_NUM 32 +#define MAX_BSSID_BLACKLIST_NUM 32 + #ifndef BESTN_MAX #define BESTN_MAX 3 #endif diff --git a/drivers/net/wireless/bcmdhd/wl_android.c b/drivers/net/wireless/bcmdhd/wl_android.c index a52baf50b6f..4c1d3183400 100644 --- a/drivers/net/wireless/bcmdhd/wl_android.c +++ b/drivers/net/wireless/bcmdhd/wl_android.c @@ -356,8 +356,9 @@ wls_parse_batching_cmd(struct net_device *dev, char *command, int total_len) " <> params\n", __FUNCTION__)); goto exit; } - while ((token2 = strsep(&pos2, - PNO_PARAM_CHANNEL_DELIMETER)) != NULL) { + + while ((token2 = strsep(&pos2, PNO_PARAM_CHANNEL_DELIMETER)) + != NULL) { if (token2 == NULL || !*token2) break; if (*token2 == '\0') @@ -368,11 +369,18 @@ wls_parse_batching_cmd(struct net_device *dev, char *command, int total_len) DHD_PNO(("band : %s\n", (*token2 == 'A')? "A" : "B")); } else { + if ((batch_params.nchan >= WL_NUMCHANNELS) || + (i >= WL_NUMCHANNELS)) { + DHD_ERROR(("Too many nchan %d\n", + batch_params.nchan)); + err = BCME_BUFTOOSHORT; + goto exit; + } batch_params.chan_list[i++] = - simple_strtol(token2, NULL, 0); + simple_strtol(token2, NULL, 0); batch_params.nchan++; - DHD_PNO(("channel :%d\n", - batch_params.chan_list[i-1])); + DHD_PNO(("channel: %d\n", + batch_params.chan_list[i-1])); } } } else if (!strncmp(param, PNO_PARAM_RTT, strlen(PNO_PARAM_MSCAN))) { @@ -1075,6 +1083,10 @@ int wl_android_priv_cmd(struct net_device *net, struct ifreq *ifr, int cmd) ret = -EINVAL; goto exit; } + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + goto exit; + } if (copy_from_user(&priv_cmd, ifr->ifr_data, sizeof(android_wifi_priv_cmd))) { ret = -EFAULT; goto exit; @@ -1456,6 +1468,18 @@ void *wifi_get_country_code(char *ccode) } #endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)) */ +bool wifi_process_partial_resume(int action) +{ +#ifdef CONFIG_PARTIALRESUME + if (wifi_control_data && wifi_control_data->partial_resume) { + return wifi_control_data->partial_resume(action); + } + return false; +#else + return false; +#endif +} + static int wifi_set_carddetect(int on) { DHD_ERROR(("%s = %d\n", __FUNCTION__, on)); diff --git a/drivers/net/wireless/bcmdhd/wl_android.h b/drivers/net/wireless/bcmdhd/wl_android.h index 394e128d7ed..90eee9a61ef 100644 --- a/drivers/net/wireless/bcmdhd/wl_android.h +++ b/drivers/net/wireless/bcmdhd/wl_android.h @@ -26,6 +26,9 @@ #include #include +#if defined(CONFIG_WIFI_CONTROL_FUNC) +#include +#endif #include /* If any feature uses the Generic Netlink Interface, put it here to enable WL_GENL @@ -60,4 +63,5 @@ int wifi_get_irq_number(unsigned long *irq_flags_ptr); int wifi_set_power(int on, unsigned long msec); int wifi_get_mac_addr(unsigned char *buf); void *wifi_get_country_code(char *ccode); +bool wifi_process_partial_resume(int action); #endif /* CONFIG_WIFI_CONTROL_FUNC */ diff --git a/drivers/net/wireless/bcmdhd/wl_cfg80211.c b/drivers/net/wireless/bcmdhd/wl_cfg80211.c index 15b2a365427..fbc5b95fab1 100644 --- a/drivers/net/wireless/bcmdhd/wl_cfg80211.c +++ b/drivers/net/wireless/bcmdhd/wl_cfg80211.c @@ -394,6 +394,8 @@ static s32 wl_notify_pfn_status(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, #ifdef GSCAN_SUPPORT static s32 wl_notify_gscan_event(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, const wl_event_msg_t *e, void *data); +static s32 wl_handle_roam_exp_event(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, + const wl_event_msg_t *e, void *data); #endif static s32 wl_notifier_change_state(struct wl_priv *wl, struct net_info *_net_info, enum wl_status state, bool set); @@ -715,7 +717,9 @@ static const u32 __wl_cipher_suites[] = { WLAN_CIPHER_SUITE_WEP104, WLAN_CIPHER_SUITE_TKIP, WLAN_CIPHER_SUITE_CCMP, +#ifdef MFP WLAN_CIPHER_SUITE_AES_CMAC, +#endif }; @@ -1012,10 +1016,13 @@ wl_validate_wps_ie(char *wps_ie, s32 wps_ie_len, bool *pbc) WL_DBG((" attr WPS_ID_CONFIG_METHODS: %x\n", HTON16(val))); } else if (subelt_id == WPS_ID_DEVICE_NAME) { char devname[100]; - memcpy(devname, subel, subelt_len); - devname[subelt_len] = '\0'; - WL_DBG((" attr WPS_ID_DEVICE_NAME: %s (len %u)\n", - devname, subelt_len)); + size_t namelen = MIN(subelt_len, sizeof(devname)); + if (namelen) { + memcpy(devname, subel, namelen); + devname[namelen - 1] = '\0'; + WL_DBG((" attr WPS_ID_DEVICE_NAME: %s (len %u)\n", + devname, subelt_len)); + } } else if (subelt_id == WPS_ID_DEVICE_PWD_ID) { valptr[0] = *subel; valptr[1] = *(subel + 1); @@ -1808,15 +1815,18 @@ static void wl_scan_prep(struct wl_scan_params *params, struct cfg80211_scan_req WL_SCAN(("Scanning all channels\n")); } n_channels = j; + /* Copy ssid array if applicable */ WL_SCAN(("### List of SSIDs to scan ###\n")); if (n_ssids > 0) { offset = offsetof(wl_scan_params_t, channel_list) + n_channels * sizeof(u16); offset = roundup(offset, sizeof(u32)); ptr = (char*)params + offset; + for (i = 0; i < n_ssids; i++) { memset(&ssid, 0, sizeof(wlc_ssid_t)); - ssid.SSID_len = request->ssids[i].ssid_len; + ssid.SSID_len = MIN(request->ssids[i].ssid_len, + DOT11_MAX_SSID_LEN); memcpy(ssid.SSID, request->ssids[i].ssid, ssid.SSID_len); if (!ssid.SSID_len) WL_SCAN(("%d: Broadcast scan\n", i)); @@ -2709,10 +2719,12 @@ wl_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *dev, WL_TRACE(("In\n")); RETURN_EIO_IF_NOT_UP(wl); WL_INFO(("JOIN BSSID:" MACDBG "\n", MAC2STRDBG(params->bssid))); - if (!params->ssid || params->ssid_len <= 0) { + + if (!params->ssid || params->ssid_len <= 0 || params->ssid_len > DOT11_MAX_SSID_LEN) { WL_ERR(("Invalid parameter\n")); return -EINVAL; } + if (wl_get_drv_status(wl, CONNECTED, dev)) { struct wlc_ssid *ssid = (struct wlc_ssid *)wl_read_prof(wl, dev, WL_PROF_SSID); u8 *bssid = (u8 *)wl_read_prof(wl, dev, WL_PROF_BSSID); @@ -3208,6 +3220,9 @@ wl_cfg80211_connect(struct wiphy *wiphy, struct net_device *dev, } #endif #ifdef WL_SCHED_SCAN + /* Locks are taken in wl_cfg80211_sched_scan_stop() + * A start scan occuring during connect is unlikely + */ if (wl->sched_scan_req) { wl_cfg80211_sched_scan_stop(wiphy, wl_to_prmry_ndev(wl)); } @@ -5720,17 +5735,16 @@ static s32 wl_cfg80211_bcn_set_params( } } - if ((info->ssid) && (info->ssid_len > 0) && - (info->ssid_len <= 32)) { + if ((info->ssid) && (info->ssid_len > 0) && (info->ssid_len <= DOT11_MAX_SSID_LEN)) { WL_DBG(("SSID (%s) len:%d \n", info->ssid, info->ssid_len)); if (dev_role == NL80211_IFTYPE_AP) { /* Store the hostapd SSID */ - memset(wl->hostapd_ssid.SSID, 0x00, 32); + memset(wl->hostapd_ssid.SSID, 0x00, DOT11_MAX_SSID_LEN); memcpy(wl->hostapd_ssid.SSID, info->ssid, info->ssid_len); wl->hostapd_ssid.SSID_len = info->ssid_len; } else { - /* P2P GO */ - memset(wl->p2p->ssid.SSID, 0x00, 32); + /* P2P GO */ + memset(wl->p2p->ssid.SSID, 0x00, DOT11_MAX_SSID_LEN); memcpy(wl->p2p->ssid.SSID, info->ssid, info->ssid_len); wl->p2p->ssid.SSID_len = info->ssid_len; } @@ -5860,11 +5874,13 @@ wl_cfg80211_bcn_bringup_ap( } memset(&join_params, 0, sizeof(join_params)); + /* join parameters starts with ssid */ join_params_size = sizeof(join_params.ssid); - memcpy(join_params.ssid.SSID, wl->hostapd_ssid.SSID, - wl->hostapd_ssid.SSID_len); - join_params.ssid.SSID_len = htod32(wl->hostapd_ssid.SSID_len); + join_params.ssid.SSID_len = MIN(wl->hostapd_ssid.SSID_len, + (uint32)DOT11_MAX_SSID_LEN); + memcpy(join_params.ssid.SSID, wl->hostapd_ssid.SSID, join_params.ssid.SSID_len); + join_params.ssid.SSID_len = htod32(join_params.ssid.SSID_len); /* create softap */ if ((err = wldev_ioctl(dev, WLC_SET_SSID, &join_params, @@ -6371,20 +6387,24 @@ wl_cfg80211_add_set_beacon(struct wiphy *wiphy, struct net_device *dev, goto fail; ie_offset = DOT11_MGMT_HDR_LEN + DOT11_BCN_PRB_FIXED_LEN; + /* find the SSID */ if ((ssid_ie = bcm_parse_tlvs((u8 *)&info->head[ie_offset], info->head_len - ie_offset, DOT11_MNG_SSID_ID)) != NULL) { if (dev_role == NL80211_IFTYPE_AP) { /* Store the hostapd SSID */ - memset(&wl->hostapd_ssid.SSID[0], 0x00, 32); - memcpy(&wl->hostapd_ssid.SSID[0], ssid_ie->data, ssid_ie->len); - wl->hostapd_ssid.SSID_len = ssid_ie->len; + memset(&wl->hostapd_ssid.SSID[0], 0x00, DOT11_MAX_SSID_LEN); + wl->hostapd_ssid.SSID_len = MIN(ssid_ie->len, + DOT11_MAX_SSID_LEN); + memcpy(&wl->hostapd_ssid.SSID[0], ssid_ie->data, + wl->hostapd_ssid.SSID_len); } else { - /* P2P GO */ - memset(&wl->p2p->ssid.SSID[0], 0x00, 32); - memcpy(wl->p2p->ssid.SSID, ssid_ie->data, ssid_ie->len); - wl->p2p->ssid.SSID_len = ssid_ie->len; + /* P2P GO */ + memset(&wl->p2p->ssid.SSID[0], 0x00, DOT11_MAX_SSID_LEN); + wl->p2p->ssid.SSID_len = MIN(ssid_ie->len, + DOT11_MAX_SSID_LEN); + memcpy(wl->p2p->ssid.SSID, ssid_ie->data, wl->p2p->ssid.SSID_len); } } @@ -6496,6 +6516,7 @@ wl_cfg80211_sched_scan_start(struct wiphy *wiphy, int ssid_cnt = 0; int i; int ret = 0; + unsigned long flags; WL_DBG(("Enter \n")); WL_PNO((">>> SCHED SCAN START\n")); @@ -6519,8 +6540,11 @@ wl_cfg80211_sched_scan_start(struct wiphy *wiphy, ssid = &request->match_sets[i].ssid; /* No need to include null ssid */ if (ssid->ssid_len) { - memcpy(ssids_local[ssid_cnt].SSID, ssid->ssid, ssid->ssid_len); - ssids_local[ssid_cnt].SSID_len = ssid->ssid_len; + ssids_local[ssid_cnt].SSID_len = MIN(ssid->ssid_len, + (uint8)DOT11_MAX_SSID_LEN); + memcpy(ssids_local[ssid_cnt].SSID, ssid->ssid, + ssids_local[ssid_cnt].SSID_len); + if (is_ssid_in_list(ssid, hidden_ssid_list, request->n_ssids)) { ssids_local[ssid_cnt].hidden = TRUE; WL_PNO((">>> PNO hidden SSID (%s) \n", ssid->ssid)); @@ -6538,7 +6562,9 @@ wl_cfg80211_sched_scan_start(struct wiphy *wiphy, WL_ERR(("PNO setup failed!! ret=%d \n", ret)); return -EINVAL; } + spin_lock_irqsave(&wl->cfgdrv_lock, flags); wl->sched_scan_req = request; + spin_unlock_irqrestore(&wl->cfgdrv_lock, flags); } else { return -EINVAL; } @@ -6550,6 +6576,7 @@ static int wl_cfg80211_sched_scan_stop(struct wiphy *wiphy, struct net_device *dev) { struct wl_priv *wl = wiphy_priv(wiphy); + unsigned long flags; WL_DBG(("Enter \n")); WL_PNO((">>> SCHED SCAN STOP\n")); @@ -6561,10 +6588,10 @@ wl_cfg80211_sched_scan_stop(struct wiphy *wiphy, struct net_device *dev) WL_PNO((">>> Sched scan running. Aborting it..\n")); wl_notify_escan_complete(wl, dev, true, true); } - - wl->sched_scan_req = NULL; - wl->sched_scan_running = FALSE; - + spin_lock_irqsave(&wl->cfgdrv_lock, flags); + wl->sched_scan_req = NULL; + wl->sched_scan_running = FALSE; + spin_unlock_irqrestore(&wl->cfgdrv_lock, flags); return 0; } #endif /* WL_SCHED_SCAN */ @@ -7413,6 +7440,39 @@ wl_notify_connect_status(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, return err; } + +#ifdef GSCAN_SUPPORT +static s32 +wl_handle_roam_exp_event(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, + const wl_event_msg_t *e, void *data) +{ + struct net_device *ndev = NULL; + u32 datalen = be32_to_cpu(e->datalen); + + if (datalen) { + wl_roam_exp_event_t *evt_data = (wl_roam_exp_event_t *)data; + if (evt_data->version == ROAM_EXP_EVENT_VERSION) { + wlc_ssid_t *ssid = &evt_data->cur_ssid; + struct wireless_dev *wdev; + ndev = cfgdev_to_wlc_ndev(cfgdev, wl); + if (ndev) { + wdev = ndev->ieee80211_ptr; + wdev->ssid_len = min(ssid->SSID_len, (uint32)DOT11_MAX_SSID_LEN); + memcpy(wdev->ssid, ssid->SSID, wdev->ssid_len); + WL_ERR(("SSID is %s\n", ssid->SSID)); + wl_update_prof(wl, ndev, NULL, ssid, WL_PROF_SSID); + } else { + WL_ERR(("NULL ndev!\n")); + } + } else { + WL_ERR(("Version mismatch %d, expected %d", evt_data->version, + ROAM_EXP_EVENT_VERSION)); + } + } + return BCME_OK; +} +#endif /* GSCAN_SUPPORT */ + static s32 wl_notify_roaming_status(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, const wl_event_msg_t *e, void *data) @@ -7798,11 +7858,28 @@ wl_notify_pfn_status(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, const wl_event_msg_t *e, void *data) { struct net_device *ndev = NULL; +#ifdef GSCAN_SUPPORT + void *ptr; + int send_evt_bytes = 0; + u32 event = be32_to_cpu(e->event_type); + struct wiphy *wiphy = wl_to_wiphy(wl); +#endif /* GSCAN_SUPPORT */ WL_ERR((">>> PNO Event\n")); ndev = cfgdev_to_wlc_ndev(cfgdev, wl); +#ifdef GSCAN_SUPPORT + ptr = dhd_dev_process_epno_result(ndev, data, event, &send_evt_bytes); + if (ptr) { + wl_cfgvendor_send_async_event(wiphy, ndev, + GOOGLE_SCAN_EPNO_EVENT, ptr, send_evt_bytes); + kfree(ptr); + } + if (!dhd_dev_is_legacy_pno_enabled(ndev)) + return 0; +#endif /* GSCAN_SUPPORT */ + #ifndef WL_SCHED_SCAN mutex_lock(&wl->usr_sync); /* TODO: Use cfg80211_sched_scan_results(wiphy); */ @@ -7888,7 +7965,17 @@ wl_notify_gscan_event(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, kfree(ptr); } break; - + case WLC_E_PFN_SSID_EXT: + ptr = dhd_dev_process_epno_result(ndev, data, event, &send_evt_bytes); + if (ptr) { + wl_cfgvendor_send_async_event(wiphy, ndev, + GOOGLE_SCAN_EPNO_EVENT, ptr, send_evt_bytes); + kfree(ptr); + } + break; + default: + WL_ERR(("Unknown event %d\n", event)); + break; } return err; } @@ -8274,10 +8361,9 @@ wl_notify_sched_scan_results(struct wl_priv *wl, struct net_device *ndev, * scan request in the form of cfg80211_scan_request. For timebeing, create * cfg80211_scan_request one out of the received PNO event. */ - memcpy(ssid[i].ssid, netinfo->pfnsubnet.SSID, - netinfo->pfnsubnet.SSID_len); - ssid[i].ssid_len = netinfo->pfnsubnet.SSID_len; - request->n_ssids++; + ssid[i].ssid_len = MIN(DOT11_MAX_SSID_LEN, netinfo->pfnsubnet.SSID_len); + memcpy(ssid[i].ssid, netinfo->pfnsubnet.SSID, ssid[i].ssid_len); + request->n_ssids++; channel_req = netinfo->pfnsubnet.channel; band = (channel_req <= CH_MAX_2G_CHANNEL) ? NL80211_BAND_2GHZ @@ -8388,6 +8474,8 @@ static void wl_init_event_handler(struct wl_priv *wl) wl->evt_handler[WLC_E_PFN_SWC] = wl_notify_gscan_event; wl->evt_handler[WLC_E_PFN_BSSID_NET_FOUND] = wl_notify_gscan_event; wl->evt_handler[WLC_E_PFN_BSSID_NET_LOST] = wl_notify_gscan_event; + wl->evt_handler[WLC_E_PFN_SSID_EXT] = wl_notify_gscan_event; + wl->evt_handler[WLC_E_ROAM_EXP_EVENT] = wl_handle_roam_exp_event; #endif /* GSCAN_SUPPORT */ #ifdef WLTDLS wl->evt_handler[WLC_E_TDLS_PEER_EVENT] = wl_tdls_event_handler; @@ -8931,11 +9019,14 @@ static s32 wl_notify_escan_complete(struct wl_priv *wl, spin_lock_irqsave(&wl->cfgdrv_lock, flags); #ifdef WL_SCHED_SCAN if (wl->sched_scan_req && !wl->scan_request) { - WL_PNO((">>> REPORTING SCHED SCAN RESULTS \n")); - if (!aborted) + int count; + + count = wl->bss_list ? wl->bss_list->count : 0; + if (!aborted) { cfg80211_sched_scan_results(wl->sched_scan_req->wiphy); + printk(">> SCHED SCAN RESULT %d\n", count); + } wl->sched_scan_running = FALSE; - wl->sched_scan_req = NULL; } #endif /* WL_SCHED_SCAN */ if (likely(wl->scan_request)) { @@ -9172,7 +9263,16 @@ static s32 wl_escan_handler(struct wl_priv *wl, bcm_struct_cfgdev *cfgdev, } wl_escan_increment_sync_id(wl, SCAN_BUF_NEXT); } +#ifdef GSCAN_SUPPORT + else if ((status == WLC_E_STATUS_ABORT) || (status == WLC_E_STATUS_NEWSCAN)) { + if (status == WLC_E_STATUS_NEWSCAN) { + WL_ERR(("WLC_E_STATUS_NEWSCAN : scan_request[%p]\n", wl->scan_request)); + WL_ERR(("sync_id[%d], bss_count[%d]\n", escan_result->sync_id, + escan_result->bss_count)); + } +#else else if (status == WLC_E_STATUS_ABORT) { +#endif /* GSCAN_SUPPORT */ wl->escan_info.escan_state = WL_ESCAN_STATE_IDLE; wl_escan_print_sync_id(status, escan_result->sync_id, wl->escan_info.cur_sync_id); @@ -10547,10 +10647,10 @@ wl_update_prof(struct wl_priv *wl, struct net_device *ndev, switch (item) { case WL_PROF_SSID: ssid = (wlc_ssid_t *) data; - memset(profile->ssid.SSID, 0, - sizeof(profile->ssid.SSID)); - memcpy(profile->ssid.SSID, ssid->SSID, ssid->SSID_len); - profile->ssid.SSID_len = ssid->SSID_len; + memset(profile->ssid.SSID, 0, sizeof(profile->ssid.SSID)); + profile->ssid.SSID_len = MIN(ssid->SSID_len, + DOT11_MAX_SSID_LEN); + memcpy(profile->ssid.SSID, ssid->SSID, profile->ssid.SSID_len); break; case WL_PROF_BSSID: if (data) @@ -10633,23 +10733,44 @@ static __used s32 wl_add_ie(struct wl_priv *wl, u8 t, u8 l, u8 *v) static void wl_update_hidden_ap_ie(struct wl_bss_info *bi, u8 *ie_stream, u32 *ie_size) { u8 *ssidie; + int32 ssid_len = MIN(bi->SSID_len, DOT11_MAX_SSID_LEN); + int32 remaining_ie_buf_len, available_buffer_len; ssidie = (u8 *)cfg80211_find_ie(WLAN_EID_SSID, ie_stream, *ie_size); + + /* + * ERROR out if + * 1. No ssid IE is FOUND or + * 2. New ssid length is > what was allocated for existing ssid (as + * we do not want to overwrite the rest of the IEs) or + * 3. If in case of erroneous buffer input where ssid length doesnt match the space + * allocated to it. + */ if (!ssidie) return; - if (ssidie[1] != bi->SSID_len) { + + available_buffer_len = ((int)(*ie_size)) - (ssidie + 2 - ie_stream); + remaining_ie_buf_len = available_buffer_len - (int)ssidie[1]; + if ((ssid_len > ssidie[1]) || (ssidie[1] > available_buffer_len)) + return; + + if (ssidie[1] != ssid_len) { if (ssidie[1]) { WL_ERR(("%s: Wrong SSID len: %d != %d\n", __FUNCTION__, ssidie[1], bi->SSID_len)); return; } - memmove(ssidie + bi->SSID_len + 2, ssidie + 2, *ie_size - (ssidie + 2 - ie_stream)); - memcpy(ssidie + 2, bi->SSID, bi->SSID_len); - *ie_size = *ie_size + bi->SSID_len; - ssidie[1] = bi->SSID_len; + + WL_ERR(("Changing the SSID Info.\n")); + memmove(ssidie + ssid_len + 2, + (ssidie + 2) + ssidie[1], remaining_ie_buf_len); + memcpy(ssidie + 2, bi->SSID, ssid_len); + *ie_size = *ie_size + ssid_len - ssidie[1]; + ssidie[1] = ssid_len; return; } + if (*(ssidie + 2) == '\0') - memcpy(ssidie + 2, bi->SSID, bi->SSID_len); + memcpy(ssidie + 2, bi->SSID, ssid_len); return; } diff --git a/drivers/net/wireless/bcmdhd/wl_cfgvendor.c b/drivers/net/wireless/bcmdhd/wl_cfgvendor.c index b26da1ab3ab..156cfc4e59c 100644 --- a/drivers/net/wireless/bcmdhd/wl_cfgvendor.c +++ b/drivers/net/wireless/bcmdhd/wl_cfgvendor.c @@ -185,6 +185,31 @@ static int wl_cfgvendor_get_feature_set_matrix(struct wiphy *wiphy, return err; } +static int wl_cfgvendor_set_pno_mac_oui(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = 0; + struct wl_priv *cfg = wiphy_priv(wiphy); + int type; + uint8 pno_random_mac_oui[DOT11_OUI_LEN]; + + type = nla_type(data); + + if (type == ANDR_WIFI_ATTRIBUTE_PNO_RANDOM_MAC_OUI) { + memcpy(pno_random_mac_oui, nla_data(data), DOT11_OUI_LEN); + + err = dhd_dev_pno_set_mac_oui(wl_to_prmry_ndev(cfg), pno_random_mac_oui); + + if (unlikely(err)) + WL_ERR(("Bad OUI, could not set:%d \n", err)); + + } else { + err = -1; + } + + return err; +} + #ifdef GSCAN_SUPPORT int wl_cfgvendor_send_hotlist_event(struct wiphy *wiphy, struct net_device *dev, void *data, int len, wl_vendor_event_t event) @@ -318,15 +343,22 @@ static int wl_cfgvendor_gscan_get_batch_results(struct wiphy *wiphy, int err = 0; struct wl_priv *cfg = wiphy_priv(wiphy); gscan_results_cache_t *results, *iter; - uint32 reply_len, complete = 0, num_results_iter; - int32 mem_needed; + uint32 reply_len, complete = 1; + int32 mem_needed, num_results_iter; wifi_gscan_result_t *ptr; uint16 num_scan_ids, num_results; struct sk_buff *skb; - struct nlattr *scan_hdr; + struct nlattr *scan_hdr, *complete_flag; + + err = dhd_dev_wait_batch_results_complete(wl_to_prmry_ndev(cfg)); + if (err != BCME_OK) + return -EBUSY; - dhd_dev_wait_batch_results_complete(wl_to_prmry_ndev(cfg)); - dhd_dev_pno_lock_access_batch_results(wl_to_prmry_ndev(cfg)); + err = dhd_dev_pno_lock_access_batch_results(wl_to_prmry_ndev(cfg)); + if (err != BCME_OK) { + WL_ERR(("Can't obtain lock to access batch results %d\n", err)); + return -EBUSY; + } results = dhd_dev_pno_get_gscan(wl_to_prmry_ndev(cfg), DHD_PNO_GET_BATCH_RESULTS, NULL, &reply_len); @@ -349,8 +381,6 @@ static int wl_cfgvendor_gscan_get_batch_results(struct wiphy *wiphy, if (mem_needed > (int32)NLMSG_DEFAULT_SIZE) { mem_needed = (int32)NLMSG_DEFAULT_SIZE; complete = 0; - } else { - complete = 1; } WL_TRACE(("complete %d mem_needed %d max_mem %d\n", complete, mem_needed, @@ -363,20 +393,25 @@ static int wl_cfgvendor_gscan_get_batch_results(struct wiphy *wiphy, return -ENOMEM; } iter = results; - - nla_put_u32(skb, GSCAN_ATTRIBUTE_SCAN_RESULTS_COMPLETE, complete); - + complete_flag = nla_reserve(skb, GSCAN_ATTRIBUTE_SCAN_RESULTS_COMPLETE, + sizeof(complete)); mem_needed = mem_needed - (SCAN_RESULTS_COMPLETE_FLAG_LEN + VENDOR_REPLY_OVERHEAD); - while (iter && ((mem_needed - GSCAN_BATCH_RESULT_HDR_LEN) > 0)) { + while (iter) { + num_results_iter = + (mem_needed - GSCAN_BATCH_RESULT_HDR_LEN)/sizeof(wifi_gscan_result_t); + if (num_results_iter <= 0 || + ((iter->tot_count - iter->tot_consumed) > num_results_iter)) + break; scan_hdr = nla_nest_start(skb, GSCAN_ATTRIBUTE_SCAN_RESULTS); + /* no more room? we are done then (for now) */ + if (scan_hdr == NULL) { + complete = 0; + break; + } nla_put_u32(skb, GSCAN_ATTRIBUTE_SCAN_ID, iter->scan_id); nla_put_u8(skb, GSCAN_ATTRIBUTE_SCAN_FLAGS, iter->flag); - num_results_iter = - (mem_needed - GSCAN_BATCH_RESULT_HDR_LEN)/sizeof(wifi_gscan_result_t); - - if ((iter->tot_count - iter->tot_consumed) < num_results_iter) - num_results_iter = iter->tot_count - iter->tot_consumed; + num_results_iter = iter->tot_count - iter->tot_consumed; nla_put_u32(skb, GSCAN_ATTRIBUTE_NUM_OF_RESULTS, num_results_iter); if (num_results_iter) { @@ -390,10 +425,9 @@ static int wl_cfgvendor_gscan_get_batch_results(struct wiphy *wiphy, (num_results_iter * sizeof(wifi_gscan_result_t)); iter = iter->next; } - + memcpy(nla_data(complete_flag), &complete, sizeof(complete)); dhd_dev_gscan_batch_cache_cleanup(wl_to_prmry_ndev(cfg)); dhd_dev_pno_unlock_access_batch_results(wl_to_prmry_ndev(cfg)); - return cfg80211_vendor_cmd_reply(skb); } @@ -453,23 +487,113 @@ static int wl_cfgvendor_enable_full_scan_result(struct wiphy *wiphy, return err; } -static int wl_cfgvendor_set_scan_cfg(struct wiphy *wiphy, - struct wireless_dev *wdev, const void *data, int len) +static int +wl_cfgvendor_set_scan_cfg_bucket(const struct nlattr *prev, + gscan_scan_params_t *scan_param, int num) +{ + struct dhd_pno_gscan_channel_bucket *ch_bucket; + int k = 0; + int type, err = 0, rem; + const struct nlattr *cur, *next; + + nla_for_each_nested(cur, prev, rem) { + type = nla_type(cur); + ch_bucket = scan_param->channel_bucket; + switch (type) { + case GSCAN_ATTRIBUTE_BUCKET_ID: + break; + case GSCAN_ATTRIBUTE_BUCKET_PERIOD: + if (nla_len(cur) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + + ch_bucket[num].bucket_freq_multiple = + nla_get_u32(cur) / MSEC_PER_SEC; + break; + case GSCAN_ATTRIBUTE_BUCKET_NUM_CHANNELS: + if (nla_len(cur) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + ch_bucket[num].num_channels = nla_get_u32(cur); + if (ch_bucket[num].num_channels > + GSCAN_MAX_CHANNELS_IN_BUCKET) { + WL_ERR(("channel range:%d,bucket:%d\n", + ch_bucket[num].num_channels, + num)); + err = -EINVAL; + goto exit; + } + break; + case GSCAN_ATTRIBUTE_BUCKET_CHANNELS: + nla_for_each_nested(next, cur, rem) { + if (k >= GSCAN_MAX_CHANNELS_IN_BUCKET) + break; + if (nla_len(next) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + ch_bucket[num].chan_list[k] = nla_get_u32(next); + k++; + } + break; + case GSCAN_ATTRIBUTE_BUCKETS_BAND: + if (nla_len(cur) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + ch_bucket[num].band = (uint16)nla_get_u32(cur); + break; + case GSCAN_ATTRIBUTE_REPORT_EVENTS: + if (nla_len(cur) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + ch_bucket[num].report_flag = (uint8)nla_get_u32(cur); + break; + case GSCAN_ATTRIBUTE_BUCKET_STEP_COUNT: + if (nla_len(cur) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + ch_bucket[num].repeat = (uint16)nla_get_u32(cur); + break; + case GSCAN_ATTRIBUTE_BUCKET_MAX_PERIOD: + if (nla_len(cur) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + ch_bucket[num].bucket_max_multiple = + nla_get_u32(cur) / MSEC_PER_SEC; + break; + default: + WL_ERR(("unknown attr type:%d\n", type)); + err = -EINVAL; + goto exit; + } + } + +exit: + return err; +} + +static int +wl_cfgvendor_set_scan_cfg(struct wiphy *wiphy, struct wireless_dev *wdev, + const void *data, int len) { int err = 0; struct wl_priv *cfg = wiphy_priv(wiphy); gscan_scan_params_t *scan_param; int j = 0; - int type, tmp, tmp1, tmp2, k = 0; - const struct nlattr *iter, *iter1, *iter2; - struct dhd_pno_gscan_channel_bucket *ch_bucket; + int type, tmp; + const struct nlattr *iter; scan_param = kzalloc(sizeof(gscan_scan_params_t), GFP_KERNEL); if (!scan_param) { WL_ERR(("Could not set GSCAN scan cfg, mem alloc failure\n")); err = -EINVAL; return err; - } scan_param->scan_fr = PNO_SCAN_MIN_FW_SEC; @@ -480,69 +604,61 @@ static int wl_cfgvendor_set_scan_cfg(struct wiphy *wiphy, break; switch (type) { - case GSCAN_ATTRIBUTE_BASE_PERIOD: - scan_param->scan_fr = nla_get_u32(iter)/1000; - break; - case GSCAN_ATTRIBUTE_NUM_BUCKETS: - scan_param->nchannel_buckets = nla_get_u32(iter); - break; - case GSCAN_ATTRIBUTE_CH_BUCKET_1: - case GSCAN_ATTRIBUTE_CH_BUCKET_2: - case GSCAN_ATTRIBUTE_CH_BUCKET_3: - case GSCAN_ATTRIBUTE_CH_BUCKET_4: - case GSCAN_ATTRIBUTE_CH_BUCKET_5: - case GSCAN_ATTRIBUTE_CH_BUCKET_6: - case GSCAN_ATTRIBUTE_CH_BUCKET_7: - nla_for_each_nested(iter1, iter, tmp1) { - type = nla_type(iter1); - ch_bucket = - scan_param->channel_bucket; + case GSCAN_ATTRIBUTE_BASE_PERIOD: + if (nla_len(iter) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + scan_param->scan_fr = nla_get_u32(iter) / MSEC_PER_SEC; + break; + case GSCAN_ATTRIBUTE_NUM_BUCKETS: + if (nla_len(iter) != sizeof(uint32)) { + err = -EINVAL; + goto exit; + } + scan_param->nchannel_buckets = nla_get_u32(iter); + if (scan_param->nchannel_buckets >= + GSCAN_MAX_CH_BUCKETS) { + WL_ERR(("ncha_buck out of range %d\n", + scan_param->nchannel_buckets)); + err = -EINVAL; + goto exit; + } + break; + case GSCAN_ATTRIBUTE_CH_BUCKET_1: + case GSCAN_ATTRIBUTE_CH_BUCKET_2: + case GSCAN_ATTRIBUTE_CH_BUCKET_3: + case GSCAN_ATTRIBUTE_CH_BUCKET_4: + case GSCAN_ATTRIBUTE_CH_BUCKET_5: + case GSCAN_ATTRIBUTE_CH_BUCKET_6: + case GSCAN_ATTRIBUTE_CH_BUCKET_7: + err = wl_cfgvendor_set_scan_cfg_bucket(iter, + scan_param, j); + if (err < 0) { + WL_ERR(("set_scan_cfg_buck error:%d\n", err)); + goto exit; + } - switch (type) { - case GSCAN_ATTRIBUTE_BUCKET_ID: - break; - case GSCAN_ATTRIBUTE_BUCKET_PERIOD: - ch_bucket[j].bucket_freq_multiple = - nla_get_u32(iter1)/1000; - break; - case GSCAN_ATTRIBUTE_BUCKET_NUM_CHANNELS: - ch_bucket[j].num_channels = - nla_get_u32(iter1); - break; - case GSCAN_ATTRIBUTE_BUCKET_CHANNELS: - nla_for_each_nested(iter2, iter1, tmp2) { - if (k >= PFN_SWC_RSSI_WINDOW_MAX) - break; - ch_bucket[j].chan_list[k] = - nla_get_u32(iter2); - k++; - } - k = 0; - break; - case GSCAN_ATTRIBUTE_BUCKETS_BAND: - ch_bucket[j].band = (uint16) - nla_get_u32(iter1); - break; - case GSCAN_ATTRIBUTE_REPORT_EVENTS: - ch_bucket[j].report_flag = (uint8) - nla_get_u32(iter1); - break; - } - } - j++; - break; + j++; + break; + default: + WL_ERR(("Unknown attr type %d\n", type)); + err = -EINVAL; + goto exit; } } - if (dhd_dev_pno_set_cfg_gscan(wl_to_prmry_ndev(cfg), - DHD_PNO_SCAN_CFG_ID, scan_param, 0) < 0) { - WL_ERR(("Could not set GSCAN scan cfg\n")); + err = dhd_dev_pno_set_cfg_gscan(wl_to_prmry_ndev(cfg), + DHD_PNO_SCAN_CFG_ID, scan_param, FALSE); + + if (err < 0) { + WL_ERR(("Could not set GSCAN scan cfg error %d\n", err)); err = -EINVAL; } +exit: kfree(scan_param); return err; - } static int wl_cfgvendor_hotlist_cfg(struct wiphy *wiphy, @@ -611,6 +727,98 @@ static int wl_cfgvendor_hotlist_cfg(struct wiphy *wiphy, kfree(hotlist_params); return err; } + +static int wl_cfgvendor_epno_cfg(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = 0; + struct wl_priv *cfg = wiphy_priv(wiphy); + dhd_epno_params_t *epno_params; + int tmp, tmp1, tmp2, type, num = 0; + const struct nlattr *outer, *inner, *iter; + uint8 flush = 0, i = 0; + uint16 num_visible_ssid = 0; + + nla_for_each_attr(iter, data, len, tmp2) { + type = nla_type(iter); + switch (type) { + case GSCAN_ATTRIBUTE_EPNO_SSID_LIST: + nla_for_each_nested(outer, iter, tmp) { + epno_params = (dhd_epno_params_t *) + dhd_dev_pno_get_gscan(wl_to_prmry_ndev(cfg), + DHD_PNO_GET_EPNO_SSID_ELEM, NULL, &num); + if (!epno_params) { + WL_ERR(("Failed to get SSID LIST buffer\n")); + err = -ENOMEM; + goto exit; + } + i++; + nla_for_each_nested(inner, outer, tmp1) { + type = nla_type(inner); + + switch (type) { + case GSCAN_ATTRIBUTE_EPNO_SSID: + memcpy(epno_params->ssid, + nla_data(inner), + DOT11_MAX_SSID_LEN); + break; + case GSCAN_ATTRIBUTE_EPNO_SSID_LEN: + len = nla_get_u8(inner); + if (len < DOT11_MAX_SSID_LEN) { + epno_params->ssid_len = len; + } else { + WL_ERR(("SSID too long %d\n", len)); + err = -EINVAL; + goto exit; + } + break; + case GSCAN_ATTRIBUTE_EPNO_RSSI: + epno_params->rssi_thresh = + (int8) nla_get_u32(inner); + break; + case GSCAN_ATTRIBUTE_EPNO_FLAGS: + epno_params->flags = + nla_get_u8(inner); + if (!(epno_params->flags & + DHD_PNO_USE_SSID)) + num_visible_ssid++; + break; + case GSCAN_ATTRIBUTE_EPNO_AUTH: + epno_params->auth = + nla_get_u8(inner); + break; + } + } + } + break; + case GSCAN_ATTRIBUTE_EPNO_SSID_NUM: + num = nla_get_u8(iter); + break; + case GSCAN_ATTRIBUTE_EPNO_FLUSH: + flush = nla_get_u8(iter); + dhd_dev_pno_set_cfg_gscan(wl_to_prmry_ndev(cfg), + DHD_PNO_EPNO_CFG_ID, NULL, flush); + break; + default: + WL_ERR(("%s: No such attribute %d\n", __FUNCTION__, type)); + err = -EINVAL; + goto exit; + } + + } + if (i != num) { + WL_ERR(("%s: num_ssid %d does not match ssids sent %d\n", __FUNCTION__, + num, i)); + err = -EINVAL; + } +exit: + /* Flush all configs if error condition */ + flush = (err < 0) ? TRUE: FALSE; + dhd_dev_pno_set_cfg_gscan(wl_to_prmry_ndev(cfg), + DHD_PNO_EPNO_CFG_ID, &num_visible_ssid, flush); + return err; +} + static int wl_cfgvendor_set_batch_scan_cfg(struct wiphy *wiphy, struct wireless_dev *wdev, const void *data, int len) { @@ -719,6 +927,315 @@ static int wl_cfgvendor_significant_change_cfg(struct wiphy *wiphy, kfree(significant_params); return err; } + +static int wl_cfgvendor_enable_lazy_roam(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = -EINVAL; + struct wl_priv *cfg = wiphy_priv(wiphy); + int type; + uint32 lazy_roam_enable_flag; + + type = nla_type(data); + + if (type == GSCAN_ATTRIBUTE_LAZY_ROAM_ENABLE) { + lazy_roam_enable_flag = nla_get_u32(data); + + err = dhd_dev_lazy_roam_enable(wl_to_prmry_ndev(cfg), + lazy_roam_enable_flag); + + if (unlikely(err)) + WL_ERR(("Could not enable lazy roam:%d \n", err)); + + } + return err; +} + +static int wl_cfgvendor_set_lazy_roam_cfg(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = 0, tmp, type; + struct wl_priv *cfg = wiphy_priv(wiphy); + wlc_roam_exp_params_t roam_param; + const struct nlattr *iter; + + memset(&roam_param, 0, sizeof(roam_param)); + + nla_for_each_attr(iter, data, len, tmp) { + type = nla_type(iter); + + switch (type) { + case GSCAN_ATTRIBUTE_A_BAND_BOOST_THRESHOLD: + roam_param.a_band_boost_threshold = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_A_BAND_PENALTY_THRESHOLD: + roam_param.a_band_penalty_threshold = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_A_BAND_BOOST_FACTOR: + roam_param.a_band_boost_factor = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_A_BAND_PENALTY_FACTOR: + roam_param.a_band_penalty_factor = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_A_BAND_MAX_BOOST: + roam_param.a_band_max_boost = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_LAZY_ROAM_HYSTERESIS: + roam_param.cur_bssid_boost = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_ALERT_ROAM_RSSI_TRIGGER: + roam_param.alert_roam_trigger_threshold = nla_get_u32(iter); + break; + } + } + + if (dhd_dev_set_lazy_roam_cfg(wl_to_prmry_ndev(cfg), &roam_param) < 0) { + WL_ERR(("Could not set batch cfg\n")); + err = -EINVAL; + } + return err; +} + +/* small helper function */ +static wl_bssid_pref_cfg_t *create_bssid_pref_cfg(uint32 num) +{ + uint32 mem_needed; + wl_bssid_pref_cfg_t *bssid_pref; + + mem_needed = sizeof(wl_bssid_pref_cfg_t); + if (num) + mem_needed += (num - 1) * sizeof(wl_bssid_pref_list_t); + bssid_pref = (wl_bssid_pref_cfg_t *) kmalloc(mem_needed, GFP_KERNEL); + return bssid_pref; +} + +static int wl_cfgvendor_set_bssid_pref(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = 0; + struct wl_priv *cfg = wiphy_priv(wiphy); + wl_bssid_pref_cfg_t *bssid_pref = NULL; + wl_bssid_pref_list_t *bssids; + int tmp, tmp1, tmp2, type; + const struct nlattr *outer, *inner, *iter; + uint32 flush = 0, i = 0, num = 0; + + /* Assumption: NUM attribute must come first */ + nla_for_each_attr(iter, data, len, tmp2) { + type = nla_type(iter); + switch (type) { + case GSCAN_ATTRIBUTE_NUM_BSSID: + num = nla_get_u32(iter); + if (num > MAX_BSSID_PREF_LIST_NUM) { + WL_ERR(("Too many Preferred BSSIDs!\n")); + err = -EINVAL; + goto exit; + } + break; + case GSCAN_ATTRIBUTE_BSSID_PREF_FLUSH: + flush = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_BSSID_PREF_LIST: + if (!num) + return -EINVAL; + if ((bssid_pref = create_bssid_pref_cfg(num)) == NULL) { + WL_ERR(("%s: Can't malloc memory\n", __FUNCTION__)); + err = -ENOMEM; + goto exit; + } + bssid_pref->count = num; + bssids = bssid_pref->bssids; + nla_for_each_nested(outer, iter, tmp) { + if (i >= num) { + WL_ERR(("CFGs don't seem right!\n")); + err = -EINVAL; + goto exit; + } + nla_for_each_nested(inner, outer, tmp1) { + type = nla_type(inner); + switch (type) { + case GSCAN_ATTRIBUTE_BSSID_PREF: + memcpy(&(bssids[i].bssid), + nla_data(inner), ETHER_ADDR_LEN); + /* not used for now */ + bssids[i].flags = 0; + break; + case GSCAN_ATTRIBUTE_RSSI_MODIFIER: + bssids[i].rssi_factor = + (int8) nla_get_u32(inner); + break; + } + } + i++; + } + break; + default: + WL_ERR(("%s: No such attribute %d\n", __FUNCTION__, type)); + break; + } + } + + if (!bssid_pref) { + /* What if only flush is desired? */ + if (flush) { + if ((bssid_pref = create_bssid_pref_cfg(0)) == NULL) { + WL_ERR(("%s: Can't malloc memory\n", __FUNCTION__)); + err = -ENOMEM; + goto exit; + } + bssid_pref->count = 0; + } else { + err = -EINVAL; + goto exit; + } + } + err = dhd_dev_set_lazy_roam_bssid_pref(wl_to_prmry_ndev(cfg), + bssid_pref, flush); +exit: + kfree(bssid_pref); + return err; +} + + +static int wl_cfgvendor_set_bssid_blacklist(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + struct wl_priv *cfg = wiphy_priv(wiphy); + maclist_t *blacklist = NULL; + int err = 0; + int type, tmp; + const struct nlattr *iter; + uint32 mem_needed = 0, flush = 0, i = 0, num = 0; + + /* Assumption: NUM attribute must come first */ + nla_for_each_attr(iter, data, len, tmp) { + type = nla_type(iter); + switch (type) { + case GSCAN_ATTRIBUTE_NUM_BSSID: + num = nla_get_u32(iter); + if (num > MAX_BSSID_BLACKLIST_NUM) { + WL_ERR(("Too many Blacklist BSSIDs!\n")); + err = -EINVAL; + goto exit; + } + break; + case GSCAN_ATTRIBUTE_BSSID_BLACKLIST_FLUSH: + flush = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_BLACKLIST_BSSID: + if (num) { + if (!blacklist) { + mem_needed = sizeof(maclist_t) + + sizeof(struct ether_addr) * (num - 1); + blacklist = (maclist_t *) + kmalloc(mem_needed, GFP_KERNEL); + if (!blacklist) { + WL_ERR(("%s: Can't malloc %d bytes\n", + __FUNCTION__, mem_needed)); + err = -ENOMEM; + goto exit; + } + blacklist->count = num; + } + if (i >= num) { + WL_ERR(("CFGs don't seem right!\n")); + err = -EINVAL; + goto exit; + } + memcpy(&(blacklist->ea[i]), + nla_data(iter), ETHER_ADDR_LEN); + i++; + } + break; + default: + WL_ERR(("%s: No such attribute %d\n", __FUNCTION__, type)); + break; + } + } + err = dhd_dev_set_blacklist_bssid(wl_to_prmry_ndev(cfg), + blacklist, mem_needed, flush); +exit: + kfree(blacklist); + return err; +} + +static int wl_cfgvendor_set_ssid_whitelist(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = 0; + struct wl_priv *cfg = wiphy_priv(wiphy); + wl_ssid_whitelist_t *ssid_whitelist = NULL; + wlc_ssid_t *ssid_elem; + int tmp, tmp2, mem_needed = 0, type; + const struct nlattr *inner, *iter; + uint32 flush = 0, i = 0, num = 0; + + /* Assumption: NUM attribute must come first */ + nla_for_each_attr(iter, data, len, tmp2) { + type = nla_type(iter); + switch (type) { + case GSCAN_ATTRIBUTE_NUM_WL_SSID: + num = nla_get_u32(iter); + if (num > MAX_SSID_WHITELIST_NUM) { + WL_ERR(("Too many WL SSIDs!\n")); + err = -EINVAL; + goto exit; + } + mem_needed = sizeof(wl_ssid_whitelist_t); + if (num) + mem_needed += (num - 1) * sizeof(ssid_info_t); + ssid_whitelist = (wl_ssid_whitelist_t *) + kzalloc(mem_needed, GFP_KERNEL); + if (ssid_whitelist == NULL) { + WL_ERR(("%s: Can't malloc %d bytes\n", + __FUNCTION__, mem_needed)); + err = -ENOMEM; + goto exit; + } + ssid_whitelist->ssid_count = num; + break; + case GSCAN_ATTRIBUTE_WL_SSID_FLUSH: + flush = nla_get_u32(iter); + break; + case GSCAN_ATTRIBUTE_WHITELIST_SSID_ELEM: + if (!num || !ssid_whitelist) { + WL_ERR(("num ssid is not set!\n")); + return -EINVAL; + } + if (i >= num) { + WL_ERR(("CFGs don't seem right!\n")); + err = -EINVAL; + goto exit; + } + ssid_elem = &ssid_whitelist->ssids[i]; + nla_for_each_nested(inner, iter, tmp) { + type = nla_type(inner); + switch (type) { + case GSCAN_ATTRIBUTE_WHITELIST_SSID: + memcpy(ssid_elem->SSID, + nla_data(inner), + DOT11_MAX_SSID_LEN); + break; + case GSCAN_ATTRIBUTE_WL_SSID_LEN: + ssid_elem->SSID_len = (uint8) + nla_get_u32(inner); + break; + } + } + i++; + break; + default: + WL_ERR(("%s: No such attribute %d\n", __FUNCTION__, type)); + break; + } + } + + err = dhd_dev_set_whitelist_ssid(wl_to_prmry_ndev(cfg), + ssid_whitelist, mem_needed, flush); +exit: + kfree(ssid_whitelist); + return err; +} #endif /* GSCAN_SUPPORT */ static int wl_cfgvendor_priv_string_handler(struct wiphy *wiphy, struct wireless_dev *wdev, const void *data, int len) @@ -750,20 +1267,39 @@ static int wl_cfgvendor_priv_string_handler(struct wiphy *wiphy, return err; } +#define NUM_CHAN 11 static int wl_cfgvendor_lstats_get_info(struct wiphy *wiphy, struct wireless_dev *wdev, const void *data, int len) { struct wl_priv *cfg = wiphy_priv(wiphy); int err = 0; wifi_iface_stat *ptr; + wifi_radio_stat *radio; wl_wme_cnt_t *wl_wme_cnt; wl_cnt_t *wl_cnt; + char *output; + uint32 reply_len = sizeof(wifi_radio_stat)+ + NUM_CHAN*sizeof(wifi_channel_stat)+ + sizeof(wifi_iface_stat); WL_INFO(("%s: Enter \n", __func__)); bzero(cfg->ioctl_buf, WLC_IOCTL_MAXLEN); - ptr = (wifi_iface_stat *)cfg->ioctl_buf; + output = cfg->ioctl_buf; + radio = (wifi_radio_stat *)output; + + radio->num_channels = NUM_CHAN; + output += sizeof(wifi_radio_stat); + output += (NUM_CHAN*sizeof(wifi_channel_stat)); + + ptr = (wifi_iface_stat *)output; + + err = (reply_len > WLC_IOCTL_MAXLEN) ? -EPERM : 0; + if (unlikely(err)) { + WL_ERR(("link stats buffer overruns (%d)\n", err)); + return err; + } err = wldev_iovar_getbuf(wl_to_prmry_ndev(cfg), "wme_counters", NULL, 0, iovar_buf, WLC_IOCTL_MAXLEN, NULL); @@ -810,14 +1346,44 @@ static int wl_cfgvendor_lstats_get_info(struct wiphy *wiphy, return err; } + ptr->num_peers = 0; + err = wl_cfgvendor_send_cmd_reply(wiphy, wl_to_prmry_ndev(cfg), - cfg->ioctl_buf, sizeof(wifi_iface_stat)); + cfg->ioctl_buf, reply_len); if (unlikely(err)) WL_ERR(("Vendor Command reply failed ret:%d \n", err)); return err; } +static int wl_cfgvendor_set_country(struct wiphy *wiphy, + struct wireless_dev *wdev, const void *data, int len) +{ + int err = BCME_ERROR, rem, type; + char country_code[WLC_CNTRY_BUF_SZ] = {0}; + const struct nlattr *iter; + + nla_for_each_attr(iter, data, len, rem) { + type = nla_type(iter); + switch (type) { + case ANDR_WIFI_ATTRIBUTE_COUNTRY: + memcpy(country_code, nla_data(iter), + MIN(nla_len(iter), WLC_CNTRY_BUF_SZ)); + break; + default: + WL_ERR(("Unknown type: %d\n", type)); + return err; + } + } + + err = wldev_set_country(wdev->netdev, country_code, true, true); + if (err < 0) { + WL_ERR(("Set country failed ret:%d\n", err)); + } + + return err; +} + static const struct wiphy_vendor_command wl_vendor_cmds [] = { { { @@ -924,7 +1490,79 @@ static const struct wiphy_vendor_command wl_vendor_cmds [] = { }, .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, .doit = wl_cfgvendor_get_feature_set_matrix - } + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = ANDR_WIFI_SET_COUNTRY + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_set_country + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = ANDR_WIFI_PNO_RANDOM_MAC_OUI + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_set_pno_mac_oui + }, +#ifdef GSCAN_SUPPORT + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = GSCAN_SUBCMD_SET_EPNO_SSID + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_epno_cfg + + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = WIFI_SUBCMD_SET_SSID_WHITELIST + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_set_ssid_whitelist + + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = WIFI_SUBCMD_SET_LAZY_ROAM_PARAMS + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_set_lazy_roam_cfg + + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = WIFI_SUBCMD_ENABLE_LAZY_ROAM + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_enable_lazy_roam + + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = WIFI_SUBCMD_SET_BSSID_PREF + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_set_bssid_pref + + }, + { + { + .vendor_id = OUI_GOOGLE, + .subcmd = WIFI_SUBCMD_SET_BSSID_BLACKLIST + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV, + .doit = wl_cfgvendor_set_bssid_blacklist + + }, +#endif /* GSCAN_SUPPORT */ }; static const struct nl80211_vendor_cmd_info wl_vendor_events [] = { @@ -937,7 +1575,8 @@ static const struct nl80211_vendor_cmd_info wl_vendor_events [] = { { OUI_GOOGLE, GOOGLE_SCAN_FULL_RESULTS_EVENT }, { OUI_GOOGLE, GOOGLE_SCAN_RTT_EVENT }, { OUI_GOOGLE, GOOGLE_SCAN_COMPLETE_EVENT }, - { OUI_GOOGLE, GOOGLE_GSCAN_GEOFENCE_LOST_EVENT } + { OUI_GOOGLE, GOOGLE_GSCAN_GEOFENCE_LOST_EVENT }, + { OUI_GOOGLE, GOOGLE_SCAN_EPNO_EVENT }, #endif /* GSCAN_SUPPORT */ }; diff --git a/drivers/net/wireless/bcmdhd/wl_cfgvendor.h b/drivers/net/wireless/bcmdhd/wl_cfgvendor.h index 75cbdb14f54..2a8f4045a41 100644 --- a/drivers/net/wireless/bcmdhd/wl_cfgvendor.h +++ b/drivers/net/wireless/bcmdhd/wl_cfgvendor.h @@ -96,10 +96,21 @@ enum wl_vendor_subcmd { GSCAN_SUBCMD_GET_CHANNEL_LIST, ANDR_WIFI_SUBCMD_GET_FEATURE_SET, ANDR_WIFI_SUBCMD_GET_FEATURE_SET_MATRIX, + ANDR_WIFI_PNO_RANDOM_MAC_OUI, + ANDR_WIFI_NODFS_CHANNELS, + ANDR_WIFI_SET_COUNTRY, + GSCAN_SUBCMD_SET_EPNO_SSID, + WIFI_SUBCMD_SET_SSID_WHITELIST, + WIFI_SUBCMD_SET_LAZY_ROAM_PARAMS, + WIFI_SUBCMD_ENABLE_LAZY_ROAM, + WIFI_SUBCMD_SET_BSSID_PREF, + WIFI_SUBCMD_SET_BSSID_BLACKLIST, /* Add more sub commands here */ GSCAN_SUBCMD_MAX, LSTATS_SUBCMD_GET_INFO = ANDROID_NL80211_SUBCMD_LSTATS_RANGE_START, + /* Add more sub commands here */ + VENDOR_SUBCMD_MAX }; enum gscan_attributes { @@ -154,6 +165,48 @@ enum gscan_attributes { GSCAN_ATTRIBUTE_MIN_BREACHING, GSCAN_ATTRIBUTE_SIGNIFICANT_CHANGE_BSSIDS, GSCAN_ATTRIBUTE_SIGNIFICANT_CHANGE_FLUSH, + + /* EPNO */ + GSCAN_ATTRIBUTE_EPNO_SSID_LIST = 70, + GSCAN_ATTRIBUTE_EPNO_SSID, + GSCAN_ATTRIBUTE_EPNO_SSID_LEN, + GSCAN_ATTRIBUTE_EPNO_RSSI, + GSCAN_ATTRIBUTE_EPNO_FLAGS, + GSCAN_ATTRIBUTE_EPNO_AUTH, + GSCAN_ATTRIBUTE_EPNO_SSID_NUM, + GSCAN_ATTRIBUTE_EPNO_FLUSH, + + /* Roam SSID Whitelist and BSSID pref */ + GSCAN_ATTRIBUTE_WHITELIST_SSID = 80, + GSCAN_ATTRIBUTE_NUM_WL_SSID, + GSCAN_ATTRIBUTE_WL_SSID_LEN, + GSCAN_ATTRIBUTE_WL_SSID_FLUSH, + GSCAN_ATTRIBUTE_WHITELIST_SSID_ELEM, + GSCAN_ATTRIBUTE_NUM_BSSID, + GSCAN_ATTRIBUTE_BSSID_PREF_LIST, + GSCAN_ATTRIBUTE_BSSID_PREF_FLUSH, + GSCAN_ATTRIBUTE_BSSID_PREF, + GSCAN_ATTRIBUTE_RSSI_MODIFIER, + + + /* Roam cfg */ + GSCAN_ATTRIBUTE_A_BAND_BOOST_THRESHOLD = 90, + GSCAN_ATTRIBUTE_A_BAND_PENALTY_THRESHOLD, + GSCAN_ATTRIBUTE_A_BAND_BOOST_FACTOR, + GSCAN_ATTRIBUTE_A_BAND_PENALTY_FACTOR, + GSCAN_ATTRIBUTE_A_BAND_MAX_BOOST, + GSCAN_ATTRIBUTE_LAZY_ROAM_HYSTERESIS, + GSCAN_ATTRIBUTE_ALERT_ROAM_RSSI_TRIGGER, + GSCAN_ATTRIBUTE_LAZY_ROAM_ENABLE, + + /* BSSID blacklist */ + GSCAN_ATTRIBUTE_BSSID_BLACKLIST_FLUSH = 100, + GSCAN_ATTRIBUTE_BLACKLIST_BSSID, + + /* Adaptive scan attributes */ + GSCAN_ATTRIBUTE_BUCKET_STEP_COUNT = 110, + GSCAN_ATTRIBUTE_BUCKET_MAX_PERIOD, + GSCAN_ATTRIBUTE_MAX }; enum gscan_bucket_attributes { @@ -185,12 +238,16 @@ typedef enum wl_vendor_event { GOOGLE_SCAN_FULL_RESULTS_EVENT, GOOGLE_SCAN_RTT_EVENT, GOOGLE_SCAN_COMPLETE_EVENT, - GOOGLE_GSCAN_GEOFENCE_LOST_EVENT + GOOGLE_GSCAN_GEOFENCE_LOST_EVENT, + GOOGLE_SCAN_EPNO_EVENT } wl_vendor_event_t; enum andr_wifi_feature_set_attr { - ANDR_WIFI_ATTRIBUTE_NUM_FEATURE_SET, - ANDR_WIFI_ATTRIBUTE_FEATURE_SET + ANDR_WIFI_ATTRIBUTE_NUM_FEATURE_SET, + ANDR_WIFI_ATTRIBUTE_FEATURE_SET, + ANDR_WIFI_ATTRIBUTE_PNO_RANDOM_MAC_OUI, + ANDR_WIFI_ATTRIBUTE_NODFS_SET, + ANDR_WIFI_ATTRIBUTE_COUNTRY }; typedef enum wl_vendor_gscan_attribute { diff --git a/drivers/net/wireless/bcmdhd/wl_iw.c b/drivers/net/wireless/bcmdhd/wl_iw.c deleted file mode 100644 index fdc4d6a8689..00000000000 --- a/drivers/net/wireless/bcmdhd/wl_iw.c +++ /dev/null @@ -1,3682 +0,0 @@ -/* - * Linux Wireless Extensions support - * - * Copyright (C) 1999-2013, Broadcom Corporation - * - * Unless you and Broadcom execute a separate written software license - * agreement governing use of this software, this software is licensed to you - * under the terms of the GNU General Public License version 2 (the "GPL"), - * available at http://www.broadcom.com/licenses/GPLv2.php, with the - * following added to such license: - * - * As a special exception, the copyright holders of this software give you - * permission to link this software with independent modules, and to copy and - * distribute the resulting executable under terms of your choice, provided that - * you also meet, for each linked independent module, the terms and conditions of - * the license of that module. An independent module is a module which is not - * derived from this software. The special exception does not apply to any - * modifications of the software. - * - * Notwithstanding the above, under no circumstances may you combine this - * software in any way with any other Broadcom software provided under a license - * other than the GPL, without Broadcom's express prior written consent. - * - * $Id: wl_iw.c 396420 2013-04-12 06:55:45Z $ - */ - -#if defined(USE_IW) -#define LINUX_PORT - -#include -#include -#include - -#include -#include -#include - -#include -#include - -typedef const struct si_pub si_t; -#include - - -#include -#include - - -/* Broadcom extensions to WEXT, linux upstream has obsoleted WEXT */ -#ifndef IW_AUTH_KEY_MGMT_FT_802_1X -#define IW_AUTH_KEY_MGMT_FT_802_1X 0x04 -#endif - -#ifndef IW_AUTH_KEY_MGMT_FT_PSK -#define IW_AUTH_KEY_MGMT_FT_PSK 0x08 -#endif - -#ifndef IW_ENC_CAPA_FW_ROAM_ENABLE -#define IW_ENC_CAPA_FW_ROAM_ENABLE 0x00000020 -#endif - - -/* FC9: wireless.h 2.6.25-14.fc9.i686 is missing these, even though WIRELESS_EXT is set to latest - * version 22. - */ -#ifndef IW_ENCODE_ALG_PMK -#define IW_ENCODE_ALG_PMK 4 -#endif -#ifndef IW_ENC_CAPA_4WAY_HANDSHAKE -#define IW_ENC_CAPA_4WAY_HANDSHAKE 0x00000010 -#endif -/* End FC9. */ - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) -#include -#endif -#if defined(SOFTAP) -struct net_device *ap_net_dev = NULL; -tsk_ctl_t ap_eth_ctl; /* apsta AP netdev waiter thread */ -#endif /* SOFTAP */ - -extern bool wl_iw_conn_status_str(uint32 event_type, uint32 status, - uint32 reason, char* stringBuf, uint buflen); - -uint wl_msg_level = WL_ERROR_VAL; - -#define MAX_WLIW_IOCTL_LEN 1024 - -/* IOCTL swapping mode for Big Endian host with Little Endian dongle. Default to off */ -#define htod32(i) i -#define htod16(i) i -#define dtoh32(i) i -#define dtoh16(i) i -#define htodchanspec(i) i -#define dtohchanspec(i) i - -extern struct iw_statistics *dhd_get_wireless_stats(struct net_device *dev); -extern int dhd_wait_pend8021x(struct net_device *dev); - -#if WIRELESS_EXT < 19 -#define IW_IOCTL_IDX(cmd) ((cmd) - SIOCIWFIRST) -#define IW_EVENT_IDX(cmd) ((cmd) - IWEVFIRST) -#endif /* WIRELESS_EXT < 19 */ - - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -#define DAEMONIZE(a) -#elif ((LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)) && \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0))) -#define DAEMONIZE(a) daemonize(a); \ - allow_signal(SIGKILL); \ - allow_signal(SIGTERM); -#else /* Linux 2.4 (w/o preemption patch) */ -#define RAISE_RX_SOFTIRQ() \ - cpu_raise_softirq(smp_processor_id(), NET_RX_SOFTIRQ) -#define DAEMONIZE(a) daemonize(); \ - do { if (a) \ - strncpy(current->comm, a, MIN(sizeof(current->comm), (strlen(a) + 1))); \ - } while (0); -#endif /* LINUX_VERSION_CODE */ - -#define ISCAN_STATE_IDLE 0 -#define ISCAN_STATE_SCANING 1 - -/* the buf lengh can be WLC_IOCTL_MAXLEN (8K) to reduce iteration */ -#define WLC_IW_ISCAN_MAXLEN 2048 -typedef struct iscan_buf { - struct iscan_buf * next; - char iscan_buf[WLC_IW_ISCAN_MAXLEN]; -} iscan_buf_t; - -typedef struct iscan_info { - struct net_device *dev; - struct timer_list timer; - uint32 timer_ms; - uint32 timer_on; - int iscan_state; - iscan_buf_t * list_hdr; - iscan_buf_t * list_cur; - - /* Thread to work on iscan */ - long sysioc_pid; - struct semaphore sysioc_sem; - struct completion sysioc_exited; - - - char ioctlbuf[WLC_IOCTL_SMLEN]; -} iscan_info_t; -iscan_info_t *g_iscan = NULL; -static void wl_iw_timerfunc(ulong data); -static void wl_iw_set_event_mask(struct net_device *dev); -static int wl_iw_iscan(iscan_info_t *iscan, wlc_ssid_t *ssid, uint16 action); - -/* priv_link becomes netdev->priv and is the link between netdev and wlif struct */ -typedef struct priv_link { - wl_iw_t *wliw; -} priv_link_t; - -/* dev to priv_link */ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)) -#define WL_DEV_LINK(dev) (priv_link_t*)(dev->priv) -#else -#define WL_DEV_LINK(dev) (priv_link_t*)netdev_priv(dev) -#endif - -/* dev to wl_iw_t */ -#define IW_DEV_IF(dev) ((wl_iw_t*)(WL_DEV_LINK(dev))->wliw) - -static void swap_key_from_BE( - wl_wsec_key_t *key -) -{ - key->index = htod32(key->index); - key->len = htod32(key->len); - key->algo = htod32(key->algo); - key->flags = htod32(key->flags); - key->rxiv.hi = htod32(key->rxiv.hi); - key->rxiv.lo = htod16(key->rxiv.lo); - key->iv_initialized = htod32(key->iv_initialized); -} - -static void swap_key_to_BE( - wl_wsec_key_t *key -) -{ - key->index = dtoh32(key->index); - key->len = dtoh32(key->len); - key->algo = dtoh32(key->algo); - key->flags = dtoh32(key->flags); - key->rxiv.hi = dtoh32(key->rxiv.hi); - key->rxiv.lo = dtoh16(key->rxiv.lo); - key->iv_initialized = dtoh32(key->iv_initialized); -} - -static int -dev_wlc_ioctl( - struct net_device *dev, - int cmd, - void *arg, - int len -) -{ - struct ifreq ifr; - wl_ioctl_t ioc; - mm_segment_t fs; - int ret; - - memset(&ioc, 0, sizeof(ioc)); - ioc.cmd = cmd; - ioc.buf = arg; - ioc.len = len; - - strcpy(ifr.ifr_name, dev->name); - ifr.ifr_data = (caddr_t) &ioc; - -#ifndef LINUX_HYBRID - /* Causes an extraneous 'up'. If specific ioctls are failing due - to device down, then we can investigate those ioctls. - */ - dev_open(dev); -#endif - - fs = get_fs(); - set_fs(get_ds()); -#if defined(WL_USE_NETDEV_OPS) - ret = dev->netdev_ops->ndo_do_ioctl(dev, &ifr, SIOCDEVPRIVATE); -#else - ret = dev->do_ioctl(dev, &ifr, SIOCDEVPRIVATE); -#endif - set_fs(fs); - - return ret; -} - -/* -set named driver variable to int value and return error indication -calling example: dev_wlc_intvar_set(dev, "arate", rate) -*/ - -static int -dev_wlc_intvar_set( - struct net_device *dev, - char *name, - int val) -{ - char buf[WLC_IOCTL_SMLEN]; - uint len; - - val = htod32(val); - len = bcm_mkiovar(name, (char *)(&val), sizeof(val), buf, sizeof(buf)); - ASSERT(len); - - return (dev_wlc_ioctl(dev, WLC_SET_VAR, buf, len)); -} - -static int -dev_iw_iovar_setbuf( - struct net_device *dev, - char *iovar, - void *param, - int paramlen, - void *bufptr, - int buflen) -{ - int iolen; - - iolen = bcm_mkiovar(iovar, param, paramlen, bufptr, buflen); - ASSERT(iolen); - BCM_REFERENCE(iolen); - - return (dev_wlc_ioctl(dev, WLC_SET_VAR, bufptr, iolen)); -} - -static int -dev_iw_iovar_getbuf( - struct net_device *dev, - char *iovar, - void *param, - int paramlen, - void *bufptr, - int buflen) -{ - int iolen; - - iolen = bcm_mkiovar(iovar, param, paramlen, bufptr, buflen); - ASSERT(iolen); - BCM_REFERENCE(iolen); - - return (dev_wlc_ioctl(dev, WLC_GET_VAR, bufptr, buflen)); -} - -#if WIRELESS_EXT > 17 -static int -dev_wlc_bufvar_set( - struct net_device *dev, - char *name, - char *buf, int len) -{ - char *ioctlbuf; - uint buflen; - int error; - - ioctlbuf = kmalloc(MAX_WLIW_IOCTL_LEN, GFP_KERNEL); - if (!ioctlbuf) - return -ENOMEM; - - buflen = bcm_mkiovar(name, buf, len, ioctlbuf, MAX_WLIW_IOCTL_LEN); - ASSERT(buflen); - error = dev_wlc_ioctl(dev, WLC_SET_VAR, ioctlbuf, buflen); - - kfree(ioctlbuf); - return error; -} -#endif /* WIRELESS_EXT > 17 */ - -/* -get named driver variable to int value and return error indication -calling example: dev_wlc_bufvar_get(dev, "arate", &rate) -*/ - -static int -dev_wlc_bufvar_get( - struct net_device *dev, - char *name, - char *buf, int buflen) -{ - char *ioctlbuf; - int error; - - uint len; - - ioctlbuf = kmalloc(MAX_WLIW_IOCTL_LEN, GFP_KERNEL); - if (!ioctlbuf) - return -ENOMEM; - len = bcm_mkiovar(name, NULL, 0, ioctlbuf, MAX_WLIW_IOCTL_LEN); - ASSERT(len); - BCM_REFERENCE(len); - error = dev_wlc_ioctl(dev, WLC_GET_VAR, (void *)ioctlbuf, MAX_WLIW_IOCTL_LEN); - if (!error) - bcopy(ioctlbuf, buf, buflen); - - kfree(ioctlbuf); - return (error); -} - -/* -get named driver variable to int value and return error indication -calling example: dev_wlc_intvar_get(dev, "arate", &rate) -*/ - -static int -dev_wlc_intvar_get( - struct net_device *dev, - char *name, - int *retval) -{ - union { - char buf[WLC_IOCTL_SMLEN]; - int val; - } var; - int error; - - uint len; - uint data_null; - - len = bcm_mkiovar(name, (char *)(&data_null), 0, (char *)(&var), sizeof(var.buf)); - ASSERT(len); - error = dev_wlc_ioctl(dev, WLC_GET_VAR, (void *)&var, len); - - *retval = dtoh32(var.val); - - return (error); -} - -/* Maintain backward compatibility */ -#if WIRELESS_EXT < 13 -struct iw_request_info -{ - __u16 cmd; /* Wireless Extension command */ - __u16 flags; /* More to come ;-) */ -}; - -typedef int (*iw_handler)(struct net_device *dev, struct iw_request_info *info, - void *wrqu, char *extra); -#endif /* WIRELESS_EXT < 13 */ - -#if WIRELESS_EXT > 12 -static int -wl_iw_set_leddc( - struct net_device *dev, - struct iw_request_info *info, - union iwreq_data *wrqu, - char *extra -) -{ - int dc = *(int *)extra; - int error; - - error = dev_wlc_intvar_set(dev, "leddc", dc); - return error; -} - -static int -wl_iw_set_vlanmode( - struct net_device *dev, - struct iw_request_info *info, - union iwreq_data *wrqu, - char *extra -) -{ - int mode = *(int *)extra; - int error; - - mode = htod32(mode); - error = dev_wlc_intvar_set(dev, "vlan_mode", mode); - return error; -} - -static int -wl_iw_set_pm( - struct net_device *dev, - struct iw_request_info *info, - union iwreq_data *wrqu, - char *extra -) -{ - int pm = *(int *)extra; - int error; - - pm = htod32(pm); - error = dev_wlc_ioctl(dev, WLC_SET_PM, &pm, sizeof(pm)); - return error; -} - -#if WIRELESS_EXT > 17 -#endif /* WIRELESS_EXT > 17 */ -#endif /* WIRELESS_EXT > 12 */ - -int -wl_iw_send_priv_event( - struct net_device *dev, - char *flag -) -{ - union iwreq_data wrqu; - char extra[IW_CUSTOM_MAX + 1]; - int cmd; - - cmd = IWEVCUSTOM; - memset(&wrqu, 0, sizeof(wrqu)); - if (strlen(flag) > sizeof(extra)) - return -1; - - strcpy(extra, flag); - wrqu.data.length = strlen(extra); - wireless_send_event(dev, cmd, &wrqu, extra); - WL_TRACE(("Send IWEVCUSTOM Event as %s\n", extra)); - - return 0; -} - -static int -wl_iw_config_commit( - struct net_device *dev, - struct iw_request_info *info, - void *zwrq, - char *extra -) -{ - wlc_ssid_t ssid; - int error; - struct sockaddr bssid; - - WL_TRACE(("%s: SIOCSIWCOMMIT\n", dev->name)); - - if ((error = dev_wlc_ioctl(dev, WLC_GET_SSID, &ssid, sizeof(ssid)))) - return error; - - ssid.SSID_len = dtoh32(ssid.SSID_len); - - if (!ssid.SSID_len) - return 0; - - bzero(&bssid, sizeof(struct sockaddr)); - if ((error = dev_wlc_ioctl(dev, WLC_REASSOC, &bssid, ETHER_ADDR_LEN))) { - WL_ERROR(("%s: WLC_REASSOC failed (%d)\n", __FUNCTION__, error)); - return error; - } - - return 0; -} - -static int -wl_iw_get_name( - struct net_device *dev, - struct iw_request_info *info, - union iwreq_data *cwrq, - char *extra -) -{ - int phytype, err; - uint band[3]; - char cap[5]; - - WL_TRACE(("%s: SIOCGIWNAME\n", dev->name)); - - cap[0] = 0; - if ((err = dev_wlc_ioctl(dev, WLC_GET_PHYTYPE, &phytype, sizeof(phytype))) < 0) - goto done; - if ((err = dev_wlc_ioctl(dev, WLC_GET_BANDLIST, band, sizeof(band))) < 0) - goto done; - - band[0] = dtoh32(band[0]); - switch (phytype) { - case WLC_PHY_TYPE_A: - strcpy(cap, "a"); - break; - case WLC_PHY_TYPE_B: - strcpy(cap, "b"); - break; - case WLC_PHY_TYPE_LP: - case WLC_PHY_TYPE_G: - if (band[0] >= 2) - strcpy(cap, "abg"); - else - strcpy(cap, "bg"); - break; - case WLC_PHY_TYPE_N: - if (band[0] >= 2) - strcpy(cap, "abgn"); - else - strcpy(cap, "bgn"); - break; - } -done: - snprintf(cwrq->name, IFNAMSIZ, "IEEE 802.11%s", cap); - return 0; -} - -static int -wl_iw_set_freq( - struct net_device *dev, - struct iw_request_info *info, - struct iw_freq *fwrq, - char *extra -) -{ - int error, chan; - uint sf = 0; - - WL_TRACE(("%s: SIOCSIWFREQ\n", dev->name)); - - /* Setting by channel number */ - if (fwrq->e == 0 && fwrq->m < MAXCHANNEL) { - chan = fwrq->m; - } - - /* Setting by frequency */ - else { - /* Convert to MHz as best we can */ - if (fwrq->e >= 6) { - fwrq->e -= 6; - while (fwrq->e--) - fwrq->m *= 10; - } else if (fwrq->e < 6) { - while (fwrq->e++ < 6) - fwrq->m /= 10; - } - /* handle 4.9GHz frequencies as Japan 4 GHz based channelization */ - if (fwrq->m > 4000 && fwrq->m < 5000) - sf = WF_CHAN_FACTOR_4_G; /* start factor for 4 GHz */ - - chan = wf_mhz2channel(fwrq->m, sf); - } - chan = htod32(chan); - if ((error = dev_wlc_ioctl(dev, WLC_SET_CHANNEL, &chan, sizeof(chan)))) - return error; - - /* -EINPROGRESS: Call commit handler */ - return -EINPROGRESS; -} - -static int -wl_iw_get_freq( - struct net_device *dev, - struct iw_request_info *info, - struct iw_freq *fwrq, - char *extra -) -{ - channel_info_t ci; - int error; - - WL_TRACE(("%s: SIOCGIWFREQ\n", dev->name)); - - if ((error = dev_wlc_ioctl(dev, WLC_GET_CHANNEL, &ci, sizeof(ci)))) - return error; - - /* Return radio channel in channel form */ - fwrq->m = dtoh32(ci.hw_channel); - fwrq->e = dtoh32(0); - return 0; -} - -static int -wl_iw_set_mode( - struct net_device *dev, - struct iw_request_info *info, - __u32 *uwrq, - char *extra -) -{ - int infra = 0, ap = 0, error = 0; - - WL_TRACE(("%s: SIOCSIWMODE\n", dev->name)); - - switch (*uwrq) { - case IW_MODE_MASTER: - infra = ap = 1; - break; - case IW_MODE_ADHOC: - case IW_MODE_AUTO: - break; - case IW_MODE_INFRA: - infra = 1; - break; - default: - return -EINVAL; - } - infra = htod32(infra); - ap = htod32(ap); - - if ((error = dev_wlc_ioctl(dev, WLC_SET_INFRA, &infra, sizeof(infra))) || - (error = dev_wlc_ioctl(dev, WLC_SET_AP, &ap, sizeof(ap)))) - return error; - - /* -EINPROGRESS: Call commit handler */ - return -EINPROGRESS; -} - -static int -wl_iw_get_mode( - struct net_device *dev, - struct iw_request_info *info, - __u32 *uwrq, - char *extra -) -{ - int error, infra = 0, ap = 0; - - WL_TRACE(("%s: SIOCGIWMODE\n", dev->name)); - - if ((error = dev_wlc_ioctl(dev, WLC_GET_INFRA, &infra, sizeof(infra))) || - (error = dev_wlc_ioctl(dev, WLC_GET_AP, &ap, sizeof(ap)))) - return error; - - infra = dtoh32(infra); - ap = dtoh32(ap); - *uwrq = infra ? ap ? IW_MODE_MASTER : IW_MODE_INFRA : IW_MODE_ADHOC; - - return 0; -} - -static int -wl_iw_get_range( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - struct iw_range *range = (struct iw_range *) extra; - static int channels[MAXCHANNEL+1]; - wl_uint32_list_t *list = (wl_uint32_list_t *) channels; - wl_rateset_t rateset; - int error, i, k; - uint sf, ch; - - int phytype; - int bw_cap = 0, sgi_tx = 0, nmode = 0; - channel_info_t ci; - uint8 nrate_list2copy = 0; - uint16 nrate_list[4][8] = { {13, 26, 39, 52, 78, 104, 117, 130}, - {14, 29, 43, 58, 87, 116, 130, 144}, - {27, 54, 81, 108, 162, 216, 243, 270}, - {30, 60, 90, 120, 180, 240, 270, 300}}; - int fbt_cap = 0; - - WL_TRACE(("%s: SIOCGIWRANGE\n", dev->name)); - - if (!extra) - return -EINVAL; - - dwrq->length = sizeof(struct iw_range); - memset(range, 0, sizeof(*range)); - - /* We don't use nwids */ - range->min_nwid = range->max_nwid = 0; - - /* Set available channels/frequencies */ - list->count = htod32(MAXCHANNEL); - if ((error = dev_wlc_ioctl(dev, WLC_GET_VALID_CHANNELS, channels, sizeof(channels)))) - return error; - for (i = 0; i < dtoh32(list->count) && i < IW_MAX_FREQUENCIES; i++) { - range->freq[i].i = dtoh32(list->element[i]); - - ch = dtoh32(list->element[i]); - if (ch <= CH_MAX_2G_CHANNEL) - sf = WF_CHAN_FACTOR_2_4_G; - else - sf = WF_CHAN_FACTOR_5_G; - - range->freq[i].m = wf_channel2mhz(ch, sf); - range->freq[i].e = 6; - } - range->num_frequency = range->num_channels = i; - - /* Link quality (use NDIS cutoffs) */ - range->max_qual.qual = 5; - /* Signal level (use RSSI) */ - range->max_qual.level = 0x100 - 200; /* -200 dBm */ - /* Noise level (use noise) */ - range->max_qual.noise = 0x100 - 200; /* -200 dBm */ - /* Signal level threshold range (?) */ - range->sensitivity = 65535; - -#if WIRELESS_EXT > 11 - /* Link quality (use NDIS cutoffs) */ - range->avg_qual.qual = 3; - /* Signal level (use RSSI) */ - range->avg_qual.level = 0x100 + WL_IW_RSSI_GOOD; - /* Noise level (use noise) */ - range->avg_qual.noise = 0x100 - 75; /* -75 dBm */ -#endif /* WIRELESS_EXT > 11 */ - - /* Set available bitrates */ - if ((error = dev_wlc_ioctl(dev, WLC_GET_CURR_RATESET, &rateset, sizeof(rateset)))) - return error; - rateset.count = dtoh32(rateset.count); - range->num_bitrates = rateset.count; - for (i = 0; i < rateset.count && i < IW_MAX_BITRATES; i++) - range->bitrate[i] = (rateset.rates[i] & 0x7f) * 500000; /* convert to bps */ - if ((error = dev_wlc_intvar_get(dev, "nmode", &nmode))) - return error; - if ((error = dev_wlc_ioctl(dev, WLC_GET_PHYTYPE, &phytype, sizeof(phytype)))) - return error; - if (nmode == 1 && ((phytype == WLC_PHY_TYPE_SSN) || (phytype == WLC_PHY_TYPE_LCN) || - (phytype == WLC_PHY_TYPE_LCN40))) { - if ((error = dev_wlc_intvar_get(dev, "mimo_bw_cap", &bw_cap))) - return error; - if ((error = dev_wlc_intvar_get(dev, "sgi_tx", &sgi_tx))) - return error; - if ((error = dev_wlc_ioctl(dev, WLC_GET_CHANNEL, &ci, sizeof(channel_info_t)))) - return error; - ci.hw_channel = dtoh32(ci.hw_channel); - - if (bw_cap == 0 || - (bw_cap == 2 && ci.hw_channel <= 14)) { - if (sgi_tx == 0) - nrate_list2copy = 0; - else - nrate_list2copy = 1; - } - if (bw_cap == 1 || - (bw_cap == 2 && ci.hw_channel >= 36)) { - if (sgi_tx == 0) - nrate_list2copy = 2; - else - nrate_list2copy = 3; - } - range->num_bitrates += 8; - ASSERT(range->num_bitrates < IW_MAX_BITRATES); - for (k = 0; i < range->num_bitrates; k++, i++) { - /* convert to bps */ - range->bitrate[i] = (nrate_list[nrate_list2copy][k]) * 500000; - } - } - - /* Set an indication of the max TCP throughput - * in bit/s that we can expect using this interface. - * May be use for QoS stuff... Jean II - */ - if ((error = dev_wlc_ioctl(dev, WLC_GET_PHYTYPE, &i, sizeof(i)))) - return error; - i = dtoh32(i); - if (i == WLC_PHY_TYPE_A) - range->throughput = 24000000; /* 24 Mbits/s */ - else - range->throughput = 1500000; /* 1.5 Mbits/s */ - - /* RTS and fragmentation thresholds */ - range->min_rts = 0; - range->max_rts = 2347; - range->min_frag = 256; - range->max_frag = 2346; - - range->max_encoding_tokens = DOT11_MAX_DEFAULT_KEYS; - range->num_encoding_sizes = 4; - range->encoding_size[0] = WEP1_KEY_SIZE; - range->encoding_size[1] = WEP128_KEY_SIZE; -#if WIRELESS_EXT > 17 - range->encoding_size[2] = TKIP_KEY_SIZE; -#else - range->encoding_size[2] = 0; -#endif - range->encoding_size[3] = AES_KEY_SIZE; - - /* Do not support power micro-management */ - range->min_pmp = 0; - range->max_pmp = 0; - range->min_pmt = 0; - range->max_pmt = 0; - range->pmp_flags = 0; - range->pm_capa = 0; - - /* Transmit Power - values are in mW */ - range->num_txpower = 2; - range->txpower[0] = 1; - range->txpower[1] = 255; - range->txpower_capa = IW_TXPOW_MWATT; - -#if WIRELESS_EXT > 10 - range->we_version_compiled = WIRELESS_EXT; - range->we_version_source = 19; - - /* Only support retry limits */ - range->retry_capa = IW_RETRY_LIMIT; - range->retry_flags = IW_RETRY_LIMIT; - range->r_time_flags = 0; - /* SRL and LRL limits */ - range->min_retry = 1; - range->max_retry = 255; - /* Retry lifetime limits unsupported */ - range->min_r_time = 0; - range->max_r_time = 0; -#endif /* WIRELESS_EXT > 10 */ - -#if WIRELESS_EXT > 17 - range->enc_capa = IW_ENC_CAPA_WPA; - range->enc_capa |= IW_ENC_CAPA_CIPHER_TKIP; - range->enc_capa |= IW_ENC_CAPA_CIPHER_CCMP; - range->enc_capa |= IW_ENC_CAPA_WPA2; - - /* Determine driver FBT capability. */ - if (dev_wlc_intvar_get(dev, "fbt_cap", &fbt_cap) == 0) { - if (fbt_cap == WLC_FBT_CAP_DRV_4WAY_AND_REASSOC) { - /* Tell the host (e.g. wpa_supplicant) to let driver do the handshake */ - range->enc_capa |= IW_ENC_CAPA_4WAY_HANDSHAKE; - } - } - -#ifdef BCMFW_ROAM_ENABLE_WEXT - /* Advertise firmware roam capability to the external supplicant */ - range->enc_capa |= IW_ENC_CAPA_FW_ROAM_ENABLE; -#endif /* BCMFW_ROAM_ENABLE_WEXT */ - - /* Event capability (kernel) */ - IW_EVENT_CAPA_SET_KERNEL(range->event_capa); - /* Event capability (driver) */ - IW_EVENT_CAPA_SET(range->event_capa, SIOCGIWAP); - IW_EVENT_CAPA_SET(range->event_capa, SIOCGIWSCAN); - IW_EVENT_CAPA_SET(range->event_capa, IWEVTXDROP); - IW_EVENT_CAPA_SET(range->event_capa, IWEVMICHAELMICFAILURE); - IW_EVENT_CAPA_SET(range->event_capa, IWEVASSOCREQIE); - IW_EVENT_CAPA_SET(range->event_capa, IWEVASSOCRESPIE); - IW_EVENT_CAPA_SET(range->event_capa, IWEVPMKIDCAND); - -#if WIRELESS_EXT >= 22 && defined(IW_SCAN_CAPA_ESSID) - /* FC7 wireless.h defines EXT 22 but doesn't define scan_capa bits */ - range->scan_capa = IW_SCAN_CAPA_ESSID; -#endif -#endif /* WIRELESS_EXT > 17 */ - - return 0; -} - -static int -rssi_to_qual(int rssi) -{ - if (rssi <= WL_IW_RSSI_NO_SIGNAL) - return 0; - else if (rssi <= WL_IW_RSSI_VERY_LOW) - return 1; - else if (rssi <= WL_IW_RSSI_LOW) - return 2; - else if (rssi <= WL_IW_RSSI_GOOD) - return 3; - else if (rssi <= WL_IW_RSSI_VERY_GOOD) - return 4; - else - return 5; -} - -static int -wl_iw_set_spy( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_iw_t *iw = IW_DEV_IF(dev); - struct sockaddr *addr = (struct sockaddr *) extra; - int i; - - WL_TRACE(("%s: SIOCSIWSPY\n", dev->name)); - - if (!extra) - return -EINVAL; - - iw->spy_num = MIN(ARRAYSIZE(iw->spy_addr), dwrq->length); - for (i = 0; i < iw->spy_num; i++) - memcpy(&iw->spy_addr[i], addr[i].sa_data, ETHER_ADDR_LEN); - memset(iw->spy_qual, 0, sizeof(iw->spy_qual)); - - return 0; -} - -static int -wl_iw_get_spy( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_iw_t *iw = IW_DEV_IF(dev); - struct sockaddr *addr = (struct sockaddr *) extra; - struct iw_quality *qual = (struct iw_quality *) &addr[iw->spy_num]; - int i; - - WL_TRACE(("%s: SIOCGIWSPY\n", dev->name)); - - if (!extra) - return -EINVAL; - - dwrq->length = iw->spy_num; - for (i = 0; i < iw->spy_num; i++) { - memcpy(addr[i].sa_data, &iw->spy_addr[i], ETHER_ADDR_LEN); - addr[i].sa_family = AF_UNIX; - memcpy(&qual[i], &iw->spy_qual[i], sizeof(struct iw_quality)); - iw->spy_qual[i].updated = 0; - } - - return 0; -} - -static int -wl_iw_set_wap( - struct net_device *dev, - struct iw_request_info *info, - struct sockaddr *awrq, - char *extra -) -{ - int error = -EINVAL; - - WL_TRACE(("%s: SIOCSIWAP\n", dev->name)); - - if (awrq->sa_family != ARPHRD_ETHER) { - WL_ERROR(("%s: Invalid Header...sa_family\n", __FUNCTION__)); - return -EINVAL; - } - - /* Ignore "auto" or "off" */ - if (ETHER_ISBCAST(awrq->sa_data) || ETHER_ISNULLADDR(awrq->sa_data)) { - scb_val_t scbval; - bzero(&scbval, sizeof(scb_val_t)); - if ((error = dev_wlc_ioctl(dev, WLC_DISASSOC, &scbval, sizeof(scb_val_t)))) { - WL_ERROR(("%s: WLC_DISASSOC failed (%d).\n", __FUNCTION__, error)); - } - return 0; - } - /* WL_ASSOC(("Assoc to %s\n", bcm_ether_ntoa((struct ether_addr *)&(awrq->sa_data), - * eabuf))); - */ - /* Reassociate to the specified AP */ - if ((error = dev_wlc_ioctl(dev, WLC_REASSOC, awrq->sa_data, ETHER_ADDR_LEN))) { - WL_ERROR(("%s: WLC_REASSOC failed (%d).\n", __FUNCTION__, error)); - return error; - } - - return 0; -} - -static int -wl_iw_get_wap( - struct net_device *dev, - struct iw_request_info *info, - struct sockaddr *awrq, - char *extra -) -{ - WL_TRACE(("%s: SIOCGIWAP\n", dev->name)); - - awrq->sa_family = ARPHRD_ETHER; - memset(awrq->sa_data, 0, ETHER_ADDR_LEN); - - /* Ignore error (may be down or disassociated) */ - (void) dev_wlc_ioctl(dev, WLC_GET_BSSID, awrq->sa_data, ETHER_ADDR_LEN); - - return 0; -} - -#if WIRELESS_EXT > 17 -static int -wl_iw_mlme( - struct net_device *dev, - struct iw_request_info *info, - struct sockaddr *awrq, - char *extra -) -{ - struct iw_mlme *mlme; - scb_val_t scbval; - int error = -EINVAL; - - WL_TRACE(("%s: SIOCSIWMLME\n", dev->name)); - - mlme = (struct iw_mlme *)extra; - if (mlme == NULL) { - WL_ERROR(("Invalid ioctl data.\n")); - return error; - } - - scbval.val = mlme->reason_code; - bcopy(&mlme->addr.sa_data, &scbval.ea, ETHER_ADDR_LEN); - - if (mlme->cmd == IW_MLME_DISASSOC) { - scbval.val = htod32(scbval.val); - error = dev_wlc_ioctl(dev, WLC_DISASSOC, &scbval, sizeof(scb_val_t)); - } - else if (mlme->cmd == IW_MLME_DEAUTH) { - scbval.val = htod32(scbval.val); - error = dev_wlc_ioctl(dev, WLC_SCB_DEAUTHENTICATE_FOR_REASON, &scbval, - sizeof(scb_val_t)); - } - else { - WL_ERROR(("%s: Invalid ioctl data.\n", __FUNCTION__)); - return error; - } - - return error; -} -#endif /* WIRELESS_EXT > 17 */ - -static int -wl_iw_get_aplist( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_scan_results_t *list; - struct sockaddr *addr = (struct sockaddr *) extra; - struct iw_quality qual[IW_MAX_AP]; - wl_bss_info_t *bi = NULL; - int error, i; - uint buflen = dwrq->length; - - WL_TRACE(("%s: SIOCGIWAPLIST\n", dev->name)); - - if (!extra) - return -EINVAL; - - /* Get scan results (too large to put on the stack) */ - list = kmalloc(buflen, GFP_KERNEL); - if (!list) - return -ENOMEM; - memset(list, 0, buflen); - list->buflen = htod32(buflen); - if ((error = dev_wlc_ioctl(dev, WLC_SCAN_RESULTS, list, buflen))) { - WL_ERROR(("%d: Scan results error %d\n", __LINE__, error)); - kfree(list); - return error; - } - list->buflen = dtoh32(list->buflen); - list->version = dtoh32(list->version); - list->count = dtoh32(list->count); - ASSERT(list->version == WL_BSS_INFO_VERSION); - - for (i = 0, dwrq->length = 0; i < list->count && dwrq->length < IW_MAX_AP; i++) { - bi = bi ? (wl_bss_info_t *)((uintptr)bi + dtoh32(bi->length)) : list->bss_info; - ASSERT(((uintptr)bi + dtoh32(bi->length)) <= ((uintptr)list + - buflen)); - - /* Infrastructure only */ - if (!(dtoh16(bi->capability) & DOT11_CAP_ESS)) - continue; - - /* BSSID */ - memcpy(addr[dwrq->length].sa_data, &bi->BSSID, ETHER_ADDR_LEN); - addr[dwrq->length].sa_family = ARPHRD_ETHER; - qual[dwrq->length].qual = rssi_to_qual(dtoh16(bi->RSSI)); - qual[dwrq->length].level = 0x100 + dtoh16(bi->RSSI); - qual[dwrq->length].noise = 0x100 + bi->phy_noise; - - /* Updated qual, level, and noise */ -#if WIRELESS_EXT > 18 - qual[dwrq->length].updated = IW_QUAL_ALL_UPDATED | IW_QUAL_DBM; -#else - qual[dwrq->length].updated = 7; -#endif /* WIRELESS_EXT > 18 */ - - dwrq->length++; - } - - kfree(list); - - if (dwrq->length) { - memcpy(&addr[dwrq->length], qual, sizeof(struct iw_quality) * dwrq->length); - /* Provided qual */ - dwrq->flags = 1; - } - - return 0; -} - -static int -wl_iw_iscan_get_aplist( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_scan_results_t *list; - iscan_buf_t * buf; - iscan_info_t *iscan = g_iscan; - - struct sockaddr *addr = (struct sockaddr *) extra; - struct iw_quality qual[IW_MAX_AP]; - wl_bss_info_t *bi = NULL; - int i; - - WL_TRACE(("%s: SIOCGIWAPLIST\n", dev->name)); - - if (!extra) - return -EINVAL; - - if ((!iscan) || (iscan->sysioc_pid < 0)) { - return wl_iw_get_aplist(dev, info, dwrq, extra); - } - - buf = iscan->list_hdr; - /* Get scan results (too large to put on the stack) */ - while (buf) { - list = &((wl_iscan_results_t*)buf->iscan_buf)->results; - ASSERT(list->version == WL_BSS_INFO_VERSION); - - bi = NULL; - for (i = 0, dwrq->length = 0; i < list->count && dwrq->length < IW_MAX_AP; i++) { - bi = bi ? (wl_bss_info_t *)((uintptr)bi + dtoh32(bi->length)) : list->bss_info; - ASSERT(((uintptr)bi + dtoh32(bi->length)) <= ((uintptr)list + - WLC_IW_ISCAN_MAXLEN)); - - /* Infrastructure only */ - if (!(dtoh16(bi->capability) & DOT11_CAP_ESS)) - continue; - - /* BSSID */ - memcpy(addr[dwrq->length].sa_data, &bi->BSSID, ETHER_ADDR_LEN); - addr[dwrq->length].sa_family = ARPHRD_ETHER; - qual[dwrq->length].qual = rssi_to_qual(dtoh16(bi->RSSI)); - qual[dwrq->length].level = 0x100 + dtoh16(bi->RSSI); - qual[dwrq->length].noise = 0x100 + bi->phy_noise; - - /* Updated qual, level, and noise */ -#if WIRELESS_EXT > 18 - qual[dwrq->length].updated = IW_QUAL_ALL_UPDATED | IW_QUAL_DBM; -#else - qual[dwrq->length].updated = 7; -#endif /* WIRELESS_EXT > 18 */ - - dwrq->length++; - } - buf = buf->next; - } - if (dwrq->length) { - memcpy(&addr[dwrq->length], qual, sizeof(struct iw_quality) * dwrq->length); - /* Provided qual */ - dwrq->flags = 1; - } - - return 0; -} - -#if WIRELESS_EXT > 13 -static int -wl_iw_set_scan( - struct net_device *dev, - struct iw_request_info *info, - union iwreq_data *wrqu, - char *extra -) -{ - wlc_ssid_t ssid; - - WL_TRACE(("%s: SIOCSIWSCAN\n", dev->name)); - - /* default Broadcast scan */ - memset(&ssid, 0, sizeof(ssid)); - -#if WIRELESS_EXT > 17 - /* check for given essid */ - if (wrqu->data.length == sizeof(struct iw_scan_req)) { - if (wrqu->data.flags & IW_SCAN_THIS_ESSID) { - struct iw_scan_req *req = (struct iw_scan_req *)extra; - ssid.SSID_len = MIN(sizeof(ssid.SSID), req->essid_len); - memcpy(ssid.SSID, req->essid, ssid.SSID_len); - ssid.SSID_len = htod32(ssid.SSID_len); - } - } -#endif - /* Ignore error (most likely scan in progress) */ - (void) dev_wlc_ioctl(dev, WLC_SCAN, &ssid, sizeof(ssid)); - - return 0; -} - -static int -wl_iw_iscan_set_scan( - struct net_device *dev, - struct iw_request_info *info, - union iwreq_data *wrqu, - char *extra -) -{ - wlc_ssid_t ssid; - iscan_info_t *iscan = g_iscan; - - WL_TRACE(("%s: SIOCSIWSCAN\n", dev->name)); - - /* use backup if our thread is not successful */ - if ((!iscan) || (iscan->sysioc_pid < 0)) { - return wl_iw_set_scan(dev, info, wrqu, extra); - } - if (iscan->iscan_state == ISCAN_STATE_SCANING) { - return 0; - } - - /* default Broadcast scan */ - memset(&ssid, 0, sizeof(ssid)); - -#if WIRELESS_EXT > 17 - /* check for given essid */ - if (wrqu->data.length == sizeof(struct iw_scan_req)) { - if (wrqu->data.flags & IW_SCAN_THIS_ESSID) { - struct iw_scan_req *req = (struct iw_scan_req *)extra; - ssid.SSID_len = MIN(sizeof(ssid.SSID), req->essid_len); - memcpy(ssid.SSID, req->essid, ssid.SSID_len); - ssid.SSID_len = htod32(ssid.SSID_len); - } - } -#endif - - iscan->list_cur = iscan->list_hdr; - iscan->iscan_state = ISCAN_STATE_SCANING; - - - wl_iw_set_event_mask(dev); - wl_iw_iscan(iscan, &ssid, WL_SCAN_ACTION_START); - - iscan->timer.expires = jiffies + msecs_to_jiffies(iscan->timer_ms); - add_timer(&iscan->timer); - iscan->timer_on = 1; - - return 0; -} - -#if WIRELESS_EXT > 17 -static bool -ie_is_wpa_ie(uint8 **wpaie, uint8 **tlvs, int *tlvs_len) -{ -/* Is this body of this tlvs entry a WPA entry? If */ -/* not update the tlvs buffer pointer/length */ - uint8 *ie = *wpaie; - - /* If the contents match the WPA_OUI and type=1 */ - if ((ie[1] >= 6) && - !bcmp((const void *)&ie[2], (const void *)(WPA_OUI "\x01"), 4)) { - return TRUE; - } - - /* point to the next ie */ - ie += ie[1] + 2; - /* calculate the length of the rest of the buffer */ - *tlvs_len -= (int)(ie - *tlvs); - /* update the pointer to the start of the buffer */ - *tlvs = ie; - return FALSE; -} - -static bool -ie_is_wps_ie(uint8 **wpsie, uint8 **tlvs, int *tlvs_len) -{ -/* Is this body of this tlvs entry a WPS entry? If */ -/* not update the tlvs buffer pointer/length */ - uint8 *ie = *wpsie; - - /* If the contents match the WPA_OUI and type=4 */ - if ((ie[1] >= 4) && - !bcmp((const void *)&ie[2], (const void *)(WPA_OUI "\x04"), 4)) { - return TRUE; - } - - /* point to the next ie */ - ie += ie[1] + 2; - /* calculate the length of the rest of the buffer */ - *tlvs_len -= (int)(ie - *tlvs); - /* update the pointer to the start of the buffer */ - *tlvs = ie; - return FALSE; -} -#endif /* WIRELESS_EXT > 17 */ - - -static int -wl_iw_handle_scanresults_ies(char **event_p, char *end, - struct iw_request_info *info, wl_bss_info_t *bi) -{ -#if WIRELESS_EXT > 17 - struct iw_event iwe; - char *event; - - event = *event_p; - if (bi->ie_length) { - /* look for wpa/rsn ies in the ie list... */ - bcm_tlv_t *ie; - uint8 *ptr = ((uint8 *)bi) + sizeof(wl_bss_info_t); - int ptr_len = bi->ie_length; - - if ((ie = bcm_parse_tlvs(ptr, ptr_len, DOT11_MNG_RSN_ID))) { - iwe.cmd = IWEVGENIE; - iwe.u.data.length = ie->len + 2; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, (char *)ie); - } - ptr = ((uint8 *)bi) + sizeof(wl_bss_info_t); - - if ((ie = bcm_parse_tlvs(ptr, ptr_len, DOT11_MNG_MDIE_ID))) { - iwe.cmd = IWEVGENIE; - iwe.u.data.length = ie->len + 2; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, (char *)ie); - } - ptr = ((uint8 *)bi) + sizeof(wl_bss_info_t); - - while ((ie = bcm_parse_tlvs(ptr, ptr_len, DOT11_MNG_WPA_ID))) { - /* look for WPS IE */ - if (ie_is_wps_ie(((uint8 **)&ie), &ptr, &ptr_len)) { - iwe.cmd = IWEVGENIE; - iwe.u.data.length = ie->len + 2; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, (char *)ie); - break; - } - } - - ptr = ((uint8 *)bi) + sizeof(wl_bss_info_t); - ptr_len = bi->ie_length; - while ((ie = bcm_parse_tlvs(ptr, ptr_len, DOT11_MNG_WPA_ID))) { - if (ie_is_wpa_ie(((uint8 **)&ie), &ptr, &ptr_len)) { - iwe.cmd = IWEVGENIE; - iwe.u.data.length = ie->len + 2; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, (char *)ie); - break; - } - } - - *event_p = event; - } - -#endif /* WIRELESS_EXT > 17 */ - return 0; -} -static int -wl_iw_get_scan( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - channel_info_t ci; - wl_scan_results_t *list; - struct iw_event iwe; - wl_bss_info_t *bi = NULL; - int error, i, j; - char *event = extra, *end = extra + dwrq->length, *value; - uint buflen = dwrq->length; - - WL_TRACE(("%s: SIOCGIWSCAN\n", dev->name)); - - if (!extra) - return -EINVAL; - - /* Check for scan in progress */ - if ((error = dev_wlc_ioctl(dev, WLC_GET_CHANNEL, &ci, sizeof(ci)))) - return error; - ci.scan_channel = dtoh32(ci.scan_channel); - if (ci.scan_channel) - return -EAGAIN; - - /* Get scan results (too large to put on the stack) */ - list = kmalloc(buflen, GFP_KERNEL); - if (!list) - return -ENOMEM; - memset(list, 0, buflen); - list->buflen = htod32(buflen); - if ((error = dev_wlc_ioctl(dev, WLC_SCAN_RESULTS, list, buflen))) { - kfree(list); - return error; - } - list->buflen = dtoh32(list->buflen); - list->version = dtoh32(list->version); - list->count = dtoh32(list->count); - - ASSERT(list->version == WL_BSS_INFO_VERSION); - - for (i = 0; i < list->count && i < IW_MAX_AP; i++) { - bi = bi ? (wl_bss_info_t *)((uintptr)bi + dtoh32(bi->length)) : list->bss_info; - ASSERT(((uintptr)bi + dtoh32(bi->length)) <= ((uintptr)list + - buflen)); - - /* First entry must be the BSSID */ - iwe.cmd = SIOCGIWAP; - iwe.u.ap_addr.sa_family = ARPHRD_ETHER; - memcpy(iwe.u.ap_addr.sa_data, &bi->BSSID, ETHER_ADDR_LEN); - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_ADDR_LEN); - - /* SSID */ - iwe.u.data.length = dtoh32(bi->SSID_len); - iwe.cmd = SIOCGIWESSID; - iwe.u.data.flags = 1; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, bi->SSID); - - /* Mode */ - if (dtoh16(bi->capability) & (DOT11_CAP_ESS | DOT11_CAP_IBSS)) { - iwe.cmd = SIOCGIWMODE; - if (dtoh16(bi->capability) & DOT11_CAP_ESS) - iwe.u.mode = IW_MODE_INFRA; - else - iwe.u.mode = IW_MODE_ADHOC; - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_UINT_LEN); - } - - /* Channel */ - iwe.cmd = SIOCGIWFREQ; - iwe.u.freq.m = wf_channel2mhz(CHSPEC_CHANNEL(bi->chanspec), - CHSPEC_CHANNEL(bi->chanspec) <= CH_MAX_2G_CHANNEL ? - WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G); - iwe.u.freq.e = 6; - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_FREQ_LEN); - - /* Channel quality */ - iwe.cmd = IWEVQUAL; - iwe.u.qual.qual = rssi_to_qual(dtoh16(bi->RSSI)); - iwe.u.qual.level = 0x100 + dtoh16(bi->RSSI); - iwe.u.qual.noise = 0x100 + bi->phy_noise; - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_QUAL_LEN); - - /* WPA, WPA2, WPS, WAPI IEs */ - wl_iw_handle_scanresults_ies(&event, end, info, bi); - - /* Encryption */ - iwe.cmd = SIOCGIWENCODE; - if (dtoh16(bi->capability) & DOT11_CAP_PRIVACY) - iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY; - else - iwe.u.data.flags = IW_ENCODE_DISABLED; - iwe.u.data.length = 0; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, (char *)event); - - /* Rates */ - if (bi->rateset.count) { - value = event + IW_EV_LCP_LEN; - iwe.cmd = SIOCGIWRATE; - /* Those two flags are ignored... */ - iwe.u.bitrate.fixed = iwe.u.bitrate.disabled = 0; - for (j = 0; j < bi->rateset.count && j < IW_MAX_BITRATES; j++) { - iwe.u.bitrate.value = (bi->rateset.rates[j] & 0x7f) * 500000; - value = IWE_STREAM_ADD_VALUE(info, event, value, end, &iwe, - IW_EV_PARAM_LEN); - } - event = value; - } - } - - kfree(list); - - dwrq->length = event - extra; - dwrq->flags = 0; /* todo */ - - return 0; -} - -static int -wl_iw_iscan_get_scan( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_scan_results_t *list; - struct iw_event iwe; - wl_bss_info_t *bi = NULL; - int ii, j; - int apcnt; - char *event = extra, *end = extra + dwrq->length, *value; - iscan_info_t *iscan = g_iscan; - iscan_buf_t * p_buf; - - WL_TRACE(("%s: SIOCGIWSCAN\n", dev->name)); - - if (!extra) - return -EINVAL; - - /* use backup if our thread is not successful */ - if ((!iscan) || (iscan->sysioc_pid < 0)) { - return wl_iw_get_scan(dev, info, dwrq, extra); - } - - /* Check for scan in progress */ - if (iscan->iscan_state == ISCAN_STATE_SCANING) - return -EAGAIN; - - apcnt = 0; - p_buf = iscan->list_hdr; - /* Get scan results */ - while (p_buf != iscan->list_cur) { - list = &((wl_iscan_results_t*)p_buf->iscan_buf)->results; - - if (list->version != WL_BSS_INFO_VERSION) { - WL_ERROR(("list->version %d != WL_BSS_INFO_VERSION\n", list->version)); - } - - bi = NULL; - for (ii = 0; ii < list->count && apcnt < IW_MAX_AP; apcnt++, ii++) { - bi = bi ? (wl_bss_info_t *)((uintptr)bi + dtoh32(bi->length)) : list->bss_info; - ASSERT(((uintptr)bi + dtoh32(bi->length)) <= ((uintptr)list + - WLC_IW_ISCAN_MAXLEN)); - - /* overflow check cover fields before wpa IEs */ - if (event + ETHER_ADDR_LEN + bi->SSID_len + IW_EV_UINT_LEN + IW_EV_FREQ_LEN + - IW_EV_QUAL_LEN >= end) - return -E2BIG; - /* First entry must be the BSSID */ - iwe.cmd = SIOCGIWAP; - iwe.u.ap_addr.sa_family = ARPHRD_ETHER; - memcpy(iwe.u.ap_addr.sa_data, &bi->BSSID, ETHER_ADDR_LEN); - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_ADDR_LEN); - - /* SSID */ - iwe.u.data.length = dtoh32(bi->SSID_len); - iwe.cmd = SIOCGIWESSID; - iwe.u.data.flags = 1; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, bi->SSID); - - /* Mode */ - if (dtoh16(bi->capability) & (DOT11_CAP_ESS | DOT11_CAP_IBSS)) { - iwe.cmd = SIOCGIWMODE; - if (dtoh16(bi->capability) & DOT11_CAP_ESS) - iwe.u.mode = IW_MODE_INFRA; - else - iwe.u.mode = IW_MODE_ADHOC; - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_UINT_LEN); - } - - /* Channel */ - iwe.cmd = SIOCGIWFREQ; - iwe.u.freq.m = wf_channel2mhz(CHSPEC_CHANNEL(bi->chanspec), - CHSPEC_CHANNEL(bi->chanspec) <= CH_MAX_2G_CHANNEL ? - WF_CHAN_FACTOR_2_4_G : WF_CHAN_FACTOR_5_G); - iwe.u.freq.e = 6; - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_FREQ_LEN); - - /* Channel quality */ - iwe.cmd = IWEVQUAL; - iwe.u.qual.qual = rssi_to_qual(dtoh16(bi->RSSI)); - iwe.u.qual.level = 0x100 + dtoh16(bi->RSSI); - iwe.u.qual.noise = 0x100 + bi->phy_noise; - event = IWE_STREAM_ADD_EVENT(info, event, end, &iwe, IW_EV_QUAL_LEN); - - /* WPA, WPA2, WPS, WAPI IEs */ - wl_iw_handle_scanresults_ies(&event, end, info, bi); - - /* Encryption */ - iwe.cmd = SIOCGIWENCODE; - if (dtoh16(bi->capability) & DOT11_CAP_PRIVACY) - iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY; - else - iwe.u.data.flags = IW_ENCODE_DISABLED; - iwe.u.data.length = 0; - event = IWE_STREAM_ADD_POINT(info, event, end, &iwe, (char *)event); - - /* Rates */ - if (bi->rateset.count <= sizeof(bi->rateset.rates)) { - if (event + IW_MAX_BITRATES*IW_EV_PARAM_LEN >= end) - return -E2BIG; - - value = event + IW_EV_LCP_LEN; - iwe.cmd = SIOCGIWRATE; - /* Those two flags are ignored... */ - iwe.u.bitrate.fixed = iwe.u.bitrate.disabled = 0; - for (j = 0; j < bi->rateset.count && j < IW_MAX_BITRATES; j++) { - iwe.u.bitrate.value = (bi->rateset.rates[j] & 0x7f) * 500000; - value = IWE_STREAM_ADD_VALUE(info, event, value, end, &iwe, - IW_EV_PARAM_LEN); - } - event = value; - } - } - p_buf = p_buf->next; - } /* while (p_buf) */ - - dwrq->length = event - extra; - dwrq->flags = 0; /* todo */ - - return 0; -} - -#endif /* WIRELESS_EXT > 13 */ - - -static int -wl_iw_set_essid( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wlc_ssid_t ssid; - int error; - - WL_TRACE(("%s: SIOCSIWESSID\n", dev->name)); - - /* default Broadcast SSID */ - memset(&ssid, 0, sizeof(ssid)); - if (dwrq->length && extra) { -#if WIRELESS_EXT > 20 - ssid.SSID_len = MIN(sizeof(ssid.SSID), dwrq->length); -#else - ssid.SSID_len = MIN(sizeof(ssid.SSID), dwrq->length-1); -#endif - memcpy(ssid.SSID, extra, ssid.SSID_len); - ssid.SSID_len = htod32(ssid.SSID_len); - - if ((error = dev_wlc_ioctl(dev, WLC_SET_SSID, &ssid, sizeof(ssid)))) - return error; - } - /* If essid null then it is "iwconfig essid off" command */ - else { - scb_val_t scbval; - bzero(&scbval, sizeof(scb_val_t)); - if ((error = dev_wlc_ioctl(dev, WLC_DISASSOC, &scbval, sizeof(scb_val_t)))) - return error; - } - return 0; -} - -static int -wl_iw_get_essid( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wlc_ssid_t ssid; - int error; - - WL_TRACE(("%s: SIOCGIWESSID\n", dev->name)); - - if (!extra) - return -EINVAL; - - if ((error = dev_wlc_ioctl(dev, WLC_GET_SSID, &ssid, sizeof(ssid)))) { - WL_ERROR(("Error getting the SSID\n")); - return error; - } - - ssid.SSID_len = dtoh32(ssid.SSID_len); - - /* Get the current SSID */ - memcpy(extra, ssid.SSID, ssid.SSID_len); - - dwrq->length = ssid.SSID_len; - - dwrq->flags = 1; /* active */ - - return 0; -} - -static int -wl_iw_set_nick( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_iw_t *iw = IW_DEV_IF(dev); - WL_TRACE(("%s: SIOCSIWNICKN\n", dev->name)); - - if (!extra) - return -EINVAL; - - /* Check the size of the string */ - if (dwrq->length > sizeof(iw->nickname)) - return -E2BIG; - - memcpy(iw->nickname, extra, dwrq->length); - iw->nickname[dwrq->length - 1] = '\0'; - - return 0; -} - -static int -wl_iw_get_nick( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_iw_t *iw = IW_DEV_IF(dev); - WL_TRACE(("%s: SIOCGIWNICKN\n", dev->name)); - - if (!extra) - return -EINVAL; - - strcpy(extra, iw->nickname); - dwrq->length = strlen(extra) + 1; - - return 0; -} - -static int wl_iw_set_rate( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - wl_rateset_t rateset; - int error, rate, i, error_bg, error_a; - - WL_TRACE(("%s: SIOCSIWRATE\n", dev->name)); - - /* Get current rateset */ - if ((error = dev_wlc_ioctl(dev, WLC_GET_CURR_RATESET, &rateset, sizeof(rateset)))) - return error; - - rateset.count = dtoh32(rateset.count); - - if (vwrq->value < 0) { - /* Select maximum rate */ - rate = rateset.rates[rateset.count - 1] & 0x7f; - } else if (vwrq->value < rateset.count) { - /* Select rate by rateset index */ - rate = rateset.rates[vwrq->value] & 0x7f; - } else { - /* Specified rate in bps */ - rate = vwrq->value / 500000; - } - - if (vwrq->fixed) { - /* - Set rate override, - Since the is a/b/g-blind, both a/bg_rate are enforced. - */ - error_bg = dev_wlc_intvar_set(dev, "bg_rate", rate); - error_a = dev_wlc_intvar_set(dev, "a_rate", rate); - - if (error_bg && error_a) - return (error_bg | error_a); - } else { - /* - clear rate override - Since the is a/b/g-blind, both a/bg_rate are enforced. - */ - /* 0 is for clearing rate override */ - error_bg = dev_wlc_intvar_set(dev, "bg_rate", 0); - /* 0 is for clearing rate override */ - error_a = dev_wlc_intvar_set(dev, "a_rate", 0); - - if (error_bg && error_a) - return (error_bg | error_a); - - /* Remove rates above selected rate */ - for (i = 0; i < rateset.count; i++) - if ((rateset.rates[i] & 0x7f) > rate) - break; - rateset.count = htod32(i); - - /* Set current rateset */ - if ((error = dev_wlc_ioctl(dev, WLC_SET_RATESET, &rateset, sizeof(rateset)))) - return error; - } - - return 0; -} - -static int wl_iw_get_rate( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, rate; - - WL_TRACE(("%s: SIOCGIWRATE\n", dev->name)); - - /* Report the current tx rate */ - if ((error = dev_wlc_ioctl(dev, WLC_GET_RATE, &rate, sizeof(rate)))) - return error; - rate = dtoh32(rate); - vwrq->value = rate * 500000; - - return 0; -} - -static int -wl_iw_set_rts( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, rts; - - WL_TRACE(("%s: SIOCSIWRTS\n", dev->name)); - - if (vwrq->disabled) - rts = DOT11_DEFAULT_RTS_LEN; - else if (vwrq->value < 0 || vwrq->value > DOT11_DEFAULT_RTS_LEN) - return -EINVAL; - else - rts = vwrq->value; - - if ((error = dev_wlc_intvar_set(dev, "rtsthresh", rts))) - return error; - - return 0; -} - -static int -wl_iw_get_rts( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, rts; - - WL_TRACE(("%s: SIOCGIWRTS\n", dev->name)); - - if ((error = dev_wlc_intvar_get(dev, "rtsthresh", &rts))) - return error; - - vwrq->value = rts; - vwrq->disabled = (rts >= DOT11_DEFAULT_RTS_LEN); - vwrq->fixed = 1; - - return 0; -} - -static int -wl_iw_set_frag( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, frag; - - WL_TRACE(("%s: SIOCSIWFRAG\n", dev->name)); - - if (vwrq->disabled) - frag = DOT11_DEFAULT_FRAG_LEN; - else if (vwrq->value < 0 || vwrq->value > DOT11_DEFAULT_FRAG_LEN) - return -EINVAL; - else - frag = vwrq->value; - - if ((error = dev_wlc_intvar_set(dev, "fragthresh", frag))) - return error; - - return 0; -} - -static int -wl_iw_get_frag( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, fragthreshold; - - WL_TRACE(("%s: SIOCGIWFRAG\n", dev->name)); - - if ((error = dev_wlc_intvar_get(dev, "fragthresh", &fragthreshold))) - return error; - - vwrq->value = fragthreshold; - vwrq->disabled = (fragthreshold >= DOT11_DEFAULT_FRAG_LEN); - vwrq->fixed = 1; - - return 0; -} - -static int -wl_iw_set_txpow( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, disable; - uint16 txpwrmw; - WL_TRACE(("%s: SIOCSIWTXPOW\n", dev->name)); - - /* Make sure radio is off or on as far as software is concerned */ - disable = vwrq->disabled ? WL_RADIO_SW_DISABLE : 0; - disable += WL_RADIO_SW_DISABLE << 16; - - disable = htod32(disable); - if ((error = dev_wlc_ioctl(dev, WLC_SET_RADIO, &disable, sizeof(disable)))) - return error; - - /* If Radio is off, nothing more to do */ - if (disable & WL_RADIO_SW_DISABLE) - return 0; - - /* Only handle mW */ - if (!(vwrq->flags & IW_TXPOW_MWATT)) - return -EINVAL; - - /* Value < 0 means just "on" or "off" */ - if (vwrq->value < 0) - return 0; - - if (vwrq->value > 0xffff) txpwrmw = 0xffff; - else txpwrmw = (uint16)vwrq->value; - - - error = dev_wlc_intvar_set(dev, "qtxpower", (int)(bcm_mw_to_qdbm(txpwrmw))); - return error; -} - -static int -wl_iw_get_txpow( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, disable, txpwrdbm; - uint8 result; - - WL_TRACE(("%s: SIOCGIWTXPOW\n", dev->name)); - - if ((error = dev_wlc_ioctl(dev, WLC_GET_RADIO, &disable, sizeof(disable))) || - (error = dev_wlc_intvar_get(dev, "qtxpower", &txpwrdbm))) - return error; - - disable = dtoh32(disable); - result = (uint8)(txpwrdbm & ~WL_TXPWR_OVERRIDE); - vwrq->value = (int32)bcm_qdbm_to_mw(result); - vwrq->fixed = 0; - vwrq->disabled = (disable & (WL_RADIO_SW_DISABLE | WL_RADIO_HW_DISABLE)) ? 1 : 0; - vwrq->flags = IW_TXPOW_MWATT; - - return 0; -} - -#if WIRELESS_EXT > 10 -static int -wl_iw_set_retry( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, lrl, srl; - - WL_TRACE(("%s: SIOCSIWRETRY\n", dev->name)); - - /* Do not handle "off" or "lifetime" */ - if (vwrq->disabled || (vwrq->flags & IW_RETRY_LIFETIME)) - return -EINVAL; - - /* Handle "[min|max] limit" */ - if (vwrq->flags & IW_RETRY_LIMIT) { - /* "max limit" or just "limit" */ -#if WIRELESS_EXT > 20 - if ((vwrq->flags & IW_RETRY_LONG) ||(vwrq->flags & IW_RETRY_MAX) || - !((vwrq->flags & IW_RETRY_SHORT) || (vwrq->flags & IW_RETRY_MIN))) { -#else - if ((vwrq->flags & IW_RETRY_MAX) || !(vwrq->flags & IW_RETRY_MIN)) { -#endif /* WIRELESS_EXT > 20 */ - - lrl = htod32(vwrq->value); - if ((error = dev_wlc_ioctl(dev, WLC_SET_LRL, &lrl, sizeof(lrl)))) - return error; - } - /* "min limit" or just "limit" */ -#if WIRELESS_EXT > 20 - if ((vwrq->flags & IW_RETRY_SHORT) ||(vwrq->flags & IW_RETRY_MIN) || - !((vwrq->flags & IW_RETRY_LONG) || (vwrq->flags & IW_RETRY_MAX))) { -#else - if ((vwrq->flags & IW_RETRY_MIN) || !(vwrq->flags & IW_RETRY_MAX)) { -#endif /* WIRELESS_EXT > 20 */ - - srl = htod32(vwrq->value); - if ((error = dev_wlc_ioctl(dev, WLC_SET_SRL, &srl, sizeof(srl)))) - return error; - } - } - - return 0; -} - -static int -wl_iw_get_retry( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, lrl, srl; - - WL_TRACE(("%s: SIOCGIWRETRY\n", dev->name)); - - vwrq->disabled = 0; /* Can't be disabled */ - - /* Do not handle lifetime queries */ - if ((vwrq->flags & IW_RETRY_TYPE) == IW_RETRY_LIFETIME) - return -EINVAL; - - /* Get retry limits */ - if ((error = dev_wlc_ioctl(dev, WLC_GET_LRL, &lrl, sizeof(lrl))) || - (error = dev_wlc_ioctl(dev, WLC_GET_SRL, &srl, sizeof(srl)))) - return error; - - lrl = dtoh32(lrl); - srl = dtoh32(srl); - - /* Note : by default, display the min retry number */ - if (vwrq->flags & IW_RETRY_MAX) { - vwrq->flags = IW_RETRY_LIMIT | IW_RETRY_MAX; - vwrq->value = lrl; - } else { - vwrq->flags = IW_RETRY_LIMIT; - vwrq->value = srl; - if (srl != lrl) - vwrq->flags |= IW_RETRY_MIN; - } - - return 0; -} -#endif /* WIRELESS_EXT > 10 */ - -static int -wl_iw_set_encode( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_wsec_key_t key; - int error, val, wsec; - - WL_TRACE(("%s: SIOCSIWENCODE\n", dev->name)); - - memset(&key, 0, sizeof(key)); - - if ((dwrq->flags & IW_ENCODE_INDEX) == 0) { - /* Find the current key */ - for (key.index = 0; key.index < DOT11_MAX_DEFAULT_KEYS; key.index++) { - val = htod32(key.index); - if ((error = dev_wlc_ioctl(dev, WLC_GET_KEY_PRIMARY, &val, sizeof(val)))) - return error; - val = dtoh32(val); - if (val) - break; - } - /* Default to 0 */ - if (key.index == DOT11_MAX_DEFAULT_KEYS) - key.index = 0; - } else { - key.index = (dwrq->flags & IW_ENCODE_INDEX) - 1; - if (key.index >= DOT11_MAX_DEFAULT_KEYS) - return -EINVAL; - } - - /* Interpret "off" to mean no encryption */ - wsec = (dwrq->flags & IW_ENCODE_DISABLED) ? 0 : WEP_ENABLED; - - if ((error = dev_wlc_intvar_set(dev, "wsec", wsec))) - return error; - - /* Old API used to pass a NULL pointer instead of IW_ENCODE_NOKEY */ - if (!extra || !dwrq->length || (dwrq->flags & IW_ENCODE_NOKEY)) { - /* Just select a new current key */ - val = htod32(key.index); - if ((error = dev_wlc_ioctl(dev, WLC_SET_KEY_PRIMARY, &val, sizeof(val)))) - return error; - } else { - key.len = dwrq->length; - - if (dwrq->length > sizeof(key.data)) - return -EINVAL; - - memcpy(key.data, extra, dwrq->length); - - key.flags = WL_PRIMARY_KEY; - switch (key.len) { - case WEP1_KEY_SIZE: - key.algo = CRYPTO_ALGO_WEP1; - break; - case WEP128_KEY_SIZE: - key.algo = CRYPTO_ALGO_WEP128; - break; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 14) - case TKIP_KEY_SIZE: - key.algo = CRYPTO_ALGO_TKIP; - break; -#endif - case AES_KEY_SIZE: - key.algo = CRYPTO_ALGO_AES_CCM; - break; - default: - return -EINVAL; - } - - /* Set the new key/index */ - swap_key_from_BE(&key); - if ((error = dev_wlc_ioctl(dev, WLC_SET_KEY, &key, sizeof(key)))) - return error; - } - - /* Interpret "restricted" to mean shared key authentication */ - val = (dwrq->flags & IW_ENCODE_RESTRICTED) ? 1 : 0; - val = htod32(val); - if ((error = dev_wlc_ioctl(dev, WLC_SET_AUTH, &val, sizeof(val)))) - return error; - - return 0; -} - -static int -wl_iw_get_encode( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_wsec_key_t key; - int error, val, wsec, auth; - - WL_TRACE(("%s: SIOCGIWENCODE\n", dev->name)); - - /* assure default values of zero for things we don't touch */ - bzero(&key, sizeof(wl_wsec_key_t)); - - if ((dwrq->flags & IW_ENCODE_INDEX) == 0) { - /* Find the current key */ - for (key.index = 0; key.index < DOT11_MAX_DEFAULT_KEYS; key.index++) { - val = key.index; - if ((error = dev_wlc_ioctl(dev, WLC_GET_KEY_PRIMARY, &val, sizeof(val)))) - return error; - val = dtoh32(val); - if (val) - break; - } - } else - key.index = (dwrq->flags & IW_ENCODE_INDEX) - 1; - - if (key.index >= DOT11_MAX_DEFAULT_KEYS) - key.index = 0; - - /* Get info */ - - if ((error = dev_wlc_ioctl(dev, WLC_GET_WSEC, &wsec, sizeof(wsec))) || - (error = dev_wlc_ioctl(dev, WLC_GET_AUTH, &auth, sizeof(auth)))) - return error; - - swap_key_to_BE(&key); - - wsec = dtoh32(wsec); - auth = dtoh32(auth); - /* Get key length */ - dwrq->length = MIN(IW_ENCODING_TOKEN_MAX, key.len); - - /* Get flags */ - dwrq->flags = key.index + 1; - if (!(wsec & (WEP_ENABLED | TKIP_ENABLED | AES_ENABLED))) { - /* Interpret "off" to mean no encryption */ - dwrq->flags |= IW_ENCODE_DISABLED; - } - if (auth) { - /* Interpret "restricted" to mean shared key authentication */ - dwrq->flags |= IW_ENCODE_RESTRICTED; - } - - /* Get key */ - if (dwrq->length && extra) - memcpy(extra, key.data, dwrq->length); - - return 0; -} - -static int -wl_iw_set_power( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, pm; - - WL_TRACE(("%s: SIOCSIWPOWER\n", dev->name)); - - pm = vwrq->disabled ? PM_OFF : PM_MAX; - - pm = htod32(pm); - if ((error = dev_wlc_ioctl(dev, WLC_SET_PM, &pm, sizeof(pm)))) - return error; - - return 0; -} - -static int -wl_iw_get_power( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error, pm; - - WL_TRACE(("%s: SIOCGIWPOWER\n", dev->name)); - - if ((error = dev_wlc_ioctl(dev, WLC_GET_PM, &pm, sizeof(pm)))) - return error; - - pm = dtoh32(pm); - vwrq->disabled = pm ? 0 : 1; - vwrq->flags = IW_POWER_ALL_R; - - return 0; -} - -#if WIRELESS_EXT > 17 -static int -wl_iw_set_wpaie( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *iwp, - char *extra -) -{ - dev_wlc_bufvar_set(dev, "wpaie", extra, iwp->length); - - return 0; -} - -static int -wl_iw_get_wpaie( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *iwp, - char *extra -) -{ - WL_TRACE(("%s: SIOCGIWGENIE\n", dev->name)); - iwp->length = 64; - dev_wlc_bufvar_get(dev, "wpaie", extra, iwp->length); - return 0; -} - -static int -wl_iw_set_encodeext( - struct net_device *dev, - struct iw_request_info *info, - struct iw_point *dwrq, - char *extra -) -{ - wl_wsec_key_t key; - int error; - struct iw_encode_ext *iwe; - - WL_TRACE(("%s: SIOCSIWENCODEEXT\n", dev->name)); - - memset(&key, 0, sizeof(key)); - iwe = (struct iw_encode_ext *)extra; - - /* disable encryption completely */ - if (dwrq->flags & IW_ENCODE_DISABLED) { - - } - - /* get the key index */ - key.index = 0; - if (dwrq->flags & IW_ENCODE_INDEX) - key.index = (dwrq->flags & IW_ENCODE_INDEX) - 1; - - key.len = iwe->key_len; - - /* Instead of bcast for ea address for default wep keys, driver needs it to be Null */ - if (!ETHER_ISMULTI(iwe->addr.sa_data)) - bcopy((void *)&iwe->addr.sa_data, (char *)&key.ea, ETHER_ADDR_LEN); - - /* check for key index change */ - if (key.len == 0) { - if (iwe->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) { - WL_WSEC(("Changing the the primary Key to %d\n", key.index)); - /* change the key index .... */ - key.index = htod32(key.index); - error = dev_wlc_ioctl(dev, WLC_SET_KEY_PRIMARY, - &key.index, sizeof(key.index)); - if (error) - return error; - } - /* key delete */ - else { - swap_key_from_BE(&key); - error = dev_wlc_ioctl(dev, WLC_SET_KEY, &key, sizeof(key)); - if (error) - return error; - } - } - /* This case is used to allow an external 802.1x supplicant - * to pass the PMK to the in-driver supplicant for use in - * the 4-way handshake. - */ - else if (iwe->alg == IW_ENCODE_ALG_PMK) { - int j; - wsec_pmk_t pmk; - char keystring[WSEC_MAX_PSK_LEN + 1]; - char* charptr = keystring; - uint len; - - /* copy the raw hex key to the appropriate format */ - for (j = 0; j < (WSEC_MAX_PSK_LEN / 2); j++) { - sprintf(charptr, "%02x", iwe->key[j]); - charptr += 2; - } - len = strlen(keystring); - pmk.key_len = htod16(len); - bcopy(keystring, pmk.key, len); - pmk.flags = htod16(WSEC_PASSPHRASE); - - error = dev_wlc_ioctl(dev, WLC_SET_WSEC_PMK, &pmk, sizeof(pmk)); - if (error) - return error; - } - - else { - if (iwe->key_len > sizeof(key.data)) - return -EINVAL; - - WL_WSEC(("Setting the key index %d\n", key.index)); - if (iwe->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) { - WL_WSEC(("key is a Primary Key\n")); - key.flags = WL_PRIMARY_KEY; - } - - bcopy((void *)iwe->key, key.data, iwe->key_len); - - if (iwe->alg == IW_ENCODE_ALG_TKIP) { - uint8 keybuf[8]; - bcopy(&key.data[24], keybuf, sizeof(keybuf)); - bcopy(&key.data[16], &key.data[24], sizeof(keybuf)); - bcopy(keybuf, &key.data[16], sizeof(keybuf)); - } - - /* rx iv */ - if (iwe->ext_flags & IW_ENCODE_EXT_RX_SEQ_VALID) { - uchar *ivptr; - ivptr = (uchar *)iwe->rx_seq; - key.rxiv.hi = (ivptr[5] << 24) | (ivptr[4] << 16) | - (ivptr[3] << 8) | ivptr[2]; - key.rxiv.lo = (ivptr[1] << 8) | ivptr[0]; - key.iv_initialized = TRUE; - } - - switch (iwe->alg) { - case IW_ENCODE_ALG_NONE: - key.algo = CRYPTO_ALGO_OFF; - break; - case IW_ENCODE_ALG_WEP: - if (iwe->key_len == WEP1_KEY_SIZE) - key.algo = CRYPTO_ALGO_WEP1; - else - key.algo = CRYPTO_ALGO_WEP128; - break; - case IW_ENCODE_ALG_TKIP: - key.algo = CRYPTO_ALGO_TKIP; - break; - case IW_ENCODE_ALG_CCMP: - key.algo = CRYPTO_ALGO_AES_CCM; - break; - default: - break; - } - swap_key_from_BE(&key); - - dhd_wait_pend8021x(dev); - - error = dev_wlc_ioctl(dev, WLC_SET_KEY, &key, sizeof(key)); - if (error) - return error; - } - return 0; -} - - -#if WIRELESS_EXT > 17 -struct { - pmkid_list_t pmkids; - pmkid_t foo[MAXPMKID-1]; -} pmkid_list; -static int -wl_iw_set_pmksa( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - struct iw_pmksa *iwpmksa; - uint i; - char eabuf[ETHER_ADDR_STR_LEN]; - pmkid_t * pmkid_array = pmkid_list.pmkids.pmkid; - - WL_TRACE(("%s: SIOCSIWPMKSA\n", dev->name)); - iwpmksa = (struct iw_pmksa *)extra; - bzero((char *)eabuf, ETHER_ADDR_STR_LEN); - if (iwpmksa->cmd == IW_PMKSA_FLUSH) { - WL_TRACE(("wl_iw_set_pmksa - IW_PMKSA_FLUSH\n")); - bzero((char *)&pmkid_list, sizeof(pmkid_list)); - } - if (iwpmksa->cmd == IW_PMKSA_REMOVE) { - pmkid_list_t pmkid, *pmkidptr; - pmkidptr = &pmkid; - bcopy(&iwpmksa->bssid.sa_data[0], &pmkidptr->pmkid[0].BSSID, ETHER_ADDR_LEN); - bcopy(&iwpmksa->pmkid[0], &pmkidptr->pmkid[0].PMKID, WPA2_PMKID_LEN); - { - uint j; - WL_TRACE(("wl_iw_set_pmksa,IW_PMKSA_REMOVE - PMKID: %s = ", - bcm_ether_ntoa(&pmkidptr->pmkid[0].BSSID, - eabuf))); - for (j = 0; j < WPA2_PMKID_LEN; j++) - WL_TRACE(("%02x ", pmkidptr->pmkid[0].PMKID[j])); - WL_TRACE(("\n")); - } - for (i = 0; i < pmkid_list.pmkids.npmkid; i++) - if (!bcmp(&iwpmksa->bssid.sa_data[0], &pmkid_array[i].BSSID, - ETHER_ADDR_LEN)) - break; - for (; i < pmkid_list.pmkids.npmkid; i++) { - bcopy(&pmkid_array[i+1].BSSID, - &pmkid_array[i].BSSID, - ETHER_ADDR_LEN); - bcopy(&pmkid_array[i+1].PMKID, - &pmkid_array[i].PMKID, - WPA2_PMKID_LEN); - } - pmkid_list.pmkids.npmkid--; - } - if (iwpmksa->cmd == IW_PMKSA_ADD) { - bcopy(&iwpmksa->bssid.sa_data[0], - &pmkid_array[pmkid_list.pmkids.npmkid].BSSID, - ETHER_ADDR_LEN); - bcopy(&iwpmksa->pmkid[0], &pmkid_array[pmkid_list.pmkids.npmkid].PMKID, - WPA2_PMKID_LEN); - { - uint j; - uint k; - k = pmkid_list.pmkids.npmkid; - BCM_REFERENCE(k); - WL_TRACE(("wl_iw_set_pmksa,IW_PMKSA_ADD - PMKID: %s = ", - bcm_ether_ntoa(&pmkid_array[k].BSSID, - eabuf))); - for (j = 0; j < WPA2_PMKID_LEN; j++) - WL_TRACE(("%02x ", pmkid_array[k].PMKID[j])); - WL_TRACE(("\n")); - } - pmkid_list.pmkids.npmkid++; - } - WL_TRACE(("PRINTING pmkid LIST - No of elements %d\n", pmkid_list.pmkids.npmkid)); - for (i = 0; i < pmkid_list.pmkids.npmkid; i++) { - uint j; - WL_TRACE(("PMKID[%d]: %s = ", i, - bcm_ether_ntoa(&pmkid_array[i].BSSID, - eabuf))); - for (j = 0; j < WPA2_PMKID_LEN; j++) - WL_TRACE(("%02x ", pmkid_array[i].PMKID[j])); - printf("\n"); - } - WL_TRACE(("\n")); - dev_wlc_bufvar_set(dev, "pmkid_info", (char *)&pmkid_list, sizeof(pmkid_list)); - return 0; -} -#endif /* WIRELESS_EXT > 17 */ - -static int -wl_iw_get_encodeext( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - WL_TRACE(("%s: SIOCGIWENCODEEXT\n", dev->name)); - return 0; -} - -static int -wl_iw_set_wpaauth( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error = 0; - int paramid; - int paramval; - uint32 cipher_combined; - int val = 0; - wl_iw_t *iw = IW_DEV_IF(dev); - - WL_TRACE(("%s: SIOCSIWAUTH\n", dev->name)); - - paramid = vwrq->flags & IW_AUTH_INDEX; - paramval = vwrq->value; - - WL_TRACE(("%s: SIOCSIWAUTH, paramid = 0x%0x, paramval = 0x%0x\n", - dev->name, paramid, paramval)); - - switch (paramid) { - - case IW_AUTH_WPA_VERSION: - /* supported wpa version disabled or wpa or wpa2 */ - if (paramval & IW_AUTH_WPA_VERSION_DISABLED) - val = WPA_AUTH_DISABLED; - else if (paramval & (IW_AUTH_WPA_VERSION_WPA)) - val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED; - else if (paramval & IW_AUTH_WPA_VERSION_WPA2) - val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; - WL_TRACE(("%s: %d: setting wpa_auth to 0x%0x\n", __FUNCTION__, __LINE__, val)); - if ((error = dev_wlc_intvar_set(dev, "wpa_auth", val))) - return error; - break; - - case IW_AUTH_CIPHER_PAIRWISE: - case IW_AUTH_CIPHER_GROUP: { - int fbt_cap = 0; - - if (paramid == IW_AUTH_CIPHER_PAIRWISE) { - iw->pwsec = paramval; - } - else { - iw->gwsec = paramval; - } - - if ((error = dev_wlc_intvar_get(dev, "wsec", &val))) - return error; - - cipher_combined = iw->gwsec | iw->pwsec; - val &= ~(WEP_ENABLED | TKIP_ENABLED | AES_ENABLED); - if (cipher_combined & (IW_AUTH_CIPHER_WEP40 | IW_AUTH_CIPHER_WEP104)) - val |= WEP_ENABLED; - if (cipher_combined & IW_AUTH_CIPHER_TKIP) - val |= TKIP_ENABLED; - if (cipher_combined & IW_AUTH_CIPHER_CCMP) - val |= AES_ENABLED; - - if (iw->privacy_invoked && !val) { - WL_WSEC(("%s: %s: 'Privacy invoked' TRUE but clearing wsec, assuming " - "we're a WPS enrollee\n", dev->name, __FUNCTION__)); - if ((error = dev_wlc_intvar_set(dev, "is_WPS_enrollee", TRUE))) { - WL_WSEC(("Failed to set iovar is_WPS_enrollee\n")); - return error; - } - } else if (val) { - if ((error = dev_wlc_intvar_set(dev, "is_WPS_enrollee", FALSE))) { - WL_WSEC(("Failed to clear iovar is_WPS_enrollee\n")); - return error; - } - } - - if ((error = dev_wlc_intvar_set(dev, "wsec", val))) - return error; - - /* Ensure in-dongle supplicant is turned on when FBT wants to do the 4-way - * handshake. - */ - if (dev_wlc_intvar_get(dev, "fbt_cap", &fbt_cap) == 0) { - if (fbt_cap == WLC_FBT_CAP_DRV_4WAY_AND_REASSOC) { - if ((paramid == IW_AUTH_CIPHER_PAIRWISE) && (val & AES_ENABLED)) { - if ((error = dev_wlc_intvar_set(dev, "sup_wpa", 1))) - return error; - } - else if (val == 0) { - if ((error = dev_wlc_intvar_set(dev, "sup_wpa", 0))) - return error; - } - } - } - break; - } - - case IW_AUTH_KEY_MGMT: - if ((error = dev_wlc_intvar_get(dev, "wpa_auth", &val))) - return error; - - if (val & (WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED)) { - if (paramval & (IW_AUTH_KEY_MGMT_FT_PSK | IW_AUTH_KEY_MGMT_PSK)) - val = WPA_AUTH_PSK; - else - val = WPA_AUTH_UNSPECIFIED; - if (paramval & (IW_AUTH_KEY_MGMT_FT_802_1X | IW_AUTH_KEY_MGMT_FT_PSK)) - val |= WPA2_AUTH_FT; - } - else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) { - if (paramval & (IW_AUTH_KEY_MGMT_FT_PSK | IW_AUTH_KEY_MGMT_PSK)) - val = WPA2_AUTH_PSK; - else - val = WPA2_AUTH_UNSPECIFIED; - if (paramval & (IW_AUTH_KEY_MGMT_FT_802_1X | IW_AUTH_KEY_MGMT_FT_PSK)) - val |= WPA2_AUTH_FT; - } - WL_TRACE(("%s: %d: setting wpa_auth to %d\n", __FUNCTION__, __LINE__, val)); - if ((error = dev_wlc_intvar_set(dev, "wpa_auth", val))) - return error; - break; - - case IW_AUTH_TKIP_COUNTERMEASURES: - dev_wlc_bufvar_set(dev, "tkip_countermeasures", (char *)¶mval, 1); - break; - - case IW_AUTH_80211_AUTH_ALG: - /* open shared */ - WL_ERROR(("Setting the D11auth %d\n", paramval)); - if (paramval & IW_AUTH_ALG_OPEN_SYSTEM) - val = 0; - else if (paramval & IW_AUTH_ALG_SHARED_KEY) - val = 1; - else - error = 1; - if (!error && (error = dev_wlc_intvar_set(dev, "auth", val))) - return error; - break; - - case IW_AUTH_WPA_ENABLED: - if (paramval == 0) { - val = 0; - WL_TRACE(("%s: %d: setting wpa_auth to %d\n", __FUNCTION__, __LINE__, val)); - error = dev_wlc_intvar_set(dev, "wpa_auth", val); - return error; - } - else { - /* If WPA is enabled, wpa_auth is set elsewhere */ - } - break; - - case IW_AUTH_DROP_UNENCRYPTED: - dev_wlc_bufvar_set(dev, "wsec_restrict", (char *)¶mval, 1); - break; - - case IW_AUTH_RX_UNENCRYPTED_EAPOL: - dev_wlc_bufvar_set(dev, "rx_unencrypted_eapol", (char *)¶mval, 1); - break; - -#if WIRELESS_EXT > 17 - - case IW_AUTH_ROAMING_CONTROL: - WL_TRACE(("%s: IW_AUTH_ROAMING_CONTROL\n", __FUNCTION__)); - /* driver control or user space app control */ - break; - - case IW_AUTH_PRIVACY_INVOKED: { - int wsec; - - if (paramval == 0) { - iw->privacy_invoked = FALSE; - if ((error = dev_wlc_intvar_set(dev, "is_WPS_enrollee", FALSE))) { - WL_WSEC(("Failed to clear iovar is_WPS_enrollee\n")); - return error; - } - } else { - iw->privacy_invoked = TRUE; - if ((error = dev_wlc_intvar_get(dev, "wsec", &wsec))) - return error; - - if (!WSEC_ENABLED(wsec)) { - /* if privacy is true, but wsec is false, we are a WPS enrollee */ - if ((error = dev_wlc_intvar_set(dev, "is_WPS_enrollee", TRUE))) { - WL_WSEC(("Failed to set iovar is_WPS_enrollee\n")); - return error; - } - } else { - if ((error = dev_wlc_intvar_set(dev, "is_WPS_enrollee", FALSE))) { - WL_WSEC(("Failed to clear iovar is_WPS_enrollee\n")); - return error; - } - } - } - break; - } - - -#endif /* WIRELESS_EXT > 17 */ - - - default: - break; - } - return 0; -} -#define VAL_PSK(_val) (((_val) & WPA_AUTH_PSK) || ((_val) & WPA2_AUTH_PSK)) - -static int -wl_iw_get_wpaauth( - struct net_device *dev, - struct iw_request_info *info, - struct iw_param *vwrq, - char *extra -) -{ - int error; - int paramid; - int paramval = 0; - int val; - wl_iw_t *iw = IW_DEV_IF(dev); - - WL_TRACE(("%s: SIOCGIWAUTH\n", dev->name)); - - paramid = vwrq->flags & IW_AUTH_INDEX; - - switch (paramid) { - case IW_AUTH_WPA_VERSION: - /* supported wpa version disabled or wpa or wpa2 */ - if ((error = dev_wlc_intvar_get(dev, "wpa_auth", &val))) - return error; - if (val & (WPA_AUTH_NONE | WPA_AUTH_DISABLED)) - paramval = IW_AUTH_WPA_VERSION_DISABLED; - else if (val & (WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED)) - paramval = IW_AUTH_WPA_VERSION_WPA; - else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) - paramval = IW_AUTH_WPA_VERSION_WPA2; - break; - - case IW_AUTH_CIPHER_PAIRWISE: - paramval = iw->pwsec; - break; - - case IW_AUTH_CIPHER_GROUP: - paramval = iw->gwsec; - break; - - case IW_AUTH_KEY_MGMT: - /* psk, 1x */ - if ((error = dev_wlc_intvar_get(dev, "wpa_auth", &val))) - return error; - if (VAL_PSK(val)) - paramval = IW_AUTH_KEY_MGMT_PSK; - else - paramval = IW_AUTH_KEY_MGMT_802_1X; - - break; - case IW_AUTH_TKIP_COUNTERMEASURES: - dev_wlc_bufvar_get(dev, "tkip_countermeasures", (char *)¶mval, 1); - break; - - case IW_AUTH_DROP_UNENCRYPTED: - dev_wlc_bufvar_get(dev, "wsec_restrict", (char *)¶mval, 1); - break; - - case IW_AUTH_RX_UNENCRYPTED_EAPOL: - dev_wlc_bufvar_get(dev, "rx_unencrypted_eapol", (char *)¶mval, 1); - break; - - case IW_AUTH_80211_AUTH_ALG: - /* open, shared, leap */ - if ((error = dev_wlc_intvar_get(dev, "auth", &val))) - return error; - if (!val) - paramval = IW_AUTH_ALG_OPEN_SYSTEM; - else - paramval = IW_AUTH_ALG_SHARED_KEY; - break; - case IW_AUTH_WPA_ENABLED: - if ((error = dev_wlc_intvar_get(dev, "wpa_auth", &val))) - return error; - if (val) - paramval = TRUE; - else - paramval = FALSE; - break; - -#if WIRELESS_EXT > 17 - - case IW_AUTH_ROAMING_CONTROL: - WL_ERROR(("%s: IW_AUTH_ROAMING_CONTROL\n", __FUNCTION__)); - /* driver control or user space app control */ - break; - - case IW_AUTH_PRIVACY_INVOKED: - paramval = iw->privacy_invoked; - break; - -#endif /* WIRELESS_EXT > 17 */ - } - vwrq->value = paramval; - return 0; -} -#endif /* WIRELESS_EXT > 17 */ - -static const iw_handler wl_iw_handler[] = -{ - (iw_handler) wl_iw_config_commit, /* SIOCSIWCOMMIT */ - (iw_handler) wl_iw_get_name, /* SIOCGIWNAME */ - (iw_handler) NULL, /* SIOCSIWNWID */ - (iw_handler) NULL, /* SIOCGIWNWID */ - (iw_handler) wl_iw_set_freq, /* SIOCSIWFREQ */ - (iw_handler) wl_iw_get_freq, /* SIOCGIWFREQ */ - (iw_handler) wl_iw_set_mode, /* SIOCSIWMODE */ - (iw_handler) wl_iw_get_mode, /* SIOCGIWMODE */ - (iw_handler) NULL, /* SIOCSIWSENS */ - (iw_handler) NULL, /* SIOCGIWSENS */ - (iw_handler) NULL, /* SIOCSIWRANGE */ - (iw_handler) wl_iw_get_range, /* SIOCGIWRANGE */ - (iw_handler) NULL, /* SIOCSIWPRIV */ - (iw_handler) NULL, /* SIOCGIWPRIV */ - (iw_handler) NULL, /* SIOCSIWSTATS */ - (iw_handler) NULL, /* SIOCGIWSTATS */ - (iw_handler) wl_iw_set_spy, /* SIOCSIWSPY */ - (iw_handler) wl_iw_get_spy, /* SIOCGIWSPY */ - (iw_handler) NULL, /* -- hole -- */ - (iw_handler) NULL, /* -- hole -- */ - (iw_handler) wl_iw_set_wap, /* SIOCSIWAP */ - (iw_handler) wl_iw_get_wap, /* SIOCGIWAP */ -#if WIRELESS_EXT > 17 - (iw_handler) wl_iw_mlme, /* SIOCSIWMLME */ -#else - (iw_handler) NULL, /* -- hole -- */ -#endif - (iw_handler) wl_iw_iscan_get_aplist, /* SIOCGIWAPLIST */ -#if WIRELESS_EXT > 13 - (iw_handler) wl_iw_iscan_set_scan, /* SIOCSIWSCAN */ - (iw_handler) wl_iw_iscan_get_scan, /* SIOCGIWSCAN */ -#else /* WIRELESS_EXT > 13 */ - (iw_handler) NULL, /* SIOCSIWSCAN */ - (iw_handler) NULL, /* SIOCGIWSCAN */ -#endif /* WIRELESS_EXT > 13 */ - (iw_handler) wl_iw_set_essid, /* SIOCSIWESSID */ - (iw_handler) wl_iw_get_essid, /* SIOCGIWESSID */ - (iw_handler) wl_iw_set_nick, /* SIOCSIWNICKN */ - (iw_handler) wl_iw_get_nick, /* SIOCGIWNICKN */ - (iw_handler) NULL, /* -- hole -- */ - (iw_handler) NULL, /* -- hole -- */ - (iw_handler) wl_iw_set_rate, /* SIOCSIWRATE */ - (iw_handler) wl_iw_get_rate, /* SIOCGIWRATE */ - (iw_handler) wl_iw_set_rts, /* SIOCSIWRTS */ - (iw_handler) wl_iw_get_rts, /* SIOCGIWRTS */ - (iw_handler) wl_iw_set_frag, /* SIOCSIWFRAG */ - (iw_handler) wl_iw_get_frag, /* SIOCGIWFRAG */ - (iw_handler) wl_iw_set_txpow, /* SIOCSIWTXPOW */ - (iw_handler) wl_iw_get_txpow, /* SIOCGIWTXPOW */ -#if WIRELESS_EXT > 10 - (iw_handler) wl_iw_set_retry, /* SIOCSIWRETRY */ - (iw_handler) wl_iw_get_retry, /* SIOCGIWRETRY */ -#endif /* WIRELESS_EXT > 10 */ - (iw_handler) wl_iw_set_encode, /* SIOCSIWENCODE */ - (iw_handler) wl_iw_get_encode, /* SIOCGIWENCODE */ - (iw_handler) wl_iw_set_power, /* SIOCSIWPOWER */ - (iw_handler) wl_iw_get_power, /* SIOCGIWPOWER */ -#if WIRELESS_EXT > 17 - (iw_handler) NULL, /* -- hole -- */ - (iw_handler) NULL, /* -- hole -- */ - (iw_handler) wl_iw_set_wpaie, /* SIOCSIWGENIE */ - (iw_handler) wl_iw_get_wpaie, /* SIOCGIWGENIE */ - (iw_handler) wl_iw_set_wpaauth, /* SIOCSIWAUTH */ - (iw_handler) wl_iw_get_wpaauth, /* SIOCGIWAUTH */ - (iw_handler) wl_iw_set_encodeext, /* SIOCSIWENCODEEXT */ - (iw_handler) wl_iw_get_encodeext, /* SIOCGIWENCODEEXT */ - (iw_handler) wl_iw_set_pmksa, /* SIOCSIWPMKSA */ -#endif /* WIRELESS_EXT > 17 */ -}; - -#if WIRELESS_EXT > 12 -enum { - WL_IW_SET_LEDDC = SIOCIWFIRSTPRIV, - WL_IW_SET_VLANMODE, - WL_IW_SET_PM, -#if WIRELESS_EXT > 17 -#endif /* WIRELESS_EXT > 17 */ - WL_IW_SET_LAST -}; - -static iw_handler wl_iw_priv_handler[] = { - wl_iw_set_leddc, - wl_iw_set_vlanmode, - wl_iw_set_pm, -#if WIRELESS_EXT > 17 -#endif /* WIRELESS_EXT > 17 */ - NULL -}; - -static struct iw_priv_args wl_iw_priv_args[] = { - { - WL_IW_SET_LEDDC, - IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, - 0, - "set_leddc" - }, - { - WL_IW_SET_VLANMODE, - IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, - 0, - "set_vlanmode" - }, - { - WL_IW_SET_PM, - IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, - 0, - "set_pm" - }, -#if WIRELESS_EXT > 17 -#endif /* WIRELESS_EXT > 17 */ - { 0, 0, 0, { 0 } } -}; - -const struct iw_handler_def wl_iw_handler_def = -{ - .num_standard = ARRAYSIZE(wl_iw_handler), - .num_private = ARRAY_SIZE(wl_iw_priv_handler), - .num_private_args = ARRAY_SIZE(wl_iw_priv_args), - .standard = (iw_handler *) wl_iw_handler, - .private = wl_iw_priv_handler, - .private_args = wl_iw_priv_args, -#if WIRELESS_EXT >= 19 - get_wireless_stats: dhd_get_wireless_stats, -#endif /* WIRELESS_EXT >= 19 */ - }; -#endif /* WIRELESS_EXT > 12 */ - -int -wl_iw_ioctl( - struct net_device *dev, - struct ifreq *rq, - int cmd -) -{ - struct iwreq *wrq = (struct iwreq *) rq; - struct iw_request_info info; - iw_handler handler; - char *extra = NULL; - size_t token_size = 1; - int max_tokens = 0, ret = 0; - - if (cmd < SIOCIWFIRST || - IW_IOCTL_IDX(cmd) >= ARRAYSIZE(wl_iw_handler) || - !(handler = wl_iw_handler[IW_IOCTL_IDX(cmd)])) - return -EOPNOTSUPP; - - switch (cmd) { - - case SIOCSIWESSID: - case SIOCGIWESSID: - case SIOCSIWNICKN: - case SIOCGIWNICKN: - max_tokens = IW_ESSID_MAX_SIZE + 1; - break; - - case SIOCSIWENCODE: - case SIOCGIWENCODE: -#if WIRELESS_EXT > 17 - case SIOCSIWENCODEEXT: - case SIOCGIWENCODEEXT: -#endif - max_tokens = IW_ENCODING_TOKEN_MAX; - break; - - case SIOCGIWRANGE: - max_tokens = sizeof(struct iw_range); - break; - - case SIOCGIWAPLIST: - token_size = sizeof(struct sockaddr) + sizeof(struct iw_quality); - max_tokens = IW_MAX_AP; - break; - -#if WIRELESS_EXT > 13 - case SIOCGIWSCAN: - if (g_iscan) - max_tokens = wrq->u.data.length; - else - max_tokens = IW_SCAN_MAX_DATA; - break; -#endif /* WIRELESS_EXT > 13 */ - - case SIOCSIWSPY: - token_size = sizeof(struct sockaddr); - max_tokens = IW_MAX_SPY; - break; - - case SIOCGIWSPY: - token_size = sizeof(struct sockaddr) + sizeof(struct iw_quality); - max_tokens = IW_MAX_SPY; - break; - default: - break; - } - - if (max_tokens && wrq->u.data.pointer) { - if (wrq->u.data.length > max_tokens) - return -E2BIG; - - if (!(extra = kmalloc(max_tokens * token_size, GFP_KERNEL))) - return -ENOMEM; - - if (copy_from_user(extra, wrq->u.data.pointer, wrq->u.data.length * token_size)) { - kfree(extra); - return -EFAULT; - } - } - - info.cmd = cmd; - info.flags = 0; - - ret = handler(dev, &info, &wrq->u, extra); - - if (extra) { - if (copy_to_user(wrq->u.data.pointer, extra, wrq->u.data.length * token_size)) { - kfree(extra); - return -EFAULT; - } - - kfree(extra); - } - - return ret; -} - -/* Convert a connection status event into a connection status string. - * Returns TRUE if a matching connection status string was found. - */ -bool -wl_iw_conn_status_str(uint32 event_type, uint32 status, uint32 reason, - char* stringBuf, uint buflen) -{ - typedef struct conn_fail_event_map_t { - uint32 inEvent; /* input: event type to match */ - uint32 inStatus; /* input: event status code to match */ - uint32 inReason; /* input: event reason code to match */ - const char* outName; /* output: failure type */ - const char* outCause; /* output: failure cause */ - } conn_fail_event_map_t; - - /* Map of WLC_E events to connection failure strings */ -# define WL_IW_DONT_CARE 9999 - const conn_fail_event_map_t event_map [] = { - /* inEvent inStatus inReason */ - /* outName outCause */ - {WLC_E_SET_SSID, WLC_E_STATUS_SUCCESS, WL_IW_DONT_CARE, - "Conn", "Success"}, - {WLC_E_SET_SSID, WLC_E_STATUS_NO_NETWORKS, WL_IW_DONT_CARE, - "Conn", "NoNetworks"}, - {WLC_E_SET_SSID, WLC_E_STATUS_FAIL, WL_IW_DONT_CARE, - "Conn", "ConfigMismatch"}, - {WLC_E_PRUNE, WL_IW_DONT_CARE, WLC_E_PRUNE_ENCR_MISMATCH, - "Conn", "EncrypMismatch"}, - {WLC_E_PRUNE, WL_IW_DONT_CARE, WLC_E_RSN_MISMATCH, - "Conn", "RsnMismatch"}, - {WLC_E_AUTH, WLC_E_STATUS_TIMEOUT, WL_IW_DONT_CARE, - "Conn", "AuthTimeout"}, - {WLC_E_AUTH, WLC_E_STATUS_FAIL, WL_IW_DONT_CARE, - "Conn", "AuthFail"}, - {WLC_E_AUTH, WLC_E_STATUS_NO_ACK, WL_IW_DONT_CARE, - "Conn", "AuthNoAck"}, - {WLC_E_REASSOC, WLC_E_STATUS_FAIL, WL_IW_DONT_CARE, - "Conn", "ReassocFail"}, - {WLC_E_REASSOC, WLC_E_STATUS_TIMEOUT, WL_IW_DONT_CARE, - "Conn", "ReassocTimeout"}, - {WLC_E_REASSOC, WLC_E_STATUS_ABORT, WL_IW_DONT_CARE, - "Conn", "ReassocAbort"}, - {WLC_E_PSK_SUP, WLC_SUP_KEYED, WL_IW_DONT_CARE, - "Sup", "ConnSuccess"}, - {WLC_E_PSK_SUP, WL_IW_DONT_CARE, WL_IW_DONT_CARE, - "Sup", "WpaHandshakeFail"}, - {WLC_E_DEAUTH_IND, WL_IW_DONT_CARE, WL_IW_DONT_CARE, - "Conn", "Deauth"}, - {WLC_E_DISASSOC_IND, WL_IW_DONT_CARE, WL_IW_DONT_CARE, - "Conn", "DisassocInd"}, - {WLC_E_DISASSOC, WL_IW_DONT_CARE, WL_IW_DONT_CARE, - "Conn", "Disassoc"} - }; - - const char* name = ""; - const char* cause = NULL; - int i; - - /* Search the event map table for a matching event */ - for (i = 0; i < sizeof(event_map)/sizeof(event_map[0]); i++) { - const conn_fail_event_map_t* row = &event_map[i]; - if (row->inEvent == event_type && - (row->inStatus == status || row->inStatus == WL_IW_DONT_CARE) && - (row->inReason == reason || row->inReason == WL_IW_DONT_CARE)) { - name = row->outName; - cause = row->outCause; - break; - } - } - - /* If found, generate a connection failure string and return TRUE */ - if (cause) { - memset(stringBuf, 0, buflen); - snprintf(stringBuf, buflen, "%s %s %02d %02d", - name, cause, status, reason); - WL_TRACE(("Connection status: %s\n", stringBuf)); - return TRUE; - } else { - return FALSE; - } -} - -#if (WIRELESS_EXT > 14) -/* Check if we have received an event that indicates connection failure - * If so, generate a connection failure report string. - * The caller supplies a buffer to hold the generated string. - */ -static bool -wl_iw_check_conn_fail(wl_event_msg_t *e, char* stringBuf, uint buflen) -{ - uint32 event = ntoh32(e->event_type); - uint32 status = ntoh32(e->status); - uint32 reason = ntoh32(e->reason); - - if (wl_iw_conn_status_str(event, status, reason, stringBuf, buflen)) { - return TRUE; - } else - { - return FALSE; - } -} -#endif /* WIRELESS_EXT > 14 */ - -#ifndef IW_CUSTOM_MAX -#define IW_CUSTOM_MAX 256 /* size of extra buffer used for translation of events */ -#endif /* IW_CUSTOM_MAX */ - -void -wl_iw_event(struct net_device *dev, wl_event_msg_t *e, void* data) -{ -#if WIRELESS_EXT > 13 - union iwreq_data wrqu; - char extra[IW_CUSTOM_MAX + 1]; - int cmd = 0; - uint32 event_type = ntoh32(e->event_type); - uint16 flags = ntoh16(e->flags); - uint32 datalen = ntoh32(e->datalen); - uint32 status = ntoh32(e->status); - - memset(&wrqu, 0, sizeof(wrqu)); - memset(extra, 0, sizeof(extra)); - - memcpy(wrqu.addr.sa_data, &e->addr, ETHER_ADDR_LEN); - wrqu.addr.sa_family = ARPHRD_ETHER; - - switch (event_type) { - case WLC_E_TXFAIL: - cmd = IWEVTXDROP; - break; -#if WIRELESS_EXT > 14 - case WLC_E_JOIN: - case WLC_E_ASSOC_IND: - case WLC_E_REASSOC_IND: - cmd = IWEVREGISTERED; - break; - case WLC_E_DEAUTH_IND: - case WLC_E_DISASSOC_IND: - cmd = SIOCGIWAP; - wrqu.data.length = strlen(extra); - bzero(wrqu.addr.sa_data, ETHER_ADDR_LEN); - bzero(&extra, ETHER_ADDR_LEN); - break; - - case WLC_E_LINK: - case WLC_E_NDIS_LINK: - cmd = SIOCGIWAP; - wrqu.data.length = strlen(extra); - if (!(flags & WLC_EVENT_MSG_LINK)) { - bzero(wrqu.addr.sa_data, ETHER_ADDR_LEN); - bzero(&extra, ETHER_ADDR_LEN); - } - break; - case WLC_E_ACTION_FRAME: - cmd = IWEVCUSTOM; - if (datalen + 1 <= sizeof(extra)) { - wrqu.data.length = datalen + 1; - extra[0] = WLC_E_ACTION_FRAME; - memcpy(&extra[1], data, datalen); - WL_TRACE(("WLC_E_ACTION_FRAME len %d \n", wrqu.data.length)); - } - break; - - case WLC_E_ACTION_FRAME_COMPLETE: - cmd = IWEVCUSTOM; - if (sizeof(status) + 1 <= sizeof(extra)) { - wrqu.data.length = sizeof(status) + 1; - extra[0] = WLC_E_ACTION_FRAME_COMPLETE; - memcpy(&extra[1], &status, sizeof(status)); - WL_TRACE(("wl_iw_event status %d \n", status)); - } - break; -#endif /* WIRELESS_EXT > 14 */ -#if WIRELESS_EXT > 17 - case WLC_E_MIC_ERROR: { - struct iw_michaelmicfailure *micerrevt = (struct iw_michaelmicfailure *)&extra; - cmd = IWEVMICHAELMICFAILURE; - wrqu.data.length = sizeof(struct iw_michaelmicfailure); - if (flags & WLC_EVENT_MSG_GROUP) - micerrevt->flags |= IW_MICFAILURE_GROUP; - else - micerrevt->flags |= IW_MICFAILURE_PAIRWISE; - memcpy(micerrevt->src_addr.sa_data, &e->addr, ETHER_ADDR_LEN); - micerrevt->src_addr.sa_family = ARPHRD_ETHER; - - break; - } - - case WLC_E_ASSOC_REQ_IE: - cmd = IWEVASSOCREQIE; - wrqu.data.length = datalen; - if (datalen < sizeof(extra)) - memcpy(extra, data, datalen); - break; - - case WLC_E_ASSOC_RESP_IE: - cmd = IWEVASSOCRESPIE; - wrqu.data.length = datalen; - if (datalen < sizeof(extra)) - memcpy(extra, data, datalen); - break; - - case WLC_E_PMKID_CACHE: { - struct iw_pmkid_cand *iwpmkidcand = (struct iw_pmkid_cand *)&extra; - pmkid_cand_list_t *pmkcandlist; - pmkid_cand_t *pmkidcand; - int count; - - if (data == NULL) - break; - - cmd = IWEVPMKIDCAND; - pmkcandlist = data; - count = ntoh32_ua((uint8 *)&pmkcandlist->npmkid_cand); - wrqu.data.length = sizeof(struct iw_pmkid_cand); - pmkidcand = pmkcandlist->pmkid_cand; - while (count) { - bzero(iwpmkidcand, sizeof(struct iw_pmkid_cand)); - if (pmkidcand->preauth) - iwpmkidcand->flags |= IW_PMKID_CAND_PREAUTH; - bcopy(&pmkidcand->BSSID, &iwpmkidcand->bssid.sa_data, - ETHER_ADDR_LEN); - wireless_send_event(dev, cmd, &wrqu, extra); - pmkidcand++; - count--; - } - break; - } -#endif /* WIRELESS_EXT > 17 */ - - case WLC_E_SCAN_COMPLETE: -#if WIRELESS_EXT > 14 - cmd = SIOCGIWSCAN; -#endif - WL_TRACE(("event WLC_E_SCAN_COMPLETE\n")); - if ((g_iscan) && (g_iscan->sysioc_pid >= 0) && - (g_iscan->iscan_state != ISCAN_STATE_IDLE)) - up(&g_iscan->sysioc_sem); - break; - - default: - /* Cannot translate event */ - break; - } - - if (cmd) { - if (cmd == SIOCGIWSCAN) - wireless_send_event(dev, cmd, &wrqu, NULL); - else - wireless_send_event(dev, cmd, &wrqu, extra); - } - -#if WIRELESS_EXT > 14 - /* Look for WLC events that indicate a connection failure. - * If found, generate an IWEVCUSTOM event. - */ - memset(extra, 0, sizeof(extra)); - if (wl_iw_check_conn_fail(e, extra, sizeof(extra))) { - cmd = IWEVCUSTOM; - wrqu.data.length = strlen(extra); - wireless_send_event(dev, cmd, &wrqu, extra); - } -#endif /* WIRELESS_EXT > 14 */ - -#endif /* WIRELESS_EXT > 13 */ -} - -int wl_iw_get_wireless_stats(struct net_device *dev, struct iw_statistics *wstats) -{ - int res = 0; - wl_cnt_t cnt; - int phy_noise; - int rssi; - scb_val_t scb_val; - - phy_noise = 0; - if ((res = dev_wlc_ioctl(dev, WLC_GET_PHY_NOISE, &phy_noise, sizeof(phy_noise)))) - goto done; - - phy_noise = dtoh32(phy_noise); - WL_TRACE(("wl_iw_get_wireless_stats phy noise=%d\n *****", phy_noise)); - - scb_val.val = 0; - if ((res = dev_wlc_ioctl(dev, WLC_GET_RSSI, &scb_val, sizeof(scb_val_t)))) - goto done; - - rssi = dtoh32(scb_val.val); - WL_TRACE(("wl_iw_get_wireless_stats rssi=%d ****** \n", rssi)); - if (rssi <= WL_IW_RSSI_NO_SIGNAL) - wstats->qual.qual = 0; - else if (rssi <= WL_IW_RSSI_VERY_LOW) - wstats->qual.qual = 1; - else if (rssi <= WL_IW_RSSI_LOW) - wstats->qual.qual = 2; - else if (rssi <= WL_IW_RSSI_GOOD) - wstats->qual.qual = 3; - else if (rssi <= WL_IW_RSSI_VERY_GOOD) - wstats->qual.qual = 4; - else - wstats->qual.qual = 5; - - /* Wraps to 0 if RSSI is 0 */ - wstats->qual.level = 0x100 + rssi; - wstats->qual.noise = 0x100 + phy_noise; -#if WIRELESS_EXT > 18 - wstats->qual.updated |= (IW_QUAL_ALL_UPDATED | IW_QUAL_DBM); -#else - wstats->qual.updated |= 7; -#endif /* WIRELESS_EXT > 18 */ - -#if WIRELESS_EXT > 11 - WL_TRACE(("wl_iw_get_wireless_stats counters=%d\n *****", (int)sizeof(wl_cnt_t))); - - memset(&cnt, 0, sizeof(wl_cnt_t)); - res = dev_wlc_bufvar_get(dev, "counters", (char *)&cnt, sizeof(wl_cnt_t)); - if (res) - { - WL_ERROR(("wl_iw_get_wireless_stats counters failed error=%d ****** \n", res)); - goto done; - } - - cnt.version = dtoh16(cnt.version); - if (cnt.version != WL_CNT_T_VERSION) { - WL_TRACE(("\tIncorrect version of counters struct: expected %d; got %d\n", - WL_CNT_T_VERSION, cnt.version)); - goto done; - } - - wstats->discard.nwid = 0; - wstats->discard.code = dtoh32(cnt.rxundec); - wstats->discard.fragment = dtoh32(cnt.rxfragerr); - wstats->discard.retries = dtoh32(cnt.txfail); - wstats->discard.misc = dtoh32(cnt.rxrunt) + dtoh32(cnt.rxgiant); - wstats->miss.beacon = 0; - - WL_TRACE(("wl_iw_get_wireless_stats counters txframe=%d txbyte=%d\n", - dtoh32(cnt.txframe), dtoh32(cnt.txbyte))); - WL_TRACE(("wl_iw_get_wireless_stats counters rxfrmtoolong=%d\n", dtoh32(cnt.rxfrmtoolong))); - WL_TRACE(("wl_iw_get_wireless_stats counters rxbadplcp=%d\n", dtoh32(cnt.rxbadplcp))); - WL_TRACE(("wl_iw_get_wireless_stats counters rxundec=%d\n", dtoh32(cnt.rxundec))); - WL_TRACE(("wl_iw_get_wireless_stats counters rxfragerr=%d\n", dtoh32(cnt.rxfragerr))); - WL_TRACE(("wl_iw_get_wireless_stats counters txfail=%d\n", dtoh32(cnt.txfail))); - WL_TRACE(("wl_iw_get_wireless_stats counters rxrunt=%d\n", dtoh32(cnt.rxrunt))); - WL_TRACE(("wl_iw_get_wireless_stats counters rxgiant=%d\n", dtoh32(cnt.rxgiant))); - -#endif /* WIRELESS_EXT > 11 */ - -done: - return res; -} - -static void -wl_iw_timerfunc(ulong data) -{ - iscan_info_t *iscan = (iscan_info_t *)data; - iscan->timer_on = 0; - if (iscan->iscan_state != ISCAN_STATE_IDLE) { - WL_TRACE(("timer trigger\n")); - up(&iscan->sysioc_sem); - } -} - -static void -wl_iw_set_event_mask(struct net_device *dev) -{ - char eventmask[WL_EVENTING_MASK_LEN]; - char iovbuf[WL_EVENTING_MASK_LEN + 12]; /* Room for "event_msgs" + '\0' + bitvec */ - - dev_iw_iovar_getbuf(dev, "event_msgs", "", 0, iovbuf, sizeof(iovbuf)); - bcopy(iovbuf, eventmask, WL_EVENTING_MASK_LEN); - setbit(eventmask, WLC_E_SCAN_COMPLETE); - dev_iw_iovar_setbuf(dev, "event_msgs", eventmask, WL_EVENTING_MASK_LEN, - iovbuf, sizeof(iovbuf)); - -} - -static int -wl_iw_iscan_prep(wl_scan_params_t *params, wlc_ssid_t *ssid) -{ - int err = 0; - - memcpy(¶ms->bssid, ðer_bcast, ETHER_ADDR_LEN); - params->bss_type = DOT11_BSSTYPE_ANY; - params->scan_type = 0; - params->nprobes = -1; - params->active_time = -1; - params->passive_time = -1; - params->home_time = -1; - params->channel_num = 0; - - params->nprobes = htod32(params->nprobes); - params->active_time = htod32(params->active_time); - params->passive_time = htod32(params->passive_time); - params->home_time = htod32(params->home_time); - if (ssid && ssid->SSID_len) - memcpy(¶ms->ssid, ssid, sizeof(wlc_ssid_t)); - - return err; -} - -static int -wl_iw_iscan(iscan_info_t *iscan, wlc_ssid_t *ssid, uint16 action) -{ - int params_size = (WL_SCAN_PARAMS_FIXED_SIZE + OFFSETOF(wl_iscan_params_t, params)); - wl_iscan_params_t *params; - int err = 0; - - if (ssid && ssid->SSID_len) { - params_size += sizeof(wlc_ssid_t); - } - params = (wl_iscan_params_t*)kmalloc(params_size, GFP_KERNEL); - if (params == NULL) { - return -ENOMEM; - } - memset(params, 0, params_size); - ASSERT(params_size < WLC_IOCTL_SMLEN); - - err = wl_iw_iscan_prep(¶ms->params, ssid); - - if (!err) { - params->version = htod32(ISCAN_REQ_VERSION); - params->action = htod16(action); - params->scan_duration = htod16(0); - - /* params_size += OFFSETOF(wl_iscan_params_t, params); */ - (void) dev_iw_iovar_setbuf(iscan->dev, "iscan", params, params_size, - iscan->ioctlbuf, WLC_IOCTL_SMLEN); - } - - kfree(params); - return err; -} - -static uint32 -wl_iw_iscan_get(iscan_info_t *iscan) -{ - iscan_buf_t * buf; - iscan_buf_t * ptr; - wl_iscan_results_t * list_buf; - wl_iscan_results_t list; - wl_scan_results_t *results; - uint32 status; - - /* buffers are allocated on demand */ - if (iscan->list_cur) { - buf = iscan->list_cur; - iscan->list_cur = buf->next; - } - else { - buf = kmalloc(sizeof(iscan_buf_t), GFP_KERNEL); - if (!buf) - return WL_SCAN_RESULTS_ABORTED; - buf->next = NULL; - if (!iscan->list_hdr) - iscan->list_hdr = buf; - else { - ptr = iscan->list_hdr; - while (ptr->next) { - ptr = ptr->next; - } - ptr->next = buf; - } - } - memset(buf->iscan_buf, 0, WLC_IW_ISCAN_MAXLEN); - list_buf = (wl_iscan_results_t*)buf->iscan_buf; - results = &list_buf->results; - results->buflen = WL_ISCAN_RESULTS_FIXED_SIZE; - results->version = 0; - results->count = 0; - - memset(&list, 0, sizeof(list)); - list.results.buflen = htod32(WLC_IW_ISCAN_MAXLEN); - (void) dev_iw_iovar_getbuf( - iscan->dev, - "iscanresults", - &list, - WL_ISCAN_RESULTS_FIXED_SIZE, - buf->iscan_buf, - WLC_IW_ISCAN_MAXLEN); - results->buflen = dtoh32(results->buflen); - results->version = dtoh32(results->version); - results->count = dtoh32(results->count); - WL_TRACE(("results->count = %d\n", results->count)); - - WL_TRACE(("results->buflen = %d\n", results->buflen)); - status = dtoh32(list_buf->status); - return status; -} - -static void wl_iw_send_scan_complete(iscan_info_t *iscan) -{ - union iwreq_data wrqu; - - memset(&wrqu, 0, sizeof(wrqu)); - - /* wext expects to get no data for SIOCGIWSCAN Event */ - wireless_send_event(iscan->dev, SIOCGIWSCAN, &wrqu, NULL); -} - -static int -_iscan_sysioc_thread(void *data) -{ - uint32 status; - iscan_info_t *iscan = (iscan_info_t *)data; - - DAEMONIZE("iscan_sysioc"); - - status = WL_SCAN_RESULTS_PARTIAL; - while (down_interruptible(&iscan->sysioc_sem) == 0) { - if (iscan->timer_on) { - del_timer(&iscan->timer); - iscan->timer_on = 0; - } - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) - rtnl_lock(); -#endif - status = wl_iw_iscan_get(iscan); -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) - rtnl_unlock(); -#endif - - switch (status) { - case WL_SCAN_RESULTS_PARTIAL: - WL_TRACE(("iscanresults incomplete\n")); -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) - rtnl_lock(); -#endif - /* make sure our buffer size is enough before going next round */ - wl_iw_iscan(iscan, NULL, WL_SCAN_ACTION_CONTINUE); -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) - rtnl_unlock(); -#endif - /* Reschedule the timer */ - iscan->timer.expires = jiffies + msecs_to_jiffies(iscan->timer_ms); - add_timer(&iscan->timer); - iscan->timer_on = 1; - break; - case WL_SCAN_RESULTS_SUCCESS: - WL_TRACE(("iscanresults complete\n")); - iscan->iscan_state = ISCAN_STATE_IDLE; - wl_iw_send_scan_complete(iscan); - break; - case WL_SCAN_RESULTS_PENDING: - WL_TRACE(("iscanresults pending\n")); - /* Reschedule the timer */ - iscan->timer.expires = jiffies + msecs_to_jiffies(iscan->timer_ms); - add_timer(&iscan->timer); - iscan->timer_on = 1; - break; - case WL_SCAN_RESULTS_ABORTED: - WL_TRACE(("iscanresults aborted\n")); - iscan->iscan_state = ISCAN_STATE_IDLE; - wl_iw_send_scan_complete(iscan); - break; - default: - WL_TRACE(("iscanresults returned unknown status %d\n", status)); - break; - } - } - complete_and_exit(&iscan->sysioc_exited, 0); -} - -int -wl_iw_attach(struct net_device *dev, void * dhdp) -{ - iscan_info_t *iscan = NULL; - - if (!dev) - return 0; - - iscan = kmalloc(sizeof(iscan_info_t), GFP_KERNEL); - if (!iscan) - return -ENOMEM; - memset(iscan, 0, sizeof(iscan_info_t)); - iscan->sysioc_pid = -1; - /* we only care about main interface so save a global here */ - g_iscan = iscan; - iscan->dev = dev; - iscan->iscan_state = ISCAN_STATE_IDLE; - - - /* Set up the timer */ - iscan->timer_ms = 2000; - init_timer(&iscan->timer); - iscan->timer.data = (ulong)iscan; - iscan->timer.function = wl_iw_timerfunc; - - sema_init(&iscan->sysioc_sem, 0); - init_completion(&iscan->sysioc_exited); - iscan->sysioc_pid = kernel_thread(_iscan_sysioc_thread, iscan, 0); - if (iscan->sysioc_pid < 0) - return -ENOMEM; - return 0; -} - -void wl_iw_detach(void) -{ - iscan_buf_t *buf; - iscan_info_t *iscan = g_iscan; - if (!iscan) - return; - if (iscan->sysioc_pid >= 0) { - KILL_PROC(iscan->sysioc_pid, SIGTERM); - wait_for_completion(&iscan->sysioc_exited); - } - - while (iscan->list_hdr) { - buf = iscan->list_hdr->next; - kfree(iscan->list_hdr); - iscan->list_hdr = buf; - } - kfree(iscan); - g_iscan = NULL; -} - -#endif /* USE_IW */ diff --git a/drivers/net/wireless/bcmdhd/wl_iw.h b/drivers/net/wireless/bcmdhd/wl_iw.h deleted file mode 100644 index df59a65c162..00000000000 --- a/drivers/net/wireless/bcmdhd/wl_iw.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Linux Wireless Extensions support - * - * Copyright (C) 1999-2013, Broadcom Corporation - * - * Unless you and Broadcom execute a separate written software license - * agreement governing use of this software, this software is licensed to you - * under the terms of the GNU General Public License version 2 (the "GPL"), - * available at http://www.broadcom.com/licenses/GPLv2.php, with the - * following added to such license: - * - * As a special exception, the copyright holders of this software give you - * permission to link this software with independent modules, and to copy and - * distribute the resulting executable under terms of your choice, provided that - * you also meet, for each linked independent module, the terms and conditions of - * the license of that module. An independent module is a module which is not - * derived from this software. The special exception does not apply to any - * modifications of the software. - * - * Notwithstanding the above, under no circumstances may you combine this - * software in any way with any other Broadcom software provided under a license - * other than the GPL, without Broadcom's express prior written consent. - * - * $Id: wl_iw.h 291086 2011-10-21 01:17:24Z $ - */ - -#ifndef _wl_iw_h_ -#define _wl_iw_h_ - -#include - -#include -#include -#include - -#define WL_SCAN_PARAMS_SSID_MAX 10 -#define GET_SSID "SSID=" -#define GET_CHANNEL "CH=" -#define GET_NPROBE "NPROBE=" -#define GET_ACTIVE_ASSOC_DWELL "ACTIVE=" -#define GET_PASSIVE_ASSOC_DWELL "PASSIVE=" -#define GET_HOME_DWELL "HOME=" -#define GET_SCAN_TYPE "TYPE=" - -#define BAND_GET_CMD "GETBAND" -#define BAND_SET_CMD "SETBAND" -#define DTIM_SKIP_GET_CMD "DTIMSKIPGET" -#define DTIM_SKIP_SET_CMD "DTIMSKIPSET" -#define SETSUSPEND_CMD "SETSUSPENDOPT" -#define PNOSSIDCLR_SET_CMD "PNOSSIDCLR" -/* Lin - Is the extra space needed? */ -#define PNOSETUP_SET_CMD "PNOSETUP " /* TLV command has extra end space */ -#define PNOENABLE_SET_CMD "PNOFORCE" -#define PNODEBUG_SET_CMD "PNODEBUG" -#define TXPOWER_SET_CMD "TXPOWER" - -#define MAC2STR(a) (a)[0], (a)[1], (a)[2], (a)[3], (a)[4], (a)[5] -#define MACSTR "%02x:%02x:%02x:%02x:%02x:%02x" - -/* Structure to keep global parameters */ -typedef struct wl_iw_extra_params { - int target_channel; /* target channel */ -} wl_iw_extra_params_t; - -struct cntry_locales_custom { - char iso_abbrev[WLC_CNTRY_BUF_SZ]; /* ISO 3166-1 country abbreviation */ - char custom_locale[WLC_CNTRY_BUF_SZ]; /* Custom firmware locale */ - int32 custom_locale_rev; /* Custom local revisin default -1 */ -}; -/* ============================================== */ -/* Defines from wlc_pub.h */ -#define WL_IW_RSSI_MINVAL -200 /* Low value, e.g. for forcing roam */ -#define WL_IW_RSSI_NO_SIGNAL -91 /* NDIS RSSI link quality cutoffs */ -#define WL_IW_RSSI_VERY_LOW -80 /* Very low quality cutoffs */ -#define WL_IW_RSSI_LOW -70 /* Low quality cutoffs */ -#define WL_IW_RSSI_GOOD -68 /* Good quality cutoffs */ -#define WL_IW_RSSI_VERY_GOOD -58 /* Very good quality cutoffs */ -#define WL_IW_RSSI_EXCELLENT -57 /* Excellent quality cutoffs */ -#define WL_IW_RSSI_INVALID 0 /* invalid RSSI value */ -#define MAX_WX_STRING 80 -#define SSID_FMT_BUF_LEN ((4 * 32) + 1) -#define isprint(c) bcm_isprint(c) -#define WL_IW_SET_ACTIVE_SCAN (SIOCIWFIRSTPRIV+1) -#define WL_IW_GET_RSSI (SIOCIWFIRSTPRIV+3) -#define WL_IW_SET_PASSIVE_SCAN (SIOCIWFIRSTPRIV+5) -#define WL_IW_GET_LINK_SPEED (SIOCIWFIRSTPRIV+7) -#define WL_IW_GET_CURR_MACADDR (SIOCIWFIRSTPRIV+9) -#define WL_IW_SET_STOP (SIOCIWFIRSTPRIV+11) -#define WL_IW_SET_START (SIOCIWFIRSTPRIV+13) - -#define G_SCAN_RESULTS 8*1024 -#define WE_ADD_EVENT_FIX 0x80 -#define G_WLAN_SET_ON 0 -#define G_WLAN_SET_OFF 1 - - -typedef struct wl_iw { - char nickname[IW_ESSID_MAX_SIZE]; - - struct iw_statistics wstats; - - int spy_num; - uint32 pwsec; /* pairwise wsec setting */ - uint32 gwsec; /* group wsec setting */ - bool privacy_invoked; /* IW_AUTH_PRIVACY_INVOKED setting */ - struct ether_addr spy_addr[IW_MAX_SPY]; - struct iw_quality spy_qual[IW_MAX_SPY]; - void *wlinfo; -} wl_iw_t; - -struct wl_ctrl { - struct timer_list *timer; - struct net_device *dev; - long sysioc_pid; - struct semaphore sysioc_sem; - struct completion sysioc_exited; -}; - - -#if WIRELESS_EXT > 12 -#include -extern const struct iw_handler_def wl_iw_handler_def; -#endif /* WIRELESS_EXT > 12 */ - -extern int wl_iw_ioctl(struct net_device *dev, struct ifreq *rq, int cmd); -extern void wl_iw_event(struct net_device *dev, wl_event_msg_t *e, void* data); -extern int wl_iw_get_wireless_stats(struct net_device *dev, struct iw_statistics *wstats); -int wl_iw_attach(struct net_device *dev, void * dhdp); -int wl_iw_send_priv_event(struct net_device *dev, char *flag); - -void wl_iw_detach(void); - -#define CSCAN_COMMAND "CSCAN " -#define CSCAN_TLV_PREFIX 'S' -#define CSCAN_TLV_VERSION 1 -#define CSCAN_TLV_SUBVERSION 0 -#define CSCAN_TLV_TYPE_SSID_IE 'S' -#define CSCAN_TLV_TYPE_CHANNEL_IE 'C' -#define CSCAN_TLV_TYPE_NPROBE_IE 'N' -#define CSCAN_TLV_TYPE_ACTIVE_IE 'A' -#define CSCAN_TLV_TYPE_PASSIVE_IE 'P' -#define CSCAN_TLV_TYPE_HOME_IE 'H' -#define CSCAN_TLV_TYPE_STYPE_IE 'T' - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27) -#define IWE_STREAM_ADD_EVENT(info, stream, ends, iwe, extra) \ - iwe_stream_add_event(info, stream, ends, iwe, extra) -#define IWE_STREAM_ADD_VALUE(info, event, value, ends, iwe, event_len) \ - iwe_stream_add_value(info, event, value, ends, iwe, event_len) -#define IWE_STREAM_ADD_POINT(info, stream, ends, iwe, extra) \ - iwe_stream_add_point(info, stream, ends, iwe, extra) -#else -#define IWE_STREAM_ADD_EVENT(info, stream, ends, iwe, extra) \ - iwe_stream_add_event(stream, ends, iwe, extra) -#define IWE_STREAM_ADD_VALUE(info, event, value, ends, iwe, event_len) \ - iwe_stream_add_value(event, value, ends, iwe, event_len) -#define IWE_STREAM_ADD_POINT(info, stream, ends, iwe, extra) \ - iwe_stream_add_point(stream, ends, iwe, extra) -#endif - -#endif /* _wl_iw_h_ */ diff --git a/drivers/platform/msm/qpnp-power-on.c b/drivers/platform/msm/qpnp-power-on.c index 67ce1b5f2a5..e8d2aef23f4 100644 --- a/drivers/platform/msm/qpnp-power-on.c +++ b/drivers/platform/msm/qpnp-power-on.c @@ -36,6 +36,7 @@ #define QPNP_PON_REASON1(base) (base + 0x8) #define QPNP_PON_WARM_RESET_REASON1(base) (base + 0xA) #define QPNP_PON_WARM_RESET_REASON2(base) (base + 0xB) +#define QPNP_POFF_REASON1(base) (base + 0xC) #define QPNP_PON_KPDPWR_S1_TIMER(base) (base + 0x40) #define QPNP_PON_KPDPWR_S2_TIMER(base) (base + 0x41) #define QPNP_PON_KPDPWR_S2_CNTL(base) (base + 0x42) @@ -122,6 +123,26 @@ static const char * const qpnp_pon_reason[] = { [7] = "Triggered from KPD (power key press)", }; +static const char * const qpnp_poff_reason[] = { + [0] = "Triggered from SOFT (Software)", + [1] = "Triggered from PS_HOLD (PS_HOLD/MSM controlled shutdown)", + [2] = "Triggered from PMIC_WD (PMIC watchdog)", + [3] = "Triggered from GP1 (Keypad_Reset1)", + [4] = "Triggered from GP2 (Keypad_Reset2)", + [5] = "Triggered from KPDPWR_AND_RESIN" + "(Simultaneous power key and reset line)", + [6] = "Triggered from RESIN_N (Reset line/Volume Down Key)", + [7] = "Triggered from KPDPWR_N (Long Power Key hold)", + [8] = "N/A", + [9] = "N/A", + [10] = "N/A", + [11] = "Triggered from CHARGER (Charger ENUM_TIMER, BOOT_DONE)", + [12] = "Triggered from TFT (Thermal Fault Tolerance)", + [13] = "Triggered from UVLO (Under Voltage Lock Out)", + [14] = "Triggered from OTST3 (Overtemp)", + [15] = "Triggered from STAGE3 (Stage 3 reset)", +}; + static int qpnp_pon_masked_write(struct qpnp_pon *pon, u16 addr, u8 mask, u8 val) { @@ -879,8 +900,9 @@ static int __devinit qpnp_pon_probe(struct spmi_device *spmi) struct device_node *itr = NULL; u32 delay = 0; int rc, sys_reset, index; - u8 pon_sts = 0; int disable = 0; + u8 pon_sts = 0, buf[2]; + u16 poff_sts = 0; pon = devm_kzalloc(&spmi->dev, sizeof(struct qpnp_pon), GFP_KERNEL); @@ -932,12 +954,31 @@ static int __devinit qpnp_pon_probe(struct spmi_device *spmi) dev_err(&pon->spmi->dev, "Unable to read PON_RESASON1 reg\n"); return rc; } + index = ffs(pon_sts); if ((index > PON_REASON_MAX) || (index < 0)) index = 0; pr_info("PMIC@SID%d Power-on reason: %s\n", pon->spmi->sid, index ? qpnp_pon_reason[index - 1] : "Unknown"); + /* POFF reason */ + rc = spmi_ext_register_readl(pon->spmi->ctrl, pon->spmi->sid, + QPNP_POFF_REASON1(pon->base), + buf, 2); + if (rc) { + dev_err(&pon->spmi->dev, "Unable to read POFF_RESASON regs\n"); + return rc; + } + poff_sts = buf[0] | (buf[1] << 8); + index = ffs(poff_sts) - 1; + if (index >= ARRAY_SIZE(qpnp_poff_reason) || index < 0) + pr_info("PMIC@SID%d: Unknown power-off reason\n", + pon->spmi->sid); + else + pr_info("PMIC@SID%d: Power-off reason: %s\n", + pon->spmi->sid, + qpnp_poff_reason[index]); + rc = of_property_read_u32(pon->spmi->dev.of_node, "qcom,pon-dbc-delay", &delay); if (rc) { diff --git a/drivers/power/max17048_battery.c b/drivers/power/max17048_battery.c index 643b74d3d97..d8a32775ece 100644 --- a/drivers/power/max17048_battery.c +++ b/drivers/power/max17048_battery.c @@ -681,7 +681,7 @@ static int qpnp_get_battery_current(int *current_ua) return ret; } - *current_ua = -i_result.result_ua; + *current_ua = i_result.result_ua; return 0; } diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index eacd46bb36b..3ec5b33b6f7 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -544,6 +544,9 @@ sg_write(struct file *filp, const char __user *buf, size_t count, loff_t * ppos) sg_io_hdr_t *hp; unsigned char cmnd[MAX_COMMAND_SIZE]; + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + return -EINVAL; + if ((!(sfp = (Sg_fd *) filp->private_data)) || (!(sdp = sfp->parentdp))) return -ENXIO; SCSI_LOG_TIMEOUT(3, printk("sg_write: %s, count=%d\n", @@ -741,8 +744,14 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp, return k; /* probably out of space --> ENOMEM */ } if (sdp->detached) { - if (srp->bio) + if (srp->bio) { + if (srp->rq->cmd != srp->rq->__cmd) + kfree(srp->rq->cmd); + blk_end_request_all(srp->rq, -EIO); + srp->rq = NULL; + } + sg_finish_rem_req(srp); return -ENODEV; } diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index f5c9d2f9b22..fcac889373a 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -32,9 +32,9 @@ struct spmii_boardinfo { static DEFINE_MUTEX(board_lock); static LIST_HEAD(board_list); static DEFINE_IDR(ctrl_idr); -static struct device_type spmi_ctrl_type = { 0 }; - -#define to_spmi(dev) platform_get_drvdata(to_platform_device(dev)) +static DEFINE_IDA(spmi_devid_ida); +static struct device_type spmi_dev_type; +static struct device_type spmi_ctrl_type; /* Forward declarations */ struct bus_type spmi_bus_type; @@ -188,22 +188,32 @@ int spmi_add_device(struct spmi_device *spmidev) { int rc; struct device *dev = get_valid_device(spmidev); + int id; if (!dev) { pr_err("%s: invalid SPMI device\n", __func__); return -EINVAL; } + id = ida_simple_get(&spmi_devid_ida, 0, 0, GFP_KERNEL); + if (id < 0) { + pr_err("No id available status = %d\n", id); + return id; + } + /* Set the device name */ - dev_set_name(dev, "%s-%p", spmidev->name, spmidev); + spmidev->id = id; + dev_set_name(dev, "%s-%d", spmidev->name, spmidev->id); /* Device may be bound to an active driver when this returns */ rc = device_add(dev); - if (rc < 0) + if (rc < 0) { + ida_simple_remove(&spmi_devid_ida, spmidev->id); dev_err(dev, "Can't add %s, status %d\n", dev_name(dev), rc); - else + } else { dev_dbg(dev, "device %s registered\n", dev_name(dev)); + } return rc; } @@ -251,6 +261,7 @@ EXPORT_SYMBOL_GPL(spmi_new_device); void spmi_remove_device(struct spmi_device *spmi_dev) { device_unregister(&spmi_dev->dev); + ida_simple_remove(&spmi_devid_ida, spmi_dev->id); } EXPORT_SYMBOL_GPL(spmi_remove_device); diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile index 8e18d4ea683..8769e325d49 100644 --- a/drivers/staging/android/Makefile +++ b/drivers/staging/android/Makefile @@ -1,3 +1,5 @@ +ccflags-y += -I$(src) # needed for trace events + obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o obj-$(CONFIG_ASHMEM) += ashmem.o obj-$(CONFIG_ANDROID_LOGGER) += logger.o diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c index 4f13907543e..740b5d215b5 100644 --- a/drivers/staging/android/ashmem.c +++ b/drivers/staging/android/ashmem.c @@ -785,11 +785,26 @@ static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return ret; } +static const struct file_operations ashmem_fops = { + .owner = THIS_MODULE, + .open = ashmem_open, + .release = ashmem_release, + .read = ashmem_read, + .llseek = ashmem_llseek, + .mmap = ashmem_mmap, + .unlocked_ioctl = ashmem_ioctl, + .compat_ioctl = ashmem_ioctl, +}; + +static struct miscdevice ashmem_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ashmem", + .fops = &ashmem_fops, +}; + static int is_ashmem_file(struct file *file) { - char fname[256], *name; - name = dentry_path(file->f_dentry, fname, 256); - return strcmp(name, "/ashmem") ? 0 : 1; + return (file->f_op == &ashmem_fops); } int get_ashmem_file(int fd, struct file **filp, struct file **vm_file, @@ -838,23 +853,6 @@ void put_ashmem_file(struct file *file) } EXPORT_SYMBOL(put_ashmem_file); -static const struct file_operations ashmem_fops = { - .owner = THIS_MODULE, - .open = ashmem_open, - .release = ashmem_release, - .read = ashmem_read, - .llseek = ashmem_llseek, - .mmap = ashmem_mmap, - .unlocked_ioctl = ashmem_ioctl, - .compat_ioctl = ashmem_ioctl, -}; - -static struct miscdevice ashmem_misc = { - .minor = MISC_DYNAMIC_MINOR, - .name = "ashmem", - .fops = &ashmem_fops, -}; - static int __init ashmem_init(void) { int ret; diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 2e4f44f6d81..2f95609f2af 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -37,8 +37,9 @@ #include #include "binder.h" +#include "binder_trace.h" -static DEFINE_MUTEX(binder_lock); +static DEFINE_MUTEX(binder_main_lock); static DEFINE_MUTEX(binder_deferred_lock); static DEFINE_MUTEX(binder_mmap_lock); @@ -501,6 +502,19 @@ static long task_close_fd(struct binder_proc *proc, unsigned int fd) return -EBADF; } +static inline void binder_lock(const char *tag) +{ + trace_binder_lock(tag); + mutex_lock(&binder_main_lock); + trace_binder_locked(tag); +} + +static inline void binder_unlock(const char *tag) +{ + trace_binder_unlock(tag); + mutex_unlock(&binder_main_lock); +} + static void binder_set_nice(long nice) { long min_nice; @@ -543,7 +557,7 @@ static void binder_insert_free_buffer(struct binder_proc *proc, binder_debug(BINDER_DEBUG_BUFFER_ALLOC, "binder: %d: add free buffer, size %zd, " - "at %p\n", proc->pid, new_buffer_size, new_buffer); + "at %pK\n", proc->pid, new_buffer_size, new_buffer); while (*p) { parent = *p; @@ -621,12 +635,14 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, struct mm_struct *mm; binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "binder: %d: %s pages %p-%p\n", proc->pid, + "binder: %d: %s pages %pK-%pK\n", proc->pid, allocate ? "allocate" : "free", start, end); if (end <= start) return 0; + trace_binder_update_page_range(proc, allocate, start, end); + if (vma) mm = NULL; else @@ -658,11 +674,11 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; BUG_ON(*page); - *page = alloc_page(GFP_KERNEL | __GFP_ZERO); + *page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); if (*page == NULL) { binder_debug(BINDER_DEBUG_TOP_ERRORS, "binder: %d: binder_alloc_buf failed " - "for page at %p\n", proc->pid, page_addr); + "for page at %pK\n", proc->pid, page_addr); goto err_alloc_page_failed; } tmp_area.addr = page_addr; @@ -672,7 +688,7 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, if (ret) { binder_debug(BINDER_DEBUG_TOP_ERRORS, "binder: %d: binder_alloc_buf failed " - "to map page at %p in kernel\n", + "to map page at %pK in kernel\n", proc->pid, page_addr); goto err_map_kernel_failed; } @@ -781,7 +797,7 @@ static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, binder_debug(BINDER_DEBUG_BUFFER_ALLOC, "binder: %d: binder_alloc_buf size %zd got buff" - "er %p size %zd\n", proc->pid, size, buffer, buffer_size); + "er %pK size %zd\n", proc->pid, size, buffer, buffer_size); has_page_addr = (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK); @@ -810,7 +826,7 @@ static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc, } binder_debug(BINDER_DEBUG_BUFFER_ALLOC, "binder: %d: binder_alloc_buf size %zd got " - "%p\n", proc->pid, size, buffer); + "%pK\n", proc->pid, size, buffer); buffer->data_size = data_size; buffer->offsets_size = offsets_size; buffer->async_transaction = is_async; @@ -850,8 +866,8 @@ static void binder_delete_free_buffer(struct binder_proc *proc, if (buffer_end_page(prev) == buffer_end_page(buffer)) free_page_end = 0; binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "binder: %d: merge free, buffer %p " - "share page with %p\n", proc->pid, buffer, prev); + "binder: %d: merge free, buffer %pK " + "share page with %pK\n", proc->pid, buffer, prev); } if (!list_is_last(&buffer->entry, &proc->buffers)) { @@ -864,15 +880,15 @@ static void binder_delete_free_buffer(struct binder_proc *proc, free_page_start = 0; binder_debug(BINDER_DEBUG_BUFFER_ALLOC, "binder: %d: merge free, buffer" - " %p share page with %p\n", proc->pid, + " %pK share page with %pK\n", proc->pid, buffer, prev); } } list_del(&buffer->entry); if (free_page_start || free_page_end) { binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "binder: %d: merge free, buffer %p do " - "not share page%s%s with with %p or %p\n", + "binder: %d: merge free, buffer %pK do " + "not share page%s%s with with %pK or %pK\n", proc->pid, buffer, free_page_start ? "" : " end", free_page_end ? "" : " start", prev, next); binder_update_page_range(proc, 0, free_page_start ? @@ -893,7 +909,7 @@ static void binder_free_buf(struct binder_proc *proc, ALIGN(buffer->offsets_size, sizeof(void *)); binder_debug(BINDER_DEBUG_BUFFER_ALLOC, - "binder: %d: binder_free_buf %p size %zd buffer" + "binder: %d: binder_free_buf %pK size %zd buffer" "_size %zd\n", proc->pid, buffer, size, buffer_size); BUG_ON(buffer->free); @@ -990,7 +1006,7 @@ static struct binder_node *binder_new_node(struct binder_proc *proc, INIT_LIST_HEAD(&node->work.entry); INIT_LIST_HEAD(&node->async_todo); binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "binder: %d:%d node %d u%p c%p created\n", + "binder: %d:%d node %d u%pK c%pK created\n", proc->pid, current->pid, node->debug_id, node->ptr, node->cookie); return node; @@ -1078,7 +1094,7 @@ static int binder_dec_node(struct binder_node *node, int strong, int internal) static struct binder_ref *binder_get_ref(struct binder_proc *proc, - uint32_t desc) + uint32_t desc, bool need_strong_ref) { struct rb_node *n = proc->refs_by_desc.rb_node; struct binder_ref *ref; @@ -1086,12 +1102,16 @@ static struct binder_ref *binder_get_ref(struct binder_proc *proc, while (n) { ref = rb_entry(n, struct binder_ref, rb_node_desc); - if (desc < ref->desc) + if (desc < ref->desc) { n = n->rb_left; - else if (desc > ref->desc) + } else if (desc > ref->desc) { n = n->rb_right; - else + } else if (need_strong_ref && !ref->strong) { + binder_user_error("tried to use weak ref as strong ref\n"); + return NULL; + } else { return ref; + } } return NULL; } @@ -1326,7 +1346,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, int debug_id = buffer->debug_id; binder_debug(BINDER_DEBUG_TRANSACTION, - "binder: %d buffer release %d, size %zd-%zd, failed at %p\n", + "binder: %d buffer release %d, size %zd-%zd, failed at %pK\n", proc->pid, buffer->debug_id, buffer->data_size, buffer->offsets_size, failed_at); @@ -1357,18 +1377,19 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, if (node == NULL) { binder_debug(BINDER_DEBUG_TOP_ERRORS, "binder: transaction release %d" - " bad node %p\n", debug_id, + " bad node %pK\n", debug_id, fp->binder); break; } binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%p\n", + " node %d u%pK\n", node->debug_id, node->ptr); binder_dec_node(node, fp->type == BINDER_TYPE_BINDER, 0); } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); + struct binder_ref *ref = binder_get_ref(proc, fp->handle, + fp->type == BINDER_TYPE_HANDLE); if (ref == NULL) { binder_debug(BINDER_DEBUG_TOP_ERRORS, "binder: transaction release %d" @@ -1468,7 +1489,7 @@ static void binder_transaction(struct binder_proc *proc, } else { if (tr->target.handle) { struct binder_ref *ref; - ref = binder_get_ref(proc, tr->target.handle); + ref = binder_get_ref(proc, tr->target.handle, true); if (ref == NULL) { binder_user_error("binder: %d:%d got " "transaction to invalid handle\n", @@ -1546,7 +1567,7 @@ static void binder_transaction(struct binder_proc *proc, if (reply) binder_debug(BINDER_DEBUG_TRANSACTION, "binder: %d:%d BC_REPLY %d -> %d:%d, " - "data %p-%p size %zd-%zd\n", + "data %pK-%pK size %zd-%zd\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_thread->pid, tr->data.ptr.buffer, tr->data.ptr.offsets, @@ -1554,7 +1575,7 @@ static void binder_transaction(struct binder_proc *proc, else binder_debug(BINDER_DEBUG_TRANSACTION, "binder: %d:%d BC_TRANSACTION %d -> " - "%d - node %d, data %p-%p size %zd-%zd\n", + "%d - node %d, data %pK-%pK size %zd-%zd\n", proc->pid, thread->pid, t->debug_id, target_proc->pid, target_node->debug_id, tr->data.ptr.buffer, tr->data.ptr.offsets, @@ -1570,6 +1591,9 @@ static void binder_transaction(struct binder_proc *proc, t->code = tr->code; t->flags = tr->flags; t->priority = task_nice(current); + + trace_binder_transaction(reply, t, target_node); + t->buffer = binder_alloc_buf(target_proc, tr->data_size, tr->offsets_size, !reply && (t->flags & TF_ONE_WAY)); if (t->buffer == NULL) { @@ -1580,6 +1604,7 @@ static void binder_transaction(struct binder_proc *proc, t->buffer->debug_id = t->debug_id; t->buffer->transaction = t; t->buffer->target_node = target_node; + trace_binder_transaction_alloc_buf(t->buffer); if (target_node) binder_inc_node(target_node, 1, 0, NULL); @@ -1632,8 +1657,8 @@ static void binder_transaction(struct binder_proc *proc, node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS); } if (fp->cookie != node->cookie) { - binder_user_error("binder: %d:%d sending u%p " - "node %d, cookie mismatch %p != %p\n", + binder_user_error("binder: %d:%d sending u%pK " + "node %d, cookie mismatch %pK != %pK\n", proc->pid, thread->pid, fp->binder, node->debug_id, fp->cookie, node->cookie); @@ -1652,18 +1677,22 @@ static void binder_transaction(struct binder_proc *proc, fp->type = BINDER_TYPE_HANDLE; else fp->type = BINDER_TYPE_WEAK_HANDLE; + fp->binder = 0; fp->handle = ref->desc; + fp->cookie = 0; binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, &thread->todo); + trace_binder_transaction_node_to_ref(t, node, ref); binder_debug(BINDER_DEBUG_TRANSACTION, - " node %d u%p -> ref %d desc %d\n", + " node %d u%pK -> ref %d desc %d\n", node->debug_id, node->ptr, ref->debug_id, ref->desc); } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); + struct binder_ref *ref = binder_get_ref(proc, fp->handle, + fp->type == BINDER_TYPE_HANDLE); if (ref == NULL) { binder_user_error("binder: %d:%d got " "transaction with invalid " @@ -1684,8 +1713,9 @@ static void binder_transaction(struct binder_proc *proc, fp->binder = ref->node->ptr; fp->cookie = ref->node->cookie; binder_inc_node(ref->node, fp->type == BINDER_TYPE_BINDER, 0, NULL); + trace_binder_transaction_ref_to_node(t, ref); binder_debug(BINDER_DEBUG_TRANSACTION, - " ref %d desc %d -> node %d u%p\n", + " ref %d desc %d -> node %d u%pK\n", ref->debug_id, ref->desc, ref->node->debug_id, ref->node->ptr); } else { @@ -1695,8 +1725,12 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + fp->binder = 0; fp->handle = new_ref->desc; + fp->cookie = 0; binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); + trace_binder_transaction_ref_to_ref(t, ref, + new_ref); binder_debug(BINDER_DEBUG_TRANSACTION, " ref %d desc %d -> ref %d desc %d (node %d)\n", ref->debug_id, ref->desc, new_ref->debug_id, @@ -1741,9 +1775,11 @@ static void binder_transaction(struct binder_proc *proc, goto err_get_unused_fd_failed; } task_fd_install(target_proc, target_fd, file); + trace_binder_transaction_fd(t, fp->handle, target_fd); binder_debug(BINDER_DEBUG_TRANSACTION, " fd %ld -> %d\n", fp->handle, target_fd); /* TODO: fput? */ + fp->binder = 0; fp->handle = target_fd; } break; @@ -1789,6 +1825,7 @@ static void binder_transaction(struct binder_proc *proc, err_bad_object_type: err_bad_offset: err_copy_data_failed: + trace_binder_transaction_failed_buffer_release(t->buffer); binder_transaction_buffer_release(target_proc, t->buffer, offp); t->buffer->transaction = NULL; binder_free_buf(target_proc, t->buffer); @@ -1834,6 +1871,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, if (get_user(cmd, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); + trace_binder_command(cmd); if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.bc)) { binder_stats.bc[_IOC_NR(cmd)]++; proc->stats.bc[_IOC_NR(cmd)]++; @@ -1864,7 +1902,9 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, ref->desc); } } else - ref = binder_get_ref(proc, target); + ref = binder_get_ref(proc, target, + cmd == BC_ACQUIRE || + cmd == BC_RELEASE); if (ref == NULL) { binder_user_error("binder: %d:%d refcou" "nt change on invalid ref %d\n", @@ -1911,7 +1951,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, node = binder_get_node(proc, node_ptr); if (node == NULL) { binder_user_error("binder: %d:%d " - "%s u%p no match\n", + "%s u%pK no match\n", proc->pid, thread->pid, cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : @@ -1920,8 +1960,8 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, break; } if (cookie != node->cookie) { - binder_user_error("binder: %d:%d %s u%p node %d" - " cookie mismatch %p != %p\n", + binder_user_error("binder: %d:%d %s u%pK node %d" + " cookie mismatch %pK != %pK\n", proc->pid, thread->pid, cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE", @@ -1978,19 +2018,19 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, buffer = binder_buffer_lookup(proc, data_ptr); if (buffer == NULL) { binder_user_error("binder: %d:%d " - "BC_FREE_BUFFER u%p no match\n", + "BC_FREE_BUFFER u%pK no match\n", proc->pid, thread->pid, data_ptr); break; } if (!buffer->allow_user_free) { binder_user_error("binder: %d:%d " - "BC_FREE_BUFFER u%p matched " + "BC_FREE_BUFFER u%pK matched " "unreturned buffer\n", proc->pid, thread->pid, data_ptr); break; } binder_debug(BINDER_DEBUG_FREE_BUFFER, - "binder: %d:%d BC_FREE_BUFFER u%p found buffer %d for %s transaction\n", + "binder: %d:%d BC_FREE_BUFFER u%pK found buffer %d for %s transaction\n", proc->pid, thread->pid, data_ptr, buffer->debug_id, buffer->transaction ? "active" : "finished"); @@ -2005,6 +2045,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, else list_move_tail(buffer->target_node->async_todo.next, &thread->todo); } + trace_binder_transaction_buffer_release(buffer); binder_transaction_buffer_release(proc, buffer, NULL); binder_free_buf(proc, buffer); break; @@ -2076,7 +2117,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, if (get_user(cookie, (void __user * __user *)ptr)) return -EFAULT; ptr += sizeof(void *); - ref = binder_get_ref(proc, target); + ref = binder_get_ref(proc, target, false); if (ref == NULL) { binder_user_error("binder: %d:%d %s " "invalid ref %d\n", @@ -2089,7 +2130,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "binder: %d:%d %s %p ref %d desc %d s %d w %d for node %d\n", + "binder: %d:%d %s %pK ref %d desc %d s %d w %d for node %d\n", proc->pid, thread->pid, cmd == BC_REQUEST_DEATH_NOTIFICATION ? "BC_REQUEST_DEATH_NOTIFICATION" : @@ -2143,7 +2184,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, "d BC_CLEAR_DEATH_NOTIFI" "CATION death notificat" "ion cookie mismatch " - "%p != %p\n", + "%pK != %pK\n", proc->pid, thread->pid, death->cookie, cookie); break; @@ -2179,11 +2220,11 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, } } binder_debug(BINDER_DEBUG_DEAD_BINDER, - "binder: %d:%d BC_DEAD_BINDER_DONE %p found %p\n", + "binder: %d:%d BC_DEAD_BINDER_DONE %pK found %pK\n", proc->pid, thread->pid, cookie, death); if (death == NULL) { binder_user_error("binder: %d:%d BC_DEAD" - "_BINDER_DONE %p not found\n", + "_BINDER_DONE %pK not found\n", proc->pid, thread->pid, cookie); break; } @@ -2214,6 +2255,7 @@ int binder_thread_write(struct binder_proc *proc, struct binder_thread *thread, void binder_stat_br(struct binder_proc *proc, struct binder_thread *thread, uint32_t cmd) { + trace_binder_return(cmd); if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.br)) { binder_stats.br[_IOC_NR(cmd)]++; proc->stats.br[_IOC_NR(cmd)]++; @@ -2260,6 +2302,7 @@ static int binder_thread_read(struct binder_proc *proc, if (put_user(thread->return_error2, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); + binder_stat_br(proc, thread, thread->return_error2); if (ptr == end) goto done; thread->return_error2 = BR_OK; @@ -2267,6 +2310,7 @@ static int binder_thread_read(struct binder_proc *proc, if (put_user(thread->return_error, (uint32_t __user *)ptr)) return -EFAULT; ptr += sizeof(uint32_t); + binder_stat_br(proc, thread, thread->return_error); thread->return_error = BR_OK; goto done; } @@ -2275,7 +2319,12 @@ static int binder_thread_read(struct binder_proc *proc, thread->looper |= BINDER_LOOPER_STATE_WAITING; if (wait_for_proc_work) proc->ready_threads++; - mutex_unlock(&binder_lock); + + binder_unlock(__func__); + + trace_binder_wait_for_work(wait_for_proc_work, + !!thread->transaction_stack, + !list_empty(&thread->todo)); if (wait_for_proc_work) { if (!(thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED))) { @@ -2299,7 +2348,9 @@ static int binder_thread_read(struct binder_proc *proc, } else ret = wait_event_freezable(thread->wait, binder_has_thread_work(thread)); } - mutex_lock(&binder_lock); + + binder_lock(__func__); + if (wait_for_proc_work) proc->ready_threads--; thread->looper &= ~BINDER_LOOPER_STATE_WAITING; @@ -2385,13 +2436,13 @@ static int binder_thread_read(struct binder_proc *proc, binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_USER_REFS, - "binder: %d:%d %s %d u%p c%p\n", + "binder: %d:%d %s %d u%pK c%pK\n", proc->pid, thread->pid, cmd_name, node->debug_id, node->ptr, node->cookie); } else { list_del_init(&w->entry); if (!weak && !strong) { binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "binder: %d:%d node %d u%p c%p deleted\n", + "binder: %d:%d node %d u%pK c%pK deleted\n", proc->pid, thread->pid, node->debug_id, node->ptr, node->cookie); rb_erase(&node->rb_node, &proc->nodes); @@ -2399,7 +2450,7 @@ static int binder_thread_read(struct binder_proc *proc, binder_stats_deleted(BINDER_STAT_NODE); } else { binder_debug(BINDER_DEBUG_INTERNAL_REFS, - "binder: %d:%d node %d u%p c%p state unchanged\n", + "binder: %d:%d node %d u%pK c%pK state unchanged\n", proc->pid, thread->pid, node->debug_id, node->ptr, node->cookie); } @@ -2422,8 +2473,9 @@ static int binder_thread_read(struct binder_proc *proc, if (put_user(death->cookie, (void * __user *)ptr)) return -EFAULT; ptr += sizeof(void *); + binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION, - "binder: %d:%d %s %p\n", + "binder: %d:%d %s %pK\n", proc->pid, thread->pid, cmd == BR_DEAD_BINDER ? "BR_DEAD_BINDER" : @@ -2489,10 +2541,11 @@ static int binder_thread_read(struct binder_proc *proc, return -EFAULT; ptr += sizeof(tr); + trace_binder_transaction_received(t); binder_stat_br(proc, thread, cmd); binder_debug(BINDER_DEBUG_TRANSACTION, "binder: %d:%d %s %d %d:%d, cmd %d" - "size %zd-%zd ptr %p-%p\n", + "size %zd-%zd ptr %pK-%pK\n", proc->pid, thread->pid, (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" : "BR_REPLY", @@ -2529,6 +2582,7 @@ static int binder_thread_read(struct binder_proc *proc, proc->pid, thread->pid); if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer)) return -EFAULT; + binder_stat_br(proc, thread, BR_SPAWN_LOOPER); } return 0; } @@ -2568,7 +2622,7 @@ static void binder_release_work(struct list_head *list) death = container_of(w, struct binder_ref_death, work); binder_debug(BINDER_DEBUG_DEAD_TRANSACTION, - "binder: undelivered death notification, %p\n", + "binder: undelivered death notification, %pK\n", death->cookie); kfree(death); binder_stats_deleted(BINDER_STAT_DEATH); @@ -2665,12 +2719,14 @@ static unsigned int binder_poll(struct file *filp, struct binder_thread *thread = NULL; int wait_for_proc_work; - mutex_lock(&binder_lock); + binder_lock(__func__); + thread = binder_get_thread(proc); wait_for_proc_work = thread->transaction_stack == NULL && list_empty(&thread->todo) && thread->return_error == BR_OK; - mutex_unlock(&binder_lock); + + binder_unlock(__func__); if (wait_for_proc_work) { if (binder_has_proc_work(proc, thread)) @@ -2698,11 +2754,13 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) /*printk(KERN_INFO "binder_ioctl: %d:%d %x %lx\n", proc->pid, current->pid, cmd, arg);*/ + trace_binder_ioctl(cmd, arg); + ret = wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); if (ret) - return ret; + goto err_unlocked; - mutex_lock(&binder_lock); + binder_lock(__func__); thread = binder_get_thread(proc); if (thread == NULL) { ret = -ENOMEM; @@ -2727,6 +2785,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (bwr.write_size > 0) { ret = binder_thread_write(proc, thread, (void __user *)bwr.write_buffer, bwr.write_size, &bwr.write_consumed); + trace_binder_write_done(ret); if (ret < 0) { bwr.read_consumed = 0; if (copy_to_user(ubuf, &bwr, sizeof(bwr))) @@ -2736,6 +2795,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } if (bwr.read_size > 0) { ret = binder_thread_read(proc, thread, (void __user *)bwr.read_buffer, bwr.read_size, &bwr.read_consumed, filp->f_flags & O_NONBLOCK); + trace_binder_read_done(ret); if (!list_empty(&proc->todo)) wake_up_interruptible(&proc->wait); if (ret < 0) { @@ -2816,12 +2876,14 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err: if (thread) thread->looper &= ~BINDER_LOOPER_STATE_NEED_RETURN; - mutex_unlock(&binder_lock); + binder_unlock(__func__); wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2); if (ret && ret != -ERESTARTSYS) binder_debug(BINDER_DEBUG_TOP_ERRORS, "binder: %d:%d ioctl %x %lx returned %d\n", proc->pid, current->pid, cmd, arg, ret); +err_unlocked: + trace_binder_ioctl_done(ret); return ret; } @@ -2898,7 +2960,7 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma) if (cache_is_vipt_aliasing()) { while (CACHE_COLOUR((vma->vm_start ^ (uint32_t)proc->buffer))) { binder_debug(BINDER_DEBUG_TOP_ERRORS, - "binder_mmap: %d %lx-%lx maps %p bad alignment\n", proc->pid, vma->vm_start, vma->vm_end, proc->buffer); + "binder_mmap: %d %lx-%lx maps %pK bad alignment\n", proc->pid, vma->vm_start, vma->vm_end, proc->buffer); vma->vm_start += PAGE_SIZE; } } @@ -2930,7 +2992,7 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma) proc->vma = vma; proc->vma_vm_mm = vma->vm_mm; - /*printk(KERN_INFO "binder_mmap: %d %lx-%lx maps %p\n", + /*printk(KERN_INFO "binder_mmap: %d %lx-%lx maps %pK\n", proc->pid, vma->vm_start, vma->vm_end, proc->buffer);*/ return 0; @@ -2967,13 +3029,16 @@ static int binder_open(struct inode *nodp, struct file *filp) INIT_LIST_HEAD(&proc->todo); init_waitqueue_head(&proc->wait); proc->default_priority = task_nice(current); - mutex_lock(&binder_lock); + + binder_lock(__func__); + binder_stats_created(BINDER_STAT_PROC); hlist_add_head(&proc->proc_node, &binder_procs); proc->pid = current->group_leader->pid; INIT_LIST_HEAD(&proc->delivered_death); filp->private_data = proc; - mutex_unlock(&binder_lock); + + binder_unlock(__func__); if (binder_debugfs_dir_entry_proc) { char strbuf[11]; @@ -3124,7 +3189,7 @@ static void binder_deferred_release(struct binder_proc *proc) void *page_addr = proc->buffer + i * PAGE_SIZE; binder_debug(BINDER_DEBUG_BUFFER_ALLOC, "binder_release: %d: " - "page %d at %p not freed\n", + "page %d at %pK not freed\n", proc->pid, i, page_addr); unmap_kernel_range((unsigned long)page_addr, @@ -3156,7 +3221,7 @@ static void binder_deferred_func(struct work_struct *work) int defer; do { - mutex_lock(&binder_lock); + binder_lock(__func__); mutex_lock(&binder_deferred_lock); if (!hlist_empty(&binder_deferred_list)) { proc = hlist_entry(binder_deferred_list.first, @@ -3183,7 +3248,7 @@ static void binder_deferred_func(struct work_struct *work) if (defer & BINDER_DEFERRED_RELEASE) binder_deferred_release(proc); /* frees proc */ - mutex_unlock(&binder_lock); + binder_unlock(__func__); if (files) put_files_struct(files); } while (proc); @@ -3207,7 +3272,7 @@ static void print_binder_transaction(struct seq_file *m, const char *prefix, struct binder_transaction *t) { seq_printf(m, - "%s %d: %p from %d:%d to %d:%d code %x flags %x pri %ld r%d", + "%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %ld r%d", prefix, t->debug_id, t, t->from ? t->from->proc->pid : 0, t->from ? t->from->pid : 0, @@ -3221,7 +3286,7 @@ static void print_binder_transaction(struct seq_file *m, const char *prefix, if (t->buffer->target_node) seq_printf(m, " node %d", t->buffer->target_node->debug_id); - seq_printf(m, " size %zd:%zd data %p\n", + seq_printf(m, " size %zd:%zd data %pK\n", t->buffer->data_size, t->buffer->offsets_size, t->buffer->data); } @@ -3229,7 +3294,7 @@ static void print_binder_transaction(struct seq_file *m, const char *prefix, static void print_binder_buffer(struct seq_file *m, const char *prefix, struct binder_buffer *buffer) { - seq_printf(m, "%s %d: %p size %zd:%zd %s\n", + seq_printf(m, "%s %d: %pK size %zd:%zd %s\n", prefix, buffer->debug_id, buffer->data, buffer->data_size, buffer->offsets_size, buffer->transaction ? "active" : "delivered"); @@ -3252,7 +3317,7 @@ static void print_binder_work(struct seq_file *m, const char *prefix, break; case BINDER_WORK_NODE: node = container_of(w, struct binder_node, work); - seq_printf(m, "%snode work %d: u%p c%p\n", + seq_printf(m, "%snode work %d: u%pK c%pK\n", prefix, node->debug_id, node->ptr, node->cookie); break; case BINDER_WORK_DEAD_BINDER: @@ -3314,7 +3379,7 @@ static void print_binder_node(struct seq_file *m, struct binder_node *node) hlist_for_each_entry(ref, pos, &node->refs, node_entry) count++; - seq_printf(m, " node %d: u%p c%p hs %d hw %d ls %d lw %d is %d iw %d", + seq_printf(m, " node %d: u%pK c%pK hs %d hw %d ls %d lw %d is %d iw %d", node->debug_id, node->ptr, node->cookie, node->has_strong_ref, node->has_weak_ref, node->local_strong_refs, node->local_weak_refs, @@ -3332,7 +3397,7 @@ static void print_binder_node(struct seq_file *m, struct binder_node *node) static void print_binder_ref(struct seq_file *m, struct binder_ref *ref) { - seq_printf(m, " ref %d: desc %d %snode %d s %d w %d d %p\n", + seq_printf(m, " ref %d: desc %d %snode %d s %d w %d d %pK\n", ref->debug_id, ref->desc, ref->node->proc ? "" : "dead ", ref->node->debug_id, ref->strong, ref->weak, ref->death); } @@ -3524,7 +3589,7 @@ static int binder_state_show(struct seq_file *m, void *unused) int do_lock = !binder_debug_no_lock; if (do_lock) - mutex_lock(&binder_lock); + binder_lock(__func__); seq_puts(m, "binder state:\n"); @@ -3536,7 +3601,7 @@ static int binder_state_show(struct seq_file *m, void *unused) hlist_for_each_entry(proc, pos, &binder_procs, proc_node) print_binder_proc(m, proc, 1); if (do_lock) - mutex_unlock(&binder_lock); + binder_unlock(__func__); return 0; } @@ -3547,7 +3612,7 @@ static int binder_stats_show(struct seq_file *m, void *unused) int do_lock = !binder_debug_no_lock; if (do_lock) - mutex_lock(&binder_lock); + binder_lock(__func__); seq_puts(m, "binder stats:\n"); @@ -3556,7 +3621,7 @@ static int binder_stats_show(struct seq_file *m, void *unused) hlist_for_each_entry(proc, pos, &binder_procs, proc_node) print_binder_proc_stats(m, proc); if (do_lock) - mutex_unlock(&binder_lock); + binder_unlock(__func__); return 0; } @@ -3567,27 +3632,39 @@ static int binder_transactions_show(struct seq_file *m, void *unused) int do_lock = !binder_debug_no_lock; if (do_lock) - mutex_lock(&binder_lock); + binder_lock(__func__); seq_puts(m, "binder transactions:\n"); hlist_for_each_entry(proc, pos, &binder_procs, proc_node) print_binder_proc(m, proc, 0); if (do_lock) - mutex_unlock(&binder_lock); + binder_unlock(__func__); return 0; } static int binder_proc_show(struct seq_file *m, void *unused) { + struct binder_proc *itr; struct binder_proc *proc = m->private; + struct hlist_node *pos; int do_lock = !binder_debug_no_lock; + bool valid_proc = false; if (do_lock) - mutex_lock(&binder_lock); - seq_puts(m, "binder proc state:\n"); - print_binder_proc(m, proc, 1); + binder_lock(__func__); + + hlist_for_each_entry(itr, pos, &binder_procs, proc_node) { + if (itr == proc) { + valid_proc = true; + break; + } + } + if (valid_proc) { + seq_puts(m, "binder proc state:\n"); + print_binder_proc(m, proc, 1); + } if (do_lock) - mutex_unlock(&binder_lock); + binder_unlock(__func__); return 0; } @@ -3682,4 +3759,7 @@ static int __init binder_init(void) device_initcall(binder_init); +#define CREATE_TRACE_POINTS +#include "binder_trace.h" + MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/binder_trace.h b/drivers/staging/android/binder_trace.h new file mode 100644 index 00000000000..82a567c2af6 --- /dev/null +++ b/drivers/staging/android/binder_trace.h @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2012 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM binder + +#if !defined(_BINDER_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _BINDER_TRACE_H + +#include + +struct binder_buffer; +struct binder_node; +struct binder_proc; +struct binder_ref; +struct binder_thread; +struct binder_transaction; + +TRACE_EVENT(binder_ioctl, + TP_PROTO(unsigned int cmd, unsigned long arg), + TP_ARGS(cmd, arg), + + TP_STRUCT__entry( + __field(unsigned int, cmd) + __field(unsigned long, arg) + ), + TP_fast_assign( + __entry->cmd = cmd; + __entry->arg = arg; + ), + TP_printk("cmd=0x%x arg=0x%lx", __entry->cmd, __entry->arg) +); + +DECLARE_EVENT_CLASS(binder_lock_class, + TP_PROTO(const char *tag), + TP_ARGS(tag), + TP_STRUCT__entry( + __field(const char *, tag) + ), + TP_fast_assign( + __entry->tag = tag; + ), + TP_printk("tag=%s", __entry->tag) +); + +#define DEFINE_BINDER_LOCK_EVENT(name) \ +DEFINE_EVENT(binder_lock_class, name, \ + TP_PROTO(const char *func), \ + TP_ARGS(func)) + +DEFINE_BINDER_LOCK_EVENT(binder_lock); +DEFINE_BINDER_LOCK_EVENT(binder_locked); +DEFINE_BINDER_LOCK_EVENT(binder_unlock); + +DECLARE_EVENT_CLASS(binder_function_return_class, + TP_PROTO(int ret), + TP_ARGS(ret), + TP_STRUCT__entry( + __field(int, ret) + ), + TP_fast_assign( + __entry->ret = ret; + ), + TP_printk("ret=%d", __entry->ret) +); + +#define DEFINE_BINDER_FUNCTION_RETURN_EVENT(name) \ +DEFINE_EVENT(binder_function_return_class, name, \ + TP_PROTO(int ret), \ + TP_ARGS(ret)) + +DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_ioctl_done); +DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_write_done); +DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_read_done); + +TRACE_EVENT(binder_wait_for_work, + TP_PROTO(bool proc_work, bool transaction_stack, bool thread_todo), + TP_ARGS(proc_work, transaction_stack, thread_todo), + + TP_STRUCT__entry( + __field(bool, proc_work) + __field(bool, transaction_stack) + __field(bool, thread_todo) + ), + TP_fast_assign( + __entry->proc_work = proc_work; + __entry->transaction_stack = transaction_stack; + __entry->thread_todo = thread_todo; + ), + TP_printk("proc_work=%d transaction_stack=%d thread_todo=%d", + __entry->proc_work, __entry->transaction_stack, + __entry->thread_todo) +); + +TRACE_EVENT(binder_transaction, + TP_PROTO(bool reply, struct binder_transaction *t, + struct binder_node *target_node), + TP_ARGS(reply, t, target_node), + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, target_node) + __field(int, to_proc) + __field(int, to_thread) + __field(int, reply) + __field(unsigned int, code) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->target_node = target_node ? target_node->debug_id : 0; + __entry->to_proc = t->to_proc->pid; + __entry->to_thread = t->to_thread ? t->to_thread->pid : 0; + __entry->reply = reply; + __entry->code = t->code; + __entry->flags = t->flags; + ), + TP_printk("transaction=%d dest_node=%d dest_proc=%d dest_thread=%d reply=%d flags=0x%x code=0x%x", + __entry->debug_id, __entry->target_node, + __entry->to_proc, __entry->to_thread, + __entry->reply, __entry->flags, __entry->code) +); + +TRACE_EVENT(binder_transaction_received, + TP_PROTO(struct binder_transaction *t), + TP_ARGS(t), + + TP_STRUCT__entry( + __field(int, debug_id) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + ), + TP_printk("transaction=%d", __entry->debug_id) +); + +TRACE_EVENT(binder_transaction_node_to_ref, + TP_PROTO(struct binder_transaction *t, struct binder_node *node, + struct binder_ref *ref), + TP_ARGS(t, node, ref), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, node_debug_id) + __field(void __user *, node_ptr) + __field(int, ref_debug_id) + __field(uint32_t, ref_desc) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->node_debug_id = node->debug_id; + __entry->node_ptr = node->ptr; + __entry->ref_debug_id = ref->debug_id; + __entry->ref_desc = ref->desc; + ), + TP_printk("transaction=%d node=%d src_ptr=0x%p ==> dest_ref=%d dest_desc=%d", + __entry->debug_id, __entry->node_debug_id, __entry->node_ptr, + __entry->ref_debug_id, __entry->ref_desc) +); + +TRACE_EVENT(binder_transaction_ref_to_node, + TP_PROTO(struct binder_transaction *t, struct binder_ref *ref), + TP_ARGS(t, ref), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, ref_debug_id) + __field(uint32_t, ref_desc) + __field(int, node_debug_id) + __field(void __user *, node_ptr) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->ref_debug_id = ref->debug_id; + __entry->ref_desc = ref->desc; + __entry->node_debug_id = ref->node->debug_id; + __entry->node_ptr = ref->node->ptr; + ), + TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%p", + __entry->debug_id, __entry->node_debug_id, + __entry->ref_debug_id, __entry->ref_desc, __entry->node_ptr) +); + +TRACE_EVENT(binder_transaction_ref_to_ref, + TP_PROTO(struct binder_transaction *t, struct binder_ref *src_ref, + struct binder_ref *dest_ref), + TP_ARGS(t, src_ref, dest_ref), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, node_debug_id) + __field(int, src_ref_debug_id) + __field(uint32_t, src_ref_desc) + __field(int, dest_ref_debug_id) + __field(uint32_t, dest_ref_desc) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->node_debug_id = src_ref->node->debug_id; + __entry->src_ref_debug_id = src_ref->debug_id; + __entry->src_ref_desc = src_ref->desc; + __entry->dest_ref_debug_id = dest_ref->debug_id; + __entry->dest_ref_desc = dest_ref->desc; + ), + TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ref=%d dest_desc=%d", + __entry->debug_id, __entry->node_debug_id, + __entry->src_ref_debug_id, __entry->src_ref_desc, + __entry->dest_ref_debug_id, __entry->dest_ref_desc) +); + +TRACE_EVENT(binder_transaction_fd, + TP_PROTO(struct binder_transaction *t, int src_fd, int dest_fd), + TP_ARGS(t, src_fd, dest_fd), + + TP_STRUCT__entry( + __field(int, debug_id) + __field(int, src_fd) + __field(int, dest_fd) + ), + TP_fast_assign( + __entry->debug_id = t->debug_id; + __entry->src_fd = src_fd; + __entry->dest_fd = dest_fd; + ), + TP_printk("transaction=%d src_fd=%d ==> dest_fd=%d", + __entry->debug_id, __entry->src_fd, __entry->dest_fd) +); + +DECLARE_EVENT_CLASS(binder_buffer_class, + TP_PROTO(struct binder_buffer *buf), + TP_ARGS(buf), + TP_STRUCT__entry( + __field(int, debug_id) + __field(size_t, data_size) + __field(size_t, offsets_size) + ), + TP_fast_assign( + __entry->debug_id = buf->debug_id; + __entry->data_size = buf->data_size; + __entry->offsets_size = buf->offsets_size; + ), + TP_printk("transaction=%d data_size=%zd offsets_size=%zd", + __entry->debug_id, __entry->data_size, __entry->offsets_size) +); + +DEFINE_EVENT(binder_buffer_class, binder_transaction_alloc_buf, + TP_PROTO(struct binder_buffer *buffer), + TP_ARGS(buffer)); + +DEFINE_EVENT(binder_buffer_class, binder_transaction_buffer_release, + TP_PROTO(struct binder_buffer *buffer), + TP_ARGS(buffer)); + +DEFINE_EVENT(binder_buffer_class, binder_transaction_failed_buffer_release, + TP_PROTO(struct binder_buffer *buffer), + TP_ARGS(buffer)); + +TRACE_EVENT(binder_update_page_range, + TP_PROTO(struct binder_proc *proc, bool allocate, + void *start, void *end), + TP_ARGS(proc, allocate, start, end), + TP_STRUCT__entry( + __field(int, proc) + __field(bool, allocate) + __field(size_t, offset) + __field(size_t, size) + ), + TP_fast_assign( + __entry->proc = proc->pid; + __entry->allocate = allocate; + __entry->offset = start - proc->buffer; + __entry->size = end - start; + ), + TP_printk("proc=%d allocate=%d offset=%zu size=%zu", + __entry->proc, __entry->allocate, + __entry->offset, __entry->size) +); + +TRACE_EVENT(binder_command, + TP_PROTO(uint32_t cmd), + TP_ARGS(cmd), + TP_STRUCT__entry( + __field(uint32_t, cmd) + ), + TP_fast_assign( + __entry->cmd = cmd; + ), + TP_printk("cmd=0x%x %s", + __entry->cmd, + _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_command_strings) ? + binder_command_strings[_IOC_NR(__entry->cmd)] : + "unknown") +); + +TRACE_EVENT(binder_return, + TP_PROTO(uint32_t cmd), + TP_ARGS(cmd), + TP_STRUCT__entry( + __field(uint32_t, cmd) + ), + TP_fast_assign( + __entry->cmd = cmd; + ), + TP_printk("cmd=0x%x %s", + __entry->cmd, + _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_return_strings) ? + binder_return_strings[_IOC_NR(__entry->cmd)] : + "unknown") +); + +#endif /* _BINDER_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE binder_trace +#include diff --git a/drivers/staging/android/lowmemorykiller.c b/drivers/staging/android/lowmemorykiller.c index d1f06032e73..ce8a2a362b7 100644 --- a/drivers/staging/android/lowmemorykiller.c +++ b/drivers/staging/android/lowmemorykiller.c @@ -40,6 +40,8 @@ #include #include #include +#include +#include #ifdef CONFIG_HIGHMEM #define _ZONE ZONE_HIGHMEM @@ -264,6 +266,7 @@ static int lowmem_shrink(struct shrinker *s, struct shrink_control *sc) int tasksize; int i; int min_score_adj = OOM_SCORE_ADJ_MAX + 1; + int minfree = 0; int selected_tasksize = 0; int selected_oom_score_adj; int array_size = ARRAY_SIZE(lowmem_adj); @@ -287,8 +290,8 @@ static int lowmem_shrink(struct shrinker *s, struct shrink_control *sc) if (lowmem_minfree_size < array_size) array_size = lowmem_minfree_size; for (i = 0; i < array_size; i++) { - if (other_free < lowmem_minfree[i] && - other_file < lowmem_minfree[i]) { + minfree = lowmem_minfree[i]; + if (other_free < minfree && other_file < minfree) { min_score_adj = lowmem_adj[i]; break; } @@ -357,13 +360,41 @@ static int lowmem_shrink(struct shrinker *s, struct shrink_control *sc) selected = p; selected_tasksize = tasksize; selected_oom_score_adj = oom_score_adj; - lowmem_print(2, "select %d (%s), adj %d, size %d, to kill\n", - p->pid, p->comm, oom_score_adj, tasksize); + lowmem_print(3, "select '%s' (%d), adj %hd, size %d, to kill\n", + p->comm, p->pid, oom_score_adj, tasksize); } if (selected) { - lowmem_print(1, "send sigkill to %d (%s), adj %d, size %d\n", - selected->pid, selected->comm, - selected_oom_score_adj, selected_tasksize); + lowmem_print(1, "Killing '%s' (%d), adj %hd,\n" \ + " to free %ldkB on behalf of '%s' (%d) because\n" \ + " cache %ldkB is below limit %ldkB for oom_score_adj %hd\n" \ + " Free memory is %ldkB above reserved.\n" \ + " Free CMA is %ldkB\n" \ + " Total reserve is %ldkB\n" \ + " Total free pages is %ldkB\n" \ + " Total file cache is %ldkB\n" \ + " GFP mask is 0x%x\n", + selected->comm, selected->pid, + selected_oom_score_adj, + selected_tasksize * (long)(PAGE_SIZE / 1024), + current->comm, current->pid, + other_file * (long)(PAGE_SIZE / 1024), + minfree * (long)(PAGE_SIZE / 1024), + min_score_adj, + other_free * (long)(PAGE_SIZE / 1024), + global_page_state(NR_FREE_CMA_PAGES) * + (long)(PAGE_SIZE / 1024), + totalreserve_pages * (long)(PAGE_SIZE / 1024), + global_page_state(NR_FREE_PAGES) * + (long)(PAGE_SIZE / 1024), + global_page_state(NR_FILE_PAGES) * + (long)(PAGE_SIZE / 1024), + sc->gfp_mask); + + if (lowmem_debug_level >= 2 && selected_oom_score_adj == 0) { + show_mem(SHOW_MEM_FILTER_NODES); + dump_tasks(NULL, NULL); + } + lowmem_deathpending_timeout = jiffies + HZ; send_sig(SIGKILL, selected, 0); set_tsk_thread_flag(selected, TIF_MEMDIE); diff --git a/drivers/thermal/qpnp-adc-tm.c b/drivers/thermal/qpnp-adc-tm.c index ccb02628c1f..7020864a245 100644 --- a/drivers/thermal/qpnp-adc-tm.c +++ b/drivers/thermal/qpnp-adc-tm.c @@ -1645,7 +1645,7 @@ static int __devinit qpnp_adc_tm_probe(struct spmi_device *spmi) pr_debug("thermal node%x\n", btm_channel_num); adc_tm->sensor[sen_idx].mode = THERMAL_DEVICE_DISABLED; adc_tm->sensor[sen_idx].thermal_node = true; - snprintf(name, sizeof(name), + snprintf(name, sizeof(name), "%s", adc_tm->adc->adc_channels[sen_idx].name); adc_tm->sensor[sen_idx].meas_interval = QPNP_ADC_TM_MEAS_INTERVAL; diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index a2034723717..a20dcb9dca8 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -2474,6 +2474,28 @@ static int tiocsetd(struct tty_struct *tty, int __user *p) return ret; } +/** + * tiocgetd - get line discipline + * @tty: tty device + * @p: pointer to user data + * + * Retrieves the line discipline id directly from the ldisc. + * + * Locking: waits for ldisc reference (in case the line discipline + * is changing or the tty is being hungup) + */ + +static int tiocgetd(struct tty_struct *tty, int __user *p) +{ + struct tty_ldisc *ld; + int ret; + + ld = tty_ldisc_ref_wait(tty); + ret = put_user(ld->ops->num, p); + tty_ldisc_deref(ld); + return ret; +} + /** * send_break - performed time break * @tty: device to break on @@ -2687,7 +2709,7 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case TIOCGSID: return tiocgsid(tty, real_tty, p); case TIOCGETD: - return put_user(tty->ldisc->ops->num, (int __user *)p); + return tiocgetd(tty, p); case TIOCSETD: return tiocsetd(tty, p); case TIOCVHANGUP: diff --git a/drivers/tty/tty_ldisc.c b/drivers/tty/tty_ldisc.c index 24b95db75d8..90ee4170c92 100644 --- a/drivers/tty/tty_ldisc.c +++ b/drivers/tty/tty_ldisc.c @@ -406,6 +406,10 @@ EXPORT_SYMBOL_GPL(tty_ldisc_flush); * they are not on hot paths so a little discipline won't do * any harm. * + * The line discipline-related tty_struct fields are reset to + * prevent the ldisc driver from re-using stale information for + * the new ldisc instance. + * * Locking: takes termios_mutex */ @@ -414,6 +418,9 @@ static void tty_set_termios_ldisc(struct tty_struct *tty, int num) mutex_lock(&tty->termios_mutex); tty->termios->c_line = num; mutex_unlock(&tty->termios_mutex); + + tty->disc_data = NULL; + tty->receive_room = 0; } /** diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index b32ccb46101..44c12421dbe 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -938,6 +938,9 @@ static int acm_probe(struct usb_interface *intf, if (quirks == NO_UNION_NORMAL) { data_interface = usb_ifnum_to_if(usb_dev, 1); control_interface = usb_ifnum_to_if(usb_dev, 0); + /* we would crash */ + if (!data_interface || !control_interface) + return -ENODEV; goto skip_normal_probe; } diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index 8df4b76465a..fe50a70d2ad 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1035,10 +1035,11 @@ static int proc_getdriver(struct dev_state *ps, void __user *arg) static int proc_connectinfo(struct dev_state *ps, void __user *arg) { - struct usbdevfs_connectinfo ci = { - .devnum = ps->dev->devnum, - .slow = ps->dev->speed == USB_SPEED_LOW - }; + struct usbdevfs_connectinfo ci; + + memset(&ci, 0, sizeof(ci)); + ci.devnum = ps->dev->devnum; + ci.slow = ps->dev->speed == USB_SPEED_LOW; if (copy_to_user(arg, &ci, sizeof(ci))) return -EFAULT; diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index 23826403b54..44274d36de6 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -417,9 +417,13 @@ static int usb_unbind_interface(struct device *dev) int usb_driver_claim_interface(struct usb_driver *driver, struct usb_interface *iface, void *priv) { - struct device *dev = &iface->dev; + struct device *dev; int retval = 0; + if (!iface) + return -ENODEV; + + dev = &iface->dev; if (dev->driver) return -EBUSY; diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 95e0785c142..71a798326c6 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -743,6 +743,7 @@ enum hub_activation_type { static void hub_init_func2(struct work_struct *ws); static void hub_init_func3(struct work_struct *ws); +static void hub_release(struct kref *kref); static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) { @@ -755,10 +756,20 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) unsigned delay; /* Continue a partial initialization */ - if (type == HUB_INIT2) - goto init2; - if (type == HUB_INIT3) + if (type == HUB_INIT2 || type == HUB_INIT3) { + device_lock(hub->intfdev); + + /* Was the hub disconnected while we were waiting? */ + if (hub->disconnected) { + device_unlock(hub->intfdev); + kref_put(&hub->kref, hub_release); + return; + } + if (type == HUB_INIT2) + goto init2; goto init3; + } + kref_get(&hub->kref); /* The superspeed hub except for root hub has to use Hub Depth * value as an offset into the route string to locate the bits @@ -952,6 +963,7 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) PREPARE_DELAYED_WORK(&hub->init_work, hub_init_func3); schedule_delayed_work(&hub->init_work, msecs_to_jiffies(delay)); + device_unlock(hub->intfdev); return; /* Continues at init3: below */ } else { msleep(delay); @@ -972,6 +984,11 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) /* Allow autosuspend if it was suppressed */ if (type <= HUB_INIT3) usb_autopm_put_interface_async(to_usb_interface(hub->intfdev)); + + if (type == HUB_INIT2 || type == HUB_INIT3) + device_unlock(hub->intfdev); + + kref_put(&hub->kref, hub_release); } /* Implement the continuations for the delays above */ diff --git a/drivers/usb/dwc3/debugfs.c b/drivers/usb/dwc3/debugfs.c index df9564640c1..893adfee8ff 100644 --- a/drivers/usb/dwc3/debugfs.c +++ b/drivers/usb/dwc3/debugfs.c @@ -660,7 +660,7 @@ static ssize_t dwc3_store_ep_num(struct file *file, const char __user *ubuf, struct seq_file *s = file->private_data; struct dwc3 *dwc = s->private; char kbuf[10]; - unsigned int num, dir; + unsigned int num, dir, temp; unsigned long flags; memset(kbuf, 0, 10); @@ -671,8 +671,15 @@ static ssize_t dwc3_store_ep_num(struct file *file, const char __user *ubuf, if (sscanf(kbuf, "%u %u", &num, &dir) != 2) return -EINVAL; + if (dir != 0 && dir != 1) + return -EINVAL; + + temp = (num << 1) + dir; + if (temp >= DWC3_ENDPOINTS_NUM) + return -EINVAL; + spin_lock_irqsave(&dwc->lock, flags); - ep_num = (num << 1) + dir; + ep_num = temp; spin_unlock_irqrestore(&dwc->lock, flags); return count; diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index c6376be7e51..289d1bf0800 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -2005,6 +2005,13 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep, */ req->request.actual += req->request.length - count; dwc3_gadget_giveback(dep, req, status); + + /* EP possibly disabled during giveback? */ + if (!(dep->flags & DWC3_EP_ENABLED)) { + dev_dbg(dwc->dev, "%s disabled while handling ep event\n", + dep->name); + return 0; + } if (s_pkt) break; if ((event->status & DEPEVT_STATUS_LST) && diff --git a/drivers/usb/gadget/android.c b/drivers/usb/gadget/android.c index afb276d1402..dcab6a0e588 100644 --- a/drivers/usb/gadget/android.c +++ b/drivers/usb/gadget/android.c @@ -57,6 +57,7 @@ #ifdef CONFIG_SND_PCM #include "f_audio_source.c" #endif +#include "f_midi.c" #include "f_mass_storage.c" #include "u_serial.c" #include "u_sdio.c" @@ -107,6 +108,12 @@ static const char longname[] = "Gadget Android"; #define ANDROID_DEVICE_NODE_NAME_LENGTH 11 +/* f_midi configuration */ +#define MIDI_INPUT_PORTS 1 +#define MIDI_OUTPUT_PORTS 1 +#define MIDI_BUFFER_SIZE 256 +#define MIDI_QUEUE_LENGTH 32 + struct android_usb_function { char *name; void *config; @@ -2077,6 +2084,60 @@ static struct android_usb_function uasp_function = { .bind_config = uasp_function_bind_config, }; +static int midi_function_init(struct android_usb_function *f, + struct usb_composite_dev *cdev) +{ + struct midi_alsa_config *config; + + config = kzalloc(sizeof(struct midi_alsa_config), GFP_KERNEL); + f->config = config; + if (!config) + return -ENOMEM; + config->card = -1; + config->device = -1; + return 0; +} + +static void midi_function_cleanup(struct android_usb_function *f) +{ + kfree(f->config); +} + +static int midi_function_bind_config(struct android_usb_function *f, + struct usb_configuration *c) +{ + struct midi_alsa_config *config = f->config; + + return f_midi_bind_config(c, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, + MIDI_INPUT_PORTS, MIDI_OUTPUT_PORTS, MIDI_BUFFER_SIZE, + MIDI_QUEUE_LENGTH, config); +} + +static ssize_t midi_alsa_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct android_usb_function *f = dev_get_drvdata(dev); + struct midi_alsa_config *config = f->config; + + /* print ALSA card and device numbers */ + return sprintf(buf, "%d %d\n", config->card, config->device); +} + +static DEVICE_ATTR(alsa, S_IRUGO, midi_alsa_show, NULL); + +static struct device_attribute *midi_function_attributes[] = { + &dev_attr_alsa, + NULL +}; + +static struct android_usb_function midi_function = { + .name = "midi", + .init = midi_function_init, + .cleanup = midi_function_cleanup, + .bind_config = midi_function_bind_config, + .attributes = midi_function_attributes, +}; + static struct android_usb_function *supported_functions[] = { &mbim_function, &ecm_qc_function, @@ -2106,6 +2167,7 @@ static struct android_usb_function *supported_functions[] = { #ifdef CONFIG_SND_PCM &audio_source_function, #endif + &midi_function, &uasp_function, NULL }; diff --git a/drivers/usb/gadget/f_mbim.c b/drivers/usb/gadget/f_mbim.c index aaacd4375cb..36cd9cabfd7 100644 --- a/drivers/usb/gadget/f_mbim.c +++ b/drivers/usb/gadget/f_mbim.c @@ -561,24 +561,24 @@ static void fmbim_ctrl_response_available(struct f_mbim *dev) unsigned long flags; int ret; - pr_debug("dev:%p portno#%d\n", dev, dev->port_num); + pr_debug("dev:%pK portno#%d\n", dev, dev->port_num); spin_lock_irqsave(&dev->lock, flags); if (!atomic_read(&dev->online)) { - pr_err("dev:%p is not online\n", dev); + pr_err("dev:%pK is not online\n", dev); spin_unlock_irqrestore(&dev->lock, flags); return; } if (!req) { - pr_err("dev:%p req is NULL\n", dev); + pr_err("dev:%pK req is NULL\n", dev); spin_unlock_irqrestore(&dev->lock, flags); return; } if (!req->buf) { - pr_err("dev:%p req->buf is NULL\n", dev); + pr_err("dev:%pK req->buf is NULL\n", dev); spin_unlock_irqrestore(&dev->lock, flags); return; } @@ -617,21 +617,21 @@ fmbim_send_cpkt_response(struct f_mbim *gr, struct ctrl_pkt *cpkt) unsigned long flags; if (!gr || !cpkt) { - pr_err("Invalid cpkt, dev:%p cpkt:%p\n", + pr_err("Invalid cpkt, dev:%pK cpkt:%pK\n", gr, cpkt); return -ENODEV; } - pr_debug("dev:%p port_num#%d\n", dev, dev->port_num); + pr_debug("dev:%pK port_num#%d\n", dev, dev->port_num); if (!atomic_read(&dev->online)) { - pr_err("dev:%p is not connected\n", dev); + pr_err("dev:%pK is not connected\n", dev); mbim_free_ctrl_pkt(cpkt); return 0; } if (dev->not_port.notify_state != MBIM_NOTIFY_RESPONSE_AVAILABLE) { - pr_err("dev:%p state=%d, recover!!\n", dev, + pr_err("dev:%pK state=%d, recover!!\n", dev, dev->not_port.notify_state); mbim_free_ctrl_pkt(cpkt); return 0; @@ -694,7 +694,7 @@ static int mbim_bam_connect(struct f_mbim *dev) enum peer_bam bam_name = (dev->xport == USB_GADGET_XPORT_BAM2BAM_IPA) ? IPA_P_BAM : A2_P_BAM; - pr_info("dev:%p portno:%d\n", dev, dev->port_num); + pr_info("dev:%pK portno:%d\n", dev, dev->port_num); src_connection_idx = usb_bam_get_connection_idx(gadget->name, bam_name, USB_TO_PEER_PERIPHERAL, dev->port_num); @@ -721,7 +721,7 @@ static int mbim_bam_connect(struct f_mbim *dev) static int mbim_bam_disconnect(struct f_mbim *dev) { - pr_info("dev:%p port:%d. Do nothing.\n", + pr_info("dev:%pK port:%d. Do nothing.\n", dev, dev->port_num); bam_data_disconnect(&dev->bam_port, dev->port_num); @@ -863,7 +863,7 @@ static void mbim_notify_complete(struct usb_ep *ep, struct usb_request *req) struct f_mbim *mbim = req->context; struct usb_cdc_notification *event = req->buf; - pr_debug("dev:%p\n", mbim); + pr_debug("dev:%pK\n", mbim); spin_lock(&mbim->lock); switch (req->status) { @@ -893,7 +893,7 @@ static void mbim_notify_complete(struct usb_ep *ep, struct usb_request *req) mbim_do_notify(mbim); spin_unlock(&mbim->lock); - pr_debug("dev:%p Exit\n", mbim); + pr_debug("dev:%pK Exit\n", mbim); } static void mbim_ep0out_complete(struct usb_ep *ep, struct usb_request *req) @@ -904,7 +904,7 @@ static void mbim_ep0out_complete(struct usb_ep *ep, struct usb_request *req) struct f_mbim *mbim = func_to_mbim(f); struct mbim_ntb_input_size *ntb = NULL; - pr_debug("dev:%p\n", mbim); + pr_debug("dev:%pK\n", mbim); req->context = NULL; if (req->status || req->actual != req->length) { @@ -942,7 +942,7 @@ static void mbim_ep0out_complete(struct usb_ep *ep, struct usb_request *req) invalid: usb_ep_set_halt(ep); - pr_err("dev:%p Failed\n", mbim); + pr_err("dev:%pK Failed\n", mbim); return; } @@ -964,7 +964,7 @@ fmbim_cmd_complete(struct usb_ep *ep, struct usb_request *req) return; } - pr_debug("dev:%p port#%d\n", dev, dev->port_num); + pr_debug("dev:%pK port#%d\n", dev, dev->port_num); cpkt = mbim_alloc_ctrl_pkt(len, GFP_ATOMIC); if (!cpkt) { @@ -1320,7 +1320,7 @@ static int mbim_set_alt(struct usb_function *f, unsigned intf, unsigned alt) return ret; } - pr_info("Set mbim port in_desc = 0x%p", + pr_info("Set mbim port in_desc = 0x%pK", mbim->bam_port.in->desc); ret = config_ep_by_speed(cdev->gadget, f, @@ -1332,7 +1332,7 @@ static int mbim_set_alt(struct usb_function *f, unsigned intf, unsigned alt) return ret; } - pr_info("Set mbim port out_desc = 0x%p", + pr_info("Set mbim port out_desc = 0x%pK", mbim->bam_port.out->desc); pr_debug("Activate mbim\n"); diff --git a/drivers/usb/gadget/f_midi.c b/drivers/usb/gadget/f_midi.c index 2f7e8f2930c..d9ff10ca737 100644 --- a/drivers/usb/gadget/f_midi.c +++ b/drivers/usb/gadget/f_midi.c @@ -66,6 +66,11 @@ struct gmidi_in_port { uint8_t data[2]; }; +struct midi_alsa_config { + int card; + int device; +}; + struct f_midi { struct usb_function func; struct usb_gadget *gadget; @@ -98,7 +103,7 @@ DECLARE_USB_MIDI_OUT_JACK_DESCRIPTOR(1); DECLARE_USB_MS_ENDPOINT_DESCRIPTOR(16); /* B.3.1 Standard AC Interface Descriptor */ -static struct usb_interface_descriptor ac_interface_desc __initdata = { +static struct usb_interface_descriptor midi_ac_interface_desc /* __initdata */ = { .bLength = USB_DT_INTERFACE_SIZE, .bDescriptorType = USB_DT_INTERFACE, /* .bInterfaceNumber = DYNAMIC */ @@ -109,7 +114,7 @@ static struct usb_interface_descriptor ac_interface_desc __initdata = { }; /* B.3.2 Class-Specific AC Interface Descriptor */ -static struct uac1_ac_header_descriptor_1 ac_header_desc __initdata = { +static struct uac1_ac_header_descriptor_1 midi_ac_header_desc /* __initdata */ = { .bLength = UAC_DT_AC_HEADER_SIZE(1), .bDescriptorType = USB_DT_CS_INTERFACE, .bDescriptorSubtype = USB_MS_HEADER, @@ -120,7 +125,7 @@ static struct uac1_ac_header_descriptor_1 ac_header_desc __initdata = { }; /* B.4.1 Standard MS Interface Descriptor */ -static struct usb_interface_descriptor ms_interface_desc __initdata = { +static struct usb_interface_descriptor ms_interface_desc /* __initdata */ = { .bLength = USB_DT_INTERFACE_SIZE, .bDescriptorType = USB_DT_INTERFACE, /* .bInterfaceNumber = DYNAMIC */ @@ -131,7 +136,7 @@ static struct usb_interface_descriptor ms_interface_desc __initdata = { }; /* B.4.2 Class-Specific MS Interface Descriptor */ -static struct usb_ms_header_descriptor ms_header_desc __initdata = { +static struct usb_ms_header_descriptor ms_header_desc /* __initdata */ = { .bLength = USB_DT_MS_HEADER_SIZE, .bDescriptorType = USB_DT_CS_INTERFACE, .bDescriptorSubtype = USB_MS_HEADER, @@ -410,7 +415,7 @@ static void f_midi_unbind(struct usb_configuration *c, struct usb_function *f) card = midi->card; midi->card = NULL; if (card) - snd_card_free(card); + snd_card_free_when_closed(card); kfree(midi->id); midi->id = NULL; @@ -734,7 +739,7 @@ static int f_midi_register_card(struct f_midi *midi) /* MIDI function driver setup/binding */ -static int __init +static int /* __init */ f_midi_bind(struct usb_configuration *c, struct usb_function *f) { struct usb_descriptor_header **midi_function; @@ -758,13 +763,13 @@ f_midi_bind(struct usb_configuration *c, struct usb_function *f) status = usb_interface_id(c, f); if (status < 0) goto fail; - ac_interface_desc.bInterfaceNumber = status; + midi_ac_interface_desc.bInterfaceNumber = status; status = usb_interface_id(c, f); if (status < 0) goto fail; ms_interface_desc.bInterfaceNumber = status; - ac_header_desc.baInterfaceNr[0] = status; + midi_ac_header_desc.baInterfaceNr[0] = status; status = -ENODEV; @@ -794,8 +799,8 @@ f_midi_bind(struct usb_configuration *c, struct usb_function *f) */ /* add the headers - these are always the same */ - midi_function[i++] = (struct usb_descriptor_header *) &ac_interface_desc; - midi_function[i++] = (struct usb_descriptor_header *) &ac_header_desc; + midi_function[i++] = (struct usb_descriptor_header *) &midi_ac_interface_desc; + midi_function[i++] = (struct usb_descriptor_header *) &midi_ac_header_desc; midi_function[i++] = (struct usb_descriptor_header *) &ms_interface_desc; /* calculate the header's wTotalLength */ @@ -884,8 +889,8 @@ f_midi_bind(struct usb_configuration *c, struct usb_function *f) /* copy descriptors, and track endpoint copies */ if (gadget_is_dualspeed(c->cdev->gadget)) { c->highspeed = true; - bulk_in_desc.wMaxPacketSize = cpu_to_le16(512); - bulk_out_desc.wMaxPacketSize = cpu_to_le16(512); + bulk_in_desc.wMaxPacketSize = cpu_to_le16(midi->buflen); + bulk_out_desc.wMaxPacketSize = cpu_to_le16(midi->buflen); f->hs_descriptors = usb_copy_descriptors(midi_function); } else { f->descriptors = usb_copy_descriptors(midi_function); @@ -918,16 +923,22 @@ f_midi_bind(struct usb_configuration *c, struct usb_function *f) * * Returns zero on success, else negative errno. */ -int __init f_midi_bind_config(struct usb_configuration *c, +int /* __init */ f_midi_bind_config(struct usb_configuration *c, int index, char *id, unsigned int in_ports, unsigned int out_ports, unsigned int buflen, - unsigned int qlen) + unsigned int qlen, + struct midi_alsa_config* config) { struct f_midi *midi; int status, i; + if (config) { + config->card = -1; + config->device = -1; + } + /* sanity check */ if (in_ports > MAX_PORTS || out_ports > MAX_PORTS) return -EINVAL; @@ -956,6 +967,10 @@ int __init f_midi_bind_config(struct usb_configuration *c, tasklet_init(&midi->tasklet, f_midi_in_tasklet, (unsigned long) midi); /* set up ALSA midi devices */ + midi->id = kstrdup(id, GFP_KERNEL); + midi->index = index; + midi->buflen = buflen; + midi->qlen = qlen; midi->in_ports = in_ports; midi->out_ports = out_ports; status = f_midi_register_card(midi); @@ -969,15 +984,16 @@ int __init f_midi_bind_config(struct usb_configuration *c, midi->func.set_alt = f_midi_set_alt; midi->func.disable = f_midi_disable; - midi->id = kstrdup(id, GFP_KERNEL); - midi->index = index; - midi->buflen = buflen; - midi->qlen = qlen; - status = usb_add_function(c, &midi->func); if (status) goto setup_fail; + + if (config) { + config->card = midi->rmidi->card->number; + config->device = midi->rmidi->device; + } + return 0; setup_fail: diff --git a/drivers/usb/gadget/u_ctrl_hsic.c b/drivers/usb/gadget/u_ctrl_hsic.c index 6143d1b8714..440d39fe972 100644 --- a/drivers/usb/gadget/u_ctrl_hsic.c +++ b/drivers/usb/gadget/u_ctrl_hsic.c @@ -563,7 +563,7 @@ static ssize_t gctrl_read_stats(struct file *file, char __user *ubuf, temp += scnprintf(buf + temp, DEBUG_BUF_SIZE - temp, "\nName: %s\n" - "#PORT:%d port: %p\n" + "#PORT:%d port: %pK\n" "to_usbhost: %lu\n" "to_modem: %lu\n" "cpkt_drp_cnt: %lu\n" diff --git a/drivers/usb/gadget/u_data_hsic.c b/drivers/usb/gadget/u_data_hsic.c index 92653dbb8b8..df45994479c 100644 --- a/drivers/usb/gadget/u_data_hsic.c +++ b/drivers/usb/gadget/u_data_hsic.c @@ -150,7 +150,7 @@ static int ghsic_data_alloc_requests(struct usb_ep *ep, struct list_head *head, struct usb_request *req; unsigned long flags; - pr_debug("%s: ep:%s head:%p num:%d cb:%p", __func__, + pr_debug("%s: ep:%s head:%pK num:%d cb:%pK", __func__, ep->name, head, num, cb); for (i = 0; i < num; i++) { @@ -266,7 +266,7 @@ static int ghsic_data_receive(void *p, void *data, size_t len) return -ENOTCONN; } - pr_debug("%s: p:%p#%d skb_len:%d\n", __func__, + pr_debug("%s: p:%pK#%d skb_len:%d\n", __func__, port, port->port_num, skb->len); spin_lock_irqsave(&port->tx_lock, flags); @@ -310,7 +310,7 @@ static void ghsic_data_write_tomdm(struct work_struct *w) } while ((skb = __skb_dequeue(&port->rx_skb_q))) { - pr_debug("%s: port:%p tom:%lu pno:%d\n", __func__, + pr_debug("%s: port:%pK tom:%lu pno:%d\n", __func__, port, port->to_modem, port->port_num); info = (struct timestamp_info *)skb->cb; @@ -418,7 +418,7 @@ static void ghsic_data_start_rx(struct gdata_port *port) struct timestamp_info *info; unsigned int created; - pr_debug("%s: port:%p\n", __func__, port); + pr_debug("%s: port:%pK\n", __func__, port); if (!port) return; @@ -475,7 +475,7 @@ static void ghsic_data_start_io(struct gdata_port *port) struct usb_ep *ep_out, *ep_in; int ret; - pr_debug("%s: port:%p\n", __func__, port); + pr_debug("%s: port:%pK\n", __func__, port); if (!port) return; @@ -527,7 +527,7 @@ static void ghsic_data_connect_w(struct work_struct *w) !test_bit(CH_READY, &port->bridge_sts)) return; - pr_debug("%s: port:%p\n", __func__, port); + pr_debug("%s: port:%pK\n", __func__, port); ret = data_bridge_open(&port->brdg); if (ret) { @@ -725,7 +725,7 @@ static int ghsic_data_port_alloc(unsigned port_num, enum gadget_type gtype) platform_driver_register(pdrv); - pr_debug("%s: port:%p portno:%d\n", __func__, port, port_num); + pr_debug("%s: port:%pK portno:%d\n", __func__, port, port_num); return 0; } @@ -834,14 +834,14 @@ int ghsic_data_connect(void *gptr, int port_num) ret = usb_ep_enable(port->in); if (ret) { - pr_err("%s: usb_ep_enable failed eptype:IN ep:%p", + pr_err("%s: usb_ep_enable failed eptype:IN ep:%pK", __func__, port->in); goto fail; } ret = usb_ep_enable(port->out); if (ret) { - pr_err("%s: usb_ep_enable failed eptype:OUT ep:%p", + pr_err("%s: usb_ep_enable failed eptype:OUT ep:%pK", __func__, port->out); usb_ep_disable(port->in); goto fail; @@ -917,7 +917,7 @@ static void dbg_timestamp(char *event, struct sk_buff * skb) write_lock_irqsave(&dbg_data.lck, flags); scnprintf(dbg_data.buf[dbg_data.idx], DBG_DATA_MSG, - "%p %u[%s] %u %u %u %u %u %u\n", + "%pK %u[%s] %u %u %u %u %u %u\n", skb, skb->len, event, info->created, info->rx_queued, info->rx_done, info->rx_done_sent, info->tx_queued, get_timestamp()); @@ -991,7 +991,7 @@ static ssize_t ghsic_data_read_stats(struct file *file, spin_lock_irqsave(&port->rx_lock, flags); temp += scnprintf(buf + temp, DEBUG_DATA_BUF_SIZE - temp, "\nName: %s\n" - "#PORT:%d port#: %p\n" + "#PORT:%d port#: %pK\n" "data_ch_open: %d\n" "data_ch_ready: %d\n" "\n******UL INFO*****\n\n" diff --git a/drivers/usb/gadget/u_ether.c b/drivers/usb/gadget/u_ether.c index 325c2b568e5..9e789c56967 100644 --- a/drivers/usb/gadget/u_ether.c +++ b/drivers/usb/gadget/u_ether.c @@ -553,6 +553,7 @@ static void tx_complete(struct usb_ep *ep, struct usb_request *req) } new_req->length = length; + new_req->complete = tx_complete; retval = usb_ep_queue(in, new_req, GFP_ATOMIC); switch (retval) { default: diff --git a/drivers/usb/host/ehci-msm2.c b/drivers/usb/host/ehci-msm2.c index 5bb87722664..e6e35e0f81e 100644 --- a/drivers/usb/host/ehci-msm2.c +++ b/drivers/usb/host/ehci-msm2.c @@ -937,7 +937,7 @@ static ssize_t debug_write_phy_data(struct file *file, const char __user *buf, memset(kbuf, 0, 10); - if (copy_from_user(kbuf, buf, count > 10 ? 10 : count)) + if (copy_from_user(kbuf, buf, min_t(size_t, sizeof(kbuf) - 1, count))) return -EFAULT; if (sscanf(kbuf, "%x", &data) != 1) @@ -962,7 +962,7 @@ static ssize_t debug_phy_write_addr(struct file *file, const char __user *buf, memset(kbuf, 0, 10); - if (copy_from_user(kbuf, buf, count > 10 ? 10 : count)) + if (copy_from_user(kbuf, buf, min_t(size_t, sizeof(kbuf) - 1, count))) return -EFAULT; if (sscanf(kbuf, "%x", &temp) != 1) diff --git a/drivers/usb/misc/iowarrior.c b/drivers/usb/misc/iowarrior.c index 4fd0dc835ae..f76b0bd4e2f 100644 --- a/drivers/usb/misc/iowarrior.c +++ b/drivers/usb/misc/iowarrior.c @@ -802,6 +802,13 @@ static int iowarrior_probe(struct usb_interface *interface, /* this one will match for the IOWarrior56 only */ dev->int_out_endpoint = endpoint; } + + if (!dev->int_in_endpoint) { + dev_err(&interface->dev, "no interrupt-in endpoint found\n"); + retval = -ENODEV; + goto error; + } + /* we have to check the report_size often, so remember it in the endianess suitable for our machine */ dev->report_size = usb_endpoint_maxp(dev->int_in_endpoint); if ((dev->interface->cur_altsetting->desc.bInterfaceNumber == 0) && diff --git a/drivers/usb/serial/cypress_m8.c b/drivers/usb/serial/cypress_m8.c index afc886c75d2..42f3a9e9954 100644 --- a/drivers/usb/serial/cypress_m8.c +++ b/drivers/usb/serial/cypress_m8.c @@ -453,6 +453,11 @@ static int generic_startup(struct usb_serial *serial) dbg("%s - port %d", __func__, port->number); + if (!port->interrupt_out_urb || !port->interrupt_in_urb) { + dev_err(&port->dev, "required endpoint is missing\n"); + return -ENODEV; + } + priv = kzalloc(sizeof(struct cypress_private), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -619,12 +624,6 @@ static int cypress_open(struct tty_struct *tty, struct usb_serial_port *port) cypress_set_termios(tty, port, &priv->tmp_termios); /* setup the port and start reading from the device */ - if (!port->interrupt_in_urb) { - dev_err(&port->dev, "%s - interrupt_in_urb is empty!\n", - __func__); - return -1; - } - usb_fill_int_urb(port->interrupt_in_urb, serial->dev, usb_rcvintpipe(serial->dev, port->interrupt_in_endpointAddress), port->interrupt_in_urb->transfer_buffer, diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c index 999f91bf70d..666c6738d92 100644 --- a/drivers/usb/serial/digi_acceleport.c +++ b/drivers/usb/serial/digi_acceleport.c @@ -1265,12 +1265,30 @@ static int digi_startup_device(struct usb_serial *serial) static int digi_startup(struct usb_serial *serial) { + struct device *dev = &serial->interface->dev; int i; struct digi_port *priv; struct digi_serial *serial_priv; dbg("digi_startup: TOP"); + /* check whether the device has the expected number of endpoints */ + if (serial->num_port_pointers < serial->type->num_ports + 1) { + dev_err(dev, "OOB endpoints missing\n"); + return -ENODEV; + } + + for (i = 0; i < serial->type->num_ports + 1 ; i++) { + if (!serial->port[i]->read_urb) { + dev_err(dev, "bulk-in endpoint missing\n"); + return -ENODEV; + } + if (!serial->port[i]->write_urb) { + dev_err(dev, "bulk-out endpoint missing\n"); + return -ENODEV; + } + } + /* allocate the private data structures for all ports */ /* number of regular ports + 1 for the out-of-band port */ for (i = 0; i < serial->type->num_ports + 1; i++) { diff --git a/drivers/usb/serial/mct_u232.c b/drivers/usb/serial/mct_u232.c index 6edd26130e2..5f2f1e1bb5b 100644 --- a/drivers/usb/serial/mct_u232.c +++ b/drivers/usb/serial/mct_u232.c @@ -400,6 +400,12 @@ static int mct_u232_startup(struct usb_serial *serial) struct mct_u232_private *priv; struct usb_serial_port *port, *rport; + /* check first to simplify error handling */ + if (!serial->port[1] || !serial->port[1]->interrupt_in_urb) { + dev_err(&port->dev, "expected endpoint missing\n"); + return -ENODEV; + } + priv = kzalloc(sizeof(struct mct_u232_private), GFP_KERNEL); if (!priv) return -ENOMEM; diff --git a/drivers/video/fbcmap.c b/drivers/video/fbcmap.c index 5c3960da755..71666c02dea 100644 --- a/drivers/video/fbcmap.c +++ b/drivers/video/fbcmap.c @@ -163,17 +163,18 @@ void fb_dealloc_cmap(struct fb_cmap *cmap) int fb_copy_cmap(const struct fb_cmap *from, struct fb_cmap *to) { - int tooff = 0, fromoff = 0; - int size; + unsigned int tooff = 0, fromoff = 0; + size_t size; if (to->start > from->start) fromoff = to->start - from->start; else tooff = from->start - to->start; - size = to->len - tooff; - if (size > (int) (from->len - fromoff)) - size = from->len - fromoff; - if (size <= 0) + if (fromoff >= from->len || tooff >= to->len) + return -EINVAL; + + size = min_t(size_t, to->len - tooff, from->len - fromoff); + if (size == 0) return -EINVAL; size *= sizeof(u16); @@ -187,17 +188,18 @@ int fb_copy_cmap(const struct fb_cmap *from, struct fb_cmap *to) int fb_cmap_to_user(const struct fb_cmap *from, struct fb_cmap_user *to) { - int tooff = 0, fromoff = 0; - int size; + unsigned int tooff = 0, fromoff = 0; + size_t size; if (to->start > from->start) fromoff = to->start - from->start; else tooff = from->start - to->start; - size = to->len - tooff; - if (size > (int) (from->len - fromoff)) - size = from->len - fromoff; - if (size <= 0) + if (fromoff >= from->len || tooff >= to->len) + return -EINVAL; + + size = min_t(size_t, to->len - tooff, from->len - fromoff); + if (size == 0) return -EINVAL; size *= sizeof(u16); diff --git a/drivers/video/msm/Kconfig b/drivers/video/msm/Kconfig index 590723a58d3..30ce98f1363 100644 --- a/drivers/video/msm/Kconfig +++ b/drivers/video/msm/Kconfig @@ -44,6 +44,11 @@ config FB_MSM_MDP_HW config FB_MSM_MDSS_COMMON bool +config MDP_DEBUG_FS + depends on DEBUG_FS + bool "MDP Debug FS" + default n + choice prompt "MDP HW version" default FB_MSM_MDP22 diff --git a/drivers/video/msm/Makefile b/drivers/video/msm/Makefile index 67c6b480a5a..d26fe580a2b 100644 --- a/drivers/video/msm/Makefile +++ b/drivers/video/msm/Makefile @@ -9,8 +9,7 @@ obj-$(CONFIG_FB_BACKLIGHT) += msm_fb_bl.o ifeq ($(CONFIG_FB_MSM_MDP_HW),y) # MDP obj-y += mdp.o - -obj-$(CONFIG_DEBUG_FS) += mdp_debugfs.o +obj-$(CONFIG_MDP_DEBUG_FS) += mdp_debugfs.o ifeq ($(CONFIG_FB_MSM_MDP40),y) obj-y += mdp4_util.o diff --git a/drivers/video/msm/mdp.c b/drivers/video/msm/mdp.c index 63a842d064d..7a59d510821 100644 --- a/drivers/video/msm/mdp.c +++ b/drivers/video/msm/mdp.c @@ -2,7 +2,7 @@ * * MSM MDP Interface (used by framebuffer core) * - * Copyright (c) 2007-2012, The Linux Foundation. All rights reserved. + * Copyright (c) 2007-2013, 2016 The Linux Foundation. All rights reserved. * Copyright (C) 2007 Google Incorporated * * This software is licensed under the terms of the GNU General Public @@ -493,6 +493,11 @@ static int mdp_lut_hw_update(struct fb_cmap *cmap) c[1] = cmap->blue; c[2] = cmap->red; + if (cmap->start > MDP_HIST_LUT_SIZE || cmap->len > MDP_HIST_LUT_SIZE || + (cmap->start + cmap->len > MDP_HIST_LUT_SIZE)) { + pr_err("mdp_lut_hw_update invalid arguments\n"); + return -EINVAL; + } for (i = 0; i < cmap->len; i++) { if (copy_from_user(&r, cmap->red++, sizeof(r)) || copy_from_user(&g, cmap->green++, sizeof(g)) || @@ -3252,7 +3257,7 @@ static int __init mdp_driver_init(void) return ret; } -#if defined(CONFIG_DEBUG_FS) +#if defined(CONFIG_MDP_DEBUG_FS) mdp_debugfs_init(); #endif diff --git a/drivers/video/msm/mdp4_util.c b/drivers/video/msm/mdp4_util.c index f8b7f2fc174..cfcccdbabae 100644 --- a/drivers/video/msm/mdp4_util.c +++ b/drivers/video/msm/mdp4_util.c @@ -2739,19 +2739,42 @@ static int mdp4_argc_process_write_req(uint32_t *offset, struct mdp_ar_gc_lut_data r[MDP_AR_GC_MAX_STAGES]; struct mdp_ar_gc_lut_data g[MDP_AR_GC_MAX_STAGES]; struct mdp_ar_gc_lut_data b[MDP_AR_GC_MAX_STAGES]; + uint8_t num_r_stages; + uint8_t num_g_stages; + uint8_t num_b_stages; + + if (get_user(num_r_stages, &pgc_ptr->num_r_stages)) { + pr_err("%s failed: num_r_stages : Invalid arg\n", __func__); + return -EFAULT; + } + + if (get_user(num_g_stages, &pgc_ptr->num_g_stages)) { + pr_err("%s failed: num_g_stages : Invalid arg\n", __func__); + return -EFAULT; + } + + if (get_user(num_b_stages, &pgc_ptr->num_b_stages)) { + pr_err("%s failed: num_b_stages : Invalid arg\n", __func__); + return -EFAULT; + } + + if ((!num_r_stages || num_r_stages > MDP_AR_GC_MAX_STAGES) || + (!num_g_stages || num_g_stages > MDP_AR_GC_MAX_STAGES) || + (!num_b_stages || num_b_stages > MDP_AR_GC_MAX_STAGES)) + return -EINVAL; ret = copy_from_user(&r[0], pgc_ptr->r_data, - pgc_ptr->num_r_stages * sizeof(struct mdp_ar_gc_lut_data)); + num_r_stages * sizeof(struct mdp_ar_gc_lut_data)); if (!ret) { ret = copy_from_user(&g[0], pgc_ptr->g_data, - pgc_ptr->num_g_stages + num_g_stages * sizeof(struct mdp_ar_gc_lut_data)); if (!ret) ret = copy_from_user(&b[0], pgc_ptr->b_data, - pgc_ptr->num_b_stages + num_b_stages * sizeof(struct mdp_ar_gc_lut_data)); } diff --git a/drivers/video/msm/mdp_debugfs.c b/drivers/video/msm/mdp_debugfs.c index d3e0c8ddd63..82acfa1f231 100644 --- a/drivers/video/msm/mdp_debugfs.c +++ b/drivers/video/msm/mdp_debugfs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2012, The Linux Foundation. All rights reserved. +/* Copyright (c) 2009-2012, 2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -39,6 +39,9 @@ #endif #define MDP_DEBUG_BUF 2048 +#define MDP_MAX_OFFSET 0xF05FC +#define MDDI_MAX_OFFSET 0xC +#define HDMI_MAX_OFFSET 0x59C static uint32 mdp_offset; static uint32 mdp_count; @@ -78,11 +81,18 @@ static ssize_t mdp_offset_write( debug_buf[count] = 0; /* end of string */ - sscanf(debug_buf, "%x %d", &off, &cnt); + if (sscanf(debug_buf, "%x %d", &off, &cnt) != 2) + return -EFAULT; if (cnt <= 0) cnt = 1; + if ((off > MDP_MAX_OFFSET) || (cnt > (MDP_MAX_OFFSET - off))) { + printk(KERN_INFO "%s: Invalid offset%x+cnt%d > %x\n", __func__, + off, cnt, MDP_MAX_OFFSET); + return -EFAULT; + } + mdp_offset = off; mdp_count = cnt; @@ -154,6 +164,14 @@ static ssize_t mdp_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDP_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset%x > %x\n", __func__, + off, MDP_MAX_OFFSET); + return -EFAULT; + } mdp_pipe_ctrl(MDP_CMD_BLOCK, MDP_BLOCK_POWER_ON, FALSE); outpdw(MDP_BASE + off, data); @@ -620,6 +638,17 @@ static void mddi_reg_write(int ndx, uint32 off, uint32 data) else base = (char *)msm_pmdh_base; + if (base == NULL) { + printk(KERN_INFO "%s: base offset is not set properly. \ + Please check if MDDI is enabled correctly\n", __func__); + return; + } + + if (off > MDDI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset=%x > %x\n", __func__, + off, MDDI_MAX_OFFSET); + return; + } mdp_pipe_ctrl(MDP_CMD_BLOCK, MDP_BLOCK_POWER_ON, FALSE); writel(data, base + off); mdp_pipe_ctrl(MDP_CMD_BLOCK, MDP_BLOCK_POWER_OFF, FALSE); @@ -682,6 +711,14 @@ static ssize_t pmdh_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDDI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset=%x > %x\n", __func__, + off, MDDI_MAX_OFFSET); + return -EFAULT; + } mddi_reg_write(0, off, data); @@ -737,6 +774,14 @@ static ssize_t emdh_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDDI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset=%x > %x\n", __func__, + off, MDDI_MAX_OFFSET); + return -EFAULT; + } mddi_reg_write(1, off, data); @@ -884,15 +929,18 @@ static ssize_t dbg_offset_write( cnt = sscanf(debug_buf, "%x %d %x", &off, &num, &base); - if (cnt < 0) - cnt = 0; + if (cnt != 3) + return -EFAULT; + + if ((off > MDP_MAX_OFFSET) || (num > (MDP_MAX_OFFSET - off))) { + printk(KERN_INFO "%s: Invalid offset%x+num%d > %x\n", __func__, + off, num, MDP_MAX_OFFSET); + return -EFAULT; + } - if (cnt >= 1) - dbg_offset = off; - if (cnt >= 2) - dbg_count = num; - if (cnt >= 3) - dbg_base = (char *)base; + dbg_offset = off; + dbg_count = num; + dbg_base = (char *)base; printk(KERN_INFO "%s: offset=%x cnt=%d base=%x\n", __func__, dbg_offset, dbg_count, (int)dbg_base); @@ -951,6 +999,14 @@ static ssize_t dbg_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDP_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset%x > %x\n", __func__, + off, MDP_MAX_OFFSET); + return -EFAULT; + } writel(data, dbg_base + off); @@ -1073,6 +1129,8 @@ static ssize_t dbg_force_ov0_blt_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x", &dbg_force_ov0_blt); + if (cnt != 1) + return -EFAULT; pr_info("%s: dbg_force_ov0_blt = %x\n", __func__, dbg_force_ov0_blt); @@ -1136,6 +1194,8 @@ static ssize_t dbg_force_ov1_blt_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x", &dbg_force_ov1_blt); + if (cnt != 1) + return -EFAULT; pr_info("%s: dbg_force_ov1_blt = %x\n", __func__, dbg_force_ov1_blt); @@ -1191,14 +1251,17 @@ static ssize_t hdmi_offset_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %d", &off, &num); + if (cnt != 2) + return -EFAULT; - if (cnt < 0) - cnt = 0; + if ((off > HDMI_MAX_OFFSET) || (num > (HDMI_MAX_OFFSET - off))) { + printk(KERN_INFO "%s: Invalid offset%x+num%d > %x\n", __func__, + off, num, HDMI_MAX_OFFSET); + return -EFAULT; + } - if (cnt >= 1) - hdmi_offset = off; - if (cnt >= 2) - hdmi_count = num; + hdmi_offset = off; + hdmi_count = num; printk(KERN_INFO "%s: offset=%x cnt=%d\n", __func__, hdmi_offset, hdmi_count); @@ -1262,6 +1325,15 @@ static ssize_t hdmi_reg_write( cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > HDMI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset%x > %x\n", __func__, + off, HDMI_MAX_OFFSET); + return -EFAULT; + } + writel(data, base + off); printk(KERN_INFO "%s: addr=%x data=%x\n", @@ -1355,14 +1427,14 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("off", 0644, dent, 0, &mdp_off_fops) + if (debugfs_create_file("off", 0600, dent, 0, &mdp_off_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: index fail\n", __FILE__, __LINE__); return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &mdp_reg_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &mdp_reg_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1402,7 +1474,7 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &pmdh_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &pmdh_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1417,7 +1489,7 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &emdh_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &emdh_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1432,21 +1504,21 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("base", 0644, dent, 0, &dbg_base_fops) + if (debugfs_create_file("base", 0600, dent, 0, &dbg_base_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: index fail\n", __FILE__, __LINE__); return -1; } - if (debugfs_create_file("off", 0644, dent, 0, &dbg_off_fops) + if (debugfs_create_file("off", 0600, dent, 0, &dbg_off_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: index fail\n", __FILE__, __LINE__); return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &dbg_reg_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &dbg_reg_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1462,14 +1534,14 @@ int mdp_debugfs_init(void) return PTR_ERR(dent); } - if (debugfs_create_file("off", 0644, dent, 0, &hdmi_off_fops) + if (debugfs_create_file("off", 0600, dent, 0, &hdmi_off_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: 'off' fail\n", __FILE__, __LINE__); return -ENOENT; } - if (debugfs_create_file("reg", 0644, dent, 0, &hdmi_reg_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &hdmi_reg_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: 'reg' fail\n", __FILE__, __LINE__); diff --git a/drivers/video/msm/mdss/mdp3.c b/drivers/video/msm/mdss/mdp3.c index 66418db5cf4..4baeef87018 100644 --- a/drivers/video/msm/mdss/mdp3.c +++ b/drivers/video/msm/mdss/mdp3.c @@ -769,7 +769,7 @@ static int mdp3_res_init(void) mdp3_res->ion_client = msm_ion_client_create(-1, mdp3_res->pdev->name); if (IS_ERR_OR_NULL(mdp3_res->ion_client)) { - pr_err("msm_ion_client_create() return error (%p)\n", + pr_err("msm_ion_client_create() return error (%pK)\n", mdp3_res->ion_client); mdp3_res->ion_client = NULL; return -EINVAL; @@ -921,7 +921,7 @@ int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) data->addr += img->offset; data->len -= img->offset; - pr_debug("mem=%d ihdl=%p buf=0x%x len=0x%x\n", img->memory_id, + pr_debug("mem=%d ihdl=%pK buf=0x%x len=0x%x\n", img->memory_id, data->srcp_ihdl, data->addr, data->len); } else { mdp3_put_img(data); @@ -1034,7 +1034,7 @@ static int mdp3_fbmem_alloc(struct msm_fb_data_type *mfd) goto ion_map_phys_err; } - pr_info("allocating %u bytes at %p (%lx phys) for fb %d\n", + pr_info("allocating %u bytes at %pK (%lx phys) for fb %d\n", size, virt, phys, mfd->index); mfd->fbi->screen_base = virt; diff --git a/drivers/video/msm/mdss/mdp3_ctrl.c b/drivers/video/msm/mdss/mdp3_ctrl.c index 1676a174187..7e39ac8d056 100644 --- a/drivers/video/msm/mdss/mdp3_ctrl.c +++ b/drivers/video/msm/mdss/mdp3_ctrl.c @@ -754,6 +754,37 @@ static int mdp3_get_metadata(struct msm_fb_data_type *mfd, return ret; } +int mdp3_validate_start_req(struct mdp_histogram_start_req *req) +{ + if (req->frame_cnt >= MDP_HISTOGRAM_FRAME_COUNT_MAX) { + pr_err("%s invalid req frame_cnt\n", __func__); + return -EINVAL; + } + if (req->bit_mask >= MDP_HISTOGRAM_BIT_MASK_MAX) { + pr_err("%s invalid req bit mask\n", __func__); + return -EINVAL; + } + if (req->block != MDP_BLOCK_DMA_P || + req->num_bins != MDP_HISTOGRAM_BIN_NUM) { + pr_err("mdp3_histogram_start invalid request\n"); + return -EINVAL; + } + return 0; +} + +int mdp3_validate_scale_config(struct mdp_bl_scale_data *data) +{ + if (data->scale > MDP_HISTOGRAM_BL_SCALE_MAX) { + pr_err("%s invalid bl_scale\n", __func__); + return -EINVAL; + } + if (data->min_lvl > MDP_HISTOGRAM_BL_LEVEL_MAX) { + pr_err("%s invalid bl_min_lvl\n", __func__); + return -EINVAL; + } + return 0; +} + static int mdp3_histogram_start(struct mdp3_session_data *session, struct mdp_histogram_start_req *req) { @@ -761,11 +792,10 @@ static int mdp3_histogram_start(struct mdp3_session_data *session, struct mdp3_dma_histogram_config histo_config; pr_debug("mdp3_histogram_start\n"); - if (req->block != MDP_BLOCK_DMA_P || - req->num_bins != MDP_HISTOGRAM_BIN_NUM) { - pr_err("mdp3_histogram_start invalid request\n"); - return -EINVAL; - } + + ret = mdp3_validate_start_req(req); + if (ret) + return ret; if (!session->dma->histo_op || !session->dma->config_histo) { @@ -923,6 +953,11 @@ static int mdp3_pp_ioctl(struct msm_fb_data_type *mfd, switch (mdp_pp.op) { case mdp_bl_scale_cfg: + ret = mdp3_validate_scale_config(&mdp_pp.data.bl_scale_data); + if (ret) { + pr_err("%s: invalid scale config\n", __func__); + break; + } ret = mdp3_bl_scale_config(mfd, (struct mdp_bl_scale_data *) &mdp_pp.data.bl_scale_data); break; @@ -996,7 +1031,8 @@ static int mdp3_ctrl_lut_update(struct msm_fb_data_type *mfd, if (!mdp3_session->dma->config_lut) return -EINVAL; - if (cmap->start + cmap->len > MDP_LUT_SIZE) { + if (cmap->start > MDP_LUT_SIZE || cmap->len > MDP_LUT_SIZE || + (cmap->start + cmap->len > MDP_LUT_SIZE)) { pr_err("mdp3_ctrl_lut_update invalid arguments\n"); return -EINVAL; } diff --git a/drivers/video/msm/mdss/mdp3_dma.c b/drivers/video/msm/mdss/mdp3_dma.c index 3b6540533ed..4e206577ea3 100644 --- a/drivers/video/msm/mdss/mdp3_dma.c +++ b/drivers/video/msm/mdss/mdp3_dma.c @@ -488,7 +488,7 @@ static int mdp3_dmap_histo_config(struct mdp3_dma *dma, struct mdp3_dma_histogram_config *histo_config) { unsigned long flag; - u32 histo_bit_mask, histo_control; + u32 histo_bit_mask = 0, histo_control = 0; u32 histo_isr_mask = MDP3_DMA_P_HIST_INTR_HIST_DONE_BIT | MDP3_DMA_P_HIST_INTR_RESET_DONE_BIT; diff --git a/drivers/video/msm/mdss/mdp3_dma.h b/drivers/video/msm/mdss/mdp3_dma.h index 518bd581b3b..c603636a822 100644 --- a/drivers/video/msm/mdss/mdp3_dma.h +++ b/drivers/video/msm/mdss/mdp3_dma.h @@ -16,6 +16,10 @@ #include +#define MDP_HISTOGRAM_BL_SCALE_MAX 1024 +#define MDP_HISTOGRAM_BL_LEVEL_MAX 255 +#define MDP_HISTOGRAM_FRAME_COUNT_MAX 0x20 +#define MDP_HISTOGRAM_BIT_MASK_MAX 0x4 #define MDP_HISTOGRAM_BIN_NUM 32 #define MDP_LUT_SIZE 256 diff --git a/drivers/video/msm/mdss/mdss_debug.c b/drivers/video/msm/mdss/mdss_debug.c index 13fba269852..b98740bb386 100644 --- a/drivers/video/msm/mdss/mdss_debug.c +++ b/drivers/video/msm/mdss/mdss_debug.c @@ -104,7 +104,7 @@ static ssize_t mdss_debug_base_offset_read(struct file *file, { struct mdss_debug_base *dbg = file->private_data; int len = 0; - char buf[24]; + char buf[24] = {'\0'}; if (!dbg) return -ENODEV; @@ -113,10 +113,10 @@ static ssize_t mdss_debug_base_offset_read(struct file *file, return 0; /* the end */ len = snprintf(buf, sizeof(buf), "0x%08x %x\n", dbg->off, dbg->cnt); - if (len < 0) + if (len < 0 || len >= sizeof(buf)) return 0; - if (copy_to_user(buff, buf, len)) + if ((count < sizeof(buf)) || copy_to_user(buff, buf, len)) return -EFAULT; *ppos += len; /* increase offset */ diff --git a/drivers/video/msm/mdss/mdss_dsi.c b/drivers/video/msm/mdss/mdss_dsi.c index e21269fca41..52660967a16 100644 --- a/drivers/video/msm/mdss/mdss_dsi.c +++ b/drivers/video/msm/mdss/mdss_dsi.c @@ -430,7 +430,7 @@ static int mdss_dsi_off(struct mdss_panel_data *pdata) ctrl_pdata = container_of(pdata, struct mdss_dsi_ctrl_pdata, panel_data); - pr_debug("%s+: ctrl=%p ndx=%d\n", __func__, + pr_debug("%s+: ctrl=%pK ndx=%d\n", __func__, ctrl_pdata, ctrl_pdata->ndx); if (pdata->panel_info.type == MIPI_CMD_PANEL) @@ -489,7 +489,7 @@ int mdss_dsi_on(struct mdss_panel_data *pdata) ctrl_pdata = container_of(pdata, struct mdss_dsi_ctrl_pdata, panel_data); - pr_debug("%s+: ctrl=%p ndx=%d\n", + pr_debug("%s+: ctrl=%pK ndx=%d\n", __func__, ctrl_pdata, ctrl_pdata->ndx); pinfo = &pdata->panel_info; @@ -694,7 +694,7 @@ int mdss_dsi_cont_splash_on(struct mdss_panel_data *pdata) ctrl_pdata = container_of(pdata, struct mdss_dsi_ctrl_pdata, panel_data); - pr_debug("%s+: ctrl=%p ndx=%d\n", __func__, + pr_debug("%s+: ctrl=%pK ndx=%d\n", __func__, ctrl_pdata, ctrl_pdata->ndx); WARN(!(ctrl_pdata->ctrl_state & CTRL_STATE_PANEL_INIT), diff --git a/drivers/video/msm/mdss/mdss_dsi_host.c b/drivers/video/msm/mdss/mdss_dsi_host.c index 3ce22e36862..9b440f444ba 100644 --- a/drivers/video/msm/mdss/mdss_dsi_host.c +++ b/drivers/video/msm/mdss/mdss_dsi_host.c @@ -68,7 +68,7 @@ void mdss_dsi_ctrl_init(struct mdss_dsi_ctrl_pdata *ctrl) if (mdss_register_irq(ctrl->dsi_hw)) pr_err("%s: mdss_register_irq failed.\n", __func__); - pr_debug("%s: ndx=%d base=%p\n", __func__, ctrl->ndx, ctrl->ctrl_base); + pr_debug("%s: ndx=%d base=%pK\n", __func__, ctrl->ndx, ctrl->ctrl_base); init_completion(&ctrl->dma_comp); init_completion(&ctrl->mdp_comp); diff --git a/drivers/video/msm/mdss/mdss_dsi_panel.c b/drivers/video/msm/mdss/mdss_dsi_panel.c index 5f80d3c9437..463be32895e 100644 --- a/drivers/video/msm/mdss/mdss_dsi_panel.c +++ b/drivers/video/msm/mdss/mdss_dsi_panel.c @@ -21,10 +21,6 @@ #include #include #include -#ifdef CONFIG_DEBUG_FS -#include -#include -#endif #include "mdss_dsi.h" @@ -286,7 +282,7 @@ static int mdss_dsi_panel_on(struct mdss_panel_data *pdata) panel_data); mipi = &pdata->panel_info.mipi; - pr_debug("%s: ctrl=%p ndx=%d\n", __func__, ctrl, ctrl->ndx); + pr_debug("%s: ctrl=%pK ndx=%d\n", __func__, ctrl, ctrl->ndx); if (ctrl->on_cmds.cmd_cnt) mdss_dsi_panel_cmds_send(ctrl, &ctrl->on_cmds); @@ -308,7 +304,7 @@ static int mdss_dsi_panel_off(struct mdss_panel_data *pdata) ctrl = container_of(pdata, struct mdss_dsi_ctrl_pdata, panel_data); - pr_debug("%s: ctrl=%p ndx=%d\n", __func__, ctrl, ctrl->ndx); + pr_debug("%s: ctrl=%pK ndx=%d\n", __func__, ctrl, ctrl->ndx); mipi = &pdata->panel_info.mipi; @@ -750,317 +746,6 @@ static int mdss_panel_parse_dt(struct platform_device *pdev, return -EINVAL; } -#ifdef CONFIG_DEBUG_FS -#define MAX_ON_CMD_SIZE (PAGE_SIZE * 8) - -ssize_t on_cmd_read(struct file *filp, char __user *user_buf, size_t count, - loff_t *ppos) -{ - int ret; - int i, j; - char *buf; - int buf_size; - struct dsi_panel_cmds *pcmds; - - struct mdss_panel_common_pdata *panel_data = filp->private_data; - if (!panel_data) - return -ENODEV; - - pcmds = &panel_data->on_cmds; - if (!pcmds) - return -EINVAL; - - buf = kzalloc(MAX_ON_CMD_SIZE, GFP_KERNEL); - - strncpy(buf, "qcom,panel-on-cmds = [", 22); - buf_size = 22; - - for (i = 0; i < pcmds->cmd_cnt; ++i) { - if (buf_size + 20 + - (pcmds->cmds[i].dchdr.dlen * 3) > - MAX_ON_CMD_SIZE - 4) { - pr_warn("Too many on_commands!(< 32KB)\n"); - ret = -EINVAL; - goto read_error; - } - - strncpy(&buf[buf_size], "\r\n", 2); - buf_size += 2; - - ret = snprintf(&buf[buf_size], 21, - "%02x %02x %02x %02x %02x %02x %02x", - pcmds->cmds[i].dchdr.dtype, - pcmds->cmds[i].dchdr.last, - pcmds->cmds[i].dchdr.vc, - pcmds->cmds[i].dchdr.ack, - pcmds->cmds[i].dchdr.wait, - (char)(pcmds->cmds[i].dchdr.dlen & 0xff00) >> 8, - (char)(pcmds->cmds[i].dchdr.dlen & 0x00ff)); - - if (ret < 0) - goto read_error; - - buf_size += ret; - - if (0 < pcmds->cmds[i].dchdr.dlen && pcmds->cmds[i].dchdr.dlen < 3) { - ret = snprintf(&buf[buf_size], 4, " %02x", - pcmds->cmds[i].payload[0]); - if (ret < 0) - goto read_error; - - buf_size += ret; - - if (pcmds->cmds[i].dchdr.dlen == 2) { - ret = snprintf(&buf[buf_size], 4, " %02x", - pcmds->cmds[i].payload[1]); - if (ret < 0) - goto read_error; - - buf_size += ret; - } - } else if (pcmds->cmds[i].dchdr.dlen > 2) { - for (j = 0; j < pcmds->cmds[i].dchdr.dlen; ++j) { - if ((j % 6) == 0) { - ret = snprintf(&buf[buf_size], 5, "\r\n%02x", - pcmds->cmds[i].payload[j]); - if (ret < 0) - goto read_error; - - buf_size += ret; - } else { - ret = snprintf(&buf[buf_size], 4, " %02x", - pcmds->cmds[i].payload[j]); - if (ret < 0) - goto read_error; - - buf_size += ret; - } - } - } else { - pr_err("Invalid data length!\n"); - ret = -EINVAL; - goto read_error; - } - } - - strncpy(&buf[buf_size], "]", 2); - buf_size += 2; - - ret = simple_read_from_buffer(user_buf, count, ppos, buf, buf_size); -read_error: - kfree(buf); - - return ret; -} - -static int parse_on_cmds(char **on_cmds, const char *buf, size_t count) -{ - int i, j; - int on_cmds_len; - char *endptr; - - i = 0; - do { - if (!strncmp(&buf[i], "qcom,panel-on-cmds", 18)) { - i += 18; - while (i < count && buf[i] != '=') - ++i; - while (i < count && buf[i] != '[') - ++i; - ++i; - break; - } - } while (++i < count); - - if (i >= count) { - pr_err("Invalid on_cmds start format!\n"); - return -EINVAL; - } - - on_cmds_len = 0; - - for (j = i; j < count; ++j) { - if (isxdigit(buf[j]) && isxdigit(buf[j + 1])) { - ++on_cmds_len; - ++j; - } else if (buf[j] == ']') - break; - } - - if (j == count) - return 0; - - if (buf[j] != ']') { - pr_err("Invalid on_cmds end format!\n"); - return -EINVAL; - } - - *on_cmds = kzalloc(on_cmds_len, GFP_KERNEL); - - if (!(*on_cmds)) - return -ENOMEM; - - j = 0; - for (i = 0; i < count; ++i) { - if (isxdigit(buf[i]) && isxdigit(buf[i + 1])) { - endptr = (char *)&buf[i + 1]; - (*on_cmds)[j] = (char)simple_strtoul(&buf[i], &endptr, 16); - ++i; - ++j; - } - } - - return on_cmds_len; -} - -static char *user_buf_total = NULL; -static int user_buf_total_pos = 0; - -ssize_t on_cmd_write(struct file *filp, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - static char *buf = NULL; - char *prev_on_cmds = NULL; - int blen = 0, len; - char *bp; - struct dsi_ctrl_hdr *dchdr; - int i, cnt; - int ret; - struct dsi_panel_cmds *pcmds; - - struct mdss_panel_common_pdata *panel_data = filp->private_data; - if (!panel_data) - return -ENODEV; - - pcmds = &panel_data->on_cmds; - if (!pcmds) - return -EINVAL; - - prev_on_cmds = buf; - - if (user_buf_total) { - if (user_buf_total_pos + count < MAX_ON_CMD_SIZE) { - memcpy(&user_buf_total[user_buf_total_pos], user_buf, count); - user_buf_total_pos += count; - blen = parse_on_cmds(&buf, user_buf_total, - user_buf_total_pos); - } else { - pr_warn("Too large file size(< 32KB)!\n"); - ret = -EINVAL; - goto write_error; - } - } else { - blen = parse_on_cmds(&buf, user_buf, count); - } - - if (blen == 0) { - if (!user_buf_total) { - user_buf_total = kzalloc(MAX_ON_CMD_SIZE, GFP_KERNEL); - if (!user_buf_total) { - ret = -ENOMEM; - goto write_error; - } - memcpy(user_buf_total, user_buf, count); - user_buf_total_pos += count; - } - return count; - } else if (blen < 0) { - pr_err("Invalid on_cmd format or length!\n"); - ret = -EINVAL; - goto write_error; - } - - /* scan dcs commands */ - bp = buf; - len = blen; - cnt = 0; - while (len > sizeof(*dchdr)) { - dchdr = (struct dsi_ctrl_hdr *)bp; - dchdr->dlen = ntohs(dchdr->dlen); - if (dchdr->dlen > len) { - pr_err("%s: dtsi cmd=%x error, len=%d", - __func__, dchdr->dtype, dchdr->dlen); - return -ENOMEM; - } - bp += sizeof(*dchdr); - len -= sizeof(*dchdr); - bp += dchdr->dlen; - len -= dchdr->dlen; - cnt++; - } - - if (len != 0) { - pr_err("%s: dcs_cmd=%x len=%d error!", - __func__, buf[0], blen); - kfree(buf); - ret = -ENOMEM; - goto write_error; - } - - kfree(pcmds->cmds); - - pcmds->cmds = kzalloc(cnt * sizeof(struct dsi_cmd_desc), - GFP_KERNEL); - if (!pcmds->cmds) { - if (buf) { - kfree(buf); - buf = prev_on_cmds; - } - ret = -ENOMEM; - goto write_error; - } - - pcmds->cmd_cnt = cnt; - pcmds->buf = buf; - pcmds->blen = blen; - - bp = buf; - len = blen; - for (i = 0; i < cnt; i++) { - dchdr = (struct dsi_ctrl_hdr *)bp; - len -= sizeof(*dchdr); - bp += sizeof(*dchdr); - pcmds->cmds[i].dchdr = *dchdr; - pcmds->cmds[i].payload = bp; - bp += dchdr->dlen; - len -= dchdr->dlen; - } - - if (prev_on_cmds) - kfree(prev_on_cmds); - - ret = count; -write_error: - if (user_buf_total) { - kfree(user_buf_total); - user_buf_total = NULL; - user_buf_total_pos = 0; - } - - return ret; -} - -static const struct file_operations on_cmd_fops = { - .open = simple_open, - .read = on_cmd_read, - .write = on_cmd_write, -}; - -static int debug_fs_init(struct mdss_panel_common_pdata *panel_data) -{ - struct dentry *dsi_base; - - dsi_base = debugfs_create_dir("mdss_dsi", NULL); - if (!dsi_base) - return -ENOMEM; - - debugfs_create_file("on_cmd", S_IRUSR | S_IRGRP | S_IWUSR, dsi_base, - panel_data, &on_cmd_fops); - - return 0; -} -#endif - static int __devinit mdss_dsi_panel_probe(struct platform_device *pdev) { int rc = 0; @@ -1090,10 +775,6 @@ static int __devinit mdss_dsi_panel_probe(struct platform_device *pdev) if (rc) return rc; -#ifdef CONFIG_DEBUG_FS - debug_fs_init(&vendor_pdata); -#endif - return 0; } diff --git a/drivers/video/msm/mdss/mdss_fb.c b/drivers/video/msm/mdss/mdss_fb.c index 0cbef4f49a6..dfa95a8950f 100644 --- a/drivers/video/msm/mdss/mdss_fb.c +++ b/drivers/video/msm/mdss/mdss_fb.c @@ -56,6 +56,7 @@ #endif #include "mdss_fb.h" +#include "mdss_mdp.h" #ifdef CONFIG_LGE_HANDLE_PANIC #include @@ -254,12 +255,77 @@ static ssize_t mdss_fb_get_split(struct device *dev, return ret; } +static int pcc_r = 32768, pcc_g = 32768, pcc_b = 32768; +static ssize_t mdss_get_rgb(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d %d %d\n", pcc_r, pcc_g, pcc_b); +} + +/** + * simple color temperature interface using polynomial color correction + * + * input values are r/g/b adjustments from 0-32768 representing 0 -> 1 + * + * example adjustment @ 3500K: + * 1.0000 / 0.5515 / 0.2520 = 32768 / 25828 / 17347 + * + * reference chart: + * http://www.vendian.org/mncharity/dir3/blackbody/UnstableURLs/bbr_color.html + */ +static ssize_t mdss_set_rgb(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + uint32_t r = 0, g = 0, b = 0; + struct mdp_pcc_cfg_data pcc_cfg; + u32 copyback = 0; + + if (count > 19) + return -EINVAL; + + sscanf(buf, "%d %d %d", &r, &g, &b); + + if (r < 0 || r > 32768) + return -EINVAL; + if (g < 0 || g > 32768) + return -EINVAL; + if (b < 0 || b > 32768) + return -EINVAL; + + pr_info("%s: r=%d g=%d b=%d\n", __func__, r, g, b); + + memset(&pcc_cfg, 0, sizeof(struct mdp_pcc_cfg_data)); + + pcc_cfg.block = MDP_LOGICAL_BLOCK_DISP_0; + if (r == 32768 && g == 32768 && b == 32768) + pcc_cfg.ops = MDP_PP_OPS_DISABLE; + else + pcc_cfg.ops = MDP_PP_OPS_ENABLE; + pcc_cfg.ops |= MDP_PP_OPS_WRITE; + pcc_cfg.r.r = r; + pcc_cfg.g.g = g; + pcc_cfg.b.b = b; + + if (mdss_mdp_pcc_config(&pcc_cfg, ©back) == 0) { + pcc_r = r; + pcc_g = g; + pcc_b = b; + return count; + } + + return -EINVAL; +} + static DEVICE_ATTR(msm_fb_type, S_IRUGO, mdss_fb_get_type, NULL); static DEVICE_ATTR(msm_fb_split, S_IRUGO, mdss_fb_get_split, NULL); +static DEVICE_ATTR(rgb, S_IRUGO | S_IWUSR | S_IWGRP, mdss_get_rgb, + mdss_set_rgb); static struct attribute *mdss_fb_attrs[] = { &dev_attr_msm_fb_type.attr, &dev_attr_msm_fb_split.attr, + &dev_attr_rgb.attr, NULL, }; @@ -884,7 +950,7 @@ static int mdss_fb_alloc_fbmem_iommu(struct msm_fb_data_type *mfd, int dom) msm_iommu_map_contig_buffer(phys, dom, 0, size, SZ_4K, 0, &mfd->iova); - pr_info("allocating %u bytes at %p (%lx phys) for fb %d\n", + pr_info("allocating %u bytes at %pK (%lx phys) for fb %d\n", size, virt, phys, mfd->index); #ifdef CONFIG_LGE_HANDLE_PANIC diff --git a/drivers/video/msm/mdss/mdss_hdmi_tx.c b/drivers/video/msm/mdss/mdss_hdmi_tx.c index 08d64883d84..c6c70d1a4c3 100644 --- a/drivers/video/msm/mdss/mdss_hdmi_tx.c +++ b/drivers/video/msm/mdss/mdss_hdmi_tx.c @@ -663,7 +663,7 @@ static int hdmi_tx_sysfs_create(struct hdmi_tx_ctrl *hdmi_ctrl, return rc; } hdmi_ctrl->kobj = &fbi->dev->kobj; - DEV_DBG("%s: sysfs group %p\n", __func__, hdmi_ctrl->kobj); + DEV_DBG("%s: sysfs group %pK\n", __func__, hdmi_ctrl->kobj); return 0; } /* hdmi_tx_sysfs_create */ diff --git a/drivers/video/msm/mdss/mdss_hdmi_util.c b/drivers/video/msm/mdss/mdss_hdmi_util.c index 711ec687933..4cf8214bc8a 100644 --- a/drivers/video/msm/mdss/mdss_hdmi_util.c +++ b/drivers/video/msm/mdss/mdss_hdmi_util.c @@ -162,7 +162,7 @@ static void hdmi_ddc_print_data(struct hdmi_tx_ddc_data *ddc_data, return; } - DEV_DBG("%s: buf=%p, d_len=0x%x, d_addr=0x%x, no_align=%d\n", + DEV_DBG("%s: buf=%pK, d_len=0x%x, d_addr=0x%x, no_align=%d\n", caller, ddc_data->data_buf, ddc_data->data_len, ddc_data->dev_addr, ddc_data->no_align); DEV_DBG("%s: offset=0x%x, req_len=0x%x, retry=%d, what=%s\n", diff --git a/drivers/video/msm/mdss/mdss_mdp.c b/drivers/video/msm/mdss/mdss_mdp.c index 53944bcfccc..8bb316f3b45 100644 --- a/drivers/video/msm/mdss/mdss_mdp.c +++ b/drivers/video/msm/mdss/mdss_mdp.c @@ -222,7 +222,7 @@ int mdss_register_irq(struct mdss_hw *hw) if (!mdss_irq_handlers[hw->hw_ndx]) mdss_irq_handlers[hw->hw_ndx] = hw; else - pr_err("panel %d's irq at %p is already registered\n", + pr_err("panel %d's irq at %pK is already registered\n", hw->hw_ndx, hw->irq_handler); spin_unlock_irqrestore(&mdss_lock, irq_flags); @@ -934,7 +934,7 @@ static u32 mdss_mdp_res_init(struct mdss_data_type *mdata) mdata->iclient = msm_ion_client_create(-1, mdata->pdev->name); if (IS_ERR_OR_NULL(mdata->iclient)) { - pr_err("msm_ion_client_create() return error (%p)\n", + pr_err("msm_ion_client_create() return error (%pK)\n", mdata->iclient); mdata->iclient = NULL; } diff --git a/drivers/video/msm/mdss/mdss_mdp.h b/drivers/video/msm/mdss/mdss_mdp.h index edfcc54ad04..055e050f140 100644 --- a/drivers/video/msm/mdss/mdss_mdp.h +++ b/drivers/video/msm/mdss/mdss_mdp.h @@ -247,6 +247,7 @@ struct mdss_mdp_img_data { u32 len; u32 flags; int p_need; + bool mapped; struct file *srcp_file; struct ion_handle *srcp_ihdl; }; @@ -506,33 +507,18 @@ int mdss_mdp_smp_setup(struct mdss_data_type *mdata, u32 cnt, u32 size); int mdss_hw_init(struct mdss_data_type *mdata); -int mdss_mdp_pa_config(struct mdss_mdp_ctl *ctl, - struct mdp_pa_cfg_data *config, - u32 *copyback); -int mdss_mdp_pcc_config(struct mdss_mdp_ctl *ctl, - struct mdp_pcc_cfg_data *cfg_ptr, - u32 *copyback); -int mdss_mdp_igc_lut_config(struct mdss_mdp_ctl *ctl, - struct mdp_igc_lut_data *config, - u32 *copyback, u32 copy_from_kernel); -int mdss_mdp_argc_config(struct mdss_mdp_ctl *ctl, - struct mdp_pgc_lut_data *config, - u32 *copyback); -int mdss_mdp_hist_lut_config(struct mdss_mdp_ctl *ctl, - struct mdp_hist_lut_data *config, - u32 *copyback); -int mdss_mdp_dither_config(struct mdss_mdp_ctl *ctl, - struct mdp_dither_cfg_data *config, - u32 *copyback); -int mdss_mdp_gamut_config(struct mdss_mdp_ctl *ctl, - struct mdp_gamut_cfg_data *config, - u32 *copyback); - -int mdss_mdp_histogram_start(struct mdss_mdp_ctl *ctl, - struct mdp_histogram_start_req *req); -int mdss_mdp_histogram_stop(struct mdss_mdp_ctl *ctl, u32 block); -int mdss_mdp_hist_collect(struct mdss_mdp_ctl *ctl, - struct mdp_histogram_data *hist); +int mdss_mdp_pa_config(struct mdp_pa_cfg_data *config, u32 *copyback); +int mdss_mdp_pcc_config(struct mdp_pcc_cfg_data *cfg_ptr, u32 *copyback); +int mdss_mdp_igc_lut_config(struct mdp_igc_lut_data *config, u32 *copyback, + u32 copy_from_kernel); +int mdss_mdp_argc_config(struct mdp_pgc_lut_data *config, u32 *copyback); +int mdss_mdp_hist_lut_config(struct mdp_hist_lut_data *config, u32 *copyback); +int mdss_mdp_dither_config(struct mdp_dither_cfg_data *config, u32 *copyback); +int mdss_mdp_gamut_config(struct mdp_gamut_cfg_data *config, u32 *copyback); + +int mdss_mdp_histogram_start(struct mdp_histogram_start_req *req); +int mdss_mdp_histogram_stop(u32 block); +int mdss_mdp_hist_collect(struct mdp_histogram_data *hist); void mdss_mdp_hist_intr_done(u32 isr); int mdss_mdp_ad_config(struct msm_fb_data_type *mfd, diff --git a/drivers/video/msm/mdss/mdss_mdp_ctl.c b/drivers/video/msm/mdss/mdss_mdp_ctl.c index f130867cd80..fc3eabec1e6 100644 --- a/drivers/video/msm/mdss/mdss_mdp_ctl.c +++ b/drivers/video/msm/mdss/mdss_mdp_ctl.c @@ -984,7 +984,7 @@ struct mdss_mdp_ctl *mdss_mdp_ctl_init(struct mdss_panel_data *pdata, ctl->dst_format = MDSS_MDP_PANEL_FORMAT_RGB888; break; } - mdss_mdp_dither_config(ctl, &dither, NULL); + mdss_mdp_dither_config(&dither, NULL); } return ctl; diff --git a/drivers/video/msm/mdss/mdss_mdp_intf_cmd.c b/drivers/video/msm/mdss/mdss_mdp_intf_cmd.c index 053d5632ece..e444e03052b 100644 --- a/drivers/video/msm/mdss/mdss_mdp_intf_cmd.c +++ b/drivers/video/msm/mdss/mdss_mdp_intf_cmd.c @@ -198,6 +198,10 @@ static int mdss_mdp_cmd_tearcheck_setup(struct mdss_mdp_ctl *ctl, int enable) static inline void mdss_mdp_cmd_clk_on(struct mdss_mdp_cmd_ctx *ctx) { unsigned long flags; + + if (!ctx->panel_on) + return; + mutex_lock(&ctx->clk_mtx); if (!ctx->clk_enabled) { ctx->clk_enabled = 1; @@ -257,6 +261,10 @@ static void mdss_mdp_cmd_readptr_done(void *arg) if (!ctx->vsync_enabled) { if (ctx->rdptr_enabled) ctx->rdptr_enabled--; + + /* keep clk on during kickoff */ + if (ctx->rdptr_enabled == 0 && ctx->koff_cnt) + ctx->rdptr_enabled++; } if (ctx->rdptr_enabled == 0) { @@ -336,6 +344,7 @@ static int mdss_mdp_cmd_add_vsync_handler(struct mdss_mdp_ctl *ctl, { struct mdss_mdp_cmd_ctx *ctx; unsigned long flags; + bool enable_rdptr = false; ctx = (struct mdss_mdp_cmd_ctx *) ctl->priv_data; if (!ctx) { @@ -347,12 +356,14 @@ static int mdss_mdp_cmd_add_vsync_handler(struct mdss_mdp_ctl *ctl, if (!handle->enabled) { handle->enabled = true; list_add(&handle->list, &ctx->vsync_handlers); - if (!handle->cmd_post_flush) - ctx->vsync_enabled = 1; + + enable_rdptr = !handle->cmd_post_flush; + if (enable_rdptr) + ctx->vsync_enabled++; } spin_unlock_irqrestore(&ctx->clk_lock, flags); - if (!handle->cmd_post_flush) + if (enable_rdptr) mdss_mdp_cmd_clk_on(ctx); return 0; @@ -361,11 +372,8 @@ static int mdss_mdp_cmd_add_vsync_handler(struct mdss_mdp_ctl *ctl, static int mdss_mdp_cmd_remove_vsync_handler(struct mdss_mdp_ctl *ctl, struct mdss_mdp_vsync_handler *handle) { - struct mdss_mdp_cmd_ctx *ctx; unsigned long flags; - struct mdss_mdp_vsync_handler *tmp; - int num_rdptr_vsync = 0; ctx = (struct mdss_mdp_cmd_ctx *) ctl->priv_data; if (!ctx) { @@ -373,19 +381,17 @@ static int mdss_mdp_cmd_remove_vsync_handler(struct mdss_mdp_ctl *ctl, return -ENODEV; } - spin_lock_irqsave(&ctx->clk_lock, flags); if (handle->enabled) { handle->enabled = false; list_del_init(&handle->list); - } - list_for_each_entry(tmp, &ctx->vsync_handlers, list) { - if (!tmp->cmd_post_flush) - num_rdptr_vsync++; - } - if (!num_rdptr_vsync) { - ctx->vsync_enabled = 0; - ctx->rdptr_enabled = VSYNC_EXPIRE_TICK; + + if (!handle->cmd_post_flush) { + if (ctx->vsync_enabled) + ctx->vsync_enabled--; + else + WARN(1, "unbalanced vsync disable"); + } } spin_unlock_irqrestore(&ctx->clk_lock, flags); return 0; @@ -428,7 +434,7 @@ static int mdss_mdp_cmd_wait4pingpong(struct mdss_mdp_ctl *ctl, void *arg) need_wait = 1; spin_unlock_irqrestore(&ctx->clk_lock, flags); - pr_debug("%s: need_wait=%d intf_num=%d ctx=%p\n", + pr_debug("%s: need_wait=%d intf_num=%d ctx=%pK\n", __func__, need_wait, ctl->intf_num, ctx); if (need_wait) { @@ -473,19 +479,19 @@ int mdss_mdp_cmd_kickoff(struct mdss_mdp_ctl *ctl, void *arg) WARN(rc, "intf %d panel on error (%d)\n", ctl->intf_num, rc); } + spin_lock_irqsave(&ctx->clk_lock, flags); + ctx->koff_cnt++; + spin_unlock_irqrestore(&ctx->clk_lock, flags); + + mdss_mdp_cmd_clk_on(ctx); + /* * tx dcs command if had any */ mdss_mdp_ctl_intf_event(ctl, MDSS_EVENT_DSI_CMDLIST_KOFF, NULL); - - mdss_mdp_cmd_clk_on(ctx); - INIT_COMPLETION(ctx->pp_comp); mdss_mdp_irq_enable(MDSS_MDP_IRQ_PING_PONG_COMP, ctx->pp_num); mdss_mdp_ctl_write(ctl, MDSS_MDP_REG_CTL_START, 1); - spin_lock_irqsave(&ctx->clk_lock, flags); - ctx->koff_cnt++; - spin_unlock_irqrestore(&ctx->clk_lock, flags); mb(); return 0; @@ -599,7 +605,7 @@ int mdss_mdp_cmd_start(struct mdss_mdp_ctl *ctl) atomic_set(&ctx->pp_done_cnt, 0); INIT_LIST_HEAD(&ctx->vsync_handlers); - pr_debug("%s: ctx=%p num=%d mixer=%d\n", __func__, + pr_debug("%s: ctx=%pK num=%d mixer=%d\n", __func__, ctx, ctx->pp_num, mixer->num); mdss_mdp_set_intr_callback(MDSS_MDP_IRQ_PING_PONG_RD_PTR, ctx->pp_num, diff --git a/drivers/video/msm/mdss/mdss_mdp_intf_video.c b/drivers/video/msm/mdss/mdss_mdp_intf_video.c index 9faf9532170..6d13b381124 100644 --- a/drivers/video/msm/mdss/mdss_mdp_intf_video.c +++ b/drivers/video/msm/mdss/mdss_mdp_intf_video.c @@ -101,7 +101,7 @@ int mdss_mdp_video_addr_setup(struct mdss_data_type *mdata, for (i = 0; i < count; i++) { head[i].base = mdata->mdp_base + offsets[i]; - pr_debug("adding Video Intf #%d offset=0x%x virt=%p\n", i, + pr_debug("adding Video Intf #%d offset=0x%x virt=%pK\n", i, offsets[i], head[i].base); head[i].ref_cnt = 0; head[i].intf_num = i + MDSS_MDP_INTF0; @@ -529,7 +529,7 @@ int mdss_mdp_video_copy_splash_screen(struct mdss_panel_data *pdata) ihdl = ion_alloc(iclient, size, SZ_1M, ION_HEAP(ION_QSECOM_HEAP_ID), 0); if (IS_ERR_OR_NULL(ihdl)) { - pr_err("unable to alloc fbmem from ion (%p)\n", ihdl); + pr_err("unable to alloc fbmem from ion (%pK)\n", ihdl); return -ENOMEM; } @@ -618,7 +618,7 @@ int mdss_mdp_video_start(struct mdss_mdp_ctl *ctl) pr_err("Intf %d already in use\n", ctl->intf_num); return -EBUSY; } - pr_debug("video Intf #%d base=%p", ctx->intf_num, ctx->base); + pr_debug("video Intf #%d base=%pK", ctx->intf_num, ctx->base); ctx->ref_cnt++; } else { pr_err("Invalid intf number: %d\n", ctl->intf_num); diff --git a/drivers/video/msm/mdss/mdss_mdp_overlay.c b/drivers/video/msm/mdss/mdss_mdp_overlay.c index 2361b975ff3..fcd320f167b 100644 --- a/drivers/video/msm/mdss/mdss_mdp_overlay.c +++ b/drivers/video/msm/mdss/mdss_mdp_overlay.c @@ -42,6 +42,7 @@ static atomic_t ov_active_panels = ATOMIC_INIT(0); static int mdss_mdp_overlay_free_fb_pipe(struct msm_fb_data_type *mfd); static int mdss_mdp_overlay_fb_parse_dt(struct msm_fb_data_type *mfd); static int mdss_mdp_overlay_off(struct msm_fb_data_type *mfd); +static void __vsync_retire_signal(struct msm_fb_data_type *mfd, int val); static int mdss_mdp_overlay_get(struct msm_fb_data_type *mfd, struct mdp_overlay *req) @@ -120,6 +121,9 @@ static int mdss_mdp_overlay_req_check(struct msm_fb_data_type *mfd, pr_err("Invalid decimation factors horz=%d vert=%d\n", req->horz_deci, req->vert_deci); return -EINVAL; + } else if (req->flags & MDP_BWC_EN) { + pr_err("Decimation can't be enabled with BWC\n"); + return -EINVAL; } } @@ -178,12 +182,15 @@ static int mdss_mdp_overlay_req_check(struct msm_fb_data_type *mfd, if (req->flags & MDP_BWC_EN) { if ((req->src.width != req->src_rect.w) || (req->src.height != req->src_rect.h)) { - pr_err("BWC: unequal src img and rect w,h\n"); + pr_err("BWC: mismatch of src img=%dx%d rect=%dx%d\n", + req->src.width, req->src.height, + req->src_rect.w, req->src_rect.h); return -EINVAL; } - if (req->flags & MDP_DECIMATION_EN) { - pr_err("Can't enable BWC decode && decimate\n"); + if ((req->flags & MDP_DECIMATION_EN) || + req->vert_deci || req->horz_deci) { + pr_err("Can't enable BWC and decimation\n"); return -EINVAL; } } @@ -314,7 +321,8 @@ static int __mdp_pipe_tune_perf(struct mdss_mdp_pipe *pipe) * requirement by applying vertical decimation and reduce * mdp clock requirement */ - if (mdata->has_decimation && (pipe->vert_deci < MAX_DECIMATION)) + if (mdata->has_decimation && (pipe->vert_deci < MAX_DECIMATION) + && !pipe->bwc_mode) pipe->vert_deci++; else return -EPERM; @@ -552,12 +560,11 @@ static int mdss_mdp_overlay_pipe_setup(struct msm_fb_data_type *mfd, pipe->pp_cfg.hist_cfg.frame_cnt; hist.bit_mask = pipe->pp_cfg.hist_cfg.bit_mask; hist.num_bins = pipe->pp_cfg.hist_cfg.num_bins; - mdss_mdp_histogram_start(pipe->mixer->ctl, - &hist); + mdss_mdp_histogram_start(&hist); } else if (pipe->pp_cfg.hist_cfg.ops & MDP_PP_OPS_DISABLE) { - mdss_mdp_histogram_stop(pipe->mixer->ctl, - pipe->pp_cfg.hist_cfg.block); + mdss_mdp_histogram_stop( + pipe->pp_cfg.hist_cfg.block); } } len = pipe->pp_cfg.hist_lut_cfg.len; @@ -1630,7 +1637,6 @@ static int mdss_mdp_pp_ioctl(struct msm_fb_data_type *mfd, struct msmfb_mdp_pp mdp_pp; u32 copyback = 0; u32 copy_from_kernel = 0; - struct mdss_overlay_private *mdp5_data = mfd_to_mdp5_data(mfd); ret = copy_from_user(&mdp_pp, argp, sizeof(mdp_pp)); if (ret) @@ -1645,14 +1651,12 @@ static int mdss_mdp_pp_ioctl(struct msm_fb_data_type *mfd, switch (mdp_pp.op) { case mdp_op_pa_cfg: - ret = mdss_mdp_pa_config(mdp5_data->ctl, - &mdp_pp.data.pa_cfg_data, + ret = mdss_mdp_pa_config(&mdp_pp.data.pa_cfg_data, ©back); break; case mdp_op_pcc_cfg: - ret = mdss_mdp_pcc_config(mdp5_data->ctl, - &mdp_pp.data.pcc_cfg_data, + ret = mdss_mdp_pcc_config(&mdp_pp.data.pcc_cfg_data, ©back); break; @@ -1660,7 +1664,6 @@ static int mdss_mdp_pp_ioctl(struct msm_fb_data_type *mfd, switch (mdp_pp.data.lut_cfg_data.lut_type) { case mdp_lut_igc: ret = mdss_mdp_igc_lut_config( - mdp5_data->ctl, (struct mdp_igc_lut_data *) &mdp_pp.data.lut_cfg_data.data, ©back, copy_from_kernel); @@ -1668,14 +1671,12 @@ static int mdss_mdp_pp_ioctl(struct msm_fb_data_type *mfd, case mdp_lut_pgc: ret = mdss_mdp_argc_config( - mdp5_data->ctl, &mdp_pp.data.lut_cfg_data.data.pgc_lut_data, ©back); break; case mdp_lut_hist: ret = mdss_mdp_hist_lut_config( - mdp5_data->ctl, (struct mdp_hist_lut_data *) &mdp_pp.data.lut_cfg_data.data, ©back); break; @@ -1687,13 +1688,11 @@ static int mdss_mdp_pp_ioctl(struct msm_fb_data_type *mfd, break; case mdp_op_dither_cfg: ret = mdss_mdp_dither_config( - mdp5_data->ctl, &mdp_pp.data.dither_cfg_data, ©back); break; case mdp_op_gamut_cfg: ret = mdss_mdp_gamut_config( - mdp5_data->ctl, &mdp_pp.data.gamut_cfg_data, ©back); break; @@ -1744,7 +1743,6 @@ static int mdss_mdp_histo_ioctl(struct msm_fb_data_type *mfd, u32 cmd, struct mdp_histogram_data hist; struct mdp_histogram_start_req hist_req; u32 block; - struct mdss_overlay_private *mdp5_data = mfd_to_mdp5_data(mfd); switch (cmd) { case MSMFB_HISTOGRAM_START: @@ -1755,7 +1753,7 @@ static int mdss_mdp_histo_ioctl(struct msm_fb_data_type *mfd, u32 cmd, if (ret) return ret; - ret = mdss_mdp_histogram_start(mdp5_data->ctl, &hist_req); + ret = mdss_mdp_histogram_start(&hist_req); break; case MSMFB_HISTOGRAM_STOP: @@ -1763,7 +1761,7 @@ static int mdss_mdp_histo_ioctl(struct msm_fb_data_type *mfd, u32 cmd, if (ret) return ret; - ret = mdss_mdp_histogram_stop(mdp5_data->ctl, block); + ret = mdss_mdp_histogram_stop(block); break; case MSMFB_HISTOGRAM: @@ -1774,7 +1772,7 @@ static int mdss_mdp_histo_ioctl(struct msm_fb_data_type *mfd, u32 cmd, if (ret) return ret; - ret = mdss_mdp_hist_collect(mdp5_data->ctl, &hist); + ret = mdss_mdp_hist_collect(&hist); if (!ret) ret = copy_to_user(argp, &hist, sizeof(hist)); break; @@ -2093,6 +2091,20 @@ static int mdss_mdp_overlay_off(struct msm_fb_data_type *mfd) mdss_mdp_overlay_kickoff(mfd); } + /* + * If retire fences are still active wait for a vsync time + * for retire fence to be updated. + * As a last resort signal the timeline if vsync doesn't arrive. + */ + if (mdp5_data->retire_cnt) { + u32 fps = mdss_panel_get_framerate(mfd->panel_info); + u32 vsync_time = 1000 / (fps ? : DEFAULT_FRAME_RATE); + + msleep(vsync_time); + + __vsync_retire_signal(mfd, mdp5_data->retire_cnt); + } + rc = mdss_mdp_ctl_stop(mdp5_data->ctl); if (rc == 0) { mdss_mdp_ctl_notifier_unregister(mdp5_data->ctl, @@ -2146,7 +2158,6 @@ static void __vsync_retire_work_handler(struct work_struct *work) { struct mdss_overlay_private *mdp5_data = container_of(work, typeof(*mdp5_data), retire_work); - struct msm_sync_pt_data *sync_pt_data; if (!mdp5_data->ctl || !mdp5_data->ctl->mfd) return; @@ -2154,12 +2165,18 @@ static void __vsync_retire_work_handler(struct work_struct *work) if (!mdp5_data->ctl->remove_vsync_handler) return; - sync_pt_data = &mdp5_data->ctl->mfd->mdp_sync_pt_data; - mutex_lock(&sync_pt_data->sync_mutex); + __vsync_retire_signal(mdp5_data->ctl->mfd, 1); +} + +static void __vsync_retire_signal(struct msm_fb_data_type *mfd, int val) +{ + struct mdss_overlay_private *mdp5_data = mfd_to_mdp5_data(mfd); + + mutex_lock(&mfd->mdp_sync_pt_data.sync_mutex); if (mdp5_data->retire_cnt > 0) { - sw_sync_timeline_inc(mdp5_data->vsync_timeline, 1); + sw_sync_timeline_inc(mdp5_data->vsync_timeline, val); - mdp5_data->retire_cnt--; + mdp5_data->retire_cnt -= min(val, mdp5_data->retire_cnt); if (mdp5_data->retire_cnt == 0) { mdss_mdp_clk_ctrl(MDP_BLOCK_POWER_ON, false); mdp5_data->ctl->remove_vsync_handler(mdp5_data->ctl, @@ -2167,7 +2184,7 @@ static void __vsync_retire_work_handler(struct work_struct *work) mdss_mdp_clk_ctrl(MDP_BLOCK_POWER_OFF, false); } } - mutex_unlock(&sync_pt_data->sync_mutex); + mutex_unlock(&mfd->mdp_sync_pt_data.sync_mutex); } static struct sync_fence * @@ -2193,7 +2210,7 @@ __vsync_retire_get_fence(struct msm_sync_pt_data *sync_pt_data) return ERR_PTR(-EPERM); } - if (mdp5_data->retire_cnt == 0) { + if (!mdp5_data->vsync_retire_handler.enabled) { mdss_mdp_clk_ctrl(MDP_BLOCK_POWER_ON, false); rc = ctl->add_vsync_handler(ctl, &mdp5_data->vsync_retire_handler); diff --git a/drivers/video/msm/mdss/mdss_mdp_pp.c b/drivers/video/msm/mdss/mdss_mdp_pp.c index fc6df9fc59e..1c3356f0d15 100644 --- a/drivers/video/msm/mdss/mdss_mdp_pp.c +++ b/drivers/video/msm/mdss/mdss_mdp_pp.c @@ -272,6 +272,10 @@ static void pp_update_argc_lut(u32 offset, struct mdp_pgc_lut_data *config); static void pp_update_hist_lut(char __iomem *base, struct mdp_hist_lut_data *cfg); +static int pp_gm_has_invalid_lut_size(struct mdp_gamut_cfg_data *config); +static void pp_gamut_config(struct mdp_gamut_cfg_data *gamut_cfg, + u32 base, + struct pp_sts_type *pp_sts); static void pp_pa_config(unsigned long flags, u32 base, struct pp_sts_type *pp_sts, struct mdp_pa_cfg *pa_config); @@ -1111,7 +1115,7 @@ static int pp_dspp_setup(u32 disp_num, struct mdss_mdp_mixer *mixer) flush_exit: writel_relaxed(opmode, basel + MDSS_MDP_REG_DSPP_OP_MODE); - mdss_mdp_ctl_write(ctl, MDSS_MDP_REG_CTL_FLUSH, BIT(13 + dspp_num)); + ctl->flush_bits |= BIT(13 + dspp_num); wmb(); dspp_exit: mdss_mdp_clk_ctrl(MDP_BLOCK_POWER_OFF, false); @@ -1324,15 +1328,12 @@ static int pp_get_dspp_num(u32 disp_num, u32 *dspp_num) return 0; } -int mdss_mdp_pa_config(struct mdss_mdp_ctl *ctl, struct mdp_pa_cfg_data *config, +int mdss_mdp_pa_config(struct mdp_pa_cfg_data *config, u32 *copyback) { int ret = 0; u32 pa_offset, disp_num, dspp_num = 0; - if (!ctl) - return -EINVAL; - if ((config->block < MDP_LOGICAL_BLOCK_DISP_0) || (config->block >= MDP_BLOCK_MAX)) return -EINVAL; @@ -1366,8 +1367,6 @@ int mdss_mdp_pa_config(struct mdss_mdp_ctl *ctl, struct mdp_pa_cfg_data *config, pa_config_exit: mutex_unlock(&mdss_pp_mutex); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } @@ -1497,16 +1496,12 @@ static void pp_update_pcc_regs(u32 offset, MDSS_MDP_REG_WRITE(offset + 8, cfg_ptr->b.rgb_1); } -int mdss_mdp_pcc_config(struct mdss_mdp_ctl *ctl, - struct mdp_pcc_cfg_data *config, +int mdss_mdp_pcc_config(struct mdp_pcc_cfg_data *config, u32 *copyback) { int ret = 0; u32 base, disp_num, dspp_num = 0; - if (!ctl) - return -EINVAL; - if ((config->block < MDP_LOGICAL_BLOCK_DISP_0) || (config->block >= MDP_BLOCK_MAX)) return -EINVAL; @@ -1536,8 +1531,6 @@ int mdss_mdp_pcc_config(struct mdss_mdp_ctl *ctl, pcc_config_exit: mutex_unlock(&mdss_pp_mutex); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } @@ -1611,22 +1604,18 @@ int mdss_mdp_limited_lut_igc_config(struct mdss_mdp_ctl *ctl) config.c0_c1_data = igc_limited; config.c2_data = igc_limited; - ret = mdss_mdp_igc_lut_config(ctl, &config, ©back, + ret = mdss_mdp_igc_lut_config(&config, ©back, copy_from_kernel); return ret; } -int mdss_mdp_igc_lut_config(struct mdss_mdp_ctl *ctl, - struct mdp_igc_lut_data *config, +int mdss_mdp_igc_lut_config(struct mdp_igc_lut_data *config, u32 *copyback, u32 copy_from_kernel) { int ret = 0; u32 tbl_idx, igc_offset, disp_num, dspp_num = 0; struct mdp_igc_lut_data local_cfg; - if (!ctl) - return -EINVAL; - if ((config->block < MDP_LOGICAL_BLOCK_DISP_0) || (config->block >= MDP_BLOCK_MAX)) return -EINVAL; @@ -1703,8 +1692,6 @@ int mdss_mdp_igc_lut_config(struct mdss_mdp_ctl *ctl, igc_config_exit: mutex_unlock(&mdss_pp_mutex); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } static void pp_update_gc_one_lut(u32 offset, @@ -1819,8 +1806,7 @@ static void pp_update_hist_lut(char __iomem *offset, writel_relaxed(1, offset + 16); } -int mdss_mdp_argc_config(struct mdss_mdp_ctl *ctl, - struct mdp_pgc_lut_data *config, +int mdss_mdp_argc_config(struct mdp_pgc_lut_data *config, u32 *copyback) { int ret = 0; @@ -1829,9 +1815,6 @@ int mdss_mdp_argc_config(struct mdss_mdp_ctl *ctl, struct mdp_pgc_lut_data *pgc_ptr; u32 tbl_size, r_size, g_size, b_size; - if (!ctl) - return -EINVAL; - if ((PP_BLOCK(config->block) < MDP_LOGICAL_BLOCK_DISP_0) || (PP_BLOCK(config->block) >= MDP_BLOCK_MAX)) return -EINVAL; @@ -1941,20 +1924,14 @@ int mdss_mdp_argc_config(struct mdss_mdp_ctl *ctl, } argc_config_exit: mutex_unlock(&mdss_pp_mutex); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } -int mdss_mdp_hist_lut_config(struct mdss_mdp_ctl *ctl, - struct mdp_hist_lut_data *config, +int mdss_mdp_hist_lut_config(struct mdp_hist_lut_data *config, u32 *copyback) { int i, ret = 0; u32 hist_offset, disp_num, dspp_num = 0; - if (!ctl) - return -EINVAL; - if ((PP_BLOCK(config->block) < MDP_LOGICAL_BLOCK_DISP_0) || (PP_BLOCK(config->block) >= MDP_BLOCK_MAX)) return -EINVAL; @@ -1998,18 +1975,13 @@ int mdss_mdp_hist_lut_config(struct mdss_mdp_ctl *ctl, } enhist_config_exit: mutex_unlock(&mdss_pp_mutex); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } -int mdss_mdp_dither_config(struct mdss_mdp_ctl *ctl, - struct mdp_dither_cfg_data *config, +int mdss_mdp_dither_config(struct mdp_dither_cfg_data *config, u32 *copyback) { u32 disp_num; - if (!ctl) - return -EINVAL; if ((config->block < MDP_LOGICAL_BLOCK_DISP_0) || (config->block >= MDP_BLOCK_MAX)) @@ -2022,15 +1994,35 @@ int mdss_mdp_dither_config(struct mdss_mdp_ctl *ctl, mdss_pp_res->dither_disp_cfg[disp_num] = *config; mdss_pp_res->pp_disp_flags[disp_num] |= PP_FLAGS_DIRTY_DITHER; mutex_unlock(&mdss_pp_mutex); - mdss_mdp_pp_setup(ctl); return 0; } -int mdss_mdp_gamut_config(struct mdss_mdp_ctl *ctl, - struct mdp_gamut_cfg_data *config, +static int pp_gm_has_invalid_lut_size(struct mdp_gamut_cfg_data *config) +{ + if (config->tbl_size[0] != GAMUT_T0_SIZE) + return -EINVAL; + if (config->tbl_size[1] != GAMUT_T1_SIZE) + return -EINVAL; + if (config->tbl_size[2] != GAMUT_T2_SIZE) + return -EINVAL; + if (config->tbl_size[3] != GAMUT_T3_SIZE) + return -EINVAL; + if (config->tbl_size[4] != GAMUT_T4_SIZE) + return -EINVAL; + if (config->tbl_size[5] != GAMUT_T5_SIZE) + return -EINVAL; + if (config->tbl_size[6] != GAMUT_T6_SIZE) + return -EINVAL; + if (config->tbl_size[7] != GAMUT_T7_SIZE) + return -EINVAL; + + return 0; +} + +int mdss_mdp_gamut_config(struct mdp_gamut_cfg_data *config, u32 *copyback) { - int i, j, size_total = 0, ret = 0; + int i, j, ret = 0; u32 offset, disp_num, dspp_num = 0; uint16_t *tbl_off; struct mdp_gamut_cfg_data local_cfg; @@ -2039,15 +2031,11 @@ int mdss_mdp_gamut_config(struct mdss_mdp_ctl *ctl, uint16_t *b_tbl[MDP_GAMUT_TABLE_NUM]; - if (!ctl) - return -EINVAL; - if ((config->block < MDP_LOGICAL_BLOCK_DISP_0) || (config->block >= MDP_BLOCK_MAX)) return -EINVAL; - for (i = 0; i < MDP_GAMUT_TABLE_NUM; i++) - size_total += config->tbl_size[i]; - if (size_total != GAMUT_TOTAL_TABLE_SIZE) + + if (pp_gm_has_invalid_lut_size(config)) return -EINVAL; mutex_lock(&mdss_pp_mutex); @@ -2164,8 +2152,6 @@ int mdss_mdp_gamut_config(struct mdss_mdp_ctl *ctl, } gamut_config_exit: mutex_unlock(&mdss_pp_mutex); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } static void pp_hist_read(char __iomem *v_base, @@ -2208,7 +2194,7 @@ static int pp_histogram_enable(struct pp_hist_col_info *hist_info, hist_info->col_state = HIST_RESET; hist_info->col_en = true; spin_unlock_irqrestore(&hist_info->hist_lock, flag); - hist_info->is_kick_ready = false; + hist_info->is_kick_ready = true; mdss_mdp_hist_irq_enable(3 << shift_bit); writel_relaxed(req->frame_cnt, ctl_base + 8); /* Kick out reset start */ @@ -2218,8 +2204,7 @@ static int pp_histogram_enable(struct pp_hist_col_info *hist_info, return ret; } -int mdss_mdp_histogram_start(struct mdss_mdp_ctl *ctl, - struct mdp_histogram_start_req *req) +int mdss_mdp_histogram_start(struct mdp_histogram_start_req *req) { u32 done_shift_bit; char __iomem *ctl_base; @@ -2229,8 +2214,6 @@ int mdss_mdp_histogram_start(struct mdss_mdp_ctl *ctl, u32 mixer_cnt, mixer_id[MDSS_MDP_INTF_MAX_LAYERMIXER]; struct mdss_mdp_pipe *pipe; struct mdss_data_type *mdata = mdss_mdp_get_mdata(); - if (!ctl) - return -EINVAL; if ((PP_BLOCK(req->block) < MDP_LOGICAL_BLOCK_DISP_0) || (PP_BLOCK(req->block) >= MDP_BLOCK_MAX)) @@ -2297,31 +2280,6 @@ int mdss_mdp_histogram_start(struct mdss_mdp_ctl *ctl, mdss_mdp_clk_ctrl(MDP_BLOCK_POWER_OFF, false); hist_exit: - if (!ret && (PP_LOCAT(req->block) == MDSS_PP_DSPP_CFG)) { - mdss_mdp_pp_setup(ctl); - /* wait for a frame to let histrogram enable itself */ - /* TODO add hysteresis value to be able to remove this sleep */ - usleep(41666); - for (i = 0; i < mixer_cnt; i++) { - dspp_num = mixer_id[i]; - hist_info = &mdss_pp_res->dspp_hist[dspp_num]; - mutex_lock(&hist_info->hist_mutex); - hist_info->is_kick_ready = true; - mutex_unlock(&hist_info->hist_mutex); - } - } else if (!ret) { - for (i = 0; i < MDSS_PP_ARG_NUM; i++) { - if (!PP_ARG(i, req->block)) - continue; - pr_info("PP_ARG(%d) = %d", i, PP_ARG(i, req->block)); - pipe = mdss_mdp_pipe_get(mdata, BIT(i)); - if (IS_ERR_OR_NULL(pipe)) - continue; - hist_info = &pipe->pp_res.hist; - hist_info->is_kick_ready = true; - mdss_mdp_pipe_unmap(pipe); - } - } return ret; } @@ -2350,7 +2308,7 @@ static int pp_histogram_disable(struct pp_hist_col_info *hist_info, return ret; } -int mdss_mdp_histogram_stop(struct mdss_mdp_ctl *ctl, u32 block) +int mdss_mdp_histogram_stop(u32 block) { int i, ret = 0; char __iomem *ctl_base; @@ -2360,9 +2318,6 @@ int mdss_mdp_histogram_stop(struct mdss_mdp_ctl *ctl, u32 block) struct mdss_mdp_pipe *pipe; struct mdss_data_type *mdata = mdss_mdp_get_mdata(); - if (!ctl) - return -EINVAL; - if ((PP_BLOCK(block) < MDP_LOGICAL_BLOCK_DISP_0) || (PP_BLOCK(block) >= MDP_BLOCK_MAX)) return -EINVAL; @@ -2427,13 +2382,10 @@ int mdss_mdp_histogram_stop(struct mdss_mdp_ctl *ctl, u32 block) hist_stop_clk: mdss_mdp_clk_ctrl(MDP_BLOCK_POWER_OFF, false); hist_stop_exit: - if (!ret && (PP_LOCAT(block) == MDSS_PP_DSPP_CFG)) - mdss_mdp_pp_setup(ctl); return ret; } -static int pp_hist_collect(struct mdss_mdp_ctl *ctl, - struct mdp_histogram_data *hist, +static int pp_hist_collect(struct mdp_histogram_data *hist, struct pp_hist_col_info *hist_info, char __iomem *ctl_base) { @@ -2457,9 +2409,6 @@ static int pp_hist_collect(struct mdss_mdp_ctl *ctl, spin_unlock_irqrestore(&hist_info->hist_lock, flag); timeout = HIST_WAIT_TIMEOUT(hist_info->frame_cnt); mutex_unlock(&hist_info->hist_mutex); - /* flush updates before wait*/ - if (PP_LOCAT(hist->block) == MDSS_PP_DSPP_CFG) - mdss_mdp_pp_setup(ctl); if (PP_LOCAT(hist->block) == MDSS_PP_SSPP_CFG) { res = container_of(hist_info, struct mdss_pipe_pp_res, hist); @@ -2520,8 +2469,7 @@ static int pp_hist_collect(struct mdss_mdp_ctl *ctl, return ret; } -int mdss_mdp_hist_collect(struct mdss_mdp_ctl *ctl, - struct mdp_histogram_data *hist) +int mdss_mdp_hist_collect(struct mdp_histogram_data *hist) { int i, j, off, ret = 0; struct pp_hist_col_info *hist_info; @@ -2535,9 +2483,6 @@ int mdss_mdp_hist_collect(struct mdss_mdp_ctl *ctl, struct mdss_mdp_pipe *pipe; struct mdss_data_type *mdata = mdss_mdp_get_mdata(); - if (!ctl) - return -EINVAL; - if ((PP_BLOCK(hist->block) < MDP_LOGICAL_BLOCK_DISP_0) || (PP_BLOCK(hist->block) >= MDP_BLOCK_MAX)) return -EINVAL; @@ -2564,7 +2509,7 @@ int mdss_mdp_hist_collect(struct mdss_mdp_ctl *ctl, hist_info = &mdss_pp_res->dspp_hist[dspp_num]; ctl_base = mdss_mdp_get_dspp_addr_off(dspp_num) + MDSS_MDP_REG_DSPP_HIST_CTL_BASE; - ret = pp_hist_collect(ctl, hist, hist_info, ctl_base); + ret = pp_hist_collect(hist, hist_info, ctl_base); if (ret) goto hist_collect_exit; } @@ -2633,7 +2578,7 @@ int mdss_mdp_hist_collect(struct mdss_mdp_ctl *ctl, hist_info = &pipe->pp_res.hist; ctl_base = pipe->base + MDSS_MDP_REG_VIG_HIST_CTL_BASE; - ret = pp_hist_collect(ctl, hist, hist_info, ctl_base); + ret = pp_hist_collect(hist, hist_info, ctl_base); mdss_mdp_pipe_unmap(pipe); if (ret) goto hist_collect_exit; @@ -2901,8 +2846,6 @@ int mdss_mdp_ad_config(struct msm_fb_data_type *mfd, } mutex_unlock(&ad->lock); ctl = mfd_to_ctl(mfd); - if (!ret) - mdss_mdp_pp_setup(ctl); return ret; } @@ -2910,7 +2853,6 @@ int mdss_mdp_ad_input(struct msm_fb_data_type *mfd, struct mdss_ad_input *input, int wait) { int ret = 0; struct mdss_ad_info *ad; - struct mdss_mdp_ctl *ctl; u32 bl; ret = mdss_mdp_get_ad(mfd, &ad); @@ -2985,8 +2927,6 @@ int mdss_mdp_ad_input(struct msm_fb_data_type *mfd, init_completion(&ad->comp); mutex_unlock(&ad->lock); } - ctl = mfd_to_ctl(mfd); - mdss_mdp_pp_setup(ctl); if (wait) { ret = wait_for_completion_interruptible_timeout( &ad->comp, HIST_WAIT_TIMEOUT(1)); @@ -3309,7 +3249,7 @@ static void pp_ad_calc_worker(struct work_struct *work) } mutex_unlock(&ad->lock); mutex_lock(&mfd->lock); - mdss_mdp_ctl_write(ctl, MDSS_MDP_REG_CTL_FLUSH, BIT(13 + ad->num)); + ctl->flush_bits |= BIT(13 + ad->num); mutex_unlock(&mfd->lock); /* Trigger update notify to wake up those waiting for display updates */ diff --git a/drivers/video/msm/mdss/mdss_mdp_util.c b/drivers/video/msm/mdss/mdss_mdp_util.c index b65d8946b2e..673cd4bc178 100644 --- a/drivers/video/msm/mdss/mdss_mdp_util.c +++ b/drivers/video/msm/mdss/mdss_mdp_util.c @@ -446,25 +446,30 @@ int mdss_mdp_put_img(struct mdss_mdp_img_data *data) pr_debug("pmem buf=0x%x\n", data->addr); data->srcp_file = NULL; } else if (!IS_ERR_OR_NULL(data->srcp_ihdl)) { - pr_debug("ion hdl=%p buf=0x%x\n", data->srcp_ihdl, data->addr); - - if (is_mdss_iommu_attached()) { - int domain; - if (data->flags & MDP_SECURE_OVERLAY_SESSION) - domain = MDSS_IOMMU_DOMAIN_SECURE; - else - domain = MDSS_IOMMU_DOMAIN_UNSECURE; - ion_unmap_iommu(iclient, data->srcp_ihdl, - mdss_get_iommu_domain(domain), 0); - - if (domain == MDSS_IOMMU_DOMAIN_SECURE) { - msm_ion_unsecure_buffer(iclient, - data->srcp_ihdl); + pr_debug("ion hdl=%pK buf=0x%x\n", data->srcp_ihdl, data->addr); + if (!iclient) { + pr_err("invalid_ion client\n"); + return -ENOMEM; + } else { + if (data->mapped) { + int domain; + if (data->flags & MDP_SECURE_OVERLAY_SESSION) + domain = MDSS_IOMMU_DOMAIN_SECURE; + else + domain = MDSS_IOMMU_DOMAIN_UNSECURE; + ion_unmap_iommu(iclient, data->srcp_ihdl, + mdss_get_iommu_domain(domain), 0); + + if (domain == MDSS_IOMMU_DOMAIN_SECURE) { + msm_ion_unsecure_buffer(iclient, + data->srcp_ihdl); + } + data->mapped = false; } + ion_free(iclient, data->srcp_ihdl); + data->srcp_ihdl = NULL; } - ion_free(iclient, data->srcp_ihdl); - data->srcp_ihdl = NULL; } else { return -ENOMEM; } @@ -538,6 +543,7 @@ int mdss_mdp_get_img(struct msmfb_data *img, struct mdss_mdp_img_data *data) if (ret && (domain == MDSS_IOMMU_DOMAIN_SECURE)) msm_ion_unsecure_buffer(iclient, data->srcp_ihdl); + data->mapped = true; } else { ret = ion_phys(iclient, data->srcp_ihdl, start, (size_t *) len); @@ -560,7 +566,7 @@ int mdss_mdp_get_img(struct msmfb_data *img, struct mdss_mdp_img_data *data) data->addr += img->offset; data->len -= img->offset; - pr_debug("mem=%d ihdl=%p buf=0x%x len=0x%x\n", img->memory_id, + pr_debug("mem=%d ihdl=%pK buf=0x%x len=0x%x\n", img->memory_id, data->srcp_ihdl, data->addr, data->len); } else { mdss_mdp_put_img(data); diff --git a/drivers/video/msm/mdss/mdss_mdp_wb.c b/drivers/video/msm/mdss/mdss_mdp_wb.c index 01a0c7ff361..362e7fc7f8b 100644 --- a/drivers/video/msm/mdss/mdss_mdp_wb.c +++ b/drivers/video/msm/mdss/mdss_mdp_wb.c @@ -94,7 +94,7 @@ struct mdss_mdp_data *mdss_mdp_wb_debug_buffer(struct msm_fb_data_type *mfd) ihdl = ion_alloc(iclient, img_size, SZ_4K, ION_HEAP(ION_SF_HEAP_ID), 0); if (IS_ERR_OR_NULL(ihdl)) { - pr_err("unable to alloc fbmem from ion (%p)\n", ihdl); + pr_err("unable to alloc fbmem from ion (%pK)\n", ihdl); return NULL; } @@ -114,7 +114,7 @@ struct mdss_mdp_data *mdss_mdp_wb_debug_buffer(struct msm_fb_data_type *mfd) img->len = img_size; } - pr_debug("ihdl=%p virt=%p phys=0x%lx iova=0x%x size=%u\n", + pr_debug("ihdl=%pK virt=%pK phys=0x%lx iova=0x%x size=%u\n", ihdl, videomemory, mdss_wb_mem, img->addr, img_size); } return &mdss_wb_buffer; @@ -136,6 +136,16 @@ int mdss_mdp_wb_set_secure(struct msm_fb_data_type *mfd, int enable) pr_debug("setting secure=%d\n", enable); + if (!ctl || !ctl->mdata) { + pr_err("%s : ctl is NULL", __func__); + return -EINVAL; + } + + if (!wb) { + pr_err("unable to start, writeback is not initialized\n"); + return -ENODEV; + } + ctl->is_secure = enable; wb->is_secure = enable; @@ -377,7 +387,7 @@ static struct mdss_mdp_wb_data *get_user_node(struct msm_fb_data_type *mfd, list_for_each_entry(node, &wb->register_queue, registered_entry) if ((node->buf_data.p[0].srcp_ihdl == ihdl) && (node->buf_info.offset == data->offset)) { - pr_debug("found fd=%d ihdl=%p off=%x addr=%x\n", + pr_debug("found fd=%d ihdl=%pK off=%x addr=%x\n", data->memory_id, ihdl, data->offset, node->buf_data.p[0].addr); @@ -425,7 +435,7 @@ static void mdss_mdp_wb_free_node(struct mdss_mdp_wb_data *node) if (node->user_alloc) { buf = &node->buf_data.p[0]; - pr_debug("free user mem_id=%d ihdl=%p, offset=%u addr=0x%x\n", + pr_debug("free user mem_id=%d ihdl=%pK, offset=%u addr=0x%x\n", node->buf_info.memory_id, buf->srcp_ihdl, node->buf_info.offset, diff --git a/drivers/video/msm/vidc/common/dec/vdec.c b/drivers/video/msm/vidc/common/dec/vdec.c index ec27b008494..62789443ed0 100644 --- a/drivers/video/msm/vidc/common/dec/vdec.c +++ b/drivers/video/msm/vidc/common/dec/vdec.c @@ -897,6 +897,12 @@ static u32 vid_dec_set_meta_buffers(struct video_client_ctx *client_ctx, vcd_meta_buffer->offset = meta_buffers->offset; vcd_meta_buffer->pmem_fd_iommu = meta_buffers->pmem_fd_iommu; + if (meta_buffers->count > MAX_META_BUFFERS) { + ERR("meta buffers maximum count reached, count = %d", + meta_buffers->count); + return false; + } + if (!vcd_get_ion_status()) { pr_err("PMEM Not available\n"); return false; @@ -1102,6 +1108,12 @@ static u32 vid_dec_set_h264_mv_buffers(struct video_client_ctx *client_ctx, vcd_h264_mv_buffer->pmem_fd = mv_data->pmem_fd; vcd_h264_mv_buffer->offset = mv_data->offset; + if (mv_data->count > MAX_MV_BUFFERS) { + ERR("MV buffers maximum count reached, count = %d", + mv_data->count); + return false; + } + if (!vcd_get_ion_status()) { pr_err("PMEM not available\n"); return false; diff --git a/drivers/video/msm/vidc/common/enc/venc.c b/drivers/video/msm/vidc/common/enc/venc.c index 823626ad18e..93945b51057 100644 --- a/drivers/video/msm/vidc/common/enc/venc.c +++ b/drivers/video/msm/vidc/common/enc/venc.c @@ -1416,6 +1416,12 @@ static long vid_enc_ioctl(struct file *file, return -EFAULT; DBG("VEN_IOCTL_GET_SEQUENCE_HDR\n"); + if (!access_ok(VERIFY_WRITE, seq_header.hdrbufptr, + seq_header.bufsize)) { + ERR("VEN_IOCTL_GET_SEQUENCE_HDR:"\ + " Userspace address verification failed.\n"); + return -EFAULT; + } result = vid_enc_get_sequence_header(client_ctx, &seq_header); if (!result) { diff --git a/fs/Kconfig b/fs/Kconfig index 1dd49481854..e3363fe2cd6 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -39,6 +39,7 @@ source "fs/gfs2/Kconfig" source "fs/ocfs2/Kconfig" source "fs/btrfs/Kconfig" source "fs/nilfs2/Kconfig" +source "fs/f2fs/Kconfig" endif # BLOCK @@ -224,6 +225,7 @@ source "fs/pstore/Kconfig" source "fs/sysv/Kconfig" source "fs/ufs/Kconfig" source "fs/exofs/Kconfig" +source "fs/f2fs/Kconfig" endif # MISC_FILESYSTEMS diff --git a/fs/Makefile b/fs/Makefile index 95cf9de6ae0..cf508e5240b 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -122,6 +122,7 @@ obj-$(CONFIG_DEBUG_FS) += debugfs/ obj-$(CONFIG_OCFS2_FS) += ocfs2/ obj-$(CONFIG_BTRFS_FS) += btrfs/ obj-$(CONFIG_GFS2_FS) += gfs2/ +obj-$(CONFIG_F2FS_FS) += f2fs/ obj-y += exofs/ # Multiple modules obj-$(CONFIG_CEPH_FS) += ceph/ obj-$(CONFIG_PSTORE) += pstore/ diff --git a/fs/aio.c b/fs/aio.c index 67a6db3e1b6..5ee63ac875a 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -111,6 +112,9 @@ static int aio_setup_ring(struct kioctx *ctx) unsigned long size; int nr_pages; + if (current->personality & READ_IMPLIES_EXEC) + return -EPERM; + /* Compensate for the ring buffer's head/tail overlap entry */ nr_events += 2; /* 1 is required, 2 for good luck */ @@ -1456,6 +1460,10 @@ static ssize_t aio_setup_vectored_rw(int type, struct kiocb *kiocb, bool compat) if (ret < 0) goto out; + ret = rw_verify_area(type, kiocb->ki_filp, &kiocb->ki_pos, ret); + if (ret < 0) + goto out; + kiocb->ki_nr_segs = kiocb->ki_nbytes; kiocb->ki_cur_seg = 0; /* ki_nbytes/left now reflect bytes instead of segs */ @@ -1467,11 +1475,17 @@ static ssize_t aio_setup_vectored_rw(int type, struct kiocb *kiocb, bool compat) return ret; } -static ssize_t aio_setup_single_vector(struct kiocb *kiocb) +static ssize_t aio_setup_single_vector(int type, struct file * file, struct kiocb *kiocb) { + int bytes; + + bytes = rw_verify_area(type, file, &kiocb->ki_pos, kiocb->ki_left); + if (bytes < 0) + return bytes; + kiocb->ki_iovec = &kiocb->ki_inline_vec; kiocb->ki_iovec->iov_base = kiocb->ki_buf; - kiocb->ki_iovec->iov_len = kiocb->ki_left; + kiocb->ki_iovec->iov_len = bytes; kiocb->ki_nr_segs = 1; kiocb->ki_cur_seg = 0; return 0; @@ -1496,10 +1510,7 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) if (unlikely(!access_ok(VERIFY_WRITE, kiocb->ki_buf, kiocb->ki_left))) break; - ret = security_file_permission(file, MAY_READ); - if (unlikely(ret)) - break; - ret = aio_setup_single_vector(kiocb); + ret = aio_setup_single_vector(READ, file, kiocb); if (ret) break; ret = -EINVAL; @@ -1514,10 +1525,7 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) if (unlikely(!access_ok(VERIFY_READ, kiocb->ki_buf, kiocb->ki_left))) break; - ret = security_file_permission(file, MAY_WRITE); - if (unlikely(ret)) - break; - ret = aio_setup_single_vector(kiocb); + ret = aio_setup_single_vector(WRITE, file, kiocb); if (ret) break; ret = -EINVAL; @@ -1528,9 +1536,6 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) ret = -EBADF; if (unlikely(!(file->f_mode & FMODE_READ))) break; - ret = security_file_permission(file, MAY_READ); - if (unlikely(ret)) - break; ret = aio_setup_vectored_rw(READ, kiocb, compat); if (ret) break; @@ -1542,9 +1547,6 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) ret = -EBADF; if (unlikely(!(file->f_mode & FMODE_WRITE))) break; - ret = security_file_permission(file, MAY_WRITE); - if (unlikely(ret)) - break; ret = aio_setup_vectored_rw(WRITE, kiocb, compat); if (ret) break; diff --git a/fs/dcache.c b/fs/dcache.c index 53c362d891e..48f701c8bed 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -2507,7 +2507,7 @@ static int prepend_path(const struct path *path, bool slash = false; int error = 0; - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); while (dentry != root->dentry || vfsmnt != root->mnt) { struct dentry * parent; @@ -2538,7 +2538,7 @@ static int prepend_path(const struct path *path, error = prepend(buffer, buflen, "/", 1); out: - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return error; global_root: @@ -2554,7 +2554,7 @@ static int prepend_path(const struct path *path, if (!slash) error = prepend(buffer, buflen, "/", 1); if (!error) - error = real_mount(vfsmnt)->mnt_ns ? 1 : 2; + error = is_mounted(vfsmnt) ? 1 : 2; goto out; } diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index ea993128155..3d0633891f2 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -2044,7 +2044,6 @@ ecryptfs_decode_from_filename(unsigned char *dst, size_t *dst_size, break; case 2: dst[dst_byte_offset++] |= (src_byte); - dst[dst_byte_offset] = 0; current_bit_offset = 0; break; } diff --git a/fs/ecryptfs/file.c b/fs/ecryptfs/file.c index 2b17f2f9b12..45fa3d8186f 100644 --- a/fs/ecryptfs/file.c +++ b/fs/ecryptfs/file.c @@ -151,6 +151,15 @@ static const struct vm_operations_struct ecryptfs_file_vm_ops = { static int ecryptfs_file_mmap(struct file *file, struct vm_area_struct *vma) { int rc; + struct file *lower_file = ecryptfs_file_to_lower(file); + + /* + * Don't allow mmap on top of file systems that don't support it + * natively. If FILESYSTEM_MAX_STACK_DEPTH > 2 or ecryptfs + * allows recursive mounting, this will need to be extended. + */ + if (!lower_file->f_op->mmap) + return -ENODEV; rc = generic_file_mmap(file, vma); if (!rc) diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index 68954937a07..043f247ec82 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -544,6 +544,13 @@ static struct dentry *ecryptfs_mount(struct file_system_type *fs_type, int flags s->s_maxbytes = path.dentry->d_sb->s_maxbytes; s->s_blocksize = path.dentry->d_sb->s_blocksize; s->s_magic = ECRYPTFS_SUPER_MAGIC; + s->s_stack_depth = path.dentry->d_sb->s_stack_depth + 1; + + rc = -EINVAL; + if (s->s_stack_depth > FILESYSTEM_MAX_STACK_DEPTH) { + pr_err("eCryptfs: maximum fs stacking depth exceeded\n"); + goto out_free; + } inode = ecryptfs_get_inode(path.dentry->d_inode, s); rc = PTR_ERR(inode); diff --git a/fs/exec.c b/fs/exec.c index b1fd2025e59..20130372cf0 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1230,7 +1230,7 @@ EXPORT_SYMBOL(install_exec_creds); /* * determine how safe it is to execute the proposed program * - the caller must hold ->cred_guard_mutex to protect against - * PTRACE_ATTACH + * PTRACE_ATTACH or seccomp thread-sync */ static int check_unsafe_exec(struct linux_binprm *bprm) { @@ -1245,6 +1245,13 @@ static int check_unsafe_exec(struct linux_binprm *bprm) bprm->unsafe |= LSM_UNSAFE_PTRACE; } + /* + * This isn't strictly necessary, but it makes it harder for LSMs to + * mess up. + */ + if (task_no_new_privs(current)) + bprm->unsafe |= LSM_UNSAFE_NO_NEW_PRIVS; + n_fs = 1; spin_lock(&p->fs->lock); rcu_read_lock(); @@ -1268,6 +1275,48 @@ static int check_unsafe_exec(struct linux_binprm *bprm) return res; } +static void bprm_fill_uid(struct linux_binprm *bprm) +{ + struct inode *inode; + unsigned int mode; + uid_t uid; + gid_t gid; + + /* clear any previous set[ug]id data from a previous binary */ + bprm->cred->euid = current_euid(); + bprm->cred->egid = current_egid(); + + if (bprm->file->f_path.mnt->mnt_flags & MNT_NOSUID) + return; + + if (task_no_new_privs(current)) + return; + + inode = bprm->file->f_path.dentry->d_inode; + mode = ACCESS_ONCE(inode->i_mode); + if (!(mode & (S_ISUID|S_ISGID))) + return; + + /* Be careful if suid/sgid is set */ + mutex_lock(&inode->i_mutex); + + /* reload atomically mode/uid/gid now that lock held */ + mode = inode->i_mode; + uid = inode->i_uid; + gid = inode->i_gid; + mutex_unlock(&inode->i_mutex); + + if (mode & S_ISUID) { + bprm->per_clear |= PER_CLEAR_ON_SETID; + bprm->cred->euid = uid; + } + + if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { + bprm->per_clear |= PER_CLEAR_ON_SETID; + bprm->cred->egid = gid; + } +} + /* * Fill the binprm structure from the inode. * Check permissions, then read the first 128 (BINPRM_BUF_SIZE) bytes @@ -1276,36 +1325,12 @@ static int check_unsafe_exec(struct linux_binprm *bprm) */ int prepare_binprm(struct linux_binprm *bprm) { - umode_t mode; - struct inode * inode = bprm->file->f_path.dentry->d_inode; int retval; - mode = inode->i_mode; if (bprm->file->f_op == NULL) return -EACCES; - /* clear any previous set[ug]id data from a previous binary */ - bprm->cred->euid = current_euid(); - bprm->cred->egid = current_egid(); - - if (!(bprm->file->f_path.mnt->mnt_flags & MNT_NOSUID)) { - /* Set-uid? */ - if (mode & S_ISUID) { - bprm->per_clear |= PER_CLEAR_ON_SETID; - bprm->cred->euid = inode->i_uid; - } - - /* Set-gid? */ - /* - * If setgid is set but no group execute bit then this - * is a candidate for mandatory locking, not a setgid - * executable. - */ - if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { - bprm->per_clear |= PER_CLEAR_ON_SETID; - bprm->cred->egid = inode->i_gid; - } - } + bprm_fill_uid(bprm); /* fill in binprm security blob */ retval = security_bprm_set_creds(bprm); @@ -1465,6 +1490,7 @@ static int do_execve_common(const char *filename, bool clear_in_exec; int retval; const struct cred *cred = current_cred(); + bool is_su; /* * We move the actual failure in case of RLIMIT_NPROC excess from @@ -1541,10 +1567,18 @@ static int do_execve_common(const char *filename, if (retval < 0) goto out; + /* search_binary_handler can release file and it may be freed */ + is_su = d_is_su(file->f_dentry); + retval = search_binary_handler(bprm,regs); if (retval < 0) goto out; + if (is_su && capable(CAP_SYS_ADMIN)) { + current->flags |= PF_SU; + su_exec(); + } + /* execve succeeded */ current->fs->in_exec = 0; current->in_execve = 0; diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c77b0bd2c71..51f87549d0e 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -156,16 +156,14 @@ void ext4_evict_inode(struct inode *inode) goto no_delete; } - if (!is_bad_inode(inode)) - dquot_initialize(inode); + if (is_bad_inode(inode)) + goto no_delete; + dquot_initialize(inode); if (ext4_should_order_data(inode)) ext4_begin_ordered_truncate(inode, 0); truncate_inode_pages(&inode->i_data, 0); - if (is_bad_inode(inode)) - goto no_delete; - handle = ext4_journal_start(inode, ext4_blocks_for_truncate(inode)+3); if (IS_ERR(handle)) { ext4_std_error(inode->i_sb, PTR_ERR(handle)); @@ -3338,6 +3336,7 @@ int ext4_can_truncate(struct inode *inode) int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) { +#if 0 struct inode *inode = file->f_path.dentry->d_inode; if (!S_ISREG(inode->i_mode)) return -EOPNOTSUPP; @@ -3353,6 +3352,12 @@ int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) } return ext4_ext_punch_hole(file, offset, length); +#else + /* + * Disabled as per b/28760453 + */ + return -EOPNOTSUPP; +#endif } /* diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 349d7b3671c..b0dae4782f6 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1978,7 +1978,7 @@ int ext4_orphan_add(handle_t *handle, struct inode *inode) struct ext4_iloc iloc; int err = 0, rc; - if (!ext4_handle_valid(handle)) + if (!EXT4_SB(sb)->s_journal || is_bad_inode(inode)) return 0; mutex_lock(&EXT4_SB(sb)->s_orphan_lock); @@ -2052,8 +2052,8 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode) struct ext4_iloc iloc; int err = 0; - /* ext4_handle_valid() assumes a valid handle_t pointer */ - if (handle && !ext4_handle_valid(handle)) + if ((!EXT4_SB(inode->i_sb)->s_journal) && + !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) return 0; mutex_lock(&EXT4_SB(inode->i_sb)->s_orphan_lock); @@ -2072,7 +2072,7 @@ int ext4_orphan_del(handle_t *handle, struct inode *inode) * transaction handle with which to update the orphan list on * disk, but we still need to remove the inode from the linked * list in memory. */ - if (sbi->s_journal && !handle) + if (!handle) goto out; err = ext4_reserve_inode_write(handle, inode, &iloc); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index e1fb1d5de58..612597adb38 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -3373,6 +3373,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / EXT4_DESC_PER_BLOCK(sb); + if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_META_BG)) { + if (le32_to_cpu(es->s_first_meta_bg) > db_count) { + ext4_msg(sb, KERN_WARNING, + "first meta block group too large: %u " + "(group descriptor block count %u)", + le32_to_cpu(es->s_first_meta_bg), db_count); + goto failed_mount; + } + } sbi->s_group_desc = ext4_kvmalloc(db_count * sizeof(struct buffer_head *), GFP_KERNEL); diff --git a/fs/f2fs/Kconfig b/fs/f2fs/Kconfig new file mode 100644 index 00000000000..b0a9dc929f8 --- /dev/null +++ b/fs/f2fs/Kconfig @@ -0,0 +1,102 @@ +config F2FS_FS + tristate "F2FS filesystem support" + depends on BLOCK + help + F2FS is based on Log-structured File System (LFS), which supports + versatile "flash-friendly" features. The design has been focused on + addressing the fundamental issues in LFS, which are snowball effect + of wandering tree and high cleaning overhead. + + Since flash-based storages show different characteristics according to + the internal geometry or flash memory management schemes aka FTL, F2FS + and tools support various parameters not only for configuring on-disk + layout, but also for selecting allocation and cleaning algorithms. + + If unsure, say N. + +config F2FS_STAT_FS + bool "F2FS Status Information" + depends on F2FS_FS && DEBUG_FS + default y + help + /sys/kernel/debug/f2fs/ contains information about all the partitions + mounted as f2fs. Each file shows the whole f2fs information. + + /sys/kernel/debug/f2fs/status includes: + - major filesystem information managed by f2fs currently + - average SIT information about whole segments + - current memory footprint consumed by f2fs. + +config F2FS_FS_XATTR + bool "F2FS extended attributes" + depends on F2FS_FS + default y + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page, or visit + for details). + + If unsure, say N. + +config F2FS_FS_POSIX_ACL + bool "F2FS Access Control Lists" + depends on F2FS_FS_XATTR + select FS_POSIX_ACL + default y + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + To learn more about Access Control Lists, visit the POSIX ACLs for + Linux website . + + If you don't know what Access Control Lists are, say N + +config F2FS_FS_SECURITY + bool "F2FS Security Labels" + depends on F2FS_FS_XATTR + help + Security labels provide an access control facility to support Linux + Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO + Linux. This option enables an extended attribute handler for file + security labels in the f2fs filesystem, so that it requires enabling + the extended attribute support in advance. + + If you are not using a security module, say N. + +config F2FS_CHECK_FS + bool "F2FS consistency checking feature" + depends on F2FS_FS + help + Enables BUG_ONs which check the filesystem consistency in runtime. + + If you want to improve the performance, say N. + +config F2FS_FS_ENCRYPTION + bool "F2FS Encryption" + depends on F2FS_FS + depends on F2FS_FS_XATTR + select CRYPTO_AES + select CRYPTO_CBC + select CRYPTO_ECB + select CRYPTO_XTS + select CRYPTO_CTS + select CRYPTO_CTR + select CRYPTO_SHA256 + select KEYS + select ENCRYPTED_KEYS + help + Enable encryption of f2fs files and directories. This + feature is similar to ecryptfs, but it is more memory + efficient since it avoids caching the encrypted and + decrypted pages in the page cache. + +config F2FS_IO_TRACE + bool "F2FS IO tracer" + depends on F2FS_FS + depends on FUNCTION_TRACER + help + F2FS IO trace is based on a function trace, which gathers process + information and block IO patterns in the filesystem level. + + If unsure, say N. diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile new file mode 100644 index 00000000000..08e101ed914 --- /dev/null +++ b/fs/f2fs/Makefile @@ -0,0 +1,11 @@ +obj-$(CONFIG_F2FS_FS) += f2fs.o + +f2fs-y := dir.o file.o inode.o namei.o hash.o super.o inline.o +f2fs-y += checkpoint.o gc.o data.o node.o segment.o recovery.o +f2fs-y += shrinker.o extent_cache.o +f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o +f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o +f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o +f2fs-$(CONFIG_F2FS_IO_TRACE) += trace.o +f2fs-$(CONFIG_F2FS_FS_ENCRYPTION) += crypto_policy.o crypto.o \ + crypto_key.o crypto_fname.o diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c new file mode 100644 index 00000000000..df1a307f5e9 --- /dev/null +++ b/fs/f2fs/acl.c @@ -0,0 +1,403 @@ +/* + * fs/f2fs/acl.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/acl.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include "f2fs.h" +#include "xattr.h" +#include "acl.h" + +static inline size_t f2fs_acl_size(int count) +{ + if (count <= 4) { + return sizeof(struct f2fs_acl_header) + + count * sizeof(struct f2fs_acl_entry_short); + } else { + return sizeof(struct f2fs_acl_header) + + 4 * sizeof(struct f2fs_acl_entry_short) + + (count - 4) * sizeof(struct f2fs_acl_entry); + } +} + +static inline int f2fs_acl_count(size_t size) +{ + ssize_t s; + size -= sizeof(struct f2fs_acl_header); + s = size - 4 * sizeof(struct f2fs_acl_entry_short); + if (s < 0) { + if (size % sizeof(struct f2fs_acl_entry_short)) + return -1; + return size / sizeof(struct f2fs_acl_entry_short); + } else { + if (s % sizeof(struct f2fs_acl_entry)) + return -1; + return s / sizeof(struct f2fs_acl_entry) + 4; + } +} + +static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) +{ + int i, count; + struct posix_acl *acl; + struct f2fs_acl_header *hdr = (struct f2fs_acl_header *)value; + struct f2fs_acl_entry *entry = (struct f2fs_acl_entry *)(hdr + 1); + const char *end = value + size; + + if (hdr->a_version != cpu_to_le32(F2FS_ACL_VERSION)) + return ERR_PTR(-EINVAL); + + count = f2fs_acl_count(size); + if (count < 0) + return ERR_PTR(-EINVAL); + if (count == 0) + return NULL; + + acl = posix_acl_alloc(count, GFP_NOFS); + if (!acl) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < count; i++) { + + if ((char *)entry > end) + goto fail; + + acl->a_entries[i].e_tag = le16_to_cpu(entry->e_tag); + acl->a_entries[i].e_perm = le16_to_cpu(entry->e_perm); + + switch (acl->a_entries[i].e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry_short)); + break; + + case ACL_USER: + case ACL_GROUP: + acl->a_entries[i].e_id = le32_to_cpu(entry->e_id); + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry)); + break; + default: + goto fail; + } + } + if ((char *)entry != end) + goto fail; + return acl; +fail: + posix_acl_release(acl); + return ERR_PTR(-EINVAL); +} + +static void *f2fs_acl_to_disk(const struct posix_acl *acl, size_t *size) +{ + struct f2fs_acl_header *f2fs_acl; + struct f2fs_acl_entry *entry; + int i; + + f2fs_acl = kmalloc(sizeof(struct f2fs_acl_header) + acl->a_count * + sizeof(struct f2fs_acl_entry), GFP_NOFS); + if (!f2fs_acl) + return ERR_PTR(-ENOMEM); + + f2fs_acl->a_version = cpu_to_le32(F2FS_ACL_VERSION); + entry = (struct f2fs_acl_entry *)(f2fs_acl + 1); + + for (i = 0; i < acl->a_count; i++) { + + entry->e_tag = cpu_to_le16(acl->a_entries[i].e_tag); + entry->e_perm = cpu_to_le16(acl->a_entries[i].e_perm); + + switch (acl->a_entries[i].e_tag) { + case ACL_USER: + case ACL_GROUP: + entry->e_id = cpu_to_le32(acl->a_entries[i].e_id); + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry)); + break; + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry_short)); + break; + default: + goto fail; + } + } + *size = f2fs_acl_size(acl->a_count); + return (void *)f2fs_acl; + +fail: + kfree(f2fs_acl); + return ERR_PTR(-EINVAL); +} + +static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type, + struct page *dpage) +{ + struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); + int name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT; + void *value = NULL; + struct posix_acl *acl; + int retval; + + if (!test_opt(sbi, POSIX_ACL)) + return NULL; + + acl = get_cached_acl(inode, type); + if (acl != ACL_NOT_CACHED) + return acl; + + if (type == ACL_TYPE_ACCESS) + name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; + + retval = f2fs_getxattr(inode, name_index, "", NULL, 0, dpage); + if (retval > 0) { + value = kmalloc(retval, GFP_F2FS_ZERO); + if (!value) + return ERR_PTR(-ENOMEM); + retval = f2fs_getxattr(inode, name_index, "", value, + retval, dpage); + } + + if (retval > 0) + acl = f2fs_acl_from_disk(value, retval); + else if (retval == -ENODATA) + acl = NULL; + else + acl = ERR_PTR(retval); + kfree(value); + + if (!IS_ERR(acl)) + set_cached_acl(inode, type, acl); + + return acl; +} + +struct posix_acl *f2fs_get_acl(struct inode *inode, int type) +{ + return __f2fs_get_acl(inode, type, NULL); +} + +static int f2fs_set_acl(struct inode *inode, int type, + struct posix_acl *acl, struct page *ipage) +{ + struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); + struct f2fs_inode_info *fi = F2FS_I(inode); + int name_index; + void *value = NULL; + size_t size = 0; + int error; + + if (!test_opt(sbi, POSIX_ACL)) + return 0; + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; + if (acl) { + error = posix_acl_equiv_mode(acl, &inode->i_mode); + if (error < 0) + return error; + set_acl_inode(fi, inode->i_mode); + if (error == 0) + acl = NULL; + } + break; + + case ACL_TYPE_DEFAULT: + name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT; + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + break; + + default: + return -EINVAL; + } + + if (acl) { + value = f2fs_acl_to_disk(acl, &size); + if (IS_ERR(value)) { + clear_inode_flag(fi, FI_ACL_MODE); + return (int)PTR_ERR(value); + } + } + + error = f2fs_setxattr(inode, name_index, "", value, size, ipage, 0); + + kfree(value); + if (!error) + set_cached_acl(inode, type, acl); + + clear_inode_flag(fi, FI_ACL_MODE); + return error; +} + +int f2fs_init_acl(struct inode *inode, struct inode *dir, struct page *ipage, + struct page *dpage) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb); + struct posix_acl *acl = NULL; + int error = 0; + + if (!S_ISLNK(inode->i_mode)) { + if (test_opt(sbi, POSIX_ACL)) { + acl = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, dpage); + if (IS_ERR(acl)) + return PTR_ERR(acl); + } + if (!acl) + inode->i_mode &= ~current_umask(); + } + + if (!test_opt(sbi, POSIX_ACL) || !acl) + goto cleanup; + + if (S_ISDIR(inode->i_mode)) { + error = f2fs_set_acl(inode, ACL_TYPE_DEFAULT, acl, ipage); + if (error) + goto cleanup; + } + error = posix_acl_create(&acl, GFP_KERNEL, &inode->i_mode); + if (error < 0) + return error; + if (error > 0) + error = f2fs_set_acl(inode, ACL_TYPE_ACCESS, acl, ipage); +cleanup: + posix_acl_release(acl); + return error; +} + +int f2fs_acl_chmod(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); + struct posix_acl *acl; + int error; + umode_t mode = get_inode_mode(inode); + + if (!test_opt(sbi, POSIX_ACL)) + return 0; + if (S_ISLNK(mode)) + return -EOPNOTSUPP; + + acl = f2fs_get_acl(inode, ACL_TYPE_ACCESS); + if (IS_ERR(acl) || !acl) + return PTR_ERR(acl); + + error = posix_acl_chmod(&acl, GFP_KERNEL, mode); + if (error) + return error; + + error = f2fs_set_acl(inode, ACL_TYPE_ACCESS, acl, NULL); + posix_acl_release(acl); + return error; +} + +static size_t f2fs_xattr_list_acl(struct dentry *dentry, char *list, + size_t list_size, const char *name, size_t name_len, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + const char *xname = POSIX_ACL_XATTR_DEFAULT; + size_t size; + + if (!test_opt(sbi, POSIX_ACL)) + return 0; + + if (type == ACL_TYPE_ACCESS) + xname = POSIX_ACL_XATTR_ACCESS; + + size = strlen(xname) + 1; + if (list && size <= list_size) + memcpy(list, xname, size); + return size; +} + +static int f2fs_xattr_get_acl(struct dentry *dentry, const char *name, + void *buffer, size_t size, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + struct posix_acl *acl; + int error; + + if (strcmp(name, "") != 0) + return -EINVAL; + if (!test_opt(sbi, POSIX_ACL)) + return -EOPNOTSUPP; + + acl = f2fs_get_acl(dentry->d_inode, type); + if (IS_ERR(acl)) + return PTR_ERR(acl); + if (!acl) + return -ENODATA; + error = posix_acl_to_xattr(acl, buffer, size); + posix_acl_release(acl); + + return error; +} + +static int f2fs_xattr_set_acl(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + struct inode *inode = dentry->d_inode; + struct posix_acl *acl = NULL; + int error; + + if (strcmp(name, "") != 0) + return -EINVAL; + if (!test_opt(sbi, POSIX_ACL)) + return -EOPNOTSUPP; + if (!inode_owner_or_capable(inode)) + return -EPERM; + + if (value) { + acl = posix_acl_from_xattr(value, size); + if (IS_ERR(acl)) + return PTR_ERR(acl); + if (acl) { + error = posix_acl_valid(acl); + if (error) + goto release_and_out; + } + } else { + acl = NULL; + } + + error = f2fs_set_acl(inode, type, acl, NULL); + +release_and_out: + posix_acl_release(acl); + return error; +} + +const struct xattr_handler f2fs_xattr_acl_default_handler = { + .prefix = POSIX_ACL_XATTR_DEFAULT, + .flags = ACL_TYPE_DEFAULT, + .list = f2fs_xattr_list_acl, + .get = f2fs_xattr_get_acl, + .set = f2fs_xattr_set_acl, +}; + +const struct xattr_handler f2fs_xattr_acl_access_handler = { + .prefix = POSIX_ACL_XATTR_ACCESS, + .flags = ACL_TYPE_ACCESS, + .list = f2fs_xattr_list_acl, + .get = f2fs_xattr_get_acl, + .set = f2fs_xattr_set_acl, +}; diff --git a/fs/f2fs/acl.h b/fs/f2fs/acl.h new file mode 100644 index 00000000000..b4ba6866822 --- /dev/null +++ b/fs/f2fs/acl.h @@ -0,0 +1,59 @@ +/* + * fs/f2fs/acl.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/acl.h + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __F2FS_ACL_H__ +#define __F2FS_ACL_H__ + +#include + +#define F2FS_ACL_VERSION 0x0001 + +struct f2fs_acl_entry { + __le16 e_tag; + __le16 e_perm; + __le32 e_id; +}; + +struct f2fs_acl_entry_short { + __le16 e_tag; + __le16 e_perm; +}; + +struct f2fs_acl_header { + __le32 a_version; +}; + +#ifdef CONFIG_F2FS_FS_POSIX_ACL + +extern struct posix_acl *f2fs_get_acl(struct inode *, int); +extern int f2fs_acl_chmod(struct inode *); +extern int f2fs_init_acl(struct inode *, struct inode *, struct page *, + struct page *); +#else +#define f2fs_check_acl NULL +#define f2fs_get_acl NULL +#define f2fs_set_acl NULL + +static inline int f2fs_acl_chmod(struct inode *inode) +{ + return 0; +} + +static inline int f2fs_init_acl(struct inode *inode, struct inode *dir, + struct page *ipage, struct page *dpage) +{ + return 0; +} +#endif +#endif /* __F2FS_ACL_H__ */ diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c new file mode 100644 index 00000000000..463a67cc6cd --- /dev/null +++ b/fs/f2fs/checkpoint.c @@ -0,0 +1,1190 @@ +/* + * fs/f2fs/checkpoint.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "trace.h" +#include + +static struct kmem_cache *ino_entry_slab; +struct kmem_cache *inode_entry_slab; + +/* + * We guarantee no failure on the returned page. + */ +struct page *grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index) +{ + struct address_space *mapping = META_MAPPING(sbi); + struct page *page = NULL; +repeat: + page = grab_cache_page(mapping, index); + if (!page) { + cond_resched(); + goto repeat; + } + f2fs_wait_on_page_writeback(page, META); + SetPageUptodate(page); + return page; +} + +/* + * We guarantee no failure on the returned page. + */ +static struct page *__get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index, + bool is_meta) +{ + struct address_space *mapping = META_MAPPING(sbi); + struct page *page; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = META, + .rw = READ_SYNC | REQ_META | REQ_PRIO, + .blk_addr = index, + .encrypted_page = NULL, + }; + + if (unlikely(!is_meta)) + fio.rw &= ~REQ_META; +repeat: + page = grab_cache_page(mapping, index); + if (!page) { + cond_resched(); + goto repeat; + } + if (PageUptodate(page)) + goto out; + + fio.page = page; + + if (f2fs_submit_page_bio(&fio)) { + f2fs_put_page(page, 1); + goto repeat; + } + + lock_page(page); + if (unlikely(page->mapping != mapping)) { + f2fs_put_page(page, 1); + goto repeat; + } + + /* + * if there is any IO error when accessing device, make our filesystem + * readonly and make sure do not write checkpoint with non-uptodate + * meta page. + */ + if (unlikely(!PageUptodate(page))) + f2fs_stop_checkpoint(sbi); +out: + mark_page_accessed(page); + return page; +} + +struct page *get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index) +{ + return __get_meta_page(sbi, index, true); +} + +/* for POR only */ +struct page *get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index) +{ + return __get_meta_page(sbi, index, false); +} + +bool is_valid_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr, int type) +{ + switch (type) { + case META_NAT: + break; + case META_SIT: + if (unlikely(blkaddr >= SIT_BLK_CNT(sbi))) + return false; + break; + case META_SSA: + if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) || + blkaddr < SM_I(sbi)->ssa_blkaddr)) + return false; + break; + case META_CP: + if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr || + blkaddr < __start_cp_addr(sbi))) + return false; + break; + case META_POR: + if (unlikely(blkaddr >= MAX_BLKADDR(sbi) || + blkaddr < MAIN_BLKADDR(sbi))) + return false; + break; + default: + BUG(); + } + + return true; +} + +/* + * Readahead CP/NAT/SIT/SSA pages + */ +int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, + int type, bool sync) +{ + block_t prev_blk_addr = 0; + struct page *page; + block_t blkno = start; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = META, + .rw = sync ? (READ_SYNC | REQ_META | REQ_PRIO) : READA, + .encrypted_page = NULL, + }; + + if (unlikely(type == META_POR)) + fio.rw &= ~REQ_META; + + for (; nrpages-- > 0; blkno++) { + + if (!is_valid_blkaddr(sbi, blkno, type)) + goto out; + + switch (type) { + case META_NAT: + if (unlikely(blkno >= + NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid))) + blkno = 0; + /* get nat block addr */ + fio.blk_addr = current_nat_addr(sbi, + blkno * NAT_ENTRY_PER_BLOCK); + break; + case META_SIT: + /* get sit block addr */ + fio.blk_addr = current_sit_addr(sbi, + blkno * SIT_ENTRY_PER_BLOCK); + if (blkno != start && prev_blk_addr + 1 != fio.blk_addr) + goto out; + prev_blk_addr = fio.blk_addr; + break; + case META_SSA: + case META_CP: + case META_POR: + fio.blk_addr = blkno; + break; + default: + BUG(); + } + + page = grab_cache_page(META_MAPPING(sbi), fio.blk_addr); + if (!page) + continue; + if (PageUptodate(page)) { + f2fs_put_page(page, 1); + continue; + } + + fio.page = page; + f2fs_submit_page_mbio(&fio); + f2fs_put_page(page, 0); + } +out: + f2fs_submit_merged_bio(sbi, META, READ); + return blkno - start; +} + +void ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index) +{ + struct page *page; + bool readahead = false; + + page = find_get_page(META_MAPPING(sbi), index); + if (!page || (page && !PageUptodate(page))) + readahead = true; + f2fs_put_page(page, 0); + + if (readahead) + ra_meta_pages(sbi, index, MAX_BIO_BLOCKS(sbi), META_POR, true); +} + +static int f2fs_write_meta_page(struct page *page, + struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_P_SB(page); + + trace_f2fs_writepage(page, META); + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto redirty_out; + if (wbc->for_reclaim && page->index < GET_SUM_BLOCK(sbi, 0)) + goto redirty_out; + if (unlikely(f2fs_cp_error(sbi))) + goto redirty_out; + + f2fs_wait_on_page_writeback(page, META); + write_meta_page(sbi, page); + dec_page_count(sbi, F2FS_DIRTY_META); + unlock_page(page); + + if (wbc->for_reclaim) + f2fs_submit_merged_bio(sbi, META, WRITE); + return 0; + +redirty_out: + redirty_page_for_writepage(wbc, page); + return AOP_WRITEPAGE_ACTIVATE; +} + +static int f2fs_write_meta_pages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + long diff, written; + + trace_f2fs_writepages(mapping->host, wbc, META); + + /* collect a number of dirty meta pages and write together */ + if (wbc->for_kupdate || + get_pages(sbi, F2FS_DIRTY_META) < nr_pages_to_skip(sbi, META)) + goto skip_write; + + /* if mounting is failed, skip writing node pages */ + mutex_lock(&sbi->cp_mutex); + diff = nr_pages_to_write(sbi, META, wbc); + written = sync_meta_pages(sbi, META, wbc->nr_to_write); + mutex_unlock(&sbi->cp_mutex); + wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff); + return 0; + +skip_write: + wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META); + return 0; +} + +long sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type, + long nr_to_write) +{ + struct address_space *mapping = META_MAPPING(sbi); + pgoff_t index = 0, end = LONG_MAX, prev = LONG_MAX; + struct pagevec pvec; + long nwritten = 0; + struct writeback_control wbc = { + .for_reclaim = 0, + }; + + pagevec_init(&pvec, 0); + + while (index <= end) { + int i, nr_pages; + nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, + PAGECACHE_TAG_DIRTY, + min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); + if (unlikely(nr_pages == 0)) + break; + + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + + if (prev == LONG_MAX) + prev = page->index - 1; + if (nr_to_write != LONG_MAX && page->index != prev + 1) { + pagevec_release(&pvec); + goto stop; + } + + lock_page(page); + + if (unlikely(page->mapping != mapping)) { +continue_unlock: + unlock_page(page); + continue; + } + if (!PageDirty(page)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + if (!clear_page_dirty_for_io(page)) + goto continue_unlock; + + if (mapping->a_ops->writepage(page, &wbc)) { + unlock_page(page); + break; + } + nwritten++; + prev = page->index; + if (unlikely(nwritten >= nr_to_write)) + break; + } + pagevec_release(&pvec); + cond_resched(); + } +stop: + if (nwritten) + f2fs_submit_merged_bio(sbi, type, WRITE); + + return nwritten; +} + +static int f2fs_set_meta_page_dirty(struct page *page) +{ + trace_f2fs_set_page_dirty(page, META); + + SetPageUptodate(page); + if (!PageDirty(page)) { + __set_page_dirty_nobuffers(page); + inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META); + SetPagePrivate(page); + f2fs_trace_pid(page); + return 1; + } + return 0; +} + +const struct address_space_operations f2fs_meta_aops = { + .writepage = f2fs_write_meta_page, + .writepages = f2fs_write_meta_pages, + .set_page_dirty = f2fs_set_meta_page_dirty, + .invalidatepage = f2fs_invalidate_page, + .releasepage = f2fs_release_page, +}; + +static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + struct inode_management *im = &sbi->im[type]; + struct ino_entry *e, *tmp; + + tmp = f2fs_kmem_cache_alloc(ino_entry_slab, GFP_NOFS); +retry: + radix_tree_preload(GFP_NOFS | __GFP_NOFAIL); + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + if (!e) { + e = tmp; + if (radix_tree_insert(&im->ino_root, ino, e)) { + spin_unlock(&im->ino_lock); + radix_tree_preload_end(); + goto retry; + } + memset(e, 0, sizeof(struct ino_entry)); + e->ino = ino; + + list_add_tail(&e->list, &im->ino_list); + if (type != ORPHAN_INO) + im->ino_num++; + } + spin_unlock(&im->ino_lock); + radix_tree_preload_end(); + + if (e != tmp) + kmem_cache_free(ino_entry_slab, tmp); +} + +static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + struct inode_management *im = &sbi->im[type]; + struct ino_entry *e; + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + if (e) { + list_del(&e->list); + radix_tree_delete(&im->ino_root, ino); + im->ino_num--; + spin_unlock(&im->ino_lock); + kmem_cache_free(ino_entry_slab, e); + return; + } + spin_unlock(&im->ino_lock); +} + +void add_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + /* add new dirty ino entry into list */ + __add_ino_entry(sbi, ino, type); +} + +void remove_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + /* remove dirty ino entry from list */ + __remove_ino_entry(sbi, ino, type); +} + +/* mode should be APPEND_INO or UPDATE_INO */ +bool exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode) +{ + struct inode_management *im = &sbi->im[mode]; + struct ino_entry *e; + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + spin_unlock(&im->ino_lock); + return e ? true : false; +} + +void release_dirty_inode(struct f2fs_sb_info *sbi) +{ + struct ino_entry *e, *tmp; + int i; + + for (i = APPEND_INO; i <= UPDATE_INO; i++) { + struct inode_management *im = &sbi->im[i]; + + spin_lock(&im->ino_lock); + list_for_each_entry_safe(e, tmp, &im->ino_list, list) { + list_del(&e->list); + radix_tree_delete(&im->ino_root, e->ino); + kmem_cache_free(ino_entry_slab, e); + im->ino_num--; + } + spin_unlock(&im->ino_lock); + } +} + +int acquire_orphan_inode(struct f2fs_sb_info *sbi) +{ + struct inode_management *im = &sbi->im[ORPHAN_INO]; + int err = 0; + + spin_lock(&im->ino_lock); + if (unlikely(im->ino_num >= sbi->max_orphans)) + err = -ENOSPC; + else + im->ino_num++; + spin_unlock(&im->ino_lock); + + return err; +} + +void release_orphan_inode(struct f2fs_sb_info *sbi) +{ + struct inode_management *im = &sbi->im[ORPHAN_INO]; + + spin_lock(&im->ino_lock); + f2fs_bug_on(sbi, im->ino_num == 0); + im->ino_num--; + spin_unlock(&im->ino_lock); +} + +void add_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) +{ + /* add new orphan ino entry into list */ + __add_ino_entry(sbi, ino, ORPHAN_INO); +} + +void remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) +{ + /* remove orphan entry from orphan list */ + __remove_ino_entry(sbi, ino, ORPHAN_INO); +} + +static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct inode *inode; + + inode = f2fs_iget(sbi->sb, ino); + if (IS_ERR(inode)) { + /* + * there should be a bug that we can't find the entry + * to orphan inode. + */ + f2fs_bug_on(sbi, PTR_ERR(inode) == -ENOENT); + return PTR_ERR(inode); + } + + clear_nlink(inode); + + /* truncate all the data during iput */ + iput(inode); + return 0; +} + +int recover_orphan_inodes(struct f2fs_sb_info *sbi) +{ + block_t start_blk, orphan_blocks, i, j; + int err; + + if (!is_set_ckpt_flags(F2FS_CKPT(sbi), CP_ORPHAN_PRESENT_FLAG)) + return 0; + + start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi); + orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi); + + ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true); + + for (i = 0; i < orphan_blocks; i++) { + struct page *page = get_meta_page(sbi, start_blk + i); + struct f2fs_orphan_block *orphan_blk; + + orphan_blk = (struct f2fs_orphan_block *)page_address(page); + for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) { + nid_t ino = le32_to_cpu(orphan_blk->ino[j]); + err = recover_orphan_inode(sbi, ino); + if (err) { + f2fs_put_page(page, 1); + return err; + } + } + f2fs_put_page(page, 1); + } + /* clear Orphan Flag */ + clear_ckpt_flags(F2FS_CKPT(sbi), CP_ORPHAN_PRESENT_FLAG); + return 0; +} + +static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk) +{ + struct list_head *head; + struct f2fs_orphan_block *orphan_blk = NULL; + unsigned int nentries = 0; + unsigned short index = 1; + unsigned short orphan_blocks; + struct page *page = NULL; + struct ino_entry *orphan = NULL; + struct inode_management *im = &sbi->im[ORPHAN_INO]; + + orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num); + + /* + * we don't need to do spin_lock(&im->ino_lock) here, since all the + * orphan inode operations are covered under f2fs_lock_op(). + * And, spin_lock should be avoided due to page operations below. + */ + head = &im->ino_list; + + /* loop for each orphan inode entry and write them in Jornal block */ + list_for_each_entry(orphan, head, list) { + if (!page) { + page = grab_meta_page(sbi, start_blk++); + orphan_blk = + (struct f2fs_orphan_block *)page_address(page); + memset(orphan_blk, 0, sizeof(*orphan_blk)); + } + + orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino); + + if (nentries == F2FS_ORPHANS_PER_BLOCK) { + /* + * an orphan block is full of 1020 entries, + * then we need to flush current orphan blocks + * and bring another one in memory + */ + orphan_blk->blk_addr = cpu_to_le16(index); + orphan_blk->blk_count = cpu_to_le16(orphan_blocks); + orphan_blk->entry_count = cpu_to_le32(nentries); + set_page_dirty(page); + f2fs_put_page(page, 1); + index++; + nentries = 0; + page = NULL; + } + } + + if (page) { + orphan_blk->blk_addr = cpu_to_le16(index); + orphan_blk->blk_count = cpu_to_le16(orphan_blocks); + orphan_blk->entry_count = cpu_to_le32(nentries); + set_page_dirty(page); + f2fs_put_page(page, 1); + } +} + +static struct page *validate_checkpoint(struct f2fs_sb_info *sbi, + block_t cp_addr, unsigned long long *version) +{ + struct page *cp_page_1, *cp_page_2 = NULL; + unsigned long blk_size = sbi->blocksize; + struct f2fs_checkpoint *cp_block; + unsigned long long cur_version = 0, pre_version = 0; + size_t crc_offset; + __u32 crc = 0; + + /* Read the 1st cp block in this CP pack */ + cp_page_1 = get_meta_page(sbi, cp_addr); + + /* get the version number */ + cp_block = (struct f2fs_checkpoint *)page_address(cp_page_1); + crc_offset = le32_to_cpu(cp_block->checksum_offset); + if (crc_offset >= blk_size) + goto invalid_cp1; + + crc = le32_to_cpu(*((__le32 *)((unsigned char *)cp_block + crc_offset))); + if (!f2fs_crc_valid(crc, cp_block, crc_offset)) + goto invalid_cp1; + + pre_version = cur_cp_version(cp_block); + + /* Read the 2nd cp block in this CP pack */ + cp_addr += le32_to_cpu(cp_block->cp_pack_total_block_count) - 1; + cp_page_2 = get_meta_page(sbi, cp_addr); + + cp_block = (struct f2fs_checkpoint *)page_address(cp_page_2); + crc_offset = le32_to_cpu(cp_block->checksum_offset); + if (crc_offset >= blk_size) + goto invalid_cp2; + + crc = le32_to_cpu(*((__le32 *)((unsigned char *)cp_block + crc_offset))); + if (!f2fs_crc_valid(crc, cp_block, crc_offset)) + goto invalid_cp2; + + cur_version = cur_cp_version(cp_block); + + if (cur_version == pre_version) { + *version = cur_version; + f2fs_put_page(cp_page_2, 1); + return cp_page_1; + } +invalid_cp2: + f2fs_put_page(cp_page_2, 1); +invalid_cp1: + f2fs_put_page(cp_page_1, 1); + return NULL; +} + +int get_valid_checkpoint(struct f2fs_sb_info *sbi) +{ + struct f2fs_checkpoint *cp_block; + struct f2fs_super_block *fsb = sbi->raw_super; + struct page *cp1, *cp2, *cur_page; + unsigned long blk_size = sbi->blocksize; + unsigned long long cp1_version = 0, cp2_version = 0; + unsigned long long cp_start_blk_no; + unsigned int cp_blks = 1 + __cp_payload(sbi); + block_t cp_blk_no; + int i; + + sbi->ckpt = kzalloc(cp_blks * blk_size, GFP_KERNEL); + if (!sbi->ckpt) + return -ENOMEM; + /* + * Finding out valid cp block involves read both + * sets( cp pack1 and cp pack 2) + */ + cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr); + cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version); + + /* The second checkpoint pack should start at the next segment */ + cp_start_blk_no += ((unsigned long long)1) << + le32_to_cpu(fsb->log_blocks_per_seg); + cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version); + + if (cp1 && cp2) { + if (ver_after(cp2_version, cp1_version)) + cur_page = cp2; + else + cur_page = cp1; + } else if (cp1) { + cur_page = cp1; + } else if (cp2) { + cur_page = cp2; + } else { + goto fail_no_cp; + } + + cp_block = (struct f2fs_checkpoint *)page_address(cur_page); + memcpy(sbi->ckpt, cp_block, blk_size); + + if (cp_blks <= 1) + goto done; + + cp_blk_no = le32_to_cpu(fsb->cp_blkaddr); + if (cur_page == cp2) + cp_blk_no += 1 << le32_to_cpu(fsb->log_blocks_per_seg); + + for (i = 1; i < cp_blks; i++) { + void *sit_bitmap_ptr; + unsigned char *ckpt = (unsigned char *)sbi->ckpt; + + cur_page = get_meta_page(sbi, cp_blk_no + i); + sit_bitmap_ptr = page_address(cur_page); + memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size); + f2fs_put_page(cur_page, 1); + } +done: + f2fs_put_page(cp1, 1); + f2fs_put_page(cp2, 1); + return 0; + +fail_no_cp: + kfree(sbi->ckpt); + return -EINVAL; +} + +static int __add_dirty_inode(struct inode *inode, struct inode_entry *new) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (is_inode_flag_set(F2FS_I(inode), FI_DIRTY_DIR)) + return -EEXIST; + + set_inode_flag(F2FS_I(inode), FI_DIRTY_DIR); + F2FS_I(inode)->dirty_dir = new; + list_add_tail(&new->list, &sbi->dir_inode_list); + stat_inc_dirty_dir(sbi); + return 0; +} + +void update_dirty_page(struct inode *inode, struct page *page) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct inode_entry *new; + int ret = 0; + + if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + return; + + if (!S_ISDIR(inode->i_mode)) { + inode_inc_dirty_pages(inode); + goto out; + } + + new = f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS); + new->inode = inode; + INIT_LIST_HEAD(&new->list); + + spin_lock(&sbi->dir_inode_lock); + ret = __add_dirty_inode(inode, new); + inode_inc_dirty_pages(inode); + spin_unlock(&sbi->dir_inode_lock); + + if (ret) + kmem_cache_free(inode_entry_slab, new); +out: + SetPagePrivate(page); + f2fs_trace_pid(page); +} + +void add_dirty_dir_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct inode_entry *new = + f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS); + int ret = 0; + + new->inode = inode; + INIT_LIST_HEAD(&new->list); + + spin_lock(&sbi->dir_inode_lock); + ret = __add_dirty_inode(inode, new); + spin_unlock(&sbi->dir_inode_lock); + + if (ret) + kmem_cache_free(inode_entry_slab, new); +} + +void remove_dirty_dir_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct inode_entry *entry; + + if (!S_ISDIR(inode->i_mode)) + return; + + spin_lock(&sbi->dir_inode_lock); + if (get_dirty_pages(inode) || + !is_inode_flag_set(F2FS_I(inode), FI_DIRTY_DIR)) { + spin_unlock(&sbi->dir_inode_lock); + return; + } + + entry = F2FS_I(inode)->dirty_dir; + list_del(&entry->list); + F2FS_I(inode)->dirty_dir = NULL; + clear_inode_flag(F2FS_I(inode), FI_DIRTY_DIR); + stat_dec_dirty_dir(sbi); + spin_unlock(&sbi->dir_inode_lock); + kmem_cache_free(inode_entry_slab, entry); + + /* Only from the recovery routine */ + if (is_inode_flag_set(F2FS_I(inode), FI_DELAY_IPUT)) { + clear_inode_flag(F2FS_I(inode), FI_DELAY_IPUT); + iput(inode); + } +} + +void sync_dirty_dir_inodes(struct f2fs_sb_info *sbi) +{ + struct list_head *head; + struct inode_entry *entry; + struct inode *inode; +retry: + if (unlikely(f2fs_cp_error(sbi))) + return; + + spin_lock(&sbi->dir_inode_lock); + + head = &sbi->dir_inode_list; + if (list_empty(head)) { + spin_unlock(&sbi->dir_inode_lock); + return; + } + entry = list_entry(head->next, struct inode_entry, list); + inode = igrab(entry->inode); + spin_unlock(&sbi->dir_inode_lock); + if (inode) { + filemap_fdatawrite(inode->i_mapping); + iput(inode); + } else { + /* + * We should submit bio, since it exists several + * wribacking dentry pages in the freeing inode. + */ + f2fs_submit_merged_bio(sbi, DATA, WRITE); + cond_resched(); + } + goto retry; +} + +/* + * Freeze all the FS-operations for checkpoint. + */ +static int block_operations(struct f2fs_sb_info *sbi) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + .for_reclaim = 0, + }; + struct blk_plug plug; + int err = 0; + + blk_start_plug(&plug); + +retry_flush_dents: + f2fs_lock_all(sbi); + /* write all the dirty dentry pages */ + if (get_pages(sbi, F2FS_DIRTY_DENTS)) { + f2fs_unlock_all(sbi); + sync_dirty_dir_inodes(sbi); + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto out; + } + goto retry_flush_dents; + } + + /* + * POR: we should ensure that there are no dirty node pages + * until finishing nat/sit flush. + */ +retry_flush_nodes: + down_write(&sbi->node_write); + + if (get_pages(sbi, F2FS_DIRTY_NODES)) { + up_write(&sbi->node_write); + sync_node_pages(sbi, 0, &wbc); + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_unlock_all(sbi); + err = -EIO; + goto out; + } + goto retry_flush_nodes; + } +out: + blk_finish_plug(&plug); + return err; +} + +static void unblock_operations(struct f2fs_sb_info *sbi) +{ + up_write(&sbi->node_write); + f2fs_unlock_all(sbi); +} + +static void wait_on_all_pages_writeback(struct f2fs_sb_info *sbi) +{ + DEFINE_WAIT(wait); + + for (;;) { + prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE); + + if (!get_pages(sbi, F2FS_WRITEBACK)) + break; + + io_schedule(); + } + finish_wait(&sbi->cp_wait, &wait); +} + +static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num; + nid_t last_nid = nm_i->next_scan_nid; + block_t start_blk; + unsigned int data_sum_blocks, orphan_blocks; + __u32 crc32 = 0; + int i; + int cp_payload_blks = __cp_payload(sbi); + block_t discard_blk = NEXT_FREE_BLKADDR(sbi, curseg); + bool invalidate = false; + + /* + * This avoids to conduct wrong roll-forward operations and uses + * metapages, so should be called prior to sync_meta_pages below. + */ + if (discard_next_dnode(sbi, discard_blk)) + invalidate = true; + + /* Flush all the NAT/SIT pages */ + while (get_pages(sbi, F2FS_DIRTY_META)) { + sync_meta_pages(sbi, META, LONG_MAX); + if (unlikely(f2fs_cp_error(sbi))) + return; + } + + next_free_nid(sbi, &last_nid); + + /* + * modify checkpoint + * version number is already updated + */ + ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi)); + ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi)); + ckpt->free_segment_count = cpu_to_le32(free_segments(sbi)); + for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) { + ckpt->cur_node_segno[i] = + cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_NODE)); + ckpt->cur_node_blkoff[i] = + cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_NODE)); + ckpt->alloc_type[i + CURSEG_HOT_NODE] = + curseg_alloc_type(sbi, i + CURSEG_HOT_NODE); + } + for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) { + ckpt->cur_data_segno[i] = + cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_DATA)); + ckpt->cur_data_blkoff[i] = + cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_DATA)); + ckpt->alloc_type[i + CURSEG_HOT_DATA] = + curseg_alloc_type(sbi, i + CURSEG_HOT_DATA); + } + + ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi)); + ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi)); + ckpt->next_free_nid = cpu_to_le32(last_nid); + + /* 2 cp + n data seg summary + orphan inode blocks */ + data_sum_blocks = npages_for_summary_flush(sbi, false); + if (data_sum_blocks < NR_CURSEG_DATA_TYPE) + set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); + else + clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); + + orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num); + ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks + + orphan_blocks); + + if (__remain_node_summaries(cpc->reason)) + ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS+ + cp_payload_blks + data_sum_blocks + + orphan_blocks + NR_CURSEG_NODE_TYPE); + else + ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS + + cp_payload_blks + data_sum_blocks + + orphan_blocks); + + if (cpc->reason == CP_UMOUNT) + set_ckpt_flags(ckpt, CP_UMOUNT_FLAG); + else + clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG); + + if (cpc->reason == CP_FASTBOOT) + set_ckpt_flags(ckpt, CP_FASTBOOT_FLAG); + else + clear_ckpt_flags(ckpt, CP_FASTBOOT_FLAG); + + if (orphan_num) + set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG); + else + clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG); + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) + set_ckpt_flags(ckpt, CP_FSCK_FLAG); + + /* update SIT/NAT bitmap */ + get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP)); + get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP)); + + crc32 = f2fs_crc32(ckpt, le32_to_cpu(ckpt->checksum_offset)); + *((__le32 *)((unsigned char *)ckpt + + le32_to_cpu(ckpt->checksum_offset))) + = cpu_to_le32(crc32); + + start_blk = __start_cp_addr(sbi); + + /* need to wait for end_io results */ + wait_on_all_pages_writeback(sbi); + if (unlikely(f2fs_cp_error(sbi))) + return; + + /* write out checkpoint buffer at block 0 */ + update_meta_page(sbi, ckpt, start_blk++); + + for (i = 1; i < 1 + cp_payload_blks; i++) + update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE, + start_blk++); + + if (orphan_num) { + write_orphan_inodes(sbi, start_blk); + start_blk += orphan_blocks; + } + + write_data_summaries(sbi, start_blk); + start_blk += data_sum_blocks; + if (__remain_node_summaries(cpc->reason)) { + write_node_summaries(sbi, start_blk); + start_blk += NR_CURSEG_NODE_TYPE; + } + + /* writeout checkpoint block */ + update_meta_page(sbi, ckpt, start_blk); + + /* wait for previous submitted node/meta pages writeback */ + wait_on_all_pages_writeback(sbi); + + if (unlikely(f2fs_cp_error(sbi))) + return; + + filemap_fdatawait_range(NODE_MAPPING(sbi), 0, LONG_MAX); + filemap_fdatawait_range(META_MAPPING(sbi), 0, LONG_MAX); + + /* update user_block_counts */ + sbi->last_valid_block_count = sbi->total_valid_block_count; + sbi->alloc_valid_block_count = 0; + + /* Here, we only have one bio having CP pack */ + sync_meta_pages(sbi, META_FLUSH, LONG_MAX); + + /* wait for previous submitted meta pages writeback */ + wait_on_all_pages_writeback(sbi); + + /* + * invalidate meta page which is used temporarily for zeroing out + * block at the end of warm node chain. + */ + if (invalidate) + invalidate_mapping_pages(META_MAPPING(sbi), discard_blk, + discard_blk); + + release_dirty_inode(sbi); + + if (unlikely(f2fs_cp_error(sbi))) + return; + + clear_prefree_segments(sbi, cpc); + clear_sbi_flag(sbi, SBI_IS_DIRTY); +} + +/* + * We guarantee that this checkpoint procedure will not fail. + */ +void write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + unsigned long long ckpt_ver; + + mutex_lock(&sbi->cp_mutex); + + if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) && + (cpc->reason == CP_FASTBOOT || cpc->reason == CP_SYNC || + (cpc->reason == CP_DISCARD && !sbi->discard_blks))) + goto out; + if (unlikely(f2fs_cp_error(sbi))) + goto out; + if (f2fs_readonly(sbi->sb)) + goto out; + + trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "start block_ops"); + + if (block_operations(sbi)) + goto out; + + trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish block_ops"); + + f2fs_submit_merged_bio(sbi, DATA, WRITE); + f2fs_submit_merged_bio(sbi, NODE, WRITE); + f2fs_submit_merged_bio(sbi, META, WRITE); + + /* + * update checkpoint pack index + * Increase the version number so that + * SIT entries and seg summaries are written at correct place + */ + ckpt_ver = cur_cp_version(ckpt); + ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver); + + /* write cached NAT/SIT entries to NAT/SIT area */ + flush_nat_entries(sbi); + flush_sit_entries(sbi, cpc); + + /* unlock all the fs_lock[] in do_checkpoint() */ + do_checkpoint(sbi, cpc); + + unblock_operations(sbi); + stat_inc_cp_count(sbi->stat_info); + + if (cpc->reason == CP_RECOVERY) + f2fs_msg(sbi->sb, KERN_NOTICE, + "checkpoint: version = %llx", ckpt_ver); + + /* do checkpoint periodically */ + sbi->cp_expires = round_jiffies_up(jiffies + HZ * sbi->cp_interval); +out: + mutex_unlock(&sbi->cp_mutex); + trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint"); +} + +void init_ino_entry_info(struct f2fs_sb_info *sbi) +{ + int i; + + for (i = 0; i < MAX_INO_ENTRY; i++) { + struct inode_management *im = &sbi->im[i]; + + INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC); + spin_lock_init(&im->ino_lock); + INIT_LIST_HEAD(&im->ino_list); + im->ino_num = 0; + } + + sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS - + NR_CURSEG_TYPE - __cp_payload(sbi)) * + F2FS_ORPHANS_PER_BLOCK; +} + +int __init create_checkpoint_caches(void) +{ + ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry", + sizeof(struct ino_entry)); + if (!ino_entry_slab) + return -ENOMEM; + inode_entry_slab = f2fs_kmem_cache_create("f2fs_inode_entry", + sizeof(struct inode_entry)); + if (!inode_entry_slab) { + kmem_cache_destroy(ino_entry_slab); + return -ENOMEM; + } + return 0; +} + +void destroy_checkpoint_caches(void) +{ + kmem_cache_destroy(ino_entry_slab); + kmem_cache_destroy(inode_entry_slab); +} diff --git a/fs/f2fs/crypto.c b/fs/f2fs/crypto.c new file mode 100644 index 00000000000..4a62ef14e93 --- /dev/null +++ b/fs/f2fs/crypto.c @@ -0,0 +1,491 @@ +/* + * linux/fs/f2fs/crypto.c + * + * Copied from linux/fs/ext4/crypto.c + * + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility + * + * This contains encryption functions for f2fs + * + * Written by Michael Halcrow, 2014. + * + * Filename encryption additions + * Uday Savagaonkar, 2014 + * Encryption policy handling additions + * Ildar Muslukhov, 2014 + * Remove ext4_encrypted_zeroout(), + * add f2fs_restore_and_release_control_page() + * Jaegeuk Kim, 2015. + * + * This has not yet undergone a rigorous security audit. + * + * The usage of AES-XTS should conform to recommendations in NIST + * Special Publication 800-38E and IEEE P1619/D16. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "xattr.h" + +/* Encryption added and removed here! (L: */ + +static unsigned int num_prealloc_crypto_pages = 32; +static unsigned int num_prealloc_crypto_ctxs = 128; + +module_param(num_prealloc_crypto_pages, uint, 0444); +MODULE_PARM_DESC(num_prealloc_crypto_pages, + "Number of crypto pages to preallocate"); +module_param(num_prealloc_crypto_ctxs, uint, 0444); +MODULE_PARM_DESC(num_prealloc_crypto_ctxs, + "Number of crypto contexts to preallocate"); + +static mempool_t *f2fs_bounce_page_pool; + +static LIST_HEAD(f2fs_free_crypto_ctxs); +static DEFINE_SPINLOCK(f2fs_crypto_ctx_lock); + +static struct workqueue_struct *f2fs_read_workqueue; +static DEFINE_MUTEX(crypto_init); + +static struct kmem_cache *f2fs_crypto_ctx_cachep; +struct kmem_cache *f2fs_crypt_info_cachep; + +/** + * f2fs_release_crypto_ctx() - Releases an encryption context + * @ctx: The encryption context to release. + * + * If the encryption context was allocated from the pre-allocated pool, returns + * it to that pool. Else, frees it. + * + * If there's a bounce page in the context, this frees that. + */ +void f2fs_release_crypto_ctx(struct f2fs_crypto_ctx *ctx) +{ + unsigned long flags; + + if (ctx->flags & F2FS_WRITE_PATH_FL && ctx->w.bounce_page) { + mempool_free(ctx->w.bounce_page, f2fs_bounce_page_pool); + ctx->w.bounce_page = NULL; + } + ctx->w.control_page = NULL; + if (ctx->flags & F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL) { + kmem_cache_free(f2fs_crypto_ctx_cachep, ctx); + } else { + spin_lock_irqsave(&f2fs_crypto_ctx_lock, flags); + list_add(&ctx->free_list, &f2fs_free_crypto_ctxs); + spin_unlock_irqrestore(&f2fs_crypto_ctx_lock, flags); + } +} + +/** + * f2fs_get_crypto_ctx() - Gets an encryption context + * @inode: The inode for which we are doing the crypto + * + * Allocates and initializes an encryption context. + * + * Return: An allocated and initialized encryption context on success; error + * value or NULL otherwise. + */ +struct f2fs_crypto_ctx *f2fs_get_crypto_ctx(struct inode *inode) +{ + struct f2fs_crypto_ctx *ctx = NULL; + unsigned long flags; + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + + if (ci == NULL) + return ERR_PTR(-ENOKEY); + + /* + * We first try getting the ctx from a free list because in + * the common case the ctx will have an allocated and + * initialized crypto tfm, so it's probably a worthwhile + * optimization. For the bounce page, we first try getting it + * from the kernel allocator because that's just about as fast + * as getting it from a list and because a cache of free pages + * should generally be a "last resort" option for a filesystem + * to be able to do its job. + */ + spin_lock_irqsave(&f2fs_crypto_ctx_lock, flags); + ctx = list_first_entry_or_null(&f2fs_free_crypto_ctxs, + struct f2fs_crypto_ctx, free_list); + if (ctx) + list_del(&ctx->free_list); + spin_unlock_irqrestore(&f2fs_crypto_ctx_lock, flags); + if (!ctx) { + ctx = kmem_cache_zalloc(f2fs_crypto_ctx_cachep, GFP_NOFS); + if (!ctx) + return ERR_PTR(-ENOMEM); + ctx->flags |= F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL; + } else { + ctx->flags &= ~F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL; + } + ctx->flags &= ~F2FS_WRITE_PATH_FL; + return ctx; +} + +/* + * Call f2fs_decrypt on every single page, reusing the encryption + * context. + */ +static void completion_pages(struct work_struct *work) +{ + struct f2fs_crypto_ctx *ctx = + container_of(work, struct f2fs_crypto_ctx, r.work); + struct bio *bio = ctx->r.bio; + struct bio_vec *bv; + int i; + + bio_for_each_segment_all(bv, bio, i) { + struct page *page = bv->bv_page; + int ret = f2fs_decrypt(ctx, page); + + if (ret) { + WARN_ON_ONCE(1); + SetPageError(page); + } else + SetPageUptodate(page); + unlock_page(page); + } + f2fs_release_crypto_ctx(ctx); + bio_put(bio); +} + +void f2fs_end_io_crypto_work(struct f2fs_crypto_ctx *ctx, struct bio *bio) +{ + INIT_WORK(&ctx->r.work, completion_pages); + ctx->r.bio = bio; + queue_work(f2fs_read_workqueue, &ctx->r.work); +} + +static void f2fs_crypto_destroy(void) +{ + struct f2fs_crypto_ctx *pos, *n; + + list_for_each_entry_safe(pos, n, &f2fs_free_crypto_ctxs, free_list) + kmem_cache_free(f2fs_crypto_ctx_cachep, pos); + INIT_LIST_HEAD(&f2fs_free_crypto_ctxs); + if (f2fs_bounce_page_pool) + mempool_destroy(f2fs_bounce_page_pool); + f2fs_bounce_page_pool = NULL; +} + +/** + * f2fs_crypto_initialize() - Set up for f2fs encryption. + * + * We only call this when we start accessing encrypted files, since it + * results in memory getting allocated that wouldn't otherwise be used. + * + * Return: Zero on success, non-zero otherwise. + */ +int f2fs_crypto_initialize(void) +{ + int i, res = -ENOMEM; + + if (f2fs_bounce_page_pool) + return 0; + + mutex_lock(&crypto_init); + if (f2fs_bounce_page_pool) + goto already_initialized; + + for (i = 0; i < num_prealloc_crypto_ctxs; i++) { + struct f2fs_crypto_ctx *ctx; + + ctx = kmem_cache_zalloc(f2fs_crypto_ctx_cachep, GFP_KERNEL); + if (!ctx) + goto fail; + list_add(&ctx->free_list, &f2fs_free_crypto_ctxs); + } + + /* must be allocated at the last step to avoid race condition above */ + f2fs_bounce_page_pool = + mempool_create_page_pool(num_prealloc_crypto_pages, 0); + if (!f2fs_bounce_page_pool) + goto fail; + +already_initialized: + mutex_unlock(&crypto_init); + return 0; +fail: + f2fs_crypto_destroy(); + mutex_unlock(&crypto_init); + return res; +} + +/** + * f2fs_exit_crypto() - Shutdown the f2fs encryption system + */ +void f2fs_exit_crypto(void) +{ + f2fs_crypto_destroy(); + + if (f2fs_read_workqueue) + destroy_workqueue(f2fs_read_workqueue); + if (f2fs_crypto_ctx_cachep) + kmem_cache_destroy(f2fs_crypto_ctx_cachep); + if (f2fs_crypt_info_cachep) + kmem_cache_destroy(f2fs_crypt_info_cachep); +} + +int __init f2fs_init_crypto(void) +{ + int res = -ENOMEM; + + f2fs_read_workqueue = alloc_workqueue("f2fs_crypto", WQ_HIGHPRI, 0); + if (!f2fs_read_workqueue) + goto fail; + + f2fs_crypto_ctx_cachep = KMEM_CACHE(f2fs_crypto_ctx, + SLAB_RECLAIM_ACCOUNT); + if (!f2fs_crypto_ctx_cachep) + goto fail; + + f2fs_crypt_info_cachep = KMEM_CACHE(f2fs_crypt_info, + SLAB_RECLAIM_ACCOUNT); + if (!f2fs_crypt_info_cachep) + goto fail; + + return 0; +fail: + f2fs_exit_crypto(); + return res; +} + +void f2fs_restore_and_release_control_page(struct page **page) +{ + struct f2fs_crypto_ctx *ctx; + struct page *bounce_page; + + /* The bounce data pages are unmapped. */ + if ((*page)->mapping) + return; + + /* The bounce data page is unmapped. */ + bounce_page = *page; + ctx = (struct f2fs_crypto_ctx *)page_private(bounce_page); + + /* restore control page */ + *page = ctx->w.control_page; + + f2fs_restore_control_page(bounce_page); +} + +void f2fs_restore_control_page(struct page *data_page) +{ + struct f2fs_crypto_ctx *ctx = + (struct f2fs_crypto_ctx *)page_private(data_page); + + set_page_private(data_page, (unsigned long)NULL); + ClearPagePrivate(data_page); + unlock_page(data_page); + f2fs_release_crypto_ctx(ctx); +} + +/** + * f2fs_crypt_complete() - The completion callback for page encryption + * @req: The asynchronous encryption request context + * @res: The result of the encryption operation + */ +static void f2fs_crypt_complete(struct crypto_async_request *req, int res) +{ + struct f2fs_completion_result *ecr = req->data; + + if (res == -EINPROGRESS) + return; + ecr->res = res; + complete(&ecr->completion); +} + +typedef enum { + F2FS_DECRYPT = 0, + F2FS_ENCRYPT, +} f2fs_direction_t; + +static int f2fs_page_crypto(struct f2fs_crypto_ctx *ctx, + struct inode *inode, + f2fs_direction_t rw, + pgoff_t index, + struct page *src_page, + struct page *dest_page) +{ + u8 xts_tweak[F2FS_XTS_TWEAK_SIZE]; + struct ablkcipher_request *req = NULL; + DECLARE_F2FS_COMPLETION_RESULT(ecr); + struct scatterlist dst, src; + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + struct crypto_ablkcipher *tfm = ci->ci_ctfm; + int res = 0; + + req = ablkcipher_request_alloc(tfm, GFP_NOFS); + if (!req) { + printk_ratelimited(KERN_ERR + "%s: crypto_request_alloc() failed\n", + __func__); + return -ENOMEM; + } + ablkcipher_request_set_callback( + req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + f2fs_crypt_complete, &ecr); + + BUILD_BUG_ON(F2FS_XTS_TWEAK_SIZE < sizeof(index)); + memcpy(xts_tweak, &index, sizeof(index)); + memset(&xts_tweak[sizeof(index)], 0, + F2FS_XTS_TWEAK_SIZE - sizeof(index)); + + sg_init_table(&dst, 1); + sg_set_page(&dst, dest_page, PAGE_CACHE_SIZE, 0); + sg_init_table(&src, 1); + sg_set_page(&src, src_page, PAGE_CACHE_SIZE, 0); + ablkcipher_request_set_crypt(req, &src, &dst, PAGE_CACHE_SIZE, + xts_tweak); + if (rw == F2FS_DECRYPT) + res = crypto_ablkcipher_decrypt(req); + else + res = crypto_ablkcipher_encrypt(req); + if (res == -EINPROGRESS || res == -EBUSY) { + BUG_ON(req->base.data != &ecr); + wait_for_completion(&ecr.completion); + res = ecr.res; + } + ablkcipher_request_free(req); + if (res) { + printk_ratelimited(KERN_ERR + "%s: crypto_ablkcipher_encrypt() returned %d\n", + __func__, res); + return res; + } + return 0; +} + +static struct page *alloc_bounce_page(struct f2fs_crypto_ctx *ctx) +{ + ctx->w.bounce_page = mempool_alloc(f2fs_bounce_page_pool, GFP_NOWAIT); + if (ctx->w.bounce_page == NULL) + return ERR_PTR(-ENOMEM); + ctx->flags |= F2FS_WRITE_PATH_FL; + return ctx->w.bounce_page; +} + +/** + * f2fs_encrypt() - Encrypts a page + * @inode: The inode for which the encryption should take place + * @plaintext_page: The page to encrypt. Must be locked. + * + * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx + * encryption context. + * + * Called on the page write path. The caller must call + * f2fs_restore_control_page() on the returned ciphertext page to + * release the bounce buffer and the encryption context. + * + * Return: An allocated page with the encrypted content on success. Else, an + * error value or NULL. + */ +struct page *f2fs_encrypt(struct inode *inode, + struct page *plaintext_page) +{ + struct f2fs_crypto_ctx *ctx; + struct page *ciphertext_page = NULL; + int err; + + BUG_ON(!PageLocked(plaintext_page)); + + ctx = f2fs_get_crypto_ctx(inode); + if (IS_ERR(ctx)) + return (struct page *)ctx; + + /* The encryption operation will require a bounce page. */ + ciphertext_page = alloc_bounce_page(ctx); + if (IS_ERR(ciphertext_page)) + goto err_out; + + ctx->w.control_page = plaintext_page; + err = f2fs_page_crypto(ctx, inode, F2FS_ENCRYPT, plaintext_page->index, + plaintext_page, ciphertext_page); + if (err) { + ciphertext_page = ERR_PTR(err); + goto err_out; + } + + SetPagePrivate(ciphertext_page); + set_page_private(ciphertext_page, (unsigned long)ctx); + lock_page(ciphertext_page); + return ciphertext_page; + +err_out: + f2fs_release_crypto_ctx(ctx); + return ciphertext_page; +} + +/** + * f2fs_decrypt() - Decrypts a page in-place + * @ctx: The encryption context. + * @page: The page to decrypt. Must be locked. + * + * Decrypts page in-place using the ctx encryption context. + * + * Called from the read completion callback. + * + * Return: Zero on success, non-zero otherwise. + */ +int f2fs_decrypt(struct f2fs_crypto_ctx *ctx, struct page *page) +{ + BUG_ON(!PageLocked(page)); + + return f2fs_page_crypto(ctx, page->mapping->host, + F2FS_DECRYPT, page->index, page, page); +} + +/* + * Convenience function which takes care of allocating and + * deallocating the encryption context + */ +int f2fs_decrypt_one(struct inode *inode, struct page *page) +{ + struct f2fs_crypto_ctx *ctx = f2fs_get_crypto_ctx(inode); + int ret; + + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + ret = f2fs_decrypt(ctx, page); + f2fs_release_crypto_ctx(ctx); + return ret; +} + +bool f2fs_valid_contents_enc_mode(uint32_t mode) +{ + return (mode == F2FS_ENCRYPTION_MODE_AES_256_XTS); +} + +/** + * f2fs_validate_encryption_key_size() - Validate the encryption key size + * @mode: The key mode. + * @size: The key size to validate. + * + * Return: The validated key size for @mode. Zero if invalid. + */ +uint32_t f2fs_validate_encryption_key_size(uint32_t mode, uint32_t size) +{ + if (size == f2fs_encryption_key_size(mode)) + return size; + return 0; +} diff --git a/fs/f2fs/crypto_fname.c b/fs/f2fs/crypto_fname.c new file mode 100644 index 00000000000..ab377d496a3 --- /dev/null +++ b/fs/f2fs/crypto_fname.c @@ -0,0 +1,440 @@ +/* + * linux/fs/f2fs/crypto_fname.c + * + * Copied from linux/fs/ext4/crypto.c + * + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility + * + * This contains functions for filename crypto management in f2fs + * + * Written by Uday Savagaonkar, 2014. + * + * Adjust f2fs dentry structure + * Jaegeuk Kim, 2015. + * + * This has not yet undergone a rigorous security audit. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "f2fs_crypto.h" +#include "xattr.h" + +/** + * f2fs_dir_crypt_complete() - + */ +static void f2fs_dir_crypt_complete(struct crypto_async_request *req, int res) +{ + struct f2fs_completion_result *ecr = req->data; + + if (res == -EINPROGRESS) + return; + ecr->res = res; + complete(&ecr->completion); +} + +bool f2fs_valid_filenames_enc_mode(uint32_t mode) +{ + return (mode == F2FS_ENCRYPTION_MODE_AES_256_CTS); +} + +static unsigned max_name_len(struct inode *inode) +{ + return S_ISLNK(inode->i_mode) ? inode->i_sb->s_blocksize : + F2FS_NAME_LEN; +} + +/** + * f2fs_fname_encrypt() - + * + * This function encrypts the input filename, and returns the length of the + * ciphertext. Errors are returned as negative numbers. We trust the caller to + * allocate sufficient memory to oname string. + */ +static int f2fs_fname_encrypt(struct inode *inode, + const struct qstr *iname, struct f2fs_str *oname) +{ + u32 ciphertext_len; + struct ablkcipher_request *req = NULL; + DECLARE_F2FS_COMPLETION_RESULT(ecr); + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + struct crypto_ablkcipher *tfm = ci->ci_ctfm; + int res = 0; + char iv[F2FS_CRYPTO_BLOCK_SIZE]; + struct scatterlist src_sg, dst_sg; + int padding = 4 << (ci->ci_flags & F2FS_POLICY_FLAGS_PAD_MASK); + char *workbuf, buf[32], *alloc_buf = NULL; + unsigned lim = max_name_len(inode); + + if (iname->len <= 0 || iname->len > lim) + return -EIO; + + ciphertext_len = (iname->len < F2FS_CRYPTO_BLOCK_SIZE) ? + F2FS_CRYPTO_BLOCK_SIZE : iname->len; + ciphertext_len = f2fs_fname_crypto_round_up(ciphertext_len, padding); + ciphertext_len = (ciphertext_len > lim) ? lim : ciphertext_len; + + if (ciphertext_len <= sizeof(buf)) { + workbuf = buf; + } else { + alloc_buf = kmalloc(ciphertext_len, GFP_NOFS); + if (!alloc_buf) + return -ENOMEM; + workbuf = alloc_buf; + } + + /* Allocate request */ + req = ablkcipher_request_alloc(tfm, GFP_NOFS); + if (!req) { + printk_ratelimited(KERN_ERR + "%s: crypto_request_alloc() failed\n", __func__); + kfree(alloc_buf); + return -ENOMEM; + } + ablkcipher_request_set_callback(req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + f2fs_dir_crypt_complete, &ecr); + + /* Copy the input */ + memcpy(workbuf, iname->name, iname->len); + if (iname->len < ciphertext_len) + memset(workbuf + iname->len, 0, ciphertext_len - iname->len); + + /* Initialize IV */ + memset(iv, 0, F2FS_CRYPTO_BLOCK_SIZE); + + /* Create encryption request */ + sg_init_one(&src_sg, workbuf, ciphertext_len); + sg_init_one(&dst_sg, oname->name, ciphertext_len); + ablkcipher_request_set_crypt(req, &src_sg, &dst_sg, ciphertext_len, iv); + res = crypto_ablkcipher_encrypt(req); + if (res == -EINPROGRESS || res == -EBUSY) { + BUG_ON(req->base.data != &ecr); + wait_for_completion(&ecr.completion); + res = ecr.res; + } + kfree(alloc_buf); + ablkcipher_request_free(req); + if (res < 0) { + printk_ratelimited(KERN_ERR + "%s: Error (error code %d)\n", __func__, res); + } + oname->len = ciphertext_len; + return res; +} + +/* + * f2fs_fname_decrypt() + * This function decrypts the input filename, and returns + * the length of the plaintext. + * Errors are returned as negative numbers. + * We trust the caller to allocate sufficient memory to oname string. + */ +static int f2fs_fname_decrypt(struct inode *inode, + const struct f2fs_str *iname, struct f2fs_str *oname) +{ + struct ablkcipher_request *req = NULL; + DECLARE_F2FS_COMPLETION_RESULT(ecr); + struct scatterlist src_sg, dst_sg; + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + struct crypto_ablkcipher *tfm = ci->ci_ctfm; + int res = 0; + char iv[F2FS_CRYPTO_BLOCK_SIZE]; + unsigned lim = max_name_len(inode); + + if (iname->len <= 0 || iname->len > lim) + return -EIO; + + /* Allocate request */ + req = ablkcipher_request_alloc(tfm, GFP_NOFS); + if (!req) { + printk_ratelimited(KERN_ERR + "%s: crypto_request_alloc() failed\n", __func__); + return -ENOMEM; + } + ablkcipher_request_set_callback(req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + f2fs_dir_crypt_complete, &ecr); + + /* Initialize IV */ + memset(iv, 0, F2FS_CRYPTO_BLOCK_SIZE); + + /* Create decryption request */ + sg_init_one(&src_sg, iname->name, iname->len); + sg_init_one(&dst_sg, oname->name, oname->len); + ablkcipher_request_set_crypt(req, &src_sg, &dst_sg, iname->len, iv); + res = crypto_ablkcipher_decrypt(req); + if (res == -EINPROGRESS || res == -EBUSY) { + BUG_ON(req->base.data != &ecr); + wait_for_completion(&ecr.completion); + res = ecr.res; + } + ablkcipher_request_free(req); + if (res < 0) { + printk_ratelimited(KERN_ERR + "%s: Error in f2fs_fname_decrypt (error code %d)\n", + __func__, res); + return res; + } + + oname->len = strnlen(oname->name, iname->len); + return oname->len; +} + +static const char *lookup_table = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; + +/** + * f2fs_fname_encode_digest() - + * + * Encodes the input digest using characters from the set [a-zA-Z0-9_+]. + * The encoded string is roughly 4/3 times the size of the input string. + */ +static int digest_encode(const char *src, int len, char *dst) +{ + int i = 0, bits = 0, ac = 0; + char *cp = dst; + + while (i < len) { + ac += (((unsigned char) src[i]) << bits); + bits += 8; + do { + *cp++ = lookup_table[ac & 0x3f]; + ac >>= 6; + bits -= 6; + } while (bits >= 6); + i++; + } + if (bits) + *cp++ = lookup_table[ac & 0x3f]; + return cp - dst; +} + +static int digest_decode(const char *src, int len, char *dst) +{ + int i = 0, bits = 0, ac = 0; + const char *p; + char *cp = dst; + + while (i < len) { + p = strchr(lookup_table, src[i]); + if (p == NULL || src[i] == 0) + return -2; + ac += (p - lookup_table) << bits; + bits += 6; + if (bits >= 8) { + *cp++ = ac & 0xff; + ac >>= 8; + bits -= 8; + } + i++; + } + if (ac) + return -1; + return cp - dst; +} + +/** + * f2fs_fname_crypto_round_up() - + * + * Return: The next multiple of block size + */ +u32 f2fs_fname_crypto_round_up(u32 size, u32 blksize) +{ + return ((size + blksize - 1) / blksize) * blksize; +} + +/** + * f2fs_fname_crypto_alloc_obuff() - + * + * Allocates an output buffer that is sufficient for the crypto operation + * specified by the context and the direction. + */ +int f2fs_fname_crypto_alloc_buffer(struct inode *inode, + u32 ilen, struct f2fs_str *crypto_str) +{ + unsigned int olen; + int padding = 16; + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + + if (ci) + padding = 4 << (ci->ci_flags & F2FS_POLICY_FLAGS_PAD_MASK); + if (padding < F2FS_CRYPTO_BLOCK_SIZE) + padding = F2FS_CRYPTO_BLOCK_SIZE; + olen = f2fs_fname_crypto_round_up(ilen, padding); + crypto_str->len = olen; + if (olen < F2FS_FNAME_CRYPTO_DIGEST_SIZE * 2) + olen = F2FS_FNAME_CRYPTO_DIGEST_SIZE * 2; + /* Allocated buffer can hold one more character to null-terminate the + * string */ + crypto_str->name = kmalloc(olen + 1, GFP_NOFS); + if (!(crypto_str->name)) + return -ENOMEM; + return 0; +} + +/** + * f2fs_fname_crypto_free_buffer() - + * + * Frees the buffer allocated for crypto operation. + */ +void f2fs_fname_crypto_free_buffer(struct f2fs_str *crypto_str) +{ + if (!crypto_str) + return; + kfree(crypto_str->name); + crypto_str->name = NULL; +} + +/** + * f2fs_fname_disk_to_usr() - converts a filename from disk space to user space + */ +int f2fs_fname_disk_to_usr(struct inode *inode, + f2fs_hash_t *hash, + const struct f2fs_str *iname, + struct f2fs_str *oname) +{ + const struct qstr qname = FSTR_TO_QSTR(iname); + char buf[24]; + int ret; + + if (is_dot_dotdot(&qname)) { + oname->name[0] = '.'; + oname->name[iname->len - 1] = '.'; + oname->len = iname->len; + return oname->len; + } + + if (F2FS_I(inode)->i_crypt_info) + return f2fs_fname_decrypt(inode, iname, oname); + + if (iname->len <= F2FS_FNAME_CRYPTO_DIGEST_SIZE) { + ret = digest_encode(iname->name, iname->len, oname->name); + oname->len = ret; + return ret; + } + if (hash) { + memcpy(buf, hash, 4); + memset(buf + 4, 0, 4); + } else + memset(buf, 0, 8); + memcpy(buf + 8, iname->name + iname->len - 16, 16); + oname->name[0] = '_'; + ret = digest_encode(buf, 24, oname->name + 1); + oname->len = ret + 1; + return ret + 1; +} + +/** + * f2fs_fname_usr_to_disk() - converts a filename from user space to disk space + */ +int f2fs_fname_usr_to_disk(struct inode *inode, + const struct qstr *iname, + struct f2fs_str *oname) +{ + int res; + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + + if (is_dot_dotdot(iname)) { + oname->name[0] = '.'; + oname->name[iname->len - 1] = '.'; + oname->len = iname->len; + return oname->len; + } + + if (ci) { + res = f2fs_fname_encrypt(inode, iname, oname); + return res; + } + /* Without a proper key, a user is not allowed to modify the filenames + * in a directory. Consequently, a user space name cannot be mapped to + * a disk-space name */ + return -EACCES; +} + +int f2fs_fname_setup_filename(struct inode *dir, const struct qstr *iname, + int lookup, struct f2fs_filename *fname) +{ + struct f2fs_crypt_info *ci; + int ret = 0, bigname = 0; + + memset(fname, 0, sizeof(struct f2fs_filename)); + fname->usr_fname = iname; + + if (!f2fs_encrypted_inode(dir) || is_dot_dotdot(iname)) { + fname->disk_name.name = (unsigned char *)iname->name; + fname->disk_name.len = iname->len; + return 0; + } + ret = f2fs_get_encryption_info(dir); + if (ret) + return ret; + ci = F2FS_I(dir)->i_crypt_info; + if (ci) { + ret = f2fs_fname_crypto_alloc_buffer(dir, iname->len, + &fname->crypto_buf); + if (ret < 0) + return ret; + ret = f2fs_fname_encrypt(dir, iname, &fname->crypto_buf); + if (ret < 0) + goto errout; + fname->disk_name.name = fname->crypto_buf.name; + fname->disk_name.len = fname->crypto_buf.len; + return 0; + } + if (!lookup) + return -EACCES; + + /* We don't have the key and we are doing a lookup; decode the + * user-supplied name + */ + if (iname->name[0] == '_') + bigname = 1; + if ((bigname && (iname->len != 33)) || + (!bigname && (iname->len > 43))) + return -ENOENT; + + fname->crypto_buf.name = kmalloc(32, GFP_KERNEL); + if (fname->crypto_buf.name == NULL) + return -ENOMEM; + ret = digest_decode(iname->name + bigname, iname->len - bigname, + fname->crypto_buf.name); + if (ret < 0) { + ret = -ENOENT; + goto errout; + } + fname->crypto_buf.len = ret; + if (bigname) { + memcpy(&fname->hash, fname->crypto_buf.name, 4); + } else { + fname->disk_name.name = fname->crypto_buf.name; + fname->disk_name.len = fname->crypto_buf.len; + } + return 0; +errout: + f2fs_fname_crypto_free_buffer(&fname->crypto_buf); + return ret; +} + +void f2fs_fname_free_filename(struct f2fs_filename *fname) +{ + kfree(fname->crypto_buf.name); + fname->crypto_buf.name = NULL; + fname->usr_fname = NULL; + fname->disk_name.name = NULL; +} diff --git a/fs/f2fs/crypto_key.c b/fs/f2fs/crypto_key.c new file mode 100644 index 00000000000..9f77de2ef31 --- /dev/null +++ b/fs/f2fs/crypto_key.c @@ -0,0 +1,254 @@ +/* + * linux/fs/f2fs/crypto_key.c + * + * Copied from linux/fs/f2fs/crypto_key.c + * + * Copyright (C) 2015, Google, Inc. + * + * This contains encryption key functions for f2fs + * + * Written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar, 2015. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "xattr.h" + +static void derive_crypt_complete(struct crypto_async_request *req, int rc) +{ + struct f2fs_completion_result *ecr = req->data; + + if (rc == -EINPROGRESS) + return; + + ecr->res = rc; + complete(&ecr->completion); +} + +/** + * f2fs_derive_key_aes() - Derive a key using AES-128-ECB + * @deriving_key: Encryption key used for derivatio. + * @source_key: Source key to which to apply derivation. + * @derived_key: Derived key. + * + * Return: Zero on success; non-zero otherwise. + */ +static int f2fs_derive_key_aes(char deriving_key[F2FS_AES_128_ECB_KEY_SIZE], + char source_key[F2FS_AES_256_XTS_KEY_SIZE], + char derived_key[F2FS_AES_256_XTS_KEY_SIZE]) +{ + int res = 0; + struct ablkcipher_request *req = NULL; + DECLARE_F2FS_COMPLETION_RESULT(ecr); + struct scatterlist src_sg, dst_sg; + struct crypto_ablkcipher *tfm = crypto_alloc_ablkcipher("ecb(aes)", 0, + 0); + + if (IS_ERR(tfm)) { + res = PTR_ERR(tfm); + tfm = NULL; + goto out; + } + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY); + req = ablkcipher_request_alloc(tfm, GFP_NOFS); + if (!req) { + res = -ENOMEM; + goto out; + } + ablkcipher_request_set_callback(req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + derive_crypt_complete, &ecr); + res = crypto_ablkcipher_setkey(tfm, deriving_key, + F2FS_AES_128_ECB_KEY_SIZE); + if (res < 0) + goto out; + + sg_init_one(&src_sg, source_key, F2FS_AES_256_XTS_KEY_SIZE); + sg_init_one(&dst_sg, derived_key, F2FS_AES_256_XTS_KEY_SIZE); + ablkcipher_request_set_crypt(req, &src_sg, &dst_sg, + F2FS_AES_256_XTS_KEY_SIZE, NULL); + res = crypto_ablkcipher_encrypt(req); + if (res == -EINPROGRESS || res == -EBUSY) { + BUG_ON(req->base.data != &ecr); + wait_for_completion(&ecr.completion); + res = ecr.res; + } +out: + if (req) + ablkcipher_request_free(req); + if (tfm) + crypto_free_ablkcipher(tfm); + return res; +} + +static void f2fs_free_crypt_info(struct f2fs_crypt_info *ci) +{ + if (!ci) + return; + + key_put(ci->ci_keyring_key); + crypto_free_ablkcipher(ci->ci_ctfm); + kmem_cache_free(f2fs_crypt_info_cachep, ci); +} + +void f2fs_free_encryption_info(struct inode *inode, struct f2fs_crypt_info *ci) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_crypt_info *prev; + + if (ci == NULL) + ci = ACCESS_ONCE(fi->i_crypt_info); + if (ci == NULL) + return; + prev = cmpxchg(&fi->i_crypt_info, ci, NULL); + if (prev != ci) + return; + + f2fs_free_crypt_info(ci); +} + +int _f2fs_get_encryption_info(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_crypt_info *crypt_info; + char full_key_descriptor[F2FS_KEY_DESC_PREFIX_SIZE + + (F2FS_KEY_DESCRIPTOR_SIZE * 2) + 1]; + struct key *keyring_key = NULL; + struct f2fs_encryption_key *master_key; + struct f2fs_encryption_context ctx; + struct user_key_payload *ukp; + struct crypto_ablkcipher *ctfm; + const char *cipher_str; + char raw_key[F2FS_MAX_KEY_SIZE]; + char mode; + int res; + + res = f2fs_crypto_initialize(); + if (res) + return res; +retry: + crypt_info = ACCESS_ONCE(fi->i_crypt_info); + if (crypt_info) { + if (!crypt_info->ci_keyring_key || + key_validate(crypt_info->ci_keyring_key) == 0) + return 0; + f2fs_free_encryption_info(inode, crypt_info); + goto retry; + } + + res = f2fs_getxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, + &ctx, sizeof(ctx), NULL); + if (res < 0) + return res; + else if (res != sizeof(ctx)) + return -EINVAL; + res = 0; + + crypt_info = kmem_cache_alloc(f2fs_crypt_info_cachep, GFP_NOFS); + if (!crypt_info) + return -ENOMEM; + + crypt_info->ci_flags = ctx.flags; + crypt_info->ci_data_mode = ctx.contents_encryption_mode; + crypt_info->ci_filename_mode = ctx.filenames_encryption_mode; + crypt_info->ci_ctfm = NULL; + crypt_info->ci_keyring_key = NULL; + memcpy(crypt_info->ci_master_key, ctx.master_key_descriptor, + sizeof(crypt_info->ci_master_key)); + if (S_ISREG(inode->i_mode)) + mode = crypt_info->ci_data_mode; + else if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) + mode = crypt_info->ci_filename_mode; + else + BUG(); + + switch (mode) { + case F2FS_ENCRYPTION_MODE_AES_256_XTS: + cipher_str = "xts(aes)"; + break; + case F2FS_ENCRYPTION_MODE_AES_256_CTS: + cipher_str = "cts(cbc(aes))"; + break; + default: + printk_once(KERN_WARNING + "f2fs: unsupported key mode %d (ino %u)\n", + mode, (unsigned) inode->i_ino); + res = -ENOKEY; + goto out; + } + + memcpy(full_key_descriptor, F2FS_KEY_DESC_PREFIX, + F2FS_KEY_DESC_PREFIX_SIZE); + sprintf(full_key_descriptor + F2FS_KEY_DESC_PREFIX_SIZE, + "%*phN", F2FS_KEY_DESCRIPTOR_SIZE, + ctx.master_key_descriptor); + full_key_descriptor[F2FS_KEY_DESC_PREFIX_SIZE + + (2 * F2FS_KEY_DESCRIPTOR_SIZE)] = '\0'; + keyring_key = request_key(&key_type_logon, full_key_descriptor, NULL); + if (IS_ERR(keyring_key)) { + res = PTR_ERR(keyring_key); + keyring_key = NULL; + goto out; + } + crypt_info->ci_keyring_key = keyring_key; + BUG_ON(keyring_key->type != &key_type_logon); + ukp = ((struct user_key_payload *)keyring_key->payload.data); + if (ukp->datalen != sizeof(struct f2fs_encryption_key)) { + res = -EINVAL; + goto out; + } + master_key = (struct f2fs_encryption_key *)ukp->data; + BUILD_BUG_ON(F2FS_AES_128_ECB_KEY_SIZE != + F2FS_KEY_DERIVATION_NONCE_SIZE); + BUG_ON(master_key->size != F2FS_AES_256_XTS_KEY_SIZE); + res = f2fs_derive_key_aes(ctx.nonce, master_key->raw, + raw_key); + if (res) + goto out; + + ctfm = crypto_alloc_ablkcipher(cipher_str, 0, 0); + if (!ctfm || IS_ERR(ctfm)) { + res = ctfm ? PTR_ERR(ctfm) : -ENOMEM; + printk(KERN_DEBUG + "%s: error %d (inode %u) allocating crypto tfm\n", + __func__, res, (unsigned) inode->i_ino); + goto out; + } + crypt_info->ci_ctfm = ctfm; + crypto_ablkcipher_clear_flags(ctfm, ~0); + crypto_tfm_set_flags(crypto_ablkcipher_tfm(ctfm), + CRYPTO_TFM_REQ_WEAK_KEY); + res = crypto_ablkcipher_setkey(ctfm, raw_key, + f2fs_encryption_key_size(mode)); + if (res) + goto out; + + memzero_explicit(raw_key, sizeof(raw_key)); + if (cmpxchg(&fi->i_crypt_info, NULL, crypt_info) != NULL) { + f2fs_free_crypt_info(crypt_info); + goto retry; + } + return 0; + +out: + if (res == -ENOKEY && !S_ISREG(inode->i_mode)) + res = 0; + + f2fs_free_crypt_info(crypt_info); + memzero_explicit(raw_key, sizeof(raw_key)); + return res; +} + +int f2fs_has_encryption_key(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + + return (fi->i_crypt_info != NULL); +} diff --git a/fs/f2fs/crypto_policy.c b/fs/f2fs/crypto_policy.c new file mode 100644 index 00000000000..d4a96af513c --- /dev/null +++ b/fs/f2fs/crypto_policy.c @@ -0,0 +1,209 @@ +/* + * copied from linux/fs/ext4/crypto_policy.c + * + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility. + * + * This contains encryption policy functions for f2fs with some modifications + * to support f2fs-specific xattr APIs. + * + * Written by Michael Halcrow, 2015. + * Modified by Jaegeuk Kim, 2015. + */ +#include +#include +#include +#include + +#include "f2fs.h" +#include "xattr.h" + +static int f2fs_inode_has_encryption_context(struct inode *inode) +{ + int res = f2fs_getxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, NULL, 0, NULL); + return (res > 0); +} + +/* + * check whether the policy is consistent with the encryption context + * for the inode + */ +static int f2fs_is_encryption_context_consistent_with_policy( + struct inode *inode, const struct f2fs_encryption_policy *policy) +{ + struct f2fs_encryption_context ctx; + int res = f2fs_getxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, &ctx, + sizeof(ctx), NULL); + + if (res != sizeof(ctx)) + return 0; + + return (memcmp(ctx.master_key_descriptor, policy->master_key_descriptor, + F2FS_KEY_DESCRIPTOR_SIZE) == 0 && + (ctx.flags == policy->flags) && + (ctx.contents_encryption_mode == + policy->contents_encryption_mode) && + (ctx.filenames_encryption_mode == + policy->filenames_encryption_mode)); +} + +static int f2fs_create_encryption_context_from_policy( + struct inode *inode, const struct f2fs_encryption_policy *policy) +{ + struct f2fs_encryption_context ctx; + + ctx.format = F2FS_ENCRYPTION_CONTEXT_FORMAT_V1; + memcpy(ctx.master_key_descriptor, policy->master_key_descriptor, + F2FS_KEY_DESCRIPTOR_SIZE); + + if (!f2fs_valid_contents_enc_mode(policy->contents_encryption_mode)) { + printk(KERN_WARNING + "%s: Invalid contents encryption mode %d\n", __func__, + policy->contents_encryption_mode); + return -EINVAL; + } + + if (!f2fs_valid_filenames_enc_mode(policy->filenames_encryption_mode)) { + printk(KERN_WARNING + "%s: Invalid filenames encryption mode %d\n", __func__, + policy->filenames_encryption_mode); + return -EINVAL; + } + + if (policy->flags & ~F2FS_POLICY_FLAGS_VALID) + return -EINVAL; + + ctx.contents_encryption_mode = policy->contents_encryption_mode; + ctx.filenames_encryption_mode = policy->filenames_encryption_mode; + ctx.flags = policy->flags; + BUILD_BUG_ON(sizeof(ctx.nonce) != F2FS_KEY_DERIVATION_NONCE_SIZE); + get_random_bytes(ctx.nonce, F2FS_KEY_DERIVATION_NONCE_SIZE); + + return f2fs_setxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, &ctx, + sizeof(ctx), NULL, XATTR_CREATE); +} + +int f2fs_process_policy(const struct f2fs_encryption_policy *policy, + struct inode *inode) +{ + if (policy->version != 0) + return -EINVAL; + + if (!S_ISDIR(inode->i_mode)) + return -EINVAL; + + if (!f2fs_inode_has_encryption_context(inode)) { + if (!f2fs_empty_dir(inode)) + return -ENOTEMPTY; + return f2fs_create_encryption_context_from_policy(inode, + policy); + } + + if (f2fs_is_encryption_context_consistent_with_policy(inode, policy)) + return 0; + + printk(KERN_WARNING "%s: Policy inconsistent with encryption context\n", + __func__); + return -EINVAL; +} + +int f2fs_get_policy(struct inode *inode, struct f2fs_encryption_policy *policy) +{ + struct f2fs_encryption_context ctx; + int res; + + if (!f2fs_encrypted_inode(inode)) + return -ENODATA; + + res = f2fs_getxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, + &ctx, sizeof(ctx), NULL); + if (res != sizeof(ctx)) + return -ENODATA; + if (ctx.format != F2FS_ENCRYPTION_CONTEXT_FORMAT_V1) + return -EINVAL; + + policy->version = 0; + policy->contents_encryption_mode = ctx.contents_encryption_mode; + policy->filenames_encryption_mode = ctx.filenames_encryption_mode; + policy->flags = ctx.flags; + memcpy(&policy->master_key_descriptor, ctx.master_key_descriptor, + F2FS_KEY_DESCRIPTOR_SIZE); + return 0; +} + +int f2fs_is_child_context_consistent_with_parent(struct inode *parent, + struct inode *child) +{ + struct f2fs_crypt_info *parent_ci, *child_ci; + int res; + + if ((parent == NULL) || (child == NULL)) { + pr_err("parent %p child %p\n", parent, child); + BUG_ON(1); + } + + /* no restrictions if the parent directory is not encrypted */ + if (!f2fs_encrypted_inode(parent)) + return 1; + /* if the child directory is not encrypted, this is always a problem */ + if (!f2fs_encrypted_inode(child)) + return 0; + res = f2fs_get_encryption_info(parent); + if (res) + return 0; + res = f2fs_get_encryption_info(child); + if (res) + return 0; + parent_ci = F2FS_I(parent)->i_crypt_info; + child_ci = F2FS_I(child)->i_crypt_info; + if (!parent_ci && !child_ci) + return 1; + if (!parent_ci || !child_ci) + return 0; + + return (memcmp(parent_ci->ci_master_key, + child_ci->ci_master_key, + F2FS_KEY_DESCRIPTOR_SIZE) == 0 && + (parent_ci->ci_data_mode == child_ci->ci_data_mode) && + (parent_ci->ci_filename_mode == child_ci->ci_filename_mode) && + (parent_ci->ci_flags == child_ci->ci_flags)); +} + +/** + * f2fs_inherit_context() - Sets a child context from its parent + * @parent: Parent inode from which the context is inherited. + * @child: Child inode that inherits the context from @parent. + * + * Return: Zero on success, non-zero otherwise + */ +int f2fs_inherit_context(struct inode *parent, struct inode *child, + struct page *ipage) +{ + struct f2fs_encryption_context ctx; + struct f2fs_crypt_info *ci; + int res; + + res = f2fs_get_encryption_info(parent); + if (res < 0) + return res; + + ci = F2FS_I(parent)->i_crypt_info; + BUG_ON(ci == NULL); + + ctx.format = F2FS_ENCRYPTION_CONTEXT_FORMAT_V1; + + ctx.contents_encryption_mode = ci->ci_data_mode; + ctx.filenames_encryption_mode = ci->ci_filename_mode; + ctx.flags = ci->ci_flags; + memcpy(ctx.master_key_descriptor, ci->ci_master_key, + F2FS_KEY_DESCRIPTOR_SIZE); + + get_random_bytes(ctx.nonce, F2FS_KEY_DERIVATION_NONCE_SIZE); + return f2fs_setxattr(child, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, &ctx, + sizeof(ctx), ipage, XATTR_CREATE); +} diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c new file mode 100644 index 00000000000..890e7363f8f --- /dev/null +++ b/fs/f2fs/data.c @@ -0,0 +1,1725 @@ +/* + * fs/f2fs/data.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "trace.h" +#include + +static void f2fs_read_end_io(struct bio *bio, int err) +{ + struct bio_vec *bvec; + int i; + + if (f2fs_bio_encrypted(bio)) { + if (err) { + f2fs_release_crypto_ctx(bio->bi_private); + } else { + f2fs_end_io_crypto_work(bio->bi_private, bio); + return; + } + } + + __bio_for_each_segment(bvec, bio, i, 0) { + struct page *page = bvec->bv_page; + + if (!err) { + SetPageUptodate(page); + } else { + ClearPageUptodate(page); + SetPageError(page); + } + unlock_page(page); + } + bio_put(bio); +} + +static void f2fs_write_end_io(struct bio *bio, int err) +{ + struct f2fs_sb_info *sbi = bio->bi_private; + struct bio_vec *bvec; + int i; + + __bio_for_each_segment(bvec, bio, i, 0) { + struct page *page = bvec->bv_page; + + f2fs_restore_and_release_control_page(&page); + + if (unlikely(err)) { + set_page_dirty(page); + set_bit(AS_EIO, &page->mapping->flags); + f2fs_stop_checkpoint(sbi); + } + end_page_writeback(page); + dec_page_count(sbi, F2FS_WRITEBACK); + } + + if (!get_pages(sbi, F2FS_WRITEBACK) && + !list_empty(&sbi->cp_wait.task_list)) + wake_up(&sbi->cp_wait); + + bio_put(bio); +} + +/* + * Low-level block read/write IO operations. + */ +static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, + int npages, bool is_read) +{ + struct bio *bio; + + bio = f2fs_bio_alloc(npages); + + bio->bi_bdev = sbi->sb->s_bdev; + bio->bi_sector = SECTOR_FROM_BLOCK(blk_addr); + bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; + bio->bi_private = is_read ? NULL : sbi; + + return bio; +} + +static void __submit_merged_bio(struct f2fs_bio_info *io) +{ + struct f2fs_io_info *fio = &io->fio; + + if (!io->bio) + return; + + if (is_read_io(fio->rw)) + trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio); + else + trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio); + + submit_bio(fio->rw, io->bio); + io->bio = NULL; +} + +void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi, + enum page_type type, int rw) +{ + enum page_type btype = PAGE_TYPE_OF_BIO(type); + struct f2fs_bio_info *io; + + io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype]; + + down_write(&io->io_rwsem); + + /* change META to META_FLUSH in the checkpoint procedure */ + if (type >= META_FLUSH) { + io->fio.type = META_FLUSH; + if (test_opt(sbi, NOBARRIER)) + io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO; + else + io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO; + } + __submit_merged_bio(io); + up_write(&io->io_rwsem); +} + +/* + * Fill the locked page with data located in the block address. + * Return unlocked page. + */ +int f2fs_submit_page_bio(struct f2fs_io_info *fio) +{ + struct bio *bio; + struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; + + trace_f2fs_submit_page_bio(page, fio); + f2fs_trace_ios(fio, 0); + + /* Allocate a new bio */ + bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw)); + + if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { + bio_put(bio); + return -EFAULT; + } + + submit_bio(fio->rw, bio); + return 0; +} + +void f2fs_submit_page_mbio(struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); + struct f2fs_bio_info *io; + bool is_read = is_read_io(fio->rw); + struct page *bio_page; + + io = is_read ? &sbi->read_io : &sbi->write_io[btype]; + + verify_block_addr(sbi, fio->blk_addr); + + down_write(&io->io_rwsem); + + if (!is_read) + inc_page_count(sbi, F2FS_WRITEBACK); + + if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 || + io->fio.rw != fio->rw)) + __submit_merged_bio(io); +alloc_new: + if (io->bio == NULL) { + int bio_blocks = MAX_BIO_BLOCKS(sbi); + + io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read); + io->fio = *fio; + } + + bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; + + if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) < + PAGE_CACHE_SIZE) { + __submit_merged_bio(io); + goto alloc_new; + } + + io->last_block_in_bio = fio->blk_addr; + f2fs_trace_ios(fio, 0); + + up_write(&io->io_rwsem); + trace_f2fs_submit_page_mbio(fio->page, fio); +} + +/* + * Lock ordering for the change of data block address: + * ->data_page + * ->node_page + * update block addresses in the node page + */ +void set_data_blkaddr(struct dnode_of_data *dn) +{ + struct f2fs_node *rn; + __le32 *addr_array; + struct page *node_page = dn->node_page; + unsigned int ofs_in_node = dn->ofs_in_node; + + f2fs_wait_on_page_writeback(node_page, NODE); + + rn = F2FS_NODE(node_page); + + /* Get physical address of data block */ + addr_array = blkaddr_in_node(rn); + addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr); + set_page_dirty(node_page); +} + +int reserve_new_block(struct dnode_of_data *dn) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + + if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) + return -EPERM; + if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) + return -ENOSPC; + + trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node); + + dn->data_blkaddr = NEW_ADDR; + set_data_blkaddr(dn); + mark_inode_dirty(dn->inode); + sync_inode_page(dn); + return 0; +} + +int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) +{ + bool need_put = dn->inode_page ? false : true; + int err; + + err = get_dnode_of_data(dn, index, ALLOC_NODE); + if (err) + return err; + + if (dn->data_blkaddr == NULL_ADDR) + err = reserve_new_block(dn); + if (err || need_put) + f2fs_put_dnode(dn); + return err; +} + +int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) +{ + struct extent_info ei; + struct inode *inode = dn->inode; + + if (f2fs_lookup_extent_cache(inode, index, &ei)) { + dn->data_blkaddr = ei.blk + index - ei.fofs; + return 0; + } + + return f2fs_reserve_block(dn, index); +} + +struct page *get_read_data_page(struct inode *inode, pgoff_t index, + int rw, bool for_write) +{ + struct address_space *mapping = inode->i_mapping; + struct dnode_of_data dn; + struct page *page; + struct extent_info ei; + int err; + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(inode), + .type = DATA, + .rw = rw, + .encrypted_page = NULL, + }; + + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) + return read_mapping_page(mapping, index, NULL); + + page = f2fs_grab_cache_page(mapping, index, for_write); + if (!page) + return ERR_PTR(-ENOMEM); + + if (f2fs_lookup_extent_cache(inode, index, &ei)) { + dn.data_blkaddr = ei.blk + index - ei.fofs; + goto got_it; + } + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, index, LOOKUP_NODE); + if (err) + goto put_err; + f2fs_put_dnode(&dn); + + if (unlikely(dn.data_blkaddr == NULL_ADDR)) { + err = -ENOENT; + goto put_err; + } +got_it: + if (PageUptodate(page)) { + unlock_page(page); + return page; + } + + /* + * A new dentry page is allocated but not able to be written, since its + * new inode page couldn't be allocated due to -ENOSPC. + * In such the case, its blkaddr can be remained as NEW_ADDR. + * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata. + */ + if (dn.data_blkaddr == NEW_ADDR) { + zero_user_segment(page, 0, PAGE_CACHE_SIZE); + SetPageUptodate(page); + unlock_page(page); + return page; + } + + fio.blk_addr = dn.data_blkaddr; + fio.page = page; + err = f2fs_submit_page_bio(&fio); + if (err) + goto put_err; + return page; + +put_err: + f2fs_put_page(page, 1); + return ERR_PTR(err); +} + +struct page *find_data_page(struct inode *inode, pgoff_t index) +{ + struct address_space *mapping = inode->i_mapping; + struct page *page; + + page = find_get_page(mapping, index); + if (page && PageUptodate(page)) + return page; + f2fs_put_page(page, 0); + + page = get_read_data_page(inode, index, READ_SYNC, false); + if (IS_ERR(page)) + return page; + + if (PageUptodate(page)) + return page; + + wait_on_page_locked(page); + if (unlikely(!PageUptodate(page))) { + f2fs_put_page(page, 0); + return ERR_PTR(-EIO); + } + return page; +} + +/* + * If it tries to access a hole, return an error. + * Because, the callers, functions in dir.c and GC, should be able to know + * whether this page exists or not. + */ +struct page *get_lock_data_page(struct inode *inode, pgoff_t index, + bool for_write) +{ + struct address_space *mapping = inode->i_mapping; + struct page *page; +repeat: + page = get_read_data_page(inode, index, READ_SYNC, for_write); + if (IS_ERR(page)) + return page; + + /* wait for read completion */ + lock_page(page); + if (unlikely(!PageUptodate(page))) { + f2fs_put_page(page, 1); + return ERR_PTR(-EIO); + } + if (unlikely(page->mapping != mapping)) { + f2fs_put_page(page, 1); + goto repeat; + } + return page; +} + +/* + * Caller ensures that this data page is never allocated. + * A new zero-filled data page is allocated in the page cache. + * + * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op(). + * Note that, ipage is set only by make_empty_dir, and if any error occur, + * ipage should be released by this function. + */ +struct page *get_new_data_page(struct inode *inode, + struct page *ipage, pgoff_t index, bool new_i_size) +{ + struct address_space *mapping = inode->i_mapping; + struct page *page; + struct dnode_of_data dn; + int err; +repeat: + page = f2fs_grab_cache_page(mapping, index, true); + if (!page) { + /* + * before exiting, we should make sure ipage will be released + * if any error occur. + */ + f2fs_put_page(ipage, 1); + return ERR_PTR(-ENOMEM); + } + + set_new_dnode(&dn, inode, ipage, NULL, 0); + err = f2fs_reserve_block(&dn, index); + if (err) { + f2fs_put_page(page, 1); + return ERR_PTR(err); + } + if (!ipage) + f2fs_put_dnode(&dn); + + if (PageUptodate(page)) + goto got_it; + + if (dn.data_blkaddr == NEW_ADDR) { + zero_user_segment(page, 0, PAGE_CACHE_SIZE); + SetPageUptodate(page); + } else { + f2fs_put_page(page, 1); + + page = get_read_data_page(inode, index, READ_SYNC, true); + if (IS_ERR(page)) + goto repeat; + + /* wait for read completion */ + lock_page(page); + } +got_it: + if (new_i_size && i_size_read(inode) < + ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) { + i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)); + /* Only the directory inode sets new_i_size */ + set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR); + } + return page; +} + +static int __allocate_data_block(struct dnode_of_data *dn) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct f2fs_inode_info *fi = F2FS_I(dn->inode); + struct f2fs_summary sum; + struct node_info ni; + int seg = CURSEG_WARM_DATA; + pgoff_t fofs; + + if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) + return -EPERM; + + dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node); + if (dn->data_blkaddr == NEW_ADDR) + goto alloc; + + if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) + return -ENOSPC; + +alloc: + get_node_info(sbi, dn->nid, &ni); + set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); + + if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page) + seg = CURSEG_DIRECT_IO; + + allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, + &sum, seg); + set_data_blkaddr(dn); + + /* update i_size */ + fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + + dn->ofs_in_node; + if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)) + i_size_write(dn->inode, + ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)); + + /* direct IO doesn't use extent cache to maximize the performance */ + f2fs_drop_largest_extent(dn->inode, fofs); + + return 0; +} + +static void __allocate_data_blocks(struct inode *inode, loff_t offset, + size_t count) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + u64 start = F2FS_BYTES_TO_BLK(offset); + u64 len = F2FS_BYTES_TO_BLK(count); + bool allocated; + u64 end_offset; + + while (len) { + f2fs_balance_fs(sbi); + f2fs_lock_op(sbi); + + /* When reading holes, we need its node page */ + set_new_dnode(&dn, inode, NULL, NULL, 0); + if (get_dnode_of_data(&dn, start, ALLOC_NODE)) + goto out; + + allocated = false; + end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); + + while (dn.ofs_in_node < end_offset && len) { + block_t blkaddr; + + if (unlikely(f2fs_cp_error(sbi))) + goto sync_out; + + blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); + if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) { + if (__allocate_data_block(&dn)) + goto sync_out; + allocated = true; + } + len--; + start++; + dn.ofs_in_node++; + } + + if (allocated) + sync_inode_page(&dn); + + f2fs_put_dnode(&dn); + f2fs_unlock_op(sbi); + } + return; + +sync_out: + if (allocated) + sync_inode_page(&dn); + f2fs_put_dnode(&dn); +out: + f2fs_unlock_op(sbi); + return; +} + +/* + * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with + * f2fs_map_blocks structure. + * If original data blocks are allocated, then give them to blockdev. + * Otherwise, + * a. preallocate requested block addresses + * b. do not use extent cache for better performance + * c. give the block addresses to blockdev + */ +static int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, + int create, int flag) +{ + unsigned int maxblocks = map->m_len; + struct dnode_of_data dn; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA; + pgoff_t pgofs, end_offset; + int err = 0, ofs = 1; + struct extent_info ei; + bool allocated = false; + + map->m_len = 0; + map->m_flags = 0; + + /* it only supports block size == page size */ + pgofs = (pgoff_t)map->m_lblk; + + if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) { + map->m_pblk = ei.blk + pgofs - ei.fofs; + map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); + map->m_flags = F2FS_MAP_MAPPED; + goto out; + } + + if (create) + f2fs_lock_op(F2FS_I_SB(inode)); + + /* When reading holes, we need its node page */ + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, pgofs, mode); + if (err) { + if (err == -ENOENT) + err = 0; + goto unlock_out; + } + + if (dn.data_blkaddr == NEW_ADDR || dn.data_blkaddr == NULL_ADDR) { + if (create) { + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto put_out; + } + err = __allocate_data_block(&dn); + if (err) + goto put_out; + allocated = true; + map->m_flags = F2FS_MAP_NEW; + } else { + if (flag != F2FS_GET_BLOCK_FIEMAP || + dn.data_blkaddr != NEW_ADDR) { + if (flag == F2FS_GET_BLOCK_BMAP) + err = -ENOENT; + goto put_out; + } + + /* + * preallocated unwritten block should be mapped + * for fiemap. + */ + if (dn.data_blkaddr == NEW_ADDR) + map->m_flags = F2FS_MAP_UNWRITTEN; + } + } + + map->m_flags |= F2FS_MAP_MAPPED; + map->m_pblk = dn.data_blkaddr; + map->m_len = 1; + + end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); + dn.ofs_in_node++; + pgofs++; + +get_next: + if (dn.ofs_in_node >= end_offset) { + if (allocated) + sync_inode_page(&dn); + allocated = false; + f2fs_put_dnode(&dn); + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, pgofs, mode); + if (err) { + if (err == -ENOENT) + err = 0; + goto unlock_out; + } + + end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); + } + + if (maxblocks > map->m_len) { + block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); + + if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) { + if (create) { + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto sync_out; + } + err = __allocate_data_block(&dn); + if (err) + goto sync_out; + allocated = true; + map->m_flags |= F2FS_MAP_NEW; + blkaddr = dn.data_blkaddr; + } else { + /* + * we only merge preallocated unwritten blocks + * for fiemap. + */ + if (flag != F2FS_GET_BLOCK_FIEMAP || + blkaddr != NEW_ADDR) + goto sync_out; + } + } + + /* Give more consecutive addresses for the readahead */ + if ((map->m_pblk != NEW_ADDR && + blkaddr == (map->m_pblk + ofs)) || + (map->m_pblk == NEW_ADDR && + blkaddr == NEW_ADDR)) { + ofs++; + dn.ofs_in_node++; + pgofs++; + map->m_len++; + goto get_next; + } + } +sync_out: + if (allocated) + sync_inode_page(&dn); +put_out: + f2fs_put_dnode(&dn); +unlock_out: + if (create) + f2fs_unlock_op(F2FS_I_SB(inode)); +out: + trace_f2fs_map_blocks(inode, map, err); + return err; +} + +static int __get_data_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create, int flag) +{ + struct f2fs_map_blocks map; + int ret; + + map.m_lblk = iblock; + map.m_len = bh->b_size >> inode->i_blkbits; + + ret = f2fs_map_blocks(inode, &map, create, flag); + if (!ret) { + map_bh(bh, inode->i_sb, map.m_pblk); + bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; + bh->b_size = map.m_len << inode->i_blkbits; + } + return ret; +} + +static int get_data_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create, int flag) +{ + return __get_data_block(inode, iblock, bh_result, create, flag); +} + +static int get_data_block_dio(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + return __get_data_block(inode, iblock, bh_result, create, + F2FS_GET_BLOCK_DIO); +} + +static int get_data_block_bmap(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + return __get_data_block(inode, iblock, bh_result, create, + F2FS_GET_BLOCK_BMAP); +} + +static inline sector_t logical_to_blk(struct inode *inode, loff_t offset) +{ + return (offset >> inode->i_blkbits); +} + +static inline loff_t blk_to_logical(struct inode *inode, sector_t blk) +{ + return (blk << inode->i_blkbits); +} + +int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct buffer_head map_bh; + sector_t start_blk, last_blk; + loff_t isize = i_size_read(inode); + u64 logical = 0, phys = 0, size = 0; + u32 flags = 0; + bool past_eof = false, whole_file = false; + int ret = 0; + + ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); + if (ret) + return ret; + + if (f2fs_has_inline_data(inode)) { + ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); + if (ret != -EAGAIN) + return ret; + } + + mutex_lock(&inode->i_mutex); + + if (len >= isize) { + whole_file = true; + len = isize; + } + + if (logical_to_blk(inode, len) == 0) + len = blk_to_logical(inode, 1); + + start_blk = logical_to_blk(inode, start); + last_blk = logical_to_blk(inode, start + len - 1); +next: + memset(&map_bh, 0, sizeof(struct buffer_head)); + map_bh.b_size = len; + + ret = get_data_block(inode, start_blk, &map_bh, 0, + F2FS_GET_BLOCK_FIEMAP); + if (ret) + goto out; + + /* HOLE */ + if (!buffer_mapped(&map_bh)) { + start_blk++; + + if (!past_eof && blk_to_logical(inode, start_blk) >= isize) + past_eof = 1; + + if (past_eof && size) { + flags |= FIEMAP_EXTENT_LAST; + ret = fiemap_fill_next_extent(fieinfo, logical, + phys, size, flags); + } else if (size) { + ret = fiemap_fill_next_extent(fieinfo, logical, + phys, size, flags); + size = 0; + } + + /* if we have holes up to/past EOF then we're done */ + if (start_blk > last_blk || past_eof || ret) + goto out; + } else { + if (start_blk > last_blk && !whole_file) { + ret = fiemap_fill_next_extent(fieinfo, logical, + phys, size, flags); + goto out; + } + + /* + * if size != 0 then we know we already have an extent + * to add, so add it. + */ + if (size) { + ret = fiemap_fill_next_extent(fieinfo, logical, + phys, size, flags); + if (ret) + goto out; + } + + logical = blk_to_logical(inode, start_blk); + phys = blk_to_logical(inode, map_bh.b_blocknr); + size = map_bh.b_size; + flags = 0; + if (buffer_unwritten(&map_bh)) + flags = FIEMAP_EXTENT_UNWRITTEN; + + start_blk += logical_to_blk(inode, size); + + /* + * If we are past the EOF, then we need to make sure as + * soon as we find a hole that the last extent we found + * is marked with FIEMAP_EXTENT_LAST + */ + if (!past_eof && logical + size >= isize) + past_eof = true; + } + cond_resched(); + if (fatal_signal_pending(current)) + ret = -EINTR; + else + goto next; +out: + if (ret == 1) + ret = 0; + + mutex_unlock(&inode->i_mutex); + return ret; +} + +/* + * This function was originally taken from fs/mpage.c, and customized for f2fs. + * Major change was from block_size == page_size in f2fs by default. + */ +static int f2fs_mpage_readpages(struct address_space *mapping, + struct list_head *pages, struct page *page, + unsigned nr_pages) +{ + struct bio *bio = NULL; + unsigned page_idx; + sector_t last_block_in_bio = 0; + struct inode *inode = mapping->host; + const unsigned blkbits = inode->i_blkbits; + const unsigned blocksize = 1 << blkbits; + sector_t block_in_file; + sector_t last_block; + sector_t last_block_in_file; + sector_t block_nr; + struct block_device *bdev = inode->i_sb->s_bdev; + struct f2fs_map_blocks map; + + map.m_pblk = 0; + map.m_lblk = 0; + map.m_len = 0; + map.m_flags = 0; + + for (page_idx = 0; nr_pages; page_idx++, nr_pages--) { + + prefetchw(&page->flags); + if (pages) { + page = list_entry(pages->prev, struct page, lru); + list_del(&page->lru); + if (add_to_page_cache_lru(page, mapping, + page->index, GFP_KERNEL)) + goto next_page; + } + + block_in_file = (sector_t)page->index; + last_block = block_in_file + nr_pages; + last_block_in_file = (i_size_read(inode) + blocksize - 1) >> + blkbits; + if (last_block > last_block_in_file) + last_block = last_block_in_file; + + /* + * Map blocks using the previous result first. + */ + if ((map.m_flags & F2FS_MAP_MAPPED) && + block_in_file > map.m_lblk && + block_in_file < (map.m_lblk + map.m_len)) + goto got_it; + + /* + * Then do more f2fs_map_blocks() calls until we are + * done with this page. + */ + map.m_flags = 0; + + if (block_in_file < last_block) { + map.m_lblk = block_in_file; + map.m_len = last_block - block_in_file; + + if (f2fs_map_blocks(inode, &map, 0, + F2FS_GET_BLOCK_READ)) + goto set_error_page; + } +got_it: + if ((map.m_flags & F2FS_MAP_MAPPED)) { + block_nr = map.m_pblk + block_in_file - map.m_lblk; + SetPageMappedToDisk(page); + + if (!PageUptodate(page) && !cleancache_get_page(page)) { + SetPageUptodate(page); + goto confused; + } + } else { + zero_user_segment(page, 0, PAGE_CACHE_SIZE); + SetPageUptodate(page); + unlock_page(page); + goto next_page; + } + + /* + * This page will go to BIO. Do we need to send this + * BIO off first? + */ + if (bio && (last_block_in_bio != block_nr - 1)) { +submit_and_realloc: + submit_bio(READ, bio); + bio = NULL; + } + if (bio == NULL) { + struct f2fs_crypto_ctx *ctx = NULL; + + if (f2fs_encrypted_inode(inode) && + S_ISREG(inode->i_mode)) { + + ctx = f2fs_get_crypto_ctx(inode); + if (IS_ERR(ctx)) + goto set_error_page; + + /* wait the page to be moved by cleaning */ + f2fs_wait_on_encrypted_page_writeback( + F2FS_I_SB(inode), block_nr); + } + + bio = bio_alloc(GFP_KERNEL, + min_t(int, nr_pages, BIO_MAX_PAGES)); + if (!bio) { + if (ctx) + f2fs_release_crypto_ctx(ctx); + goto set_error_page; + } + bio->bi_bdev = bdev; + bio->bi_sector = SECTOR_FROM_BLOCK(block_nr); + bio->bi_end_io = f2fs_read_end_io; + bio->bi_private = ctx; + } + + if (bio_add_page(bio, page, blocksize, 0) < blocksize) + goto submit_and_realloc; + + last_block_in_bio = block_nr; + goto next_page; +set_error_page: + SetPageError(page); + zero_user_segment(page, 0, PAGE_CACHE_SIZE); + unlock_page(page); + goto next_page; +confused: + if (bio) { + submit_bio(READ, bio); + bio = NULL; + } + unlock_page(page); +next_page: + if (pages) + page_cache_release(page); + } + BUG_ON(pages && !list_empty(pages)); + if (bio) + submit_bio(READ, bio); + return 0; +} + +static int f2fs_read_data_page(struct file *file, struct page *page) +{ + struct inode *inode = page->mapping->host; + int ret = -EAGAIN; + + trace_f2fs_readpage(page, DATA); + + /* If the file has inline data, try to read it directly */ + if (f2fs_has_inline_data(inode)) + ret = f2fs_read_inline_data(inode, page); + if (ret == -EAGAIN) + ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1); + return ret; +} + +static int f2fs_read_data_pages(struct file *file, + struct address_space *mapping, + struct list_head *pages, unsigned nr_pages) +{ + struct inode *inode = file->f_mapping->host; + struct page *page = list_entry(pages->prev, struct page, lru); + + trace_f2fs_readpages(inode, page, nr_pages); + + /* If the file has inline data, skip readpages */ + if (f2fs_has_inline_data(inode)) + return 0; + + return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages); +} + +int do_write_data_page(struct f2fs_io_info *fio) +{ + struct page *page = fio->page; + struct inode *inode = page->mapping->host; + struct dnode_of_data dn; + int err = 0; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); + if (err) + return err; + + fio->blk_addr = dn.data_blkaddr; + + /* This page is already truncated */ + if (fio->blk_addr == NULL_ADDR) { + ClearPageUptodate(page); + goto out_writepage; + } + + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { + + /* wait for GCed encrypted page writeback */ + f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode), + fio->blk_addr); + + fio->encrypted_page = f2fs_encrypt(inode, fio->page); + if (IS_ERR(fio->encrypted_page)) { + err = PTR_ERR(fio->encrypted_page); + goto out_writepage; + } + } + + set_page_writeback(page); + + /* + * If current allocation needs SSR, + * it had better in-place writes for updated data. + */ + if (unlikely(fio->blk_addr != NEW_ADDR && + !is_cold_data(page) && + need_inplace_update(inode))) { + rewrite_data_page(fio); + set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE); + trace_f2fs_do_write_data_page(page, IPU); + } else { + write_data_page(&dn, fio); + set_data_blkaddr(&dn); + f2fs_update_extent_cache(&dn); + trace_f2fs_do_write_data_page(page, OPU); + set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); + if (page->index == 0) + set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); + } +out_writepage: + f2fs_put_dnode(&dn); + return err; +} + +static int f2fs_write_data_page(struct page *page, + struct writeback_control *wbc) +{ + struct inode *inode = page->mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + loff_t i_size = i_size_read(inode); + const pgoff_t end_index = ((unsigned long long) i_size) + >> PAGE_CACHE_SHIFT; + unsigned offset = 0; + bool need_balance_fs = false; + int err = 0; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = DATA, + .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE, + .page = page, + .encrypted_page = NULL, + }; + + trace_f2fs_writepage(page, DATA); + + if (page->index < end_index) + goto write; + + /* + * If the offset is out-of-range of file size, + * this page does not have to be written to disk. + */ + offset = i_size & (PAGE_CACHE_SIZE - 1); + if ((page->index >= end_index + 1) || !offset) + goto out; + + zero_user_segment(page, offset, PAGE_CACHE_SIZE); +write: + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto redirty_out; + if (f2fs_is_drop_cache(inode)) + goto out; + if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim && + available_free_memory(sbi, BASE_CHECK)) + goto redirty_out; + + /* Dentry blocks are controlled by checkpoint */ + if (S_ISDIR(inode->i_mode)) { + if (unlikely(f2fs_cp_error(sbi))) + goto redirty_out; + err = do_write_data_page(&fio); + goto done; + } + + /* we should bypass data pages to proceed the kworkder jobs */ + if (unlikely(f2fs_cp_error(sbi))) { + SetPageError(page); + goto out; + } + + if (!wbc->for_reclaim) + need_balance_fs = true; + else if (has_not_enough_free_secs(sbi, 0)) + goto redirty_out; + + err = -EAGAIN; + f2fs_lock_op(sbi); + if (f2fs_has_inline_data(inode)) + err = f2fs_write_inline_data(inode, page); + if (err == -EAGAIN) + err = do_write_data_page(&fio); + f2fs_unlock_op(sbi); +done: + if (err && err != -ENOENT) + goto redirty_out; + + clear_cold_data(page); +out: + inode_dec_dirty_pages(inode); + if (err) + ClearPageUptodate(page); + unlock_page(page); + if (need_balance_fs) + f2fs_balance_fs(sbi); + if (wbc->for_reclaim) + f2fs_submit_merged_bio(sbi, DATA, WRITE); + return 0; + +redirty_out: + redirty_page_for_writepage(wbc, page); + return AOP_WRITEPAGE_ACTIVATE; +} + +static int __f2fs_writepage(struct page *page, struct writeback_control *wbc, + void *data) +{ + struct address_space *mapping = data; + int ret = mapping->a_ops->writepage(page, wbc); + mapping_set_error(mapping, ret); + return ret; +} + +/* + * This function was copied from write_cche_pages from mm/page-writeback.c. + * The major change is making write step of cold data page separately from + * warm/hot data page. + */ +static int f2fs_write_cache_pages(struct address_space *mapping, + struct writeback_control *wbc, writepage_t writepage, + void *data) +{ + int ret = 0; + int done = 0; + struct pagevec pvec; + int nr_pages; + pgoff_t uninitialized_var(writeback_index); + pgoff_t index; + pgoff_t end; /* Inclusive */ + pgoff_t done_index; + int cycled; + int range_whole = 0; + int tag; + int step = 0; + + pagevec_init(&pvec, 0); +next: + if (wbc->range_cyclic) { + writeback_index = mapping->writeback_index; /* prev offset */ + index = writeback_index; + if (index == 0) + cycled = 1; + else + cycled = 0; + end = -1; + } else { + index = wbc->range_start >> PAGE_CACHE_SHIFT; + end = wbc->range_end >> PAGE_CACHE_SHIFT; + if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) + range_whole = 1; + cycled = 1; /* ignore range_cyclic tests */ + } + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag = PAGECACHE_TAG_TOWRITE; + else + tag = PAGECACHE_TAG_DIRTY; +retry: + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && (index <= end)) { + int i; + + nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, + min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1); + if (nr_pages == 0) + break; + + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + + if (page->index > end) { + done = 1; + break; + } + + done_index = page->index; + + lock_page(page); + + if (unlikely(page->mapping != mapping)) { +continue_unlock: + unlock_page(page); + continue; + } + + if (!PageDirty(page)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + if (step == is_cold_data(page)) + goto continue_unlock; + + if (PageWriteback(page)) { + if (wbc->sync_mode != WB_SYNC_NONE) + f2fs_wait_on_page_writeback(page, DATA); + else + goto continue_unlock; + } + + BUG_ON(PageWriteback(page)); + if (!clear_page_dirty_for_io(page)) + goto continue_unlock; + + ret = (*writepage)(page, wbc, data); + if (unlikely(ret)) { + if (ret == AOP_WRITEPAGE_ACTIVATE) { + unlock_page(page); + ret = 0; + } else { + done_index = page->index + 1; + done = 1; + break; + } + } + + if (--wbc->nr_to_write <= 0 && + wbc->sync_mode == WB_SYNC_NONE) { + done = 1; + break; + } + } + pagevec_release(&pvec); + cond_resched(); + } + + if (step < 1) { + step++; + goto next; + } + + if (!cycled && !done) { + cycled = 1; + index = 0; + end = writeback_index - 1; + goto retry; + } + if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) + mapping->writeback_index = done_index; + + return ret; +} + +static int f2fs_write_data_pages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + bool locked = false; + int ret; + long diff; + + trace_f2fs_writepages(mapping->host, wbc, DATA); + + /* deal with chardevs and other special file */ + if (!mapping->a_ops->writepage) + return 0; + + /* skip writing if there is no dirty page in this inode */ + if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) + return 0; + + if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && + get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && + available_free_memory(sbi, DIRTY_DENTS)) + goto skip_write; + + /* during POR, we don't need to trigger writepage at all. */ + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto skip_write; + + diff = nr_pages_to_write(sbi, DATA, wbc); + + if (!S_ISDIR(inode->i_mode)) { + mutex_lock(&sbi->writepages); + locked = true; + } + ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping); + f2fs_submit_merged_bio(sbi, DATA, WRITE); + if (locked) + mutex_unlock(&sbi->writepages); + + remove_dirty_dir_inode(inode); + + wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); + return ret; + +skip_write: + wbc->pages_skipped += get_dirty_pages(inode); + return 0; +} + +static void f2fs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, 0, inode->i_size); + truncate_blocks(inode, inode->i_size, true); + } +} + +static int f2fs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct page *page = NULL; + struct page *ipage; + pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; + struct dnode_of_data dn; + int err = 0; + + trace_f2fs_write_begin(inode, pos, len, flags); + + f2fs_balance_fs(sbi); + + /* + * We should check this at this moment to avoid deadlock on inode page + * and #0 page. The locking rule for inline_data conversion should be: + * lock_page(page #0) -> lock_page(inode_page) + */ + if (index != 0) { + err = f2fs_convert_inline_inode(inode); + if (err) + goto fail; + } +repeat: + page = grab_cache_page_write_begin(mapping, index, flags); + if (!page) { + err = -ENOMEM; + goto fail; + } + + *pagep = page; + + f2fs_lock_op(sbi); + + /* check inline_data */ + ipage = get_node_page(sbi, inode->i_ino); + if (IS_ERR(ipage)) { + err = PTR_ERR(ipage); + goto unlock_fail; + } + + set_new_dnode(&dn, inode, ipage, ipage, 0); + + if (f2fs_has_inline_data(inode)) { + if (pos + len <= MAX_INLINE_DATA) { + read_inline_data(page, ipage); + set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); + sync_inode_page(&dn); + goto put_next; + } + err = f2fs_convert_inline_page(&dn, page); + if (err) + goto put_fail; + } + + err = f2fs_get_block(&dn, index); + if (err) + goto put_fail; +put_next: + f2fs_put_dnode(&dn); + f2fs_unlock_op(sbi); + + f2fs_wait_on_page_writeback(page, DATA); + + /* wait for GCed encrypted page writeback */ + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) + f2fs_wait_on_encrypted_page_writeback(sbi, dn.data_blkaddr); + + if (len == PAGE_CACHE_SIZE) + goto out_update; + if (PageUptodate(page)) + goto out_clear; + + if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { + unsigned start = pos & (PAGE_CACHE_SIZE - 1); + unsigned end = start + len; + + /* Reading beyond i_size is simple: memset to zero */ + zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE); + goto out_update; + } + + if (dn.data_blkaddr == NEW_ADDR) { + zero_user_segment(page, 0, PAGE_CACHE_SIZE); + } else { + struct f2fs_io_info fio = { + .sbi = sbi, + .type = DATA, + .rw = READ_SYNC, + .blk_addr = dn.data_blkaddr, + .page = page, + .encrypted_page = NULL, + }; + err = f2fs_submit_page_bio(&fio); + if (err) + goto fail; + + lock_page(page); + if (unlikely(!PageUptodate(page))) { + err = -EIO; + goto fail; + } + if (unlikely(page->mapping != mapping)) { + f2fs_put_page(page, 1); + goto repeat; + } + + /* avoid symlink page */ + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { + err = f2fs_decrypt_one(inode, page); + if (err) + goto fail; + } + } +out_update: + SetPageUptodate(page); +out_clear: + clear_cold_data(page); + return 0; + +put_fail: + f2fs_put_dnode(&dn); +unlock_fail: + f2fs_unlock_op(sbi); +fail: + f2fs_put_page(page, 1); + f2fs_write_failed(mapping, pos + len); + return err; +} + +static int f2fs_write_end(struct file *file, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + struct inode *inode = page->mapping->host; + + trace_f2fs_write_end(inode, pos, len, copied); + + set_page_dirty(page); + + if (pos + copied > i_size_read(inode)) { + i_size_write(inode, pos + copied); + mark_inode_dirty(inode); + update_inode_page(inode); + } + + f2fs_put_page(page, 1); + return copied; +} + +static ssize_t check_direct_IO(struct inode *inode, int rw, + const struct iovec *iov, loff_t offset, unsigned long nr_segs) +{ + unsigned blocksize_mask = inode->i_sb->s_blocksize - 1; + int seg, i; + size_t size; + unsigned long addr; + ssize_t retval = -EINVAL; + loff_t end = offset; + + if (offset & blocksize_mask) + return -EINVAL; + + /* Check the memory alignment. Blocks cannot straddle pages */ + for (seg = 0; seg < nr_segs; seg++) { + addr = (unsigned long)iov[seg].iov_base; + size = iov[seg].iov_len; + end += size; + if ((addr & blocksize_mask) || (size & blocksize_mask)) + goto out; + + /* If this is a write we don't need to check anymore */ + if (rw & WRITE) + continue; + + /* + * Check to make sure we don't have duplicate iov_base's in this + * iovec, if so return EINVAL, otherwise we'll get csum errors + * when reading back. + */ + for (i = seg + 1; i < nr_segs; i++) { + if (iov[seg].iov_base == iov[i].iov_base) + goto out; + } + } + retval = 0; +out: + return retval; +} + +static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb, + const struct iovec *iov, loff_t offset, + unsigned long nr_segs) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + size_t count = iov_length(iov, nr_segs); + int err; + + /* we don't need to use inline_data strictly */ + if (f2fs_has_inline_data(inode)) { + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) + return 0; + + err = check_direct_IO(inode, rw, iov, offset, nr_segs); + if (err) + return err; + + trace_f2fs_direct_IO_enter(inode, offset, count, rw); + + if (rw & WRITE) { + __allocate_data_blocks(inode, offset, count); + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { + err = -EIO; + goto out; + } + } + + err = blockdev_direct_IO(rw, iocb, inode, iov, offset, nr_segs, + get_data_block_dio); +out: + if (err < 0 && (rw & WRITE)) + f2fs_write_failed(mapping, offset + count); + + trace_f2fs_direct_IO_exit(inode, offset, count, rw, err); + + return err; +} + +void f2fs_invalidate_page(struct page *page, unsigned long offset) +{ + struct inode *inode = page->mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (inode->i_ino >= F2FS_ROOT_INO(sbi) && (offset % PAGE_CACHE_SIZE)) + return; + + if (PageDirty(page)) { + if (inode->i_ino == F2FS_META_INO(sbi)) + dec_page_count(sbi, F2FS_DIRTY_META); + else if (inode->i_ino == F2FS_NODE_INO(sbi)) + dec_page_count(sbi, F2FS_DIRTY_NODES); + else + inode_dec_dirty_pages(inode); + } + + /* This is atomic written page, keep Private */ + if (IS_ATOMIC_WRITTEN_PAGE(page)) + return; + + ClearPagePrivate(page); +} + +int f2fs_release_page(struct page *page, gfp_t wait) +{ + /* If this is dirty page, keep PagePrivate */ + if (PageDirty(page)) + return 0; + + /* This is atomic written page, keep Private */ + if (IS_ATOMIC_WRITTEN_PAGE(page)) + return 0; + + ClearPagePrivate(page); + return 1; +} + +static int f2fs_set_data_page_dirty(struct page *page) +{ + struct address_space *mapping = page->mapping; + struct inode *inode = mapping->host; + + trace_f2fs_set_page_dirty(page, DATA); + + SetPageUptodate(page); + + if (f2fs_is_atomic_file(inode)) { + if (!IS_ATOMIC_WRITTEN_PAGE(page)) { + register_inmem_page(inode, page); + return 1; + } + /* + * Previously, this page has been registered, we just + * return here. + */ + return 0; + } + + if (!PageDirty(page)) { + __set_page_dirty_nobuffers(page); + update_dirty_page(inode, page); + return 1; + } + return 0; +} + +static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *inode = mapping->host; + + if (f2fs_has_inline_data(inode)) + return 0; + + /* make sure allocating whole blocks */ + if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) + filemap_write_and_wait(mapping); + + return generic_block_bmap(mapping, block, get_data_block_bmap); +} + +const struct address_space_operations f2fs_dblock_aops = { + .readpage = f2fs_read_data_page, + .readpages = f2fs_read_data_pages, + .writepage = f2fs_write_data_page, + .writepages = f2fs_write_data_pages, + .write_begin = f2fs_write_begin, + .write_end = f2fs_write_end, + .set_page_dirty = f2fs_set_data_page_dirty, + .invalidatepage = f2fs_invalidate_page, + .releasepage = f2fs_release_page, + .direct_IO = f2fs_direct_IO, + .bmap = f2fs_bmap, +}; diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c new file mode 100644 index 00000000000..478e5d54154 --- /dev/null +++ b/fs/f2fs/debug.c @@ -0,0 +1,432 @@ +/* + * f2fs debugging statistics + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * Copyright (c) 2012 Linux Foundation + * Copyright (c) 2012 Greg Kroah-Hartman + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "gc.h" + +static LIST_HEAD(f2fs_stat_list); +static struct dentry *f2fs_debugfs_root; +static DEFINE_MUTEX(f2fs_stat_mutex); + +static void update_general_status(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + int i; + + /* validation check of the segment numbers */ + si->hit_largest = atomic64_read(&sbi->read_hit_largest); + si->hit_cached = atomic64_read(&sbi->read_hit_cached); + si->hit_rbtree = atomic64_read(&sbi->read_hit_rbtree); + si->hit_total = si->hit_largest + si->hit_cached + si->hit_rbtree; + si->total_ext = atomic64_read(&sbi->total_hit_ext); + si->ext_tree = sbi->total_ext_tree; + si->ext_node = atomic_read(&sbi->total_ext_node); + si->ndirty_node = get_pages(sbi, F2FS_DIRTY_NODES); + si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS); + si->ndirty_dirs = sbi->n_dirty_dirs; + si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META); + si->inmem_pages = get_pages(sbi, F2FS_INMEM_PAGES); + si->wb_pages = get_pages(sbi, F2FS_WRITEBACK); + si->total_count = (int)sbi->user_block_count / sbi->blocks_per_seg; + si->rsvd_segs = reserved_segments(sbi); + si->overp_segs = overprovision_segments(sbi); + si->valid_count = valid_user_blocks(sbi); + si->valid_node_count = valid_node_count(sbi); + si->valid_inode_count = valid_inode_count(sbi); + si->inline_xattr = atomic_read(&sbi->inline_xattr); + si->inline_inode = atomic_read(&sbi->inline_inode); + si->inline_dir = atomic_read(&sbi->inline_dir); + si->utilization = utilization(sbi); + + si->free_segs = free_segments(sbi); + si->free_secs = free_sections(sbi); + si->prefree_count = prefree_segments(sbi); + si->dirty_count = dirty_segments(sbi); + si->node_pages = NODE_MAPPING(sbi)->nrpages; + si->meta_pages = META_MAPPING(sbi)->nrpages; + si->nats = NM_I(sbi)->nat_cnt; + si->dirty_nats = NM_I(sbi)->dirty_nat_cnt; + si->sits = MAIN_SEGS(sbi); + si->dirty_sits = SIT_I(sbi)->dirty_sentries; + si->fnids = NM_I(sbi)->fcnt; + si->bg_gc = sbi->bg_gc; + si->util_free = (int)(free_user_blocks(sbi) >> sbi->log_blocks_per_seg) + * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) + / 2; + si->util_valid = (int)(written_block_count(sbi) >> + sbi->log_blocks_per_seg) + * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) + / 2; + si->util_invalid = 50 - si->util_free - si->util_valid; + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_NODE; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i); + si->curseg[i] = curseg->segno; + si->cursec[i] = curseg->segno / sbi->segs_per_sec; + si->curzone[i] = si->cursec[i] / sbi->secs_per_zone; + } + + for (i = 0; i < 2; i++) { + si->segment_count[i] = sbi->segment_count[i]; + si->block_count[i] = sbi->block_count[i]; + } + + si->inplace_count = atomic_read(&sbi->inplace_count); +} + +/* + * This function calculates BDF of every segments + */ +static void update_sit_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + unsigned long long blks_per_sec, hblks_per_sec, total_vblocks; + unsigned long long bimodal, dist; + unsigned int segno, vblocks; + int ndirty = 0; + + bimodal = 0; + total_vblocks = 0; + blks_per_sec = sbi->segs_per_sec * (1 << sbi->log_blocks_per_seg); + hblks_per_sec = blks_per_sec / 2; + for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { + vblocks = get_valid_blocks(sbi, segno, sbi->segs_per_sec); + dist = abs(vblocks - hblks_per_sec); + bimodal += dist * dist; + + if (vblocks > 0 && vblocks < blks_per_sec) { + total_vblocks += vblocks; + ndirty++; + } + } + dist = div_u64(MAIN_SECS(sbi) * hblks_per_sec * hblks_per_sec, 100); + si->bimodal = div64_u64(bimodal, dist); + if (si->dirty_count) + si->avg_vblocks = div_u64(total_vblocks, ndirty); + else + si->avg_vblocks = 0; +} + +/* + * This function calculates memory footprint. + */ +static void update_mem_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + unsigned npages; + int i; + + if (si->base_mem) + goto get_cache; + + si->base_mem = sizeof(struct f2fs_sb_info) + sbi->sb->s_blocksize; + si->base_mem += 2 * sizeof(struct f2fs_inode_info); + si->base_mem += sizeof(*sbi->ckpt); + + /* build sm */ + si->base_mem += sizeof(struct f2fs_sm_info); + + /* build sit */ + si->base_mem += sizeof(struct sit_info); + si->base_mem += MAIN_SEGS(sbi) * sizeof(struct seg_entry); + si->base_mem += f2fs_bitmap_size(MAIN_SEGS(sbi)); + si->base_mem += 3 * SIT_VBLOCK_MAP_SIZE * MAIN_SEGS(sbi); + si->base_mem += SIT_VBLOCK_MAP_SIZE; + if (sbi->segs_per_sec > 1) + si->base_mem += MAIN_SECS(sbi) * sizeof(struct sec_entry); + si->base_mem += __bitmap_size(sbi, SIT_BITMAP); + + /* build free segmap */ + si->base_mem += sizeof(struct free_segmap_info); + si->base_mem += f2fs_bitmap_size(MAIN_SEGS(sbi)); + si->base_mem += f2fs_bitmap_size(MAIN_SECS(sbi)); + + /* build curseg */ + si->base_mem += sizeof(struct curseg_info) * NR_CURSEG_TYPE; + si->base_mem += PAGE_CACHE_SIZE * NR_CURSEG_TYPE; + + /* build dirty segmap */ + si->base_mem += sizeof(struct dirty_seglist_info); + si->base_mem += NR_DIRTY_TYPE * f2fs_bitmap_size(MAIN_SEGS(sbi)); + si->base_mem += f2fs_bitmap_size(MAIN_SECS(sbi)); + + /* build nm */ + si->base_mem += sizeof(struct f2fs_nm_info); + si->base_mem += __bitmap_size(sbi, NAT_BITMAP); + +get_cache: + si->cache_mem = 0; + + /* build gc */ + if (sbi->gc_thread) + si->cache_mem += sizeof(struct f2fs_gc_kthread); + + /* build merge flush thread */ + if (SM_I(sbi)->cmd_control_info) + si->cache_mem += sizeof(struct flush_cmd_control); + + /* free nids */ + si->cache_mem += NM_I(sbi)->fcnt * sizeof(struct free_nid); + si->cache_mem += NM_I(sbi)->nat_cnt * sizeof(struct nat_entry); + si->cache_mem += NM_I(sbi)->dirty_nat_cnt * + sizeof(struct nat_entry_set); + si->cache_mem += si->inmem_pages * sizeof(struct inmem_pages); + si->cache_mem += sbi->n_dirty_dirs * sizeof(struct inode_entry); + for (i = 0; i <= UPDATE_INO; i++) + si->cache_mem += sbi->im[i].ino_num * sizeof(struct ino_entry); + si->cache_mem += sbi->total_ext_tree * sizeof(struct extent_tree); + si->cache_mem += atomic_read(&sbi->total_ext_node) * + sizeof(struct extent_node); + + si->page_mem = 0; + npages = NODE_MAPPING(sbi)->nrpages; + si->page_mem += (unsigned long long)npages << PAGE_CACHE_SHIFT; + npages = META_MAPPING(sbi)->nrpages; + si->page_mem += (unsigned long long)npages << PAGE_CACHE_SHIFT; +} + +static int stat_show(struct seq_file *s, void *v) +{ + struct f2fs_stat_info *si; + int i = 0; + int j; + + mutex_lock(&f2fs_stat_mutex); + list_for_each_entry(si, &f2fs_stat_list, stat_list) { + char devname[BDEVNAME_SIZE]; + + update_general_status(si->sbi); + + seq_printf(s, "\n=====[ partition info(%s). #%d ]=====\n", + bdevname(si->sbi->sb->s_bdev, devname), i++); + seq_printf(s, "[SB: 1] [CP: 2] [SIT: %d] [NAT: %d] ", + si->sit_area_segs, si->nat_area_segs); + seq_printf(s, "[SSA: %d] [MAIN: %d", + si->ssa_area_segs, si->main_area_segs); + seq_printf(s, "(OverProv:%d Resv:%d)]\n\n", + si->overp_segs, si->rsvd_segs); + seq_printf(s, "Utilization: %d%% (%d valid blocks)\n", + si->utilization, si->valid_count); + seq_printf(s, " - Node: %u (Inode: %u, ", + si->valid_node_count, si->valid_inode_count); + seq_printf(s, "Other: %u)\n - Data: %u\n", + si->valid_node_count - si->valid_inode_count, + si->valid_count - si->valid_node_count); + seq_printf(s, " - Inline_xattr Inode: %u\n", + si->inline_xattr); + seq_printf(s, " - Inline_data Inode: %u\n", + si->inline_inode); + seq_printf(s, " - Inline_dentry Inode: %u\n", + si->inline_dir); + seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n", + si->main_area_segs, si->main_area_sections, + si->main_area_zones); + seq_printf(s, " - COLD data: %d, %d, %d\n", + si->curseg[CURSEG_COLD_DATA], + si->cursec[CURSEG_COLD_DATA], + si->curzone[CURSEG_COLD_DATA]); + seq_printf(s, " - WARM data: %d, %d, %d\n", + si->curseg[CURSEG_WARM_DATA], + si->cursec[CURSEG_WARM_DATA], + si->curzone[CURSEG_WARM_DATA]); + seq_printf(s, " - HOT data: %d, %d, %d\n", + si->curseg[CURSEG_HOT_DATA], + si->cursec[CURSEG_HOT_DATA], + si->curzone[CURSEG_HOT_DATA]); + seq_printf(s, " - Dir dnode: %d, %d, %d\n", + si->curseg[CURSEG_HOT_NODE], + si->cursec[CURSEG_HOT_NODE], + si->curzone[CURSEG_HOT_NODE]); + seq_printf(s, " - File dnode: %d, %d, %d\n", + si->curseg[CURSEG_WARM_NODE], + si->cursec[CURSEG_WARM_NODE], + si->curzone[CURSEG_WARM_NODE]); + seq_printf(s, " - Indir nodes: %d, %d, %d\n", + si->curseg[CURSEG_COLD_NODE], + si->cursec[CURSEG_COLD_NODE], + si->curzone[CURSEG_COLD_NODE]); + seq_printf(s, "\n - Valid: %d\n - Dirty: %d\n", + si->main_area_segs - si->dirty_count - + si->prefree_count - si->free_segs, + si->dirty_count); + seq_printf(s, " - Prefree: %d\n - Free: %d (%d)\n\n", + si->prefree_count, si->free_segs, si->free_secs); + seq_printf(s, "CP calls: %d\n", si->cp_count); + seq_printf(s, "GC calls: %d (BG: %d)\n", + si->call_count, si->bg_gc); + seq_printf(s, " - data segments : %d (%d)\n", + si->data_segs, si->bg_data_segs); + seq_printf(s, " - node segments : %d (%d)\n", + si->node_segs, si->bg_node_segs); + seq_printf(s, "Try to move %d blocks (BG: %d)\n", si->tot_blks, + si->bg_data_blks + si->bg_node_blks); + seq_printf(s, " - data blocks : %d (%d)\n", si->data_blks, + si->bg_data_blks); + seq_printf(s, " - node blocks : %d (%d)\n", si->node_blks, + si->bg_node_blks); + seq_puts(s, "\nExtent Cache:\n"); + seq_printf(s, " - Hit Count: L1-1:%llu L1-2:%llu L2:%llu\n", + si->hit_largest, si->hit_cached, + si->hit_rbtree); + seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", + !si->total_ext ? 0 : + div64_u64(si->hit_total * 100, si->total_ext), + si->hit_total, si->total_ext); + seq_printf(s, " - Inner Struct Count: tree: %d, node: %d\n", + si->ext_tree, si->ext_node); + seq_puts(s, "\nBalancing F2FS Async:\n"); + seq_printf(s, " - inmem: %4d, wb: %4d\n", + si->inmem_pages, si->wb_pages); + seq_printf(s, " - nodes: %4d in %4d\n", + si->ndirty_node, si->node_pages); + seq_printf(s, " - dents: %4d in dirs:%4d\n", + si->ndirty_dent, si->ndirty_dirs); + seq_printf(s, " - meta: %4d in %4d\n", + si->ndirty_meta, si->meta_pages); + seq_printf(s, " - NATs: %9d/%9d\n - SITs: %9d/%9d\n", + si->dirty_nats, si->nats, si->dirty_sits, si->sits); + seq_printf(s, " - free_nids: %9d\n", + si->fnids); + seq_puts(s, "\nDistribution of User Blocks:"); + seq_puts(s, " [ valid | invalid | free ]\n"); + seq_puts(s, " ["); + + for (j = 0; j < si->util_valid; j++) + seq_putc(s, '-'); + seq_putc(s, '|'); + + for (j = 0; j < si->util_invalid; j++) + seq_putc(s, '-'); + seq_putc(s, '|'); + + for (j = 0; j < si->util_free; j++) + seq_putc(s, '-'); + seq_puts(s, "]\n\n"); + seq_printf(s, "IPU: %u blocks\n", si->inplace_count); + seq_printf(s, "SSR: %u blocks in %u segments\n", + si->block_count[SSR], si->segment_count[SSR]); + seq_printf(s, "LFS: %u blocks in %u segments\n", + si->block_count[LFS], si->segment_count[LFS]); + + /* segment usage info */ + update_sit_info(si->sbi); + seq_printf(s, "\nBDF: %u, avg. vblocks: %u\n", + si->bimodal, si->avg_vblocks); + + /* memory footprint */ + update_mem_info(si->sbi); + seq_printf(s, "\nMemory: %llu KB\n", + (si->base_mem + si->cache_mem + si->page_mem) >> 10); + seq_printf(s, " - static: %llu KB\n", + si->base_mem >> 10); + seq_printf(s, " - cached: %llu KB\n", + si->cache_mem >> 10); + seq_printf(s, " - paged : %llu KB\n", + si->page_mem >> 10); + } + mutex_unlock(&f2fs_stat_mutex); + return 0; +} + +static int stat_open(struct inode *inode, struct file *file) +{ + return single_open(file, stat_show, inode->i_private); +} + +static const struct file_operations stat_fops = { + .open = stat_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int f2fs_build_stats(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_stat_info *si; + + si = kzalloc(sizeof(struct f2fs_stat_info), GFP_KERNEL); + if (!si) + return -ENOMEM; + + si->all_area_segs = le32_to_cpu(raw_super->segment_count); + si->sit_area_segs = le32_to_cpu(raw_super->segment_count_sit); + si->nat_area_segs = le32_to_cpu(raw_super->segment_count_nat); + si->ssa_area_segs = le32_to_cpu(raw_super->segment_count_ssa); + si->main_area_segs = le32_to_cpu(raw_super->segment_count_main); + si->main_area_sections = le32_to_cpu(raw_super->section_count); + si->main_area_zones = si->main_area_sections / + le32_to_cpu(raw_super->secs_per_zone); + si->sbi = sbi; + sbi->stat_info = si; + + atomic64_set(&sbi->total_hit_ext, 0); + atomic64_set(&sbi->read_hit_rbtree, 0); + atomic64_set(&sbi->read_hit_largest, 0); + atomic64_set(&sbi->read_hit_cached, 0); + + atomic_set(&sbi->inline_xattr, 0); + atomic_set(&sbi->inline_inode, 0); + atomic_set(&sbi->inline_dir, 0); + atomic_set(&sbi->inplace_count, 0); + + mutex_lock(&f2fs_stat_mutex); + list_add_tail(&si->stat_list, &f2fs_stat_list); + mutex_unlock(&f2fs_stat_mutex); + + return 0; +} + +void f2fs_destroy_stats(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + + mutex_lock(&f2fs_stat_mutex); + list_del(&si->stat_list); + mutex_unlock(&f2fs_stat_mutex); + + kfree(si); +} + +void __init f2fs_create_root_stats(void) +{ + struct dentry *file; + + f2fs_debugfs_root = debugfs_create_dir("f2fs", NULL); + if (!f2fs_debugfs_root) + return; + + file = debugfs_create_file("status", S_IRUGO, f2fs_debugfs_root, + NULL, &stat_fops); + if (!file) { + debugfs_remove(f2fs_debugfs_root); + f2fs_debugfs_root = NULL; + } +} + +void f2fs_destroy_root_stats(void) +{ + if (!f2fs_debugfs_root) + return; + + debugfs_remove_recursive(f2fs_debugfs_root); + f2fs_debugfs_root = NULL; +} diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c new file mode 100644 index 00000000000..f11d32b1c03 --- /dev/null +++ b/fs/f2fs/dir.c @@ -0,0 +1,902 @@ +/* + * fs/f2fs/dir.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include "f2fs.h" +#include "node.h" +#include "acl.h" +#include "xattr.h" + +static unsigned long dir_blocks(struct inode *inode) +{ + return ((unsigned long long) (i_size_read(inode) + PAGE_CACHE_SIZE - 1)) + >> PAGE_CACHE_SHIFT; +} + +static unsigned int dir_buckets(unsigned int level, int dir_level) +{ + if (level + dir_level < MAX_DIR_HASH_DEPTH / 2) + return 1 << (level + dir_level); + else + return MAX_DIR_BUCKETS; +} + +static unsigned int bucket_blocks(unsigned int level) +{ + if (level < MAX_DIR_HASH_DEPTH / 2) + return 2; + else + return 4; +} + +unsigned char f2fs_filetype_table[F2FS_FT_MAX] = { + [F2FS_FT_UNKNOWN] = DT_UNKNOWN, + [F2FS_FT_REG_FILE] = DT_REG, + [F2FS_FT_DIR] = DT_DIR, + [F2FS_FT_CHRDEV] = DT_CHR, + [F2FS_FT_BLKDEV] = DT_BLK, + [F2FS_FT_FIFO] = DT_FIFO, + [F2FS_FT_SOCK] = DT_SOCK, + [F2FS_FT_SYMLINK] = DT_LNK, +}; + +#define S_SHIFT 12 +static unsigned char f2fs_type_by_mode[S_IFMT >> S_SHIFT] = { + [S_IFREG >> S_SHIFT] = F2FS_FT_REG_FILE, + [S_IFDIR >> S_SHIFT] = F2FS_FT_DIR, + [S_IFCHR >> S_SHIFT] = F2FS_FT_CHRDEV, + [S_IFBLK >> S_SHIFT] = F2FS_FT_BLKDEV, + [S_IFIFO >> S_SHIFT] = F2FS_FT_FIFO, + [S_IFSOCK >> S_SHIFT] = F2FS_FT_SOCK, + [S_IFLNK >> S_SHIFT] = F2FS_FT_SYMLINK, +}; + +void set_de_type(struct f2fs_dir_entry *de, umode_t mode) +{ + de->file_type = f2fs_type_by_mode[(mode & S_IFMT) >> S_SHIFT]; +} + +static unsigned long dir_block_index(unsigned int level, + int dir_level, unsigned int idx) +{ + unsigned long i; + unsigned long bidx = 0; + + for (i = 0; i < level; i++) + bidx += dir_buckets(i, dir_level) * bucket_blocks(i); + bidx += idx * bucket_blocks(level); + return bidx; +} + +static struct f2fs_dir_entry *find_in_block(struct page *dentry_page, + struct f2fs_filename *fname, + f2fs_hash_t namehash, + int *max_slots, + struct page **res_page) +{ + struct f2fs_dentry_block *dentry_blk; + struct f2fs_dir_entry *de; + struct f2fs_dentry_ptr d; + + dentry_blk = (struct f2fs_dentry_block *)kmap(dentry_page); + + make_dentry_ptr(NULL, &d, (void *)dentry_blk, 1); + de = find_target_dentry(fname, namehash, max_slots, &d); + if (de) + *res_page = dentry_page; + else + kunmap(dentry_page); + + /* + * For the most part, it should be a bug when name_len is zero. + * We stop here for figuring out where the bugs has occurred. + */ + f2fs_bug_on(F2FS_P_SB(dentry_page), d.max < 0); + return de; +} + +struct f2fs_dir_entry *find_target_dentry(struct f2fs_filename *fname, + f2fs_hash_t namehash, int *max_slots, + struct f2fs_dentry_ptr *d) +{ + struct f2fs_dir_entry *de; + unsigned long bit_pos = 0; + int max_len = 0; + struct f2fs_str de_name = FSTR_INIT(NULL, 0); + struct f2fs_str *name = &fname->disk_name; + + if (max_slots) + *max_slots = 0; + while (bit_pos < d->max) { + if (!test_bit_le(bit_pos, d->bitmap)) { + bit_pos++; + max_len++; + continue; + } + + de = &d->dentry[bit_pos]; + + /* encrypted case */ + de_name.name = d->filename[bit_pos]; + de_name.len = le16_to_cpu(de->name_len); + + /* show encrypted name */ + if (fname->hash) { + if (de->hash_code == fname->hash) + goto found; + } else if (de_name.len == name->len && + de->hash_code == namehash && + !memcmp(de_name.name, name->name, name->len)) + goto found; + + if (max_slots && max_len > *max_slots) + *max_slots = max_len; + max_len = 0; + + /* remain bug on condition */ + if (unlikely(!de->name_len)) + d->max = -1; + + bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); + } + + de = NULL; +found: + if (max_slots && max_len > *max_slots) + *max_slots = max_len; + return de; +} + +static struct f2fs_dir_entry *find_in_level(struct inode *dir, + unsigned int level, + struct f2fs_filename *fname, + struct page **res_page) +{ + struct qstr name = FSTR_TO_QSTR(&fname->disk_name); + int s = GET_DENTRY_SLOTS(name.len); + unsigned int nbucket, nblock; + unsigned int bidx, end_block; + struct page *dentry_page; + struct f2fs_dir_entry *de = NULL; + bool room = false; + int max_slots; + f2fs_hash_t namehash; + + namehash = f2fs_dentry_hash(&name); + + f2fs_bug_on(F2FS_I_SB(dir), level > MAX_DIR_HASH_DEPTH); + + nbucket = dir_buckets(level, F2FS_I(dir)->i_dir_level); + nblock = bucket_blocks(level); + + bidx = dir_block_index(level, F2FS_I(dir)->i_dir_level, + le32_to_cpu(namehash) % nbucket); + end_block = bidx + nblock; + + for (; bidx < end_block; bidx++) { + /* no need to allocate new dentry pages to all the indices */ + dentry_page = find_data_page(dir, bidx); + if (IS_ERR(dentry_page)) { + room = true; + continue; + } + + de = find_in_block(dentry_page, fname, namehash, &max_slots, + res_page); + if (de) + break; + + if (max_slots >= s) + room = true; + f2fs_put_page(dentry_page, 0); + } + + if (!de && room && F2FS_I(dir)->chash != namehash) { + F2FS_I(dir)->chash = namehash; + F2FS_I(dir)->clevel = level; + } + + return de; +} + +/* + * Find an entry in the specified directory with the wanted name. + * It returns the page where the entry was found (as a parameter - res_page), + * and the entry itself. Page is returned mapped and unlocked. + * Entry is guaranteed to be valid. + */ +struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir, + struct qstr *child, struct page **res_page) +{ + unsigned long npages = dir_blocks(dir); + struct f2fs_dir_entry *de = NULL; + unsigned int max_depth; + unsigned int level; + struct f2fs_filename fname; + int err; + + *res_page = NULL; + + err = f2fs_fname_setup_filename(dir, child, 1, &fname); + if (err) + return NULL; + + if (f2fs_has_inline_dentry(dir)) { + de = find_in_inline_dir(dir, &fname, res_page); + goto out; + } + + if (npages == 0) + goto out; + + max_depth = F2FS_I(dir)->i_current_depth; + + for (level = 0; level < max_depth; level++) { + de = find_in_level(dir, level, &fname, res_page); + if (de) + break; + } +out: + f2fs_fname_free_filename(&fname); + return de; +} + +struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct page **p) +{ + struct page *page; + struct f2fs_dir_entry *de; + struct f2fs_dentry_block *dentry_blk; + + if (f2fs_has_inline_dentry(dir)) + return f2fs_parent_inline_dir(dir, p); + + page = get_lock_data_page(dir, 0, false); + if (IS_ERR(page)) + return NULL; + + dentry_blk = kmap(page); + de = &dentry_blk->dentry[1]; + *p = page; + unlock_page(page); + return de; +} + +ino_t f2fs_inode_by_name(struct inode *dir, struct qstr *qstr) +{ + ino_t res = 0; + struct f2fs_dir_entry *de; + struct page *page; + + de = f2fs_find_entry(dir, qstr, &page); + if (de) { + res = le32_to_cpu(de->ino); + f2fs_dentry_kunmap(dir, page); + f2fs_put_page(page, 0); + } + + return res; +} + +void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, + struct page *page, struct inode *inode) +{ + enum page_type type = f2fs_has_inline_dentry(dir) ? NODE : DATA; + lock_page(page); + f2fs_wait_on_page_writeback(page, type); + de->ino = cpu_to_le32(inode->i_ino); + set_de_type(de, inode->i_mode); + f2fs_dentry_kunmap(dir, page); + set_page_dirty(page); + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + mark_inode_dirty(dir); + + f2fs_put_page(page, 1); +} + +static void init_dent_inode(const struct qstr *name, struct page *ipage) +{ + struct f2fs_inode *ri; + + f2fs_wait_on_page_writeback(ipage, NODE); + + /* copy name info. to this inode page */ + ri = F2FS_INODE(ipage); + ri->i_namelen = cpu_to_le32(name->len); + memcpy(ri->i_name, name->name, name->len); + set_page_dirty(ipage); +} + +int update_dent_inode(struct inode *inode, struct inode *to, + const struct qstr *name) +{ + struct page *page; + + if (file_enc_name(to)) + return 0; + + page = get_node_page(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(page)) + return PTR_ERR(page); + + init_dent_inode(name, page); + f2fs_put_page(page, 1); + + return 0; +} + +void do_make_empty_dir(struct inode *inode, struct inode *parent, + struct f2fs_dentry_ptr *d) +{ + struct f2fs_dir_entry *de; + + de = &d->dentry[0]; + de->name_len = cpu_to_le16(1); + de->hash_code = 0; + de->ino = cpu_to_le32(inode->i_ino); + memcpy(d->filename[0], ".", 1); + set_de_type(de, inode->i_mode); + + de = &d->dentry[1]; + de->hash_code = 0; + de->name_len = cpu_to_le16(2); + de->ino = cpu_to_le32(parent->i_ino); + memcpy(d->filename[1], "..", 2); + set_de_type(de, parent->i_mode); + + test_and_set_bit_le(0, (void *)d->bitmap); + test_and_set_bit_le(1, (void *)d->bitmap); +} + +static int make_empty_dir(struct inode *inode, + struct inode *parent, struct page *page) +{ + struct page *dentry_page; + struct f2fs_dentry_block *dentry_blk; + struct f2fs_dentry_ptr d; + + if (f2fs_has_inline_dentry(inode)) + return make_empty_inline_dir(inode, parent, page); + + dentry_page = get_new_data_page(inode, page, 0, true); + if (IS_ERR(dentry_page)) + return PTR_ERR(dentry_page); + + dentry_blk = kmap_atomic(dentry_page); + + make_dentry_ptr(NULL, &d, (void *)dentry_blk, 1); + do_make_empty_dir(inode, parent, &d); + + kunmap_atomic(dentry_blk); + + set_page_dirty(dentry_page); + f2fs_put_page(dentry_page, 1); + return 0; +} + +struct page *init_inode_metadata(struct inode *inode, struct inode *dir, + const struct qstr *name, struct page *dpage) +{ + struct page *page; + int err; + + if (is_inode_flag_set(F2FS_I(inode), FI_NEW_INODE)) { + page = new_inode_page(inode); + if (IS_ERR(page)) + return page; + + if (S_ISDIR(inode->i_mode)) { + err = make_empty_dir(inode, dir, page); + if (err) + goto error; + } + + err = f2fs_init_acl(inode, dir, page, dpage); + if (err) + goto put_error; + + err = f2fs_init_security(inode, dir, name, page); + if (err) + goto put_error; + + if (f2fs_encrypted_inode(dir) && f2fs_may_encrypt(inode)) { + err = f2fs_inherit_context(dir, inode, page); + if (err) + goto put_error; + } + } else { + page = get_node_page(F2FS_I_SB(dir), inode->i_ino); + if (IS_ERR(page)) + return page; + + set_cold_node(inode, page); + } + + if (name) + init_dent_inode(name, page); + + /* + * This file should be checkpointed during fsync. + * We lost i_pino from now on. + */ + if (is_inode_flag_set(F2FS_I(inode), FI_INC_LINK)) { + file_lost_pino(inode); + /* + * If link the tmpfile to alias through linkat path, + * we should remove this inode from orphan list. + */ + if (inode->i_nlink == 0) + remove_orphan_inode(F2FS_I_SB(dir), inode->i_ino); + inc_nlink(inode); + } + return page; + +put_error: + f2fs_put_page(page, 1); +error: + /* once the failed inode becomes a bad inode, i_mode is S_IFREG */ + truncate_inode_pages(&inode->i_data, 0); + truncate_blocks(inode, 0, false); + remove_dirty_dir_inode(inode); + remove_inode_page(inode); + return ERR_PTR(err); +} + +void update_parent_metadata(struct inode *dir, struct inode *inode, + unsigned int current_depth) +{ + if (inode && is_inode_flag_set(F2FS_I(inode), FI_NEW_INODE)) { + if (S_ISDIR(inode->i_mode)) { + inc_nlink(dir); + set_inode_flag(F2FS_I(dir), FI_UPDATE_DIR); + } + clear_inode_flag(F2FS_I(inode), FI_NEW_INODE); + } + dir->i_mtime = dir->i_ctime = CURRENT_TIME; + mark_inode_dirty(dir); + + if (F2FS_I(dir)->i_current_depth != current_depth) { + F2FS_I(dir)->i_current_depth = current_depth; + set_inode_flag(F2FS_I(dir), FI_UPDATE_DIR); + } + + if (inode && is_inode_flag_set(F2FS_I(inode), FI_INC_LINK)) + clear_inode_flag(F2FS_I(inode), FI_INC_LINK); +} + +int room_for_filename(const void *bitmap, int slots, int max_slots) +{ + int bit_start = 0; + int zero_start, zero_end; +next: + zero_start = find_next_zero_bit_le(bitmap, max_slots, bit_start); + if (zero_start >= max_slots) + return max_slots; + + zero_end = find_next_bit_le(bitmap, max_slots, zero_start); + if (zero_end - zero_start >= slots) + return zero_start; + + bit_start = zero_end + 1; + + if (zero_end + 1 >= max_slots) + return max_slots; + goto next; +} + +void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d, + const struct qstr *name, f2fs_hash_t name_hash, + unsigned int bit_pos) +{ + struct f2fs_dir_entry *de; + int slots = GET_DENTRY_SLOTS(name->len); + int i; + + de = &d->dentry[bit_pos]; + de->hash_code = name_hash; + de->name_len = cpu_to_le16(name->len); + memcpy(d->filename[bit_pos], name->name, name->len); + de->ino = cpu_to_le32(ino); + set_de_type(de, mode); + for (i = 0; i < slots; i++) + test_and_set_bit_le(bit_pos + i, (void *)d->bitmap); +} + +/* + * Caller should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op(). + */ +int __f2fs_add_link(struct inode *dir, const struct qstr *name, + struct inode *inode, nid_t ino, umode_t mode) +{ + unsigned int bit_pos; + unsigned int level; + unsigned int current_depth; + unsigned long bidx, block; + f2fs_hash_t dentry_hash; + unsigned int nbucket, nblock; + struct page *dentry_page = NULL; + struct f2fs_dentry_block *dentry_blk = NULL; + struct f2fs_dentry_ptr d; + struct page *page = NULL; + struct f2fs_filename fname; + struct qstr new_name; + int slots, err; + + err = f2fs_fname_setup_filename(dir, name, 0, &fname); + if (err) + return err; + + new_name.name = fname_name(&fname); + new_name.len = fname_len(&fname); + + if (f2fs_has_inline_dentry(dir)) { + err = f2fs_add_inline_entry(dir, &new_name, inode, ino, mode); + if (!err || err != -EAGAIN) + goto out; + else + err = 0; + } + + level = 0; + slots = GET_DENTRY_SLOTS(new_name.len); + dentry_hash = f2fs_dentry_hash(&new_name); + + current_depth = F2FS_I(dir)->i_current_depth; + if (F2FS_I(dir)->chash == dentry_hash) { + level = F2FS_I(dir)->clevel; + F2FS_I(dir)->chash = 0; + } + +start: + if (unlikely(current_depth == MAX_DIR_HASH_DEPTH)) { + err = -ENOSPC; + goto out; + } + + /* Increase the depth, if required */ + if (level == current_depth) + ++current_depth; + + nbucket = dir_buckets(level, F2FS_I(dir)->i_dir_level); + nblock = bucket_blocks(level); + + bidx = dir_block_index(level, F2FS_I(dir)->i_dir_level, + (le32_to_cpu(dentry_hash) % nbucket)); + + for (block = bidx; block <= (bidx + nblock - 1); block++) { + dentry_page = get_new_data_page(dir, NULL, block, true); + if (IS_ERR(dentry_page)) { + err = PTR_ERR(dentry_page); + goto out; + } + + dentry_blk = kmap(dentry_page); + bit_pos = room_for_filename(&dentry_blk->dentry_bitmap, + slots, NR_DENTRY_IN_BLOCK); + if (bit_pos < NR_DENTRY_IN_BLOCK) + goto add_dentry; + + kunmap(dentry_page); + f2fs_put_page(dentry_page, 1); + } + + /* Move to next level to find the empty slot for new dentry */ + ++level; + goto start; +add_dentry: + f2fs_wait_on_page_writeback(dentry_page, DATA); + + if (inode) { + down_write(&F2FS_I(inode)->i_sem); + page = init_inode_metadata(inode, dir, &new_name, NULL); + if (IS_ERR(page)) { + err = PTR_ERR(page); + goto fail; + } + if (f2fs_encrypted_inode(dir)) + file_set_enc_name(inode); + } + + make_dentry_ptr(NULL, &d, (void *)dentry_blk, 1); + f2fs_update_dentry(ino, mode, &d, &new_name, dentry_hash, bit_pos); + + set_page_dirty(dentry_page); + + if (inode) { + /* we don't need to mark_inode_dirty now */ + F2FS_I(inode)->i_pino = dir->i_ino; + update_inode(inode, page); + f2fs_put_page(page, 1); + } + + update_parent_metadata(dir, inode, current_depth); +fail: + if (inode) + up_write(&F2FS_I(inode)->i_sem); + + if (is_inode_flag_set(F2FS_I(dir), FI_UPDATE_DIR)) { + update_inode_page(dir); + clear_inode_flag(F2FS_I(dir), FI_UPDATE_DIR); + } + kunmap(dentry_page); + f2fs_put_page(dentry_page, 1); +out: + f2fs_fname_free_filename(&fname); + return err; +} + +int f2fs_do_tmpfile(struct inode *inode, struct inode *dir) +{ + struct page *page; + int err = 0; + + down_write(&F2FS_I(inode)->i_sem); + page = init_inode_metadata(inode, dir, NULL, NULL); + if (IS_ERR(page)) { + err = PTR_ERR(page); + goto fail; + } + /* we don't need to mark_inode_dirty now */ + update_inode(inode, page); + f2fs_put_page(page, 1); + + clear_inode_flag(F2FS_I(inode), FI_NEW_INODE); +fail: + up_write(&F2FS_I(inode)->i_sem); + return err; +} + +void f2fs_drop_nlink(struct inode *dir, struct inode *inode, struct page *page) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + + down_write(&F2FS_I(inode)->i_sem); + + if (S_ISDIR(inode->i_mode)) { + drop_nlink(dir); + if (page) + update_inode(dir, page); + else + update_inode_page(dir); + } + inode->i_ctime = CURRENT_TIME; + + drop_nlink(inode); + if (S_ISDIR(inode->i_mode)) { + drop_nlink(inode); + i_size_write(inode, 0); + } + up_write(&F2FS_I(inode)->i_sem); + update_inode_page(inode); + + if (inode->i_nlink == 0) + add_orphan_inode(sbi, inode->i_ino); + else + release_orphan_inode(sbi); +} + +/* + * It only removes the dentry from the dentry page, corresponding name + * entry in name page does not need to be touched during deletion. + */ +void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, + struct inode *dir, struct inode *inode) +{ + struct f2fs_dentry_block *dentry_blk; + unsigned int bit_pos; + int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len)); + int i; + + if (f2fs_has_inline_dentry(dir)) + return f2fs_delete_inline_entry(dentry, page, dir, inode); + + lock_page(page); + f2fs_wait_on_page_writeback(page, DATA); + + dentry_blk = page_address(page); + bit_pos = dentry - dentry_blk->dentry; + for (i = 0; i < slots; i++) + test_and_clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap); + + /* Let's check and deallocate this dentry page */ + bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, + NR_DENTRY_IN_BLOCK, + 0); + kunmap(page); /* kunmap - pair of f2fs_find_entry */ + set_page_dirty(page); + + dir->i_ctime = dir->i_mtime = CURRENT_TIME; + + if (inode) + f2fs_drop_nlink(dir, inode, NULL); + + if (bit_pos == NR_DENTRY_IN_BLOCK && + !truncate_hole(dir, page->index, page->index + 1)) { + clear_page_dirty_for_io(page); + ClearPagePrivate(page); + ClearPageUptodate(page); + inode_dec_dirty_pages(dir); + } + f2fs_put_page(page, 1); +} + +bool f2fs_empty_dir(struct inode *dir) +{ + unsigned long bidx; + struct page *dentry_page; + unsigned int bit_pos; + struct f2fs_dentry_block *dentry_blk; + unsigned long nblock = dir_blocks(dir); + + if (f2fs_has_inline_dentry(dir)) + return f2fs_empty_inline_dir(dir); + + for (bidx = 0; bidx < nblock; bidx++) { + dentry_page = get_lock_data_page(dir, bidx, false); + if (IS_ERR(dentry_page)) { + if (PTR_ERR(dentry_page) == -ENOENT) + continue; + else + return false; + } + + dentry_blk = kmap_atomic(dentry_page); + if (bidx == 0) + bit_pos = 2; + else + bit_pos = 0; + bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, + NR_DENTRY_IN_BLOCK, + bit_pos); + kunmap_atomic(dentry_blk); + + f2fs_put_page(dentry_page, 1); + + if (bit_pos < NR_DENTRY_IN_BLOCK) + return false; + } + return true; +} + +bool f2fs_fill_dentries(struct file *file, void *dirent, filldir_t filldir, + struct f2fs_dentry_ptr *d, unsigned int n, unsigned int bit_pos, + struct f2fs_str *fstr) +{ + unsigned int start_bit_pos = bit_pos; + unsigned char d_type; + struct f2fs_dir_entry *de = NULL; + struct f2fs_str de_name = FSTR_INIT(NULL, 0); + unsigned char *types = f2fs_filetype_table; + int over; + + while (bit_pos < d->max) { + d_type = DT_UNKNOWN; + bit_pos = find_next_bit_le(d->bitmap, d->max, bit_pos); + if (bit_pos >= d->max) + break; + + de = &d->dentry[bit_pos]; + + if (types && de->file_type < F2FS_FT_MAX) + d_type = types[de->file_type]; + + de_name.name = d->filename[bit_pos]; + de_name.len = le16_to_cpu(de->name_len); + + if (f2fs_encrypted_inode(d->inode)) { + int save_len = fstr->len; + int ret; + + de_name.name = kmalloc(de_name.len, GFP_NOFS); + if (!de_name.name) + return false; + + memcpy(de_name.name, d->filename[bit_pos], de_name.len); + + ret = f2fs_fname_disk_to_usr(d->inode, &de->hash_code, + &de_name, fstr); + kfree(de_name.name); + if (ret < 0) + return true; + + de_name = *fstr; + fstr->len = save_len; + } + + over = filldir(dirent, de_name.name, de_name.len, + (n * d->max) + bit_pos, + le32_to_cpu(de->ino), d_type); + if (over) { + file->f_pos += bit_pos - start_bit_pos; + return true; + } + + bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); + } + return false; +} + +static int f2fs_readdir(struct file *file, void *dirent, filldir_t filldir) +{ + unsigned long pos = file->f_pos; + unsigned int bit_pos = 0; + struct inode *inode = file_inode(file); + unsigned long npages = dir_blocks(inode); + struct f2fs_dentry_block *dentry_blk = NULL; + struct page *dentry_page = NULL; + struct file_ra_state *ra = &file->f_ra; + struct f2fs_dentry_ptr d; + struct f2fs_str fstr = FSTR_INIT(NULL, 0); + unsigned int n = 0; + int err = 0; + + if (f2fs_encrypted_inode(inode)) { + err = f2fs_get_encryption_info(inode); + if (err) + return err; + + err = f2fs_fname_crypto_alloc_buffer(inode, F2FS_NAME_LEN, + &fstr); + if (err < 0) + return err; + } + + if (f2fs_has_inline_dentry(inode)) { + err = f2fs_read_inline_dir(file, dirent, filldir, &fstr); + goto out; + } + + bit_pos = (pos % NR_DENTRY_IN_BLOCK); + n = (pos / NR_DENTRY_IN_BLOCK); + + /* readahead for multi pages of dir */ + if (npages - n > 1 && !ra_has_index(ra, n)) + page_cache_sync_readahead(inode->i_mapping, ra, file, n, + min(npages - n, (pgoff_t)MAX_DIR_RA_PAGES)); + + for (; n < npages; n++) { + dentry_page = get_lock_data_page(inode, n, false); + if (IS_ERR(dentry_page)) + continue; + + dentry_blk = kmap(dentry_page); + + make_dentry_ptr(inode, &d, (void *)dentry_blk, 1); + + if (f2fs_fill_dentries(file, dirent, filldir, &d, n, bit_pos, &fstr)) + goto stop; + + bit_pos = 0; + file->f_pos = (n + 1) * NR_DENTRY_IN_BLOCK; + kunmap(dentry_page); + f2fs_put_page(dentry_page, 1); + dentry_page = NULL; + } +stop: + if (dentry_page && !IS_ERR(dentry_page)) { + kunmap(dentry_page); + f2fs_put_page(dentry_page, 1); + } +out: + f2fs_fname_crypto_free_buffer(&fstr); + return err; +} + +const struct file_operations f2fs_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .readdir = f2fs_readdir, + .fsync = f2fs_sync_file, + .unlocked_ioctl = f2fs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = f2fs_compat_ioctl, +#endif +}; diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c new file mode 100644 index 00000000000..7ddba812e11 --- /dev/null +++ b/fs/f2fs/extent_cache.c @@ -0,0 +1,748 @@ +/* + * f2fs extent cache support + * + * Copyright (c) 2015 Motorola Mobility + * Copyright (c) 2015 Samsung Electronics + * Authors: Jaegeuk Kim + * Chao Yu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +#include "f2fs.h" +#include "node.h" +#include + +static struct kmem_cache *extent_tree_slab; +static struct kmem_cache *extent_node_slab; + +static struct extent_node *__attach_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei, + struct rb_node *parent, struct rb_node **p) +{ + struct extent_node *en; + + en = kmem_cache_alloc(extent_node_slab, GFP_ATOMIC); + if (!en) + return NULL; + + en->ei = *ei; + INIT_LIST_HEAD(&en->list); + + rb_link_node(&en->rb_node, parent, p); + rb_insert_color(&en->rb_node, &et->root); + et->count++; + atomic_inc(&sbi->total_ext_node); + return en; +} + +static void __detach_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_node *en) +{ + rb_erase(&en->rb_node, &et->root); + et->count--; + atomic_dec(&sbi->total_ext_node); + + if (et->cached_en == en) + et->cached_en = NULL; +} + +static struct extent_tree *__grab_extent_tree(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et; + nid_t ino = inode->i_ino; + + down_write(&sbi->extent_tree_lock); + et = radix_tree_lookup(&sbi->extent_tree_root, ino); + if (!et) { + et = f2fs_kmem_cache_alloc(extent_tree_slab, GFP_NOFS); + f2fs_radix_tree_insert(&sbi->extent_tree_root, ino, et); + memset(et, 0, sizeof(struct extent_tree)); + et->ino = ino; + et->root = RB_ROOT; + et->cached_en = NULL; + rwlock_init(&et->lock); + atomic_set(&et->refcount, 0); + et->count = 0; + sbi->total_ext_tree++; + } + atomic_inc(&et->refcount); + up_write(&sbi->extent_tree_lock); + + /* never died until evict_inode */ + F2FS_I(inode)->extent_tree = et; + + return et; +} + +static struct extent_node *__lookup_extent_tree(struct f2fs_sb_info *sbi, + struct extent_tree *et, unsigned int fofs) +{ + struct rb_node *node = et->root.rb_node; + struct extent_node *en = et->cached_en; + + if (en) { + struct extent_info *cei = &en->ei; + + if (cei->fofs <= fofs && cei->fofs + cei->len > fofs) { + stat_inc_cached_node_hit(sbi); + return en; + } + } + + while (node) { + en = rb_entry(node, struct extent_node, rb_node); + + if (fofs < en->ei.fofs) { + node = node->rb_left; + } else if (fofs >= en->ei.fofs + en->ei.len) { + node = node->rb_right; + } else { + stat_inc_rbtree_node_hit(sbi); + return en; + } + } + return NULL; +} + +static struct extent_node *__init_extent_tree(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei) +{ + struct rb_node **p = &et->root.rb_node; + struct extent_node *en; + + en = __attach_extent_node(sbi, et, ei, NULL, p); + if (!en) + return NULL; + + et->largest = en->ei; + et->cached_en = en; + return en; +} + +static unsigned int __free_extent_tree(struct f2fs_sb_info *sbi, + struct extent_tree *et, bool free_all) +{ + struct rb_node *node, *next; + struct extent_node *en; + unsigned int count = et->count; + + node = rb_first(&et->root); + while (node) { + next = rb_next(node); + en = rb_entry(node, struct extent_node, rb_node); + + if (free_all) { + spin_lock(&sbi->extent_lock); + if (!list_empty(&en->list)) + list_del_init(&en->list); + spin_unlock(&sbi->extent_lock); + } + + if (free_all || list_empty(&en->list)) { + __detach_extent_node(sbi, et, en); + kmem_cache_free(extent_node_slab, en); + } + node = next; + } + + return count - et->count; +} + +static void __drop_largest_extent(struct inode *inode, + pgoff_t fofs, unsigned int len) +{ + struct extent_info *largest = &F2FS_I(inode)->extent_tree->largest; + + if (fofs < largest->fofs + largest->len && fofs + len > largest->fofs) + largest->len = 0; +} + +void f2fs_drop_largest_extent(struct inode *inode, pgoff_t fofs) +{ + if (!f2fs_may_extent_tree(inode)) + return; + + __drop_largest_extent(inode, fofs, 1); +} + +void f2fs_init_extent_tree(struct inode *inode, struct f2fs_extent *i_ext) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et; + struct extent_node *en; + struct extent_info ei; + + if (!f2fs_may_extent_tree(inode)) + return; + + et = __grab_extent_tree(inode); + + if (!i_ext || le32_to_cpu(i_ext->len) < F2FS_MIN_EXTENT_LEN) + return; + + set_extent_info(&ei, le32_to_cpu(i_ext->fofs), + le32_to_cpu(i_ext->blk), le32_to_cpu(i_ext->len)); + + write_lock(&et->lock); + if (et->count) + goto out; + + en = __init_extent_tree(sbi, et, &ei); + if (en) { + spin_lock(&sbi->extent_lock); + list_add_tail(&en->list, &sbi->extent_list); + spin_unlock(&sbi->extent_lock); + } +out: + write_unlock(&et->lock); +} + +static bool f2fs_lookup_extent_tree(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_node *en; + bool ret = false; + + f2fs_bug_on(sbi, !et); + + trace_f2fs_lookup_extent_tree_start(inode, pgofs); + + read_lock(&et->lock); + + if (et->largest.fofs <= pgofs && + et->largest.fofs + et->largest.len > pgofs) { + *ei = et->largest; + ret = true; + stat_inc_largest_node_hit(sbi); + goto out; + } + + en = __lookup_extent_tree(sbi, et, pgofs); + if (en) { + *ei = en->ei; + spin_lock(&sbi->extent_lock); + if (!list_empty(&en->list)) + list_move_tail(&en->list, &sbi->extent_list); + et->cached_en = en; + spin_unlock(&sbi->extent_lock); + ret = true; + } +out: + stat_inc_total_hit(sbi); + read_unlock(&et->lock); + + trace_f2fs_lookup_extent_tree_end(inode, pgofs, ei); + return ret; +} + + +/* + * lookup extent at @fofs, if hit, return the extent + * if not, return NULL and + * @prev_ex: extent before fofs + * @next_ex: extent after fofs + * @insert_p: insert point for new extent at fofs + * in order to simpfy the insertion after. + * tree must stay unchanged between lookup and insertion. + */ +static struct extent_node *__lookup_extent_tree_ret(struct extent_tree *et, + unsigned int fofs, + struct extent_node **prev_ex, + struct extent_node **next_ex, + struct rb_node ***insert_p, + struct rb_node **insert_parent) +{ + struct rb_node **pnode = &et->root.rb_node; + struct rb_node *parent = NULL, *tmp_node; + struct extent_node *en = et->cached_en; + + *insert_p = NULL; + *insert_parent = NULL; + *prev_ex = NULL; + *next_ex = NULL; + + if (RB_EMPTY_ROOT(&et->root)) + return NULL; + + if (en) { + struct extent_info *cei = &en->ei; + + if (cei->fofs <= fofs && cei->fofs + cei->len > fofs) + goto lookup_neighbors; + } + + while (*pnode) { + parent = *pnode; + en = rb_entry(*pnode, struct extent_node, rb_node); + + if (fofs < en->ei.fofs) + pnode = &(*pnode)->rb_left; + else if (fofs >= en->ei.fofs + en->ei.len) + pnode = &(*pnode)->rb_right; + else + goto lookup_neighbors; + } + + *insert_p = pnode; + *insert_parent = parent; + + en = rb_entry(parent, struct extent_node, rb_node); + tmp_node = parent; + if (parent && fofs > en->ei.fofs) + tmp_node = rb_next(parent); + *next_ex = tmp_node ? + rb_entry(tmp_node, struct extent_node, rb_node) : NULL; + + tmp_node = parent; + if (parent && fofs < en->ei.fofs) + tmp_node = rb_prev(parent); + *prev_ex = tmp_node ? + rb_entry(tmp_node, struct extent_node, rb_node) : NULL; + return NULL; + +lookup_neighbors: + if (fofs == en->ei.fofs) { + /* lookup prev node for merging backward later */ + tmp_node = rb_prev(&en->rb_node); + *prev_ex = tmp_node ? + rb_entry(tmp_node, struct extent_node, rb_node) : NULL; + } + if (fofs == en->ei.fofs + en->ei.len - 1) { + /* lookup next node for merging frontward later */ + tmp_node = rb_next(&en->rb_node); + *next_ex = tmp_node ? + rb_entry(tmp_node, struct extent_node, rb_node) : NULL; + } + return en; +} + +static struct extent_node *__try_merge_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei, + struct extent_node **den, + struct extent_node *prev_ex, + struct extent_node *next_ex) +{ + struct extent_node *en = NULL; + + if (prev_ex && __is_back_mergeable(ei, &prev_ex->ei)) { + prev_ex->ei.len += ei->len; + ei = &prev_ex->ei; + en = prev_ex; + } + + if (next_ex && __is_front_mergeable(ei, &next_ex->ei)) { + if (en) { + __detach_extent_node(sbi, et, prev_ex); + *den = prev_ex; + } + next_ex->ei.fofs = ei->fofs; + next_ex->ei.blk = ei->blk; + next_ex->ei.len += ei->len; + en = next_ex; + } + + if (en) { + __try_update_largest_extent(et, en); + et->cached_en = en; + } + return en; +} + +static struct extent_node *__insert_extent_tree(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei, + struct rb_node **insert_p, + struct rb_node *insert_parent) +{ + struct rb_node **p = &et->root.rb_node; + struct rb_node *parent = NULL; + struct extent_node *en = NULL; + + if (insert_p && insert_parent) { + parent = insert_parent; + p = insert_p; + goto do_insert; + } + + while (*p) { + parent = *p; + en = rb_entry(parent, struct extent_node, rb_node); + + if (ei->fofs < en->ei.fofs) + p = &(*p)->rb_left; + else if (ei->fofs >= en->ei.fofs + en->ei.len) + p = &(*p)->rb_right; + else + f2fs_bug_on(sbi, 1); + } +do_insert: + en = __attach_extent_node(sbi, et, ei, parent, p); + if (!en) + return NULL; + + __try_update_largest_extent(et, en); + et->cached_en = en; + return en; +} + +static unsigned int f2fs_update_extent_tree_range(struct inode *inode, + pgoff_t fofs, block_t blkaddr, unsigned int len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree; + struct extent_node *en = NULL, *en1 = NULL; + struct extent_node *prev_en = NULL, *next_en = NULL; + struct extent_info ei, dei, prev; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + unsigned int end = fofs + len; + unsigned int pos = (unsigned int)fofs; + + if (!et) + return false; + + trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, len); + + write_lock(&et->lock); + + if (is_inode_flag_set(F2FS_I(inode), FI_NO_EXTENT)) { + write_unlock(&et->lock); + return false; + } + + prev = et->largest; + dei.len = 0; + + /* + * drop largest extent before lookup, in case it's already + * been shrunk from extent tree + */ + __drop_largest_extent(inode, fofs, len); + + /* 1. lookup first extent node in range [fofs, fofs + len - 1] */ + en = __lookup_extent_tree_ret(et, fofs, &prev_en, &next_en, + &insert_p, &insert_parent); + if (!en) + en = next_en; + + /* 2. invlidate all extent nodes in range [fofs, fofs + len - 1] */ + while (en && en->ei.fofs < end) { + unsigned int org_end; + int parts = 0; /* # of parts current extent split into */ + + next_en = en1 = NULL; + + dei = en->ei; + org_end = dei.fofs + dei.len; + f2fs_bug_on(sbi, pos >= org_end); + + if (pos > dei.fofs && pos - dei.fofs >= F2FS_MIN_EXTENT_LEN) { + en->ei.len = pos - en->ei.fofs; + prev_en = en; + parts = 1; + } + + if (end < org_end && org_end - end >= F2FS_MIN_EXTENT_LEN) { + if (parts) { + set_extent_info(&ei, end, + end - dei.fofs + dei.blk, + org_end - end); + en1 = __insert_extent_tree(sbi, et, &ei, + NULL, NULL); + next_en = en1; + } else { + en->ei.fofs = end; + en->ei.blk += end - dei.fofs; + en->ei.len -= end - dei.fofs; + next_en = en; + } + parts++; + } + + if (!next_en) { + struct rb_node *node = rb_next(&en->rb_node); + + next_en = node ? + rb_entry(node, struct extent_node, rb_node) + : NULL; + } + + if (parts) + __try_update_largest_extent(et, en); + else + __detach_extent_node(sbi, et, en); + + /* + * if original extent is split into zero or two parts, extent + * tree has been altered by deletion or insertion, therefore + * invalidate pointers regard to tree. + */ + if (parts != 1) { + insert_p = NULL; + insert_parent = NULL; + } + + /* update in global extent list */ + spin_lock(&sbi->extent_lock); + if (!parts && !list_empty(&en->list)) + list_del(&en->list); + if (en1) + list_add_tail(&en1->list, &sbi->extent_list); + spin_unlock(&sbi->extent_lock); + + /* release extent node */ + if (!parts) + kmem_cache_free(extent_node_slab, en); + + en = next_en; + } + + /* 3. update extent in extent cache */ + if (blkaddr) { + struct extent_node *den = NULL; + + set_extent_info(&ei, fofs, blkaddr, len); + en1 = __try_merge_extent_node(sbi, et, &ei, &den, + prev_en, next_en); + if (!en1) + en1 = __insert_extent_tree(sbi, et, &ei, + insert_p, insert_parent); + + /* give up extent_cache, if split and small updates happen */ + if (dei.len >= 1 && + prev.len < F2FS_MIN_EXTENT_LEN && + et->largest.len < F2FS_MIN_EXTENT_LEN) { + et->largest.len = 0; + set_inode_flag(F2FS_I(inode), FI_NO_EXTENT); + } + + spin_lock(&sbi->extent_lock); + if (en1) { + if (list_empty(&en1->list)) + list_add_tail(&en1->list, &sbi->extent_list); + else + list_move_tail(&en1->list, &sbi->extent_list); + } + if (den && !list_empty(&den->list)) + list_del(&den->list); + spin_unlock(&sbi->extent_lock); + + if (den) + kmem_cache_free(extent_node_slab, den); + } + + if (is_inode_flag_set(F2FS_I(inode), FI_NO_EXTENT)) + __free_extent_tree(sbi, et, true); + + write_unlock(&et->lock); + + return !__is_extent_same(&prev, &et->largest); +} + +unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) +{ + struct extent_tree *treevec[EXT_TREE_VEC_SIZE]; + struct extent_node *en, *tmp; + unsigned long ino = F2FS_ROOT_INO(sbi); + struct radix_tree_root *root = &sbi->extent_tree_root; + unsigned int found; + unsigned int node_cnt = 0, tree_cnt = 0; + int remained; + + if (!test_opt(sbi, EXTENT_CACHE)) + return 0; + + if (!down_write_trylock(&sbi->extent_tree_lock)) + goto out; + + /* 1. remove unreferenced extent tree */ + while ((found = radix_tree_gang_lookup(root, + (void **)treevec, ino, EXT_TREE_VEC_SIZE))) { + unsigned i; + + ino = treevec[found - 1]->ino + 1; + for (i = 0; i < found; i++) { + struct extent_tree *et = treevec[i]; + + if (!atomic_read(&et->refcount)) { + write_lock(&et->lock); + node_cnt += __free_extent_tree(sbi, et, true); + write_unlock(&et->lock); + + radix_tree_delete(root, et->ino); + kmem_cache_free(extent_tree_slab, et); + sbi->total_ext_tree--; + tree_cnt++; + + if (node_cnt + tree_cnt >= nr_shrink) + goto unlock_out; + } + } + } + up_write(&sbi->extent_tree_lock); + + /* 2. remove LRU extent entries */ + if (!down_write_trylock(&sbi->extent_tree_lock)) + goto out; + + remained = nr_shrink - (node_cnt + tree_cnt); + + spin_lock(&sbi->extent_lock); + list_for_each_entry_safe(en, tmp, &sbi->extent_list, list) { + if (!remained--) + break; + list_del_init(&en->list); + } + spin_unlock(&sbi->extent_lock); + + /* + * reset ino for searching victims from beginning of global extent tree. + */ + ino = F2FS_ROOT_INO(sbi); + + while ((found = radix_tree_gang_lookup(root, + (void **)treevec, ino, EXT_TREE_VEC_SIZE))) { + unsigned i; + + ino = treevec[found - 1]->ino + 1; + for (i = 0; i < found; i++) { + struct extent_tree *et = treevec[i]; + + write_lock(&et->lock); + node_cnt += __free_extent_tree(sbi, et, false); + write_unlock(&et->lock); + + if (node_cnt + tree_cnt >= nr_shrink) + goto unlock_out; + } + } +unlock_out: + up_write(&sbi->extent_tree_lock); +out: + trace_f2fs_shrink_extent_tree(sbi, node_cnt, tree_cnt); + + return node_cnt + tree_cnt; +} + +unsigned int f2fs_destroy_extent_node(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree; + unsigned int node_cnt = 0; + + if (!et) + return 0; + + write_lock(&et->lock); + node_cnt = __free_extent_tree(sbi, et, true); + write_unlock(&et->lock); + + return node_cnt; +} + +void f2fs_destroy_extent_tree(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree; + unsigned int node_cnt = 0; + + if (!et) + return; + + if (inode->i_nlink && !is_bad_inode(inode) && et->count) { + atomic_dec(&et->refcount); + return; + } + + /* free all extent info belong to this extent tree */ + node_cnt = f2fs_destroy_extent_node(inode); + + /* delete extent tree entry in radix tree */ + down_write(&sbi->extent_tree_lock); + atomic_dec(&et->refcount); + f2fs_bug_on(sbi, atomic_read(&et->refcount) || et->count); + radix_tree_delete(&sbi->extent_tree_root, inode->i_ino); + kmem_cache_free(extent_tree_slab, et); + sbi->total_ext_tree--; + up_write(&sbi->extent_tree_lock); + + F2FS_I(inode)->extent_tree = NULL; + + trace_f2fs_destroy_extent_tree(inode, node_cnt); +} + +bool f2fs_lookup_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei) +{ + if (!f2fs_may_extent_tree(inode)) + return false; + + return f2fs_lookup_extent_tree(inode, pgofs, ei); +} + +void f2fs_update_extent_cache(struct dnode_of_data *dn) +{ + struct f2fs_inode_info *fi = F2FS_I(dn->inode); + pgoff_t fofs; + + if (!f2fs_may_extent_tree(dn->inode)) + return; + + f2fs_bug_on(F2FS_I_SB(dn->inode), dn->data_blkaddr == NEW_ADDR); + + + fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + + dn->ofs_in_node; + + if (f2fs_update_extent_tree_range(dn->inode, fofs, dn->data_blkaddr, 1)) + sync_inode_page(dn); +} + +void f2fs_update_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, block_t blkaddr, unsigned int len) + +{ + if (!f2fs_may_extent_tree(dn->inode)) + return; + + if (f2fs_update_extent_tree_range(dn->inode, fofs, blkaddr, len)) + sync_inode_page(dn); +} + +void init_extent_cache_info(struct f2fs_sb_info *sbi) +{ + INIT_RADIX_TREE(&sbi->extent_tree_root, GFP_NOIO); + init_rwsem(&sbi->extent_tree_lock); + INIT_LIST_HEAD(&sbi->extent_list); + spin_lock_init(&sbi->extent_lock); + sbi->total_ext_tree = 0; + atomic_set(&sbi->total_ext_node, 0); +} + +int __init create_extent_cache(void) +{ + extent_tree_slab = f2fs_kmem_cache_create("f2fs_extent_tree", + sizeof(struct extent_tree)); + if (!extent_tree_slab) + return -ENOMEM; + extent_node_slab = f2fs_kmem_cache_create("f2fs_extent_node", + sizeof(struct extent_node)); + if (!extent_node_slab) { + kmem_cache_destroy(extent_tree_slab); + return -ENOMEM; + } + return 0; +} + +void destroy_extent_cache(void) +{ + kmem_cache_destroy(extent_node_slab); + kmem_cache_destroy(extent_tree_slab); +} diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h new file mode 100644 index 00000000000..182154c6103 --- /dev/null +++ b/fs/f2fs/f2fs.h @@ -0,0 +1,2227 @@ +/* + * fs/f2fs/f2fs.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _LINUX_F2FS_H +#define _LINUX_F2FS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_F2FS_CHECK_FS +#define f2fs_bug_on(sbi, condition) BUG_ON(condition) +#define f2fs_down_write(x, y) down_write(x) +#else +#define f2fs_bug_on(sbi, condition) \ + do { \ + if (unlikely(condition)) { \ + WARN_ON(1); \ + set_sbi_flag(sbi, SBI_NEED_FSCK); \ + } \ + } while (0) +#define f2fs_down_write(x, y) down_write(x) +#endif + +/* + * For mount options + */ +#define F2FS_SUPER_MAGIC 0xF2F52010 /* F2FS Magic Number */ +#define F2FS_MOUNT_BG_GC 0x00000001 +#define F2FS_MOUNT_DISABLE_ROLL_FORWARD 0x00000002 +#define F2FS_MOUNT_DISCARD 0x00000004 +#define F2FS_MOUNT_NOHEAP 0x00000008 +#define F2FS_MOUNT_XATTR_USER 0x00000010 +#define F2FS_MOUNT_POSIX_ACL 0x00000020 +#define F2FS_MOUNT_DISABLE_EXT_IDENTIFY 0x00000040 +#define F2FS_MOUNT_INLINE_XATTR 0x00000080 +#define F2FS_MOUNT_INLINE_DATA 0x00000100 +#define F2FS_MOUNT_INLINE_DENTRY 0x00000200 +#define F2FS_MOUNT_FLUSH_MERGE 0x00000400 +#define F2FS_MOUNT_NOBARRIER 0x00000800 +#define F2FS_MOUNT_FASTBOOT 0x00001000 +#define F2FS_MOUNT_EXTENT_CACHE 0x00002000 +#define F2FS_MOUNT_FORCE_FG_GC 0x00004000 + +#define clear_opt(sbi, option) (sbi->mount_opt.opt &= ~F2FS_MOUNT_##option) +#define set_opt(sbi, option) (sbi->mount_opt.opt |= F2FS_MOUNT_##option) +#define test_opt(sbi, option) (sbi->mount_opt.opt & F2FS_MOUNT_##option) + +#define ver_after(a, b) (typecheck(unsigned long long, a) && \ + typecheck(unsigned long long, b) && \ + ((long long)((a) - (b)) > 0)) + +typedef u32 block_t; /* + * should not change u32, since it is the on-disk block + * address format, __le32. + */ +typedef u32 nid_t; + +struct f2fs_mount_info { + unsigned int opt; +}; + +#define F2FS_FEATURE_ENCRYPT 0x0001 + +#define F2FS_HAS_FEATURE(sb, mask) \ + ((F2FS_SB(sb)->raw_super->feature & cpu_to_le32(mask)) != 0) +#define F2FS_SET_FEATURE(sb, mask) \ + F2FS_SB(sb)->raw_super->feature |= cpu_to_le32(mask) +#define F2FS_CLEAR_FEATURE(sb, mask) \ + F2FS_SB(sb)->raw_super->feature &= ~cpu_to_le32(mask) + +#define CRCPOLY_LE 0xedb88320 + +static inline __u32 f2fs_crc32(void *buf, size_t len) +{ + unsigned char *p = (unsigned char *)buf; + __u32 crc = F2FS_SUPER_MAGIC; + int i; + + while (len--) { + crc ^= *p++; + for (i = 0; i < 8; i++) + crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_LE : 0); + } + return crc; +} + +static inline bool f2fs_crc_valid(__u32 blk_crc, void *buf, size_t buf_size) +{ + return f2fs_crc32(buf, buf_size) == blk_crc; +} + +/* + * For checkpoint manager + */ +enum { + NAT_BITMAP, + SIT_BITMAP +}; + +enum { + CP_UMOUNT, + CP_FASTBOOT, + CP_SYNC, + CP_RECOVERY, + CP_DISCARD, +}; + +#define DEF_BATCHED_TRIM_SECTIONS 32 +#define BATCHED_TRIM_SEGMENTS(sbi) \ + (SM_I(sbi)->trim_sections * (sbi)->segs_per_sec) +#define BATCHED_TRIM_BLOCKS(sbi) \ + (BATCHED_TRIM_SEGMENTS(sbi) << (sbi)->log_blocks_per_seg) +#define DEF_CP_INTERVAL 60 /* 60 secs */ + +struct cp_control { + int reason; + __u64 trim_start; + __u64 trim_end; + __u64 trim_minlen; + __u64 trimmed; +}; + +/* + * For CP/NAT/SIT/SSA readahead + */ +enum { + META_CP, + META_NAT, + META_SIT, + META_SSA, + META_POR, +}; + +/* for the list of ino */ +enum { + ORPHAN_INO, /* for orphan ino list */ + APPEND_INO, /* for append ino list */ + UPDATE_INO, /* for update ino list */ + MAX_INO_ENTRY, /* max. list */ +}; + +struct ino_entry { + struct list_head list; /* list head */ + nid_t ino; /* inode number */ +}; + +/* + * for the list of directory inodes or gc inodes. + * NOTE: there are two slab users for this structure, if we add/modify/delete + * fields in structure for one of slab users, it may affect fields or size of + * other one, in this condition, it's better to split both of slab and related + * data structure. + */ +struct inode_entry { + struct list_head list; /* list head */ + struct inode *inode; /* vfs inode pointer */ +}; + +/* for the list of blockaddresses to be discarded */ +struct discard_entry { + struct list_head list; /* list head */ + block_t blkaddr; /* block address to be discarded */ + int len; /* # of consecutive blocks of the discard */ +}; + +/* for the list of fsync inodes, used only during recovery */ +struct fsync_inode_entry { + struct list_head list; /* list head */ + struct inode *inode; /* vfs inode pointer */ + block_t blkaddr; /* block address locating the last fsync */ + block_t last_dentry; /* block address locating the last dentry */ + block_t last_inode; /* block address locating the last inode */ +}; + +#define nats_in_cursum(sum) (le16_to_cpu(sum->n_nats)) +#define sits_in_cursum(sum) (le16_to_cpu(sum->n_sits)) + +#define nat_in_journal(sum, i) (sum->nat_j.entries[i].ne) +#define nid_in_journal(sum, i) (sum->nat_j.entries[i].nid) +#define sit_in_journal(sum, i) (sum->sit_j.entries[i].se) +#define segno_in_journal(sum, i) (sum->sit_j.entries[i].segno) + +#define MAX_NAT_JENTRIES(sum) (NAT_JOURNAL_ENTRIES - nats_in_cursum(sum)) +#define MAX_SIT_JENTRIES(sum) (SIT_JOURNAL_ENTRIES - sits_in_cursum(sum)) + +static inline int update_nats_in_cursum(struct f2fs_summary_block *rs, int i) +{ + int before = nats_in_cursum(rs); + rs->n_nats = cpu_to_le16(before + i); + return before; +} + +static inline int update_sits_in_cursum(struct f2fs_summary_block *rs, int i) +{ + int before = sits_in_cursum(rs); + rs->n_sits = cpu_to_le16(before + i); + return before; +} + +static inline bool __has_cursum_space(struct f2fs_summary_block *sum, int size, + int type) +{ + if (type == NAT_JOURNAL) + return size <= MAX_NAT_JENTRIES(sum); + return size <= MAX_SIT_JENTRIES(sum); +} + +/* + * ioctl commands + */ +#define F2FS_IOC_GETFLAGS FS_IOC_GETFLAGS +#define F2FS_IOC_SETFLAGS FS_IOC_SETFLAGS +#define F2FS_IOC_GETVERSION FS_IOC_GETVERSION +#define FS_IOC_SHUTDOWN _IOR('X', 125, __u32) /* Shutdown */ + +/* + * Flags for going down operation used by FS_IOC_GOINGDOWN + */ +#define FS_GOING_DOWN_FULLSYNC 0x0 /* going down with full sync */ +#define FS_GOING_DOWN_METASYNC 0x1 /* going down with metadata */ +#define FS_GOING_DOWN_NOSYNC 0x2 /* going down */ +#define FS_GOING_DOWN_METAFLUSH 0x3 /* going down with meta flush */ + +#define F2FS_IOCTL_MAGIC 0xf5 +#define F2FS_IOC_START_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 1) +#define F2FS_IOC_COMMIT_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 2) +#define F2FS_IOC_START_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 3) +#define F2FS_IOC_RELEASE_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 4) +#define F2FS_IOC_ABORT_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 5) +#define F2FS_IOC_GARBAGE_COLLECT _IO(F2FS_IOCTL_MAGIC, 6) +#define F2FS_IOC_WRITE_CHECKPOINT _IO(F2FS_IOCTL_MAGIC, 7) + +#define F2FS_IOC_SET_ENCRYPTION_POLICY \ + _IOR('f', 19, struct f2fs_encryption_policy) +#define F2FS_IOC_GET_ENCRYPTION_PWSALT \ + _IOW('f', 20, __u8[16]) +#define F2FS_IOC_GET_ENCRYPTION_POLICY \ + _IOW('f', 21, struct f2fs_encryption_policy) + +#if defined(__KERNEL__) && defined(CONFIG_COMPAT) +/* + * ioctl commands in 32 bit emulation + */ +#define F2FS_IOC32_GETFLAGS FS_IOC32_GETFLAGS +#define F2FS_IOC32_SETFLAGS FS_IOC32_SETFLAGS +#endif + +/* + * For INODE and NODE manager + */ +/* for directory operations */ +struct f2fs_str { + unsigned char *name; + u32 len; +}; + +struct f2fs_filename { + const struct qstr *usr_fname; + struct f2fs_str disk_name; + f2fs_hash_t hash; +#ifdef CONFIG_F2FS_FS_ENCRYPTION + struct f2fs_str crypto_buf; +#endif +}; + +#define QSTR_INIT(n, l) { .name = n, .len = l } +#define FSTR_INIT(n, l) { .name = n, .len = l } +#define FSTR_TO_QSTR(f) QSTR_INIT((f)->name, (f)->len) +#define fname_name(p) ((p)->disk_name.name) +#define fname_len(p) ((p)->disk_name.len) + +struct f2fs_dentry_ptr { + struct inode *inode; + const void *bitmap; + struct f2fs_dir_entry *dentry; + __u8 (*filename)[F2FS_SLOT_LEN]; + int max; +}; + +static inline void make_dentry_ptr(struct inode *inode, + struct f2fs_dentry_ptr *d, void *src, int type) +{ + d->inode = inode; + + if (type == 1) { + struct f2fs_dentry_block *t = (struct f2fs_dentry_block *)src; + d->max = NR_DENTRY_IN_BLOCK; + d->bitmap = &t->dentry_bitmap; + d->dentry = t->dentry; + d->filename = t->filename; + } else { + struct f2fs_inline_dentry *t = (struct f2fs_inline_dentry *)src; + d->max = NR_INLINE_DENTRY; + d->bitmap = &t->dentry_bitmap; + d->dentry = t->dentry; + d->filename = t->filename; + } +} + +/* + * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1 + * as its node offset to distinguish from index node blocks. + * But some bits are used to mark the node block. + */ +#define XATTR_NODE_OFFSET ((((unsigned int)-1) << OFFSET_BIT_SHIFT) \ + >> OFFSET_BIT_SHIFT) +enum { + ALLOC_NODE, /* allocate a new node page if needed */ + LOOKUP_NODE, /* look up a node without readahead */ + LOOKUP_NODE_RA, /* + * look up a node with readahead called + * by get_data_block. + */ +}; + +#define F2FS_LINK_MAX 0xffffffff /* maximum link count per file */ + +#define MAX_DIR_RA_PAGES 4 /* maximum ra pages of dir */ + +/* vector size for gang look-up from extent cache that consists of radix tree */ +#define EXT_TREE_VEC_SIZE 64 + +/* for in-memory extent cache entry */ +#define F2FS_MIN_EXTENT_LEN 64 /* minimum extent length */ + +/* number of extent info in extent cache we try to shrink */ +#define EXTENT_CACHE_SHRINK_NUMBER 128 + +struct extent_info { + unsigned int fofs; /* start offset in a file */ + u32 blk; /* start block address of the extent */ + unsigned int len; /* length of the extent */ +}; + +struct extent_node { + struct rb_node rb_node; /* rb node located in rb-tree */ + struct list_head list; /* node in global extent list of sbi */ + struct extent_info ei; /* extent info */ +}; + +struct extent_tree { + nid_t ino; /* inode number */ + struct rb_root root; /* root of extent info rb-tree */ + struct extent_node *cached_en; /* recently accessed extent node */ + struct extent_info largest; /* largested extent info */ + rwlock_t lock; /* protect extent info rb-tree */ + atomic_t refcount; /* reference count of rb-tree */ + unsigned int count; /* # of extent node in rb-tree*/ +}; + +/* + * This structure is taken from ext4_map_blocks. + * + * Note that, however, f2fs uses NEW and MAPPED flags for f2fs_map_blocks(). + */ +#define F2FS_MAP_NEW (1 << BH_New) +#define F2FS_MAP_MAPPED (1 << BH_Mapped) +#define F2FS_MAP_UNWRITTEN (1 << BH_Unwritten) +#define F2FS_MAP_FLAGS (F2FS_MAP_NEW | F2FS_MAP_MAPPED |\ + F2FS_MAP_UNWRITTEN) + +struct f2fs_map_blocks { + block_t m_pblk; + block_t m_lblk; + unsigned int m_len; + unsigned int m_flags; +}; + +/* for flag in get_data_block */ +#define F2FS_GET_BLOCK_READ 0 +#define F2FS_GET_BLOCK_DIO 1 +#define F2FS_GET_BLOCK_FIEMAP 2 +#define F2FS_GET_BLOCK_BMAP 3 + +/* + * i_advise uses FADVISE_XXX_BIT. We can add additional hints later. + */ +#define FADVISE_COLD_BIT 0x01 +#define FADVISE_LOST_PINO_BIT 0x02 +#define FADVISE_ENCRYPT_BIT 0x04 +#define FADVISE_ENC_NAME_BIT 0x08 + +#define file_is_cold(inode) is_file(inode, FADVISE_COLD_BIT) +#define file_wrong_pino(inode) is_file(inode, FADVISE_LOST_PINO_BIT) +#define file_set_cold(inode) set_file(inode, FADVISE_COLD_BIT) +#define file_lost_pino(inode) set_file(inode, FADVISE_LOST_PINO_BIT) +#define file_clear_cold(inode) clear_file(inode, FADVISE_COLD_BIT) +#define file_got_pino(inode) clear_file(inode, FADVISE_LOST_PINO_BIT) +#define file_is_encrypt(inode) is_file(inode, FADVISE_ENCRYPT_BIT) +#define file_set_encrypt(inode) set_file(inode, FADVISE_ENCRYPT_BIT) +#define file_clear_encrypt(inode) clear_file(inode, FADVISE_ENCRYPT_BIT) +#define file_enc_name(inode) is_file(inode, FADVISE_ENC_NAME_BIT) +#define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT) + +/* Encryption algorithms */ +#define F2FS_ENCRYPTION_MODE_INVALID 0 +#define F2FS_ENCRYPTION_MODE_AES_256_XTS 1 +#define F2FS_ENCRYPTION_MODE_AES_256_GCM 2 +#define F2FS_ENCRYPTION_MODE_AES_256_CBC 3 +#define F2FS_ENCRYPTION_MODE_AES_256_CTS 4 + +#include "f2fs_crypto.h" + +#define DEF_DIR_LEVEL 0 + +struct f2fs_inode_info { + struct inode vfs_inode; /* serve a vfs inode */ + unsigned long i_flags; /* keep an inode flags for ioctl */ + unsigned char i_advise; /* use to give file attribute hints */ + unsigned char i_dir_level; /* use for dentry level for large dir */ + unsigned int i_current_depth; /* use only in directory structure */ + unsigned int i_pino; /* parent inode number */ + umode_t i_acl_mode; /* keep file acl mode temporarily */ + + /* Use below internally in f2fs*/ + unsigned long flags; /* use to pass per-file flags */ + struct rw_semaphore i_sem; /* protect fi info */ + atomic_t dirty_pages; /* # of dirty pages */ + f2fs_hash_t chash; /* hash value of given file name */ + unsigned int clevel; /* maximum level of given file name */ + nid_t i_xattr_nid; /* node id that contains xattrs */ + unsigned long long xattr_ver; /* cp version of xattr modification */ + struct inode_entry *dirty_dir; /* the pointer of dirty dir */ + + struct list_head inmem_pages; /* inmemory pages managed by f2fs */ + struct mutex inmem_lock; /* lock for inmemory pages */ + + struct extent_tree *extent_tree; /* cached extent_tree entry */ + +#ifdef CONFIG_F2FS_FS_ENCRYPTION + /* Encryption params */ + struct f2fs_crypt_info *i_crypt_info; +#endif +}; + +static inline void get_extent_info(struct extent_info *ext, + struct f2fs_extent i_ext) +{ + ext->fofs = le32_to_cpu(i_ext.fofs); + ext->blk = le32_to_cpu(i_ext.blk); + ext->len = le32_to_cpu(i_ext.len); +} + +static inline void set_raw_extent(struct extent_info *ext, + struct f2fs_extent *i_ext) +{ + i_ext->fofs = cpu_to_le32(ext->fofs); + i_ext->blk = cpu_to_le32(ext->blk); + i_ext->len = cpu_to_le32(ext->len); +} + +static inline void set_extent_info(struct extent_info *ei, unsigned int fofs, + u32 blk, unsigned int len) +{ + ei->fofs = fofs; + ei->blk = blk; + ei->len = len; +} + +static inline bool __is_extent_same(struct extent_info *ei1, + struct extent_info *ei2) +{ + return (ei1->fofs == ei2->fofs && ei1->blk == ei2->blk && + ei1->len == ei2->len); +} + +static inline bool __is_extent_mergeable(struct extent_info *back, + struct extent_info *front) +{ + return (back->fofs + back->len == front->fofs && + back->blk + back->len == front->blk); +} + +static inline bool __is_back_mergeable(struct extent_info *cur, + struct extent_info *back) +{ + return __is_extent_mergeable(back, cur); +} + +static inline bool __is_front_mergeable(struct extent_info *cur, + struct extent_info *front) +{ + return __is_extent_mergeable(cur, front); +} + +static inline void __try_update_largest_extent(struct extent_tree *et, + struct extent_node *en) +{ + if (en->ei.len > et->largest.len) + et->largest = en->ei; +} + +struct f2fs_nm_info { + block_t nat_blkaddr; /* base disk address of NAT */ + nid_t max_nid; /* maximum possible node ids */ + nid_t available_nids; /* maximum available node ids */ + nid_t next_scan_nid; /* the next nid to be scanned */ + unsigned int ram_thresh; /* control the memory footprint */ + unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */ + + /* NAT cache management */ + struct radix_tree_root nat_root;/* root of the nat entry cache */ + struct radix_tree_root nat_set_root;/* root of the nat set cache */ + struct rw_semaphore nat_tree_lock; /* protect nat_tree_lock */ + struct list_head nat_entries; /* cached nat entry list (clean) */ + unsigned int nat_cnt; /* the # of cached nat entries */ + unsigned int dirty_nat_cnt; /* total num of nat entries in set */ + + /* free node ids management */ + struct radix_tree_root free_nid_root;/* root of the free_nid cache */ + struct list_head free_nid_list; /* a list for free nids */ + spinlock_t free_nid_list_lock; /* protect free nid list */ + unsigned int fcnt; /* the number of free node id */ + struct mutex build_lock; /* lock for build free nids */ + + /* for checkpoint */ + char *nat_bitmap; /* NAT bitmap pointer */ + int bitmap_size; /* bitmap size */ +}; + +/* + * this structure is used as one of function parameters. + * all the information are dedicated to a given direct node block determined + * by the data offset in a file. + */ +struct dnode_of_data { + struct inode *inode; /* vfs inode pointer */ + struct page *inode_page; /* its inode page, NULL is possible */ + struct page *node_page; /* cached direct node page */ + nid_t nid; /* node id of the direct node block */ + unsigned int ofs_in_node; /* data offset in the node page */ + bool inode_page_locked; /* inode page is locked or not */ + block_t data_blkaddr; /* block address of the node block */ +}; + +static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode, + struct page *ipage, struct page *npage, nid_t nid) +{ + memset(dn, 0, sizeof(*dn)); + dn->inode = inode; + dn->inode_page = ipage; + dn->node_page = npage; + dn->nid = nid; +} + +/* + * For SIT manager + * + * By default, there are 6 active log areas across the whole main area. + * When considering hot and cold data separation to reduce cleaning overhead, + * we split 3 for data logs and 3 for node logs as hot, warm, and cold types, + * respectively. + * In the current design, you should not change the numbers intentionally. + * Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6 + * logs individually according to the underlying devices. (default: 6) + * Just in case, on-disk layout covers maximum 16 logs that consist of 8 for + * data and 8 for node logs. + */ +#define NR_CURSEG_DATA_TYPE (3) +#define NR_CURSEG_NODE_TYPE (3) +#define NR_CURSEG_TYPE (NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE) + +enum { + CURSEG_HOT_DATA = 0, /* directory entry blocks */ + CURSEG_WARM_DATA, /* data blocks */ + CURSEG_COLD_DATA, /* multimedia or GCed data blocks */ + CURSEG_HOT_NODE, /* direct node blocks of directory files */ + CURSEG_WARM_NODE, /* direct node blocks of normal files */ + CURSEG_COLD_NODE, /* indirect node blocks */ + NO_CHECK_TYPE, + CURSEG_DIRECT_IO, /* to use for the direct IO path */ +}; + +struct flush_cmd { + struct completion wait; + struct llist_node llnode; + int ret; +}; + +struct flush_cmd_control { + struct task_struct *f2fs_issue_flush; /* flush thread */ + wait_queue_head_t flush_wait_queue; /* waiting queue for wake-up */ + struct llist_head issue_list; /* list for command issue */ + struct llist_node *dispatch_list; /* list for command dispatch */ +}; + +struct f2fs_sm_info { + struct sit_info *sit_info; /* whole segment information */ + struct free_segmap_info *free_info; /* free segment information */ + struct dirty_seglist_info *dirty_info; /* dirty segment information */ + struct curseg_info *curseg_array; /* active segment information */ + + block_t seg0_blkaddr; /* block address of 0'th segment */ + block_t main_blkaddr; /* start block address of main area */ + block_t ssa_blkaddr; /* start block address of SSA area */ + + unsigned int segment_count; /* total # of segments */ + unsigned int main_segments; /* # of segments in main area */ + unsigned int reserved_segments; /* # of reserved segments */ + unsigned int ovp_segments; /* # of overprovision segments */ + + /* a threshold to reclaim prefree segments */ + unsigned int rec_prefree_segments; + + /* for small discard management */ + struct list_head discard_list; /* 4KB discard list */ + int nr_discards; /* # of discards in the list */ + int max_discards; /* max. discards to be issued */ + + /* for batched trimming */ + unsigned int trim_sections; /* # of sections to trim */ + + struct list_head sit_entry_set; /* sit entry set list */ + + unsigned int ipu_policy; /* in-place-update policy */ + unsigned int min_ipu_util; /* in-place-update threshold */ + unsigned int min_fsync_blocks; /* threshold for fsync */ + + /* for flush command control */ + struct flush_cmd_control *cmd_control_info; + +}; + +/* + * For superblock + */ +/* + * COUNT_TYPE for monitoring + * + * f2fs monitors the number of several block types such as on-writeback, + * dirty dentry blocks, dirty node blocks, and dirty meta blocks. + */ +enum count_type { + F2FS_WRITEBACK, + F2FS_DIRTY_DENTS, + F2FS_DIRTY_NODES, + F2FS_DIRTY_META, + F2FS_INMEM_PAGES, + NR_COUNT_TYPE, +}; + +/* + * The below are the page types of bios used in submit_bio(). + * The available types are: + * DATA User data pages. It operates as async mode. + * NODE Node pages. It operates as async mode. + * META FS metadata pages such as SIT, NAT, CP. + * NR_PAGE_TYPE The number of page types. + * META_FLUSH Make sure the previous pages are written + * with waiting the bio's completion + * ... Only can be used with META. + */ +#define PAGE_TYPE_OF_BIO(type) ((type) > META ? META : (type)) +enum page_type { + DATA, + NODE, + META, + NR_PAGE_TYPE, + META_FLUSH, + INMEM, /* the below types are used by tracepoints only. */ + INMEM_DROP, + IPU, + OPU, +}; + +struct f2fs_io_info { + struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */ + enum page_type type; /* contains DATA/NODE/META/META_FLUSH */ + int rw; /* contains R/RS/W/WS with REQ_META/REQ_PRIO */ + block_t blk_addr; /* block address to be written */ + struct page *page; /* page to be written */ + struct page *encrypted_page; /* encrypted page */ +}; + +#define is_read_io(rw) (((rw) & 1) == READ) + +struct f2fs_bio_info { + struct f2fs_sb_info *sbi; /* f2fs superblock */ + struct bio *bio; /* bios to merge */ + sector_t last_block_in_bio; /* last block number */ + struct f2fs_io_info fio; /* store buffered io info. */ + struct rw_semaphore io_rwsem; /* blocking op for bio */ +}; + +/* for inner inode cache management */ +struct inode_management { + struct radix_tree_root ino_root; /* ino entry array */ + spinlock_t ino_lock; /* for ino entry lock */ + struct list_head ino_list; /* inode list head */ + unsigned long ino_num; /* number of entries */ +}; + +/* For s_flag in struct f2fs_sb_info */ +enum { + SBI_IS_DIRTY, /* dirty flag for checkpoint */ + SBI_IS_CLOSE, /* specify unmounting */ + SBI_NEED_FSCK, /* need fsck.f2fs to fix */ + SBI_POR_DOING, /* recovery is doing or not */ +}; + +struct f2fs_sb_info { + struct super_block *sb; /* pointer to VFS super block */ + struct proc_dir_entry *s_proc; /* proc entry */ + struct buffer_head *raw_super_buf; /* buffer head of raw sb */ + struct f2fs_super_block *raw_super; /* raw super block pointer */ + int s_flag; /* flags for sbi */ + + /* for node-related operations */ + struct f2fs_nm_info *nm_info; /* node manager */ + struct inode *node_inode; /* cache node blocks */ + + /* for segment-related operations */ + struct f2fs_sm_info *sm_info; /* segment manager */ + + /* for bio operations */ + struct f2fs_bio_info read_io; /* for read bios */ + struct f2fs_bio_info write_io[NR_PAGE_TYPE]; /* for write bios */ + + /* for checkpoint */ + struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ + struct inode *meta_inode; /* cache meta blocks */ + struct mutex cp_mutex; /* checkpoint procedure lock */ + struct rw_semaphore cp_rwsem; /* blocking FS operations */ + struct rw_semaphore node_write; /* locking node writes */ + struct mutex writepages; /* mutex for writepages() */ + wait_queue_head_t cp_wait; + long cp_expires, cp_interval; /* next expected periodic cp */ + + struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */ + + /* for orphan inode, use 0'th array */ + unsigned int max_orphans; /* max orphan inodes */ + + /* for directory inode management */ + struct list_head dir_inode_list; /* dir inode list */ + spinlock_t dir_inode_lock; /* for dir inode list lock */ + + /* for extent tree cache */ + struct radix_tree_root extent_tree_root;/* cache extent cache entries */ + struct rw_semaphore extent_tree_lock; /* locking extent radix tree */ + struct list_head extent_list; /* lru list for shrinker */ + spinlock_t extent_lock; /* locking extent lru list */ + int total_ext_tree; /* extent tree count */ + atomic_t total_ext_node; /* extent info count */ + + /* basic filesystem units */ + unsigned int log_sectors_per_block; /* log2 sectors per block */ + unsigned int log_blocksize; /* log2 block size */ + unsigned int blocksize; /* block size */ + unsigned int root_ino_num; /* root inode number*/ + unsigned int node_ino_num; /* node inode number*/ + unsigned int meta_ino_num; /* meta inode number*/ + unsigned int log_blocks_per_seg; /* log2 blocks per segment */ + unsigned int blocks_per_seg; /* blocks per segment */ + unsigned int segs_per_sec; /* segments per section */ + unsigned int secs_per_zone; /* sections per zone */ + unsigned int total_sections; /* total section count */ + unsigned int total_node_count; /* total node block count */ + unsigned int total_valid_node_count; /* valid node block count */ + unsigned int total_valid_inode_count; /* valid inode count */ + int active_logs; /* # of active logs */ + int dir_level; /* directory level */ + + block_t user_block_count; /* # of user blocks */ + block_t total_valid_block_count; /* # of valid blocks */ + block_t alloc_valid_block_count; /* # of allocated blocks */ + block_t discard_blks; /* discard command candidats */ + block_t last_valid_block_count; /* for recovery */ + u32 s_next_generation; /* for NFS support */ + atomic_t nr_pages[NR_COUNT_TYPE]; /* # of pages, see count_type */ + + struct f2fs_mount_info mount_opt; /* mount options */ + + /* for cleaning operations */ + struct mutex gc_mutex; /* mutex for GC */ + struct f2fs_gc_kthread *gc_thread; /* GC thread */ + unsigned int cur_victim_sec; /* current victim section num */ + + /* maximum # of trials to find a victim segment for SSR and GC */ + unsigned int max_victim_search; + + /* + * for stat information. + * one is for the LFS mode, and the other is for the SSR mode. + */ +#ifdef CONFIG_F2FS_STAT_FS + struct f2fs_stat_info *stat_info; /* FS status information */ + unsigned int segment_count[2]; /* # of allocated segments */ + unsigned int block_count[2]; /* # of allocated blocks */ + atomic_t inplace_count; /* # of inplace update */ + atomic64_t total_hit_ext; /* # of lookup extent cache */ + atomic64_t read_hit_rbtree; /* # of hit rbtree extent node */ + atomic64_t read_hit_largest; /* # of hit largest extent node */ + atomic64_t read_hit_cached; /* # of hit cached extent node */ + atomic_t inline_xattr; /* # of inline_xattr inodes */ + atomic_t inline_inode; /* # of inline_data inodes */ + atomic_t inline_dir; /* # of inline_dentry inodes */ + int bg_gc; /* background gc calls */ + unsigned int n_dirty_dirs; /* # of dir inodes */ +#endif + unsigned int last_victim[2]; /* last victim segment # */ + spinlock_t stat_lock; /* lock for stat operations */ + + /* For sysfs suppport */ + struct kobject s_kobj; + struct completion s_kobj_unregister; + + /* For shrinker support */ + struct list_head s_list; + struct mutex umount_mutex; + unsigned int shrinker_run_no; +}; + +/* + * Inline functions + */ +static inline struct f2fs_inode_info *F2FS_I(struct inode *inode) +{ + return container_of(inode, struct f2fs_inode_info, vfs_inode); +} + +static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode) +{ + return F2FS_SB(inode->i_sb); +} + +static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping) +{ + return F2FS_I_SB(mapping->host); +} + +static inline struct f2fs_sb_info *F2FS_P_SB(struct page *page) +{ + return F2FS_M_SB(page->mapping); +} + +static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_super_block *)(sbi->raw_super); +} + +static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_checkpoint *)(sbi->ckpt); +} + +static inline struct f2fs_node *F2FS_NODE(struct page *page) +{ + return (struct f2fs_node *)page_address(page); +} + +static inline struct f2fs_inode *F2FS_INODE(struct page *page) +{ + return &((struct f2fs_node *)page_address(page))->i; +} + +static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_nm_info *)(sbi->nm_info); +} + +static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_sm_info *)(sbi->sm_info); +} + +static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi) +{ + return (struct sit_info *)(SM_I(sbi)->sit_info); +} + +static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi) +{ + return (struct free_segmap_info *)(SM_I(sbi)->free_info); +} + +static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi) +{ + return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info); +} + +static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi) +{ + return sbi->meta_inode->i_mapping; +} + +static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi) +{ + return sbi->node_inode->i_mapping; +} + +static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type) +{ + return sbi->s_flag & (0x01 << type); +} + +static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) +{ + sbi->s_flag |= (0x01 << type); +} + +static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) +{ + sbi->s_flag &= ~(0x01 << type); +} + +static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp) +{ + return le64_to_cpu(cp->checkpoint_ver); +} + +static inline bool is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f) +{ + unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags); + return ckpt_flags & f; +} + +static inline void set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f) +{ + unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags); + ckpt_flags |= f; + cp->ckpt_flags = cpu_to_le32(ckpt_flags); +} + +static inline void clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f) +{ + unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags); + ckpt_flags &= (~f); + cp->ckpt_flags = cpu_to_le32(ckpt_flags); +} + +static inline void f2fs_lock_op(struct f2fs_sb_info *sbi) +{ + down_read(&sbi->cp_rwsem); +} + +static inline void f2fs_unlock_op(struct f2fs_sb_info *sbi) +{ + up_read(&sbi->cp_rwsem); +} + +static inline void f2fs_lock_all(struct f2fs_sb_info *sbi) +{ + f2fs_down_write(&sbi->cp_rwsem, &sbi->cp_mutex); +} + +static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi) +{ + up_write(&sbi->cp_rwsem); +} + +static inline int __get_cp_reason(struct f2fs_sb_info *sbi) +{ + int reason = CP_SYNC; + + if (test_opt(sbi, FASTBOOT)) + reason = CP_FASTBOOT; + if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) + reason = CP_UMOUNT; + return reason; +} + +static inline bool __remain_node_summaries(int reason) +{ + return (reason == CP_UMOUNT || reason == CP_FASTBOOT); +} + +static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi) +{ + return (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG) || + is_set_ckpt_flags(F2FS_CKPT(sbi), CP_FASTBOOT_FLAG)); +} + +/* + * Check whether the given nid is within node id range. + */ +static inline int check_nid_range(struct f2fs_sb_info *sbi, nid_t nid) +{ + if (unlikely(nid < F2FS_ROOT_INO(sbi))) + return -EINVAL; + if (unlikely(nid >= NM_I(sbi)->max_nid)) + return -EINVAL; + return 0; +} + +#define F2FS_DEFAULT_ALLOCATED_BLOCKS 1 + +/* + * Check whether the inode has blocks or not + */ +static inline int F2FS_HAS_BLOCKS(struct inode *inode) +{ + if (F2FS_I(inode)->i_xattr_nid) + return inode->i_blocks > F2FS_DEFAULT_ALLOCATED_BLOCKS + 1; + else + return inode->i_blocks > F2FS_DEFAULT_ALLOCATED_BLOCKS; +} + +static inline bool f2fs_has_xattr_block(unsigned int ofs) +{ + return ofs == XATTR_NODE_OFFSET; +} + +static inline bool inc_valid_block_count(struct f2fs_sb_info *sbi, + struct inode *inode, blkcnt_t count) +{ + block_t valid_block_count; + + spin_lock(&sbi->stat_lock); + valid_block_count = + sbi->total_valid_block_count + (block_t)count; + if (unlikely(valid_block_count > sbi->user_block_count)) { + spin_unlock(&sbi->stat_lock); + return false; + } + inode->i_blocks += count; + sbi->total_valid_block_count = valid_block_count; + sbi->alloc_valid_block_count += (block_t)count; + spin_unlock(&sbi->stat_lock); + return true; +} + +static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, + struct inode *inode, + blkcnt_t count) +{ + spin_lock(&sbi->stat_lock); + f2fs_bug_on(sbi, sbi->total_valid_block_count < (block_t) count); + f2fs_bug_on(sbi, inode->i_blocks < count); + inode->i_blocks -= count; + sbi->total_valid_block_count -= (block_t)count; + spin_unlock(&sbi->stat_lock); +} + +static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) +{ + atomic_inc(&sbi->nr_pages[count_type]); + set_sbi_flag(sbi, SBI_IS_DIRTY); +} + +static inline void inode_inc_dirty_pages(struct inode *inode) +{ + atomic_inc(&F2FS_I(inode)->dirty_pages); + if (S_ISDIR(inode->i_mode)) + inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_DENTS); +} + +static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type) +{ + atomic_dec(&sbi->nr_pages[count_type]); +} + +static inline void inode_dec_dirty_pages(struct inode *inode) +{ + if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + return; + + atomic_dec(&F2FS_I(inode)->dirty_pages); + + if (S_ISDIR(inode->i_mode)) + dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_DENTS); +} + +static inline int get_pages(struct f2fs_sb_info *sbi, int count_type) +{ + return atomic_read(&sbi->nr_pages[count_type]); +} + +static inline int get_dirty_pages(struct inode *inode) +{ + return atomic_read(&F2FS_I(inode)->dirty_pages); +} + +static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type) +{ + unsigned int pages_per_sec = sbi->segs_per_sec * + (1 << sbi->log_blocks_per_seg); + return ((get_pages(sbi, block_type) + pages_per_sec - 1) + >> sbi->log_blocks_per_seg) / sbi->segs_per_sec; +} + +static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi) +{ + return sbi->total_valid_block_count; +} + +static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + + /* return NAT or SIT bitmap */ + if (flag == NAT_BITMAP) + return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize); + else if (flag == SIT_BITMAP) + return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize); + + return 0; +} + +static inline block_t __cp_payload(struct f2fs_sb_info *sbi) +{ + return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload); +} + +static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + int offset; + + if (__cp_payload(sbi) > 0) { + if (flag == NAT_BITMAP) + return &ckpt->sit_nat_version_bitmap; + else + return (unsigned char *)ckpt + F2FS_BLKSIZE; + } else { + offset = (flag == NAT_BITMAP) ? + le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0; + return &ckpt->sit_nat_version_bitmap + offset; + } +} + +static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi) +{ + block_t start_addr; + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + unsigned long long ckpt_version = cur_cp_version(ckpt); + + start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + + /* + * odd numbered checkpoint should at cp segment 0 + * and even segment must be at cp segment 1 + */ + if (!(ckpt_version & 1)) + start_addr += sbi->blocks_per_seg; + + return start_addr; +} + +static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) +{ + return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); +} + +static inline bool inc_valid_node_count(struct f2fs_sb_info *sbi, + struct inode *inode) +{ + block_t valid_block_count; + unsigned int valid_node_count; + + spin_lock(&sbi->stat_lock); + + valid_block_count = sbi->total_valid_block_count + 1; + if (unlikely(valid_block_count > sbi->user_block_count)) { + spin_unlock(&sbi->stat_lock); + return false; + } + + valid_node_count = sbi->total_valid_node_count + 1; + if (unlikely(valid_node_count > sbi->total_node_count)) { + spin_unlock(&sbi->stat_lock); + return false; + } + + if (inode) + inode->i_blocks++; + + sbi->alloc_valid_block_count++; + sbi->total_valid_node_count++; + sbi->total_valid_block_count++; + spin_unlock(&sbi->stat_lock); + + return true; +} + +static inline void dec_valid_node_count(struct f2fs_sb_info *sbi, + struct inode *inode) +{ + spin_lock(&sbi->stat_lock); + + f2fs_bug_on(sbi, !sbi->total_valid_block_count); + f2fs_bug_on(sbi, !sbi->total_valid_node_count); + f2fs_bug_on(sbi, !inode->i_blocks); + + inode->i_blocks--; + sbi->total_valid_node_count--; + sbi->total_valid_block_count--; + + spin_unlock(&sbi->stat_lock); +} + +static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi) +{ + return sbi->total_valid_node_count; +} + +static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi) +{ + spin_lock(&sbi->stat_lock); + f2fs_bug_on(sbi, sbi->total_valid_inode_count == sbi->total_node_count); + sbi->total_valid_inode_count++; + spin_unlock(&sbi->stat_lock); +} + +static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi) +{ + spin_lock(&sbi->stat_lock); + f2fs_bug_on(sbi, !sbi->total_valid_inode_count); + sbi->total_valid_inode_count--; + spin_unlock(&sbi->stat_lock); +} + +static inline unsigned int valid_inode_count(struct f2fs_sb_info *sbi) +{ + return sbi->total_valid_inode_count; +} + +static inline struct page *f2fs_grab_cache_page(struct address_space *mapping, + pgoff_t index, bool for_write) +{ + if (!for_write) + return grab_cache_page(mapping, index); + return grab_cache_page_write_begin(mapping, index, AOP_FLAG_NOFS); +} + +static inline void f2fs_copy_page(struct page *src, struct page *dst) +{ + char *src_kaddr = kmap(src); + char *dst_kaddr = kmap(dst); + + memcpy(dst_kaddr, src_kaddr, PAGE_SIZE); + kunmap(dst); + kunmap(src); +} + +static inline void f2fs_put_page(struct page *page, int unlock) +{ + if (!page) + return; + + if (unlock) { + f2fs_bug_on(F2FS_P_SB(page), !PageLocked(page)); + unlock_page(page); + } + page_cache_release(page); +} + +static inline void f2fs_put_dnode(struct dnode_of_data *dn) +{ + if (dn->node_page) + f2fs_put_page(dn->node_page, 1); + if (dn->inode_page && dn->node_page != dn->inode_page) + f2fs_put_page(dn->inode_page, 0); + dn->node_page = NULL; + dn->inode_page = NULL; +} + +static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name, + size_t size) +{ + return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL); +} + +static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep, + gfp_t flags) +{ + void *entry; + + entry = kmem_cache_alloc(cachep, flags); + if (!entry) + entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL); + return entry; +} + +static inline struct bio *f2fs_bio_alloc(int npages) +{ + struct bio *bio; + + /* No failure on bio allocation */ + bio = bio_alloc(GFP_NOIO, npages); + if (!bio) + bio = bio_alloc(GFP_NOIO | __GFP_NOFAIL, npages); + return bio; +} + +static inline void f2fs_radix_tree_insert(struct radix_tree_root *root, + unsigned long index, void *item) +{ + while (radix_tree_insert(root, index, item)) + cond_resched(); +} + +#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino) + +static inline bool IS_INODE(struct page *page) +{ + struct f2fs_node *p = F2FS_NODE(page); + return RAW_IS_INODE(p); +} + +static inline __le32 *blkaddr_in_node(struct f2fs_node *node) +{ + return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr; +} + +static inline block_t datablock_addr(struct page *node_page, + unsigned int offset) +{ + struct f2fs_node *raw_node; + __le32 *addr_array; + raw_node = F2FS_NODE(node_page); + addr_array = blkaddr_in_node(raw_node); + return le32_to_cpu(addr_array[offset]); +} + +static inline int f2fs_test_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = 1 << (7 - (nr & 0x07)); + return mask & *addr; +} + +static inline void f2fs_set_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = 1 << (7 - (nr & 0x07)); + *addr |= mask; +} + +static inline void f2fs_clear_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = 1 << (7 - (nr & 0x07)); + *addr &= ~mask; +} + +static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr) +{ + int mask; + int ret; + + addr += (nr >> 3); + mask = 1 << (7 - (nr & 0x07)); + ret = mask & *addr; + *addr |= mask; + return ret; +} + +static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr) +{ + int mask; + int ret; + + addr += (nr >> 3); + mask = 1 << (7 - (nr & 0x07)); + ret = mask & *addr; + *addr &= ~mask; + return ret; +} + +static inline void f2fs_change_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = 1 << (7 - (nr & 0x07)); + *addr ^= mask; +} + +/* used for f2fs_inode_info->flags */ +enum { + FI_NEW_INODE, /* indicate newly allocated inode */ + FI_DIRTY_INODE, /* indicate inode is dirty or not */ + FI_DIRTY_DIR, /* indicate directory has dirty pages */ + FI_INC_LINK, /* need to increment i_nlink */ + FI_ACL_MODE, /* indicate acl mode */ + FI_NO_ALLOC, /* should not allocate any blocks */ + FI_FREE_NID, /* free allocated nide */ + FI_UPDATE_DIR, /* should update inode block for consistency */ + FI_DELAY_IPUT, /* used for the recovery */ + FI_NO_EXTENT, /* not to use the extent cache */ + FI_INLINE_XATTR, /* used for inline xattr */ + FI_INLINE_DATA, /* used for inline data*/ + FI_INLINE_DENTRY, /* used for inline dentry */ + FI_APPEND_WRITE, /* inode has appended data */ + FI_UPDATE_WRITE, /* inode has in-place-update data */ + FI_NEED_IPU, /* used for ipu per file */ + FI_ATOMIC_FILE, /* indicate atomic file */ + FI_VOLATILE_FILE, /* indicate volatile file */ + FI_FIRST_BLOCK_WRITTEN, /* indicate #0 data block was written */ + FI_DROP_CACHE, /* drop dirty page cache */ + FI_DATA_EXIST, /* indicate data exists */ + FI_INLINE_DOTS, /* indicate inline dot dentries */ +}; + +static inline void set_inode_flag(struct f2fs_inode_info *fi, int flag) +{ + if (!test_bit(flag, &fi->flags)) + set_bit(flag, &fi->flags); +} + +static inline int is_inode_flag_set(struct f2fs_inode_info *fi, int flag) +{ + return test_bit(flag, &fi->flags); +} + +static inline void clear_inode_flag(struct f2fs_inode_info *fi, int flag) +{ + if (test_bit(flag, &fi->flags)) + clear_bit(flag, &fi->flags); +} + +static inline void set_acl_inode(struct f2fs_inode_info *fi, umode_t mode) +{ + fi->i_acl_mode = mode; + set_inode_flag(fi, FI_ACL_MODE); +} + +static inline void get_inline_info(struct f2fs_inode_info *fi, + struct f2fs_inode *ri) +{ + if (ri->i_inline & F2FS_INLINE_XATTR) + set_inode_flag(fi, FI_INLINE_XATTR); + if (ri->i_inline & F2FS_INLINE_DATA) + set_inode_flag(fi, FI_INLINE_DATA); + if (ri->i_inline & F2FS_INLINE_DENTRY) + set_inode_flag(fi, FI_INLINE_DENTRY); + if (ri->i_inline & F2FS_DATA_EXIST) + set_inode_flag(fi, FI_DATA_EXIST); + if (ri->i_inline & F2FS_INLINE_DOTS) + set_inode_flag(fi, FI_INLINE_DOTS); +} + +static inline void set_raw_inline(struct f2fs_inode_info *fi, + struct f2fs_inode *ri) +{ + ri->i_inline = 0; + + if (is_inode_flag_set(fi, FI_INLINE_XATTR)) + ri->i_inline |= F2FS_INLINE_XATTR; + if (is_inode_flag_set(fi, FI_INLINE_DATA)) + ri->i_inline |= F2FS_INLINE_DATA; + if (is_inode_flag_set(fi, FI_INLINE_DENTRY)) + ri->i_inline |= F2FS_INLINE_DENTRY; + if (is_inode_flag_set(fi, FI_DATA_EXIST)) + ri->i_inline |= F2FS_DATA_EXIST; + if (is_inode_flag_set(fi, FI_INLINE_DOTS)) + ri->i_inline |= F2FS_INLINE_DOTS; +} + +static inline int f2fs_has_inline_xattr(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_INLINE_XATTR); +} + +static inline unsigned int addrs_per_inode(struct f2fs_inode_info *fi) +{ + if (f2fs_has_inline_xattr(&fi->vfs_inode)) + return DEF_ADDRS_PER_INODE - F2FS_INLINE_XATTR_ADDRS; + return DEF_ADDRS_PER_INODE; +} + +static inline void *inline_xattr_addr(struct page *page) +{ + struct f2fs_inode *ri = F2FS_INODE(page); + return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE - + F2FS_INLINE_XATTR_ADDRS]); +} + +static inline int inline_xattr_size(struct inode *inode) +{ + if (f2fs_has_inline_xattr(inode)) + return F2FS_INLINE_XATTR_ADDRS << 2; + else + return 0; +} + +static inline int f2fs_has_inline_data(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_INLINE_DATA); +} + +static inline void f2fs_clear_inline_inode(struct inode *inode) +{ + clear_inode_flag(F2FS_I(inode), FI_INLINE_DATA); + clear_inode_flag(F2FS_I(inode), FI_DATA_EXIST); +} + +static inline int f2fs_exist_data(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_DATA_EXIST); +} + +static inline int f2fs_has_inline_dots(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_INLINE_DOTS); +} + +static inline bool f2fs_is_atomic_file(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_ATOMIC_FILE); +} + +static inline bool f2fs_is_volatile_file(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_VOLATILE_FILE); +} + +static inline bool f2fs_is_first_block_written(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); +} + +static inline bool f2fs_is_drop_cache(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_DROP_CACHE); +} + +static inline void *inline_data_addr(struct page *page) +{ + struct f2fs_inode *ri = F2FS_INODE(page); + return (void *)&(ri->i_addr[1]); +} + +static inline int f2fs_has_inline_dentry(struct inode *inode) +{ + return is_inode_flag_set(F2FS_I(inode), FI_INLINE_DENTRY); +} + +static inline void f2fs_dentry_kunmap(struct inode *dir, struct page *page) +{ + if (!f2fs_has_inline_dentry(dir)) + kunmap(page); +} + +static inline int is_file(struct inode *inode, int type) +{ + return F2FS_I(inode)->i_advise & type; +} + +static inline void set_file(struct inode *inode, int type) +{ + F2FS_I(inode)->i_advise |= type; +} + +static inline void clear_file(struct inode *inode, int type) +{ + F2FS_I(inode)->i_advise &= ~type; +} + +static inline int f2fs_readonly(struct super_block *sb) +{ + return sb->s_flags & MS_RDONLY; +} + +static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi) +{ + return is_set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG); +} + +static inline void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi) +{ + set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG); + sbi->sb->s_flags |= MS_RDONLY; +} + +static inline struct inode *file_inode(struct file *f) +{ + return f->f_path.dentry->d_inode; +} + +static inline bool is_dot_dotdot(const struct qstr *str) +{ + if (str->len == 1 && str->name[0] == '.') + return true; + + if (str->len == 2 && str->name[0] == '.' && str->name[1] == '.') + return true; + + return false; +} + +static inline bool f2fs_may_extent_tree(struct inode *inode) +{ + mode_t mode = inode->i_mode; + + if (!test_opt(F2FS_I_SB(inode), EXTENT_CACHE) || + is_inode_flag_set(F2FS_I(inode), FI_NO_EXTENT)) + return false; + + return S_ISREG(mode); +} + +static inline void *f2fs_kvmalloc(size_t size, gfp_t flags) +{ + void *ret; + + ret = kmalloc(size, flags | __GFP_NOWARN); + if (!ret) + ret = __vmalloc(size, flags, PAGE_KERNEL); + return ret; +} + +static inline void *f2fs_kvzalloc(size_t size, gfp_t flags) +{ + void *ret; + + ret = kzalloc(size, flags | __GFP_NOWARN); + if (!ret) + ret = __vmalloc(size, flags | __GFP_ZERO, PAGE_KERNEL); + return ret; +} + +static inline void f2fs_kvfree(void *ptr) +{ + if (is_vmalloc_addr(ptr)) + vfree(ptr); + else + kfree(ptr); +} + +#define get_inode_mode(i) \ + ((is_inode_flag_set(F2FS_I(i), FI_ACL_MODE)) ? \ + (F2FS_I(i)->i_acl_mode) : ((i)->i_mode)) + +/* get offset of first page in next direct node */ +#define PGOFS_OF_NEXT_DNODE(pgofs, fi) \ + ((pgofs < ADDRS_PER_INODE(fi)) ? ADDRS_PER_INODE(fi) : \ + (pgofs - ADDRS_PER_INODE(fi) + ADDRS_PER_BLOCK) / \ + ADDRS_PER_BLOCK * ADDRS_PER_BLOCK + ADDRS_PER_INODE(fi)) + +/* + * file.c + */ +int f2fs_sync_file(struct file *, loff_t, loff_t, int); +void truncate_data_blocks(struct dnode_of_data *); +int truncate_blocks(struct inode *, u64, bool); +int f2fs_truncate(struct inode *, bool); +int f2fs_getattr(struct vfsmount *, struct dentry *, struct kstat *); +int f2fs_setattr(struct dentry *, struct iattr *); +int truncate_hole(struct inode *, pgoff_t, pgoff_t); +int truncate_data_blocks_range(struct dnode_of_data *, int); +long f2fs_ioctl(struct file *, unsigned int, unsigned long); +long f2fs_compat_ioctl(struct file *, unsigned int, unsigned long); + +/* + * inode.c + */ +void f2fs_set_inode_flags(struct inode *); +struct inode *f2fs_iget(struct super_block *, unsigned long); +int try_to_free_nats(struct f2fs_sb_info *, int); +void update_inode(struct inode *, struct page *); +void update_inode_page(struct inode *); +int f2fs_write_inode(struct inode *, struct writeback_control *); +void f2fs_evict_inode(struct inode *); +void handle_failed_inode(struct inode *); + +/* + * namei.c + */ +struct dentry *f2fs_get_parent(struct dentry *child); + +/* + * dir.c + */ +extern unsigned char f2fs_filetype_table[F2FS_FT_MAX]; +void set_de_type(struct f2fs_dir_entry *, umode_t); +struct f2fs_dir_entry *find_target_dentry(struct f2fs_filename *, + f2fs_hash_t, int *, struct f2fs_dentry_ptr *); +bool f2fs_fill_dentries(struct file *, void *, filldir_t, + struct f2fs_dentry_ptr *, unsigned int, unsigned int, struct f2fs_str *); +void do_make_empty_dir(struct inode *, struct inode *, + struct f2fs_dentry_ptr *); +struct page *init_inode_metadata(struct inode *, struct inode *, + const struct qstr *, struct page *); +void update_parent_metadata(struct inode *, struct inode *, unsigned int); +int room_for_filename(const void *, int, int); +void f2fs_drop_nlink(struct inode *, struct inode *, struct page *); +struct f2fs_dir_entry *f2fs_find_entry(struct inode *, struct qstr *, + struct page **); +struct f2fs_dir_entry *f2fs_parent_dir(struct inode *, struct page **); +ino_t f2fs_inode_by_name(struct inode *, struct qstr *); +void f2fs_set_link(struct inode *, struct f2fs_dir_entry *, + struct page *, struct inode *); +int update_dent_inode(struct inode *, struct inode *, const struct qstr *); +void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *, + const struct qstr *, f2fs_hash_t , unsigned int); +int __f2fs_add_link(struct inode *, const struct qstr *, struct inode *, nid_t, + umode_t); +void f2fs_delete_entry(struct f2fs_dir_entry *, struct page *, struct inode *, + struct inode *); +int f2fs_do_tmpfile(struct inode *, struct inode *); +bool f2fs_empty_dir(struct inode *); + +static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode) +{ + return __f2fs_add_link(dentry->d_parent->d_inode, &dentry->d_name, + inode, inode->i_ino, inode->i_mode); +} + +/* + * super.c + */ +int f2fs_commit_super(struct f2fs_sb_info *, bool); +int f2fs_sync_fs(struct super_block *, int); +extern __printf(3, 4) +void f2fs_msg(struct super_block *, const char *, const char *, ...); + +/* + * hash.c + */ +f2fs_hash_t f2fs_dentry_hash(const struct qstr *); + +/* + * node.c + */ +struct dnode_of_data; +struct node_info; + +bool available_free_memory(struct f2fs_sb_info *, int); +int need_dentry_mark(struct f2fs_sb_info *, nid_t); +bool is_checkpointed_node(struct f2fs_sb_info *, nid_t); +bool need_inode_block_update(struct f2fs_sb_info *, nid_t); +void get_node_info(struct f2fs_sb_info *, nid_t, struct node_info *); +int get_dnode_of_data(struct dnode_of_data *, pgoff_t, int); +int truncate_inode_blocks(struct inode *, pgoff_t); +int truncate_xattr_node(struct inode *, struct page *); +int wait_on_node_pages_writeback(struct f2fs_sb_info *, nid_t); +int remove_inode_page(struct inode *); +struct page *new_inode_page(struct inode *); +struct page *new_node_page(struct dnode_of_data *, unsigned int, struct page *); +void ra_node_page(struct f2fs_sb_info *, nid_t); +struct page *get_node_page(struct f2fs_sb_info *, pgoff_t); +struct page *get_node_page_ra(struct page *, int); +void sync_inode_page(struct dnode_of_data *); +int sync_node_pages(struct f2fs_sb_info *, nid_t, struct writeback_control *); +bool alloc_nid(struct f2fs_sb_info *, nid_t *); +void alloc_nid_done(struct f2fs_sb_info *, nid_t); +void alloc_nid_failed(struct f2fs_sb_info *, nid_t); +int try_to_free_nids(struct f2fs_sb_info *, int); +void recover_inline_xattr(struct inode *, struct page *); +void recover_xattr_data(struct inode *, struct page *, block_t); +int recover_inode_page(struct f2fs_sb_info *, struct page *); +int restore_node_summary(struct f2fs_sb_info *, unsigned int, + struct f2fs_summary_block *); +void flush_nat_entries(struct f2fs_sb_info *); +int build_node_manager(struct f2fs_sb_info *); +void destroy_node_manager(struct f2fs_sb_info *); +int __init create_node_manager_caches(void); +void destroy_node_manager_caches(void); + +/* + * segment.c + */ +void register_inmem_page(struct inode *, struct page *); +int commit_inmem_pages(struct inode *, bool); +void f2fs_balance_fs(struct f2fs_sb_info *); +void f2fs_balance_fs_bg(struct f2fs_sb_info *); +int f2fs_issue_flush(struct f2fs_sb_info *); +int create_flush_cmd_control(struct f2fs_sb_info *); +void destroy_flush_cmd_control(struct f2fs_sb_info *); +void invalidate_blocks(struct f2fs_sb_info *, block_t); +bool is_checkpointed_data(struct f2fs_sb_info *, block_t); +void refresh_sit_entry(struct f2fs_sb_info *, block_t, block_t); +void clear_prefree_segments(struct f2fs_sb_info *, struct cp_control *); +void release_discard_addrs(struct f2fs_sb_info *); +bool discard_next_dnode(struct f2fs_sb_info *, block_t); +int npages_for_summary_flush(struct f2fs_sb_info *, bool); +void allocate_new_segments(struct f2fs_sb_info *); +int f2fs_trim_fs(struct f2fs_sb_info *, struct fstrim_range *); +struct page *get_sum_page(struct f2fs_sb_info *, unsigned int); +void update_meta_page(struct f2fs_sb_info *, void *, block_t); +void write_meta_page(struct f2fs_sb_info *, struct page *); +void write_node_page(unsigned int, struct f2fs_io_info *); +void write_data_page(struct dnode_of_data *, struct f2fs_io_info *); +void rewrite_data_page(struct f2fs_io_info *); +void f2fs_replace_block(struct f2fs_sb_info *, struct dnode_of_data *, + block_t, block_t, unsigned char, bool); +void allocate_data_block(struct f2fs_sb_info *, struct page *, + block_t, block_t *, struct f2fs_summary *, int); +void f2fs_wait_on_page_writeback(struct page *, enum page_type); +void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *, block_t); +void write_data_summaries(struct f2fs_sb_info *, block_t); +void write_node_summaries(struct f2fs_sb_info *, block_t); +int lookup_journal_in_cursum(struct f2fs_summary_block *, + int, unsigned int, int); +void flush_sit_entries(struct f2fs_sb_info *, struct cp_control *); +int build_segment_manager(struct f2fs_sb_info *); +void destroy_segment_manager(struct f2fs_sb_info *); +int __init create_segment_manager_caches(void); +void destroy_segment_manager_caches(void); + +/* + * checkpoint.c + */ +struct page *grab_meta_page(struct f2fs_sb_info *, pgoff_t); +struct page *get_meta_page(struct f2fs_sb_info *, pgoff_t); +struct page *get_tmp_page(struct f2fs_sb_info *, pgoff_t); +bool is_valid_blkaddr(struct f2fs_sb_info *, block_t, int); +int ra_meta_pages(struct f2fs_sb_info *, block_t, int, int, bool); +void ra_meta_pages_cond(struct f2fs_sb_info *, pgoff_t); +long sync_meta_pages(struct f2fs_sb_info *, enum page_type, long); +void add_dirty_inode(struct f2fs_sb_info *, nid_t, int type); +void remove_dirty_inode(struct f2fs_sb_info *, nid_t, int type); +void release_dirty_inode(struct f2fs_sb_info *); +bool exist_written_data(struct f2fs_sb_info *, nid_t, int); +int acquire_orphan_inode(struct f2fs_sb_info *); +void release_orphan_inode(struct f2fs_sb_info *); +void add_orphan_inode(struct f2fs_sb_info *, nid_t); +void remove_orphan_inode(struct f2fs_sb_info *, nid_t); +int recover_orphan_inodes(struct f2fs_sb_info *); +int get_valid_checkpoint(struct f2fs_sb_info *); +void update_dirty_page(struct inode *, struct page *); +void add_dirty_dir_inode(struct inode *); +void remove_dirty_dir_inode(struct inode *); +void sync_dirty_dir_inodes(struct f2fs_sb_info *); +void write_checkpoint(struct f2fs_sb_info *, struct cp_control *); +void init_ino_entry_info(struct f2fs_sb_info *); +int __init create_checkpoint_caches(void); +void destroy_checkpoint_caches(void); + +/* + * data.c + */ +void f2fs_submit_merged_bio(struct f2fs_sb_info *, enum page_type, int); +int f2fs_submit_page_bio(struct f2fs_io_info *); +void f2fs_submit_page_mbio(struct f2fs_io_info *); +void set_data_blkaddr(struct dnode_of_data *); +int reserve_new_block(struct dnode_of_data *); +int f2fs_get_block(struct dnode_of_data *, pgoff_t); +int f2fs_reserve_block(struct dnode_of_data *, pgoff_t); +struct page *get_read_data_page(struct inode *, pgoff_t, int, bool); +struct page *find_data_page(struct inode *, pgoff_t); +struct page *get_lock_data_page(struct inode *, pgoff_t, bool); +struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); +int do_write_data_page(struct f2fs_io_info *); +int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64); +void f2fs_invalidate_page(struct page *, unsigned long); +int f2fs_release_page(struct page *, gfp_t); + +/* + * gc.c + */ +int start_gc_thread(struct f2fs_sb_info *); +void stop_gc_thread(struct f2fs_sb_info *); +block_t start_bidx_of_node(unsigned int, struct f2fs_inode_info *); +int f2fs_gc(struct f2fs_sb_info *, bool); +void build_gc_manager(struct f2fs_sb_info *); + +/* + * recovery.c + */ +int recover_fsync_data(struct f2fs_sb_info *); +bool space_for_roll_forward(struct f2fs_sb_info *); + +/* + * debug.c + */ +#ifdef CONFIG_F2FS_STAT_FS +struct f2fs_stat_info { + struct list_head stat_list; + struct f2fs_sb_info *sbi; + int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs; + int main_area_segs, main_area_sections, main_area_zones; + unsigned long long hit_largest, hit_cached, hit_rbtree; + unsigned long long hit_total, total_ext; + int ext_tree, ext_node; + int ndirty_node, ndirty_dent, ndirty_dirs, ndirty_meta; + int nats, dirty_nats, sits, dirty_sits, fnids; + int total_count, utilization; + int bg_gc, inmem_pages, wb_pages; + int inline_xattr, inline_inode, inline_dir; + unsigned int valid_count, valid_node_count, valid_inode_count; + unsigned int bimodal, avg_vblocks; + int util_free, util_valid, util_invalid; + int rsvd_segs, overp_segs; + int dirty_count, node_pages, meta_pages; + int prefree_count, call_count, cp_count; + int tot_segs, node_segs, data_segs, free_segs, free_secs; + int bg_node_segs, bg_data_segs; + int tot_blks, data_blks, node_blks; + int bg_data_blks, bg_node_blks; + int curseg[NR_CURSEG_TYPE]; + int cursec[NR_CURSEG_TYPE]; + int curzone[NR_CURSEG_TYPE]; + + unsigned int segment_count[2]; + unsigned int block_count[2]; + unsigned int inplace_count; + unsigned long long base_mem, cache_mem, page_mem; +}; + +static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_stat_info *)sbi->stat_info; +} + +#define stat_inc_cp_count(si) ((si)->cp_count++) +#define stat_inc_call_count(si) ((si)->call_count++) +#define stat_inc_bggc_count(sbi) ((sbi)->bg_gc++) +#define stat_inc_dirty_dir(sbi) ((sbi)->n_dirty_dirs++) +#define stat_dec_dirty_dir(sbi) ((sbi)->n_dirty_dirs--) +#define stat_inc_total_hit(sbi) (atomic64_inc(&(sbi)->total_hit_ext)) +#define stat_inc_rbtree_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_rbtree)) +#define stat_inc_largest_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_largest)) +#define stat_inc_cached_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_cached)) +#define stat_inc_inline_xattr(inode) \ + do { \ + if (f2fs_has_inline_xattr(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->inline_xattr)); \ + } while (0) +#define stat_dec_inline_xattr(inode) \ + do { \ + if (f2fs_has_inline_xattr(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->inline_xattr)); \ + } while (0) +#define stat_inc_inline_inode(inode) \ + do { \ + if (f2fs_has_inline_data(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->inline_inode)); \ + } while (0) +#define stat_dec_inline_inode(inode) \ + do { \ + if (f2fs_has_inline_data(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->inline_inode)); \ + } while (0) +#define stat_inc_inline_dir(inode) \ + do { \ + if (f2fs_has_inline_dentry(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->inline_dir)); \ + } while (0) +#define stat_dec_inline_dir(inode) \ + do { \ + if (f2fs_has_inline_dentry(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->inline_dir)); \ + } while (0) +#define stat_inc_seg_type(sbi, curseg) \ + ((sbi)->segment_count[(curseg)->alloc_type]++) +#define stat_inc_block_count(sbi, curseg) \ + ((sbi)->block_count[(curseg)->alloc_type]++) +#define stat_inc_inplace_blocks(sbi) \ + (atomic_inc(&(sbi)->inplace_count)) +#define stat_inc_seg_count(sbi, type, gc_type) \ + do { \ + struct f2fs_stat_info *si = F2FS_STAT(sbi); \ + (si)->tot_segs++; \ + if (type == SUM_TYPE_DATA) { \ + si->data_segs++; \ + si->bg_data_segs += (gc_type == BG_GC) ? 1 : 0; \ + } else { \ + si->node_segs++; \ + si->bg_node_segs += (gc_type == BG_GC) ? 1 : 0; \ + } \ + } while (0) + +#define stat_inc_tot_blk_count(si, blks) \ + (si->tot_blks += (blks)) + +#define stat_inc_data_blk_count(sbi, blks, gc_type) \ + do { \ + struct f2fs_stat_info *si = F2FS_STAT(sbi); \ + stat_inc_tot_blk_count(si, blks); \ + si->data_blks += (blks); \ + si->bg_data_blks += (gc_type == BG_GC) ? (blks) : 0; \ + } while (0) + +#define stat_inc_node_blk_count(sbi, blks, gc_type) \ + do { \ + struct f2fs_stat_info *si = F2FS_STAT(sbi); \ + stat_inc_tot_blk_count(si, blks); \ + si->node_blks += (blks); \ + si->bg_node_blks += (gc_type == BG_GC) ? (blks) : 0; \ + } while (0) + +int f2fs_build_stats(struct f2fs_sb_info *); +void f2fs_destroy_stats(struct f2fs_sb_info *); +void __init f2fs_create_root_stats(void); +void f2fs_destroy_root_stats(void); +#else +#define stat_inc_cp_count(si) +#define stat_inc_call_count(si) +#define stat_inc_bggc_count(si) +#define stat_inc_dirty_dir(sbi) +#define stat_dec_dirty_dir(sbi) +#define stat_inc_total_hit(sb) +#define stat_inc_rbtree_node_hit(sb) +#define stat_inc_largest_node_hit(sbi) +#define stat_inc_cached_node_hit(sbi) +#define stat_inc_inline_xattr(inode) +#define stat_dec_inline_xattr(inode) +#define stat_inc_inline_inode(inode) +#define stat_dec_inline_inode(inode) +#define stat_inc_inline_dir(inode) +#define stat_dec_inline_dir(inode) +#define stat_inc_seg_type(sbi, curseg) +#define stat_inc_block_count(sbi, curseg) +#define stat_inc_inplace_blocks(sbi) +#define stat_inc_seg_count(sbi, type, gc_type) +#define stat_inc_tot_blk_count(si, blks) +#define stat_inc_data_blk_count(sbi, blks, gc_type) +#define stat_inc_node_blk_count(sbi, blks, gc_type) + +static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; } +static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { } +static inline void __init f2fs_create_root_stats(void) { } +static inline void f2fs_destroy_root_stats(void) { } +#endif + +extern const struct file_operations f2fs_dir_operations; +extern const struct file_operations f2fs_file_operations; +extern const struct inode_operations f2fs_file_inode_operations; +extern const struct address_space_operations f2fs_dblock_aops; +extern const struct address_space_operations f2fs_node_aops; +extern const struct address_space_operations f2fs_meta_aops; +extern const struct inode_operations f2fs_dir_inode_operations; +extern const struct inode_operations f2fs_symlink_inode_operations; +extern const struct inode_operations f2fs_encrypted_symlink_inode_operations; +extern const struct inode_operations f2fs_special_inode_operations; +extern struct kmem_cache *inode_entry_slab; + +/* + * inline.c + */ +bool f2fs_may_inline_data(struct inode *); +bool f2fs_may_inline_dentry(struct inode *); +void read_inline_data(struct page *, struct page *); +bool truncate_inline_inode(struct page *, u64); +int f2fs_read_inline_data(struct inode *, struct page *); +int f2fs_convert_inline_page(struct dnode_of_data *, struct page *); +int f2fs_convert_inline_inode(struct inode *); +int f2fs_write_inline_data(struct inode *, struct page *); +bool recover_inline_data(struct inode *, struct page *); +struct f2fs_dir_entry *find_in_inline_dir(struct inode *, + struct f2fs_filename *, struct page **); +struct f2fs_dir_entry *f2fs_parent_inline_dir(struct inode *, struct page **); +int make_empty_inline_dir(struct inode *inode, struct inode *, struct page *); +int f2fs_add_inline_entry(struct inode *, const struct qstr *, struct inode *, + nid_t, umode_t); +void f2fs_delete_inline_entry(struct f2fs_dir_entry *, struct page *, + struct inode *, struct inode *); +bool f2fs_empty_inline_dir(struct inode *); +int f2fs_read_inline_dir(struct file *, void *, filldir_t, struct f2fs_str *); +int f2fs_inline_data_fiemap(struct inode *, + struct fiemap_extent_info *, __u64, __u64); + +/* + * shrinker.c + */ +int f2fs_shrink_count(struct shrinker *, struct shrink_control *); +int f2fs_shrink_scan(struct shrinker *, struct shrink_control *); +void f2fs_join_shrinker(struct f2fs_sb_info *); +void f2fs_leave_shrinker(struct f2fs_sb_info *); + +/* + * extent_cache.c + */ +unsigned int f2fs_shrink_extent_tree(struct f2fs_sb_info *, int); +void f2fs_drop_largest_extent(struct inode *, pgoff_t); +void f2fs_init_extent_tree(struct inode *, struct f2fs_extent *); +unsigned int f2fs_destroy_extent_node(struct inode *); +void f2fs_destroy_extent_tree(struct inode *); +bool f2fs_lookup_extent_cache(struct inode *, pgoff_t, struct extent_info *); +void f2fs_update_extent_cache(struct dnode_of_data *); +void f2fs_update_extent_cache_range(struct dnode_of_data *dn, + pgoff_t, block_t, unsigned int); +void init_extent_cache_info(struct f2fs_sb_info *); +int __init create_extent_cache(void); +void destroy_extent_cache(void); + +/* + * crypto support + */ +static inline int f2fs_encrypted_inode(struct inode *inode) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + return file_is_encrypt(inode); +#else + return 0; +#endif +} + +static inline void f2fs_set_encrypted_inode(struct inode *inode) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + file_set_encrypt(inode); +#endif +} + +static inline bool f2fs_bio_encrypted(struct bio *bio) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + return unlikely(bio->bi_private != NULL); +#else + return false; +#endif +} + +static inline int f2fs_sb_has_crypto(struct super_block *sb) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_ENCRYPT); +#else + return 0; +#endif +} + +static inline bool f2fs_may_encrypt(struct inode *inode) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + mode_t mode = inode->i_mode; + + return (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)); +#else + return 0; +#endif +} + +/* crypto_policy.c */ +int f2fs_is_child_context_consistent_with_parent(struct inode *, + struct inode *); +int f2fs_inherit_context(struct inode *, struct inode *, struct page *); +int f2fs_process_policy(const struct f2fs_encryption_policy *, struct inode *); +int f2fs_get_policy(struct inode *, struct f2fs_encryption_policy *); + +/* crypt.c */ +extern struct kmem_cache *f2fs_crypt_info_cachep; +bool f2fs_valid_contents_enc_mode(uint32_t); +uint32_t f2fs_validate_encryption_key_size(uint32_t, uint32_t); +struct f2fs_crypto_ctx *f2fs_get_crypto_ctx(struct inode *); +void f2fs_release_crypto_ctx(struct f2fs_crypto_ctx *); +struct page *f2fs_encrypt(struct inode *, struct page *); +int f2fs_decrypt(struct f2fs_crypto_ctx *, struct page *); +int f2fs_decrypt_one(struct inode *, struct page *); +void f2fs_end_io_crypto_work(struct f2fs_crypto_ctx *, struct bio *); + +/* crypto_key.c */ +void f2fs_free_encryption_info(struct inode *, struct f2fs_crypt_info *); +int _f2fs_get_encryption_info(struct inode *inode); + +/* crypto_fname.c */ +bool f2fs_valid_filenames_enc_mode(uint32_t); +u32 f2fs_fname_crypto_round_up(u32, u32); +int f2fs_fname_crypto_alloc_buffer(struct inode *, u32, struct f2fs_str *); +int f2fs_fname_disk_to_usr(struct inode *, f2fs_hash_t *, + const struct f2fs_str *, struct f2fs_str *); +int f2fs_fname_usr_to_disk(struct inode *, const struct qstr *, + struct f2fs_str *); + +#ifdef CONFIG_F2FS_FS_ENCRYPTION +void f2fs_restore_and_release_control_page(struct page **); +void f2fs_restore_control_page(struct page *); + +int __init f2fs_init_crypto(void); +int f2fs_crypto_initialize(void); +void f2fs_exit_crypto(void); + +int f2fs_has_encryption_key(struct inode *); + +static inline int f2fs_get_encryption_info(struct inode *inode) +{ + struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info; + + if (!ci || + (ci->ci_keyring_key && + (ci->ci_keyring_key->flags & ((1 << KEY_FLAG_INVALIDATED) | + (1 << KEY_FLAG_REVOKED) | + (1 << KEY_FLAG_DEAD))))) + return _f2fs_get_encryption_info(inode); + return 0; +} + +void f2fs_fname_crypto_free_buffer(struct f2fs_str *); +int f2fs_fname_setup_filename(struct inode *, const struct qstr *, + int lookup, struct f2fs_filename *); +void f2fs_fname_free_filename(struct f2fs_filename *); +#else +static inline void f2fs_restore_and_release_control_page(struct page **p) { } +static inline void f2fs_restore_control_page(struct page *p) { } + +static inline int __init f2fs_init_crypto(void) { return 0; } +static inline void f2fs_exit_crypto(void) { } + +static inline int f2fs_has_encryption_key(struct inode *i) { return 0; } +static inline int f2fs_get_encryption_info(struct inode *i) { return 0; } +static inline void f2fs_fname_crypto_free_buffer(struct f2fs_str *p) { } + +static inline int f2fs_fname_setup_filename(struct inode *dir, + const struct qstr *iname, + int lookup, struct f2fs_filename *fname) +{ + memset(fname, 0, sizeof(struct f2fs_filename)); + fname->usr_fname = iname; + fname->disk_name.name = (unsigned char *)iname->name; + fname->disk_name.len = iname->len; + return 0; +} + +static inline void f2fs_fname_free_filename(struct f2fs_filename *fname) { } +#endif +#endif diff --git a/fs/f2fs/f2fs_crypto.h b/fs/f2fs/f2fs_crypto.h new file mode 100644 index 00000000000..c2c1c2b63b2 --- /dev/null +++ b/fs/f2fs/f2fs_crypto.h @@ -0,0 +1,151 @@ +/* + * linux/fs/f2fs/f2fs_crypto.h + * + * Copied from linux/fs/ext4/ext4_crypto.h + * + * Copyright (C) 2015, Google, Inc. + * + * This contains encryption header content for f2fs + * + * Written by Michael Halcrow, 2015. + * Modified by Jaegeuk Kim, 2015. + */ +#ifndef _F2FS_CRYPTO_H +#define _F2FS_CRYPTO_H + +#include + +#define F2FS_KEY_DESCRIPTOR_SIZE 8 + +/* Policy provided via an ioctl on the topmost directory */ +struct f2fs_encryption_policy { + char version; + char contents_encryption_mode; + char filenames_encryption_mode; + char flags; + char master_key_descriptor[F2FS_KEY_DESCRIPTOR_SIZE]; +} __attribute__((__packed__)); + +#define F2FS_ENCRYPTION_CONTEXT_FORMAT_V1 1 +#define F2FS_KEY_DERIVATION_NONCE_SIZE 16 + +#define F2FS_POLICY_FLAGS_PAD_4 0x00 +#define F2FS_POLICY_FLAGS_PAD_8 0x01 +#define F2FS_POLICY_FLAGS_PAD_16 0x02 +#define F2FS_POLICY_FLAGS_PAD_32 0x03 +#define F2FS_POLICY_FLAGS_PAD_MASK 0x03 +#define F2FS_POLICY_FLAGS_VALID 0x03 + +/** + * Encryption context for inode + * + * Protector format: + * 1 byte: Protector format (1 = this version) + * 1 byte: File contents encryption mode + * 1 byte: File names encryption mode + * 1 byte: Flags + * 8 bytes: Master Key descriptor + * 16 bytes: Encryption Key derivation nonce + */ +struct f2fs_encryption_context { + char format; + char contents_encryption_mode; + char filenames_encryption_mode; + char flags; + char master_key_descriptor[F2FS_KEY_DESCRIPTOR_SIZE]; + char nonce[F2FS_KEY_DERIVATION_NONCE_SIZE]; +} __attribute__((__packed__)); + +/* Encryption parameters */ +#define F2FS_XTS_TWEAK_SIZE 16 +#define F2FS_AES_128_ECB_KEY_SIZE 16 +#define F2FS_AES_256_GCM_KEY_SIZE 32 +#define F2FS_AES_256_CBC_KEY_SIZE 32 +#define F2FS_AES_256_CTS_KEY_SIZE 32 +#define F2FS_AES_256_XTS_KEY_SIZE 64 +#define F2FS_MAX_KEY_SIZE 64 + +#define F2FS_KEY_DESC_PREFIX "f2fs:" +#define F2FS_KEY_DESC_PREFIX_SIZE 5 + +struct f2fs_encryption_key { + __u32 mode; + char raw[F2FS_MAX_KEY_SIZE]; + __u32 size; +} __attribute__((__packed__)); + +struct f2fs_crypt_info { + char ci_data_mode; + char ci_filename_mode; + char ci_flags; + struct crypto_ablkcipher *ci_ctfm; + struct key *ci_keyring_key; + char ci_master_key[F2FS_KEY_DESCRIPTOR_SIZE]; +}; + +#define F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL 0x00000001 +#define F2FS_WRITE_PATH_FL 0x00000002 + +struct f2fs_crypto_ctx { + union { + struct { + struct page *bounce_page; /* Ciphertext page */ + struct page *control_page; /* Original page */ + } w; + struct { + struct bio *bio; + struct work_struct work; + } r; + struct list_head free_list; /* Free list */ + }; + char flags; /* Flags */ +}; + +struct f2fs_completion_result { + struct completion completion; + int res; +}; + +#define DECLARE_F2FS_COMPLETION_RESULT(ecr) \ + struct f2fs_completion_result ecr = { \ + COMPLETION_INITIALIZER((ecr).completion), 0 } + +static inline int f2fs_encryption_key_size(int mode) +{ + switch (mode) { + case F2FS_ENCRYPTION_MODE_AES_256_XTS: + return F2FS_AES_256_XTS_KEY_SIZE; + case F2FS_ENCRYPTION_MODE_AES_256_GCM: + return F2FS_AES_256_GCM_KEY_SIZE; + case F2FS_ENCRYPTION_MODE_AES_256_CBC: + return F2FS_AES_256_CBC_KEY_SIZE; + case F2FS_ENCRYPTION_MODE_AES_256_CTS: + return F2FS_AES_256_CTS_KEY_SIZE; + default: + BUG(); + } + return 0; +} + +#define F2FS_FNAME_NUM_SCATTER_ENTRIES 4 +#define F2FS_CRYPTO_BLOCK_SIZE 16 +#define F2FS_FNAME_CRYPTO_DIGEST_SIZE 32 + +/** + * For encrypted symlinks, the ciphertext length is stored at the beginning + * of the string in little-endian format. + */ +struct f2fs_encrypted_symlink_data { + __le16 len; + char encrypted_path[1]; +} __attribute__((__packed__)); + +/** + * This function is used to calculate the disk space required to + * store a filename of length l in encrypted symlink format. + */ +static inline u32 encrypted_symlink_data_len(u32 l) +{ + return (l + sizeof(struct f2fs_encrypted_symlink_data) - 1); +} +#endif /* _F2FS_CRYPTO_H */ diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c new file mode 100644 index 00000000000..30a8d8cbe8f --- /dev/null +++ b/fs/f2fs/file.c @@ -0,0 +1,1741 @@ +/* + * fs/f2fs/file.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" +#include "acl.h" +#include "gc.h" +#include "trace.h" +#include + +static int f2fs_vm_page_mkwrite(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + struct page *page = vmf->page; + struct inode *inode = file_inode(vma->vm_file); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + int err; + + f2fs_balance_fs(sbi); + + vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); + + f2fs_bug_on(sbi, f2fs_has_inline_data(inode)); + + /* block allocation */ + f2fs_lock_op(sbi); + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_reserve_block(&dn, page->index); + if (err) { + f2fs_unlock_op(sbi); + goto out; + } + f2fs_put_dnode(&dn); + f2fs_unlock_op(sbi); + + file_update_time(vma->vm_file); + lock_page(page); + if (unlikely(page->mapping != inode->i_mapping || + page_offset(page) > i_size_read(inode) || + !PageUptodate(page))) { + unlock_page(page); + err = -EFAULT; + goto out; + } + + /* + * check to see if the page is mapped already (no holes) + */ + if (PageMappedToDisk(page)) + goto mapped; + + /* page is wholly or partially inside EOF */ + if (((loff_t)(page->index + 1) << PAGE_CACHE_SHIFT) > + i_size_read(inode)) { + unsigned offset; + offset = i_size_read(inode) & ~PAGE_CACHE_MASK; + zero_user_segment(page, offset, PAGE_CACHE_SIZE); + } + set_page_dirty(page); + SetPageUptodate(page); + + trace_f2fs_vm_page_mkwrite(page, DATA); +mapped: + /* fill the page */ + f2fs_wait_on_page_writeback(page, DATA); + + /* wait for GCed encrypted page writeback */ + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) + f2fs_wait_on_encrypted_page_writeback(sbi, dn.data_blkaddr); + + /* if gced page is attached, don't write to cold segment */ + clear_cold_data(page); +out: + return block_page_mkwrite_return(err); +} + +static const struct vm_operations_struct f2fs_file_vm_ops = { + .fault = filemap_fault, + .page_mkwrite = f2fs_vm_page_mkwrite, +}; + +static int get_parent_ino(struct inode *inode, nid_t *pino) +{ + struct dentry *dentry; + + inode = igrab(inode); + dentry = d_find_any_alias(inode); + iput(inode); + if (!dentry) + return 0; + + if (update_dent_inode(inode, inode, &dentry->d_name)) { + dput(dentry); + return 0; + } + + *pino = parent_ino(dentry); + dput(dentry); + return 1; +} + +static inline bool need_do_checkpoint(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + bool need_cp = false; + + if (!S_ISREG(inode->i_mode) || inode->i_nlink != 1) + need_cp = true; + else if (file_enc_name(inode) && need_dentry_mark(sbi, inode->i_ino)) + need_cp = true; + else if (file_wrong_pino(inode)) + need_cp = true; + else if (!space_for_roll_forward(sbi)) + need_cp = true; + else if (!is_checkpointed_node(sbi, F2FS_I(inode)->i_pino)) + need_cp = true; + else if (F2FS_I(inode)->xattr_ver == cur_cp_version(F2FS_CKPT(sbi))) + need_cp = true; + else if (test_opt(sbi, FASTBOOT)) + need_cp = true; + else if (sbi->active_logs == 2) + need_cp = true; + + return need_cp; +} + +static bool need_inode_page_update(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct page *i = find_get_page(NODE_MAPPING(sbi), ino); + bool ret = false; + /* But we need to avoid that there are some inode updates */ + if ((i && PageDirty(i)) || need_inode_block_update(sbi, ino)) + ret = true; + f2fs_put_page(i, 0); + return ret; +} + +static void try_to_fix_pino(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + nid_t pino; + + down_write(&fi->i_sem); + fi->xattr_ver = 0; + if (file_wrong_pino(inode) && inode->i_nlink == 1 && + get_parent_ino(inode, &pino)) { + fi->i_pino = pino; + file_got_pino(inode); + up_write(&fi->i_sem); + + mark_inode_dirty_sync(inode); + f2fs_write_inode(inode, NULL); + } else { + up_write(&fi->i_sem); + } +} + +int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t ino = inode->i_ino; + int ret = 0; + bool need_cp = false; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + .for_reclaim = 0, + }; + + if (unlikely(f2fs_readonly(inode->i_sb))) + return 0; + + trace_f2fs_sync_file_enter(inode); + + /* if fdatasync is triggered, let's do in-place-update */ + if (get_dirty_pages(inode) <= SM_I(sbi)->min_fsync_blocks) + set_inode_flag(fi, FI_NEED_IPU); + ret = filemap_write_and_wait_range(inode->i_mapping, start, end); + clear_inode_flag(fi, FI_NEED_IPU); + + if (ret) { + trace_f2fs_sync_file_exit(inode, need_cp, datasync, ret); + return ret; + } + + /* if the inode is dirty, let's recover all the time */ + if (!datasync) { + f2fs_write_inode(inode, NULL); + goto go_write; + } + + /* + * if there is no written data, don't waste time to write recovery info. + */ + if (!is_inode_flag_set(fi, FI_APPEND_WRITE) && + !exist_written_data(sbi, ino, APPEND_INO)) { + + /* it may call write_inode just prior to fsync */ + if (need_inode_page_update(sbi, ino)) + goto go_write; + + if (is_inode_flag_set(fi, FI_UPDATE_WRITE) || + exist_written_data(sbi, ino, UPDATE_INO)) + goto flush_out; + goto out; + } +go_write: + /* guarantee free sections for fsync */ + f2fs_balance_fs(sbi); + + /* + * Both of fdatasync() and fsync() are able to be recovered from + * sudden-power-off. + */ + down_read(&fi->i_sem); + need_cp = need_do_checkpoint(inode); + up_read(&fi->i_sem); + + if (need_cp) { + /* all the dirty node pages should be flushed for POR */ + ret = f2fs_sync_fs(inode->i_sb, 1); + + /* + * We've secured consistency through sync_fs. Following pino + * will be used only for fsynced inodes after checkpoint. + */ + try_to_fix_pino(inode); + clear_inode_flag(fi, FI_APPEND_WRITE); + clear_inode_flag(fi, FI_UPDATE_WRITE); + goto out; + } +sync_nodes: + sync_node_pages(sbi, ino, &wbc); + + /* if cp_error was enabled, we should avoid infinite loop */ + if (unlikely(f2fs_cp_error(sbi))) + goto out; + + if (need_inode_block_update(sbi, ino)) { + mark_inode_dirty_sync(inode); + f2fs_write_inode(inode, NULL); + goto sync_nodes; + } + + ret = wait_on_node_pages_writeback(sbi, ino); + if (ret) + goto out; + + /* once recovery info is written, don't need to tack this */ + remove_dirty_inode(sbi, ino, APPEND_INO); + clear_inode_flag(fi, FI_APPEND_WRITE); +flush_out: + remove_dirty_inode(sbi, ino, UPDATE_INO); + clear_inode_flag(fi, FI_UPDATE_WRITE); + ret = f2fs_issue_flush(sbi); +out: + trace_f2fs_sync_file_exit(inode, need_cp, datasync, ret); + f2fs_trace_ios(NULL, 1); + return ret; +} + +static pgoff_t __get_first_dirty_index(struct address_space *mapping, + pgoff_t pgofs, int whence) +{ + struct pagevec pvec; + int nr_pages; + + if (whence != SEEK_DATA) + return 0; + + /* find first dirty page index */ + pagevec_init(&pvec, 0); + nr_pages = pagevec_lookup_tag(&pvec, mapping, &pgofs, + PAGECACHE_TAG_DIRTY, 1); + pgofs = nr_pages ? pvec.pages[0]->index : LONG_MAX; + pagevec_release(&pvec); + return pgofs; +} + +static bool __found_offset(block_t blkaddr, pgoff_t dirty, pgoff_t pgofs, + int whence) +{ + switch (whence) { + case SEEK_DATA: + if ((blkaddr == NEW_ADDR && dirty == pgofs) || + (blkaddr != NEW_ADDR && blkaddr != NULL_ADDR)) + return true; + break; + case SEEK_HOLE: + if (blkaddr == NULL_ADDR) + return true; + break; + } + return false; +} + +static inline int unsigned_offsets(struct file *file) +{ + return file->f_mode & FMODE_UNSIGNED_OFFSET; +} + +static loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize) +{ + if (offset < 0 && !unsigned_offsets(file)) + return -EINVAL; + if (offset > maxsize) + return -EINVAL; + + if (offset != file->f_pos) { + file->f_pos = offset; + file->f_version = 0; + } + return offset; +} + +static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxbytes = inode->i_sb->s_maxbytes; + struct dnode_of_data dn; + pgoff_t pgofs, end_offset, dirty; + loff_t data_ofs = offset; + loff_t isize; + int err = 0; + + mutex_lock(&inode->i_mutex); + + isize = i_size_read(inode); + if (offset >= isize) + goto fail; + + /* handle inline data case */ + if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) { + if (whence == SEEK_HOLE) + data_ofs = isize; + goto found; + } + + pgofs = (pgoff_t)(offset >> PAGE_CACHE_SHIFT); + + dirty = __get_first_dirty_index(inode->i_mapping, pgofs, whence); + + for (; data_ofs < isize; data_ofs = (loff_t)pgofs << PAGE_CACHE_SHIFT) { + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, pgofs, LOOKUP_NODE_RA); + if (err && err != -ENOENT) { + goto fail; + } else if (err == -ENOENT) { + /* direct node does not exists */ + if (whence == SEEK_DATA) { + pgofs = PGOFS_OF_NEXT_DNODE(pgofs, + F2FS_I(inode)); + continue; + } else { + goto found; + } + } + + end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); + + /* find data/hole in dnode block */ + for (; dn.ofs_in_node < end_offset; + dn.ofs_in_node++, pgofs++, + data_ofs = (loff_t)pgofs << PAGE_CACHE_SHIFT) { + block_t blkaddr; + blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); + + if (__found_offset(blkaddr, dirty, pgofs, whence)) { + f2fs_put_dnode(&dn); + goto found; + } + } + f2fs_put_dnode(&dn); + } + + if (whence == SEEK_DATA) + goto fail; +found: + if (whence == SEEK_HOLE && data_ofs > isize) + data_ofs = isize; + mutex_unlock(&inode->i_mutex); + return vfs_setpos(file, data_ofs, maxbytes); +fail: + mutex_unlock(&inode->i_mutex); + return -ENXIO; +} + +static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxbytes = inode->i_sb->s_maxbytes; + + switch (whence) { + case SEEK_SET: + case SEEK_CUR: + case SEEK_END: + return generic_file_llseek_size(file, offset, whence, + maxbytes); + case SEEK_DATA: + case SEEK_HOLE: + if (offset < 0) + return -ENXIO; + return f2fs_seek_block(file, offset, whence); + } + + return -EINVAL; +} + +static int f2fs_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct inode *inode = file_inode(file); + + if (f2fs_encrypted_inode(inode)) { + int err = f2fs_get_encryption_info(inode); + if (err) + return 0; + } + + /* we don't need to use inline_data strictly */ + if (f2fs_has_inline_data(inode)) { + int err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + file_accessed(file); + vma->vm_ops = &f2fs_file_vm_ops; + return 0; +} + +static int f2fs_file_open(struct inode *inode, struct file *filp) +{ + int ret = generic_file_open(inode, filp); + + if (!ret && f2fs_encrypted_inode(inode)) { + ret = f2fs_get_encryption_info(inode); + if (ret) + ret = -EACCES; + } + return ret; +} + +int truncate_data_blocks_range(struct dnode_of_data *dn, int count) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct f2fs_node *raw_node; + int nr_free = 0, ofs = dn->ofs_in_node, len = count; + __le32 *addr; + + raw_node = F2FS_NODE(dn->node_page); + addr = blkaddr_in_node(raw_node) + ofs; + + for (; count > 0; count--, addr++, dn->ofs_in_node++) { + block_t blkaddr = le32_to_cpu(*addr); + if (blkaddr == NULL_ADDR) + continue; + + dn->data_blkaddr = NULL_ADDR; + set_data_blkaddr(dn); + invalidate_blocks(sbi, blkaddr); + if (dn->ofs_in_node == 0 && IS_INODE(dn->node_page)) + clear_inode_flag(F2FS_I(dn->inode), + FI_FIRST_BLOCK_WRITTEN); + nr_free++; + } + + if (nr_free) { + pgoff_t fofs; + /* + * once we invalidate valid blkaddr in range [ofs, ofs + count], + * we will invalidate all blkaddr in the whole range. + */ + fofs = start_bidx_of_node(ofs_of_node(dn->node_page), + F2FS_I(dn->inode)) + ofs; + f2fs_update_extent_cache_range(dn, fofs, 0, len); + dec_valid_block_count(sbi, dn->inode, nr_free); + set_page_dirty(dn->node_page); + sync_inode_page(dn); + } + dn->ofs_in_node = ofs; + + trace_f2fs_truncate_data_blocks_range(dn->inode, dn->nid, + dn->ofs_in_node, nr_free); + return nr_free; +} + +void truncate_data_blocks(struct dnode_of_data *dn) +{ + truncate_data_blocks_range(dn, ADDRS_PER_BLOCK); +} + +static int truncate_partial_data_page(struct inode *inode, u64 from, + bool cache_only) +{ + unsigned offset = from & (PAGE_CACHE_SIZE - 1); + pgoff_t index = from >> PAGE_CACHE_SHIFT; + struct address_space *mapping = inode->i_mapping; + struct page *page; + + if (!offset && !cache_only) + return 0; + + if (cache_only) { + page = f2fs_grab_cache_page(mapping, index, false); + if (page && PageUptodate(page)) + goto truncate_out; + f2fs_put_page(page, 1); + return 0; + } + + page = get_lock_data_page(inode, index, true); + if (IS_ERR(page)) + return 0; +truncate_out: + f2fs_wait_on_page_writeback(page, DATA); + zero_user(page, offset, PAGE_CACHE_SIZE - offset); + if (!cache_only || !f2fs_encrypted_inode(inode) || !S_ISREG(inode->i_mode)) + set_page_dirty(page); + f2fs_put_page(page, 1); + return 0; +} + +int truncate_blocks(struct inode *inode, u64 from, bool lock) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int blocksize = inode->i_sb->s_blocksize; + struct dnode_of_data dn; + pgoff_t free_from; + int count = 0, err = 0; + struct page *ipage; + bool truncate_page = false; + + trace_f2fs_truncate_blocks_enter(inode, from); + + free_from = (pgoff_t)F2FS_BYTES_TO_BLK(from + blocksize - 1); + + if (lock) + f2fs_lock_op(sbi); + + ipage = get_node_page(sbi, inode->i_ino); + if (IS_ERR(ipage)) { + err = PTR_ERR(ipage); + goto out; + } + + if (f2fs_has_inline_data(inode)) { + if (truncate_inline_inode(ipage, from)) + set_page_dirty(ipage); + f2fs_put_page(ipage, 1); + truncate_page = true; + goto out; + } + + set_new_dnode(&dn, inode, ipage, NULL, 0); + err = get_dnode_of_data(&dn, free_from, LOOKUP_NODE); + if (err) { + if (err == -ENOENT) + goto free_next; + goto out; + } + + count = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); + + count -= dn.ofs_in_node; + f2fs_bug_on(sbi, count < 0); + + if (dn.ofs_in_node || IS_INODE(dn.node_page)) { + truncate_data_blocks_range(&dn, count); + free_from += count; + } + + f2fs_put_dnode(&dn); +free_next: + err = truncate_inode_blocks(inode, free_from); +out: + if (lock) + f2fs_unlock_op(sbi); + + /* lastly zero out the first data page */ + if (!err) + err = truncate_partial_data_page(inode, from, truncate_page); + + trace_f2fs_truncate_blocks_exit(inode, err); + return err; +} + +int f2fs_truncate(struct inode *inode, bool lock) +{ + int err; + + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + return 0; + + trace_f2fs_truncate(inode); + + /* we should check inline_data size */ + if (f2fs_has_inline_data(inode) && !f2fs_may_inline_data(inode)) { + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + err = truncate_blocks(inode, i_size_read(inode), lock); + if (err) + return err; + + inode->i_mtime = inode->i_ctime = CURRENT_TIME; + mark_inode_dirty(inode); + return 0; +} + +int f2fs_getattr(struct vfsmount *mnt, + struct dentry *dentry, struct kstat *stat) +{ + struct inode *inode = dentry->d_inode; + generic_fillattr(inode, stat); + stat->blocks <<= 3; + return 0; +} + +#ifdef CONFIG_F2FS_FS_POSIX_ACL +static void __setattr_copy(struct inode *inode, const struct iattr *attr) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + unsigned int ia_valid = attr->ia_valid; + + if (ia_valid & ATTR_UID) + inode->i_uid = attr->ia_uid; + if (ia_valid & ATTR_GID) + inode->i_gid = attr->ia_gid; + if (ia_valid & ATTR_ATIME) + inode->i_atime = timespec_trunc(attr->ia_atime, + inode->i_sb->s_time_gran); + if (ia_valid & ATTR_MTIME) + inode->i_mtime = timespec_trunc(attr->ia_mtime, + inode->i_sb->s_time_gran); + if (ia_valid & ATTR_CTIME) + inode->i_ctime = timespec_trunc(attr->ia_ctime, + inode->i_sb->s_time_gran); + if (ia_valid & ATTR_MODE) { + umode_t mode = attr->ia_mode; + + if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) + mode &= ~S_ISGID; + set_acl_inode(fi, mode); + } +} +#else +#define __setattr_copy setattr_copy +#endif + +int f2fs_setattr(struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = dentry->d_inode; + struct f2fs_inode_info *fi = F2FS_I(inode); + int err; + + err = inode_change_ok(inode, attr); + if (err) + return err; + + if (attr->ia_valid & ATTR_SIZE) { + if (f2fs_encrypted_inode(inode) && + f2fs_get_encryption_info(inode)) + return -EACCES; + + if (attr->ia_size <= i_size_read(inode)) { + truncate_setsize(inode, attr->ia_size); + err = f2fs_truncate(inode, true); + if (err) + return err; + f2fs_balance_fs(F2FS_I_SB(inode)); + } else { + /* + * do not trim all blocks after i_size if target size is + * larger than i_size. + */ + truncate_setsize(inode, attr->ia_size); + inode->i_mtime = inode->i_ctime = CURRENT_TIME; + } + } + + __setattr_copy(inode, attr); + + if (attr->ia_valid & ATTR_MODE) { + err = f2fs_acl_chmod(inode); + if (err || is_inode_flag_set(fi, FI_ACL_MODE)) { + inode->i_mode = fi->i_acl_mode; + clear_inode_flag(fi, FI_ACL_MODE); + } + } + + mark_inode_dirty(inode); + return err; +} + +const struct inode_operations f2fs_file_inode_operations = { + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .get_acl = f2fs_get_acl, +#ifdef CONFIG_F2FS_FS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = f2fs_listxattr, + .removexattr = generic_removexattr, +#endif + .fiemap = f2fs_fiemap, +}; + +static int fill_zero(struct inode *inode, pgoff_t index, + loff_t start, loff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct page *page; + + if (!len) + return 0; + + f2fs_balance_fs(sbi); + + f2fs_lock_op(sbi); + page = get_new_data_page(inode, NULL, index, false); + f2fs_unlock_op(sbi); + + if (IS_ERR(page)) + return PTR_ERR(page); + + f2fs_wait_on_page_writeback(page, DATA); + zero_user(page, start, len); + set_page_dirty(page); + f2fs_put_page(page, 1); + return 0; +} + +int truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end) +{ + int err; + + while (pg_start < pg_end) { + struct dnode_of_data dn; + pgoff_t end_offset, count; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, pg_start, LOOKUP_NODE); + if (err) { + if (err == -ENOENT) { + pg_start++; + continue; + } + return err; + } + + end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); + count = min(end_offset - dn.ofs_in_node, pg_end - pg_start); + + f2fs_bug_on(F2FS_I_SB(inode), count == 0 || count > end_offset); + + truncate_data_blocks_range(&dn, count); + f2fs_put_dnode(&dn); + + pg_start += count; + } + return 0; +} + +static int punch_hole(struct inode *inode, loff_t offset, loff_t len) +{ + pgoff_t pg_start, pg_end; + loff_t off_start, off_end; + int ret = 0; + + if (f2fs_has_inline_data(inode)) { + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + } + + pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT; + pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT; + + off_start = offset & (PAGE_CACHE_SIZE - 1); + off_end = (offset + len) & (PAGE_CACHE_SIZE - 1); + + if (pg_start == pg_end) { + ret = fill_zero(inode, pg_start, off_start, + off_end - off_start); + if (ret) + return ret; + } else { + if (off_start) { + ret = fill_zero(inode, pg_start++, off_start, + PAGE_CACHE_SIZE - off_start); + if (ret) + return ret; + } + if (off_end) { + ret = fill_zero(inode, pg_end, 0, off_end); + if (ret) + return ret; + } + + if (pg_start < pg_end) { + struct address_space *mapping = inode->i_mapping; + loff_t blk_start, blk_end; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + f2fs_balance_fs(sbi); + + blk_start = (loff_t)pg_start << PAGE_CACHE_SHIFT; + blk_end = (loff_t)pg_end << PAGE_CACHE_SHIFT; + truncate_inode_pages_range(mapping, blk_start, + blk_end - 1); + + f2fs_lock_op(sbi); + ret = truncate_hole(inode, pg_start, pg_end); + f2fs_unlock_op(sbi); + } + } + + return ret; +} + +static int __exchange_data_block(struct inode *inode, pgoff_t src, + pgoff_t dst, bool full) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + block_t new_addr; + bool do_replace = false; + int ret; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = get_dnode_of_data(&dn, src, LOOKUP_NODE_RA); + if (ret && ret != -ENOENT) { + return ret; + } else if (ret == -ENOENT) { + new_addr = NULL_ADDR; + } else { + new_addr = dn.data_blkaddr; + if (!is_checkpointed_data(sbi, new_addr)) { + dn.data_blkaddr = NULL_ADDR; + /* do not invalidate this block address */ + set_data_blkaddr(&dn); + f2fs_update_extent_cache(&dn); + do_replace = true; + } + f2fs_put_dnode(&dn); + } + + if (new_addr == NULL_ADDR) + return full ? truncate_hole(inode, dst, dst + 1) : 0; + + if (do_replace) { + struct page *ipage = get_node_page(sbi, inode->i_ino); + struct node_info ni; + + if (IS_ERR(ipage)) { + ret = PTR_ERR(ipage); + goto err_out; + } + + set_new_dnode(&dn, inode, ipage, NULL, 0); + ret = f2fs_reserve_block(&dn, dst); + if (ret) + goto err_out; + + truncate_data_blocks_range(&dn, 1); + + get_node_info(sbi, dn.nid, &ni); + f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr, + ni.version, true); + f2fs_put_dnode(&dn); + } else { + struct page *psrc, *pdst; + + psrc = get_lock_data_page(inode, src, true); + if (IS_ERR(psrc)) + return PTR_ERR(psrc); + pdst = get_new_data_page(inode, NULL, dst, false); + if (IS_ERR(pdst)) { + f2fs_put_page(psrc, 1); + return PTR_ERR(pdst); + } + f2fs_copy_page(psrc, pdst); + set_page_dirty(pdst); + f2fs_put_page(pdst, 1); + f2fs_put_page(psrc, 1); + + return truncate_hole(inode, src, src + 1); + } + return 0; + +err_out: + if (!get_dnode_of_data(&dn, src, LOOKUP_NODE)) { + dn.data_blkaddr = new_addr; + set_data_blkaddr(&dn); + f2fs_update_extent_cache(&dn); + f2fs_put_dnode(&dn); + } + return ret; +} + +static int f2fs_do_collapse(struct inode *inode, pgoff_t start, pgoff_t end) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + pgoff_t nrpages = (i_size_read(inode) + PAGE_SIZE - 1) / PAGE_SIZE; + int ret = 0; + + for (; end < nrpages; start++, end++) { + f2fs_balance_fs(sbi); + f2fs_lock_op(sbi); + ret = __exchange_data_block(inode, end, start, true); + f2fs_unlock_op(sbi); + if (ret) + break; + } + return ret; +} + +static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) +{ + pgoff_t pg_start, pg_end; + loff_t new_size; + int ret; + + if (offset + len >= i_size_read(inode)) + return -EINVAL; + + /* collapse range should be aligned to block size of f2fs. */ + if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1)) + return -EINVAL; + + f2fs_balance_fs(F2FS_I_SB(inode)); + + if (f2fs_has_inline_data(inode)) { + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + } + + pg_start = offset >> PAGE_CACHE_SHIFT; + pg_end = (offset + len) >> PAGE_CACHE_SHIFT; + + /* write out all dirty pages from offset */ + ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + if (ret) + return ret; + + truncate_pagecache(inode, 0, offset); + + ret = f2fs_do_collapse(inode, pg_start, pg_end); + if (ret) + return ret; + + /* write out all moved pages, if possible */ + filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + truncate_pagecache(inode, 0, offset); + + new_size = i_size_read(inode) - len; + truncate_pagecache(inode, 0, new_size); + + ret = truncate_blocks(inode, new_size, true); + if (!ret) + i_size_write(inode, new_size); + + return ret; +} + +static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, + int mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct address_space *mapping = inode->i_mapping; + pgoff_t index, pg_start, pg_end; + loff_t new_size = i_size_read(inode); + loff_t off_start, off_end; + int ret = 0; + + ret = inode_newsize_ok(inode, (len + offset)); + if (ret) + return ret; + + f2fs_balance_fs(sbi); + + if (f2fs_has_inline_data(inode)) { + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + } + + ret = filemap_write_and_wait_range(mapping, offset, offset + len - 1); + if (ret) + return ret; + + truncate_pagecache_range(inode, offset, offset + len - 1); + + pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT; + pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT; + + off_start = offset & (PAGE_CACHE_SIZE - 1); + off_end = (offset + len) & (PAGE_CACHE_SIZE - 1); + + if (pg_start == pg_end) { + ret = fill_zero(inode, pg_start, off_start, + off_end - off_start); + if (ret) + return ret; + + if (offset + len > new_size) + new_size = offset + len; + new_size = max_t(loff_t, new_size, offset + len); + } else { + if (off_start) { + ret = fill_zero(inode, pg_start++, off_start, + PAGE_CACHE_SIZE - off_start); + if (ret) + return ret; + + new_size = max_t(loff_t, new_size, + (loff_t)pg_start << PAGE_CACHE_SHIFT); + } + + for (index = pg_start; index < pg_end; index++) { + struct dnode_of_data dn; + struct page *ipage; + + f2fs_lock_op(sbi); + + ipage = get_node_page(sbi, inode->i_ino); + if (IS_ERR(ipage)) { + ret = PTR_ERR(ipage); + f2fs_unlock_op(sbi); + goto out; + } + + set_new_dnode(&dn, inode, ipage, NULL, 0); + ret = f2fs_reserve_block(&dn, index); + if (ret) { + f2fs_unlock_op(sbi); + goto out; + } + + if (dn.data_blkaddr != NEW_ADDR) { + invalidate_blocks(sbi, dn.data_blkaddr); + + dn.data_blkaddr = NEW_ADDR; + set_data_blkaddr(&dn); + + dn.data_blkaddr = NULL_ADDR; + f2fs_update_extent_cache(&dn); + } + f2fs_put_dnode(&dn); + f2fs_unlock_op(sbi); + + new_size = max_t(loff_t, new_size, + (loff_t)(index + 1) << PAGE_CACHE_SHIFT); + } + + if (off_end) { + ret = fill_zero(inode, pg_end, 0, off_end); + if (ret) + goto out; + + new_size = max_t(loff_t, new_size, offset + len); + } + } + +out: + if (!(mode & FALLOC_FL_KEEP_SIZE) && i_size_read(inode) < new_size) { + i_size_write(inode, new_size); + mark_inode_dirty(inode); + update_inode_page(inode); + } + + return ret; +} + +static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + pgoff_t pg_start, pg_end, delta, nrpages, idx; + loff_t new_size; + int ret = 0; + + new_size = i_size_read(inode) + len; + if (new_size > inode->i_sb->s_maxbytes) + return -EFBIG; + + if (offset >= i_size_read(inode)) + return -EINVAL; + + /* insert range should be aligned to block size of f2fs. */ + if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1)) + return -EINVAL; + + f2fs_balance_fs(sbi); + + if (f2fs_has_inline_data(inode)) { + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + } + + ret = truncate_blocks(inode, i_size_read(inode), true); + if (ret) + return ret; + + /* write out all dirty pages from offset */ + ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + if (ret) + return ret; + + truncate_pagecache(inode, 0, offset); + + pg_start = offset >> PAGE_CACHE_SHIFT; + pg_end = (offset + len) >> PAGE_CACHE_SHIFT; + delta = pg_end - pg_start; + nrpages = (i_size_read(inode) + PAGE_SIZE - 1) / PAGE_SIZE; + + for (idx = nrpages - 1; idx >= pg_start && idx != -1; idx--) { + f2fs_lock_op(sbi); + ret = __exchange_data_block(inode, idx, idx + delta, false); + f2fs_unlock_op(sbi); + if (ret) + break; + } + + /* write out all moved pages, if possible */ + filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + truncate_pagecache(inode, 0, offset); + + if (!ret) + i_size_write(inode, new_size); + return ret; +} + +static int expand_inode_data(struct inode *inode, loff_t offset, + loff_t len, int mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + pgoff_t index, pg_start, pg_end; + loff_t new_size = i_size_read(inode); + loff_t off_start, off_end; + int ret = 0; + + f2fs_balance_fs(sbi); + + ret = inode_newsize_ok(inode, (len + offset)); + if (ret) + return ret; + + if (f2fs_has_inline_data(inode)) { + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + } + + pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT; + pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT; + + off_start = offset & (PAGE_CACHE_SIZE - 1); + off_end = (offset + len) & (PAGE_CACHE_SIZE - 1); + + f2fs_lock_op(sbi); + + for (index = pg_start; index <= pg_end; index++) { + struct dnode_of_data dn; + + if (index == pg_end && !off_end) + goto noalloc; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_reserve_block(&dn, index); + if (ret) + break; +noalloc: + if (pg_start == pg_end) + new_size = offset + len; + else if (index == pg_start && off_start) + new_size = (loff_t)(index + 1) << PAGE_CACHE_SHIFT; + else if (index == pg_end) + new_size = ((loff_t)index << PAGE_CACHE_SHIFT) + + off_end; + else + new_size += PAGE_CACHE_SIZE; + } + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + i_size_read(inode) < new_size) { + i_size_write(inode, new_size); + mark_inode_dirty(inode); + update_inode_page(inode); + } + f2fs_unlock_op(sbi); + + return ret; +} + +#define FALLOC_FL_COLLAPSE_RANGE 0X08 +#define FALLOC_FL_ZERO_RANGE 0X10 +#define FALLOC_FL_INSERT_RANGE 0X20 + +static long f2fs_fallocate(struct file *file, int mode, + loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + long ret = 0; + + /* f2fs only support ->fallocate for regular file */ + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (f2fs_encrypted_inode(inode) && + (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE))) + return -EOPNOTSUPP; + + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | + FALLOC_FL_INSERT_RANGE)) + return -EOPNOTSUPP; + + mutex_lock(&inode->i_mutex); + + if (mode & FALLOC_FL_PUNCH_HOLE) { + if (offset >= inode->i_size) + goto out; + + ret = punch_hole(inode, offset, len); + } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { + ret = f2fs_collapse_range(inode, offset, len); + } else if (mode & FALLOC_FL_ZERO_RANGE) { + ret = f2fs_zero_range(inode, offset, len, mode); + } else if (mode & FALLOC_FL_INSERT_RANGE) { + ret = f2fs_insert_range(inode, offset, len); + } else { + ret = expand_inode_data(inode, offset, len, mode); + } + + if (!ret) { + inode->i_mtime = inode->i_ctime = CURRENT_TIME; + mark_inode_dirty(inode); + } + +out: + mutex_unlock(&inode->i_mutex); + + trace_f2fs_fallocate(inode, mode, offset, len, ret); + return ret; +} + +static int f2fs_release_file(struct inode *inode, struct file *filp) +{ + /* some remained atomic pages should discarded */ + if (f2fs_is_atomic_file(inode)) + commit_inmem_pages(inode, true); + if (f2fs_is_volatile_file(inode)) { + set_inode_flag(F2FS_I(inode), FI_DROP_CACHE); + filemap_fdatawrite(inode->i_mapping); + clear_inode_flag(F2FS_I(inode), FI_DROP_CACHE); + } + return 0; +} + +#define F2FS_REG_FLMASK (~(FS_DIRSYNC_FL | FS_TOPDIR_FL)) +#define F2FS_OTHER_FLMASK (FS_NODUMP_FL | FS_NOATIME_FL) + +static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags) +{ + if (S_ISDIR(mode)) + return flags; + else if (S_ISREG(mode)) + return flags & F2FS_REG_FLMASK; + else + return flags & F2FS_OTHER_FLMASK; +} + +static int f2fs_ioc_getflags(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_inode_info *fi = F2FS_I(inode); + unsigned int flags = fi->i_flags & FS_FL_USER_VISIBLE; + return put_user(flags, (int __user *)arg); +} + +static int f2fs_ioc_setflags(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_inode_info *fi = F2FS_I(inode); + unsigned int flags = fi->i_flags & FS_FL_USER_VISIBLE; + unsigned int oldflags; + int ret; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + if (!inode_owner_or_capable(inode)) { + ret = -EACCES; + goto out; + } + + if (get_user(flags, (int __user *)arg)) { + ret = -EFAULT; + goto out; + } + + flags = f2fs_mask_flags(inode->i_mode, flags); + + mutex_lock(&inode->i_mutex); + + oldflags = fi->i_flags; + + if ((flags ^ oldflags) & (FS_APPEND_FL | FS_IMMUTABLE_FL)) { + if (!capable(CAP_LINUX_IMMUTABLE)) { + mutex_unlock(&inode->i_mutex); + ret = -EPERM; + goto out; + } + } + + flags = flags & FS_FL_USER_MODIFIABLE; + flags |= oldflags & ~FS_FL_USER_MODIFIABLE; + fi->i_flags = flags; + mutex_unlock(&inode->i_mutex); + + f2fs_set_inode_flags(inode); + inode->i_ctime = CURRENT_TIME; + mark_inode_dirty(inode); +out: + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_getversion(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + + return put_user(inode->i_generation, (int __user *)arg); +} + +static int f2fs_ioc_start_atomic_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + int ret; + + if (!inode_owner_or_capable(inode)) + return -EACCES; + + f2fs_balance_fs(F2FS_I_SB(inode)); + + if (f2fs_is_atomic_file(inode)) + return 0; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + set_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE); + return 0; +} + +static int f2fs_ioc_commit_atomic_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + int ret; + + if (!inode_owner_or_capable(inode)) + return -EACCES; + + if (f2fs_is_volatile_file(inode)) + return 0; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + if (f2fs_is_atomic_file(inode)) { + clear_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE); + ret = commit_inmem_pages(inode, false); + if (ret) + goto err_out; + } + + ret = f2fs_sync_file(filp, 0, LLONG_MAX, 0); +err_out: + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_start_volatile_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + int ret; + + if (!inode_owner_or_capable(inode)) + return -EACCES; + + if (f2fs_is_volatile_file(inode)) + return 0; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + set_inode_flag(F2FS_I(inode), FI_VOLATILE_FILE); + return 0; +} + +static int f2fs_ioc_release_volatile_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + + if (!inode_owner_or_capable(inode)) + return -EACCES; + + if (!f2fs_is_volatile_file(inode)) + return 0; + + if (!f2fs_is_first_block_written(inode)) + return truncate_partial_data_page(inode, 0, true); + + return punch_hole(inode, 0, F2FS_BLKSIZE); +} + +static int f2fs_ioc_abort_volatile_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + int ret; + + if (!inode_owner_or_capable(inode)) + return -EACCES; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + f2fs_balance_fs(F2FS_I_SB(inode)); + + clear_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE); + clear_inode_flag(F2FS_I(inode), FI_VOLATILE_FILE); + commit_inmem_pages(inode, true); + + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct super_block *sb = sbi->sb; + __u32 in; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(in, (__u32 __user *)arg)) + return -EFAULT; + + switch (in) { + case FS_GOING_DOWN_FULLSYNC: + sb = freeze_bdev(sb->s_bdev); + if (sb && !IS_ERR(sb)) { + f2fs_stop_checkpoint(sbi); + thaw_bdev(sb->s_bdev, sb); + } + break; + case FS_GOING_DOWN_METASYNC: + /* do checkpoint only */ + f2fs_sync_fs(sb, 1); + f2fs_stop_checkpoint(sbi); + break; + case FS_GOING_DOWN_NOSYNC: + f2fs_stop_checkpoint(sbi); + break; + case FS_GOING_DOWN_METAFLUSH: + sync_meta_pages(sbi, META, LONG_MAX); + f2fs_stop_checkpoint(sbi); + break; + default: + return -EINVAL; + } + return 0; +} + +static int f2fs_ioc_fitrim(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct super_block *sb = inode->i_sb; + struct request_queue *q = bdev_get_queue(sb->s_bdev); + struct fstrim_range range; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!blk_queue_discard(q)) + return -EOPNOTSUPP; + + if (copy_from_user(&range, (struct fstrim_range __user *)arg, + sizeof(range))) + return -EFAULT; + + range.minlen = max((unsigned int)range.minlen, + q->limits.discard_granularity); + ret = f2fs_trim_fs(F2FS_SB(sb), &range); + if (ret < 0) + return ret; + + if (copy_to_user((struct fstrim_range __user *)arg, &range, + sizeof(range))) + return -EFAULT; + return 0; +} + +static bool uuid_is_nonzero(__u8 u[16]) +{ + int i; + + for (i = 0; i < 16; i++) + if (u[i]) + return true; + return false; +} + +static int f2fs_ioc_set_encryption_policy(struct file *filp, unsigned long arg) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + struct f2fs_encryption_policy policy; + struct inode *inode = file_inode(filp); + + if (copy_from_user(&policy, (struct f2fs_encryption_policy __user *)arg, + sizeof(policy))) + return -EFAULT; + + return f2fs_process_policy(&policy, inode); +#else + return -EOPNOTSUPP; +#endif +} + +static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg) +{ +#ifdef CONFIG_F2FS_FS_ENCRYPTION + struct f2fs_encryption_policy policy; + struct inode *inode = file_inode(filp); + int err; + + err = f2fs_get_policy(inode, &policy); + if (err) + return err; + + if (copy_to_user((struct f2fs_encryption_policy __user *)arg, &policy, + sizeof(policy))) + return -EFAULT; + return 0; +#else + return -EOPNOTSUPP; +#endif +} + +static int f2fs_ioc_get_encryption_pwsalt(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int err; + + if (!f2fs_sb_has_crypto(inode->i_sb)) + return -EOPNOTSUPP; + + if (uuid_is_nonzero(sbi->raw_super->encrypt_pw_salt)) + goto got_it; + + err = mnt_want_write_file(filp); + if (err) + return err; + + /* update superblock with uuid */ + generate_random_uuid(sbi->raw_super->encrypt_pw_salt); + + err = f2fs_commit_super(sbi, false); + + mnt_drop_write_file(filp); + if (err) { + /* undo new data */ + memset(sbi->raw_super->encrypt_pw_salt, 0, 16); + return err; + } +got_it: + if (copy_to_user((__u8 __user *)arg, sbi->raw_super->encrypt_pw_salt, + 16)) + return -EFAULT; + return 0; +} + +static int f2fs_ioc_gc(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + __u32 sync; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(sync, (__u32 __user *)arg)) + return -EFAULT; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (!sync) { + if (!mutex_trylock(&sbi->gc_mutex)) + return -EBUSY; + } else { + mutex_lock(&sbi->gc_mutex); + } + + return f2fs_gc(sbi, sync); +} + +static int f2fs_ioc_write_checkpoint(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct cp_control cpc; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + cpc.reason = __get_cp_reason(sbi); + + mutex_lock(&sbi->gc_mutex); + write_checkpoint(sbi, &cpc); + mutex_unlock(&sbi->gc_mutex); + + return 0; +} + +long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case F2FS_IOC_GETFLAGS: + return f2fs_ioc_getflags(filp, arg); + case F2FS_IOC_SETFLAGS: + return f2fs_ioc_setflags(filp, arg); + case F2FS_IOC_GETVERSION: + return f2fs_ioc_getversion(filp, arg); + case F2FS_IOC_START_ATOMIC_WRITE: + return f2fs_ioc_start_atomic_write(filp); + case F2FS_IOC_COMMIT_ATOMIC_WRITE: + return f2fs_ioc_commit_atomic_write(filp); + case F2FS_IOC_START_VOLATILE_WRITE: + return f2fs_ioc_start_volatile_write(filp); + case F2FS_IOC_RELEASE_VOLATILE_WRITE: + return f2fs_ioc_release_volatile_write(filp); + case F2FS_IOC_ABORT_VOLATILE_WRITE: + return f2fs_ioc_abort_volatile_write(filp); + case FS_IOC_SHUTDOWN: + return f2fs_ioc_shutdown(filp, arg); + case FITRIM: + return f2fs_ioc_fitrim(filp, arg); + case F2FS_IOC_SET_ENCRYPTION_POLICY: + return f2fs_ioc_set_encryption_policy(filp, arg); + case F2FS_IOC_GET_ENCRYPTION_POLICY: + return f2fs_ioc_get_encryption_policy(filp, arg); + case F2FS_IOC_GET_ENCRYPTION_PWSALT: + return f2fs_ioc_get_encryption_pwsalt(filp, arg); + case F2FS_IOC_GARBAGE_COLLECT: + return f2fs_ioc_gc(filp, arg); + case F2FS_IOC_WRITE_CHECKPOINT: + return f2fs_ioc_write_checkpoint(filp, arg); + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case F2FS_IOC32_GETFLAGS: + cmd = F2FS_IOC_GETFLAGS; + break; + case F2FS_IOC32_SETFLAGS: + cmd = F2FS_IOC_SETFLAGS; + break; + default: + return -ENOIOCTLCMD; + } + return f2fs_ioctl(file, cmd, (unsigned long) compat_ptr(arg)); +} +#endif + +const struct file_operations f2fs_file_operations = { + .llseek = f2fs_llseek, + .read = do_sync_read, + .write = do_sync_write, + .aio_read = generic_file_aio_read, + .aio_write = generic_file_aio_write, + .open = f2fs_file_open, + .release = f2fs_release_file, + .mmap = f2fs_file_mmap, + .fsync = f2fs_sync_file, + .fallocate = f2fs_fallocate, + .unlocked_ioctl = f2fs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = f2fs_compat_ioctl, +#endif + .splice_read = generic_file_splice_read, + .splice_write = generic_file_splice_write, +}; diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c new file mode 100644 index 00000000000..ee32e72ab80 --- /dev/null +++ b/fs/f2fs/gc.c @@ -0,0 +1,886 @@ +/* + * fs/f2fs/gc.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "gc.h" +#include + +static int gc_thread_func(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head; + long wait_ms; + + wait_ms = gc_th->min_sleep_time; + + do { + if (try_to_freeze()) + continue; + else + wait_event_interruptible_timeout(*wq, + kthread_should_stop(), + msecs_to_jiffies(wait_ms)); + if (kthread_should_stop()) + break; + + if (sbi->sb->s_frozen >= SB_FREEZE_WRITE) { + increase_sleep_time(gc_th, &wait_ms); + continue; + } + + /* + * [GC triggering condition] + * 0. GC is not conducted currently. + * 1. There are enough dirty segments. + * 2. IO subsystem is idle by checking the # of writeback pages. + * 3. IO subsystem is idle by checking the # of requests in + * bdev's request list. + * + * Note) We have to avoid triggering GCs frequently. + * Because it is possible that some segments can be + * invalidated soon after by user update or deletion. + * So, I'd like to wait some time to collect dirty segments. + */ + if (!mutex_trylock(&sbi->gc_mutex)) + continue; + + if (!is_idle(sbi)) { + increase_sleep_time(gc_th, &wait_ms); + mutex_unlock(&sbi->gc_mutex); + continue; + } + + if (has_enough_invalid_blocks(sbi)) + decrease_sleep_time(gc_th, &wait_ms); + else + increase_sleep_time(gc_th, &wait_ms); + + stat_inc_bggc_count(sbi); + + /* if return value is not zero, no victim was selected */ + if (f2fs_gc(sbi, test_opt(sbi, FORCE_FG_GC))) + wait_ms = gc_th->no_gc_sleep_time; + + trace_f2fs_background_gc(sbi->sb, wait_ms, + prefree_segments(sbi), free_segments(sbi)); + + /* balancing f2fs's metadata periodically */ + f2fs_balance_fs_bg(sbi); + + } while (!kthread_should_stop()); + return 0; +} + +int start_gc_thread(struct f2fs_sb_info *sbi) +{ + struct f2fs_gc_kthread *gc_th; + dev_t dev = sbi->sb->s_bdev->bd_dev; + int err = 0; + + gc_th = kmalloc(sizeof(struct f2fs_gc_kthread), GFP_KERNEL); + if (!gc_th) { + err = -ENOMEM; + goto out; + } + + gc_th->min_sleep_time = DEF_GC_THREAD_MIN_SLEEP_TIME; + gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME; + gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME; + + gc_th->gc_idle = 0; + + sbi->gc_thread = gc_th; + init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head); + sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi, + "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(gc_th->f2fs_gc_task)) { + err = PTR_ERR(gc_th->f2fs_gc_task); + kfree(gc_th); + sbi->gc_thread = NULL; + } +out: + return err; +} + +void stop_gc_thread(struct f2fs_sb_info *sbi) +{ + struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + if (!gc_th) + return; + kthread_stop(gc_th->f2fs_gc_task); + kfree(gc_th); + sbi->gc_thread = NULL; +} + +static int select_gc_type(struct f2fs_gc_kthread *gc_th, int gc_type) +{ + int gc_mode = (gc_type == BG_GC) ? GC_CB : GC_GREEDY; + + if (gc_th && gc_th->gc_idle) { + if (gc_th->gc_idle == 1) + gc_mode = GC_CB; + else if (gc_th->gc_idle == 2) + gc_mode = GC_GREEDY; + } + return gc_mode; +} + +static void select_policy(struct f2fs_sb_info *sbi, int gc_type, + int type, struct victim_sel_policy *p) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + if (p->alloc_mode == SSR) { + p->gc_mode = GC_GREEDY; + p->dirty_segmap = dirty_i->dirty_segmap[type]; + p->max_search = dirty_i->nr_dirty[type]; + p->ofs_unit = 1; + } else { + p->gc_mode = select_gc_type(sbi->gc_thread, gc_type); + p->dirty_segmap = dirty_i->dirty_segmap[DIRTY]; + p->max_search = dirty_i->nr_dirty[DIRTY]; + p->ofs_unit = sbi->segs_per_sec; + } + + if (p->max_search > sbi->max_victim_search) + p->max_search = sbi->max_victim_search; + + p->offset = sbi->last_victim[p->gc_mode]; +} + +static unsigned int get_max_cost(struct f2fs_sb_info *sbi, + struct victim_sel_policy *p) +{ + /* SSR allocates in a segment unit */ + if (p->alloc_mode == SSR) + return 1 << sbi->log_blocks_per_seg; + if (p->gc_mode == GC_GREEDY) + return (1 << sbi->log_blocks_per_seg) * p->ofs_unit; + else if (p->gc_mode == GC_CB) + return UINT_MAX; + else /* No other gc_mode */ + return 0; +} + +static unsigned int check_bg_victims(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int secno; + + /* + * If the gc_type is FG_GC, we can select victim segments + * selected by background GC before. + * Those segments guarantee they have small valid blocks. + */ + for_each_set_bit(secno, dirty_i->victim_secmap, MAIN_SECS(sbi)) { + if (sec_usage_check(sbi, secno)) + continue; + clear_bit(secno, dirty_i->victim_secmap); + return secno * sbi->segs_per_sec; + } + return NULL_SEGNO; +} + +static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int secno = GET_SECNO(sbi, segno); + unsigned int start = secno * sbi->segs_per_sec; + unsigned long long mtime = 0; + unsigned int vblocks; + unsigned char age = 0; + unsigned char u; + unsigned int i; + + for (i = 0; i < sbi->segs_per_sec; i++) + mtime += get_seg_entry(sbi, start + i)->mtime; + vblocks = get_valid_blocks(sbi, segno, sbi->segs_per_sec); + + mtime = div_u64(mtime, sbi->segs_per_sec); + vblocks = div_u64(vblocks, sbi->segs_per_sec); + + u = (vblocks * 100) >> sbi->log_blocks_per_seg; + + /* Handle if the system time has changed by the user */ + if (mtime < sit_i->min_mtime) + sit_i->min_mtime = mtime; + if (mtime > sit_i->max_mtime) + sit_i->max_mtime = mtime; + if (sit_i->max_mtime != sit_i->min_mtime) + age = 100 - div64_u64(100 * (mtime - sit_i->min_mtime), + sit_i->max_mtime - sit_i->min_mtime); + + return UINT_MAX - ((100 * (100 - u) * age) / (100 + u)); +} + +static inline unsigned int get_gc_cost(struct f2fs_sb_info *sbi, + unsigned int segno, struct victim_sel_policy *p) +{ + if (p->alloc_mode == SSR) + return get_seg_entry(sbi, segno)->ckpt_valid_blocks; + + /* alloc_mode == LFS */ + if (p->gc_mode == GC_GREEDY) + return get_valid_blocks(sbi, segno, sbi->segs_per_sec); + else + return get_cb_cost(sbi, segno); +} + +/* + * This function is called from two paths. + * One is garbage collection and the other is SSR segment selection. + * When it is called during GC, it just gets a victim segment + * and it does not remove it from dirty seglist. + * When it is called from SSR segment selection, it finds a segment + * which has minimum valid blocks and removes it from dirty seglist. + */ +static int get_victim_by_default(struct f2fs_sb_info *sbi, + unsigned int *result, int gc_type, int type, char alloc_mode) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + struct victim_sel_policy p; + unsigned int secno, max_cost; + unsigned int last_segment = MAIN_SEGS(sbi); + int nsearched = 0; + + mutex_lock(&dirty_i->seglist_lock); + + p.alloc_mode = alloc_mode; + select_policy(sbi, gc_type, type, &p); + + p.min_segno = NULL_SEGNO; + p.min_cost = max_cost = get_max_cost(sbi, &p); + + if (p.max_search == 0) + goto out; + + if (p.alloc_mode == LFS && gc_type == FG_GC) { + p.min_segno = check_bg_victims(sbi); + if (p.min_segno != NULL_SEGNO) + goto got_it; + } + + while (1) { + unsigned long cost; + unsigned int segno; + + segno = find_next_bit(p.dirty_segmap, last_segment, p.offset); + if (segno >= last_segment) { + if (sbi->last_victim[p.gc_mode]) { + last_segment = sbi->last_victim[p.gc_mode]; + sbi->last_victim[p.gc_mode] = 0; + p.offset = 0; + continue; + } + break; + } + + p.offset = segno + p.ofs_unit; + if (p.ofs_unit > 1) + p.offset -= segno % p.ofs_unit; + + secno = GET_SECNO(sbi, segno); + + if (sec_usage_check(sbi, secno)) + continue; + if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap)) + continue; + + cost = get_gc_cost(sbi, segno, &p); + + if (p.min_cost > cost) { + p.min_segno = segno; + p.min_cost = cost; + } else if (unlikely(cost == max_cost)) { + continue; + } + + if (nsearched++ >= p.max_search) { + sbi->last_victim[p.gc_mode] = segno; + break; + } + } + if (p.min_segno != NULL_SEGNO) { +got_it: + if (p.alloc_mode == LFS) { + secno = GET_SECNO(sbi, p.min_segno); + if (gc_type == FG_GC) + sbi->cur_victim_sec = secno; + else + set_bit(secno, dirty_i->victim_secmap); + } + *result = (p.min_segno / p.ofs_unit) * p.ofs_unit; + + trace_f2fs_get_victim(sbi->sb, type, gc_type, &p, + sbi->cur_victim_sec, + prefree_segments(sbi), free_segments(sbi)); + } +out: + mutex_unlock(&dirty_i->seglist_lock); + + return (p.min_segno == NULL_SEGNO) ? 0 : 1; +} + +static const struct victim_selection default_v_ops = { + .get_victim = get_victim_by_default, +}; + +static struct inode *find_gc_inode(struct gc_inode_list *gc_list, nid_t ino) +{ + struct inode_entry *ie; + + ie = radix_tree_lookup(&gc_list->iroot, ino); + if (ie) + return ie->inode; + return NULL; +} + +static void add_gc_inode(struct gc_inode_list *gc_list, struct inode *inode) +{ + struct inode_entry *new_ie; + + if (inode == find_gc_inode(gc_list, inode->i_ino)) { + iput(inode); + return; + } + new_ie = f2fs_kmem_cache_alloc(inode_entry_slab, GFP_NOFS); + new_ie->inode = inode; + + f2fs_radix_tree_insert(&gc_list->iroot, inode->i_ino, new_ie); + list_add_tail(&new_ie->list, &gc_list->ilist); +} + +static void put_gc_inode(struct gc_inode_list *gc_list) +{ + struct inode_entry *ie, *next_ie; + list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) { + radix_tree_delete(&gc_list->iroot, ie->inode->i_ino); + iput(ie->inode); + list_del(&ie->list); + kmem_cache_free(inode_entry_slab, ie); + } +} + +static int check_valid_map(struct f2fs_sb_info *sbi, + unsigned int segno, int offset) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct seg_entry *sentry; + int ret; + + mutex_lock(&sit_i->sentry_lock); + sentry = get_seg_entry(sbi, segno); + ret = f2fs_test_bit(offset, sentry->cur_valid_map); + mutex_unlock(&sit_i->sentry_lock); + return ret; +} + +/* + * This function compares node address got in summary with that in NAT. + * On validity, copy that node with cold status, otherwise (invalid node) + * ignore that. + */ +static int gc_node_segment(struct f2fs_sb_info *sbi, + struct f2fs_summary *sum, unsigned int segno, int gc_type) +{ + bool initial = true; + struct f2fs_summary *entry; + block_t start_addr; + int off; + + start_addr = START_BLOCK(sbi, segno); + +next_step: + entry = sum; + + for (off = 0; off < sbi->blocks_per_seg; off++, entry++) { + nid_t nid = le32_to_cpu(entry->nid); + struct page *node_page; + struct node_info ni; + + /* stop BG_GC if there is not enough free sections. */ + if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0)) + return 0; + + if (check_valid_map(sbi, segno, off) == 0) + continue; + + if (initial) { + ra_node_page(sbi, nid); + continue; + } + node_page = get_node_page(sbi, nid); + if (IS_ERR(node_page)) + continue; + + /* block may become invalid during get_node_page */ + if (check_valid_map(sbi, segno, off) == 0) { + f2fs_put_page(node_page, 1); + continue; + } + + get_node_info(sbi, nid, &ni); + if (ni.blk_addr != start_addr + off) { + f2fs_put_page(node_page, 1); + continue; + } + + /* set page dirty and write it */ + if (gc_type == FG_GC) { + f2fs_wait_on_page_writeback(node_page, NODE); + set_page_dirty(node_page); + } else { + if (!PageWriteback(node_page)) + set_page_dirty(node_page); + } + f2fs_put_page(node_page, 1); + stat_inc_node_blk_count(sbi, 1, gc_type); + } + + if (initial) { + initial = false; + goto next_step; + } + + if (gc_type == FG_GC) { + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + .for_reclaim = 0, + }; + sync_node_pages(sbi, 0, &wbc); + + /* return 1 only if FG_GC succefully reclaimed one */ + if (get_valid_blocks(sbi, segno, 1) == 0) + return 1; + } + return 0; +} + +/* + * Calculate start block index indicating the given node offset. + * Be careful, caller should give this node offset only indicating direct node + * blocks. If any node offsets, which point the other types of node blocks such + * as indirect or double indirect node blocks, are given, it must be a caller's + * bug. + */ +block_t start_bidx_of_node(unsigned int node_ofs, struct f2fs_inode_info *fi) +{ + unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4; + unsigned int bidx; + + if (node_ofs == 0) + return 0; + + if (node_ofs <= 2) { + bidx = node_ofs - 1; + } else if (node_ofs <= indirect_blks) { + int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1); + bidx = node_ofs - 2 - dec; + } else { + int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1); + bidx = node_ofs - 5 - dec; + } + return bidx * ADDRS_PER_BLOCK + ADDRS_PER_INODE(fi); +} + +static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, + struct node_info *dni, block_t blkaddr, unsigned int *nofs) +{ + struct page *node_page; + nid_t nid; + unsigned int ofs_in_node; + block_t source_blkaddr; + + nid = le32_to_cpu(sum->nid); + ofs_in_node = le16_to_cpu(sum->ofs_in_node); + + node_page = get_node_page(sbi, nid); + if (IS_ERR(node_page)) + return false; + + get_node_info(sbi, nid, dni); + + if (sum->version != dni->version) { + f2fs_put_page(node_page, 1); + return false; + } + + *nofs = ofs_of_node(node_page); + source_blkaddr = datablock_addr(node_page, ofs_in_node); + f2fs_put_page(node_page, 1); + + if (source_blkaddr != blkaddr) + return false; + return true; +} + +static void move_encrypted_block(struct inode *inode, block_t bidx) +{ + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(inode), + .type = DATA, + .rw = READ_SYNC, + .encrypted_page = NULL, + }; + struct dnode_of_data dn; + struct f2fs_summary sum; + struct node_info ni; + struct page *page; + int err; + + /* do not read out */ + page = f2fs_grab_cache_page(inode->i_mapping, bidx, false); + if (!page) + return; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, bidx, LOOKUP_NODE); + if (err) + goto out; + + if (unlikely(dn.data_blkaddr == NULL_ADDR)) { + ClearPageUptodate(page); + goto put_out; + } + + /* + * don't cache encrypted data into meta inode until previous dirty + * data were writebacked to avoid racing between GC and flush. + */ + f2fs_wait_on_page_writeback(page, DATA); + + get_node_info(fio.sbi, dn.nid, &ni); + set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version); + + /* read page */ + fio.page = page; + fio.blk_addr = dn.data_blkaddr; + + fio.encrypted_page = grab_cache_page(META_MAPPING(fio.sbi), fio.blk_addr); + if (!fio.encrypted_page) + goto put_out; + + err = f2fs_submit_page_bio(&fio); + if (err) + goto put_page_out; + + /* write page */ + lock_page(fio.encrypted_page); + + if (unlikely(!PageUptodate(fio.encrypted_page))) + goto put_page_out; + if (unlikely(fio.encrypted_page->mapping != META_MAPPING(fio.sbi))) + goto put_page_out; + + set_page_dirty(fio.encrypted_page); + f2fs_wait_on_page_writeback(fio.encrypted_page, DATA); + if (clear_page_dirty_for_io(fio.encrypted_page)) + dec_page_count(fio.sbi, F2FS_DIRTY_META); + + set_page_writeback(fio.encrypted_page); + + /* allocate block address */ + f2fs_wait_on_page_writeback(dn.node_page, NODE); + allocate_data_block(fio.sbi, NULL, fio.blk_addr, + &fio.blk_addr, &sum, CURSEG_COLD_DATA); + fio.rw = WRITE_SYNC; + f2fs_submit_page_mbio(&fio); + + dn.data_blkaddr = fio.blk_addr; + set_data_blkaddr(&dn); + f2fs_update_extent_cache(&dn); + set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); + if (page->index == 0) + set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); +put_page_out: + f2fs_put_page(fio.encrypted_page, 1); +put_out: + f2fs_put_dnode(&dn); +out: + f2fs_put_page(page, 1); +} + +static void move_data_page(struct inode *inode, block_t bidx, int gc_type) +{ + struct page *page; + + page = get_lock_data_page(inode, bidx, true); + if (IS_ERR(page)) + return; + + if (gc_type == BG_GC) { + if (PageWriteback(page)) + goto out; + set_page_dirty(page); + set_cold_data(page); + } else { + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(inode), + .type = DATA, + .rw = WRITE_SYNC, + .page = page, + .encrypted_page = NULL, + }; + set_page_dirty(page); + f2fs_wait_on_page_writeback(page, DATA); + if (clear_page_dirty_for_io(page)) + inode_dec_dirty_pages(inode); + set_cold_data(page); + do_write_data_page(&fio); + clear_cold_data(page); + } +out: + f2fs_put_page(page, 1); +} + +/* + * This function tries to get parent node of victim data block, and identifies + * data block validity. If the block is valid, copy that with cold status and + * modify parent node. + * If the parent node is not valid or the data block address is different, + * the victim data block is ignored. + */ +static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, + struct gc_inode_list *gc_list, unsigned int segno, int gc_type) +{ + struct super_block *sb = sbi->sb; + struct f2fs_summary *entry; + block_t start_addr; + int off; + int phase = 0; + + start_addr = START_BLOCK(sbi, segno); + +next_step: + entry = sum; + + for (off = 0; off < sbi->blocks_per_seg; off++, entry++) { + struct page *data_page; + struct inode *inode; + struct node_info dni; /* dnode info for the data */ + unsigned int ofs_in_node, nofs; + block_t start_bidx; + + /* stop BG_GC if there is not enough free sections. */ + if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0)) + return 0; + + if (check_valid_map(sbi, segno, off) == 0) + continue; + + if (phase == 0) { + ra_node_page(sbi, le32_to_cpu(entry->nid)); + continue; + } + + /* Get an inode by ino with checking validity */ + if (!is_alive(sbi, entry, &dni, start_addr + off, &nofs)) + continue; + + if (phase == 1) { + ra_node_page(sbi, dni.ino); + continue; + } + + ofs_in_node = le16_to_cpu(entry->ofs_in_node); + + if (phase == 2) { + inode = f2fs_iget(sb, dni.ino); + if (IS_ERR(inode) || is_bad_inode(inode)) + continue; + + /* if encrypted inode, let's go phase 3 */ + if (f2fs_encrypted_inode(inode) && + S_ISREG(inode->i_mode)) { + add_gc_inode(gc_list, inode); + continue; + } + + start_bidx = start_bidx_of_node(nofs, F2FS_I(inode)); + data_page = get_read_data_page(inode, + start_bidx + ofs_in_node, READA, true); + if (IS_ERR(data_page)) { + iput(inode); + continue; + } + + f2fs_put_page(data_page, 0); + add_gc_inode(gc_list, inode); + continue; + } + + /* phase 3 */ + inode = find_gc_inode(gc_list, dni.ino); + if (inode) { + start_bidx = start_bidx_of_node(nofs, F2FS_I(inode)) + + ofs_in_node; + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) + move_encrypted_block(inode, start_bidx); + else + move_data_page(inode, start_bidx, gc_type); + stat_inc_data_blk_count(sbi, 1, gc_type); + } + } + + if (++phase < 4) + goto next_step; + + if (gc_type == FG_GC) { + f2fs_submit_merged_bio(sbi, DATA, WRITE); + + /* return 1 only if FG_GC succefully reclaimed one */ + if (get_valid_blocks(sbi, segno, 1) == 0) + return 1; + } + return 0; +} + +static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim, + int gc_type) +{ + struct sit_info *sit_i = SIT_I(sbi); + int ret; + + mutex_lock(&sit_i->sentry_lock); + ret = DIRTY_I(sbi)->v_ops->get_victim(sbi, victim, gc_type, + NO_CHECK_TYPE, LFS); + mutex_unlock(&sit_i->sentry_lock); + return ret; +} + +static int do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int segno, + struct gc_inode_list *gc_list, int gc_type) +{ + struct page *sum_page; + struct f2fs_summary_block *sum; + struct blk_plug plug; + int nfree = 0; + + /* read segment summary of victim */ + sum_page = get_sum_page(sbi, segno); + + blk_start_plug(&plug); + + sum = page_address(sum_page); + + /* + * this is to avoid deadlock: + * - lock_page(sum_page) - f2fs_replace_block + * - check_valid_map() - mutex_lock(sentry_lock) + * - mutex_lock(sentry_lock) - change_curseg() + * - lock_page(sum_page) + */ + unlock_page(sum_page); + + switch (GET_SUM_TYPE((&sum->footer))) { + case SUM_TYPE_NODE: + nfree = gc_node_segment(sbi, sum->entries, segno, gc_type); + break; + case SUM_TYPE_DATA: + nfree = gc_data_segment(sbi, sum->entries, gc_list, + segno, gc_type); + break; + } + blk_finish_plug(&plug); + + stat_inc_seg_count(sbi, GET_SUM_TYPE((&sum->footer)), gc_type); + stat_inc_call_count(sbi->stat_info); + + f2fs_put_page(sum_page, 0); + return nfree; +} + +int f2fs_gc(struct f2fs_sb_info *sbi, bool sync) +{ + unsigned int segno, i; + int gc_type = sync ? FG_GC : BG_GC; + int sec_freed = 0; + int ret = -EINVAL; + struct cp_control cpc; + struct gc_inode_list gc_list = { + .ilist = LIST_HEAD_INIT(gc_list.ilist), + .iroot = RADIX_TREE_INIT(GFP_NOFS), + }; + + cpc.reason = __get_cp_reason(sbi); +gc_more: + segno = NULL_SEGNO; + + if (unlikely(!(sbi->sb->s_flags & MS_ACTIVE))) + goto stop; + if (unlikely(f2fs_cp_error(sbi))) + goto stop; + + if (gc_type == BG_GC && has_not_enough_free_secs(sbi, sec_freed)) { + gc_type = FG_GC; + if (__get_victim(sbi, &segno, gc_type) || prefree_segments(sbi)) + write_checkpoint(sbi, &cpc); + } + + if (segno == NULL_SEGNO && !__get_victim(sbi, &segno, gc_type)) + goto stop; + ret = 0; + + /* readahead multi ssa blocks those have contiguous address */ + if (sbi->segs_per_sec > 1) + ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), sbi->segs_per_sec, + META_SSA, true); + + for (i = 0; i < sbi->segs_per_sec; i++) { + /* + * for FG_GC case, halt gcing left segments once failed one + * of segments in selected section to avoid long latency. + */ + if (!do_garbage_collect(sbi, segno + i, &gc_list, gc_type) && + gc_type == FG_GC) + break; + } + + if (i == sbi->segs_per_sec && gc_type == FG_GC) + sec_freed++; + + if (gc_type == FG_GC) + sbi->cur_victim_sec = NULL_SEGNO; + + if (!sync) { + if (has_not_enough_free_secs(sbi, sec_freed)) + goto gc_more; + + if (gc_type == FG_GC) + write_checkpoint(sbi, &cpc); + } +stop: + mutex_unlock(&sbi->gc_mutex); + + put_gc_inode(&gc_list); + + if (sync) + ret = sec_freed ? 0 : -EAGAIN; + return ret; +} + +void build_gc_manager(struct f2fs_sb_info *sbi) +{ + DIRTY_I(sbi)->v_ops = &default_v_ops; +} diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h new file mode 100644 index 00000000000..9091e0c9ded --- /dev/null +++ b/fs/f2fs/gc.h @@ -0,0 +1,110 @@ +/* + * fs/f2fs/gc.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#define GC_THREAD_MIN_WB_PAGES 1 /* + * a threshold to determine + * whether IO subsystem is idle + * or not + */ +#define DEF_GC_THREAD_MIN_SLEEP_TIME 30000 /* milliseconds */ +#define DEF_GC_THREAD_MAX_SLEEP_TIME 60000 +#define DEF_GC_THREAD_NOGC_SLEEP_TIME 300000 /* wait 5 min */ +#define LIMIT_INVALID_BLOCK 40 /* percentage over total user space */ +#define LIMIT_FREE_BLOCK 40 /* percentage over invalid + free space */ + +/* Search max. number of dirty segments to select a victim segment */ +#define DEF_MAX_VICTIM_SEARCH 4096 /* covers 8GB */ + +struct f2fs_gc_kthread { + struct task_struct *f2fs_gc_task; + wait_queue_head_t gc_wait_queue_head; + + /* for gc sleep time */ + unsigned int min_sleep_time; + unsigned int max_sleep_time; + unsigned int no_gc_sleep_time; + + /* for changing gc mode */ + unsigned int gc_idle; +}; + +struct gc_inode_list { + struct list_head ilist; + struct radix_tree_root iroot; +}; + +/* + * inline functions + */ +static inline block_t free_user_blocks(struct f2fs_sb_info *sbi) +{ + if (free_segments(sbi) < overprovision_segments(sbi)) + return 0; + else + return (free_segments(sbi) - overprovision_segments(sbi)) + << sbi->log_blocks_per_seg; +} + +static inline block_t limit_invalid_user_blocks(struct f2fs_sb_info *sbi) +{ + return (long)(sbi->user_block_count * LIMIT_INVALID_BLOCK) / 100; +} + +static inline block_t limit_free_user_blocks(struct f2fs_sb_info *sbi) +{ + block_t reclaimable_user_blocks = sbi->user_block_count - + written_block_count(sbi); + return (long)(reclaimable_user_blocks * LIMIT_FREE_BLOCK) / 100; +} + +static inline void increase_sleep_time(struct f2fs_gc_kthread *gc_th, + long *wait) +{ + if (*wait == gc_th->no_gc_sleep_time) + return; + + *wait += gc_th->min_sleep_time; + if (*wait > gc_th->max_sleep_time) + *wait = gc_th->max_sleep_time; +} + +static inline void decrease_sleep_time(struct f2fs_gc_kthread *gc_th, + long *wait) +{ + if (*wait == gc_th->no_gc_sleep_time) + *wait = gc_th->max_sleep_time; + + *wait -= gc_th->min_sleep_time; + if (*wait <= gc_th->min_sleep_time) + *wait = gc_th->min_sleep_time; +} + +static inline bool has_enough_invalid_blocks(struct f2fs_sb_info *sbi) +{ + block_t invalid_user_blocks = sbi->user_block_count - + written_block_count(sbi); + /* + * Background GC is triggered with the following conditions. + * 1. There are a number of invalid blocks. + * 2. There is not enough free space. + */ + if (invalid_user_blocks > limit_invalid_user_blocks(sbi) && + free_user_blocks(sbi) < limit_free_user_blocks(sbi)) + return true; + return false; +} + +static inline int is_idle(struct f2fs_sb_info *sbi) +{ + struct block_device *bdev = sbi->sb->s_bdev; + struct request_queue *q = bdev_get_queue(bdev); + struct request_list *rl = &q->rq; + return !(rl->count[BLK_RW_SYNC]) && !(rl->count[BLK_RW_ASYNC]); +} diff --git a/fs/f2fs/hash.c b/fs/f2fs/hash.c new file mode 100644 index 00000000000..71b7206c431 --- /dev/null +++ b/fs/f2fs/hash.c @@ -0,0 +1,103 @@ +/* + * fs/f2fs/hash.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext3/hash.c + * + * Copyright (C) 2002 by Theodore Ts'o + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include + +#include "f2fs.h" + +/* + * Hashing code copied from ext3 + */ +#define DELTA 0x9E3779B9 + +static void TEA_transform(unsigned int buf[4], unsigned int const in[]) +{ + __u32 sum = 0; + __u32 b0 = buf[0], b1 = buf[1]; + __u32 a = in[0], b = in[1], c = in[2], d = in[3]; + int n = 16; + + do { + sum += DELTA; + b0 += ((b1 << 4)+a) ^ (b1+sum) ^ ((b1 >> 5)+b); + b1 += ((b0 << 4)+c) ^ (b0+sum) ^ ((b0 >> 5)+d); + } while (--n); + + buf[0] += b0; + buf[1] += b1; +} + +static void str2hashbuf(const unsigned char *msg, size_t len, + unsigned int *buf, int num) +{ + unsigned pad, val; + int i; + + pad = (__u32)len | ((__u32)len << 8); + pad |= pad << 16; + + val = pad; + if (len > num * 4) + len = num * 4; + for (i = 0; i < len; i++) { + if ((i % 4) == 0) + val = pad; + val = msg[i] + (val << 8); + if ((i % 4) == 3) { + *buf++ = val; + val = pad; + num--; + } + } + if (--num >= 0) + *buf++ = val; + while (--num >= 0) + *buf++ = pad; +} + +f2fs_hash_t f2fs_dentry_hash(const struct qstr *name_info) +{ + __u32 hash; + f2fs_hash_t f2fs_hash; + const unsigned char *p; + __u32 in[8], buf[4]; + const unsigned char *name = name_info->name; + size_t len = name_info->len; + + if (is_dot_dotdot(name_info)) + return 0; + + /* Initialize the default seed for the hash checksum functions */ + buf[0] = 0x67452301; + buf[1] = 0xefcdab89; + buf[2] = 0x98badcfe; + buf[3] = 0x10325476; + + p = name; + while (1) { + str2hashbuf(p, len, in, 4); + TEA_transform(buf, in); + p += 16; + if (len <= 16) + break; + len -= 16; + } + hash = buf[0]; + f2fs_hash = cpu_to_le32(hash & ~F2FS_HASH_COL_BIT); + return f2fs_hash; +} diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c new file mode 100644 index 00000000000..4d22fa72eca --- /dev/null +++ b/fs/f2fs/inline.c @@ -0,0 +1,612 @@ +/* + * fs/f2fs/inline.c + * Copyright (c) 2013, Intel Corporation + * Authors: Huajun Li + * Haicheng Li + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +#include "f2fs.h" +#include "node.h" + +bool f2fs_may_inline_data(struct inode *inode) +{ + if (!test_opt(F2FS_I_SB(inode), INLINE_DATA)) + return false; + + if (f2fs_is_atomic_file(inode)) + return false; + + if (!S_ISREG(inode->i_mode) && !S_ISLNK(inode->i_mode)) + return false; + + if (i_size_read(inode) > MAX_INLINE_DATA) + return false; + + if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) + return false; + + return true; +} + +bool f2fs_may_inline_dentry(struct inode *inode) +{ + if (!test_opt(F2FS_I_SB(inode), INLINE_DENTRY)) + return false; + + if (!S_ISDIR(inode->i_mode)) + return false; + + return true; +} + +void read_inline_data(struct page *page, struct page *ipage) +{ + void *src_addr, *dst_addr; + + if (PageUptodate(page)) + return; + + f2fs_bug_on(F2FS_P_SB(page), page->index); + + zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE); + + /* Copy the whole inline data block */ + src_addr = inline_data_addr(ipage); + dst_addr = kmap_atomic(page); + memcpy(dst_addr, src_addr, MAX_INLINE_DATA); + flush_dcache_page(page); + kunmap_atomic(dst_addr); + SetPageUptodate(page); +} + +bool truncate_inline_inode(struct page *ipage, u64 from) +{ + void *addr; + + if (from >= MAX_INLINE_DATA) + return false; + + addr = inline_data_addr(ipage); + + f2fs_wait_on_page_writeback(ipage, NODE); + memset(addr + from, 0, MAX_INLINE_DATA - from); + + return true; +} + +int f2fs_read_inline_data(struct inode *inode, struct page *page) +{ + struct page *ipage; + + ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ipage)) { + unlock_page(page); + return PTR_ERR(ipage); + } + + if (!f2fs_has_inline_data(inode)) { + f2fs_put_page(ipage, 1); + return -EAGAIN; + } + + if (page->index) + zero_user_segment(page, 0, PAGE_CACHE_SIZE); + else + read_inline_data(page, ipage); + + SetPageUptodate(page); + f2fs_put_page(ipage, 1); + unlock_page(page); + return 0; +} + +int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page) +{ + void *src_addr, *dst_addr; + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(dn->inode), + .type = DATA, + .rw = WRITE_SYNC | REQ_PRIO, + .page = page, + .encrypted_page = NULL, + }; + int dirty, err; + + f2fs_bug_on(F2FS_I_SB(dn->inode), page->index); + + if (!f2fs_exist_data(dn->inode)) + goto clear_out; + + err = f2fs_reserve_block(dn, 0); + if (err) + return err; + + f2fs_wait_on_page_writeback(page, DATA); + + if (PageUptodate(page)) + goto no_update; + + zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE); + + /* Copy the whole inline data block */ + src_addr = inline_data_addr(dn->inode_page); + dst_addr = kmap_atomic(page); + memcpy(dst_addr, src_addr, MAX_INLINE_DATA); + flush_dcache_page(page); + kunmap_atomic(dst_addr); + SetPageUptodate(page); +no_update: + set_page_dirty(page); + + /* clear dirty state */ + dirty = clear_page_dirty_for_io(page); + + /* write data page to try to make data consistent */ + set_page_writeback(page); + fio.blk_addr = dn->data_blkaddr; + write_data_page(dn, &fio); + set_data_blkaddr(dn); + f2fs_update_extent_cache(dn); + f2fs_wait_on_page_writeback(page, DATA); + if (dirty) + inode_dec_dirty_pages(dn->inode); + + /* this converted inline_data should be recovered. */ + set_inode_flag(F2FS_I(dn->inode), FI_APPEND_WRITE); + + /* clear inline data and flag after data writeback */ + truncate_inline_inode(dn->inode_page, 0); +clear_out: + stat_dec_inline_inode(dn->inode); + f2fs_clear_inline_inode(dn->inode); + sync_inode_page(dn); + f2fs_put_dnode(dn); + return 0; +} + +int f2fs_convert_inline_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + struct page *ipage, *page; + int err = 0; + + page = grab_cache_page(inode->i_mapping, 0); + if (!page) + return -ENOMEM; + + f2fs_lock_op(sbi); + + ipage = get_node_page(sbi, inode->i_ino); + if (IS_ERR(ipage)) { + err = PTR_ERR(ipage); + goto out; + } + + set_new_dnode(&dn, inode, ipage, ipage, 0); + + if (f2fs_has_inline_data(inode)) + err = f2fs_convert_inline_page(&dn, page); + + f2fs_put_dnode(&dn); +out: + f2fs_unlock_op(sbi); + + f2fs_put_page(page, 1); + return err; +} + +int f2fs_write_inline_data(struct inode *inode, struct page *page) +{ + void *src_addr, *dst_addr; + struct dnode_of_data dn; + int err; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = get_dnode_of_data(&dn, 0, LOOKUP_NODE); + if (err) + return err; + + if (!f2fs_has_inline_data(inode)) { + f2fs_put_dnode(&dn); + return -EAGAIN; + } + + f2fs_bug_on(F2FS_I_SB(inode), page->index); + + f2fs_wait_on_page_writeback(dn.inode_page, NODE); + src_addr = kmap_atomic(page); + dst_addr = inline_data_addr(dn.inode_page); + memcpy(dst_addr, src_addr, MAX_INLINE_DATA); + kunmap_atomic(src_addr); + + set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); + set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); + + sync_inode_page(&dn); + f2fs_put_dnode(&dn); + return 0; +} + +bool recover_inline_data(struct inode *inode, struct page *npage) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode *ri = NULL; + void *src_addr, *dst_addr; + struct page *ipage; + + /* + * The inline_data recovery policy is as follows. + * [prev.] [next] of inline_data flag + * o o -> recover inline_data + * o x -> remove inline_data, and then recover data blocks + * x o -> remove inline_data, and then recover inline_data + * x x -> recover data blocks + */ + if (IS_INODE(npage)) + ri = F2FS_INODE(npage); + + if (f2fs_has_inline_data(inode) && + ri && (ri->i_inline & F2FS_INLINE_DATA)) { +process_inline: + ipage = get_node_page(sbi, inode->i_ino); + f2fs_bug_on(sbi, IS_ERR(ipage)); + + f2fs_wait_on_page_writeback(ipage, NODE); + + src_addr = inline_data_addr(npage); + dst_addr = inline_data_addr(ipage); + memcpy(dst_addr, src_addr, MAX_INLINE_DATA); + + set_inode_flag(F2FS_I(inode), FI_INLINE_DATA); + set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); + + update_inode(inode, ipage); + f2fs_put_page(ipage, 1); + return true; + } + + if (f2fs_has_inline_data(inode)) { + ipage = get_node_page(sbi, inode->i_ino); + f2fs_bug_on(sbi, IS_ERR(ipage)); + if (!truncate_inline_inode(ipage, 0)) + return false; + f2fs_clear_inline_inode(inode); + update_inode(inode, ipage); + f2fs_put_page(ipage, 1); + } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) { + if (truncate_blocks(inode, 0, false)) + return false; + goto process_inline; + } + return false; +} + +struct f2fs_dir_entry *find_in_inline_dir(struct inode *dir, + struct f2fs_filename *fname, struct page **res_page) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb); + struct f2fs_inline_dentry *inline_dentry; + struct qstr name = FSTR_TO_QSTR(&fname->disk_name); + struct f2fs_dir_entry *de; + struct f2fs_dentry_ptr d; + struct page *ipage; + f2fs_hash_t namehash; + + ipage = get_node_page(sbi, dir->i_ino); + if (IS_ERR(ipage)) + return NULL; + + namehash = f2fs_dentry_hash(&name); + + inline_dentry = inline_data_addr(ipage); + + make_dentry_ptr(NULL, &d, (void *)inline_dentry, 2); + de = find_target_dentry(fname, namehash, NULL, &d); + unlock_page(ipage); + if (de) + *res_page = ipage; + else + f2fs_put_page(ipage, 0); + + /* + * For the most part, it should be a bug when name_len is zero. + * We stop here for figuring out where the bugs has occurred. + */ + f2fs_bug_on(sbi, d.max < 0); + return de; +} + +struct f2fs_dir_entry *f2fs_parent_inline_dir(struct inode *dir, + struct page **p) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct page *ipage; + struct f2fs_dir_entry *de; + struct f2fs_inline_dentry *dentry_blk; + + ipage = get_node_page(sbi, dir->i_ino); + if (IS_ERR(ipage)) + return NULL; + + dentry_blk = inline_data_addr(ipage); + de = &dentry_blk->dentry[1]; + *p = ipage; + unlock_page(ipage); + return de; +} + +int make_empty_inline_dir(struct inode *inode, struct inode *parent, + struct page *ipage) +{ + struct f2fs_inline_dentry *dentry_blk; + struct f2fs_dentry_ptr d; + + dentry_blk = inline_data_addr(ipage); + + make_dentry_ptr(NULL, &d, (void *)dentry_blk, 2); + do_make_empty_dir(inode, parent, &d); + + set_page_dirty(ipage); + + /* update i_size to MAX_INLINE_DATA */ + if (i_size_read(inode) < MAX_INLINE_DATA) { + i_size_write(inode, MAX_INLINE_DATA); + set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR); + } + return 0; +} + +/* + * NOTE: ipage is grabbed by caller, but if any error occurs, we should + * release ipage in this function. + */ +static int f2fs_convert_inline_dir(struct inode *dir, struct page *ipage, + struct f2fs_inline_dentry *inline_dentry) +{ + struct page *page; + struct dnode_of_data dn; + struct f2fs_dentry_block *dentry_blk; + int err; + + page = grab_cache_page(dir->i_mapping, 0); + if (!page) { + f2fs_put_page(ipage, 1); + return -ENOMEM; + } + + set_new_dnode(&dn, dir, ipage, NULL, 0); + err = f2fs_reserve_block(&dn, 0); + if (err) + goto out; + + f2fs_wait_on_page_writeback(page, DATA); + zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE); + + dentry_blk = kmap_atomic(page); + + /* copy data from inline dentry block to new dentry block */ + memcpy(dentry_blk->dentry_bitmap, inline_dentry->dentry_bitmap, + INLINE_DENTRY_BITMAP_SIZE); + memset(dentry_blk->dentry_bitmap + INLINE_DENTRY_BITMAP_SIZE, 0, + SIZE_OF_DENTRY_BITMAP - INLINE_DENTRY_BITMAP_SIZE); + /* + * we do not need to zero out remainder part of dentry and filename + * field, since we have used bitmap for marking the usage status of + * them, besides, we can also ignore copying/zeroing reserved space + * of dentry block, because them haven't been used so far. + */ + memcpy(dentry_blk->dentry, inline_dentry->dentry, + sizeof(struct f2fs_dir_entry) * NR_INLINE_DENTRY); + memcpy(dentry_blk->filename, inline_dentry->filename, + NR_INLINE_DENTRY * F2FS_SLOT_LEN); + + kunmap_atomic(dentry_blk); + SetPageUptodate(page); + set_page_dirty(page); + + /* clear inline dir and flag after data writeback */ + truncate_inline_inode(ipage, 0); + + stat_dec_inline_dir(dir); + clear_inode_flag(F2FS_I(dir), FI_INLINE_DENTRY); + + if (i_size_read(dir) < PAGE_CACHE_SIZE) { + i_size_write(dir, PAGE_CACHE_SIZE); + set_inode_flag(F2FS_I(dir), FI_UPDATE_DIR); + } + + sync_inode_page(&dn); +out: + f2fs_put_page(page, 1); + return err; +} + +int f2fs_add_inline_entry(struct inode *dir, const struct qstr *name, + struct inode *inode, nid_t ino, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct page *ipage; + unsigned int bit_pos; + f2fs_hash_t name_hash; + size_t namelen = name->len; + struct f2fs_inline_dentry *dentry_blk = NULL; + struct f2fs_dentry_ptr d; + int slots = GET_DENTRY_SLOTS(namelen); + struct page *page = NULL; + int err = 0; + + ipage = get_node_page(sbi, dir->i_ino); + if (IS_ERR(ipage)) + return PTR_ERR(ipage); + + dentry_blk = inline_data_addr(ipage); + bit_pos = room_for_filename(&dentry_blk->dentry_bitmap, + slots, NR_INLINE_DENTRY); + if (bit_pos >= NR_INLINE_DENTRY) { + err = f2fs_convert_inline_dir(dir, ipage, dentry_blk); + if (err) + return err; + err = -EAGAIN; + goto out; + } + + if (inode) { + down_write(&F2FS_I(inode)->i_sem); + page = init_inode_metadata(inode, dir, name, ipage); + if (IS_ERR(page)) { + err = PTR_ERR(page); + goto fail; + } + } + + f2fs_wait_on_page_writeback(ipage, NODE); + + name_hash = f2fs_dentry_hash(name); + make_dentry_ptr(NULL, &d, (void *)dentry_blk, 2); + f2fs_update_dentry(ino, mode, &d, name, name_hash, bit_pos); + + set_page_dirty(ipage); + + /* we don't need to mark_inode_dirty now */ + if (inode) { + F2FS_I(inode)->i_pino = dir->i_ino; + update_inode(inode, page); + f2fs_put_page(page, 1); + } + + update_parent_metadata(dir, inode, 0); +fail: + if (inode) + up_write(&F2FS_I(inode)->i_sem); + + if (is_inode_flag_set(F2FS_I(dir), FI_UPDATE_DIR)) { + update_inode(dir, ipage); + clear_inode_flag(F2FS_I(dir), FI_UPDATE_DIR); + } +out: + f2fs_put_page(ipage, 1); + return err; +} + +void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, struct page *page, + struct inode *dir, struct inode *inode) +{ + struct f2fs_inline_dentry *inline_dentry; + int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len)); + unsigned int bit_pos; + int i; + + lock_page(page); + f2fs_wait_on_page_writeback(page, NODE); + + inline_dentry = inline_data_addr(page); + bit_pos = dentry - inline_dentry->dentry; + for (i = 0; i < slots; i++) + test_and_clear_bit_le(bit_pos + i, + &inline_dentry->dentry_bitmap); + + set_page_dirty(page); + + dir->i_ctime = dir->i_mtime = CURRENT_TIME; + + if (inode) + f2fs_drop_nlink(dir, inode, page); + + f2fs_put_page(page, 1); +} + +bool f2fs_empty_inline_dir(struct inode *dir) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct page *ipage; + unsigned int bit_pos = 2; + struct f2fs_inline_dentry *dentry_blk; + + ipage = get_node_page(sbi, dir->i_ino); + if (IS_ERR(ipage)) + return false; + + dentry_blk = inline_data_addr(ipage); + bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, + NR_INLINE_DENTRY, + bit_pos); + + f2fs_put_page(ipage, 1); + + if (bit_pos < NR_INLINE_DENTRY) + return false; + + return true; +} + +int f2fs_read_inline_dir(struct file *file, void *dirent, filldir_t filldir, + struct f2fs_str *fstr) +{ + unsigned long pos = file->f_pos; + unsigned int bit_pos = 0; + struct inode *inode = file_inode(file); + struct f2fs_inline_dentry *inline_dentry = NULL; + struct page *ipage = NULL; + struct f2fs_dentry_ptr d; + + if (pos >= NR_INLINE_DENTRY) + return 0; + + bit_pos = (pos % NR_INLINE_DENTRY); + + ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ipage)) + return PTR_ERR(ipage); + + inline_dentry = inline_data_addr(ipage); + + make_dentry_ptr(inode, &d, (void *)inline_dentry, 2); + + if (!f2fs_fill_dentries(file, dirent, filldir, &d, 0, bit_pos, fstr)) + file->f_pos = NR_INLINE_DENTRY; + + f2fs_put_page(ipage, 1); + return 0; +} + +int f2fs_inline_data_fiemap(struct inode *inode, + struct fiemap_extent_info *fieinfo, __u64 start, __u64 len) +{ + __u64 byteaddr, ilen; + __u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED | + FIEMAP_EXTENT_LAST; + struct node_info ni; + struct page *ipage; + int err = 0; + + ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ipage)) + return PTR_ERR(ipage); + + if (!f2fs_has_inline_data(inode)) { + err = -EAGAIN; + goto out; + } + + ilen = min_t(size_t, MAX_INLINE_DATA, i_size_read(inode)); + if (start >= ilen) + goto out; + if (start + len < ilen) + ilen = start + len; + ilen -= start; + + get_node_info(F2FS_I_SB(inode), inode->i_ino, &ni); + byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits; + byteaddr += (char *)inline_data_addr(ipage) - (char *)F2FS_INODE(ipage); + err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags); +out: + f2fs_put_page(ipage, 1); + return err; +} diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c new file mode 100644 index 00000000000..ad480e595a1 --- /dev/null +++ b/fs/f2fs/inode.c @@ -0,0 +1,429 @@ +/* + * fs/f2fs/inode.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" + +#include + +void f2fs_set_inode_flags(struct inode *inode) +{ + unsigned int flags = F2FS_I(inode)->i_flags; + + inode->i_flags &= ~(S_SYNC | S_APPEND | S_IMMUTABLE | + S_NOATIME | S_DIRSYNC); + + if (flags & FS_SYNC_FL) + inode->i_flags |= S_SYNC; + if (flags & FS_APPEND_FL) + inode->i_flags |= S_APPEND; + if (flags & FS_IMMUTABLE_FL) + inode->i_flags |= S_IMMUTABLE; + if (flags & FS_NOATIME_FL) + inode->i_flags |= S_NOATIME; + if (flags & FS_DIRSYNC_FL) + inode->i_flags |= S_DIRSYNC; +} + +static void __get_inode_rdev(struct inode *inode, struct f2fs_inode *ri) +{ + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + if (ri->i_addr[0]) + inode->i_rdev = + old_decode_dev(le32_to_cpu(ri->i_addr[0])); + else + inode->i_rdev = + new_decode_dev(le32_to_cpu(ri->i_addr[1])); + } +} + +static bool __written_first_block(struct f2fs_inode *ri) +{ + block_t addr = le32_to_cpu(ri->i_addr[0]); + + if (addr != NEW_ADDR && addr != NULL_ADDR) + return true; + return false; +} + +static void __set_inode_rdev(struct inode *inode, struct f2fs_inode *ri) +{ + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + if (old_valid_dev(inode->i_rdev)) { + ri->i_addr[0] = + cpu_to_le32(old_encode_dev(inode->i_rdev)); + ri->i_addr[1] = 0; + } else { + ri->i_addr[0] = 0; + ri->i_addr[1] = + cpu_to_le32(new_encode_dev(inode->i_rdev)); + ri->i_addr[2] = 0; + } + } +} + +static void __recover_inline_status(struct inode *inode, struct page *ipage) +{ + void *inline_data = inline_data_addr(ipage); + __le32 *start = inline_data; + __le32 *end = start + MAX_INLINE_DATA / sizeof(__le32); + + while (start < end) { + if (*start++) { + f2fs_wait_on_page_writeback(ipage, NODE); + + set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); + set_raw_inline(F2FS_I(inode), F2FS_INODE(ipage)); + set_page_dirty(ipage); + return; + } + } + return; +} + +static int do_read_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct page *node_page; + struct f2fs_inode *ri; + + /* Check if ino is within scope */ + if (check_nid_range(sbi, inode->i_ino)) { + f2fs_msg(inode->i_sb, KERN_ERR, "bad inode number: %lu", + (unsigned long) inode->i_ino); + WARN_ON(1); + return -EINVAL; + } + + node_page = get_node_page(sbi, inode->i_ino); + if (IS_ERR(node_page)) + return PTR_ERR(node_page); + + ri = F2FS_INODE(node_page); + + inode->i_mode = le16_to_cpu(ri->i_mode); + inode->i_uid = le32_to_cpu(ri->i_uid); + inode->i_gid = le32_to_cpu(ri->i_gid); + set_nlink(inode, le32_to_cpu(ri->i_links)); + inode->i_size = le64_to_cpu(ri->i_size); + inode->i_blocks = le64_to_cpu(ri->i_blocks); + + inode->i_atime.tv_sec = le64_to_cpu(ri->i_atime); + inode->i_ctime.tv_sec = le64_to_cpu(ri->i_ctime); + inode->i_mtime.tv_sec = le64_to_cpu(ri->i_mtime); + inode->i_atime.tv_nsec = le32_to_cpu(ri->i_atime_nsec); + inode->i_ctime.tv_nsec = le32_to_cpu(ri->i_ctime_nsec); + inode->i_mtime.tv_nsec = le32_to_cpu(ri->i_mtime_nsec); + inode->i_generation = le32_to_cpu(ri->i_generation); + + fi->i_current_depth = le32_to_cpu(ri->i_current_depth); + fi->i_xattr_nid = le32_to_cpu(ri->i_xattr_nid); + fi->i_flags = le32_to_cpu(ri->i_flags); + fi->flags = 0; + fi->i_advise = ri->i_advise; + fi->i_pino = le32_to_cpu(ri->i_pino); + fi->i_dir_level = ri->i_dir_level; + + f2fs_init_extent_tree(inode, &ri->i_ext); + + get_inline_info(fi, ri); + + /* check data exist */ + if (f2fs_has_inline_data(inode) && !f2fs_exist_data(inode)) + __recover_inline_status(inode, node_page); + + /* get rdev by using inline_info */ + __get_inode_rdev(inode, ri); + + if (__written_first_block(ri)) + set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); + + f2fs_put_page(node_page, 1); + + stat_inc_inline_xattr(inode); + stat_inc_inline_inode(inode); + stat_inc_inline_dir(inode); + + return 0; +} + +struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct inode *inode; + int ret = 0; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode->i_state & I_NEW)) { + trace_f2fs_iget(inode); + return inode; + } + if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi)) + goto make_now; + + ret = do_read_inode(inode); + if (ret) + goto bad_inode; +make_now: + if (ino == F2FS_NODE_INO(sbi)) { + inode->i_mapping->a_ops = &f2fs_node_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_ZERO); + } else if (ino == F2FS_META_INO(sbi)) { + inode->i_mapping->a_ops = &f2fs_meta_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_ZERO); + } else if (S_ISREG(inode->i_mode)) { + inode->i_op = &f2fs_file_inode_operations; + inode->i_fop = &f2fs_file_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &f2fs_dir_inode_operations; + inode->i_fop = &f2fs_dir_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_HIGH_ZERO); + } else if (S_ISLNK(inode->i_mode)) { + if (f2fs_encrypted_inode(inode)) + inode->i_op = &f2fs_encrypted_symlink_inode_operations; + else + inode->i_op = &f2fs_symlink_inode_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + inode->i_op = &f2fs_special_inode_operations; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + } else { + ret = -EIO; + goto bad_inode; + } + unlock_new_inode(inode); + trace_f2fs_iget(inode); + return inode; + +bad_inode: + iget_failed(inode); + trace_f2fs_iget_exit(inode, ret); + return ERR_PTR(ret); +} + +void update_inode(struct inode *inode, struct page *node_page) +{ + struct f2fs_inode *ri; + + f2fs_wait_on_page_writeback(node_page, NODE); + + ri = F2FS_INODE(node_page); + + ri->i_mode = cpu_to_le16(inode->i_mode); + ri->i_advise = F2FS_I(inode)->i_advise; + ri->i_uid = cpu_to_le32(inode->i_uid); + ri->i_gid = cpu_to_le32(inode->i_gid); + ri->i_links = cpu_to_le32(inode->i_nlink); + ri->i_size = cpu_to_le64(i_size_read(inode)); + ri->i_blocks = cpu_to_le64(inode->i_blocks); + + if (F2FS_I(inode)->extent_tree) + set_raw_extent(&F2FS_I(inode)->extent_tree->largest, + &ri->i_ext); + else + memset(&ri->i_ext, 0, sizeof(ri->i_ext)); + set_raw_inline(F2FS_I(inode), ri); + + ri->i_atime = cpu_to_le64(inode->i_atime.tv_sec); + ri->i_ctime = cpu_to_le64(inode->i_ctime.tv_sec); + ri->i_mtime = cpu_to_le64(inode->i_mtime.tv_sec); + ri->i_atime_nsec = cpu_to_le32(inode->i_atime.tv_nsec); + ri->i_ctime_nsec = cpu_to_le32(inode->i_ctime.tv_nsec); + ri->i_mtime_nsec = cpu_to_le32(inode->i_mtime.tv_nsec); + ri->i_current_depth = cpu_to_le32(F2FS_I(inode)->i_current_depth); + ri->i_xattr_nid = cpu_to_le32(F2FS_I(inode)->i_xattr_nid); + ri->i_flags = cpu_to_le32(F2FS_I(inode)->i_flags); + ri->i_pino = cpu_to_le32(F2FS_I(inode)->i_pino); + ri->i_generation = cpu_to_le32(inode->i_generation); + ri->i_dir_level = F2FS_I(inode)->i_dir_level; + + __set_inode_rdev(inode, ri); + set_cold_node(inode, node_page); + set_page_dirty(node_page); + + clear_inode_flag(F2FS_I(inode), FI_DIRTY_INODE); +} + +void update_inode_page(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct page *node_page; +retry: + node_page = get_node_page(sbi, inode->i_ino); + if (IS_ERR(node_page)) { + int err = PTR_ERR(node_page); + if (err == -ENOMEM) { + cond_resched(); + goto retry; + } else if (err != -ENOENT) { + f2fs_stop_checkpoint(sbi); + } + return; + } + update_inode(inode, node_page); + f2fs_put_page(node_page, 1); +} + +int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (inode->i_ino == F2FS_NODE_INO(sbi) || + inode->i_ino == F2FS_META_INO(sbi)) + return 0; + + if (!is_inode_flag_set(F2FS_I(inode), FI_DIRTY_INODE)) + return 0; + + /* + * We need to balance fs here to prevent from producing dirty node pages + * during the urgent cleaning time when runing out of free sections. + */ + update_inode_page(inode); + + f2fs_balance_fs(sbi); + return 0; +} + +/* + * Called at the last iput() if i_nlink is zero + */ +void f2fs_evict_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + nid_t xnid = fi->i_xattr_nid; + int err = 0; + + /* some remained atomic pages should discarded */ + if (f2fs_is_atomic_file(inode)) + commit_inmem_pages(inode, true); + + trace_f2fs_evict_inode(inode); + truncate_inode_pages(&inode->i_data, 0); + + if (inode->i_ino == F2FS_NODE_INO(sbi) || + inode->i_ino == F2FS_META_INO(sbi)) + goto out_clear; + + f2fs_bug_on(sbi, get_dirty_pages(inode)); + remove_dirty_dir_inode(inode); + + f2fs_destroy_extent_tree(inode); + + if (inode->i_nlink || is_bad_inode(inode)) + goto no_delete; + + set_inode_flag(fi, FI_NO_ALLOC); + i_size_write(inode, 0); + + if (F2FS_HAS_BLOCKS(inode)) + err = f2fs_truncate(inode, true); + + if (!err) { + f2fs_lock_op(sbi); + err = remove_inode_page(inode); + f2fs_unlock_op(sbi); + } + +no_delete: + stat_dec_inline_xattr(inode); + stat_dec_inline_dir(inode); + stat_dec_inline_inode(inode); + + invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, inode->i_ino); + if (xnid) + invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid); + if (is_inode_flag_set(fi, FI_APPEND_WRITE)) + add_dirty_inode(sbi, inode->i_ino, APPEND_INO); + if (is_inode_flag_set(fi, FI_UPDATE_WRITE)) + add_dirty_inode(sbi, inode->i_ino, UPDATE_INO); + if (is_inode_flag_set(fi, FI_FREE_NID)) { + if (err && err != -ENOENT) + alloc_nid_done(sbi, inode->i_ino); + else + alloc_nid_failed(sbi, inode->i_ino); + clear_inode_flag(fi, FI_FREE_NID); + } + + if (err && err != -ENOENT) { + if (!exist_written_data(sbi, inode->i_ino, ORPHAN_INO)) { + /* + * get here because we failed to release resource + * of inode previously, reminder our user to run fsck + * for fixing. + */ + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_msg(sbi->sb, KERN_WARNING, + "inode (ino:%lu) resource leak, run fsck " + "to fix this issue!", inode->i_ino); + } + } +out_clear: +#ifdef CONFIG_F2FS_FS_ENCRYPTION + if (fi->i_crypt_info) + f2fs_free_encryption_info(inode, fi->i_crypt_info); +#endif + end_writeback(inode); +} + +/* caller should call f2fs_lock_op() */ +void handle_failed_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int err = 0; + + clear_nlink(inode); + make_bad_inode(inode); + unlock_new_inode(inode); + + i_size_write(inode, 0); + if (F2FS_HAS_BLOCKS(inode)) + err = f2fs_truncate(inode, false); + + if (!err) + err = remove_inode_page(inode); + + /* + * if we skip truncate_node in remove_inode_page bacause we failed + * before, it's better to find another way to release resource of + * this inode (e.g. valid block count, node block or nid). Here we + * choose to add this inode to orphan list, so that we can call iput + * for releasing in orphan recovery flow. + * + * Note: we should add inode to orphan list before f2fs_unlock_op() + * so we can prevent losing this orphan when encoutering checkpoint + * and following suddenly power-off. + */ + if (err && err != -ENOENT) { + err = acquire_orphan_inode(sbi); + if (!err) + add_orphan_inode(sbi, inode->i_ino); + } + + set_inode_flag(F2FS_I(inode), FI_FREE_NID); + f2fs_unlock_op(sbi); + + /* iput will drop the inode object */ + iput(inode); +} diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c new file mode 100644 index 00000000000..469d9000b51 --- /dev/null +++ b/fs/f2fs/namei.c @@ -0,0 +1,802 @@ +/* + * fs/f2fs/namei.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "xattr.h" +#include "acl.h" +#include + +static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + nid_t ino; + struct inode *inode; + bool nid_free = false; + int err; + + inode = new_inode(dir->i_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + f2fs_lock_op(sbi); + if (!alloc_nid(sbi, &ino)) { + f2fs_unlock_op(sbi); + err = -ENOSPC; + goto fail; + } + f2fs_unlock_op(sbi); + + inode_init_owner(inode, dir, mode); + + inode->i_ino = ino; + inode->i_blocks = 0; + inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; + inode->i_generation = sbi->s_next_generation++; + + err = insert_inode_locked(inode); + if (err) { + err = -EINVAL; + nid_free = true; + goto fail; + } + + /* If the directory encrypted, then we should encrypt the inode. */ + if (f2fs_encrypted_inode(dir) && f2fs_may_encrypt(inode)) + f2fs_set_encrypted_inode(inode); + + if (f2fs_may_inline_data(inode)) + set_inode_flag(F2FS_I(inode), FI_INLINE_DATA); + if (f2fs_may_inline_dentry(inode)) + set_inode_flag(F2FS_I(inode), FI_INLINE_DENTRY); + + f2fs_init_extent_tree(inode, NULL); + + stat_inc_inline_xattr(inode); + stat_inc_inline_inode(inode); + stat_inc_inline_dir(inode); + + trace_f2fs_new_inode(inode, 0); + mark_inode_dirty(inode); + return inode; + +fail: + trace_f2fs_new_inode(inode, err); + make_bad_inode(inode); + if (nid_free) + set_inode_flag(F2FS_I(inode), FI_FREE_NID); + iput(inode); + return ERR_PTR(err); +} + +static int is_multimedia_file(const unsigned char *s, const char *sub) +{ + size_t slen = strlen(s); + size_t sublen = strlen(sub); + + /* + * filename format of multimedia file should be defined as: + * "filename + '.' + extension". + */ + if (slen < sublen + 2) + return 0; + + if (s[slen - sublen - 1] != '.') + return 0; + + return !strncasecmp(s + slen - sublen, sub, sublen); +} + +/* + * Set multimedia files as cold files for hot/cold data separation + */ +static inline void set_cold_files(struct f2fs_sb_info *sbi, struct inode *inode, + const unsigned char *name) +{ + int i; + __u8 (*extlist)[8] = sbi->raw_super->extension_list; + + int count = le32_to_cpu(sbi->raw_super->extension_count); + for (i = 0; i < count; i++) { + if (is_multimedia_file(name, extlist[i])) { + file_set_cold(inode); + break; + } + } +} + +static int f2fs_create(struct inode *dir, struct dentry *dentry, umode_t mode, + struct nameidata *nd) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct inode *inode; + nid_t ino = 0; + int err; + + f2fs_balance_fs(sbi); + + inode = f2fs_new_inode(dir, mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (!test_opt(sbi, DISABLE_EXT_IDENTIFY)) + set_cold_files(sbi, inode, dentry->d_name.name); + + inode->i_op = &f2fs_file_inode_operations; + inode->i_fop = &f2fs_file_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + ino = inode->i_ino; + + f2fs_lock_op(sbi); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi); + + alloc_nid_done(sbi, ino); + + d_instantiate(dentry, inode); + unlock_new_inode(inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + return 0; +out: + handle_failed_inode(inode); + return err; +} + +static int f2fs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct inode *inode = old_dentry->d_inode; + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + int err; + + if (f2fs_encrypted_inode(dir) && + !f2fs_is_child_context_consistent_with_parent(dir, inode)) + return -EPERM; + + f2fs_balance_fs(sbi); + + inode->i_ctime = CURRENT_TIME; + ihold(inode); + + set_inode_flag(F2FS_I(inode), FI_INC_LINK); + f2fs_lock_op(sbi); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi); + + d_instantiate(dentry, inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + return 0; +out: + clear_inode_flag(F2FS_I(inode), FI_INC_LINK); + iput(inode); + f2fs_unlock_op(sbi); + return err; +} + +struct dentry *f2fs_get_parent(struct dentry *child) +{ + struct qstr dotdot = {.len = 2, .name = ".."}; + unsigned long ino = f2fs_inode_by_name(child->d_inode, &dotdot); + if (!ino) + return ERR_PTR(-ENOENT); + return d_obtain_alias(f2fs_iget(child->d_inode->i_sb, ino)); +} + +static int __recover_dot_dentries(struct inode *dir, nid_t pino) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct qstr dot = {.len = 1, .name = "."}; + struct qstr dotdot = {.len = 2, .name = ".."}; + struct f2fs_dir_entry *de; + struct page *page; + int err = 0; + + f2fs_lock_op(sbi); + + de = f2fs_find_entry(dir, &dot, &page); + if (de) { + f2fs_dentry_kunmap(dir, page); + f2fs_put_page(page, 0); + } else { + err = __f2fs_add_link(dir, &dot, NULL, dir->i_ino, S_IFDIR); + if (err) + goto out; + } + + de = f2fs_find_entry(dir, &dotdot, &page); + if (de) { + f2fs_dentry_kunmap(dir, page); + f2fs_put_page(page, 0); + } else { + err = __f2fs_add_link(dir, &dotdot, NULL, pino, S_IFDIR); + } +out: + if (!err) { + clear_inode_flag(F2FS_I(dir), FI_INLINE_DOTS); + mark_inode_dirty(dir); + } + + f2fs_unlock_op(sbi); + return err; +} + +static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry, + struct nameidata *nd) +{ + struct inode *inode = NULL; + struct f2fs_dir_entry *de; + struct page *page; + nid_t ino; + int err = 0; + + if (dentry->d_name.len > F2FS_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + de = f2fs_find_entry(dir, &dentry->d_name, &page); + if (!de) + return d_splice_alias(inode, dentry); + + ino = le32_to_cpu(de->ino); + f2fs_dentry_kunmap(dir, page); + f2fs_put_page(page, 0); + + inode = f2fs_iget(dir->i_sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + if (f2fs_has_inline_dots(inode)) { + err = __recover_dot_dentries(inode, dir->i_ino); + if (err) + goto err_out; + } + return d_splice_alias(inode, dentry); + +err_out: + iget_failed(inode); + return ERR_PTR(err); +} + +static int f2fs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct inode *inode = dentry->d_inode; + struct f2fs_dir_entry *de; + struct page *page; + int err = -ENOENT; + + trace_f2fs_unlink_enter(dir, dentry); + f2fs_balance_fs(sbi); + + de = f2fs_find_entry(dir, &dentry->d_name, &page); + if (!de) + goto fail; + + f2fs_lock_op(sbi); + err = acquire_orphan_inode(sbi); + if (err) { + f2fs_unlock_op(sbi); + f2fs_dentry_kunmap(dir, page); + f2fs_put_page(page, 0); + goto fail; + } + f2fs_delete_entry(de, page, dir, inode); + f2fs_unlock_op(sbi); + + /* In order to evict this inode, we set it dirty */ + mark_inode_dirty(inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); +fail: + trace_f2fs_unlink_exit(inode, err); + return err; +} + +static void *f2fs_follow_link(struct dentry *dentry, struct nameidata *nd) +{ + struct page *page; + char *link; + + page = page_follow_link_light(dentry, nd); + if (IS_ERR(page)) + return page; + + link = nd_get_link(nd); + if (IS_ERR(link)) + return link; + + /* this is broken symlink case */ + if (*link == 0) { + kunmap(page); + page_cache_release(page); + return ERR_PTR(-ENOENT); + } + return page; +} + +static int f2fs_symlink(struct inode *dir, struct dentry *dentry, + const char *symname) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct inode *inode; + size_t len = strlen(symname); + size_t p_len; + char *p_str; + struct f2fs_str disk_link = FSTR_INIT(NULL, 0); + struct f2fs_encrypted_symlink_data *sd = NULL; + int err; + + if (len > dir->i_sb->s_blocksize) + return -ENAMETOOLONG; + + f2fs_balance_fs(sbi); + + inode = f2fs_new_inode(dir, S_IFLNK | S_IRWXUGO); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (f2fs_encrypted_inode(inode)) + inode->i_op = &f2fs_encrypted_symlink_inode_operations; + else + inode->i_op = &f2fs_symlink_inode_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + + f2fs_lock_op(sbi); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi); + alloc_nid_done(sbi, inode->i_ino); + + if (f2fs_encrypted_inode(dir)) { + struct qstr istr = QSTR_INIT(symname, len); + + err = f2fs_get_encryption_info(inode); + if (err) + goto err_out; + + err = f2fs_fname_crypto_alloc_buffer(inode, len, &disk_link); + if (err) + goto err_out; + + err = f2fs_fname_usr_to_disk(inode, &istr, &disk_link); + if (err < 0) + goto err_out; + + p_len = encrypted_symlink_data_len(disk_link.len) + 1; + + if (p_len > dir->i_sb->s_blocksize) { + err = -ENAMETOOLONG; + goto err_out; + } + + sd = kzalloc(p_len, GFP_NOFS); + if (!sd) { + err = -ENOMEM; + goto err_out; + } + memcpy(sd->encrypted_path, disk_link.name, disk_link.len); + sd->len = cpu_to_le16(disk_link.len); + p_str = (char *)sd; + } else { + p_len = len + 1; + p_str = (char *)symname; + } + + err = page_symlink(inode, p_str, p_len); + +err_out: + d_instantiate(dentry, inode); + unlock_new_inode(inode); + + /* + * Let's flush symlink data in order to avoid broken symlink as much as + * possible. Nevertheless, fsyncing is the best way, but there is no + * way to get a file descriptor in order to flush that. + * + * Note that, it needs to do dir->fsync to make this recoverable. + * If the symlink path is stored into inline_data, there is no + * performance regression. + */ + if (!err) { + filemap_write_and_wait_range(inode->i_mapping, 0, p_len - 1); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + } else { + f2fs_unlink(dir, dentry); + } + + kfree(sd); + f2fs_fname_crypto_free_buffer(&disk_link); + return err; +out: + handle_failed_inode(inode); + return err; +} + +static int f2fs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct inode *inode; + int err; + + f2fs_balance_fs(sbi); + + inode = f2fs_new_inode(dir, S_IFDIR | mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_op = &f2fs_dir_inode_operations; + inode->i_fop = &f2fs_dir_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_HIGH_ZERO); + + set_inode_flag(F2FS_I(inode), FI_INC_LINK); + f2fs_lock_op(sbi); + err = f2fs_add_link(dentry, inode); + if (err) + goto out_fail; + f2fs_unlock_op(sbi); + + alloc_nid_done(sbi, inode->i_ino); + + d_instantiate(dentry, inode); + unlock_new_inode(inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + return 0; + +out_fail: + clear_inode_flag(F2FS_I(inode), FI_INC_LINK); + handle_failed_inode(inode); + return err; +} + +static int f2fs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + if (f2fs_empty_dir(inode)) + return f2fs_unlink(dir, dentry); + return -ENOTEMPTY; +} + +static int f2fs_mknod(struct inode *dir, struct dentry *dentry, + umode_t mode, dev_t rdev) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct inode *inode; + int err = 0; + + if (!new_valid_dev(rdev)) + return -EINVAL; + + f2fs_balance_fs(sbi); + + inode = f2fs_new_inode(dir, mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + init_special_inode(inode, inode->i_mode, rdev); + inode->i_op = &f2fs_special_inode_operations; + + f2fs_lock_op(sbi); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi); + + alloc_nid_done(sbi, inode->i_ino); + + d_instantiate(dentry, inode); + unlock_new_inode(inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + return 0; +out: + handle_failed_inode(inode); + return err; +} + +static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir); + struct inode *old_inode = old_dentry->d_inode; + struct inode *new_inode = new_dentry->d_inode; + struct page *old_dir_page; + struct page *old_page, *new_page; + struct f2fs_dir_entry *old_dir_entry = NULL; + struct f2fs_dir_entry *old_entry; + struct f2fs_dir_entry *new_entry; + int err = -ENOENT; + + if ((old_dir != new_dir) && f2fs_encrypted_inode(new_dir) && + !f2fs_is_child_context_consistent_with_parent(new_dir, + old_inode)) { + err = -EPERM; + goto out; + } + + f2fs_balance_fs(sbi); + + old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_page); + if (!old_entry) + goto out; + + if (S_ISDIR(old_inode->i_mode)) { + err = -EIO; + old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_page); + if (!old_dir_entry) + goto out_old; + } + + if (new_inode) { + + err = -ENOTEMPTY; + if (old_dir_entry && !f2fs_empty_dir(new_inode)) + goto out_dir; + + err = -ENOENT; + new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, + &new_page); + if (!new_entry) + goto out_dir; + + f2fs_lock_op(sbi); + + err = acquire_orphan_inode(sbi); + if (err) + goto put_out_dir; + + if (update_dent_inode(old_inode, new_inode, + &new_dentry->d_name)) { + release_orphan_inode(sbi); + goto put_out_dir; + } + + f2fs_set_link(new_dir, new_entry, new_page, old_inode); + + new_inode->i_ctime = CURRENT_TIME; + down_write(&F2FS_I(new_inode)->i_sem); + if (old_dir_entry) + drop_nlink(new_inode); + drop_nlink(new_inode); + up_write(&F2FS_I(new_inode)->i_sem); + + mark_inode_dirty(new_inode); + + if (!new_inode->i_nlink) + add_orphan_inode(sbi, new_inode->i_ino); + else + release_orphan_inode(sbi); + + update_inode_page(old_inode); + update_inode_page(new_inode); + } else { + f2fs_lock_op(sbi); + + err = f2fs_add_link(new_dentry, old_inode); + if (err) { + f2fs_unlock_op(sbi); + goto out_dir; + } + + if (old_dir_entry) { + inc_nlink(new_dir); + update_inode_page(new_dir); + } + } + + down_write(&F2FS_I(old_inode)->i_sem); + file_lost_pino(old_inode); + if (new_inode && file_enc_name(new_inode)) + file_set_enc_name(old_inode); + up_write(&F2FS_I(old_inode)->i_sem); + + old_inode->i_ctime = CURRENT_TIME; + mark_inode_dirty(old_inode); + + f2fs_delete_entry(old_entry, old_page, old_dir, NULL); + + if (old_dir_entry) { + if (old_dir != new_dir) { + f2fs_set_link(old_inode, old_dir_entry, + old_dir_page, new_dir); + update_inode_page(old_inode); + } else { + f2fs_dentry_kunmap(old_inode, old_dir_page); + f2fs_put_page(old_dir_page, 0); + } + drop_nlink(old_dir); + mark_inode_dirty(old_dir); + update_inode_page(old_dir); + } + + f2fs_unlock_op(sbi); + + if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) + f2fs_sync_fs(sbi->sb, 1); + return 0; + +put_out_dir: + f2fs_unlock_op(sbi); + f2fs_dentry_kunmap(new_dir, new_page); + f2fs_put_page(new_page, 0); +out_dir: + if (old_dir_entry) { + f2fs_dentry_kunmap(old_inode, old_dir_page); + f2fs_put_page(old_dir_page, 0); + } +out_old: + f2fs_dentry_kunmap(old_dir, old_page); + f2fs_put_page(old_page, 0); +out: + return err; +} + +#ifdef CONFIG_F2FS_FS_ENCRYPTION +static void *f2fs_encrypted_follow_link(struct dentry *dentry, + struct nameidata *nd) +{ + struct page *cpage = NULL; + char *caddr, *paddr = NULL; + struct f2fs_str cstr; + struct f2fs_str pstr = FSTR_INIT(NULL, 0); + struct inode *inode = dentry->d_inode; + struct f2fs_encrypted_symlink_data *sd; + loff_t size = min_t(loff_t, i_size_read(inode), PAGE_SIZE - 1); + u32 max_size = inode->i_sb->s_blocksize; + int res; + + res = f2fs_get_encryption_info(inode); + if (res) + return ERR_PTR(res); + + cpage = read_mapping_page(inode->i_mapping, 0, NULL); + if (IS_ERR(cpage)) + return cpage; + caddr = kmap(cpage); + caddr[size] = 0; + + /* Symlink is encrypted */ + sd = (struct f2fs_encrypted_symlink_data *)caddr; + cstr.len = le16_to_cpu(sd->len); + cstr.name = kmalloc(cstr.len, GFP_NOFS); + if (!cstr.name) { + res = -ENOMEM; + goto errout; + } + memcpy(cstr.name, sd->encrypted_path, cstr.len); + + /* this is broken symlink case */ + if (cstr.name[0] == 0 && cstr.len == 0) { + res = -ENOENT; + goto errout; + } + + if ((cstr.len + sizeof(struct f2fs_encrypted_symlink_data) - 1) > + max_size) { + /* Symlink data on the disk is corrupted */ + res = -EIO; + goto errout; + } + res = f2fs_fname_crypto_alloc_buffer(inode, cstr.len, &pstr); + if (res) + goto errout; + + res = f2fs_fname_disk_to_usr(inode, NULL, &cstr, &pstr); + if (res < 0) + goto errout; + + kfree(cstr.name); + + paddr = pstr.name; + + /* Null-terminate the name */ + paddr[res] = '\0'; + nd_set_link(nd, paddr); + + kunmap(cpage); + page_cache_release(cpage); + return NULL; +errout: + kfree(cstr.name); + f2fs_fname_crypto_free_buffer(&pstr); + kunmap(cpage); + page_cache_release(cpage); + return ERR_PTR(res); +} + +void kfree_put_link(struct dentry *dentry, struct nameidata *nd, + void *cookie) +{ + char *s = nd_get_link(nd); + if (!IS_ERR(s)) + kfree(s); +} + +const struct inode_operations f2fs_encrypted_symlink_inode_operations = { + .readlink = generic_readlink, + .follow_link = f2fs_encrypted_follow_link, + .put_link = kfree_put_link, + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = f2fs_listxattr, + .removexattr = generic_removexattr, +}; +#endif + +const struct inode_operations f2fs_dir_inode_operations = { + .create = f2fs_create, + .lookup = f2fs_lookup, + .link = f2fs_link, + .unlink = f2fs_unlink, + .symlink = f2fs_symlink, + .mkdir = f2fs_mkdir, + .rmdir = f2fs_rmdir, + .mknod = f2fs_mknod, + .rename = f2fs_rename, + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .get_acl = f2fs_get_acl, +#ifdef CONFIG_F2FS_FS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = f2fs_listxattr, + .removexattr = generic_removexattr, +#endif +}; + +const struct inode_operations f2fs_symlink_inode_operations = { + .readlink = generic_readlink, + .follow_link = f2fs_follow_link, + .put_link = page_put_link, + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, +#ifdef CONFIG_F2FS_FS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = f2fs_listxattr, + .removexattr = generic_removexattr, +#endif +}; + +const struct inode_operations f2fs_special_inode_operations = { + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .get_acl = f2fs_get_acl, +#ifdef CONFIG_F2FS_FS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = f2fs_listxattr, + .removexattr = generic_removexattr, +#endif +}; diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c new file mode 100644 index 00000000000..413d7724bdf --- /dev/null +++ b/fs/f2fs/node.c @@ -0,0 +1,2140 @@ +/* + * fs/f2fs/node.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "trace.h" +#include + +#define on_build_free_nids(nmi) mutex_is_locked(&nm_i->build_lock) + +static struct kmem_cache *nat_entry_slab; +static struct kmem_cache *free_nid_slab; +static struct kmem_cache *nat_entry_set_slab; + +bool available_free_memory(struct f2fs_sb_info *sbi, int type) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct sysinfo val; + unsigned long avail_ram; + unsigned long mem_size = 0; + bool res = false; + + si_meminfo(&val); + + /* only uses low memory */ + avail_ram = val.totalram - val.totalhigh; + + /* + * give 25%, 25%, 50%, 50%, 50% memory for each components respectively + */ + if (type == FREE_NIDS) { + mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >> + PAGE_CACHE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); + } else if (type == NAT_ENTRIES) { + mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >> + PAGE_CACHE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); + } else if (type == DIRTY_DENTS) { + if (sbi->sb->s_bdi->dirty_exceeded) + return false; + mem_size = get_pages(sbi, F2FS_DIRTY_DENTS); + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + } else if (type == INO_ENTRIES) { + int i; + + for (i = 0; i <= UPDATE_INO; i++) + mem_size += (sbi->im[i].ino_num * + sizeof(struct ino_entry)) >> PAGE_CACHE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + } else if (type == EXTENT_CACHE) { + mem_size = (sbi->total_ext_tree * sizeof(struct extent_tree) + + atomic_read(&sbi->total_ext_node) * + sizeof(struct extent_node)) >> PAGE_CACHE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + } else { + if (sbi->sb->s_bdi->dirty_exceeded) + return false; + } + return res; +} + +static void clear_node_page_dirty(struct page *page) +{ + struct address_space *mapping = page->mapping; + unsigned int long flags; + + if (PageDirty(page)) { + spin_lock_irqsave(&mapping->tree_lock, flags); + radix_tree_tag_clear(&mapping->page_tree, + page_index(page), + PAGECACHE_TAG_DIRTY); + spin_unlock_irqrestore(&mapping->tree_lock, flags); + + clear_page_dirty_for_io(page); + dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); + } + ClearPageUptodate(page); +} + +static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid) +{ + pgoff_t index = current_nat_addr(sbi, nid); + return get_meta_page(sbi, index); +} + +static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct page *src_page; + struct page *dst_page; + pgoff_t src_off; + pgoff_t dst_off; + void *src_addr; + void *dst_addr; + struct f2fs_nm_info *nm_i = NM_I(sbi); + + src_off = current_nat_addr(sbi, nid); + dst_off = next_nat_addr(sbi, src_off); + + /* get current nat block page with lock */ + src_page = get_meta_page(sbi, src_off); + dst_page = grab_meta_page(sbi, dst_off); + f2fs_bug_on(sbi, PageDirty(src_page)); + + src_addr = page_address(src_page); + dst_addr = page_address(dst_page); + memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE); + set_page_dirty(dst_page); + f2fs_put_page(src_page, 1); + + set_to_next_nat(nm_i, nid); + + return dst_page; +} + +static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n) +{ + return radix_tree_lookup(&nm_i->nat_root, n); +} + +static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i, + nid_t start, unsigned int nr, struct nat_entry **ep) +{ + return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr); +} + +static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e) +{ + list_del(&e->list); + radix_tree_delete(&nm_i->nat_root, nat_get_nid(e)); + nm_i->nat_cnt--; + kmem_cache_free(nat_entry_slab, e); +} + +static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i, + struct nat_entry *ne) +{ + nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid); + struct nat_entry_set *head; + + if (get_nat_flag(ne, IS_DIRTY)) + return; + + head = radix_tree_lookup(&nm_i->nat_set_root, set); + if (!head) { + head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_NOFS); + + INIT_LIST_HEAD(&head->entry_list); + INIT_LIST_HEAD(&head->set_list); + head->set = set; + head->entry_cnt = 0; + f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head); + } + list_move_tail(&ne->list, &head->entry_list); + nm_i->dirty_nat_cnt++; + head->entry_cnt++; + set_nat_flag(ne, IS_DIRTY, true); +} + +static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i, + struct nat_entry *ne) +{ + nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid); + struct nat_entry_set *head; + + head = radix_tree_lookup(&nm_i->nat_set_root, set); + if (head) { + list_move_tail(&ne->list, &nm_i->nat_entries); + set_nat_flag(ne, IS_DIRTY, false); + head->entry_cnt--; + nm_i->dirty_nat_cnt--; + } +} + +static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i, + nid_t start, unsigned int nr, struct nat_entry_set **ep) +{ + return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep, + start, nr); +} + +int need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + bool need = false; + + down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid); + if (e) { + if (!get_nat_flag(e, IS_CHECKPOINTED) && + !get_nat_flag(e, HAS_FSYNCED_INODE)) + need = true; + } + up_read(&nm_i->nat_tree_lock); + return need; +} + +bool is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + bool is_cp = true; + + down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid); + if (e && !get_nat_flag(e, IS_CHECKPOINTED)) + is_cp = false; + up_read(&nm_i->nat_tree_lock); + return is_cp; +} + +bool need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + bool need_update = true; + + down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, ino); + if (e && get_nat_flag(e, HAS_LAST_FSYNC) && + (get_nat_flag(e, IS_CHECKPOINTED) || + get_nat_flag(e, HAS_FSYNCED_INODE))) + need_update = false; + up_read(&nm_i->nat_tree_lock); + return need_update; +} + +static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid) +{ + struct nat_entry *new; + + new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_NOFS); + f2fs_radix_tree_insert(&nm_i->nat_root, nid, new); + memset(new, 0, sizeof(struct nat_entry)); + nat_set_nid(new, nid); + nat_reset_flag(new); + list_add_tail(&new->list, &nm_i->nat_entries); + nm_i->nat_cnt++; + return new; +} + +static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid, + struct f2fs_nat_entry *ne) +{ + struct nat_entry *e; + + down_write(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid); + if (!e) { + e = grab_nat_entry(nm_i, nid); + node_info_from_raw_nat(&e->ni, ne); + } + up_write(&nm_i->nat_tree_lock); +} + +static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, + block_t new_blkaddr, bool fsync_done) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + + down_write(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, ni->nid); + if (!e) { + e = grab_nat_entry(nm_i, ni->nid); + copy_node_info(&e->ni, ni); + f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR); + } else if (new_blkaddr == NEW_ADDR) { + /* + * when nid is reallocated, + * previous nat entry can be remained in nat cache. + * So, reinitialize it with new information. + */ + copy_node_info(&e->ni, ni); + f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR); + } + + /* sanity check */ + f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr); + f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR && + new_blkaddr == NULL_ADDR); + f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR && + new_blkaddr == NEW_ADDR); + f2fs_bug_on(sbi, nat_get_blkaddr(e) != NEW_ADDR && + nat_get_blkaddr(e) != NULL_ADDR && + new_blkaddr == NEW_ADDR); + + /* increment version no as node is removed */ + if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) { + unsigned char version = nat_get_version(e); + nat_set_version(e, inc_node_version(version)); + + /* in order to reuse the nid */ + if (nm_i->next_scan_nid > ni->nid) + nm_i->next_scan_nid = ni->nid; + } + + /* change address */ + nat_set_blkaddr(e, new_blkaddr); + if (new_blkaddr == NEW_ADDR || new_blkaddr == NULL_ADDR) + set_nat_flag(e, IS_CHECKPOINTED, false); + __set_nat_cache_dirty(nm_i, e); + + /* update fsync_mark if its inode nat entry is still alive */ + if (ni->nid != ni->ino) + e = __lookup_nat_cache(nm_i, ni->ino); + if (e) { + if (fsync_done && ni->nid == ni->ino) + set_nat_flag(e, HAS_FSYNCED_INODE, true); + set_nat_flag(e, HAS_LAST_FSYNC, fsync_done); + } + up_write(&nm_i->nat_tree_lock); +} + +int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + int nr = nr_shrink; + + if (!down_write_trylock(&nm_i->nat_tree_lock)) + return 0; + + while (nr_shrink && !list_empty(&nm_i->nat_entries)) { + struct nat_entry *ne; + ne = list_first_entry(&nm_i->nat_entries, + struct nat_entry, list); + __del_from_nat_cache(nm_i, ne); + nr_shrink--; + } + up_write(&nm_i->nat_tree_lock); + return nr - nr_shrink; +} + +/* + * This function always returns success + */ +void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + nid_t start_nid = START_NID(nid); + struct f2fs_nat_block *nat_blk; + struct page *page = NULL; + struct f2fs_nat_entry ne; + struct nat_entry *e; + int i; + + ni->nid = nid; + + /* Check nat cache */ + down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid); + if (e) { + ni->ino = nat_get_ino(e); + ni->blk_addr = nat_get_blkaddr(e); + ni->version = nat_get_version(e); + } + up_read(&nm_i->nat_tree_lock); + if (e) + return; + + memset(&ne, 0, sizeof(struct f2fs_nat_entry)); + + /* Check current segment summary */ + mutex_lock(&curseg->curseg_mutex); + i = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 0); + if (i >= 0) { + ne = nat_in_journal(sum, i); + node_info_from_raw_nat(ni, &ne); + } + mutex_unlock(&curseg->curseg_mutex); + if (i >= 0) + goto cache; + + /* Fill node_info from nat page */ + page = get_current_nat_page(sbi, start_nid); + nat_blk = (struct f2fs_nat_block *)page_address(page); + ne = nat_blk->entries[nid - start_nid]; + node_info_from_raw_nat(ni, &ne); + f2fs_put_page(page, 1); +cache: + /* cache nat entry */ + cache_nat_entry(NM_I(sbi), nid, &ne); +} + +/* + * The maximum depth is four. + * Offset[0] will have raw inode offset. + */ +static int get_node_path(struct f2fs_inode_info *fi, long block, + int offset[4], unsigned int noffset[4]) +{ + const long direct_index = ADDRS_PER_INODE(fi); + const long direct_blks = ADDRS_PER_BLOCK; + const long dptrs_per_blk = NIDS_PER_BLOCK; + const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK; + const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK; + int n = 0; + int level = 0; + + noffset[0] = 0; + + if (block < direct_index) { + offset[n] = block; + goto got; + } + block -= direct_index; + if (block < direct_blks) { + offset[n++] = NODE_DIR1_BLOCK; + noffset[n] = 1; + offset[n] = block; + level = 1; + goto got; + } + block -= direct_blks; + if (block < direct_blks) { + offset[n++] = NODE_DIR2_BLOCK; + noffset[n] = 2; + offset[n] = block; + level = 1; + goto got; + } + block -= direct_blks; + if (block < indirect_blks) { + offset[n++] = NODE_IND1_BLOCK; + noffset[n] = 3; + offset[n++] = block / direct_blks; + noffset[n] = 4 + offset[n - 1]; + offset[n] = block % direct_blks; + level = 2; + goto got; + } + block -= indirect_blks; + if (block < indirect_blks) { + offset[n++] = NODE_IND2_BLOCK; + noffset[n] = 4 + dptrs_per_blk; + offset[n++] = block / direct_blks; + noffset[n] = 5 + dptrs_per_blk + offset[n - 1]; + offset[n] = block % direct_blks; + level = 2; + goto got; + } + block -= indirect_blks; + if (block < dindirect_blks) { + offset[n++] = NODE_DIND_BLOCK; + noffset[n] = 5 + (dptrs_per_blk * 2); + offset[n++] = block / indirect_blks; + noffset[n] = 6 + (dptrs_per_blk * 2) + + offset[n - 1] * (dptrs_per_blk + 1); + offset[n++] = (block / direct_blks) % dptrs_per_blk; + noffset[n] = 7 + (dptrs_per_blk * 2) + + offset[n - 2] * (dptrs_per_blk + 1) + + offset[n - 1]; + offset[n] = block % direct_blks; + level = 3; + goto got; + } else { + BUG(); + } +got: + return level; +} + +/* + * Caller should call f2fs_put_dnode(dn). + * Also, it should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op() only if ro is not set RDONLY_NODE. + * In the case of RDONLY_NODE, we don't need to care about mutex. + */ +int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct page *npage[4]; + struct page *parent = NULL; + int offset[4]; + unsigned int noffset[4]; + nid_t nids[4]; + int level, i; + int err = 0; + + level = get_node_path(F2FS_I(dn->inode), index, offset, noffset); + + nids[0] = dn->inode->i_ino; + npage[0] = dn->inode_page; + + if (!npage[0]) { + npage[0] = get_node_page(sbi, nids[0]); + if (IS_ERR(npage[0])) + return PTR_ERR(npage[0]); + } + + /* if inline_data is set, should not report any block indices */ + if (f2fs_has_inline_data(dn->inode) && index) { + err = -ENOENT; + f2fs_put_page(npage[0], 1); + goto release_out; + } + + parent = npage[0]; + if (level != 0) + nids[1] = get_nid(parent, offset[0], true); + dn->inode_page = npage[0]; + dn->inode_page_locked = true; + + /* get indirect or direct nodes */ + for (i = 1; i <= level; i++) { + bool done = false; + + if (!nids[i] && mode == ALLOC_NODE) { + /* alloc new node */ + if (!alloc_nid(sbi, &(nids[i]))) { + err = -ENOSPC; + goto release_pages; + } + + dn->nid = nids[i]; + npage[i] = new_node_page(dn, noffset[i], NULL); + if (IS_ERR(npage[i])) { + alloc_nid_failed(sbi, nids[i]); + err = PTR_ERR(npage[i]); + goto release_pages; + } + + set_nid(parent, offset[i - 1], nids[i], i == 1); + alloc_nid_done(sbi, nids[i]); + done = true; + } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) { + npage[i] = get_node_page_ra(parent, offset[i - 1]); + if (IS_ERR(npage[i])) { + err = PTR_ERR(npage[i]); + goto release_pages; + } + done = true; + } + if (i == 1) { + dn->inode_page_locked = false; + unlock_page(parent); + } else { + f2fs_put_page(parent, 1); + } + + if (!done) { + npage[i] = get_node_page(sbi, nids[i]); + if (IS_ERR(npage[i])) { + err = PTR_ERR(npage[i]); + f2fs_put_page(npage[0], 0); + goto release_out; + } + } + if (i < level) { + parent = npage[i]; + nids[i + 1] = get_nid(parent, offset[i], false); + } + } + dn->nid = nids[level]; + dn->ofs_in_node = offset[level]; + dn->node_page = npage[level]; + dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node); + return 0; + +release_pages: + f2fs_put_page(parent, 1); + if (i > 1) + f2fs_put_page(npage[0], 0); +release_out: + dn->inode_page = NULL; + dn->node_page = NULL; + return err; +} + +static void truncate_node(struct dnode_of_data *dn) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct node_info ni; + + get_node_info(sbi, dn->nid, &ni); + if (dn->inode->i_blocks == 0) { + f2fs_bug_on(sbi, ni.blk_addr != NULL_ADDR); + goto invalidate; + } + f2fs_bug_on(sbi, ni.blk_addr == NULL_ADDR); + + /* Deallocate node address */ + invalidate_blocks(sbi, ni.blk_addr); + dec_valid_node_count(sbi, dn->inode); + set_node_addr(sbi, &ni, NULL_ADDR, false); + + if (dn->nid == dn->inode->i_ino) { + remove_orphan_inode(sbi, dn->nid); + dec_valid_inode_count(sbi); + } else { + sync_inode_page(dn); + } +invalidate: + clear_node_page_dirty(dn->node_page); + set_sbi_flag(sbi, SBI_IS_DIRTY); + + f2fs_put_page(dn->node_page, 1); + + invalidate_mapping_pages(NODE_MAPPING(sbi), + dn->node_page->index, dn->node_page->index); + + dn->node_page = NULL; + trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr); +} + +static int truncate_dnode(struct dnode_of_data *dn) +{ + struct page *page; + + if (dn->nid == 0) + return 1; + + /* get direct node */ + page = get_node_page(F2FS_I_SB(dn->inode), dn->nid); + if (IS_ERR(page) && PTR_ERR(page) == -ENOENT) + return 1; + else if (IS_ERR(page)) + return PTR_ERR(page); + + /* Make dnode_of_data for parameter */ + dn->node_page = page; + dn->ofs_in_node = 0; + truncate_data_blocks(dn); + truncate_node(dn); + return 1; +} + +static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, + int ofs, int depth) +{ + struct dnode_of_data rdn = *dn; + struct page *page; + struct f2fs_node *rn; + nid_t child_nid; + unsigned int child_nofs; + int freed = 0; + int i, ret; + + if (dn->nid == 0) + return NIDS_PER_BLOCK + 1; + + trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr); + + page = get_node_page(F2FS_I_SB(dn->inode), dn->nid); + if (IS_ERR(page)) { + trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(page)); + return PTR_ERR(page); + } + + rn = F2FS_NODE(page); + if (depth < 3) { + for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) { + child_nid = le32_to_cpu(rn->in.nid[i]); + if (child_nid == 0) + continue; + rdn.nid = child_nid; + ret = truncate_dnode(&rdn); + if (ret < 0) + goto out_err; + set_nid(page, i, 0, false); + } + } else { + child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1; + for (i = ofs; i < NIDS_PER_BLOCK; i++) { + child_nid = le32_to_cpu(rn->in.nid[i]); + if (child_nid == 0) { + child_nofs += NIDS_PER_BLOCK + 1; + continue; + } + rdn.nid = child_nid; + ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1); + if (ret == (NIDS_PER_BLOCK + 1)) { + set_nid(page, i, 0, false); + child_nofs += ret; + } else if (ret < 0 && ret != -ENOENT) { + goto out_err; + } + } + freed = child_nofs; + } + + if (!ofs) { + /* remove current indirect node */ + dn->node_page = page; + truncate_node(dn); + freed++; + } else { + f2fs_put_page(page, 1); + } + trace_f2fs_truncate_nodes_exit(dn->inode, freed); + return freed; + +out_err: + f2fs_put_page(page, 1); + trace_f2fs_truncate_nodes_exit(dn->inode, ret); + return ret; +} + +static int truncate_partial_nodes(struct dnode_of_data *dn, + struct f2fs_inode *ri, int *offset, int depth) +{ + struct page *pages[2]; + nid_t nid[3]; + nid_t child_nid; + int err = 0; + int i; + int idx = depth - 2; + + nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]); + if (!nid[0]) + return 0; + + /* get indirect nodes in the path */ + for (i = 0; i < idx + 1; i++) { + /* reference count'll be increased */ + pages[i] = get_node_page(F2FS_I_SB(dn->inode), nid[i]); + if (IS_ERR(pages[i])) { + err = PTR_ERR(pages[i]); + idx = i - 1; + goto fail; + } + nid[i + 1] = get_nid(pages[i], offset[i + 1], false); + } + + /* free direct nodes linked to a partial indirect node */ + for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) { + child_nid = get_nid(pages[idx], i, false); + if (!child_nid) + continue; + dn->nid = child_nid; + err = truncate_dnode(dn); + if (err < 0) + goto fail; + set_nid(pages[idx], i, 0, false); + } + + if (offset[idx + 1] == 0) { + dn->node_page = pages[idx]; + dn->nid = nid[idx]; + truncate_node(dn); + } else { + f2fs_put_page(pages[idx], 1); + } + offset[idx]++; + offset[idx + 1] = 0; + idx--; +fail: + for (i = idx; i >= 0; i--) + f2fs_put_page(pages[i], 1); + + trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err); + + return err; +} + +/* + * All the block addresses of data and nodes should be nullified. + */ +int truncate_inode_blocks(struct inode *inode, pgoff_t from) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int err = 0, cont = 1; + int level, offset[4], noffset[4]; + unsigned int nofs = 0; + struct f2fs_inode *ri; + struct dnode_of_data dn; + struct page *page; + + trace_f2fs_truncate_inode_blocks_enter(inode, from); + + level = get_node_path(F2FS_I(inode), from, offset, noffset); +restart: + page = get_node_page(sbi, inode->i_ino); + if (IS_ERR(page)) { + trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(page)); + return PTR_ERR(page); + } + + set_new_dnode(&dn, inode, page, NULL, 0); + unlock_page(page); + + ri = F2FS_INODE(page); + switch (level) { + case 0: + case 1: + nofs = noffset[1]; + break; + case 2: + nofs = noffset[1]; + if (!offset[level - 1]) + goto skip_partial; + err = truncate_partial_nodes(&dn, ri, offset, level); + if (err < 0 && err != -ENOENT) + goto fail; + nofs += 1 + NIDS_PER_BLOCK; + break; + case 3: + nofs = 5 + 2 * NIDS_PER_BLOCK; + if (!offset[level - 1]) + goto skip_partial; + err = truncate_partial_nodes(&dn, ri, offset, level); + if (err < 0 && err != -ENOENT) + goto fail; + break; + default: + BUG(); + } + +skip_partial: + while (cont) { + dn.nid = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]); + switch (offset[0]) { + case NODE_DIR1_BLOCK: + case NODE_DIR2_BLOCK: + err = truncate_dnode(&dn); + break; + + case NODE_IND1_BLOCK: + case NODE_IND2_BLOCK: + err = truncate_nodes(&dn, nofs, offset[1], 2); + break; + + case NODE_DIND_BLOCK: + err = truncate_nodes(&dn, nofs, offset[1], 3); + cont = 0; + break; + + default: + BUG(); + } + if (err < 0 && err != -ENOENT) + goto fail; + if (offset[1] == 0 && + ri->i_nid[offset[0] - NODE_DIR1_BLOCK]) { + lock_page(page); + if (unlikely(page->mapping != NODE_MAPPING(sbi))) { + f2fs_put_page(page, 1); + goto restart; + } + f2fs_wait_on_page_writeback(page, NODE); + ri->i_nid[offset[0] - NODE_DIR1_BLOCK] = 0; + set_page_dirty(page); + unlock_page(page); + } + offset[1] = 0; + offset[0]++; + nofs += err; + } +fail: + f2fs_put_page(page, 0); + trace_f2fs_truncate_inode_blocks_exit(inode, err); + return err > 0 ? 0 : err; +} + +int truncate_xattr_node(struct inode *inode, struct page *page) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t nid = F2FS_I(inode)->i_xattr_nid; + struct dnode_of_data dn; + struct page *npage; + + if (!nid) + return 0; + + npage = get_node_page(sbi, nid); + if (IS_ERR(npage)) + return PTR_ERR(npage); + + F2FS_I(inode)->i_xattr_nid = 0; + + /* need to do checkpoint during fsync */ + F2FS_I(inode)->xattr_ver = cur_cp_version(F2FS_CKPT(sbi)); + + set_new_dnode(&dn, inode, page, npage, nid); + + if (page) + dn.inode_page_locked = true; + truncate_node(&dn); + return 0; +} + +/* + * Caller should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op(). + */ +int remove_inode_page(struct inode *inode) +{ + struct dnode_of_data dn; + int err; + + set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); + err = get_dnode_of_data(&dn, 0, LOOKUP_NODE); + if (err) + return err; + + err = truncate_xattr_node(inode, dn.inode_page); + if (err) { + f2fs_put_dnode(&dn); + return err; + } + + /* remove potential inline_data blocks */ + if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode)) + truncate_data_blocks_range(&dn, 1); + + /* 0 is possible, after f2fs_new_inode() has failed */ + f2fs_bug_on(F2FS_I_SB(inode), + inode->i_blocks != 0 && inode->i_blocks != 1); + + /* will put inode & node pages */ + truncate_node(&dn); + return 0; +} + +struct page *new_inode_page(struct inode *inode) +{ + struct dnode_of_data dn; + + /* allocate inode page for new inode */ + set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); + + /* caller should f2fs_put_page(page, 1); */ + return new_node_page(&dn, 0, NULL); +} + +struct page *new_node_page(struct dnode_of_data *dn, + unsigned int ofs, struct page *ipage) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct node_info old_ni, new_ni; + struct page *page; + int err; + + if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) + return ERR_PTR(-EPERM); + + page = grab_cache_page(NODE_MAPPING(sbi), dn->nid); + if (!page) + return ERR_PTR(-ENOMEM); + + if (unlikely(!inc_valid_node_count(sbi, dn->inode))) { + err = -ENOSPC; + goto fail; + } + + get_node_info(sbi, dn->nid, &old_ni); + + /* Reinitialize old_ni with new node page */ + f2fs_bug_on(sbi, old_ni.blk_addr != NULL_ADDR); + new_ni = old_ni; + new_ni.ino = dn->inode->i_ino; + set_node_addr(sbi, &new_ni, NEW_ADDR, false); + + f2fs_wait_on_page_writeback(page, NODE); + fill_node_footer(page, dn->nid, dn->inode->i_ino, ofs, true); + set_cold_node(dn->inode, page); + SetPageUptodate(page); + set_page_dirty(page); + + if (f2fs_has_xattr_block(ofs)) + F2FS_I(dn->inode)->i_xattr_nid = dn->nid; + + dn->node_page = page; + if (ipage) + update_inode(dn->inode, ipage); + else + sync_inode_page(dn); + if (ofs == 0) + inc_valid_inode_count(sbi); + + return page; + +fail: + clear_node_page_dirty(page); + f2fs_put_page(page, 1); + return ERR_PTR(err); +} + +/* + * Caller should do after getting the following values. + * 0: f2fs_put_page(page, 0) + * LOCKED_PAGE or error: f2fs_put_page(page, 1) + */ +static int read_node_page(struct page *page, int rw) +{ + struct f2fs_sb_info *sbi = F2FS_P_SB(page); + struct node_info ni; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = NODE, + .rw = rw, + .page = page, + .encrypted_page = NULL, + }; + + get_node_info(sbi, page->index, &ni); + + if (unlikely(ni.blk_addr == NULL_ADDR)) { + ClearPageUptodate(page); + return -ENOENT; + } + + if (PageUptodate(page)) + return LOCKED_PAGE; + + fio.blk_addr = ni.blk_addr; + return f2fs_submit_page_bio(&fio); +} + +/* + * Readahead a node page + */ +void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct page *apage; + int err; + + apage = find_get_page(NODE_MAPPING(sbi), nid); + if (apage && PageUptodate(apage)) { + f2fs_put_page(apage, 0); + return; + } + f2fs_put_page(apage, 0); + + apage = grab_cache_page(NODE_MAPPING(sbi), nid); + if (!apage) + return; + + err = read_node_page(apage, READA); + f2fs_put_page(apage, err ? 1 : 0); +} + +struct page *get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid) +{ + struct page *page; + int err; +repeat: + page = grab_cache_page(NODE_MAPPING(sbi), nid); + if (!page) + return ERR_PTR(-ENOMEM); + + err = read_node_page(page, READ_SYNC); + if (err < 0) { + f2fs_put_page(page, 1); + return ERR_PTR(err); + } else if (err != LOCKED_PAGE) { + lock_page(page); + } + + if (unlikely(!PageUptodate(page) || nid != nid_of_node(page))) { + ClearPageUptodate(page); + f2fs_put_page(page, 1); + return ERR_PTR(-EIO); + } + if (unlikely(page->mapping != NODE_MAPPING(sbi))) { + f2fs_put_page(page, 1); + goto repeat; + } + mark_page_accessed(page); + return page; +} + +/* + * Return a locked page for the desired node page. + * And, readahead MAX_RA_NODE number of node pages. + */ +struct page *get_node_page_ra(struct page *parent, int start) +{ + struct f2fs_sb_info *sbi = F2FS_P_SB(parent); + struct blk_plug plug; + struct page *page; + int err, i, end; + nid_t nid; + + /* First, try getting the desired direct node. */ + nid = get_nid(parent, start, false); + if (!nid) + return ERR_PTR(-ENOENT); +repeat: + page = grab_cache_page(NODE_MAPPING(sbi), nid); + if (!page) + return ERR_PTR(-ENOMEM); + + err = read_node_page(page, READ_SYNC); + if (err < 0) { + f2fs_put_page(page, 1); + return ERR_PTR(err); + } else if (err == LOCKED_PAGE) { + goto page_hit; + } + + blk_start_plug(&plug); + + /* Then, try readahead for siblings of the desired node */ + end = start + MAX_RA_NODE; + end = min(end, NIDS_PER_BLOCK); + for (i = start + 1; i < end; i++) { + nid = get_nid(parent, i, false); + if (!nid) + continue; + ra_node_page(sbi, nid); + } + + blk_finish_plug(&plug); + + lock_page(page); + if (unlikely(page->mapping != NODE_MAPPING(sbi))) { + f2fs_put_page(page, 1); + goto repeat; + } +page_hit: + if (unlikely(!PageUptodate(page))) { + f2fs_put_page(page, 1); + return ERR_PTR(-EIO); + } + mark_page_accessed(page); + return page; +} + +void sync_inode_page(struct dnode_of_data *dn) +{ + if (IS_INODE(dn->node_page) || dn->inode_page == dn->node_page) { + update_inode(dn->inode, dn->node_page); + } else if (dn->inode_page) { + if (!dn->inode_page_locked) + lock_page(dn->inode_page); + update_inode(dn->inode, dn->inode_page); + if (!dn->inode_page_locked) + unlock_page(dn->inode_page); + } else { + update_inode_page(dn->inode); + } +} + +int sync_node_pages(struct f2fs_sb_info *sbi, nid_t ino, + struct writeback_control *wbc) +{ + pgoff_t index, end; + struct pagevec pvec; + int step = ino ? 2 : 0; + int nwritten = 0, wrote = 0; + + pagevec_init(&pvec, 0); + +next_step: + index = 0; + end = LONG_MAX; + + while (index <= end) { + int i, nr_pages; + nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, + PAGECACHE_TAG_DIRTY, + min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); + if (nr_pages == 0) + break; + + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + + /* + * flushing sequence with step: + * 0. indirect nodes + * 1. dentry dnodes + * 2. file dnodes + */ + if (step == 0 && IS_DNODE(page)) + continue; + if (step == 1 && (!IS_DNODE(page) || + is_cold_node(page))) + continue; + if (step == 2 && (!IS_DNODE(page) || + !is_cold_node(page))) + continue; + + /* + * If an fsync mode, + * we should not skip writing node pages. + */ + if (ino && ino_of_node(page) == ino) + lock_page(page); + else if (!trylock_page(page)) + continue; + + if (unlikely(page->mapping != NODE_MAPPING(sbi))) { +continue_unlock: + unlock_page(page); + continue; + } + if (ino && ino_of_node(page) != ino) + goto continue_unlock; + + if (!PageDirty(page)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + if (!clear_page_dirty_for_io(page)) + goto continue_unlock; + + /* called by fsync() */ + if (ino && IS_DNODE(page)) { + set_fsync_mark(page, 1); + if (IS_INODE(page)) + set_dentry_mark(page, + need_dentry_mark(sbi, ino)); + nwritten++; + } else { + set_fsync_mark(page, 0); + set_dentry_mark(page, 0); + } + + if (NODE_MAPPING(sbi)->a_ops->writepage(page, wbc)) + unlock_page(page); + else + wrote++; + + if (--wbc->nr_to_write == 0) + break; + } + pagevec_release(&pvec); + cond_resched(); + + if (wbc->nr_to_write == 0) { + step = 2; + break; + } + } + + if (step < 2) { + step++; + goto next_step; + } + + if (wrote) + f2fs_submit_merged_bio(sbi, NODE, WRITE); + return nwritten; +} + +int wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, nid_t ino) +{ + pgoff_t index = 0, end = LONG_MAX; + struct pagevec pvec; + int ret2 = 0, ret = 0; + + pagevec_init(&pvec, 0); + + while (index <= end) { + int i, nr_pages; + nr_pages = pagevec_lookup_tag(&pvec, NODE_MAPPING(sbi), &index, + PAGECACHE_TAG_WRITEBACK, + min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); + if (nr_pages == 0) + break; + + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + + /* until radix tree lookup accepts end_index */ + if (unlikely(page->index > end)) + continue; + + if (ino && ino_of_node(page) == ino) { + f2fs_wait_on_page_writeback(page, NODE); + if (TestClearPageError(page)) + ret = -EIO; + } + } + pagevec_release(&pvec); + cond_resched(); + } + + if (unlikely(test_and_clear_bit(AS_ENOSPC, &NODE_MAPPING(sbi)->flags))) + ret2 = -ENOSPC; + if (unlikely(test_and_clear_bit(AS_EIO, &NODE_MAPPING(sbi)->flags))) + ret2 = -EIO; + if (!ret) + ret = ret2; + return ret; +} + +static int f2fs_write_node_page(struct page *page, + struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_P_SB(page); + nid_t nid; + struct node_info ni; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = NODE, + .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE, + .page = page, + .encrypted_page = NULL, + }; + + trace_f2fs_writepage(page, NODE); + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto redirty_out; + if (unlikely(f2fs_cp_error(sbi))) + goto redirty_out; + + f2fs_wait_on_page_writeback(page, NODE); + + /* get old block addr of this node page */ + nid = nid_of_node(page); + f2fs_bug_on(sbi, page->index != nid); + + if (wbc->for_reclaim) { + if (!down_read_trylock(&sbi->node_write)) + goto redirty_out; + } else { + down_read(&sbi->node_write); + } + + get_node_info(sbi, nid, &ni); + + /* This page is already truncated */ + if (unlikely(ni.blk_addr == NULL_ADDR)) { + ClearPageUptodate(page); + dec_page_count(sbi, F2FS_DIRTY_NODES); + up_read(&sbi->node_write); + unlock_page(page); + return 0; + } + + set_page_writeback(page); + fio.blk_addr = ni.blk_addr; + write_node_page(nid, &fio); + set_node_addr(sbi, &ni, fio.blk_addr, is_fsync_dnode(page)); + dec_page_count(sbi, F2FS_DIRTY_NODES); + up_read(&sbi->node_write); + unlock_page(page); + + if (wbc->for_reclaim) + f2fs_submit_merged_bio(sbi, NODE, WRITE); + + return 0; + +redirty_out: + redirty_page_for_writepage(wbc, page); + return AOP_WRITEPAGE_ACTIVATE; +} + +static int f2fs_write_node_pages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + long diff; + + trace_f2fs_writepages(mapping->host, wbc, NODE); + + /* balancing f2fs's metadata in background */ + f2fs_balance_fs_bg(sbi); + + /* collect a number of dirty node pages and write together */ + if (get_pages(sbi, F2FS_DIRTY_NODES) < nr_pages_to_skip(sbi, NODE)) + goto skip_write; + + diff = nr_pages_to_write(sbi, NODE, wbc); + wbc->sync_mode = WB_SYNC_NONE; + sync_node_pages(sbi, 0, wbc); + wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); + return 0; + +skip_write: + wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES); + return 0; +} + +static int f2fs_set_node_page_dirty(struct page *page) +{ + trace_f2fs_set_page_dirty(page, NODE); + + SetPageUptodate(page); + if (!PageDirty(page)) { + __set_page_dirty_nobuffers(page); + inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES); + SetPagePrivate(page); + f2fs_trace_pid(page); + return 1; + } + return 0; +} + +/* + * Structure of the f2fs node operations + */ +const struct address_space_operations f2fs_node_aops = { + .writepage = f2fs_write_node_page, + .writepages = f2fs_write_node_pages, + .set_page_dirty = f2fs_set_node_page_dirty, + .invalidatepage = f2fs_invalidate_page, + .releasepage = f2fs_release_page, +}; + +static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i, + nid_t n) +{ + return radix_tree_lookup(&nm_i->free_nid_root, n); +} + +static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i, + struct free_nid *i) +{ + list_del(&i->list); + radix_tree_delete(&nm_i->free_nid_root, i->nid); +} + +static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i; + struct nat_entry *ne; + bool allocated = false; + + if (!available_free_memory(sbi, FREE_NIDS)) + return -1; + + /* 0 nid should not be used */ + if (unlikely(nid == 0)) + return 0; + + if (build) { + /* do not add allocated nids */ + down_read(&nm_i->nat_tree_lock); + ne = __lookup_nat_cache(nm_i, nid); + if (ne && + (!get_nat_flag(ne, IS_CHECKPOINTED) || + nat_get_blkaddr(ne) != NULL_ADDR)) + allocated = true; + up_read(&nm_i->nat_tree_lock); + if (allocated) + return 0; + } + + i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS); + i->nid = nid; + i->state = NID_NEW; + + if (radix_tree_preload(GFP_NOFS)) { + kmem_cache_free(free_nid_slab, i); + return 0; + } + + spin_lock(&nm_i->free_nid_list_lock); + if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) { + spin_unlock(&nm_i->free_nid_list_lock); + radix_tree_preload_end(); + kmem_cache_free(free_nid_slab, i); + return 0; + } + list_add_tail(&i->list, &nm_i->free_nid_list); + nm_i->fcnt++; + spin_unlock(&nm_i->free_nid_list_lock); + radix_tree_preload_end(); + return 1; +} + +static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid) +{ + struct free_nid *i; + bool need_free = false; + + spin_lock(&nm_i->free_nid_list_lock); + i = __lookup_free_nid_list(nm_i, nid); + if (i && i->state == NID_NEW) { + __del_from_free_nid_list(nm_i, i); + nm_i->fcnt--; + need_free = true; + } + spin_unlock(&nm_i->free_nid_list_lock); + + if (need_free) + kmem_cache_free(free_nid_slab, i); +} + +static void scan_nat_page(struct f2fs_sb_info *sbi, + struct page *nat_page, nid_t start_nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct f2fs_nat_block *nat_blk = page_address(nat_page); + block_t blk_addr; + int i; + + i = start_nid % NAT_ENTRY_PER_BLOCK; + + for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) { + + if (unlikely(start_nid >= nm_i->max_nid)) + break; + + blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr); + f2fs_bug_on(sbi, blk_addr == NEW_ADDR); + if (blk_addr == NULL_ADDR) { + if (add_free_nid(sbi, start_nid, true) < 0) + break; + } + } +} + +static void build_free_nids(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + int i = 0; + nid_t nid = nm_i->next_scan_nid; + + /* Enough entries */ + if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK) + return; + + /* readahead nat pages to be scanned */ + ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES, + META_NAT, true); + + while (1) { + struct page *page = get_current_nat_page(sbi, nid); + + scan_nat_page(sbi, page, nid); + f2fs_put_page(page, 1); + + nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK)); + if (unlikely(nid >= nm_i->max_nid)) + nid = 0; + + if (++i >= FREE_NID_PAGES) + break; + } + + /* go to the next free nat pages to find free nids abundantly */ + nm_i->next_scan_nid = nid; + + /* find free nids from current sum_pages */ + mutex_lock(&curseg->curseg_mutex); + for (i = 0; i < nats_in_cursum(sum); i++) { + block_t addr = le32_to_cpu(nat_in_journal(sum, i).block_addr); + nid = le32_to_cpu(nid_in_journal(sum, i)); + if (addr == NULL_ADDR) + add_free_nid(sbi, nid, true); + else + remove_free_nid(nm_i, nid); + } + mutex_unlock(&curseg->curseg_mutex); + + ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid), + nm_i->ra_nid_pages, META_NAT, false); +} + +/* + * If this function returns success, caller can obtain a new nid + * from second parameter of this function. + * The returned nid could be used ino as well as nid when inode is created. + */ +bool alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i = NULL; +retry: + if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids)) + return false; + + spin_lock(&nm_i->free_nid_list_lock); + + /* We should not use stale free nids created by build_free_nids */ + if (nm_i->fcnt && !on_build_free_nids(nm_i)) { + struct node_info ni; + + f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list)); + list_for_each_entry(i, &nm_i->free_nid_list, list) + if (i->state == NID_NEW) + break; + + f2fs_bug_on(sbi, i->state != NID_NEW); + *nid = i->nid; + i->state = NID_ALLOC; + nm_i->fcnt--; + spin_unlock(&nm_i->free_nid_list_lock); + + /* check nid is allocated already */ + get_node_info(sbi, *nid, &ni); + if (ni.blk_addr != NULL_ADDR) { + alloc_nid_done(sbi, *nid); + goto retry; + } + return true; + } + spin_unlock(&nm_i->free_nid_list_lock); + + /* Let's scan nat pages and its caches to get free nids */ + mutex_lock(&nm_i->build_lock); + build_free_nids(sbi); + mutex_unlock(&nm_i->build_lock); + goto retry; +} + +/* + * alloc_nid() should be called prior to this function. + */ +void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i; + + spin_lock(&nm_i->free_nid_list_lock); + i = __lookup_free_nid_list(nm_i, nid); + f2fs_bug_on(sbi, !i || i->state != NID_ALLOC); + __del_from_free_nid_list(nm_i, i); + spin_unlock(&nm_i->free_nid_list_lock); + + kmem_cache_free(free_nid_slab, i); +} + +/* + * alloc_nid() should be called prior to this function. + */ +void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i; + bool need_free = false; + + if (!nid) + return; + + spin_lock(&nm_i->free_nid_list_lock); + i = __lookup_free_nid_list(nm_i, nid); + f2fs_bug_on(sbi, !i || i->state != NID_ALLOC); + if (!available_free_memory(sbi, FREE_NIDS)) { + __del_from_free_nid_list(nm_i, i); + need_free = true; + } else { + i->state = NID_NEW; + nm_i->fcnt++; + } + spin_unlock(&nm_i->free_nid_list_lock); + + if (need_free) + kmem_cache_free(free_nid_slab, i); +} + +int try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i, *next; + int nr = nr_shrink; + + if (!mutex_trylock(&nm_i->build_lock)) + return 0; + + spin_lock(&nm_i->free_nid_list_lock); + list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) { + if (nr_shrink <= 0 || nm_i->fcnt <= NAT_ENTRY_PER_BLOCK) + break; + if (i->state == NID_ALLOC) + continue; + __del_from_free_nid_list(nm_i, i); + kmem_cache_free(free_nid_slab, i); + nm_i->fcnt--; + nr_shrink--; + } + spin_unlock(&nm_i->free_nid_list_lock); + mutex_unlock(&nm_i->build_lock); + + return nr - nr_shrink; +} + +void recover_inline_xattr(struct inode *inode, struct page *page) +{ + void *src_addr, *dst_addr; + size_t inline_size; + struct page *ipage; + struct f2fs_inode *ri; + + ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); + f2fs_bug_on(F2FS_I_SB(inode), IS_ERR(ipage)); + + ri = F2FS_INODE(page); + if (!(ri->i_inline & F2FS_INLINE_XATTR)) { + clear_inode_flag(F2FS_I(inode), FI_INLINE_XATTR); + goto update_inode; + } + + dst_addr = inline_xattr_addr(ipage); + src_addr = inline_xattr_addr(page); + inline_size = inline_xattr_size(inode); + + f2fs_wait_on_page_writeback(ipage, NODE); + memcpy(dst_addr, src_addr, inline_size); +update_inode: + update_inode(inode, ipage); + f2fs_put_page(ipage, 1); +} + +void recover_xattr_data(struct inode *inode, struct page *page, block_t blkaddr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid; + nid_t new_xnid = nid_of_node(page); + struct node_info ni; + + /* 1: invalidate the previous xattr nid */ + if (!prev_xnid) + goto recover_xnid; + + /* Deallocate node address */ + get_node_info(sbi, prev_xnid, &ni); + f2fs_bug_on(sbi, ni.blk_addr == NULL_ADDR); + invalidate_blocks(sbi, ni.blk_addr); + dec_valid_node_count(sbi, inode); + set_node_addr(sbi, &ni, NULL_ADDR, false); + +recover_xnid: + /* 2: allocate new xattr nid */ + if (unlikely(!inc_valid_node_count(sbi, inode))) + f2fs_bug_on(sbi, 1); + + remove_free_nid(NM_I(sbi), new_xnid); + get_node_info(sbi, new_xnid, &ni); + ni.ino = inode->i_ino; + set_node_addr(sbi, &ni, NEW_ADDR, false); + F2FS_I(inode)->i_xattr_nid = new_xnid; + + /* 3: update xattr blkaddr */ + refresh_sit_entry(sbi, NEW_ADDR, blkaddr); + set_node_addr(sbi, &ni, blkaddr, false); + + update_inode_page(inode); +} + +int recover_inode_page(struct f2fs_sb_info *sbi, struct page *page) +{ + struct f2fs_inode *src, *dst; + nid_t ino = ino_of_node(page); + struct node_info old_ni, new_ni; + struct page *ipage; + + get_node_info(sbi, ino, &old_ni); + + if (unlikely(old_ni.blk_addr != NULL_ADDR)) + return -EINVAL; + + ipage = grab_cache_page(NODE_MAPPING(sbi), ino); + if (!ipage) + return -ENOMEM; + + /* Should not use this inode from free nid list */ + remove_free_nid(NM_I(sbi), ino); + + SetPageUptodate(ipage); + fill_node_footer(ipage, ino, ino, 0, true); + + src = F2FS_INODE(page); + dst = F2FS_INODE(ipage); + + memcpy(dst, src, (unsigned long)&src->i_ext - (unsigned long)src); + dst->i_size = 0; + dst->i_blocks = cpu_to_le64(1); + dst->i_links = cpu_to_le32(1); + dst->i_xattr_nid = 0; + dst->i_inline = src->i_inline & F2FS_INLINE_XATTR; + + new_ni = old_ni; + new_ni.ino = ino; + + if (unlikely(!inc_valid_node_count(sbi, NULL))) + WARN_ON(1); + set_node_addr(sbi, &new_ni, NEW_ADDR, false); + inc_valid_inode_count(sbi); + set_page_dirty(ipage); + f2fs_put_page(ipage, 1); + return 0; +} + +int restore_node_summary(struct f2fs_sb_info *sbi, + unsigned int segno, struct f2fs_summary_block *sum) +{ + struct f2fs_node *rn; + struct f2fs_summary *sum_entry; + block_t addr; + int bio_blocks = MAX_BIO_BLOCKS(sbi); + int i, idx, last_offset, nrpages; + + /* scan the node segment */ + last_offset = sbi->blocks_per_seg; + addr = START_BLOCK(sbi, segno); + sum_entry = &sum->entries[0]; + + for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { + nrpages = min(last_offset - i, bio_blocks); + + /* readahead node pages */ + ra_meta_pages(sbi, addr, nrpages, META_POR, true); + + for (idx = addr; idx < addr + nrpages; idx++) { + struct page *page = get_tmp_page(sbi, idx); + + rn = F2FS_NODE(page); + sum_entry->nid = rn->footer.nid; + sum_entry->version = 0; + sum_entry->ofs_in_node = 0; + sum_entry++; + f2fs_put_page(page, 1); + } + + invalidate_mapping_pages(META_MAPPING(sbi), addr, + addr + nrpages); + } + return 0; +} + +static void remove_nats_in_journal(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + int i; + + mutex_lock(&curseg->curseg_mutex); + for (i = 0; i < nats_in_cursum(sum); i++) { + struct nat_entry *ne; + struct f2fs_nat_entry raw_ne; + nid_t nid = le32_to_cpu(nid_in_journal(sum, i)); + + raw_ne = nat_in_journal(sum, i); + + down_write(&nm_i->nat_tree_lock); + ne = __lookup_nat_cache(nm_i, nid); + if (!ne) { + ne = grab_nat_entry(nm_i, nid); + node_info_from_raw_nat(&ne->ni, &raw_ne); + } + __set_nat_cache_dirty(nm_i, ne); + up_write(&nm_i->nat_tree_lock); + } + update_nats_in_cursum(sum, -i); + mutex_unlock(&curseg->curseg_mutex); +} + +static void __adjust_nat_entry_set(struct nat_entry_set *nes, + struct list_head *head, int max) +{ + struct nat_entry_set *cur; + + if (nes->entry_cnt >= max) + goto add_out; + + list_for_each_entry(cur, head, set_list) { + if (cur->entry_cnt >= nes->entry_cnt) { + list_add(&nes->set_list, cur->set_list.prev); + return; + } + } +add_out: + list_add_tail(&nes->set_list, head); +} + +static void __flush_nat_entry_set(struct f2fs_sb_info *sbi, + struct nat_entry_set *set) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK; + bool to_journal = true; + struct f2fs_nat_block *nat_blk; + struct nat_entry *ne, *cur; + struct page *page = NULL; + struct f2fs_nm_info *nm_i = NM_I(sbi); + + /* + * there are two steps to flush nat entries: + * #1, flush nat entries to journal in current hot data summary block. + * #2, flush nat entries to nat page. + */ + if (!__has_cursum_space(sum, set->entry_cnt, NAT_JOURNAL)) + to_journal = false; + + if (to_journal) { + mutex_lock(&curseg->curseg_mutex); + } else { + page = get_next_nat_page(sbi, start_nid); + nat_blk = page_address(page); + f2fs_bug_on(sbi, !nat_blk); + } + + /* flush dirty nats in nat entry set */ + list_for_each_entry_safe(ne, cur, &set->entry_list, list) { + struct f2fs_nat_entry *raw_ne; + nid_t nid = nat_get_nid(ne); + int offset; + + if (nat_get_blkaddr(ne) == NEW_ADDR) + continue; + + if (to_journal) { + offset = lookup_journal_in_cursum(sum, + NAT_JOURNAL, nid, 1); + f2fs_bug_on(sbi, offset < 0); + raw_ne = &nat_in_journal(sum, offset); + nid_in_journal(sum, offset) = cpu_to_le32(nid); + } else { + raw_ne = &nat_blk->entries[nid - start_nid]; + } + raw_nat_from_node_info(raw_ne, &ne->ni); + + down_write(&NM_I(sbi)->nat_tree_lock); + nat_reset_flag(ne); + __clear_nat_cache_dirty(NM_I(sbi), ne); + up_write(&NM_I(sbi)->nat_tree_lock); + + if (nat_get_blkaddr(ne) == NULL_ADDR) + add_free_nid(sbi, nid, false); + } + + if (to_journal) + mutex_unlock(&curseg->curseg_mutex); + else + f2fs_put_page(page, 1); + + f2fs_bug_on(sbi, set->entry_cnt); + + down_write(&nm_i->nat_tree_lock); + radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set); + up_write(&nm_i->nat_tree_lock); + kmem_cache_free(nat_entry_set_slab, set); +} + +/* + * This function is called during the checkpointing process. + */ +void flush_nat_entries(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + struct nat_entry_set *setvec[SETVEC_SIZE]; + struct nat_entry_set *set, *tmp; + unsigned int found; + nid_t set_idx = 0; + LIST_HEAD(sets); + + if (!nm_i->dirty_nat_cnt) + return; + /* + * if there are no enough space in journal to store dirty nat + * entries, remove all entries from journal and merge them + * into nat entry set. + */ + if (!__has_cursum_space(sum, nm_i->dirty_nat_cnt, NAT_JOURNAL)) + remove_nats_in_journal(sbi); + + down_write(&nm_i->nat_tree_lock); + while ((found = __gang_lookup_nat_set(nm_i, + set_idx, SETVEC_SIZE, setvec))) { + unsigned idx; + set_idx = setvec[found - 1]->set + 1; + for (idx = 0; idx < found; idx++) + __adjust_nat_entry_set(setvec[idx], &sets, + MAX_NAT_JENTRIES(sum)); + } + up_write(&nm_i->nat_tree_lock); + + /* flush dirty nats in nat entry set */ + list_for_each_entry_safe(set, tmp, &sets, set_list) + __flush_nat_entry_set(sbi, set); + + f2fs_bug_on(sbi, nm_i->dirty_nat_cnt); +} + +static int init_node_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi); + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned char *version_bitmap; + unsigned int nat_segs, nat_blocks; + + nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr); + + /* segment_count_nat includes pair segment so divide to 2. */ + nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1; + nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg); + + nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks; + + /* not used nids: 0, node, meta, (and root counted as valid node) */ + nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM; + nm_i->fcnt = 0; + nm_i->nat_cnt = 0; + nm_i->ram_thresh = DEF_RAM_THRESHOLD; + nm_i->ra_nid_pages = DEF_RA_NID_PAGES; + + INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC); + INIT_LIST_HEAD(&nm_i->free_nid_list); + INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO); + INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO); + INIT_LIST_HEAD(&nm_i->nat_entries); + + mutex_init(&nm_i->build_lock); + spin_lock_init(&nm_i->free_nid_list_lock); + init_rwsem(&nm_i->nat_tree_lock); + + nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid); + nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP); + version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP); + if (!version_bitmap) + return -EFAULT; + + nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size, + GFP_KERNEL); + if (!nm_i->nat_bitmap) + return -ENOMEM; + return 0; +} + +int build_node_manager(struct f2fs_sb_info *sbi) +{ + int err; + + sbi->nm_info = kzalloc(sizeof(struct f2fs_nm_info), GFP_KERNEL); + if (!sbi->nm_info) + return -ENOMEM; + + err = init_node_manager(sbi); + if (err) + return err; + + build_free_nids(sbi); + return 0; +} + +void destroy_node_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i, *next_i; + struct nat_entry *natvec[NATVEC_SIZE]; + struct nat_entry_set *setvec[SETVEC_SIZE]; + nid_t nid = 0; + unsigned int found; + + if (!nm_i) + return; + + /* destroy free nid list */ + spin_lock(&nm_i->free_nid_list_lock); + list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) { + f2fs_bug_on(sbi, i->state == NID_ALLOC); + __del_from_free_nid_list(nm_i, i); + nm_i->fcnt--; + spin_unlock(&nm_i->free_nid_list_lock); + kmem_cache_free(free_nid_slab, i); + spin_lock(&nm_i->free_nid_list_lock); + } + f2fs_bug_on(sbi, nm_i->fcnt); + spin_unlock(&nm_i->free_nid_list_lock); + + /* destroy nat cache */ + down_write(&nm_i->nat_tree_lock); + while ((found = __gang_lookup_nat_cache(nm_i, + nid, NATVEC_SIZE, natvec))) { + unsigned idx; + + nid = nat_get_nid(natvec[found - 1]) + 1; + for (idx = 0; idx < found; idx++) + __del_from_nat_cache(nm_i, natvec[idx]); + } + f2fs_bug_on(sbi, nm_i->nat_cnt); + + /* destroy nat set cache */ + nid = 0; + while ((found = __gang_lookup_nat_set(nm_i, + nid, SETVEC_SIZE, setvec))) { + unsigned idx; + + nid = setvec[found - 1]->set + 1; + for (idx = 0; idx < found; idx++) { + /* entry_cnt is not zero, when cp_error was occurred */ + f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list)); + radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set); + kmem_cache_free(nat_entry_set_slab, setvec[idx]); + } + } + up_write(&nm_i->nat_tree_lock); + + kfree(nm_i->nat_bitmap); + sbi->nm_info = NULL; + kfree(nm_i); +} + +int __init create_node_manager_caches(void) +{ + nat_entry_slab = f2fs_kmem_cache_create("nat_entry", + sizeof(struct nat_entry)); + if (!nat_entry_slab) + goto fail; + + free_nid_slab = f2fs_kmem_cache_create("free_nid", + sizeof(struct free_nid)); + if (!free_nid_slab) + goto destroy_nat_entry; + + nat_entry_set_slab = f2fs_kmem_cache_create("nat_entry_set", + sizeof(struct nat_entry_set)); + if (!nat_entry_set_slab) + goto destroy_free_nid; + return 0; + +destroy_free_nid: + kmem_cache_destroy(free_nid_slab); +destroy_nat_entry: + kmem_cache_destroy(nat_entry_slab); +fail: + return -ENOMEM; +} + +void destroy_node_manager_caches(void) +{ + kmem_cache_destroy(nat_entry_set_slab); + kmem_cache_destroy(free_nid_slab); + kmem_cache_destroy(nat_entry_slab); +} diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h new file mode 100644 index 00000000000..e4fffd2d98c --- /dev/null +++ b/fs/f2fs/node.h @@ -0,0 +1,396 @@ +/* + * fs/f2fs/node.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/* start node id of a node block dedicated to the given node id */ +#define START_NID(nid) ((nid / NAT_ENTRY_PER_BLOCK) * NAT_ENTRY_PER_BLOCK) + +/* node block offset on the NAT area dedicated to the given start node id */ +#define NAT_BLOCK_OFFSET(start_nid) (start_nid / NAT_ENTRY_PER_BLOCK) + +/* # of pages to perform synchronous readahead before building free nids */ +#define FREE_NID_PAGES 4 + +#define DEF_RA_NID_PAGES 4 /* # of nid pages to be readaheaded */ + +/* maximum readahead size for node during getting data blocks */ +#define MAX_RA_NODE 128 + +/* control the memory footprint threshold (10MB per 1GB ram) */ +#define DEF_RAM_THRESHOLD 10 + +/* vector size for gang look-up from nat cache that consists of radix tree */ +#define NATVEC_SIZE 64 +#define SETVEC_SIZE 32 + +/* return value for read_node_page */ +#define LOCKED_PAGE 1 + +/* For flag in struct node_info */ +enum { + IS_CHECKPOINTED, /* is it checkpointed before? */ + HAS_FSYNCED_INODE, /* is the inode fsynced before? */ + HAS_LAST_FSYNC, /* has the latest node fsync mark? */ + IS_DIRTY, /* this nat entry is dirty? */ +}; + +/* + * For node information + */ +struct node_info { + nid_t nid; /* node id */ + nid_t ino; /* inode number of the node's owner */ + block_t blk_addr; /* block address of the node */ + unsigned char version; /* version of the node */ + unsigned char flag; /* for node information bits */ +}; + +struct nat_entry { + struct list_head list; /* for clean or dirty nat list */ + struct node_info ni; /* in-memory node information */ +}; + +#define nat_get_nid(nat) (nat->ni.nid) +#define nat_set_nid(nat, n) (nat->ni.nid = n) +#define nat_get_blkaddr(nat) (nat->ni.blk_addr) +#define nat_set_blkaddr(nat, b) (nat->ni.blk_addr = b) +#define nat_get_ino(nat) (nat->ni.ino) +#define nat_set_ino(nat, i) (nat->ni.ino = i) +#define nat_get_version(nat) (nat->ni.version) +#define nat_set_version(nat, v) (nat->ni.version = v) + +#define inc_node_version(version) (++version) + +static inline void copy_node_info(struct node_info *dst, + struct node_info *src) +{ + dst->nid = src->nid; + dst->ino = src->ino; + dst->blk_addr = src->blk_addr; + dst->version = src->version; + /* should not copy flag here */ +} + +static inline void set_nat_flag(struct nat_entry *ne, + unsigned int type, bool set) +{ + unsigned char mask = 0x01 << type; + if (set) + ne->ni.flag |= mask; + else + ne->ni.flag &= ~mask; +} + +static inline bool get_nat_flag(struct nat_entry *ne, unsigned int type) +{ + unsigned char mask = 0x01 << type; + return ne->ni.flag & mask; +} + +static inline void nat_reset_flag(struct nat_entry *ne) +{ + /* these states can be set only after checkpoint was done */ + set_nat_flag(ne, IS_CHECKPOINTED, true); + set_nat_flag(ne, HAS_FSYNCED_INODE, false); + set_nat_flag(ne, HAS_LAST_FSYNC, true); +} + +static inline void node_info_from_raw_nat(struct node_info *ni, + struct f2fs_nat_entry *raw_ne) +{ + ni->ino = le32_to_cpu(raw_ne->ino); + ni->blk_addr = le32_to_cpu(raw_ne->block_addr); + ni->version = raw_ne->version; +} + +static inline void raw_nat_from_node_info(struct f2fs_nat_entry *raw_ne, + struct node_info *ni) +{ + raw_ne->ino = cpu_to_le32(ni->ino); + raw_ne->block_addr = cpu_to_le32(ni->blk_addr); + raw_ne->version = ni->version; +} + +enum mem_type { + FREE_NIDS, /* indicates the free nid list */ + NAT_ENTRIES, /* indicates the cached nat entry */ + DIRTY_DENTS, /* indicates dirty dentry pages */ + INO_ENTRIES, /* indicates inode entries */ + EXTENT_CACHE, /* indicates extent cache */ + BASE_CHECK, /* check kernel status */ +}; + +struct nat_entry_set { + struct list_head set_list; /* link with other nat sets */ + struct list_head entry_list; /* link with dirty nat entries */ + nid_t set; /* set number*/ + unsigned int entry_cnt; /* the # of nat entries in set */ +}; + +/* + * For free nid mangement + */ +enum nid_state { + NID_NEW, /* newly added to free nid list */ + NID_ALLOC /* it is allocated */ +}; + +struct free_nid { + struct list_head list; /* for free node id list */ + nid_t nid; /* node id */ + int state; /* in use or not: NID_NEW or NID_ALLOC */ +}; + +static inline void next_free_nid(struct f2fs_sb_info *sbi, nid_t *nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *fnid; + + spin_lock(&nm_i->free_nid_list_lock); + if (nm_i->fcnt <= 0) { + spin_unlock(&nm_i->free_nid_list_lock); + return; + } + fnid = list_entry(nm_i->free_nid_list.next, struct free_nid, list); + *nid = fnid->nid; + spin_unlock(&nm_i->free_nid_list_lock); +} + +/* + * inline functions + */ +static inline void get_nat_bitmap(struct f2fs_sb_info *sbi, void *addr) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + memcpy(addr, nm_i->nat_bitmap, nm_i->bitmap_size); +} + +static inline pgoff_t current_nat_addr(struct f2fs_sb_info *sbi, nid_t start) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + pgoff_t block_off; + pgoff_t block_addr; + int seg_off; + + block_off = NAT_BLOCK_OFFSET(start); + seg_off = block_off >> sbi->log_blocks_per_seg; + + block_addr = (pgoff_t)(nm_i->nat_blkaddr + + (seg_off << sbi->log_blocks_per_seg << 1) + + (block_off & ((1 << sbi->log_blocks_per_seg) - 1))); + + if (f2fs_test_bit(block_off, nm_i->nat_bitmap)) + block_addr += sbi->blocks_per_seg; + + return block_addr; +} + +static inline pgoff_t next_nat_addr(struct f2fs_sb_info *sbi, + pgoff_t block_addr) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + + block_addr -= nm_i->nat_blkaddr; + if ((block_addr >> sbi->log_blocks_per_seg) % 2) + block_addr -= sbi->blocks_per_seg; + else + block_addr += sbi->blocks_per_seg; + + return block_addr + nm_i->nat_blkaddr; +} + +static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid) +{ + unsigned int block_off = NAT_BLOCK_OFFSET(start_nid); + + f2fs_change_bit(block_off, nm_i->nat_bitmap); +} + +static inline void fill_node_footer(struct page *page, nid_t nid, + nid_t ino, unsigned int ofs, bool reset) +{ + struct f2fs_node *rn = F2FS_NODE(page); + unsigned int old_flag = 0; + + if (reset) + memset(rn, 0, sizeof(*rn)); + else + old_flag = le32_to_cpu(rn->footer.flag); + + rn->footer.nid = cpu_to_le32(nid); + rn->footer.ino = cpu_to_le32(ino); + + /* should remain old flag bits such as COLD_BIT_SHIFT */ + rn->footer.flag = cpu_to_le32((ofs << OFFSET_BIT_SHIFT) | + (old_flag & OFFSET_BIT_MASK)); +} + +static inline void copy_node_footer(struct page *dst, struct page *src) +{ + struct f2fs_node *src_rn = F2FS_NODE(src); + struct f2fs_node *dst_rn = F2FS_NODE(dst); + memcpy(&dst_rn->footer, &src_rn->footer, sizeof(struct node_footer)); +} + +static inline void fill_node_footer_blkaddr(struct page *page, block_t blkaddr) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_P_SB(page)); + struct f2fs_node *rn = F2FS_NODE(page); + + rn->footer.cp_ver = ckpt->checkpoint_ver; + rn->footer.next_blkaddr = cpu_to_le32(blkaddr); +} + +static inline nid_t ino_of_node(struct page *node_page) +{ + struct f2fs_node *rn = F2FS_NODE(node_page); + return le32_to_cpu(rn->footer.ino); +} + +static inline nid_t nid_of_node(struct page *node_page) +{ + struct f2fs_node *rn = F2FS_NODE(node_page); + return le32_to_cpu(rn->footer.nid); +} + +static inline unsigned int ofs_of_node(struct page *node_page) +{ + struct f2fs_node *rn = F2FS_NODE(node_page); + unsigned flag = le32_to_cpu(rn->footer.flag); + return flag >> OFFSET_BIT_SHIFT; +} + +static inline unsigned long long cpver_of_node(struct page *node_page) +{ + struct f2fs_node *rn = F2FS_NODE(node_page); + return le64_to_cpu(rn->footer.cp_ver); +} + +static inline block_t next_blkaddr_of_node(struct page *node_page) +{ + struct f2fs_node *rn = F2FS_NODE(node_page); + return le32_to_cpu(rn->footer.next_blkaddr); +} + +/* + * f2fs assigns the following node offsets described as (num). + * N = NIDS_PER_BLOCK + * + * Inode block (0) + * |- direct node (1) + * |- direct node (2) + * |- indirect node (3) + * | `- direct node (4 => 4 + N - 1) + * |- indirect node (4 + N) + * | `- direct node (5 + N => 5 + 2N - 1) + * `- double indirect node (5 + 2N) + * `- indirect node (6 + 2N) + * `- direct node + * ...... + * `- indirect node ((6 + 2N) + x(N + 1)) + * `- direct node + * ...... + * `- indirect node ((6 + 2N) + (N - 1)(N + 1)) + * `- direct node + */ +static inline bool IS_DNODE(struct page *node_page) +{ + unsigned int ofs = ofs_of_node(node_page); + + if (f2fs_has_xattr_block(ofs)) + return false; + + if (ofs == 3 || ofs == 4 + NIDS_PER_BLOCK || + ofs == 5 + 2 * NIDS_PER_BLOCK) + return false; + if (ofs >= 6 + 2 * NIDS_PER_BLOCK) { + ofs -= 6 + 2 * NIDS_PER_BLOCK; + if (!((long int)ofs % (NIDS_PER_BLOCK + 1))) + return false; + } + return true; +} + +static inline void set_nid(struct page *p, int off, nid_t nid, bool i) +{ + struct f2fs_node *rn = F2FS_NODE(p); + + f2fs_wait_on_page_writeback(p, NODE); + + if (i) + rn->i.i_nid[off - NODE_DIR1_BLOCK] = cpu_to_le32(nid); + else + rn->in.nid[off] = cpu_to_le32(nid); + set_page_dirty(p); +} + +static inline nid_t get_nid(struct page *p, int off, bool i) +{ + struct f2fs_node *rn = F2FS_NODE(p); + + if (i) + return le32_to_cpu(rn->i.i_nid[off - NODE_DIR1_BLOCK]); + return le32_to_cpu(rn->in.nid[off]); +} + +/* + * Coldness identification: + * - Mark cold files in f2fs_inode_info + * - Mark cold node blocks in their node footer + * - Mark cold data pages in page cache + */ +static inline int is_cold_data(struct page *page) +{ + return PageChecked(page); +} + +static inline void set_cold_data(struct page *page) +{ + SetPageChecked(page); +} + +static inline void clear_cold_data(struct page *page) +{ + ClearPageChecked(page); +} + +static inline int is_node(struct page *page, int type) +{ + struct f2fs_node *rn = F2FS_NODE(page); + return le32_to_cpu(rn->footer.flag) & (1 << type); +} + +#define is_cold_node(page) is_node(page, COLD_BIT_SHIFT) +#define is_fsync_dnode(page) is_node(page, FSYNC_BIT_SHIFT) +#define is_dent_dnode(page) is_node(page, DENT_BIT_SHIFT) + +static inline void set_cold_node(struct inode *inode, struct page *page) +{ + struct f2fs_node *rn = F2FS_NODE(page); + unsigned int flag = le32_to_cpu(rn->footer.flag); + + if (S_ISDIR(inode->i_mode)) + flag &= ~(0x1 << COLD_BIT_SHIFT); + else + flag |= (0x1 << COLD_BIT_SHIFT); + rn->footer.flag = cpu_to_le32(flag); +} + +static inline void set_mark(struct page *page, int mark, int type) +{ + struct f2fs_node *rn = F2FS_NODE(page); + unsigned int flag = le32_to_cpu(rn->footer.flag); + if (mark) + flag |= (0x1 << type); + else + flag &= ~(0x1 << type); + rn->footer.flag = cpu_to_le32(flag); +} +#define set_dentry_mark(page, mark) set_mark(page, mark, DENT_BIT_SHIFT) +#define set_fsync_mark(page, mark) set_mark(page, mark, FSYNC_BIT_SHIFT) diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c new file mode 100644 index 00000000000..6a3f04fa34e --- /dev/null +++ b/fs/f2fs/recovery.c @@ -0,0 +1,603 @@ +/* + * fs/f2fs/recovery.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include "f2fs.h" +#include "node.h" +#include "segment.h" + +/* + * Roll forward recovery scenarios. + * + * [Term] F: fsync_mark, D: dentry_mark + * + * 1. inode(x) | CP | inode(x) | dnode(F) + * -> Update the latest inode(x). + * + * 2. inode(x) | CP | inode(F) | dnode(F) + * -> No problem. + * + * 3. inode(x) | CP | dnode(F) | inode(x) + * -> Recover to the latest dnode(F), and drop the last inode(x) + * + * 4. inode(x) | CP | dnode(F) | inode(F) + * -> No problem. + * + * 5. CP | inode(x) | dnode(F) + * -> The inode(DF) was missing. Should drop this dnode(F). + * + * 6. CP | inode(DF) | dnode(F) + * -> No problem. + * + * 7. CP | dnode(F) | inode(DF) + * -> If f2fs_iget fails, then goto next to find inode(DF). + * + * 8. CP | dnode(F) | inode(x) + * -> If f2fs_iget fails, then goto next to find inode(DF). + * But it will fail due to no inode(DF). + */ + +static struct kmem_cache *fsync_entry_slab; + +bool space_for_roll_forward(struct f2fs_sb_info *sbi) +{ + if (sbi->last_valid_block_count + sbi->alloc_valid_block_count + > sbi->user_block_count) + return false; + return true; +} + +static struct fsync_inode_entry *get_fsync_inode(struct list_head *head, + nid_t ino) +{ + struct fsync_inode_entry *entry; + + list_for_each_entry(entry, head, list) + if (entry->inode->i_ino == ino) + return entry; + + return NULL; +} + +static int recover_dentry(struct inode *inode, struct page *ipage) +{ + struct f2fs_inode *raw_inode = F2FS_INODE(ipage); + nid_t pino = le32_to_cpu(raw_inode->i_pino); + struct f2fs_dir_entry *de; + struct qstr name; + struct page *page; + struct inode *dir, *einode; + int err = 0; + + dir = f2fs_iget(inode->i_sb, pino); + if (IS_ERR(dir)) { + err = PTR_ERR(dir); + goto out; + } + + if (file_enc_name(inode)) { + iput(dir); + return 0; + } + + name.len = le32_to_cpu(raw_inode->i_namelen); + name.name = raw_inode->i_name; + + if (unlikely(name.len > F2FS_NAME_LEN)) { + WARN_ON(1); + err = -ENAMETOOLONG; + goto out_err; + } +retry: + de = f2fs_find_entry(dir, &name, &page); + if (de && inode->i_ino == le32_to_cpu(de->ino)) + goto out_unmap_put; + + if (de) { + einode = f2fs_iget(inode->i_sb, le32_to_cpu(de->ino)); + if (IS_ERR(einode)) { + WARN_ON(1); + err = PTR_ERR(einode); + if (err == -ENOENT) + err = -EEXIST; + goto out_unmap_put; + } + err = acquire_orphan_inode(F2FS_I_SB(inode)); + if (err) { + iput(einode); + goto out_unmap_put; + } + f2fs_delete_entry(de, page, dir, einode); + iput(einode); + goto retry; + } + err = __f2fs_add_link(dir, &name, inode, inode->i_ino, inode->i_mode); + if (err) + goto out_err; + + if (is_inode_flag_set(F2FS_I(dir), FI_DELAY_IPUT)) { + iput(dir); + } else { + add_dirty_dir_inode(dir); + set_inode_flag(F2FS_I(dir), FI_DELAY_IPUT); + } + + goto out; + +out_unmap_put: + f2fs_dentry_kunmap(dir, page); + f2fs_put_page(page, 0); +out_err: + iput(dir); +out: + f2fs_msg(inode->i_sb, KERN_NOTICE, + "%s: ino = %x, name = %s, dir = %lx, err = %d", + __func__, ino_of_node(ipage), raw_inode->i_name, + IS_ERR(dir) ? 0 : dir->i_ino, err); + return err; +} + +static void recover_inode(struct inode *inode, struct page *page) +{ + struct f2fs_inode *raw = F2FS_INODE(page); + char *name; + + inode->i_mode = le16_to_cpu(raw->i_mode); + i_size_write(inode, le64_to_cpu(raw->i_size)); + inode->i_atime.tv_sec = le64_to_cpu(raw->i_mtime); + inode->i_ctime.tv_sec = le64_to_cpu(raw->i_ctime); + inode->i_mtime.tv_sec = le64_to_cpu(raw->i_mtime); + inode->i_atime.tv_nsec = le32_to_cpu(raw->i_mtime_nsec); + inode->i_ctime.tv_nsec = le32_to_cpu(raw->i_ctime_nsec); + inode->i_mtime.tv_nsec = le32_to_cpu(raw->i_mtime_nsec); + + if (file_enc_name(inode)) + name = ""; + else + name = F2FS_INODE(page)->i_name; + + f2fs_msg(inode->i_sb, KERN_NOTICE, "recover_inode: ino = %x, name = %s", + ino_of_node(page), name); +} + +static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head) +{ + unsigned long long cp_ver = cur_cp_version(F2FS_CKPT(sbi)); + struct curseg_info *curseg; + struct page *page = NULL; + block_t blkaddr; + int err = 0; + + /* get node pages in the current segment */ + curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); + blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + + ra_meta_pages(sbi, blkaddr, 1, META_POR, true); + + while (1) { + struct fsync_inode_entry *entry; + + if (!is_valid_blkaddr(sbi, blkaddr, META_POR)) + return 0; + + page = get_tmp_page(sbi, blkaddr); + + if (cp_ver != cpver_of_node(page)) + break; + + if (!is_fsync_dnode(page)) + goto next; + + entry = get_fsync_inode(head, ino_of_node(page)); + if (!entry) { + if (IS_INODE(page) && is_dent_dnode(page)) { + err = recover_inode_page(sbi, page); + if (err) + break; + } + + /* add this fsync inode to the list */ + entry = kmem_cache_alloc(fsync_entry_slab, GFP_F2FS_ZERO); + if (!entry) { + err = -ENOMEM; + break; + } + /* + * CP | dnode(F) | inode(DF) + * For this case, we should not give up now. + */ + entry->inode = f2fs_iget(sbi->sb, ino_of_node(page)); + if (IS_ERR(entry->inode)) { + err = PTR_ERR(entry->inode); + kmem_cache_free(fsync_entry_slab, entry); + if (err == -ENOENT) { + err = 0; + goto next; + } + break; + } + list_add_tail(&entry->list, head); + } + entry->blkaddr = blkaddr; + + if (IS_INODE(page)) { + entry->last_inode = blkaddr; + if (is_dent_dnode(page)) + entry->last_dentry = blkaddr; + } +next: + /* check next segment */ + blkaddr = next_blkaddr_of_node(page); + f2fs_put_page(page, 1); + + ra_meta_pages_cond(sbi, blkaddr); + } + f2fs_put_page(page, 1); + return err; +} + +static void destroy_fsync_dnodes(struct list_head *head) +{ + struct fsync_inode_entry *entry, *tmp; + + list_for_each_entry_safe(entry, tmp, head, list) { + iput(entry->inode); + list_del(&entry->list); + kmem_cache_free(fsync_entry_slab, entry); + } +} + +static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, + block_t blkaddr, struct dnode_of_data *dn) +{ + struct seg_entry *sentry; + unsigned int segno = GET_SEGNO(sbi, blkaddr); + unsigned short blkoff = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + struct f2fs_summary_block *sum_node; + struct f2fs_summary sum; + struct page *sum_page, *node_page; + struct dnode_of_data tdn = *dn; + nid_t ino, nid; + struct inode *inode; + unsigned int offset; + block_t bidx; + int i; + + sentry = get_seg_entry(sbi, segno); + if (!f2fs_test_bit(blkoff, sentry->cur_valid_map)) + return 0; + + /* Get the previous summary */ + for (i = CURSEG_WARM_DATA; i <= CURSEG_COLD_DATA; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i); + if (curseg->segno == segno) { + sum = curseg->sum_blk->entries[blkoff]; + goto got_it; + } + } + + sum_page = get_sum_page(sbi, segno); + sum_node = (struct f2fs_summary_block *)page_address(sum_page); + sum = sum_node->entries[blkoff]; + f2fs_put_page(sum_page, 1); +got_it: + /* Use the locked dnode page and inode */ + nid = le32_to_cpu(sum.nid); + if (dn->inode->i_ino == nid) { + tdn.nid = nid; + if (!dn->inode_page_locked) + lock_page(dn->inode_page); + tdn.node_page = dn->inode_page; + tdn.ofs_in_node = le16_to_cpu(sum.ofs_in_node); + goto truncate_out; + } else if (dn->nid == nid) { + tdn.ofs_in_node = le16_to_cpu(sum.ofs_in_node); + goto truncate_out; + } + + /* Get the node page */ + node_page = get_node_page(sbi, nid); + if (IS_ERR(node_page)) + return PTR_ERR(node_page); + + offset = ofs_of_node(node_page); + ino = ino_of_node(node_page); + f2fs_put_page(node_page, 1); + + if (ino != dn->inode->i_ino) { + /* Deallocate previous index in the node page */ + inode = f2fs_iget(sbi->sb, ino); + if (IS_ERR(inode)) + return PTR_ERR(inode); + } else { + inode = dn->inode; + } + + bidx = start_bidx_of_node(offset, F2FS_I(inode)) + + le16_to_cpu(sum.ofs_in_node); + + /* + * if inode page is locked, unlock temporarily, but its reference + * count keeps alive. + */ + if (ino == dn->inode->i_ino && dn->inode_page_locked) + unlock_page(dn->inode_page); + + set_new_dnode(&tdn, inode, NULL, NULL, 0); + if (get_dnode_of_data(&tdn, bidx, LOOKUP_NODE)) + goto out; + + if (tdn.data_blkaddr == blkaddr) + truncate_data_blocks_range(&tdn, 1); + + f2fs_put_dnode(&tdn); +out: + if (ino != dn->inode->i_ino) + iput(inode); + else if (dn->inode_page_locked) + lock_page(dn->inode_page); + return 0; + +truncate_out: + if (datablock_addr(tdn.node_page, tdn.ofs_in_node) == blkaddr) + truncate_data_blocks_range(&tdn, 1); + if (dn->inode->i_ino == nid && !dn->inode_page_locked) + unlock_page(dn->inode_page); + return 0; +} + +static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, + struct page *page, block_t blkaddr) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + unsigned int start, end; + struct dnode_of_data dn; + struct node_info ni; + int err = 0, recovered = 0; + + /* step 1: recover xattr */ + if (IS_INODE(page)) { + recover_inline_xattr(inode, page); + } else if (f2fs_has_xattr_block(ofs_of_node(page))) { + /* + * Deprecated; xattr blocks should be found from cold log. + * But, we should remain this for backward compatibility. + */ + recover_xattr_data(inode, page, blkaddr); + goto out; + } + + /* step 2: recover inline data */ + if (recover_inline_data(inode, page)) + goto out; + + /* step 3: recover data indices */ + start = start_bidx_of_node(ofs_of_node(page), fi); + end = start + ADDRS_PER_PAGE(page, fi); + + set_new_dnode(&dn, inode, NULL, NULL, 0); + + err = get_dnode_of_data(&dn, start, ALLOC_NODE); + if (err) + goto out; + + f2fs_wait_on_page_writeback(dn.node_page, NODE); + + get_node_info(sbi, dn.nid, &ni); + f2fs_bug_on(sbi, ni.ino != ino_of_node(page)); + f2fs_bug_on(sbi, ofs_of_node(dn.node_page) != ofs_of_node(page)); + + for (; start < end; start++, dn.ofs_in_node++) { + block_t src, dest; + + src = datablock_addr(dn.node_page, dn.ofs_in_node); + dest = datablock_addr(page, dn.ofs_in_node); + + /* skip recovering if dest is the same as src */ + if (src == dest) + continue; + + /* dest is invalid, just invalidate src block */ + if (dest == NULL_ADDR) { + truncate_data_blocks_range(&dn, 1); + continue; + } + + /* + * dest is reserved block, invalidate src block + * and then reserve one new block in dnode page. + */ + if (dest == NEW_ADDR) { + truncate_data_blocks_range(&dn, 1); + err = reserve_new_block(&dn); + f2fs_bug_on(sbi, err); + continue; + } + + /* dest is valid block, try to recover from src to dest */ + if (is_valid_blkaddr(sbi, dest, META_POR)) { + + if (src == NULL_ADDR) { + err = reserve_new_block(&dn); + /* We should not get -ENOSPC */ + f2fs_bug_on(sbi, err); + } + + /* Check the previous node page having this index */ + err = check_index_in_prev_nodes(sbi, dest, &dn); + if (err) + goto err; + + /* write dummy data page */ + f2fs_replace_block(sbi, &dn, src, dest, + ni.version, false); + recovered++; + } + } + + if (IS_INODE(dn.node_page)) + sync_inode_page(&dn); + + copy_node_footer(dn.node_page, page); + fill_node_footer(dn.node_page, dn.nid, ni.ino, + ofs_of_node(page), false); + set_page_dirty(dn.node_page); +err: + f2fs_put_dnode(&dn); +out: + f2fs_msg(sbi->sb, KERN_NOTICE, + "recover_data: ino = %lx, recovered = %d blocks, err = %d", + inode->i_ino, recovered, err); + return err; +} + +static int recover_data(struct f2fs_sb_info *sbi, + struct list_head *head, int type) +{ + unsigned long long cp_ver = cur_cp_version(F2FS_CKPT(sbi)); + struct curseg_info *curseg; + struct page *page = NULL; + int err = 0; + block_t blkaddr; + + /* get node pages in the current segment */ + curseg = CURSEG_I(sbi, type); + blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + + while (1) { + struct fsync_inode_entry *entry; + + if (!is_valid_blkaddr(sbi, blkaddr, META_POR)) + break; + + ra_meta_pages_cond(sbi, blkaddr); + + page = get_tmp_page(sbi, blkaddr); + + if (cp_ver != cpver_of_node(page)) { + f2fs_put_page(page, 1); + break; + } + + entry = get_fsync_inode(head, ino_of_node(page)); + if (!entry) + goto next; + /* + * inode(x) | CP | inode(x) | dnode(F) + * In this case, we can lose the latest inode(x). + * So, call recover_inode for the inode update. + */ + if (entry->last_inode == blkaddr) + recover_inode(entry->inode, page); + if (entry->last_dentry == blkaddr) { + err = recover_dentry(entry->inode, page); + if (err) { + f2fs_put_page(page, 1); + break; + } + } + err = do_recover_data(sbi, entry->inode, page, blkaddr); + if (err) { + f2fs_put_page(page, 1); + break; + } + + if (entry->blkaddr == blkaddr) { + iput(entry->inode); + list_del(&entry->list); + kmem_cache_free(fsync_entry_slab, entry); + } +next: + /* check next segment */ + blkaddr = next_blkaddr_of_node(page); + f2fs_put_page(page, 1); + } + if (!err) + allocate_new_segments(sbi); + return err; +} + +int recover_fsync_data(struct f2fs_sb_info *sbi) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); + struct list_head inode_list; + block_t blkaddr; + int err; + bool need_writecp = false; + + fsync_entry_slab = f2fs_kmem_cache_create("f2fs_fsync_inode_entry", + sizeof(struct fsync_inode_entry)); + if (!fsync_entry_slab) + return -ENOMEM; + + INIT_LIST_HEAD(&inode_list); + + /* prevent checkpoint */ + mutex_lock(&sbi->cp_mutex); + + blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + + /* step #1: find fsynced inode numbers */ + err = find_fsync_dnodes(sbi, &inode_list); + if (err) + goto out; + + if (list_empty(&inode_list)) + goto out; + + need_writecp = true; + + /* step #2: recover data */ + err = recover_data(sbi, &inode_list, CURSEG_WARM_NODE); + if (!err) + f2fs_bug_on(sbi, !list_empty(&inode_list)); +out: + destroy_fsync_dnodes(&inode_list); + kmem_cache_destroy(fsync_entry_slab); + + /* truncate meta pages to be used by the recovery */ + truncate_inode_pages_range(META_MAPPING(sbi), + (loff_t)MAIN_BLKADDR(sbi) << PAGE_CACHE_SHIFT, -1); + + if (err) { + truncate_inode_pages(NODE_MAPPING(sbi), 0); + truncate_inode_pages(META_MAPPING(sbi), 0); + } + + clear_sbi_flag(sbi, SBI_POR_DOING); + if (err) { + bool invalidate = false; + + if (discard_next_dnode(sbi, blkaddr)) + invalidate = true; + + /* Flush all the NAT/SIT pages */ + while (get_pages(sbi, F2FS_DIRTY_META)) + sync_meta_pages(sbi, META, LONG_MAX); + + /* invalidate temporary meta page */ + if (invalidate) + invalidate_mapping_pages(META_MAPPING(sbi), + blkaddr, blkaddr); + + set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG); + mutex_unlock(&sbi->cp_mutex); + } else if (need_writecp) { + struct cp_control cpc = { + .reason = CP_RECOVERY, + }; + mutex_unlock(&sbi->cp_mutex); + write_checkpoint(sbi, &cpc); + } else { + mutex_unlock(&sbi->cp_mutex); + } + return err; +} diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c new file mode 100644 index 00000000000..efebae3a6c3 --- /dev/null +++ b/fs/f2fs/segment.c @@ -0,0 +1,2560 @@ +/* + * fs/f2fs/segment.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "segment.h" +#include "node.h" +#include "trace.h" +#include + +#define __reverse_ffz(x) __reverse_ffs(~(x)) + +static struct kmem_cache *discard_entry_slab; +static struct kmem_cache *sit_entry_set_slab; +static struct kmem_cache *inmem_entry_slab; + +static unsigned long __reverse_ulong(unsigned char *str) +{ + unsigned long tmp = 0; + int shift = 24, idx = 0; + +#if BITS_PER_LONG == 64 + shift = 56; +#endif + while (shift >= 0) { + tmp |= (unsigned long)str[idx++] << shift; + shift -= BITS_PER_BYTE; + } + return tmp; +} + +/** + * Copied from latest lib/llist.c + * llist_for_each_entry_safe - iterate over some deleted entries of + * lock-less list of given type + * safe against removal of list entry + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @node: the first entry of deleted list entries. + * @member: the name of the llist_node with the struct. + * + * In general, some entries of the lock-less list can be traversed + * safely only after being removed from list, so start with an entry + * instead of list head. + * + * If being used on entries deleted from lock-less list directly, the + * traverse order is from the newest to the oldest added entry. If + * you want to traverse from the oldest to the newest, you must + * reverse the order by yourself before traversing. + */ +#define llist_for_each_entry_safe(pos, n, node, member) \ + for (pos = llist_entry((node), typeof(*pos), member); \ + &pos->member != NULL && \ + (n = llist_entry(pos->member.next, typeof(*n), member), true); \ + pos = n) + +/** + * Copied from latest lib/llist.c + * llist_reverse_order - reverse order of a llist chain + * @head: first item of the list to be reversed + * + * Reverse the order of a chain of llist entries and return the + * new first entry. + */ +struct llist_node *llist_reverse_order(struct llist_node *head) +{ + struct llist_node *new_head = NULL; + + while (head) { + struct llist_node *tmp = head; + head = head->next; + tmp->next = new_head; + new_head = tmp; + } + + return new_head; +} + +/** + * Copied from latest linux/list.h + * list_last_entry - get the last element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + * + * Note, that list is expected to be not empty. + */ +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) + +/* + * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since + * MSB and LSB are reversed in a byte by f2fs_set_bit. + */ +static inline unsigned long __reverse_ffs(unsigned long word) +{ + int num = 0; + +#if BITS_PER_LONG == 64 + if ((word & 0xffffffff00000000UL) == 0) + num += 32; + else + word >>= 32; +#endif + if ((word & 0xffff0000) == 0) + num += 16; + else + word >>= 16; + + if ((word & 0xff00) == 0) + num += 8; + else + word >>= 8; + + if ((word & 0xf0) == 0) + num += 4; + else + word >>= 4; + + if ((word & 0xc) == 0) + num += 2; + else + word >>= 2; + + if ((word & 0x2) == 0) + num += 1; + return num; +} + +/* + * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because + * f2fs_set_bit makes MSB and LSB reversed in a byte. + * Example: + * MSB <--> LSB + * f2fs_set_bit(0, bitmap) => 1000 0000 + * f2fs_set_bit(7, bitmap) => 0000 0001 + */ +static unsigned long __find_rev_next_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + const unsigned long *p = addr + BIT_WORD(offset); + unsigned long result = offset & ~(BITS_PER_LONG - 1); + unsigned long tmp; + + if (offset >= size) + return size; + + size -= result; + offset %= BITS_PER_LONG; + if (!offset) + goto aligned; + + tmp = __reverse_ulong((unsigned char *)p); + tmp &= ~0UL >> offset; + + if (size < BITS_PER_LONG) + goto found_first; + if (tmp) + goto found_middle; + + size -= BITS_PER_LONG; + result += BITS_PER_LONG; + p++; +aligned: + while (size & ~(BITS_PER_LONG-1)) { + tmp = __reverse_ulong((unsigned char *)p); + if (tmp) + goto found_middle; + result += BITS_PER_LONG; + size -= BITS_PER_LONG; + p++; + } + if (!size) + return result; + + tmp = __reverse_ulong((unsigned char *)p); +found_first: + tmp &= (~0UL << (BITS_PER_LONG - size)); + if (!tmp) /* Are any bits set? */ + return result + size; /* Nope. */ +found_middle: + return result + __reverse_ffs(tmp); +} + +static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + const unsigned long *p = addr + BIT_WORD(offset); + unsigned long result = offset & ~(BITS_PER_LONG - 1); + unsigned long tmp; + + if (offset >= size) + return size; + + size -= result; + offset %= BITS_PER_LONG; + if (!offset) + goto aligned; + + tmp = __reverse_ulong((unsigned char *)p); + tmp |= ~((~0UL << offset) >> offset); + + if (size < BITS_PER_LONG) + goto found_first; + if (tmp != ~0UL) + goto found_middle; + + size -= BITS_PER_LONG; + result += BITS_PER_LONG; + p++; +aligned: + while (size & ~(BITS_PER_LONG - 1)) { + tmp = __reverse_ulong((unsigned char *)p); + if (tmp != ~0UL) + goto found_middle; + result += BITS_PER_LONG; + size -= BITS_PER_LONG; + p++; + } + if (!size) + return result; + + tmp = __reverse_ulong((unsigned char *)p); +found_first: + tmp |= ~(~0UL << (BITS_PER_LONG - size)); + if (tmp == ~0UL) /* Are any bits zero? */ + return result + size; /* Nope. */ +found_middle: + return result + __reverse_ffz(tmp); +} + +void register_inmem_page(struct inode *inode, struct page *page) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct inmem_pages *new; + + f2fs_trace_pid(page); + + set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE); + SetPagePrivate(page); + + new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); + + /* add atomic page indices to the list */ + new->page = page; + INIT_LIST_HEAD(&new->list); + + /* increase reference count with clean state */ + mutex_lock(&fi->inmem_lock); + get_page(page); + list_add_tail(&new->list, &fi->inmem_pages); + inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); + mutex_unlock(&fi->inmem_lock); + + trace_f2fs_register_inmem_page(page, INMEM); +} + +int commit_inmem_pages(struct inode *inode, bool abort) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct inmem_pages *cur, *tmp; + bool submit_bio = false; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = DATA, + .rw = WRITE_SYNC | REQ_PRIO, + .encrypted_page = NULL, + }; + int err = 0; + + /* + * The abort is true only when f2fs_evict_inode is called. + * Basically, the f2fs_evict_inode doesn't produce any data writes, so + * that we don't need to call f2fs_balance_fs. + * Otherwise, f2fs_gc in f2fs_balance_fs can wait forever until this + * inode becomes free by iget_locked in f2fs_iget. + */ + if (!abort) { + f2fs_balance_fs(sbi); + f2fs_lock_op(sbi); + } + + mutex_lock(&fi->inmem_lock); + list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) { + lock_page(cur->page); + if (!abort) { + if (cur->page->mapping == inode->i_mapping) { + set_page_dirty(cur->page); + f2fs_wait_on_page_writeback(cur->page, DATA); + if (clear_page_dirty_for_io(cur->page)) + inode_dec_dirty_pages(inode); + trace_f2fs_commit_inmem_page(cur->page, INMEM); + fio.page = cur->page; + err = do_write_data_page(&fio); + if (err) { + unlock_page(cur->page); + break; + } + clear_cold_data(cur->page); + submit_bio = true; + } + } else { + trace_f2fs_commit_inmem_page(cur->page, INMEM_DROP); + } + set_page_private(cur->page, 0); + ClearPagePrivate(cur->page); + f2fs_put_page(cur->page, 1); + + list_del(&cur->list); + kmem_cache_free(inmem_entry_slab, cur); + dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES); + } + mutex_unlock(&fi->inmem_lock); + + if (!abort) { + f2fs_unlock_op(sbi); + if (submit_bio) + f2fs_submit_merged_bio(sbi, DATA, WRITE); + } + return err; +} + +/* + * This function balances dirty node and dentry pages. + * In addition, it controls garbage collection. + */ +void f2fs_balance_fs(struct f2fs_sb_info *sbi) +{ + /* + * We should do GC or end up with checkpoint, if there are so many dirty + * dir/node pages without enough free segments. + */ + if (has_not_enough_free_secs(sbi, 0)) { + mutex_lock(&sbi->gc_mutex); + f2fs_gc(sbi, false); + } +} + +void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) +{ + /* try to shrink extent cache when there is no enough memory */ + if (!available_free_memory(sbi, EXTENT_CACHE)) + f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER); + + /* check the # of cached NAT entries */ + if (!available_free_memory(sbi, NAT_ENTRIES)) + try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); + + if (!available_free_memory(sbi, FREE_NIDS)) + try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES); + + /* checkpoint is the only way to shrink partial cached entries */ + if (!available_free_memory(sbi, NAT_ENTRIES) || + excess_prefree_segs(sbi) || + !available_free_memory(sbi, INO_ENTRIES) || + jiffies > sbi->cp_expires) + f2fs_sync_fs(sbi->sb, true); +} + +struct __submit_bio_ret { + struct completion event; + int error; +}; + +static void __submit_bio_wait_endio(struct bio *bio, int error) +{ + struct __submit_bio_ret *ret = bio->bi_private; + + ret->error = error; + complete(&ret->event); +} + +static int __submit_bio_wait(int rw, struct bio *bio) +{ + struct __submit_bio_ret ret; + + rw |= REQ_SYNC; + init_completion(&ret.event); + bio->bi_private = &ret; + bio->bi_end_io = __submit_bio_wait_endio; + submit_bio(rw, bio); + wait_for_completion(&ret.event); + + return ret.error; +} + +static int issue_flush_thread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; + wait_queue_head_t *q = &fcc->flush_wait_queue; +repeat: + if (kthread_should_stop()) + return 0; + + if (!llist_empty(&fcc->issue_list)) { + struct bio *bio; + struct flush_cmd *cmd, *next; + int ret; + + bio = f2fs_bio_alloc(0); + + fcc->dispatch_list = llist_del_all(&fcc->issue_list); + fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); + + bio->bi_bdev = sbi->sb->s_bdev; + ret = __submit_bio_wait(WRITE_FLUSH, bio); + + llist_for_each_entry_safe(cmd, next, + fcc->dispatch_list, llnode) { + cmd->ret = ret; + complete(&cmd->wait); + } + bio_put(bio); + fcc->dispatch_list = NULL; + } + + wait_event_interruptible(*q, + kthread_should_stop() || !llist_empty(&fcc->issue_list)); + goto repeat; +} + +int f2fs_issue_flush(struct f2fs_sb_info *sbi) +{ + struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; + struct flush_cmd cmd; + + trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER), + test_opt(sbi, FLUSH_MERGE)); + + if (test_opt(sbi, NOBARRIER)) + return 0; + + if (!test_opt(sbi, FLUSH_MERGE)) { + struct bio *bio = f2fs_bio_alloc(0); + int ret; + + bio->bi_bdev = sbi->sb->s_bdev; + ret = __submit_bio_wait(WRITE_FLUSH, bio); + bio_put(bio); + return ret; + } + + init_completion(&cmd.wait); + + llist_add(&cmd.llnode, &fcc->issue_list); + + if (!fcc->dispatch_list) + wake_up(&fcc->flush_wait_queue); + + wait_for_completion(&cmd.wait); + + return cmd.ret; +} + +int create_flush_cmd_control(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + struct flush_cmd_control *fcc; + int err = 0; + + fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL); + if (!fcc) + return -ENOMEM; + init_waitqueue_head(&fcc->flush_wait_queue); + init_llist_head(&fcc->issue_list); + SM_I(sbi)->cmd_control_info = fcc; + fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, + "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(fcc->f2fs_issue_flush)) { + err = PTR_ERR(fcc->f2fs_issue_flush); + kfree(fcc); + SM_I(sbi)->cmd_control_info = NULL; + return err; + } + + return err; +} + +void destroy_flush_cmd_control(struct f2fs_sb_info *sbi) +{ + struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; + + if (fcc && fcc->f2fs_issue_flush) + kthread_stop(fcc->f2fs_issue_flush); + kfree(fcc); + SM_I(sbi)->cmd_control_info = NULL; +} + +static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, + enum dirty_type dirty_type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + /* need not be added */ + if (IS_CURSEG(sbi, segno)) + return; + + if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) + dirty_i->nr_dirty[dirty_type]++; + + if (dirty_type == DIRTY) { + struct seg_entry *sentry = get_seg_entry(sbi, segno); + enum dirty_type t = sentry->type; + + if (unlikely(t >= DIRTY)) { + f2fs_bug_on(sbi, 1); + return; + } + if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) + dirty_i->nr_dirty[t]++; + } +} + +static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, + enum dirty_type dirty_type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) + dirty_i->nr_dirty[dirty_type]--; + + if (dirty_type == DIRTY) { + struct seg_entry *sentry = get_seg_entry(sbi, segno); + enum dirty_type t = sentry->type; + + if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) + dirty_i->nr_dirty[t]--; + + if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0) + clear_bit(GET_SECNO(sbi, segno), + dirty_i->victim_secmap); + } +} + +/* + * Should not occur error such as -ENOMEM. + * Adding dirty entry into seglist is not critical operation. + * If a given segment is one of current working segments, it won't be added. + */ +static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned short valid_blocks; + + if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) + return; + + mutex_lock(&dirty_i->seglist_lock); + + valid_blocks = get_valid_blocks(sbi, segno, 0); + + if (valid_blocks == 0) { + __locate_dirty_segment(sbi, segno, PRE); + __remove_dirty_segment(sbi, segno, DIRTY); + } else if (valid_blocks < sbi->blocks_per_seg) { + __locate_dirty_segment(sbi, segno, DIRTY); + } else { + /* Recovery routine with SSR needs this */ + __remove_dirty_segment(sbi, segno, DIRTY); + } + + mutex_unlock(&dirty_i->seglist_lock); +} + +static int f2fs_issue_discard(struct f2fs_sb_info *sbi, + block_t blkstart, block_t blklen) +{ + sector_t start = SECTOR_FROM_BLOCK(blkstart); + sector_t len = SECTOR_FROM_BLOCK(blklen); + struct seg_entry *se; + unsigned int offset; + block_t i; + + for (i = blkstart; i < blkstart + blklen; i++) { + se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); + offset = GET_BLKOFF_FROM_SEG0(sbi, i); + + if (!f2fs_test_and_set_bit(offset, se->discard_map)) + sbi->discard_blks--; + } + trace_f2fs_issue_discard(sbi->sb, blkstart, blklen); + return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0); +} + +bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + int err = -ENOTSUPP; + + if (test_opt(sbi, DISCARD)) { + struct seg_entry *se = get_seg_entry(sbi, + GET_SEGNO(sbi, blkaddr)); + unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + + if (f2fs_test_bit(offset, se->discard_map)) + return false; + + err = f2fs_issue_discard(sbi, blkaddr, 1); + } + + if (err) { + update_meta_page(sbi, NULL, blkaddr); + return true; + } + return false; +} + +static void __add_discard_entry(struct f2fs_sb_info *sbi, + struct cp_control *cpc, struct seg_entry *se, + unsigned int start, unsigned int end) +{ + struct list_head *head = &SM_I(sbi)->discard_list; + struct discard_entry *new, *last; + + if (!list_empty(head)) { + last = list_last_entry(head, struct discard_entry, list); + if (START_BLOCK(sbi, cpc->trim_start) + start == + last->blkaddr + last->len) { + last->len += end - start; + goto done; + } + } + + new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS); + INIT_LIST_HEAD(&new->list); + new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start; + new->len = end - start; + list_add_tail(&new->list, head); +done: + SM_I(sbi)->nr_discards += end - start; +} + +static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); + int max_blocks = sbi->blocks_per_seg; + struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); + unsigned long *cur_map = (unsigned long *)se->cur_valid_map; + unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; + unsigned long *discard_map = (unsigned long *)se->discard_map; + unsigned long *dmap = SIT_I(sbi)->tmp_map; + unsigned int start = 0, end = -1; + bool force = (cpc->reason == CP_DISCARD); + int i; + + if (se->valid_blocks == max_blocks) + return; + + if (!force) { + if (!test_opt(sbi, DISCARD) || !se->valid_blocks || + SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards) + return; + } + + /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ + for (i = 0; i < entries; i++) + dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : + (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; + + while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) { + start = __find_rev_next_bit(dmap, max_blocks, end + 1); + if (start >= max_blocks) + break; + + end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); + __add_discard_entry(sbi, cpc, se, start, end); + } +} + +void release_discard_addrs(struct f2fs_sb_info *sbi) +{ + struct list_head *head = &(SM_I(sbi)->discard_list); + struct discard_entry *entry, *this; + + /* drop caches */ + list_for_each_entry_safe(entry, this, head, list) { + list_del(&entry->list); + kmem_cache_free(discard_entry_slab, entry); + } +} + +/* + * Should call clear_prefree_segments after checkpoint is done. + */ +static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int segno; + + mutex_lock(&dirty_i->seglist_lock); + for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) + __set_test_and_free(sbi, segno); + mutex_unlock(&dirty_i->seglist_lock); +} + +void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct list_head *head = &(SM_I(sbi)->discard_list); + struct discard_entry *entry, *this; + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; + unsigned int start = 0, end = -1; + + mutex_lock(&dirty_i->seglist_lock); + + while (1) { + int i; + start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); + if (start >= MAIN_SEGS(sbi)) + break; + end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), + start + 1); + + for (i = start; i < end; i++) + clear_bit(i, prefree_map); + + dirty_i->nr_dirty[PRE] -= end - start; + + if (!test_opt(sbi, DISCARD)) + continue; + + f2fs_issue_discard(sbi, START_BLOCK(sbi, start), + (end - start) << sbi->log_blocks_per_seg); + } + mutex_unlock(&dirty_i->seglist_lock); + + /* send small discards */ + list_for_each_entry_safe(entry, this, head, list) { + if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen) + goto skip; + f2fs_issue_discard(sbi, entry->blkaddr, entry->len); + cpc->trimmed += entry->len; +skip: + list_del(&entry->list); + SM_I(sbi)->nr_discards -= entry->len; + kmem_cache_free(discard_entry_slab, entry); + } +} + +static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + + if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { + sit_i->dirty_sentries++; + return false; + } + + return true; +} + +static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, + unsigned int segno, int modified) +{ + struct seg_entry *se = get_seg_entry(sbi, segno); + se->type = type; + if (modified) + __mark_sit_entry_dirty(sbi, segno); +} + +static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) +{ + struct seg_entry *se; + unsigned int segno, offset; + long int new_vblocks; + + segno = GET_SEGNO(sbi, blkaddr); + + se = get_seg_entry(sbi, segno); + new_vblocks = se->valid_blocks + del; + offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + + f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || + (new_vblocks > sbi->blocks_per_seg))); + + se->valid_blocks = new_vblocks; + se->mtime = get_mtime(sbi); + SIT_I(sbi)->max_mtime = se->mtime; + + /* Update valid block bitmap */ + if (del > 0) { + if (f2fs_test_and_set_bit(offset, se->cur_valid_map)) + f2fs_bug_on(sbi, 1); + if (!f2fs_test_and_set_bit(offset, se->discard_map)) + sbi->discard_blks--; + } else { + if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map)) + f2fs_bug_on(sbi, 1); + if (f2fs_test_and_clear_bit(offset, se->discard_map)) + sbi->discard_blks++; + } + if (!f2fs_test_bit(offset, se->ckpt_valid_map)) + se->ckpt_valid_blocks += del; + + __mark_sit_entry_dirty(sbi, segno); + + /* update total number of valid blocks to be written in ckpt area */ + SIT_I(sbi)->written_valid_blocks += del; + + if (sbi->segs_per_sec > 1) + get_sec_entry(sbi, segno)->valid_blocks += del; +} + +void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new) +{ + update_sit_entry(sbi, new, 1); + if (GET_SEGNO(sbi, old) != NULL_SEGNO) + update_sit_entry(sbi, old, -1); + + locate_dirty_segment(sbi, GET_SEGNO(sbi, old)); + locate_dirty_segment(sbi, GET_SEGNO(sbi, new)); +} + +void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) +{ + unsigned int segno = GET_SEGNO(sbi, addr); + struct sit_info *sit_i = SIT_I(sbi); + + f2fs_bug_on(sbi, addr == NULL_ADDR); + if (addr == NEW_ADDR) + return; + + /* add it into sit main buffer */ + mutex_lock(&sit_i->sentry_lock); + + update_sit_entry(sbi, addr, -1); + + /* add it into dirty seglist */ + locate_dirty_segment(sbi, segno); + + mutex_unlock(&sit_i->sentry_lock); +} + +bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int segno, offset; + struct seg_entry *se; + bool is_cp = false; + + if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) + return true; + + mutex_lock(&sit_i->sentry_lock); + + segno = GET_SEGNO(sbi, blkaddr); + se = get_seg_entry(sbi, segno); + offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + + if (f2fs_test_bit(offset, se->ckpt_valid_map)) + is_cp = true; + + mutex_unlock(&sit_i->sentry_lock); + + return is_cp; +} + +/* + * This function should be resided under the curseg_mutex lock + */ +static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, + struct f2fs_summary *sum) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + void *addr = curseg->sum_blk; + addr += curseg->next_blkoff * sizeof(struct f2fs_summary); + memcpy(addr, sum, sizeof(struct f2fs_summary)); +} + +/* + * Calculate the number of current summary pages for writing + */ +int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) +{ + int valid_sum_count = 0; + int i, sum_in_page; + + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + if (sbi->ckpt->alloc_type[i] == SSR) + valid_sum_count += sbi->blocks_per_seg; + else { + if (for_ra) + valid_sum_count += le16_to_cpu( + F2FS_CKPT(sbi)->cur_data_blkoff[i]); + else + valid_sum_count += curseg_blkoff(sbi, i); + } + } + + sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE - + SUM_FOOTER_SIZE) / SUMMARY_SIZE; + if (valid_sum_count <= sum_in_page) + return 1; + else if ((valid_sum_count - sum_in_page) <= + (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) + return 2; + return 3; +} + +/* + * Caller should put this summary page + */ +struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) +{ + return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); +} + +void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr) +{ + struct page *page = grab_meta_page(sbi, blk_addr); + void *dst = page_address(page); + + if (src) + memcpy(dst, src, PAGE_CACHE_SIZE); + else + memset(dst, 0, PAGE_CACHE_SIZE); + set_page_dirty(page); + f2fs_put_page(page, 1); +} + +static void write_sum_page(struct f2fs_sb_info *sbi, + struct f2fs_summary_block *sum_blk, block_t blk_addr) +{ + update_meta_page(sbi, (void *)sum_blk, blk_addr); +} + +static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int segno = curseg->segno + 1; + struct free_segmap_info *free_i = FREE_I(sbi); + + if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec) + return !test_bit(segno, free_i->free_segmap); + return 0; +} + +/* + * Find a new segment from the free segments bitmap to right order + * This function should be returned with success, otherwise BUG + */ +static void get_new_segment(struct f2fs_sb_info *sbi, + unsigned int *newseg, bool new_sec, int dir) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int segno, secno, zoneno; + unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; + unsigned int hint = *newseg / sbi->segs_per_sec; + unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg); + unsigned int left_start = hint; + bool init = true; + int go_left = 0; + int i; + + spin_lock(&free_i->segmap_lock); + + if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { + segno = find_next_zero_bit(free_i->free_segmap, + MAIN_SEGS(sbi), *newseg + 1); + if (segno - *newseg < sbi->segs_per_sec - + (*newseg % sbi->segs_per_sec)) + goto got_it; + } +find_other_zone: + secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); + if (secno >= MAIN_SECS(sbi)) { + if (dir == ALLOC_RIGHT) { + secno = find_next_zero_bit(free_i->free_secmap, + MAIN_SECS(sbi), 0); + f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi)); + } else { + go_left = 1; + left_start = hint - 1; + } + } + if (go_left == 0) + goto skip_left; + + while (test_bit(left_start, free_i->free_secmap)) { + if (left_start > 0) { + left_start--; + continue; + } + left_start = find_next_zero_bit(free_i->free_secmap, + MAIN_SECS(sbi), 0); + f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi)); + break; + } + secno = left_start; +skip_left: + hint = secno; + segno = secno * sbi->segs_per_sec; + zoneno = secno / sbi->secs_per_zone; + + /* give up on finding another zone */ + if (!init) + goto got_it; + if (sbi->secs_per_zone == 1) + goto got_it; + if (zoneno == old_zoneno) + goto got_it; + if (dir == ALLOC_LEFT) { + if (!go_left && zoneno + 1 >= total_zones) + goto got_it; + if (go_left && zoneno == 0) + goto got_it; + } + for (i = 0; i < NR_CURSEG_TYPE; i++) + if (CURSEG_I(sbi, i)->zone == zoneno) + break; + + if (i < NR_CURSEG_TYPE) { + /* zone is in user, try another */ + if (go_left) + hint = zoneno * sbi->secs_per_zone - 1; + else if (zoneno + 1 >= total_zones) + hint = 0; + else + hint = (zoneno + 1) * sbi->secs_per_zone; + init = false; + goto find_other_zone; + } +got_it: + /* set it as dirty segment in free segmap */ + f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); + __set_inuse(sbi, segno); + *newseg = segno; + spin_unlock(&free_i->segmap_lock); +} + +static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + struct summary_footer *sum_footer; + + curseg->segno = curseg->next_segno; + curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno); + curseg->next_blkoff = 0; + curseg->next_segno = NULL_SEGNO; + + sum_footer = &(curseg->sum_blk->footer); + memset(sum_footer, 0, sizeof(struct summary_footer)); + if (IS_DATASEG(type)) + SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); + if (IS_NODESEG(type)) + SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); + __set_sit_entry_type(sbi, type, curseg->segno, modified); +} + +/* + * Allocate a current working segment. + * This function always allocates a free segment in LFS manner. + */ +static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int segno = curseg->segno; + int dir = ALLOC_LEFT; + + write_sum_page(sbi, curseg->sum_blk, + GET_SUM_BLOCK(sbi, segno)); + if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA) + dir = ALLOC_RIGHT; + + if (test_opt(sbi, NOHEAP)) + dir = ALLOC_RIGHT; + + get_new_segment(sbi, &segno, new_sec, dir); + curseg->next_segno = segno; + reset_curseg(sbi, type, 1); + curseg->alloc_type = LFS; +} + +static void __next_free_blkoff(struct f2fs_sb_info *sbi, + struct curseg_info *seg, block_t start) +{ + struct seg_entry *se = get_seg_entry(sbi, seg->segno); + int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); + unsigned long *target_map = SIT_I(sbi)->tmp_map; + unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; + unsigned long *cur_map = (unsigned long *)se->cur_valid_map; + int i, pos; + + for (i = 0; i < entries; i++) + target_map[i] = ckpt_map[i] | cur_map[i]; + + pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); + + seg->next_blkoff = pos; +} + +/* + * If a segment is written by LFS manner, next block offset is just obtained + * by increasing the current block offset. However, if a segment is written by + * SSR manner, next block offset obtained by calling __next_free_blkoff + */ +static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, + struct curseg_info *seg) +{ + if (seg->alloc_type == SSR) + __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); + else + seg->next_blkoff++; +} + +/* + * This function always allocates a used segment(from dirty seglist) by SSR + * manner, so it should recover the existing segment information of valid blocks + */ +static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int new_segno = curseg->next_segno; + struct f2fs_summary_block *sum_node; + struct page *sum_page; + + write_sum_page(sbi, curseg->sum_blk, + GET_SUM_BLOCK(sbi, curseg->segno)); + __set_test_and_inuse(sbi, new_segno); + + mutex_lock(&dirty_i->seglist_lock); + __remove_dirty_segment(sbi, new_segno, PRE); + __remove_dirty_segment(sbi, new_segno, DIRTY); + mutex_unlock(&dirty_i->seglist_lock); + + reset_curseg(sbi, type, 1); + curseg->alloc_type = SSR; + __next_free_blkoff(sbi, curseg, 0); + + if (reuse) { + sum_page = get_sum_page(sbi, new_segno); + sum_node = (struct f2fs_summary_block *)page_address(sum_page); + memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); + f2fs_put_page(sum_page, 1); + } +} + +static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; + + if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0)) + return v_ops->get_victim(sbi, + &(curseg)->next_segno, BG_GC, type, SSR); + + /* For data segments, let's do SSR more intensively */ + for (; type >= CURSEG_HOT_DATA; type--) + if (v_ops->get_victim(sbi, &(curseg)->next_segno, + BG_GC, type, SSR)) + return 1; + return 0; +} + +/* + * flush out current segment and replace it with new segment + * This function should be returned with success, otherwise BUG + */ +static void allocate_segment_by_default(struct f2fs_sb_info *sbi, + int type, bool force) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + + if (force) + new_curseg(sbi, type, true); + else if (type == CURSEG_WARM_NODE) + new_curseg(sbi, type, false); + else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) + new_curseg(sbi, type, false); + else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) + change_curseg(sbi, type, true); + else + new_curseg(sbi, type, false); + + stat_inc_seg_type(sbi, curseg); +} + +static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int old_segno; + + old_segno = curseg->segno; + SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true); + locate_dirty_segment(sbi, old_segno); +} + +void allocate_new_segments(struct f2fs_sb_info *sbi) +{ + int i; + + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) + __allocate_new_segments(sbi, i); +} + +static const struct segment_allocation default_salloc_ops = { + .allocate_segment = allocate_segment_by_default, +}; + +int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) +{ + __u64 start = F2FS_BYTES_TO_BLK(range->start); + __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; + unsigned int start_segno, end_segno; + struct cp_control cpc; + + if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) + return -EINVAL; + + cpc.trimmed = 0; + if (end <= MAIN_BLKADDR(sbi)) + goto out; + + /* start/end segment number in main_area */ + start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); + end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : + GET_SEGNO(sbi, end); + cpc.reason = CP_DISCARD; + cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); + + /* do checkpoint to issue discard commands safely */ + for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) { + cpc.trim_start = start_segno; + + if (sbi->discard_blks == 0) + break; + else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi)) + cpc.trim_end = end_segno; + else + cpc.trim_end = min_t(unsigned int, + rounddown(start_segno + + BATCHED_TRIM_SEGMENTS(sbi), + sbi->segs_per_sec) - 1, end_segno); + + mutex_lock(&sbi->gc_mutex); + write_checkpoint(sbi, &cpc); + mutex_unlock(&sbi->gc_mutex); + } +out: + range->len = F2FS_BLK_TO_BYTES(cpc.trimmed); + return 0; +} + +static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + if (curseg->next_blkoff < sbi->blocks_per_seg) + return true; + return false; +} + +static int __get_segment_type_2(struct page *page, enum page_type p_type) +{ + if (p_type == DATA) + return CURSEG_HOT_DATA; + else + return CURSEG_HOT_NODE; +} + +static int __get_segment_type_4(struct page *page, enum page_type p_type) +{ + if (p_type == DATA) { + struct inode *inode = page->mapping->host; + + if (S_ISDIR(inode->i_mode)) + return CURSEG_HOT_DATA; + else + return CURSEG_COLD_DATA; + } else { + if (IS_DNODE(page) && is_cold_node(page)) + return CURSEG_WARM_NODE; + else + return CURSEG_COLD_NODE; + } +} + +static int __get_segment_type_6(struct page *page, enum page_type p_type) +{ + if (p_type == DATA) { + struct inode *inode = page->mapping->host; + + if (S_ISDIR(inode->i_mode)) + return CURSEG_HOT_DATA; + else if (is_cold_data(page) || file_is_cold(inode)) + return CURSEG_COLD_DATA; + else + return CURSEG_WARM_DATA; + } else { + if (IS_DNODE(page)) + return is_cold_node(page) ? CURSEG_WARM_NODE : + CURSEG_HOT_NODE; + else + return CURSEG_COLD_NODE; + } +} + +static int __get_segment_type(struct page *page, enum page_type p_type) +{ + switch (F2FS_P_SB(page)->active_logs) { + case 2: + return __get_segment_type_2(page, p_type); + case 4: + return __get_segment_type_4(page, p_type); + } + /* NR_CURSEG_TYPE(6) logs by default */ + f2fs_bug_on(F2FS_P_SB(page), + F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE); + return __get_segment_type_6(page, p_type); +} + +void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, + block_t old_blkaddr, block_t *new_blkaddr, + struct f2fs_summary *sum, int type) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct curseg_info *curseg; + bool direct_io = (type == CURSEG_DIRECT_IO); + + type = direct_io ? CURSEG_WARM_DATA : type; + + curseg = CURSEG_I(sbi, type); + + mutex_lock(&curseg->curseg_mutex); + mutex_lock(&sit_i->sentry_lock); + + /* direct_io'ed data is aligned to the segment for better performance */ + if (direct_io && curseg->next_blkoff && + !has_not_enough_free_secs(sbi, 0)) + __allocate_new_segments(sbi, type); + + *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + + /* + * __add_sum_entry should be resided under the curseg_mutex + * because, this function updates a summary entry in the + * current summary block. + */ + __add_sum_entry(sbi, type, sum); + + __refresh_next_blkoff(sbi, curseg); + + stat_inc_block_count(sbi, curseg); + + if (!__has_curseg_space(sbi, type)) + sit_i->s_ops->allocate_segment(sbi, type, false); + /* + * SIT information should be updated before segment allocation, + * since SSR needs latest valid block information. + */ + refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr); + + mutex_unlock(&sit_i->sentry_lock); + + if (page && IS_NODESEG(type)) + fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); + + mutex_unlock(&curseg->curseg_mutex); +} + +static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) +{ + int type = __get_segment_type(fio->page, fio->type); + + allocate_data_block(fio->sbi, fio->page, fio->blk_addr, + &fio->blk_addr, sum, type); + + /* writeout dirty page into bdev */ + f2fs_submit_page_mbio(fio); +} + +void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) +{ + struct f2fs_io_info fio = { + .sbi = sbi, + .type = META, + .rw = WRITE_SYNC | REQ_META | REQ_PRIO, + .blk_addr = page->index, + .page = page, + .encrypted_page = NULL, + }; + + if (unlikely(page->index >= MAIN_BLKADDR(sbi))) + fio.rw &= ~REQ_META; + + set_page_writeback(page); + f2fs_submit_page_mbio(&fio); +} + +void write_node_page(unsigned int nid, struct f2fs_io_info *fio) +{ + struct f2fs_summary sum; + + set_summary(&sum, nid, 0, 0); + do_write_page(&sum, fio); +} + +void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + struct f2fs_summary sum; + struct node_info ni; + + f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); + get_node_info(sbi, dn->nid, &ni); + set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); + do_write_page(&sum, fio); + dn->data_blkaddr = fio->blk_addr; +} + +void rewrite_data_page(struct f2fs_io_info *fio) +{ + stat_inc_inplace_blocks(fio->sbi); + f2fs_submit_page_mbio(fio); +} + +static void __f2fs_replace_block(struct f2fs_sb_info *sbi, + struct f2fs_summary *sum, + block_t old_blkaddr, block_t new_blkaddr, + bool recover_curseg) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct curseg_info *curseg; + unsigned int segno, old_cursegno; + struct seg_entry *se; + int type; + unsigned short old_blkoff; + + segno = GET_SEGNO(sbi, new_blkaddr); + se = get_seg_entry(sbi, segno); + type = se->type; + + if (!recover_curseg) { + /* for recovery flow */ + if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { + if (old_blkaddr == NULL_ADDR) + type = CURSEG_COLD_DATA; + else + type = CURSEG_WARM_DATA; + } + } else { + if (!IS_CURSEG(sbi, segno)) + type = CURSEG_WARM_DATA; + } + + curseg = CURSEG_I(sbi, type); + + mutex_lock(&curseg->curseg_mutex); + mutex_lock(&sit_i->sentry_lock); + + old_cursegno = curseg->segno; + old_blkoff = curseg->next_blkoff; + + /* change the current segment */ + if (segno != curseg->segno) { + curseg->next_segno = segno; + change_curseg(sbi, type, true); + } + + curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); + __add_sum_entry(sbi, type, sum); + + if (!recover_curseg) + update_sit_entry(sbi, new_blkaddr, 1); + if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) + update_sit_entry(sbi, old_blkaddr, -1); + + locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); + locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); + + locate_dirty_segment(sbi, old_cursegno); + + if (recover_curseg) { + if (old_cursegno != curseg->segno) { + curseg->next_segno = old_cursegno; + change_curseg(sbi, type, true); + } + curseg->next_blkoff = old_blkoff; + } + + mutex_unlock(&sit_i->sentry_lock); + mutex_unlock(&curseg->curseg_mutex); +} + +void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, + block_t old_addr, block_t new_addr, + unsigned char version, bool recover_curseg) +{ + struct f2fs_summary sum; + + set_summary(&sum, dn->nid, dn->ofs_in_node, version); + + __f2fs_replace_block(sbi, &sum, old_addr, new_addr, recover_curseg); + + dn->data_blkaddr = new_addr; + set_data_blkaddr(dn); + f2fs_update_extent_cache(dn); +} + +static inline bool is_merged_page(struct f2fs_sb_info *sbi, + struct page *page, enum page_type type) +{ + enum page_type btype = PAGE_TYPE_OF_BIO(type); + struct f2fs_bio_info *io = &sbi->write_io[btype]; + struct bio_vec *bvec; + struct page *target; + int i; + + down_read(&io->io_rwsem); + if (!io->bio) { + up_read(&io->io_rwsem); + return false; + } + + __bio_for_each_segment(bvec, io->bio, i, 0) { + + if (bvec->bv_page->mapping) { + target = bvec->bv_page; + } else { + struct f2fs_crypto_ctx *ctx; + + /* encrypted page */ + ctx = (struct f2fs_crypto_ctx *)page_private( + bvec->bv_page); + target = ctx->w.control_page; + } + + if (page == target) { + up_read(&io->io_rwsem); + return true; + } + } + + up_read(&io->io_rwsem); + return false; +} + +void f2fs_wait_on_page_writeback(struct page *page, + enum page_type type) +{ + if (PageWriteback(page)) { + struct f2fs_sb_info *sbi = F2FS_P_SB(page); + + if (is_merged_page(sbi, page, type)) + f2fs_submit_merged_bio(sbi, type, WRITE); + wait_on_page_writeback(page); + } +} + +void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi, + block_t blkaddr) +{ + struct page *cpage; + + if (blkaddr == NEW_ADDR) + return; + + f2fs_bug_on(sbi, blkaddr == NULL_ADDR); + + cpage = find_lock_page(META_MAPPING(sbi), blkaddr); + if (cpage) { + f2fs_wait_on_page_writeback(cpage, DATA); + f2fs_put_page(cpage, 1); + } +} + +static int read_compacted_summaries(struct f2fs_sb_info *sbi) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct curseg_info *seg_i; + unsigned char *kaddr; + struct page *page; + block_t start; + int i, j, offset; + + start = start_sum_block(sbi); + + page = get_meta_page(sbi, start++); + kaddr = (unsigned char *)page_address(page); + + /* Step 1: restore nat cache */ + seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); + memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE); + + /* Step 2: restore sit cache */ + seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); + memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE, + SUM_JOURNAL_SIZE); + offset = 2 * SUM_JOURNAL_SIZE; + + /* Step 3: restore summary entries */ + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + unsigned short blk_off; + unsigned int segno; + + seg_i = CURSEG_I(sbi, i); + segno = le32_to_cpu(ckpt->cur_data_segno[i]); + blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); + seg_i->next_segno = segno; + reset_curseg(sbi, i, 0); + seg_i->alloc_type = ckpt->alloc_type[i]; + seg_i->next_blkoff = blk_off; + + if (seg_i->alloc_type == SSR) + blk_off = sbi->blocks_per_seg; + + for (j = 0; j < blk_off; j++) { + struct f2fs_summary *s; + s = (struct f2fs_summary *)(kaddr + offset); + seg_i->sum_blk->entries[j] = *s; + offset += SUMMARY_SIZE; + if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE - + SUM_FOOTER_SIZE) + continue; + + f2fs_put_page(page, 1); + page = NULL; + + page = get_meta_page(sbi, start++); + kaddr = (unsigned char *)page_address(page); + offset = 0; + } + } + f2fs_put_page(page, 1); + return 0; +} + +static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_summary_block *sum; + struct curseg_info *curseg; + struct page *new; + unsigned short blk_off; + unsigned int segno = 0; + block_t blk_addr = 0; + + /* get segment number and block addr */ + if (IS_DATASEG(type)) { + segno = le32_to_cpu(ckpt->cur_data_segno[type]); + blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - + CURSEG_HOT_DATA]); + if (__exist_node_summaries(sbi)) + blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); + else + blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); + } else { + segno = le32_to_cpu(ckpt->cur_node_segno[type - + CURSEG_HOT_NODE]); + blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - + CURSEG_HOT_NODE]); + if (__exist_node_summaries(sbi)) + blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, + type - CURSEG_HOT_NODE); + else + blk_addr = GET_SUM_BLOCK(sbi, segno); + } + + new = get_meta_page(sbi, blk_addr); + sum = (struct f2fs_summary_block *)page_address(new); + + if (IS_NODESEG(type)) { + if (__exist_node_summaries(sbi)) { + struct f2fs_summary *ns = &sum->entries[0]; + int i; + for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { + ns->version = 0; + ns->ofs_in_node = 0; + } + } else { + int err; + + err = restore_node_summary(sbi, segno, sum); + if (err) { + f2fs_put_page(new, 1); + return err; + } + } + } + + /* set uncompleted segment to curseg */ + curseg = CURSEG_I(sbi, type); + mutex_lock(&curseg->curseg_mutex); + memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE); + curseg->next_segno = segno; + reset_curseg(sbi, type, 0); + curseg->alloc_type = ckpt->alloc_type[type]; + curseg->next_blkoff = blk_off; + mutex_unlock(&curseg->curseg_mutex); + f2fs_put_page(new, 1); + return 0; +} + +static int restore_curseg_summaries(struct f2fs_sb_info *sbi) +{ + int type = CURSEG_HOT_DATA; + int err; + + if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) { + int npages = npages_for_summary_flush(sbi, true); + + if (npages >= 2) + ra_meta_pages(sbi, start_sum_block(sbi), npages, + META_CP, true); + + /* restore for compacted data summary */ + if (read_compacted_summaries(sbi)) + return -EINVAL; + type = CURSEG_HOT_NODE; + } + + if (__exist_node_summaries(sbi)) + ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type), + NR_CURSEG_TYPE - type, META_CP, true); + + for (; type <= CURSEG_COLD_NODE; type++) { + err = read_normal_summaries(sbi, type); + if (err) + return err; + } + + return 0; +} + +static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct page *page; + unsigned char *kaddr; + struct f2fs_summary *summary; + struct curseg_info *seg_i; + int written_size = 0; + int i, j; + + page = grab_meta_page(sbi, blkaddr++); + kaddr = (unsigned char *)page_address(page); + + /* Step 1: write nat cache */ + seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); + memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE); + written_size += SUM_JOURNAL_SIZE; + + /* Step 2: write sit cache */ + seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); + memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits, + SUM_JOURNAL_SIZE); + written_size += SUM_JOURNAL_SIZE; + + /* Step 3: write summary entries */ + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + unsigned short blkoff; + seg_i = CURSEG_I(sbi, i); + if (sbi->ckpt->alloc_type[i] == SSR) + blkoff = sbi->blocks_per_seg; + else + blkoff = curseg_blkoff(sbi, i); + + for (j = 0; j < blkoff; j++) { + if (!page) { + page = grab_meta_page(sbi, blkaddr++); + kaddr = (unsigned char *)page_address(page); + written_size = 0; + } + summary = (struct f2fs_summary *)(kaddr + written_size); + *summary = seg_i->sum_blk->entries[j]; + written_size += SUMMARY_SIZE; + + if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE - + SUM_FOOTER_SIZE) + continue; + + set_page_dirty(page); + f2fs_put_page(page, 1); + page = NULL; + } + } + if (page) { + set_page_dirty(page); + f2fs_put_page(page, 1); + } +} + +static void write_normal_summaries(struct f2fs_sb_info *sbi, + block_t blkaddr, int type) +{ + int i, end; + if (IS_DATASEG(type)) + end = type + NR_CURSEG_DATA_TYPE; + else + end = type + NR_CURSEG_NODE_TYPE; + + for (i = type; i < end; i++) { + struct curseg_info *sum = CURSEG_I(sbi, i); + mutex_lock(&sum->curseg_mutex); + write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type)); + mutex_unlock(&sum->curseg_mutex); + } +} + +void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) +{ + if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) + write_compacted_summaries(sbi, start_blk); + else + write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); +} + +void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) +{ + write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); +} + +int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type, + unsigned int val, int alloc) +{ + int i; + + if (type == NAT_JOURNAL) { + for (i = 0; i < nats_in_cursum(sum); i++) { + if (le32_to_cpu(nid_in_journal(sum, i)) == val) + return i; + } + if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) + return update_nats_in_cursum(sum, 1); + } else if (type == SIT_JOURNAL) { + for (i = 0; i < sits_in_cursum(sum); i++) + if (le32_to_cpu(segno_in_journal(sum, i)) == val) + return i; + if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES) + return update_sits_in_cursum(sum, 1); + } + return -1; +} + +static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + return get_meta_page(sbi, current_sit_addr(sbi, segno)); +} + +static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, + unsigned int start) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct page *src_page, *dst_page; + pgoff_t src_off, dst_off; + void *src_addr, *dst_addr; + + src_off = current_sit_addr(sbi, start); + dst_off = next_sit_addr(sbi, src_off); + + /* get current sit block page without lock */ + src_page = get_meta_page(sbi, src_off); + dst_page = grab_meta_page(sbi, dst_off); + f2fs_bug_on(sbi, PageDirty(src_page)); + + src_addr = page_address(src_page); + dst_addr = page_address(dst_page); + memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE); + + set_page_dirty(dst_page); + f2fs_put_page(src_page, 1); + + set_to_next_sit(sit_i, start); + + return dst_page; +} + +static struct sit_entry_set *grab_sit_entry_set(void) +{ + struct sit_entry_set *ses = + f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS); + + ses->entry_cnt = 0; + INIT_LIST_HEAD(&ses->set_list); + return ses; +} + +static void release_sit_entry_set(struct sit_entry_set *ses) +{ + list_del(&ses->set_list); + kmem_cache_free(sit_entry_set_slab, ses); +} + +static void adjust_sit_entry_set(struct sit_entry_set *ses, + struct list_head *head) +{ + struct sit_entry_set *next = ses; + + if (list_is_last(&ses->set_list, head)) + return; + + list_for_each_entry_continue(next, head, set_list) + if (ses->entry_cnt <= next->entry_cnt) + break; + + list_move_tail(&ses->set_list, &next->set_list); +} + +static void add_sit_entry(unsigned int segno, struct list_head *head) +{ + struct sit_entry_set *ses; + unsigned int start_segno = START_SEGNO(segno); + + list_for_each_entry(ses, head, set_list) { + if (ses->start_segno == start_segno) { + ses->entry_cnt++; + adjust_sit_entry_set(ses, head); + return; + } + } + + ses = grab_sit_entry_set(); + + ses->start_segno = start_segno; + ses->entry_cnt++; + list_add(&ses->set_list, head); +} + +static void add_sits_in_set(struct f2fs_sb_info *sbi) +{ + struct f2fs_sm_info *sm_info = SM_I(sbi); + struct list_head *set_list = &sm_info->sit_entry_set; + unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; + unsigned int segno; + + for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) + add_sit_entry(segno, set_list); +} + +static void remove_sits_in_journal(struct f2fs_sb_info *sbi) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + int i; + + for (i = sits_in_cursum(sum) - 1; i >= 0; i--) { + unsigned int segno; + bool dirtied; + + segno = le32_to_cpu(segno_in_journal(sum, i)); + dirtied = __mark_sit_entry_dirty(sbi, segno); + + if (!dirtied) + add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); + } + update_sits_in_cursum(sum, -sits_in_cursum(sum)); +} + +/* + * CP calls this function, which flushes SIT entries including sit_journal, + * and moves prefree segs to free segs. + */ +void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned long *bitmap = sit_i->dirty_sentries_bitmap; + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + struct sit_entry_set *ses, *tmp; + struct list_head *head = &SM_I(sbi)->sit_entry_set; + bool to_journal = true; + struct seg_entry *se; + + mutex_lock(&curseg->curseg_mutex); + mutex_lock(&sit_i->sentry_lock); + + if (!sit_i->dirty_sentries) + goto out; + + /* + * add and account sit entries of dirty bitmap in sit entry + * set temporarily + */ + add_sits_in_set(sbi); + + /* + * if there are no enough space in journal to store dirty sit + * entries, remove all entries from journal and add and account + * them in sit entry set. + */ + if (!__has_cursum_space(sum, sit_i->dirty_sentries, SIT_JOURNAL)) + remove_sits_in_journal(sbi); + + /* + * there are two steps to flush sit entries: + * #1, flush sit entries to journal in current cold data summary block. + * #2, flush sit entries to sit page. + */ + list_for_each_entry_safe(ses, tmp, head, set_list) { + struct page *page = NULL; + struct f2fs_sit_block *raw_sit = NULL; + unsigned int start_segno = ses->start_segno; + unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, + (unsigned long)MAIN_SEGS(sbi)); + unsigned int segno = start_segno; + + if (to_journal && + !__has_cursum_space(sum, ses->entry_cnt, SIT_JOURNAL)) + to_journal = false; + + if (!to_journal) { + page = get_next_sit_page(sbi, start_segno); + raw_sit = page_address(page); + } + + /* flush dirty sit entries in region of current sit set */ + for_each_set_bit_from(segno, bitmap, end) { + int offset, sit_offset; + + se = get_seg_entry(sbi, segno); + + /* add discard candidates */ + if (cpc->reason != CP_DISCARD) { + cpc->trim_start = segno; + add_discard_addrs(sbi, cpc); + } + + if (to_journal) { + offset = lookup_journal_in_cursum(sum, + SIT_JOURNAL, segno, 1); + f2fs_bug_on(sbi, offset < 0); + segno_in_journal(sum, offset) = + cpu_to_le32(segno); + seg_info_to_raw_sit(se, + &sit_in_journal(sum, offset)); + } else { + sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); + seg_info_to_raw_sit(se, + &raw_sit->entries[sit_offset]); + } + + __clear_bit(segno, bitmap); + sit_i->dirty_sentries--; + ses->entry_cnt--; + } + + if (!to_journal) + f2fs_put_page(page, 1); + + f2fs_bug_on(sbi, ses->entry_cnt); + release_sit_entry_set(ses); + } + + f2fs_bug_on(sbi, !list_empty(head)); + f2fs_bug_on(sbi, sit_i->dirty_sentries); +out: + if (cpc->reason == CP_DISCARD) { + for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) + add_discard_addrs(sbi, cpc); + } + mutex_unlock(&sit_i->sentry_lock); + mutex_unlock(&curseg->curseg_mutex); + + set_prefree_as_free_segments(sbi); +} + +static int build_sit_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct sit_info *sit_i; + unsigned int sit_segs, start; + char *src_bitmap, *dst_bitmap; + unsigned int bitmap_size; + + /* allocate memory for SIT information */ + sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL); + if (!sit_i) + return -ENOMEM; + + SM_I(sbi)->sit_info = sit_i; + + sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) * + sizeof(struct seg_entry), GFP_KERNEL); + if (!sit_i->sentries) + return -ENOMEM; + + bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); + sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL); + if (!sit_i->dirty_sentries_bitmap) + return -ENOMEM; + + for (start = 0; start < MAIN_SEGS(sbi); start++) { + sit_i->sentries[start].cur_valid_map + = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); + sit_i->sentries[start].ckpt_valid_map + = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); + sit_i->sentries[start].discard_map + = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); + if (!sit_i->sentries[start].cur_valid_map || + !sit_i->sentries[start].ckpt_valid_map || + !sit_i->sentries[start].discard_map) + return -ENOMEM; + } + + sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); + if (!sit_i->tmp_map) + return -ENOMEM; + + if (sbi->segs_per_sec > 1) { + sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) * + sizeof(struct sec_entry), GFP_KERNEL); + if (!sit_i->sec_entries) + return -ENOMEM; + } + + /* get information related with SIT */ + sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; + + /* setup SIT bitmap from ckeckpoint pack */ + bitmap_size = __bitmap_size(sbi, SIT_BITMAP); + src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); + + dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); + if (!dst_bitmap) + return -ENOMEM; + + /* init SIT information */ + sit_i->s_ops = &default_salloc_ops; + + sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); + sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; + sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count); + sit_i->sit_bitmap = dst_bitmap; + sit_i->bitmap_size = bitmap_size; + sit_i->dirty_sentries = 0; + sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; + sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); + sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec; + mutex_init(&sit_i->sentry_lock); + return 0; +} + +static int build_free_segmap(struct f2fs_sb_info *sbi) +{ + struct free_segmap_info *free_i; + unsigned int bitmap_size, sec_bitmap_size; + + /* allocate memory for free segmap information */ + free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL); + if (!free_i) + return -ENOMEM; + + SM_I(sbi)->free_info = free_i; + + bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); + free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL); + if (!free_i->free_segmap) + return -ENOMEM; + + sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); + free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL); + if (!free_i->free_secmap) + return -ENOMEM; + + /* set all segments as dirty temporarily */ + memset(free_i->free_segmap, 0xff, bitmap_size); + memset(free_i->free_secmap, 0xff, sec_bitmap_size); + + /* init free segmap information */ + free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); + free_i->free_segments = 0; + free_i->free_sections = 0; + spin_lock_init(&free_i->segmap_lock); + return 0; +} + +static int build_curseg(struct f2fs_sb_info *sbi) +{ + struct curseg_info *array; + int i; + + array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL); + if (!array) + return -ENOMEM; + + SM_I(sbi)->curseg_array = array; + + for (i = 0; i < NR_CURSEG_TYPE; i++) { + mutex_init(&array[i].curseg_mutex); + array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL); + if (!array[i].sum_blk) + return -ENOMEM; + array[i].segno = NULL_SEGNO; + array[i].next_blkoff = 0; + } + return restore_curseg_summaries(sbi); +} + +static void build_sit_entries(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); + struct f2fs_summary_block *sum = curseg->sum_blk; + int sit_blk_cnt = SIT_BLK_CNT(sbi); + unsigned int i, start, end; + unsigned int readed, start_blk = 0; + int nrpages = MAX_BIO_BLOCKS(sbi); + + do { + readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true); + + start = start_blk * sit_i->sents_per_block; + end = (start_blk + readed) * sit_i->sents_per_block; + + for (; start < end && start < MAIN_SEGS(sbi); start++) { + struct seg_entry *se = &sit_i->sentries[start]; + struct f2fs_sit_block *sit_blk; + struct f2fs_sit_entry sit; + struct page *page; + + mutex_lock(&curseg->curseg_mutex); + for (i = 0; i < sits_in_cursum(sum); i++) { + if (le32_to_cpu(segno_in_journal(sum, i)) + == start) { + sit = sit_in_journal(sum, i); + mutex_unlock(&curseg->curseg_mutex); + goto got_it; + } + } + mutex_unlock(&curseg->curseg_mutex); + + page = get_current_sit_page(sbi, start); + sit_blk = (struct f2fs_sit_block *)page_address(page); + sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; + f2fs_put_page(page, 1); +got_it: + check_block_count(sbi, start, &sit); + seg_info_from_raw_sit(se, &sit); + + /* build discard map only one time */ + memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE); + sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks; + + if (sbi->segs_per_sec > 1) { + struct sec_entry *e = get_sec_entry(sbi, start); + e->valid_blocks += se->valid_blocks; + } + } + start_blk += readed; + } while (start_blk < sit_blk_cnt); +} + +static void init_free_segmap(struct f2fs_sb_info *sbi) +{ + unsigned int start; + int type; + + for (start = 0; start < MAIN_SEGS(sbi); start++) { + struct seg_entry *sentry = get_seg_entry(sbi, start); + if (!sentry->valid_blocks) + __set_free(sbi, start); + } + + /* set use the current segments */ + for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { + struct curseg_info *curseg_t = CURSEG_I(sbi, type); + __set_test_and_inuse(sbi, curseg_t->segno); + } +} + +static void init_dirty_segmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int segno = 0, offset = 0; + unsigned short valid_blocks; + + while (1) { + /* find dirty segment based on free segmap */ + segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); + if (segno >= MAIN_SEGS(sbi)) + break; + offset = segno + 1; + valid_blocks = get_valid_blocks(sbi, segno, 0); + if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) + continue; + if (valid_blocks > sbi->blocks_per_seg) { + f2fs_bug_on(sbi, 1); + continue; + } + mutex_lock(&dirty_i->seglist_lock); + __locate_dirty_segment(sbi, segno, DIRTY); + mutex_unlock(&dirty_i->seglist_lock); + } +} + +static int init_victim_secmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); + + dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL); + if (!dirty_i->victim_secmap) + return -ENOMEM; + return 0; +} + +static int build_dirty_segmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i; + unsigned int bitmap_size, i; + + /* allocate memory for dirty segments list information */ + dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL); + if (!dirty_i) + return -ENOMEM; + + SM_I(sbi)->dirty_info = dirty_i; + mutex_init(&dirty_i->seglist_lock); + + bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); + + for (i = 0; i < NR_DIRTY_TYPE; i++) { + dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL); + if (!dirty_i->dirty_segmap[i]) + return -ENOMEM; + } + + init_dirty_segmap(sbi); + return init_victim_secmap(sbi); +} + +/* + * Update min, max modified time for cost-benefit GC algorithm + */ +static void init_min_max_mtime(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int segno; + + mutex_lock(&sit_i->sentry_lock); + + sit_i->min_mtime = LLONG_MAX; + + for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) { + unsigned int i; + unsigned long long mtime = 0; + + for (i = 0; i < sbi->segs_per_sec; i++) + mtime += get_seg_entry(sbi, segno + i)->mtime; + + mtime = div_u64(mtime, sbi->segs_per_sec); + + if (sit_i->min_mtime > mtime) + sit_i->min_mtime = mtime; + } + sit_i->max_mtime = get_mtime(sbi); + mutex_unlock(&sit_i->sentry_lock); +} + +int build_segment_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_sm_info *sm_info; + int err; + + sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL); + if (!sm_info) + return -ENOMEM; + + /* init sm info */ + sbi->sm_info = sm_info; + sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); + sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); + sm_info->segment_count = le32_to_cpu(raw_super->segment_count); + sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); + sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); + sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); + sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); + sm_info->rec_prefree_segments = sm_info->main_segments * + DEF_RECLAIM_PREFREE_SEGMENTS / 100; + sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC; + sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; + sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; + + INIT_LIST_HEAD(&sm_info->discard_list); + sm_info->nr_discards = 0; + sm_info->max_discards = 0; + + sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS; + + INIT_LIST_HEAD(&sm_info->sit_entry_set); + + if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) { + err = create_flush_cmd_control(sbi); + if (err) + return err; + } + + err = build_sit_info(sbi); + if (err) + return err; + err = build_free_segmap(sbi); + if (err) + return err; + err = build_curseg(sbi); + if (err) + return err; + + /* reinit free segmap based on SIT */ + build_sit_entries(sbi); + + init_free_segmap(sbi); + err = build_dirty_segmap(sbi); + if (err) + return err; + + init_min_max_mtime(sbi); + return 0; +} + +static void discard_dirty_segmap(struct f2fs_sb_info *sbi, + enum dirty_type dirty_type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + mutex_lock(&dirty_i->seglist_lock); + f2fs_kvfree(dirty_i->dirty_segmap[dirty_type]); + dirty_i->nr_dirty[dirty_type] = 0; + mutex_unlock(&dirty_i->seglist_lock); +} + +static void destroy_victim_secmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + f2fs_kvfree(dirty_i->victim_secmap); +} + +static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + int i; + + if (!dirty_i) + return; + + /* discard pre-free/dirty segments list */ + for (i = 0; i < NR_DIRTY_TYPE; i++) + discard_dirty_segmap(sbi, i); + + destroy_victim_secmap(sbi); + SM_I(sbi)->dirty_info = NULL; + kfree(dirty_i); +} + +static void destroy_curseg(struct f2fs_sb_info *sbi) +{ + struct curseg_info *array = SM_I(sbi)->curseg_array; + int i; + + if (!array) + return; + SM_I(sbi)->curseg_array = NULL; + for (i = 0; i < NR_CURSEG_TYPE; i++) + kfree(array[i].sum_blk); + kfree(array); +} + +static void destroy_free_segmap(struct f2fs_sb_info *sbi) +{ + struct free_segmap_info *free_i = SM_I(sbi)->free_info; + if (!free_i) + return; + SM_I(sbi)->free_info = NULL; + f2fs_kvfree(free_i->free_segmap); + f2fs_kvfree(free_i->free_secmap); + kfree(free_i); +} + +static void destroy_sit_info(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int start; + + if (!sit_i) + return; + + if (sit_i->sentries) { + for (start = 0; start < MAIN_SEGS(sbi); start++) { + kfree(sit_i->sentries[start].cur_valid_map); + kfree(sit_i->sentries[start].ckpt_valid_map); + kfree(sit_i->sentries[start].discard_map); + } + } + kfree(sit_i->tmp_map); + + f2fs_kvfree(sit_i->sentries); + f2fs_kvfree(sit_i->sec_entries); + f2fs_kvfree(sit_i->dirty_sentries_bitmap); + + SM_I(sbi)->sit_info = NULL; + kfree(sit_i->sit_bitmap); + kfree(sit_i); +} + +void destroy_segment_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_sm_info *sm_info = SM_I(sbi); + + if (!sm_info) + return; + destroy_flush_cmd_control(sbi); + destroy_dirty_segmap(sbi); + destroy_curseg(sbi); + destroy_free_segmap(sbi); + destroy_sit_info(sbi); + sbi->sm_info = NULL; + kfree(sm_info); +} + +int __init create_segment_manager_caches(void) +{ + discard_entry_slab = f2fs_kmem_cache_create("discard_entry", + sizeof(struct discard_entry)); + if (!discard_entry_slab) + goto fail; + + sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set", + sizeof(struct sit_entry_set)); + if (!sit_entry_set_slab) + goto destory_discard_entry; + + inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry", + sizeof(struct inmem_pages)); + if (!inmem_entry_slab) + goto destroy_sit_entry_set; + return 0; + +destroy_sit_entry_set: + kmem_cache_destroy(sit_entry_set_slab); +destory_discard_entry: + kmem_cache_destroy(discard_entry_slab); +fail: + return -ENOMEM; +} + +void destroy_segment_manager_caches(void) +{ + kmem_cache_destroy(sit_entry_set_slab); + kmem_cache_destroy(discard_entry_slab); + kmem_cache_destroy(inmem_entry_slab); +} diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h new file mode 100644 index 00000000000..3bbeca13f70 --- /dev/null +++ b/fs/f2fs/segment.h @@ -0,0 +1,733 @@ +/* + * fs/f2fs/segment.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include + +/* constant macro */ +#define NULL_SEGNO ((unsigned int)(~0)) +#define NULL_SECNO ((unsigned int)(~0)) + +#define DEF_RECLAIM_PREFREE_SEGMENTS 5 /* 5% over total segments */ + +/* L: Logical segment # in volume, R: Relative segment # in main area */ +#define GET_L2R_SEGNO(free_i, segno) (segno - free_i->start_segno) +#define GET_R2L_SEGNO(free_i, segno) (segno + free_i->start_segno) + +#define IS_DATASEG(t) (t <= CURSEG_COLD_DATA) +#define IS_NODESEG(t) (t >= CURSEG_HOT_NODE) + +#define IS_CURSEG(sbi, seg) \ + ((seg == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno) || \ + (seg == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno) || \ + (seg == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno) || \ + (seg == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno) || \ + (seg == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno) || \ + (seg == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno)) + +#define IS_CURSEC(sbi, secno) \ + ((secno == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno / \ + sbi->segs_per_sec) || \ + (secno == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno / \ + sbi->segs_per_sec) || \ + (secno == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno / \ + sbi->segs_per_sec) || \ + (secno == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno / \ + sbi->segs_per_sec) || \ + (secno == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno / \ + sbi->segs_per_sec) || \ + (secno == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno / \ + sbi->segs_per_sec)) \ + +#define MAIN_BLKADDR(sbi) (SM_I(sbi)->main_blkaddr) +#define SEG0_BLKADDR(sbi) (SM_I(sbi)->seg0_blkaddr) + +#define MAIN_SEGS(sbi) (SM_I(sbi)->main_segments) +#define MAIN_SECS(sbi) (sbi->total_sections) + +#define TOTAL_SEGS(sbi) (SM_I(sbi)->segment_count) +#define TOTAL_BLKS(sbi) (TOTAL_SEGS(sbi) << sbi->log_blocks_per_seg) + +#define MAX_BLKADDR(sbi) (SEG0_BLKADDR(sbi) + TOTAL_BLKS(sbi)) +#define SEGMENT_SIZE(sbi) (1ULL << (sbi->log_blocksize + \ + sbi->log_blocks_per_seg)) + +#define START_BLOCK(sbi, segno) (SEG0_BLKADDR(sbi) + \ + (GET_R2L_SEGNO(FREE_I(sbi), segno) << sbi->log_blocks_per_seg)) + +#define NEXT_FREE_BLKADDR(sbi, curseg) \ + (START_BLOCK(sbi, curseg->segno) + curseg->next_blkoff) + +#define GET_SEGOFF_FROM_SEG0(sbi, blk_addr) ((blk_addr) - SEG0_BLKADDR(sbi)) +#define GET_SEGNO_FROM_SEG0(sbi, blk_addr) \ + (GET_SEGOFF_FROM_SEG0(sbi, blk_addr) >> sbi->log_blocks_per_seg) +#define GET_BLKOFF_FROM_SEG0(sbi, blk_addr) \ + (GET_SEGOFF_FROM_SEG0(sbi, blk_addr) & (sbi->blocks_per_seg - 1)) + +#define GET_SEGNO(sbi, blk_addr) \ + (((blk_addr == NULL_ADDR) || (blk_addr == NEW_ADDR)) ? \ + NULL_SEGNO : GET_L2R_SEGNO(FREE_I(sbi), \ + GET_SEGNO_FROM_SEG0(sbi, blk_addr))) +#define GET_SECNO(sbi, segno) \ + ((segno) / sbi->segs_per_sec) +#define GET_ZONENO_FROM_SEGNO(sbi, segno) \ + ((segno / sbi->segs_per_sec) / sbi->secs_per_zone) + +#define GET_SUM_BLOCK(sbi, segno) \ + ((sbi->sm_info->ssa_blkaddr) + segno) + +#define GET_SUM_TYPE(footer) ((footer)->entry_type) +#define SET_SUM_TYPE(footer, type) ((footer)->entry_type = type) + +#define SIT_ENTRY_OFFSET(sit_i, segno) \ + (segno % sit_i->sents_per_block) +#define SIT_BLOCK_OFFSET(segno) \ + (segno / SIT_ENTRY_PER_BLOCK) +#define START_SEGNO(segno) \ + (SIT_BLOCK_OFFSET(segno) * SIT_ENTRY_PER_BLOCK) +#define SIT_BLK_CNT(sbi) \ + ((MAIN_SEGS(sbi) + SIT_ENTRY_PER_BLOCK - 1) / SIT_ENTRY_PER_BLOCK) +#define f2fs_bitmap_size(nr) \ + (BITS_TO_LONGS(nr) * sizeof(unsigned long)) + +#define SECTOR_FROM_BLOCK(blk_addr) \ + (((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK) +#define SECTOR_TO_BLOCK(sectors) \ + (sectors >> F2FS_LOG_SECTORS_PER_BLOCK) +#define MAX_BIO_BLOCKS(sbi) \ + ((int)min((int)max_hw_blocks(sbi), BIO_MAX_PAGES)) + +/* + * indicate a block allocation direction: RIGHT and LEFT. + * RIGHT means allocating new sections towards the end of volume. + * LEFT means the opposite direction. + */ +enum { + ALLOC_RIGHT = 0, + ALLOC_LEFT +}; + +/* + * In the victim_sel_policy->alloc_mode, there are two block allocation modes. + * LFS writes data sequentially with cleaning operations. + * SSR (Slack Space Recycle) reuses obsolete space without cleaning operations. + */ +enum { + LFS = 0, + SSR +}; + +/* + * In the victim_sel_policy->gc_mode, there are two gc, aka cleaning, modes. + * GC_CB is based on cost-benefit algorithm. + * GC_GREEDY is based on greedy algorithm. + */ +enum { + GC_CB = 0, + GC_GREEDY +}; + +/* + * BG_GC means the background cleaning job. + * FG_GC means the on-demand cleaning job. + * FORCE_FG_GC means on-demand cleaning job in background. + */ +enum { + BG_GC = 0, + FG_GC, + FORCE_FG_GC, +}; + +/* for a function parameter to select a victim segment */ +struct victim_sel_policy { + int alloc_mode; /* LFS or SSR */ + int gc_mode; /* GC_CB or GC_GREEDY */ + unsigned long *dirty_segmap; /* dirty segment bitmap */ + unsigned int max_search; /* maximum # of segments to search */ + unsigned int offset; /* last scanned bitmap offset */ + unsigned int ofs_unit; /* bitmap search unit */ + unsigned int min_cost; /* minimum cost */ + unsigned int min_segno; /* segment # having min. cost */ +}; + +struct seg_entry { + unsigned short valid_blocks; /* # of valid blocks */ + unsigned char *cur_valid_map; /* validity bitmap of blocks */ + /* + * # of valid blocks and the validity bitmap stored in the the last + * checkpoint pack. This information is used by the SSR mode. + */ + unsigned short ckpt_valid_blocks; + unsigned char *ckpt_valid_map; + unsigned char *discard_map; + unsigned char type; /* segment type like CURSEG_XXX_TYPE */ + unsigned long long mtime; /* modification time of the segment */ +}; + +struct sec_entry { + unsigned int valid_blocks; /* # of valid blocks in a section */ +}; + +struct segment_allocation { + void (*allocate_segment)(struct f2fs_sb_info *, int, bool); +}; + +/* + * this value is set in page as a private data which indicate that + * the page is atomically written, and it is in inmem_pages list. + */ +#define ATOMIC_WRITTEN_PAGE 0x0000ffff + +#define IS_ATOMIC_WRITTEN_PAGE(page) \ + (page_private(page) == (unsigned long)ATOMIC_WRITTEN_PAGE) + +struct inmem_pages { + struct list_head list; + struct page *page; +}; + +struct sit_info { + const struct segment_allocation *s_ops; + + block_t sit_base_addr; /* start block address of SIT area */ + block_t sit_blocks; /* # of blocks used by SIT area */ + block_t written_valid_blocks; /* # of valid blocks in main area */ + char *sit_bitmap; /* SIT bitmap pointer */ + unsigned int bitmap_size; /* SIT bitmap size */ + + unsigned long *tmp_map; /* bitmap for temporal use */ + unsigned long *dirty_sentries_bitmap; /* bitmap for dirty sentries */ + unsigned int dirty_sentries; /* # of dirty sentries */ + unsigned int sents_per_block; /* # of SIT entries per block */ + struct mutex sentry_lock; /* to protect SIT cache */ + struct seg_entry *sentries; /* SIT segment-level cache */ + struct sec_entry *sec_entries; /* SIT section-level cache */ + + /* for cost-benefit algorithm in cleaning procedure */ + unsigned long long elapsed_time; /* elapsed time after mount */ + unsigned long long mounted_time; /* mount time */ + unsigned long long min_mtime; /* min. modification time */ + unsigned long long max_mtime; /* max. modification time */ +}; + +struct free_segmap_info { + unsigned int start_segno; /* start segment number logically */ + unsigned int free_segments; /* # of free segments */ + unsigned int free_sections; /* # of free sections */ + spinlock_t segmap_lock; /* free segmap lock */ + unsigned long *free_segmap; /* free segment bitmap */ + unsigned long *free_secmap; /* free section bitmap */ +}; + +/* Notice: The order of dirty type is same with CURSEG_XXX in f2fs.h */ +enum dirty_type { + DIRTY_HOT_DATA, /* dirty segments assigned as hot data logs */ + DIRTY_WARM_DATA, /* dirty segments assigned as warm data logs */ + DIRTY_COLD_DATA, /* dirty segments assigned as cold data logs */ + DIRTY_HOT_NODE, /* dirty segments assigned as hot node logs */ + DIRTY_WARM_NODE, /* dirty segments assigned as warm node logs */ + DIRTY_COLD_NODE, /* dirty segments assigned as cold node logs */ + DIRTY, /* to count # of dirty segments */ + PRE, /* to count # of entirely obsolete segments */ + NR_DIRTY_TYPE +}; + +struct dirty_seglist_info { + const struct victim_selection *v_ops; /* victim selction operation */ + unsigned long *dirty_segmap[NR_DIRTY_TYPE]; + struct mutex seglist_lock; /* lock for segment bitmaps */ + int nr_dirty[NR_DIRTY_TYPE]; /* # of dirty segments */ + unsigned long *victim_secmap; /* background GC victims */ +}; + +/* victim selection function for cleaning and SSR */ +struct victim_selection { + int (*get_victim)(struct f2fs_sb_info *, unsigned int *, + int, int, char); +}; + +/* for active log information */ +struct curseg_info { + struct mutex curseg_mutex; /* lock for consistency */ + struct f2fs_summary_block *sum_blk; /* cached summary block */ + unsigned char alloc_type; /* current allocation type */ + unsigned int segno; /* current segment number */ + unsigned short next_blkoff; /* next block offset to write */ + unsigned int zone; /* current zone number */ + unsigned int next_segno; /* preallocated segment */ +}; + +struct sit_entry_set { + struct list_head set_list; /* link with all sit sets */ + unsigned int start_segno; /* start segno of sits in set */ + unsigned int entry_cnt; /* the # of sit entries in set */ +}; + +/* + * inline functions + */ +static inline struct curseg_info *CURSEG_I(struct f2fs_sb_info *sbi, int type) +{ + return (struct curseg_info *)(SM_I(sbi)->curseg_array + type); +} + +static inline struct seg_entry *get_seg_entry(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + return &sit_i->sentries[segno]; +} + +static inline struct sec_entry *get_sec_entry(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + return &sit_i->sec_entries[GET_SECNO(sbi, segno)]; +} + +static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi, + unsigned int segno, int section) +{ + /* + * In order to get # of valid blocks in a section instantly from many + * segments, f2fs manages two counting structures separately. + */ + if (section > 1) + return get_sec_entry(sbi, segno)->valid_blocks; + else + return get_seg_entry(sbi, segno)->valid_blocks; +} + +static inline void seg_info_from_raw_sit(struct seg_entry *se, + struct f2fs_sit_entry *rs) +{ + se->valid_blocks = GET_SIT_VBLOCKS(rs); + se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs); + memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); + memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); + se->type = GET_SIT_TYPE(rs); + se->mtime = le64_to_cpu(rs->mtime); +} + +static inline void seg_info_to_raw_sit(struct seg_entry *se, + struct f2fs_sit_entry *rs) +{ + unsigned short raw_vblocks = (se->type << SIT_VBLOCKS_SHIFT) | + se->valid_blocks; + rs->vblocks = cpu_to_le16(raw_vblocks); + memcpy(rs->valid_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE); + memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); + se->ckpt_valid_blocks = se->valid_blocks; + rs->mtime = cpu_to_le64(se->mtime); +} + +static inline unsigned int find_next_inuse(struct free_segmap_info *free_i, + unsigned int max, unsigned int segno) +{ + unsigned int ret; + spin_lock(&free_i->segmap_lock); + ret = find_next_bit(free_i->free_segmap, max, segno); + spin_unlock(&free_i->segmap_lock); + return ret; +} + +static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = segno / sbi->segs_per_sec; + unsigned int start_segno = secno * sbi->segs_per_sec; + unsigned int next; + + spin_lock(&free_i->segmap_lock); + clear_bit(segno, free_i->free_segmap); + free_i->free_segments++; + + next = find_next_bit(free_i->free_segmap, + start_segno + sbi->segs_per_sec, start_segno); + if (next >= start_segno + sbi->segs_per_sec) { + clear_bit(secno, free_i->free_secmap); + free_i->free_sections++; + } + spin_unlock(&free_i->segmap_lock); +} + +static inline void __set_inuse(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = segno / sbi->segs_per_sec; + set_bit(segno, free_i->free_segmap); + free_i->free_segments--; + if (!test_and_set_bit(secno, free_i->free_secmap)) + free_i->free_sections--; +} + +static inline void __set_test_and_free(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = segno / sbi->segs_per_sec; + unsigned int start_segno = secno * sbi->segs_per_sec; + unsigned int next; + + spin_lock(&free_i->segmap_lock); + if (test_and_clear_bit(segno, free_i->free_segmap)) { + free_i->free_segments++; + + next = find_next_bit(free_i->free_segmap, + start_segno + sbi->segs_per_sec, start_segno); + if (next >= start_segno + sbi->segs_per_sec) { + if (test_and_clear_bit(secno, free_i->free_secmap)) + free_i->free_sections++; + } + } + spin_unlock(&free_i->segmap_lock); +} + +static inline void __set_test_and_inuse(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = segno / sbi->segs_per_sec; + spin_lock(&free_i->segmap_lock); + if (!test_and_set_bit(segno, free_i->free_segmap)) { + free_i->free_segments--; + if (!test_and_set_bit(secno, free_i->free_secmap)) + free_i->free_sections--; + } + spin_unlock(&free_i->segmap_lock); +} + +static inline void get_sit_bitmap(struct f2fs_sb_info *sbi, + void *dst_addr) +{ + struct sit_info *sit_i = SIT_I(sbi); + memcpy(dst_addr, sit_i->sit_bitmap, sit_i->bitmap_size); +} + +static inline block_t written_block_count(struct f2fs_sb_info *sbi) +{ + return SIT_I(sbi)->written_valid_blocks; +} + +static inline unsigned int free_segments(struct f2fs_sb_info *sbi) +{ + return FREE_I(sbi)->free_segments; +} + +static inline int reserved_segments(struct f2fs_sb_info *sbi) +{ + return SM_I(sbi)->reserved_segments; +} + +static inline unsigned int free_sections(struct f2fs_sb_info *sbi) +{ + return FREE_I(sbi)->free_sections; +} + +static inline unsigned int prefree_segments(struct f2fs_sb_info *sbi) +{ + return DIRTY_I(sbi)->nr_dirty[PRE]; +} + +static inline unsigned int dirty_segments(struct f2fs_sb_info *sbi) +{ + return DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_DATA] + + DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_DATA] + + DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_DATA] + + DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_NODE] + + DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_NODE] + + DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_NODE]; +} + +static inline int overprovision_segments(struct f2fs_sb_info *sbi) +{ + return SM_I(sbi)->ovp_segments; +} + +static inline int overprovision_sections(struct f2fs_sb_info *sbi) +{ + return ((unsigned int) overprovision_segments(sbi)) / sbi->segs_per_sec; +} + +static inline int reserved_sections(struct f2fs_sb_info *sbi) +{ + return ((unsigned int) reserved_segments(sbi)) / sbi->segs_per_sec; +} + +static inline bool need_SSR(struct f2fs_sb_info *sbi) +{ + int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); + int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); + return free_sections(sbi) <= (node_secs + 2 * dent_secs + + reserved_sections(sbi) + 1); +} + +static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, int freed) +{ + int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); + int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + return false; + + return (free_sections(sbi) + freed) <= (node_secs + 2 * dent_secs + + reserved_sections(sbi)); +} + +static inline bool excess_prefree_segs(struct f2fs_sb_info *sbi) +{ + return prefree_segments(sbi) > SM_I(sbi)->rec_prefree_segments; +} + +static inline int utilization(struct f2fs_sb_info *sbi) +{ + return div_u64((u64)valid_user_blocks(sbi) * 100, + sbi->user_block_count); +} + +/* + * Sometimes f2fs may be better to drop out-of-place update policy. + * And, users can control the policy through sysfs entries. + * There are five policies with triggering conditions as follows. + * F2FS_IPU_FORCE - all the time, + * F2FS_IPU_SSR - if SSR mode is activated, + * F2FS_IPU_UTIL - if FS utilization is over threashold, + * F2FS_IPU_SSR_UTIL - if SSR mode is activated and FS utilization is over + * threashold, + * F2FS_IPU_FSYNC - activated in fsync path only for high performance flash + * storages. IPU will be triggered only if the # of dirty + * pages over min_fsync_blocks. + * F2FS_IPUT_DISABLE - disable IPU. (=default option) + */ +#define DEF_MIN_IPU_UTIL 70 +#define DEF_MIN_FSYNC_BLOCKS 8 + +enum { + F2FS_IPU_FORCE, + F2FS_IPU_SSR, + F2FS_IPU_UTIL, + F2FS_IPU_SSR_UTIL, + F2FS_IPU_FSYNC, +}; + +static inline bool need_inplace_update(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int policy = SM_I(sbi)->ipu_policy; + + /* IPU can be done only for the user data */ + if (S_ISDIR(inode->i_mode) || f2fs_is_atomic_file(inode)) + return false; + + if (policy & (0x1 << F2FS_IPU_FORCE)) + return true; + if (policy & (0x1 << F2FS_IPU_SSR) && need_SSR(sbi)) + return true; + if (policy & (0x1 << F2FS_IPU_UTIL) && + utilization(sbi) > SM_I(sbi)->min_ipu_util) + return true; + if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && need_SSR(sbi) && + utilization(sbi) > SM_I(sbi)->min_ipu_util) + return true; + + /* this is only set during fdatasync */ + if (policy & (0x1 << F2FS_IPU_FSYNC) && + is_inode_flag_set(F2FS_I(inode), FI_NEED_IPU)) + return true; + + return false; +} + +static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi, + int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + return curseg->segno; +} + +static inline unsigned char curseg_alloc_type(struct f2fs_sb_info *sbi, + int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + return curseg->alloc_type; +} + +static inline unsigned short curseg_blkoff(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + return curseg->next_blkoff; +} + +static inline void check_seg_range(struct f2fs_sb_info *sbi, unsigned int segno) +{ + f2fs_bug_on(sbi, segno > TOTAL_SEGS(sbi) - 1); +} + +static inline void verify_block_addr(struct f2fs_sb_info *sbi, block_t blk_addr) +{ + f2fs_bug_on(sbi, blk_addr < SEG0_BLKADDR(sbi) + || blk_addr >= MAX_BLKADDR(sbi)); +} + +/* + * Summary block is always treated as an invalid block + */ +static inline void check_block_count(struct f2fs_sb_info *sbi, + int segno, struct f2fs_sit_entry *raw_sit) +{ +#ifdef CONFIG_F2FS_CHECK_FS + bool is_valid = test_bit_le(0, raw_sit->valid_map) ? true : false; + int valid_blocks = 0; + int cur_pos = 0, next_pos; + + /* check bitmap with valid block count */ + do { + if (is_valid) { + next_pos = find_next_zero_bit_le(&raw_sit->valid_map, + sbi->blocks_per_seg, + cur_pos); + valid_blocks += next_pos - cur_pos; + } else + next_pos = find_next_bit_le(&raw_sit->valid_map, + sbi->blocks_per_seg, + cur_pos); + cur_pos = next_pos; + is_valid = !is_valid; + } while (cur_pos < sbi->blocks_per_seg); + BUG_ON(GET_SIT_VBLOCKS(raw_sit) != valid_blocks); +#endif + /* check segment usage, and check boundary of a given segment number */ + f2fs_bug_on(sbi, GET_SIT_VBLOCKS(raw_sit) > sbi->blocks_per_seg + || segno > TOTAL_SEGS(sbi) - 1); +} + +static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi, + unsigned int start) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int offset = SIT_BLOCK_OFFSET(start); + block_t blk_addr = sit_i->sit_base_addr + offset; + + check_seg_range(sbi, start); + + /* calculate sit block address */ + if (f2fs_test_bit(offset, sit_i->sit_bitmap)) + blk_addr += sit_i->sit_blocks; + + return blk_addr; +} + +static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi, + pgoff_t block_addr) +{ + struct sit_info *sit_i = SIT_I(sbi); + block_addr -= sit_i->sit_base_addr; + if (block_addr < sit_i->sit_blocks) + block_addr += sit_i->sit_blocks; + else + block_addr -= sit_i->sit_blocks; + + return block_addr + sit_i->sit_base_addr; +} + +static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start) +{ + unsigned int block_off = SIT_BLOCK_OFFSET(start); + + f2fs_change_bit(block_off, sit_i->sit_bitmap); +} + +static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + return sit_i->elapsed_time + CURRENT_TIME_SEC.tv_sec - + sit_i->mounted_time; +} + +static inline void set_summary(struct f2fs_summary *sum, nid_t nid, + unsigned int ofs_in_node, unsigned char version) +{ + sum->nid = cpu_to_le32(nid); + sum->ofs_in_node = cpu_to_le16(ofs_in_node); + sum->version = version; +} + +static inline block_t start_sum_block(struct f2fs_sb_info *sbi) +{ + return __start_cp_addr(sbi) + + le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); +} + +static inline block_t sum_blk_addr(struct f2fs_sb_info *sbi, int base, int type) +{ + return __start_cp_addr(sbi) + + le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_total_block_count) + - (base + 1) + type; +} + +static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno) +{ + if (IS_CURSEC(sbi, secno) || (sbi->cur_victim_sec == secno)) + return true; + return false; +} + +static inline unsigned int max_hw_blocks(struct f2fs_sb_info *sbi) +{ + struct block_device *bdev = sbi->sb->s_bdev; + struct request_queue *q = bdev_get_queue(bdev); + return SECTOR_TO_BLOCK(queue_max_sectors(q)); +} + +/* + * It is very important to gather dirty pages and write at once, so that we can + * submit a big bio without interfering other data writes. + * By default, 512 pages for directory data, + * 512 pages (2MB) * 3 for three types of nodes, and + * max_bio_blocks for meta are set. + */ +static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) +{ + if (sbi->sb->s_bdi->dirty_exceeded) + return 0; + + if (type == DATA) + return sbi->blocks_per_seg; + else if (type == NODE) + return 3 * sbi->blocks_per_seg; + else if (type == META) + return MAX_BIO_BLOCKS(sbi); + else + return 0; +} + +/* + * When writing pages, it'd better align nr_to_write for segment size. + */ +static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type, + struct writeback_control *wbc) +{ + long nr_to_write, desired; + + if (wbc->sync_mode != WB_SYNC_NONE) + return 0; + + nr_to_write = wbc->nr_to_write; + + if (type == DATA) + desired = 4096; + else if (type == NODE) + desired = 3 * max_hw_blocks(sbi); + else + desired = MAX_BIO_BLOCKS(sbi); + + wbc->nr_to_write = desired; + return desired - nr_to_write; +} diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c new file mode 100644 index 00000000000..420b233d3de --- /dev/null +++ b/fs/f2fs/shrinker.c @@ -0,0 +1,139 @@ +/* + * f2fs shrinker support + * the basic infra was copied from fs/ubifs/shrinker.c + * + * Copyright (c) 2015 Motorola Mobility + * Copyright (c) 2015 Jaegeuk Kim + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include + +#include "f2fs.h" + +static LIST_HEAD(f2fs_list); +static DEFINE_SPINLOCK(f2fs_list_lock); +static unsigned int shrinker_run_no; + +static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi) +{ + return NM_I(sbi)->nat_cnt - NM_I(sbi)->dirty_nat_cnt; +} + +static unsigned long __count_free_nids(struct f2fs_sb_info *sbi) +{ + if (NM_I(sbi)->fcnt > NAT_ENTRY_PER_BLOCK) + return NM_I(sbi)->fcnt - NAT_ENTRY_PER_BLOCK; + return 0; +} + +static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi) +{ + return sbi->total_ext_tree + atomic_read(&sbi->total_ext_node); +} + +int f2fs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct f2fs_sb_info *sbi; + struct list_head *p; + unsigned long count = 0; + + spin_lock(&f2fs_list_lock); + p = f2fs_list.next; + while (p != &f2fs_list) { + sbi = list_entry(p, struct f2fs_sb_info, s_list); + + /* stop f2fs_put_super */ + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + spin_unlock(&f2fs_list_lock); + + /* count extent cache entries */ + count += __count_extent_cache(sbi); + + /* shrink clean nat cache entries */ + count += __count_nat_entries(sbi); + + /* count free nids cache entries */ + count += __count_free_nids(sbi); + + spin_lock(&f2fs_list_lock); + p = p->next; + mutex_unlock(&sbi->umount_mutex); + } + spin_unlock(&f2fs_list_lock); + return count; +} + +int f2fs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long nr = sc->nr_to_scan; + struct f2fs_sb_info *sbi; + struct list_head *p; + unsigned int run_no; + unsigned long freed = 0; + + spin_lock(&f2fs_list_lock); + do { + run_no = ++shrinker_run_no; + } while (run_no == 0); + p = f2fs_list.next; + while (p != &f2fs_list) { + sbi = list_entry(p, struct f2fs_sb_info, s_list); + + if (sbi->shrinker_run_no == run_no) + break; + + /* stop f2fs_put_super */ + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + spin_unlock(&f2fs_list_lock); + + sbi->shrinker_run_no = run_no; + + /* shrink extent cache entries */ + freed += f2fs_shrink_extent_tree(sbi, nr >> 1); + + /* shrink clean nat cache entries */ + if (freed < nr) + freed += try_to_free_nats(sbi, nr - freed); + + /* shrink free nids cache entries */ + if (freed < nr) + freed += try_to_free_nids(sbi, nr - freed); + + spin_lock(&f2fs_list_lock); + p = p->next; + list_move_tail(&sbi->s_list, &f2fs_list); + mutex_unlock(&sbi->umount_mutex); + if (freed >= nr) + break; + } + spin_unlock(&f2fs_list_lock); + return f2fs_shrink_count(NULL, NULL); +} + +void f2fs_join_shrinker(struct f2fs_sb_info *sbi) +{ + spin_lock(&f2fs_list_lock); + list_add_tail(&sbi->s_list, &f2fs_list); + spin_unlock(&f2fs_list_lock); +} + +void f2fs_leave_shrinker(struct f2fs_sb_info *sbi) +{ + f2fs_shrink_extent_tree(sbi, __count_extent_cache(sbi)); + + spin_lock(&f2fs_list_lock); + list_del(&sbi->s_list); + spin_unlock(&f2fs_list_lock); +} diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c new file mode 100644 index 00000000000..95ee32c0923 --- /dev/null +++ b/fs/f2fs/super.c @@ -0,0 +1,1515 @@ +/* + * fs/f2fs/super.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" +#include "gc.h" +#include "trace.h" + +#define CREATE_TRACE_POINTS +#include + +static struct proc_dir_entry *f2fs_proc_root; +static struct kmem_cache *f2fs_inode_cachep; +static struct kset *f2fs_kset; + +/* f2fs-wide shrinker description */ +static struct shrinker f2fs_shrinker_info = { + .shrink = f2fs_shrink_scan, + .seeks = DEFAULT_SEEKS, +}; + +enum { + Opt_gc_background, + Opt_disable_roll_forward, + Opt_norecovery, + Opt_discard, + Opt_noheap, + Opt_user_xattr, + Opt_nouser_xattr, + Opt_acl, + Opt_noacl, + Opt_active_logs, + Opt_disable_ext_identify, + Opt_inline_xattr, + Opt_inline_data, + Opt_inline_dentry, + Opt_flush_merge, + Opt_nobarrier, + Opt_fastboot, + Opt_extent_cache, + Opt_noextent_cache, + Opt_noinline_data, + Opt_err, +}; + +static match_table_t f2fs_tokens = { + {Opt_gc_background, "background_gc=%s"}, + {Opt_disable_roll_forward, "disable_roll_forward"}, + {Opt_norecovery, "norecovery"}, + {Opt_discard, "discard"}, + {Opt_noheap, "no_heap"}, + {Opt_user_xattr, "user_xattr"}, + {Opt_nouser_xattr, "nouser_xattr"}, + {Opt_acl, "acl"}, + {Opt_noacl, "noacl"}, + {Opt_active_logs, "active_logs=%u"}, + {Opt_disable_ext_identify, "disable_ext_identify"}, + {Opt_inline_xattr, "inline_xattr"}, + {Opt_inline_data, "inline_data"}, + {Opt_inline_dentry, "inline_dentry"}, + {Opt_flush_merge, "flush_merge"}, + {Opt_nobarrier, "nobarrier"}, + {Opt_fastboot, "fastboot"}, + {Opt_extent_cache, "extent_cache"}, + {Opt_noextent_cache, "noextent_cache"}, + {Opt_noinline_data, "noinline_data"}, + {Opt_err, NULL}, +}; + +/* Sysfs support for f2fs */ +enum { + GC_THREAD, /* struct f2fs_gc_thread */ + SM_INFO, /* struct f2fs_sm_info */ + NM_INFO, /* struct f2fs_nm_info */ + F2FS_SBI, /* struct f2fs_sb_info */ +}; + +struct f2fs_attr { + struct attribute attr; + ssize_t (*show)(struct f2fs_attr *, struct f2fs_sb_info *, char *); + ssize_t (*store)(struct f2fs_attr *, struct f2fs_sb_info *, + const char *, size_t); + int struct_type; + int offset; +}; + +static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) +{ + if (struct_type == GC_THREAD) + return (unsigned char *)sbi->gc_thread; + else if (struct_type == SM_INFO) + return (unsigned char *)SM_I(sbi); + else if (struct_type == NM_INFO) + return (unsigned char *)NM_I(sbi); + else if (struct_type == F2FS_SBI) + return (unsigned char *)sbi; + return NULL; +} + +static ssize_t f2fs_sbi_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + unsigned char *ptr = NULL; + unsigned int *ui; + + ptr = __struct_ptr(sbi, a->struct_type); + if (!ptr) + return -EINVAL; + + ui = (unsigned int *)(ptr + a->offset); + + return snprintf(buf, PAGE_SIZE, "%u\n", *ui); +} + +static ssize_t f2fs_sbi_store(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + const char *buf, size_t count) +{ + unsigned char *ptr; + unsigned long t; + unsigned int *ui; + ssize_t ret; + + ptr = __struct_ptr(sbi, a->struct_type); + if (!ptr) + return -EINVAL; + + ui = (unsigned int *)(ptr + a->offset); + + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret < 0) + return ret; + *ui = t; + return count; +} + +static ssize_t f2fs_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->show ? a->show(a, sbi, buf) : 0; +} + +static ssize_t f2fs_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->store ? a->store(a, sbi, buf, len) : 0; +} + +static void f2fs_sb_release(struct kobject *kobj) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_kobj); + complete(&sbi->s_kobj_unregister); +} + +#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset) \ +static struct f2fs_attr f2fs_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .show = _show, \ + .store = _store, \ + .struct_type = _struct_type, \ + .offset = _offset \ +} + +#define F2FS_RW_ATTR(struct_type, struct_name, name, elname) \ + F2FS_ATTR_OFFSET(struct_type, name, 0644, \ + f2fs_sbi_show, f2fs_sbi_store, \ + offsetof(struct struct_name, elname)) + +F2FS_RW_ATTR(GC_THREAD, f2fs_gc_kthread, gc_min_sleep_time, min_sleep_time); +F2FS_RW_ATTR(GC_THREAD, f2fs_gc_kthread, gc_max_sleep_time, max_sleep_time); +F2FS_RW_ATTR(GC_THREAD, f2fs_gc_kthread, gc_no_gc_sleep_time, no_gc_sleep_time); +F2FS_RW_ATTR(GC_THREAD, f2fs_gc_kthread, gc_idle, gc_idle); +F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, reclaim_segments, rec_prefree_segments); +F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, max_small_discards, max_discards); +F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, batched_trim_sections, trim_sections); +F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, ipu_policy, ipu_policy); +F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, min_ipu_util, min_ipu_util); +F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, min_fsync_blocks, min_fsync_blocks); +F2FS_RW_ATTR(NM_INFO, f2fs_nm_info, ram_thresh, ram_thresh); +F2FS_RW_ATTR(NM_INFO, f2fs_nm_info, ra_nid_pages, ra_nid_pages); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, max_victim_search, max_victim_search); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, dir_level, dir_level); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, cp_interval, cp_interval); + +#define ATTR_LIST(name) (&f2fs_attr_##name.attr) +static struct attribute *f2fs_attrs[] = { + ATTR_LIST(gc_min_sleep_time), + ATTR_LIST(gc_max_sleep_time), + ATTR_LIST(gc_no_gc_sleep_time), + ATTR_LIST(gc_idle), + ATTR_LIST(reclaim_segments), + ATTR_LIST(max_small_discards), + ATTR_LIST(batched_trim_sections), + ATTR_LIST(ipu_policy), + ATTR_LIST(min_ipu_util), + ATTR_LIST(min_fsync_blocks), + ATTR_LIST(max_victim_search), + ATTR_LIST(dir_level), + ATTR_LIST(ram_thresh), + ATTR_LIST(ra_nid_pages), + ATTR_LIST(cp_interval), + NULL, +}; + +static const struct sysfs_ops f2fs_attr_ops = { + .show = f2fs_attr_show, + .store = f2fs_attr_store, +}; + +static struct kobj_type f2fs_ktype = { + .default_attrs = f2fs_attrs, + .sysfs_ops = &f2fs_attr_ops, + .release = f2fs_sb_release, +}; + +void f2fs_msg(struct super_block *sb, const char *level, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk("%sF2FS-fs (%s): %pV\n", level, sb->s_id, &vaf); + va_end(args); +} + +static void init_once(void *foo) +{ + struct f2fs_inode_info *fi = (struct f2fs_inode_info *) foo; + + inode_init_once(&fi->vfs_inode); +} + +static int parse_options(struct super_block *sb, char *options) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct request_queue *q; + substring_t args[MAX_OPT_ARGS]; + char *p, *name; + int arg = 0; + + if (!options) + return 0; + + while ((p = strsep(&options, ",")) != NULL) { + int token; + if (!*p) + continue; + /* + * Initialize args struct so we know whether arg was + * found; some options take optional arguments. + */ + args[0].to = args[0].from = NULL; + token = match_token(p, f2fs_tokens, args); + + switch (token) { + case Opt_gc_background: + name = match_strdup(&args[0]); + + if (!name) + return -ENOMEM; + if (strlen(name) == 2 && !strncmp(name, "on", 2)) { + set_opt(sbi, BG_GC); + clear_opt(sbi, FORCE_FG_GC); + } else if (strlen(name) == 3 && !strncmp(name, "off", 3)) { + clear_opt(sbi, BG_GC); + clear_opt(sbi, FORCE_FG_GC); + } else if (strlen(name) == 4 && !strncmp(name, "sync", 4)) { + set_opt(sbi, BG_GC); + set_opt(sbi, FORCE_FG_GC); + } else { + kfree(name); + return -EINVAL; + } + kfree(name); + break; + case Opt_disable_roll_forward: + set_opt(sbi, DISABLE_ROLL_FORWARD); + break; + case Opt_norecovery: + /* this option mounts f2fs with ro */ + set_opt(sbi, DISABLE_ROLL_FORWARD); + if (!f2fs_readonly(sb)) + return -EINVAL; + break; + case Opt_discard: + q = bdev_get_queue(sb->s_bdev); + if (blk_queue_discard(q)) { + set_opt(sbi, DISCARD); + } else { + f2fs_msg(sb, KERN_WARNING, + "mounting with \"discard\" option, but " + "the device does not support discard"); + } + break; + case Opt_noheap: + set_opt(sbi, NOHEAP); + break; +#ifdef CONFIG_F2FS_FS_XATTR + case Opt_user_xattr: + set_opt(sbi, XATTR_USER); + break; + case Opt_nouser_xattr: + clear_opt(sbi, XATTR_USER); + break; + case Opt_inline_xattr: + set_opt(sbi, INLINE_XATTR); + break; +#else + case Opt_user_xattr: + f2fs_msg(sb, KERN_INFO, + "user_xattr options not supported"); + break; + case Opt_nouser_xattr: + f2fs_msg(sb, KERN_INFO, + "nouser_xattr options not supported"); + break; + case Opt_inline_xattr: + f2fs_msg(sb, KERN_INFO, + "inline_xattr options not supported"); + break; +#endif +#ifdef CONFIG_F2FS_FS_POSIX_ACL + case Opt_acl: + set_opt(sbi, POSIX_ACL); + break; + case Opt_noacl: + clear_opt(sbi, POSIX_ACL); + break; +#else + case Opt_acl: + f2fs_msg(sb, KERN_INFO, "acl options not supported"); + break; + case Opt_noacl: + f2fs_msg(sb, KERN_INFO, "noacl options not supported"); + break; +#endif + case Opt_active_logs: + if (args->from && match_int(args, &arg)) + return -EINVAL; + if (arg != 2 && arg != 4 && arg != NR_CURSEG_TYPE) + return -EINVAL; + sbi->active_logs = arg; + break; + case Opt_disable_ext_identify: + set_opt(sbi, DISABLE_EXT_IDENTIFY); + break; + case Opt_inline_data: + set_opt(sbi, INLINE_DATA); + break; + case Opt_inline_dentry: + set_opt(sbi, INLINE_DENTRY); + break; + case Opt_flush_merge: + set_opt(sbi, FLUSH_MERGE); + break; + case Opt_nobarrier: + set_opt(sbi, NOBARRIER); + break; + case Opt_fastboot: + set_opt(sbi, FASTBOOT); + break; + case Opt_extent_cache: + set_opt(sbi, EXTENT_CACHE); + break; + case Opt_noextent_cache: + clear_opt(sbi, EXTENT_CACHE); + break; + case Opt_noinline_data: + clear_opt(sbi, INLINE_DATA); + break; + default: + f2fs_msg(sb, KERN_ERR, + "Unrecognized mount option \"%s\" or missing value", + p); + return -EINVAL; + } + } + return 0; +} + +static struct inode *f2fs_alloc_inode(struct super_block *sb) +{ + struct f2fs_inode_info *fi; + + fi = kmem_cache_alloc(f2fs_inode_cachep, GFP_F2FS_ZERO); + if (!fi) + return NULL; + + init_once((void *) fi); + + /* Initialize f2fs-specific inode info */ + fi->vfs_inode.i_version = 1; + atomic_set(&fi->dirty_pages, 0); + fi->i_current_depth = 1; + fi->i_advise = 0; + init_rwsem(&fi->i_sem); + INIT_LIST_HEAD(&fi->inmem_pages); + mutex_init(&fi->inmem_lock); + + set_inode_flag(fi, FI_NEW_INODE); + + if (test_opt(F2FS_SB(sb), INLINE_XATTR)) + set_inode_flag(fi, FI_INLINE_XATTR); + + /* Will be used by directory only */ + fi->i_dir_level = F2FS_SB(sb)->dir_level; + +#ifdef CONFIG_F2FS_FS_ENCRYPTION + fi->i_crypt_info = NULL; +#endif + return &fi->vfs_inode; +} + +static int f2fs_drop_inode(struct inode *inode) +{ + /* + * This is to avoid a deadlock condition like below. + * writeback_single_inode(inode) + * - f2fs_write_data_page + * - f2fs_gc -> iput -> evict + * - inode_wait_for_writeback(inode) + */ + if (!inode_unhashed(inode) && inode->i_state & I_SYNC) { + if (!inode->i_nlink && !is_bad_inode(inode)) { + /* to avoid evict_inode call simultaneously */ + atomic_inc(&inode->i_count); + spin_unlock(&inode->i_lock); + + /* some remained atomic pages should discarded */ + if (f2fs_is_atomic_file(inode)) + commit_inmem_pages(inode, true); + + i_size_write(inode, 0); + + if (F2FS_HAS_BLOCKS(inode)) + f2fs_truncate(inode, true); + +#ifdef CONFIG_F2FS_FS_ENCRYPTION + if (F2FS_I(inode)->i_crypt_info) + f2fs_free_encryption_info(inode, + F2FS_I(inode)->i_crypt_info); +#endif + spin_lock(&inode->i_lock); + atomic_dec(&inode->i_count); + } + return 0; + } + return generic_drop_inode(inode); +} + +/* + * f2fs_dirty_inode() is called from __mark_inode_dirty() + * + * We should call set_dirty_inode to write the dirty inode through write_inode. + */ +static void f2fs_dirty_inode(struct inode *inode, int flags) +{ + set_inode_flag(F2FS_I(inode), FI_DIRTY_INODE); +} + +static void f2fs_i_callback(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + kmem_cache_free(f2fs_inode_cachep, F2FS_I(inode)); +} + +static void f2fs_destroy_inode(struct inode *inode) +{ + call_rcu(&inode->i_rcu, f2fs_i_callback); +} + +static void f2fs_put_super(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (sbi->s_proc) { + remove_proc_entry("segment_info", sbi->s_proc); + remove_proc_entry(sb->s_id, f2fs_proc_root); + } + kobject_del(&sbi->s_kobj); + + stop_gc_thread(sbi); + + /* prevent remaining shrinker jobs */ + mutex_lock(&sbi->umount_mutex); + + /* + * We don't need to do checkpoint when superblock is clean. + * But, the previous checkpoint was not done by umount, it needs to do + * clean checkpoint again. + */ + if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) || + !is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG)) { + struct cp_control cpc = { + .reason = CP_UMOUNT, + }; + write_checkpoint(sbi, &cpc); + } + + /* write_checkpoint can update stat informaion */ + f2fs_destroy_stats(sbi); + + /* + * normally superblock is clean, so we need to release this. + * In addition, EIO will skip do checkpoint, we need this as well. + */ + release_dirty_inode(sbi); + release_discard_addrs(sbi); + + f2fs_leave_shrinker(sbi); + mutex_unlock(&sbi->umount_mutex); + + iput(sbi->node_inode); + iput(sbi->meta_inode); + + /* destroy f2fs internal modules */ + destroy_node_manager(sbi); + destroy_segment_manager(sbi); + + kfree(sbi->ckpt); + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + + sb->s_fs_info = NULL; + brelse(sbi->raw_super_buf); + kfree(sbi); +} + +int f2fs_sync_fs(struct super_block *sb, int sync) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + trace_f2fs_sync_fs(sb, sync); + + if (sync) { + struct cp_control cpc; + + cpc.reason = __get_cp_reason(sbi); + + mutex_lock(&sbi->gc_mutex); + write_checkpoint(sbi, &cpc); + mutex_unlock(&sbi->gc_mutex); + } else { + f2fs_balance_fs(sbi); + } + f2fs_trace_ios(NULL, 1); + + return 0; +} + +static int f2fs_freeze(struct super_block *sb) +{ + int err; + + if (f2fs_readonly(sb)) + return 0; + + err = f2fs_sync_fs(sb, 1); + return err; +} + +static int f2fs_unfreeze(struct super_block *sb) +{ + return 0; +} + +static int f2fs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + block_t total_count, user_block_count, start_count, ovp_count; + + total_count = le64_to_cpu(sbi->raw_super->block_count); + user_block_count = sbi->user_block_count; + start_count = le32_to_cpu(sbi->raw_super->segment0_blkaddr); + ovp_count = SM_I(sbi)->ovp_segments << sbi->log_blocks_per_seg; + buf->f_type = F2FS_SUPER_MAGIC; + buf->f_bsize = sbi->blocksize; + + buf->f_blocks = total_count - start_count; + buf->f_bfree = buf->f_blocks - valid_user_blocks(sbi) - ovp_count; + buf->f_bavail = user_block_count - valid_user_blocks(sbi); + + buf->f_files = sbi->total_node_count - F2FS_RESERVED_NODE_NUM; + buf->f_ffree = buf->f_files - valid_inode_count(sbi); + + buf->f_namelen = F2FS_NAME_LEN; + buf->f_fsid.val[0] = (u32)id; + buf->f_fsid.val[1] = (u32)(id >> 32); + + return 0; +} + +static int f2fs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct f2fs_sb_info *sbi = F2FS_SB(root->d_sb); + + if (!f2fs_readonly(sbi->sb) && test_opt(sbi, BG_GC)) { + if (test_opt(sbi, FORCE_FG_GC)) + seq_printf(seq, ",background_gc=%s", "sync"); + else + seq_printf(seq, ",background_gc=%s", "on"); + } else { + seq_printf(seq, ",background_gc=%s", "off"); + } + if (test_opt(sbi, DISABLE_ROLL_FORWARD)) + seq_puts(seq, ",disable_roll_forward"); + if (test_opt(sbi, DISCARD)) + seq_puts(seq, ",discard"); + if (test_opt(sbi, NOHEAP)) + seq_puts(seq, ",no_heap_alloc"); +#ifdef CONFIG_F2FS_FS_XATTR + if (test_opt(sbi, XATTR_USER)) + seq_puts(seq, ",user_xattr"); + else + seq_puts(seq, ",nouser_xattr"); + if (test_opt(sbi, INLINE_XATTR)) + seq_puts(seq, ",inline_xattr"); +#endif +#ifdef CONFIG_F2FS_FS_POSIX_ACL + if (test_opt(sbi, POSIX_ACL)) + seq_puts(seq, ",acl"); + else + seq_puts(seq, ",noacl"); +#endif + if (test_opt(sbi, DISABLE_EXT_IDENTIFY)) + seq_puts(seq, ",disable_ext_identify"); + if (test_opt(sbi, INLINE_DATA)) + seq_puts(seq, ",inline_data"); + else + seq_puts(seq, ",noinline_data"); + if (test_opt(sbi, INLINE_DENTRY)) + seq_puts(seq, ",inline_dentry"); + if (!f2fs_readonly(sbi->sb) && test_opt(sbi, FLUSH_MERGE)) + seq_puts(seq, ",flush_merge"); + if (test_opt(sbi, NOBARRIER)) + seq_puts(seq, ",nobarrier"); + if (test_opt(sbi, FASTBOOT)) + seq_puts(seq, ",fastboot"); + if (test_opt(sbi, EXTENT_CACHE)) + seq_puts(seq, ",extent_cache"); + else + seq_puts(seq, ",noextent_cache"); + seq_printf(seq, ",active_logs=%u", sbi->active_logs); + + return 0; +} + +static int segment_info_seq_show(struct seq_file *seq, void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + unsigned int total_segs = + le32_to_cpu(sbi->raw_super->segment_count_main); + int i; + + seq_puts(seq, "format: segment_type|valid_blocks\n" + "segment_type(0:HD, 1:WD, 2:CD, 3:HN, 4:WN, 5:CN)\n"); + + for (i = 0; i < total_segs; i++) { + struct seg_entry *se = get_seg_entry(sbi, i); + + if ((i % 10) == 0) + seq_printf(seq, "%-10d", i); + seq_printf(seq, "%d|%-3u", se->type, + get_valid_blocks(sbi, i, 1)); + if ((i % 10) == 9 || i == (total_segs - 1)) + seq_putc(seq, '\n'); + else + seq_putc(seq, ' '); + } + + return 0; +} + +static int segment_info_open_fs(struct inode *inode, struct file *file) +{ + return single_open(file, segment_info_seq_show, PDE(inode)->data); +} + +static const struct file_operations f2fs_seq_segment_info_fops = { + .owner = THIS_MODULE, + .open = segment_info_open_fs, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static void default_options(struct f2fs_sb_info *sbi) +{ + /* init some FS parameters */ + sbi->active_logs = NR_CURSEG_TYPE; + + set_opt(sbi, BG_GC); + set_opt(sbi, INLINE_DATA); + set_opt(sbi, EXTENT_CACHE); + +#ifdef CONFIG_F2FS_FS_XATTR + set_opt(sbi, XATTR_USER); +#endif +#ifdef CONFIG_F2FS_FS_POSIX_ACL + set_opt(sbi, POSIX_ACL); +#endif +} + +static int f2fs_remount(struct super_block *sb, int *flags, char *data) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct f2fs_mount_info org_mount_opt; + int err, active_logs; + bool need_restart_gc = false; + bool need_stop_gc = false; + bool no_extent_cache = !test_opt(sbi, EXTENT_CACHE); + + sync_filesystem(sb); + + /* + * Save the old mount options in case we + * need to restore them. + */ + org_mount_opt = sbi->mount_opt; + active_logs = sbi->active_logs; + + sbi->mount_opt.opt = 0; + default_options(sbi); + + /* parse mount options */ + err = parse_options(sb, data); + if (err) + goto restore_opts; + + /* + * Previous and new state of filesystem is RO, + * so skip checking GC and FLUSH_MERGE conditions. + */ + if (f2fs_readonly(sb) && (*flags & MS_RDONLY)) + goto skip; + + /* disallow enable/disable extent_cache dynamically */ + if (no_extent_cache == !!test_opt(sbi, EXTENT_CACHE)) { + err = -EINVAL; + f2fs_msg(sbi->sb, KERN_WARNING, + "switch extent_cache option is not allowed"); + goto restore_opts; + } + + /* + * We stop the GC thread if FS is mounted as RO + * or if background_gc = off is passed in mount + * option. Also sync the filesystem. + */ + if ((*flags & MS_RDONLY) || !test_opt(sbi, BG_GC)) { + if (sbi->gc_thread) { + stop_gc_thread(sbi); + f2fs_sync_fs(sb, 1); + need_restart_gc = true; + } + } else if (!sbi->gc_thread) { + err = start_gc_thread(sbi); + if (err) + goto restore_opts; + need_stop_gc = true; + } + + /* + * We stop issue flush thread if FS is mounted as RO + * or if flush_merge is not passed in mount option. + */ + if ((*flags & MS_RDONLY) || !test_opt(sbi, FLUSH_MERGE)) { + destroy_flush_cmd_control(sbi); + } else if (!SM_I(sbi)->cmd_control_info) { + err = create_flush_cmd_control(sbi); + if (err) + goto restore_gc; + } +skip: + /* Update the POSIXACL Flag */ + sb->s_flags = (sb->s_flags & ~MS_POSIXACL) | + (test_opt(sbi, POSIX_ACL) ? MS_POSIXACL : 0); + return 0; +restore_gc: + if (need_restart_gc) { + if (start_gc_thread(sbi)) + f2fs_msg(sbi->sb, KERN_WARNING, + "background gc thread has stopped"); + } else if (need_stop_gc) { + stop_gc_thread(sbi); + } +restore_opts: + sbi->mount_opt = org_mount_opt; + sbi->active_logs = active_logs; + return err; +} + +static struct super_operations f2fs_sops = { + .alloc_inode = f2fs_alloc_inode, + .drop_inode = f2fs_drop_inode, + .destroy_inode = f2fs_destroy_inode, + .write_inode = f2fs_write_inode, + .dirty_inode = f2fs_dirty_inode, + .show_options = f2fs_show_options, + .evict_inode = f2fs_evict_inode, + .put_super = f2fs_put_super, + .sync_fs = f2fs_sync_fs, + .freeze_fs = f2fs_freeze, + .unfreeze_fs = f2fs_unfreeze, + .statfs = f2fs_statfs, + .remount_fs = f2fs_remount, +}; + +static struct inode *f2fs_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct inode *inode; + + if (check_nid_range(sbi, ino)) + return ERR_PTR(-ESTALE); + + /* + * f2fs_iget isn't quite right if the inode is currently unallocated! + * However f2fs_iget currently does appropriate checks to handle stale + * inodes so everything is OK. + */ + inode = f2fs_iget(sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (unlikely(generation && inode->i_generation != generation)) { + /* we didn't find the right inode.. */ + iput(inode); + return ERR_PTR(-ESTALE); + } + return inode; +} + +static struct dentry *f2fs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + f2fs_nfs_get_inode); +} + +static struct dentry *f2fs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + f2fs_nfs_get_inode); +} + +static const struct export_operations f2fs_export_ops = { + .fh_to_dentry = f2fs_fh_to_dentry, + .fh_to_parent = f2fs_fh_to_parent, + .get_parent = f2fs_get_parent, +}; + +static loff_t max_file_size(unsigned bits) +{ + loff_t result = (DEF_ADDRS_PER_INODE - F2FS_INLINE_XATTR_ADDRS); + loff_t leaf_count = ADDRS_PER_BLOCK; + + /* two direct node blocks */ + result += (leaf_count * 2); + + /* two indirect node blocks */ + leaf_count *= NIDS_PER_BLOCK; + result += (leaf_count * 2); + + /* one double indirect node block */ + leaf_count *= NIDS_PER_BLOCK; + result += leaf_count; + + result <<= bits; + return result; +} + +static int sanity_check_raw_super(struct super_block *sb, + struct f2fs_super_block *raw_super) +{ + unsigned int blocksize; + + if (F2FS_SUPER_MAGIC != le32_to_cpu(raw_super->magic)) { + f2fs_msg(sb, KERN_INFO, + "Magic Mismatch, valid(0x%x) - read(0x%x)", + F2FS_SUPER_MAGIC, le32_to_cpu(raw_super->magic)); + return 1; + } + + /* Currently, support only 4KB page cache size */ + if (F2FS_BLKSIZE != PAGE_CACHE_SIZE) { + f2fs_msg(sb, KERN_INFO, + "Invalid page_cache_size (%lu), supports only 4KB\n", + PAGE_CACHE_SIZE); + return 1; + } + + /* Currently, support only 4KB block size */ + blocksize = 1 << le32_to_cpu(raw_super->log_blocksize); + if (blocksize != F2FS_BLKSIZE) { + f2fs_msg(sb, KERN_INFO, + "Invalid blocksize (%u), supports only 4KB\n", + blocksize); + return 1; + } + + /* Currently, support 512/1024/2048/4096 bytes sector size */ + if (le32_to_cpu(raw_super->log_sectorsize) > + F2FS_MAX_LOG_SECTOR_SIZE || + le32_to_cpu(raw_super->log_sectorsize) < + F2FS_MIN_LOG_SECTOR_SIZE) { + f2fs_msg(sb, KERN_INFO, "Invalid log sectorsize (%u)", + le32_to_cpu(raw_super->log_sectorsize)); + return 1; + } + if (le32_to_cpu(raw_super->log_sectors_per_block) + + le32_to_cpu(raw_super->log_sectorsize) != + F2FS_MAX_LOG_SECTOR_SIZE) { + f2fs_msg(sb, KERN_INFO, + "Invalid log sectors per block(%u) log sectorsize(%u)", + le32_to_cpu(raw_super->log_sectors_per_block), + le32_to_cpu(raw_super->log_sectorsize)); + return 1; + } + return 0; +} + +static int sanity_check_ckpt(struct f2fs_sb_info *sbi) +{ + unsigned int total, fsmeta; + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + + total = le32_to_cpu(raw_super->segment_count); + fsmeta = le32_to_cpu(raw_super->segment_count_ckpt); + fsmeta += le32_to_cpu(raw_super->segment_count_sit); + fsmeta += le32_to_cpu(raw_super->segment_count_nat); + fsmeta += le32_to_cpu(ckpt->rsvd_segment_count); + fsmeta += le32_to_cpu(raw_super->segment_count_ssa); + + if (unlikely(fsmeta >= total)) + return 1; + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_msg(sbi->sb, KERN_ERR, "A bug case: need to run fsck"); + return 1; + } + return 0; +} + +static void init_sb_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = sbi->raw_super; + int i; + + sbi->log_sectors_per_block = + le32_to_cpu(raw_super->log_sectors_per_block); + sbi->log_blocksize = le32_to_cpu(raw_super->log_blocksize); + sbi->blocksize = 1 << sbi->log_blocksize; + sbi->log_blocks_per_seg = le32_to_cpu(raw_super->log_blocks_per_seg); + sbi->blocks_per_seg = 1 << sbi->log_blocks_per_seg; + sbi->segs_per_sec = le32_to_cpu(raw_super->segs_per_sec); + sbi->secs_per_zone = le32_to_cpu(raw_super->secs_per_zone); + sbi->total_sections = le32_to_cpu(raw_super->section_count); + sbi->total_node_count = + (le32_to_cpu(raw_super->segment_count_nat) / 2) + * sbi->blocks_per_seg * NAT_ENTRY_PER_BLOCK; + sbi->root_ino_num = le32_to_cpu(raw_super->root_ino); + sbi->node_ino_num = le32_to_cpu(raw_super->node_ino); + sbi->meta_ino_num = le32_to_cpu(raw_super->meta_ino); + sbi->cur_victim_sec = NULL_SECNO; + sbi->max_victim_search = DEF_MAX_VICTIM_SEARCH; + + for (i = 0; i < NR_COUNT_TYPE; i++) + atomic_set(&sbi->nr_pages[i], 0); + + sbi->dir_level = DEF_DIR_LEVEL; + sbi->cp_interval = DEF_CP_INTERVAL; + clear_sbi_flag(sbi, SBI_NEED_FSCK); + + INIT_LIST_HEAD(&sbi->s_list); + mutex_init(&sbi->umount_mutex); +} + +/* + * Read f2fs raw super block. + * Because we have two copies of super block, so read the first one at first, + * if the first one is invalid, move to read the second one. + */ +static int read_raw_super_block(struct super_block *sb, + struct f2fs_super_block **raw_super, + struct buffer_head **raw_super_buf, + int *recovery) +{ + int block = 0; + struct buffer_head *buffer; + struct f2fs_super_block *super; + int err = 0; + +retry: + buffer = sb_bread(sb, block); + if (!buffer) { + *recovery = 1; + f2fs_msg(sb, KERN_ERR, "Unable to read %dth superblock", + block + 1); + if (block == 0) { + block++; + goto retry; + } else { + err = -EIO; + goto out; + } + } + + super = (struct f2fs_super_block *) + ((char *)(buffer)->b_data + F2FS_SUPER_OFFSET); + + /* sanity checking of raw super */ + if (sanity_check_raw_super(sb, super)) { + brelse(buffer); + *recovery = 1; + f2fs_msg(sb, KERN_ERR, + "Can't find valid F2FS filesystem in %dth superblock", + block + 1); + if (block == 0) { + block++; + goto retry; + } else { + err = -EINVAL; + goto out; + } + } + + if (!*raw_super) { + *raw_super_buf = buffer; + *raw_super = super; + } else { + /* already have a valid superblock */ + brelse(buffer); + } + + /* check the validity of the second superblock */ + if (block == 0) { + block++; + goto retry; + } + +out: + /* No valid superblock */ + if (!*raw_super) + return err; + + return 0; +} + +int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover) +{ + struct buffer_head *sbh = sbi->raw_super_buf; + sector_t block = sbh->b_blocknr; + int err; + + /* write back-up superblock first */ + sbh->b_blocknr = block ? 0 : 1; + mark_buffer_dirty(sbh); + err = sync_dirty_buffer(sbh); + + sbh->b_blocknr = block; + + /* if we are in recovery path, skip writing valid superblock */ + if (recover || err) + goto out; + + /* write current valid superblock */ + mark_buffer_dirty(sbh); + err = sync_dirty_buffer(sbh); +out: + clear_buffer_write_io_error(sbh); + set_buffer_uptodate(sbh); + return err; +} + +static int f2fs_fill_super(struct super_block *sb, void *data, int silent) +{ + struct f2fs_sb_info *sbi; + struct f2fs_super_block *raw_super; + struct buffer_head *raw_super_buf; + struct inode *root; + long err; + bool retry = true, need_fsck = false; + char *options = NULL; + int recovery, i; + +try_onemore: + err = -EINVAL; + raw_super = NULL; + raw_super_buf = NULL; + recovery = 0; + + /* allocate memory for f2fs-specific super block info */ + sbi = kzalloc(sizeof(struct f2fs_sb_info), GFP_KERNEL); + if (!sbi) + return -ENOMEM; + + /* set a block size */ + if (unlikely(!sb_set_blocksize(sb, F2FS_BLKSIZE))) { + f2fs_msg(sb, KERN_ERR, "unable to set blocksize"); + goto free_sbi; + } + + err = read_raw_super_block(sb, &raw_super, &raw_super_buf, &recovery); + if (err) + goto free_sbi; + + sb->s_fs_info = sbi; + default_options(sbi); + /* parse mount options */ + options = kstrdup((const char *)data, GFP_KERNEL); + if (data && !options) { + err = -ENOMEM; + goto free_sb_buf; + } + + err = parse_options(sb, options); + if (err) + goto free_options; + + sb->s_maxbytes = max_file_size(le32_to_cpu(raw_super->log_blocksize)); + sb->s_max_links = F2FS_LINK_MAX; + get_random_bytes(&sbi->s_next_generation, sizeof(u32)); + + sb->s_op = &f2fs_sops; + sb->s_xattr = f2fs_xattr_handlers; + sb->s_export_op = &f2fs_export_ops; + sb->s_magic = F2FS_SUPER_MAGIC; + sb->s_time_gran = 1; + sb->s_flags = (sb->s_flags & ~MS_POSIXACL) | + (test_opt(sbi, POSIX_ACL) ? MS_POSIXACL : 0); + memcpy(sb->s_uuid, raw_super->uuid, sizeof(raw_super->uuid)); + + /* init f2fs-specific super block info */ + sbi->sb = sb; + sbi->raw_super = raw_super; + sbi->raw_super_buf = raw_super_buf; + mutex_init(&sbi->gc_mutex); + mutex_init(&sbi->writepages); + mutex_init(&sbi->cp_mutex); + init_rwsem(&sbi->node_write); + + /* disallow all the data/node/meta page writes */ + set_sbi_flag(sbi, SBI_POR_DOING); + spin_lock_init(&sbi->stat_lock); + + init_rwsem(&sbi->read_io.io_rwsem); + sbi->read_io.sbi = sbi; + sbi->read_io.bio = NULL; + for (i = 0; i < NR_PAGE_TYPE; i++) { + init_rwsem(&sbi->write_io[i].io_rwsem); + sbi->write_io[i].sbi = sbi; + sbi->write_io[i].bio = NULL; + } + + init_rwsem(&sbi->cp_rwsem); + init_waitqueue_head(&sbi->cp_wait); + init_sb_info(sbi); + + /* get an inode for meta space */ + sbi->meta_inode = f2fs_iget(sb, F2FS_META_INO(sbi)); + if (IS_ERR(sbi->meta_inode)) { + f2fs_msg(sb, KERN_ERR, "Failed to read F2FS meta data inode"); + err = PTR_ERR(sbi->meta_inode); + goto free_options; + } + + err = get_valid_checkpoint(sbi); + if (err) { + f2fs_msg(sb, KERN_ERR, "Failed to get valid F2FS checkpoint"); + goto free_meta_inode; + } + + /* sanity checking of checkpoint */ + err = -EINVAL; + if (sanity_check_ckpt(sbi)) { + f2fs_msg(sb, KERN_ERR, "Invalid F2FS checkpoint"); + goto free_cp; + } + + sbi->total_valid_node_count = + le32_to_cpu(sbi->ckpt->valid_node_count); + sbi->total_valid_inode_count = + le32_to_cpu(sbi->ckpt->valid_inode_count); + sbi->user_block_count = le64_to_cpu(sbi->ckpt->user_block_count); + sbi->total_valid_block_count = + le64_to_cpu(sbi->ckpt->valid_block_count); + sbi->last_valid_block_count = sbi->total_valid_block_count; + sbi->alloc_valid_block_count = 0; + INIT_LIST_HEAD(&sbi->dir_inode_list); + spin_lock_init(&sbi->dir_inode_lock); + + init_extent_cache_info(sbi); + + init_ino_entry_info(sbi); + + /* setup f2fs internal modules */ + err = build_segment_manager(sbi); + if (err) { + f2fs_msg(sb, KERN_ERR, + "Failed to initialize F2FS segment manager"); + goto free_sm; + } + err = build_node_manager(sbi); + if (err) { + f2fs_msg(sb, KERN_ERR, + "Failed to initialize F2FS node manager"); + goto free_nm; + } + + build_gc_manager(sbi); + + /* get an inode for node space */ + sbi->node_inode = f2fs_iget(sb, F2FS_NODE_INO(sbi)); + if (IS_ERR(sbi->node_inode)) { + f2fs_msg(sb, KERN_ERR, "Failed to read node inode"); + err = PTR_ERR(sbi->node_inode); + goto free_nm; + } + + f2fs_join_shrinker(sbi); + + /* if there are nt orphan nodes free them */ + err = recover_orphan_inodes(sbi); + if (err) + goto free_node_inode; + + /* read root inode and dentry */ + root = f2fs_iget(sb, F2FS_ROOT_INO(sbi)); + if (IS_ERR(root)) { + f2fs_msg(sb, KERN_ERR, "Failed to read root inode"); + err = PTR_ERR(root); + goto free_node_inode; + } + if (!S_ISDIR(root->i_mode) || !root->i_blocks || !root->i_size) { + iput(root); + err = -EINVAL; + goto free_node_inode; + } + + sb->s_root = d_make_root(root); /* allocate root dentry */ + if (!sb->s_root) { + err = -ENOMEM; + goto free_root_inode; + } + + err = f2fs_build_stats(sbi); + if (err) + goto free_root_inode; + + if (f2fs_proc_root) + sbi->s_proc = proc_mkdir(sb->s_id, f2fs_proc_root); + + if (sbi->s_proc) + proc_create_data("segment_info", S_IRUGO, sbi->s_proc, + &f2fs_seq_segment_info_fops, sb); + + sbi->s_kobj.kset = f2fs_kset; + init_completion(&sbi->s_kobj_unregister); + err = kobject_init_and_add(&sbi->s_kobj, &f2fs_ktype, NULL, + "%s", sb->s_id); + if (err) + goto free_proc; + + /* recover fsynced data */ + if (!test_opt(sbi, DISABLE_ROLL_FORWARD)) { + /* + * mount should be failed, when device has readonly mode, and + * previous checkpoint was not done by clean system shutdown. + */ + if (bdev_read_only(sb->s_bdev) && + !is_set_ckpt_flags(sbi->ckpt, CP_UMOUNT_FLAG)) { + err = -EROFS; + goto free_kobj; + } + + if (need_fsck) + set_sbi_flag(sbi, SBI_NEED_FSCK); + + err = recover_fsync_data(sbi); + if (err) { + need_fsck = true; + f2fs_msg(sb, KERN_ERR, + "Cannot recover all fsync data errno=%ld", err); + goto free_kobj; + } + } + /* recover_fsync_data() cleared this already */ + clear_sbi_flag(sbi, SBI_POR_DOING); + + /* + * If filesystem is not mounted as read-only then + * do start the gc_thread. + */ + if (test_opt(sbi, BG_GC) && !f2fs_readonly(sb)) { + /* After POR, we can run background GC thread.*/ + err = start_gc_thread(sbi); + if (err) + goto free_kobj; + } + kfree(options); + + /* recover broken superblock */ + if (recovery && !f2fs_readonly(sb) && !bdev_read_only(sb->s_bdev)) { + f2fs_msg(sb, KERN_INFO, "Recover invalid superblock"); + f2fs_commit_super(sbi, true); + } + + sbi->cp_expires = round_jiffies_up(jiffies); + + return 0; + +free_kobj: + kobject_del(&sbi->s_kobj); +free_proc: + if (sbi->s_proc) { + remove_proc_entry("segment_info", sbi->s_proc); + remove_proc_entry(sb->s_id, f2fs_proc_root); + } + f2fs_destroy_stats(sbi); +free_root_inode: + dput(sb->s_root); + sb->s_root = NULL; +free_node_inode: + mutex_lock(&sbi->umount_mutex); + f2fs_leave_shrinker(sbi); + iput(sbi->node_inode); + mutex_unlock(&sbi->umount_mutex); +free_nm: + destroy_node_manager(sbi); +free_sm: + destroy_segment_manager(sbi); +free_cp: + kfree(sbi->ckpt); +free_meta_inode: + make_bad_inode(sbi->meta_inode); + iput(sbi->meta_inode); +free_options: + kfree(options); +free_sb_buf: + brelse(raw_super_buf); +free_sbi: + kfree(sbi); + + /* give only one another chance */ + if (retry) { + retry = false; + shrink_dcache_sb(sb); + goto try_onemore; + } + return err; +} + +static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) +{ + return mount_bdev(fs_type, flags, dev_name, data, f2fs_fill_super); +} + +static void kill_f2fs_super(struct super_block *sb) +{ + if (sb->s_root) + set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE); + kill_block_super(sb); +} + +static struct file_system_type f2fs_fs_type = { + .owner = THIS_MODULE, + .name = "f2fs", + .mount = f2fs_mount, + .kill_sb = kill_f2fs_super, + .fs_flags = FS_REQUIRES_DEV, +}; + +static int __init init_inodecache(void) +{ + f2fs_inode_cachep = f2fs_kmem_cache_create("f2fs_inode_cache", + sizeof(struct f2fs_inode_info)); + if (!f2fs_inode_cachep) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(f2fs_inode_cachep); +} + +static int __init init_f2fs_fs(void) +{ + int err; + + f2fs_build_trace_ios(); + + err = init_inodecache(); + if (err) + goto fail; + err = create_node_manager_caches(); + if (err) + goto free_inodecache; + err = create_segment_manager_caches(); + if (err) + goto free_node_manager_caches; + err = create_checkpoint_caches(); + if (err) + goto free_segment_manager_caches; + err = create_extent_cache(); + if (err) + goto free_checkpoint_caches; + f2fs_kset = kset_create_and_add("f2fs", NULL, fs_kobj); + if (!f2fs_kset) { + err = -ENOMEM; + goto free_extent_cache; + } + err = f2fs_init_crypto(); + if (err) + goto free_kset; + + register_shrinker(&f2fs_shrinker_info); + + err = register_filesystem(&f2fs_fs_type); + if (err) + goto free_shrinker; + f2fs_create_root_stats(); + f2fs_proc_root = proc_mkdir("fs/f2fs", NULL); + return 0; + +free_shrinker: + unregister_shrinker(&f2fs_shrinker_info); + f2fs_exit_crypto(); +free_kset: + kset_unregister(f2fs_kset); +free_extent_cache: + destroy_extent_cache(); +free_checkpoint_caches: + destroy_checkpoint_caches(); +free_segment_manager_caches: + destroy_segment_manager_caches(); +free_node_manager_caches: + destroy_node_manager_caches(); +free_inodecache: + destroy_inodecache(); +fail: + return err; +} + +static void __exit exit_f2fs_fs(void) +{ + remove_proc_entry("fs/f2fs", NULL); + f2fs_destroy_root_stats(); + unregister_shrinker(&f2fs_shrinker_info); + unregister_filesystem(&f2fs_fs_type); + f2fs_exit_crypto(); + destroy_extent_cache(); + destroy_checkpoint_caches(); + destroy_segment_manager_caches(); + destroy_node_manager_caches(); + destroy_inodecache(); + kset_unregister(f2fs_kset); + f2fs_destroy_trace_ios(); +} + +module_init(init_f2fs_fs) +module_exit(exit_f2fs_fs) + +MODULE_AUTHOR("Samsung Electronics's Praesto Team"); +MODULE_DESCRIPTION("Flash Friendly File System"); +MODULE_LICENSE("GPL"); diff --git a/fs/f2fs/trace.c b/fs/f2fs/trace.c new file mode 100644 index 00000000000..145fb659ad4 --- /dev/null +++ b/fs/f2fs/trace.c @@ -0,0 +1,159 @@ +/* + * f2fs IO tracer + * + * Copyright (c) 2014 Motorola Mobility + * Copyright (c) 2014 Jaegeuk Kim + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include + +#include "f2fs.h" +#include "trace.h" + +static RADIX_TREE(pids, GFP_ATOMIC); +static spinlock_t pids_lock; +static struct last_io_info last_io; + +static inline void __print_last_io(void) +{ + if (!last_io.len) + return; + + trace_printk("%3x:%3x %4x %-16s %2x %5x %12x %4x\n", + last_io.major, last_io.minor, + last_io.pid, "----------------", + last_io.type, + last_io.fio.rw, last_io.fio.blk_addr, + last_io.len); + memset(&last_io, 0, sizeof(last_io)); +} + +static int __file_type(struct inode *inode, pid_t pid) +{ + if (f2fs_is_atomic_file(inode)) + return __ATOMIC_FILE; + else if (f2fs_is_volatile_file(inode)) + return __VOLATILE_FILE; + else if (S_ISDIR(inode->i_mode)) + return __DIR_FILE; + else if (inode->i_ino == F2FS_NODE_INO(F2FS_I_SB(inode))) + return __NODE_FILE; + else if (inode->i_ino == F2FS_META_INO(F2FS_I_SB(inode))) + return __META_FILE; + else if (pid) + return __NORMAL_FILE; + else + return __MISC_FILE; +} + +void f2fs_trace_pid(struct page *page) +{ + struct inode *inode = page->mapping->host; + pid_t pid = task_pid_nr(current); + void *p; + + page->private = pid; + + if (radix_tree_preload(GFP_NOFS)) + return; + + spin_lock(&pids_lock); + p = radix_tree_lookup(&pids, pid); + if (p == current) + goto out; + if (p) + radix_tree_delete(&pids, pid); + + f2fs_radix_tree_insert(&pids, pid, current); + + trace_printk("%3x:%3x %4x %-16s\n", + MAJOR(inode->i_sb->s_dev), MINOR(inode->i_sb->s_dev), + pid, current->comm); +out: + spin_unlock(&pids_lock); + radix_tree_preload_end(); +} + +void f2fs_trace_ios(struct f2fs_io_info *fio, int flush) +{ + struct inode *inode; + pid_t pid; + int major, minor; + + if (flush) { + __print_last_io(); + return; + } + + inode = fio->page->mapping->host; + pid = page_private(fio->page); + + major = MAJOR(inode->i_sb->s_dev); + minor = MINOR(inode->i_sb->s_dev); + + if (last_io.major == major && last_io.minor == minor && + last_io.pid == pid && + last_io.type == __file_type(inode, pid) && + last_io.fio.rw == fio->rw && + last_io.fio.blk_addr + last_io.len == fio->blk_addr) { + last_io.len++; + return; + } + + __print_last_io(); + + last_io.major = major; + last_io.minor = minor; + last_io.pid = pid; + last_io.type = __file_type(inode, pid); + last_io.fio = *fio; + last_io.len = 1; + return; +} + +void f2fs_build_trace_ios(void) +{ + spin_lock_init(&pids_lock); +} + +#define PIDVEC_SIZE 128 +static unsigned int gang_lookup_pids(pid_t *results, unsigned long first_index, + unsigned int max_items) +{ + struct radix_tree_iter iter; + void **slot; + unsigned int ret = 0; + + if (unlikely(!max_items)) + return 0; + + radix_tree_for_each_slot(slot, &pids, &iter, first_index) { + results[ret] = iter.index; + if (++ret == PIDVEC_SIZE) + break; + } + return ret; +} + +void f2fs_destroy_trace_ios(void) +{ + pid_t pid[PIDVEC_SIZE]; + pid_t next_pid = 0; + unsigned int found; + + spin_lock(&pids_lock); + while ((found = gang_lookup_pids(pid, next_pid, PIDVEC_SIZE))) { + unsigned idx; + + next_pid = pid[found - 1] + 1; + for (idx = 0; idx < found; idx++) + radix_tree_delete(&pids, pid[idx]); + } + spin_unlock(&pids_lock); +} diff --git a/fs/f2fs/trace.h b/fs/f2fs/trace.h new file mode 100644 index 00000000000..67db24ac1e8 --- /dev/null +++ b/fs/f2fs/trace.h @@ -0,0 +1,46 @@ +/* + * f2fs IO tracer + * + * Copyright (c) 2014 Motorola Mobility + * Copyright (c) 2014 Jaegeuk Kim + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __F2FS_TRACE_H__ +#define __F2FS_TRACE_H__ + +#ifdef CONFIG_F2FS_IO_TRACE +#include + +enum file_type { + __NORMAL_FILE, + __DIR_FILE, + __NODE_FILE, + __META_FILE, + __ATOMIC_FILE, + __VOLATILE_FILE, + __MISC_FILE, +}; + +struct last_io_info { + int major, minor; + pid_t pid; + enum file_type type; + struct f2fs_io_info fio; + block_t len; +}; + +extern void f2fs_trace_pid(struct page *); +extern void f2fs_trace_ios(struct f2fs_io_info *, int); +extern void f2fs_build_trace_ios(void); +extern void f2fs_destroy_trace_ios(void); +#else +#define f2fs_trace_pid(p) +#define f2fs_trace_ios(i, n) +#define f2fs_build_trace_ios() +#define f2fs_destroy_trace_ios() + +#endif +#endif /* __F2FS_TRACE_H__ */ diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c new file mode 100644 index 00000000000..dd0646a5687 --- /dev/null +++ b/fs/f2fs/xattr.c @@ -0,0 +1,623 @@ +/* + * fs/f2fs/xattr.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/xattr.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher + * + * Fix by Harrison Xing . + * Extended attributes for symlinks and special files added per + * suggestion of Luka Renko . + * xattr consolidation Copyright (c) 2004 James Morris , + * Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include "f2fs.h" +#include "xattr.h" + +static size_t f2fs_xattr_generic_list(struct dentry *dentry, char *list, + size_t list_size, const char *name, size_t len, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + int total_len, prefix_len = 0; + const char *prefix = NULL; + + switch (type) { + case F2FS_XATTR_INDEX_USER: + if (!test_opt(sbi, XATTR_USER)) + return -EOPNOTSUPP; + prefix = XATTR_USER_PREFIX; + prefix_len = XATTR_USER_PREFIX_LEN; + break; + case F2FS_XATTR_INDEX_TRUSTED: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + prefix = XATTR_TRUSTED_PREFIX; + prefix_len = XATTR_TRUSTED_PREFIX_LEN; + break; + case F2FS_XATTR_INDEX_SECURITY: + prefix = XATTR_SECURITY_PREFIX; + prefix_len = XATTR_SECURITY_PREFIX_LEN; + break; + default: + return -EINVAL; + } + + total_len = prefix_len + len + 1; + if (list && total_len <= list_size) { + memcpy(list, prefix, prefix_len); + memcpy(list + prefix_len, name, len); + list[prefix_len + len] = '\0'; + } + return total_len; +} + +static int f2fs_xattr_generic_get(struct dentry *dentry, const char *name, + void *buffer, size_t size, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + + switch (type) { + case F2FS_XATTR_INDEX_USER: + if (!test_opt(sbi, XATTR_USER)) + return -EOPNOTSUPP; + break; + case F2FS_XATTR_INDEX_TRUSTED: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + break; + case F2FS_XATTR_INDEX_SECURITY: + break; + default: + return -EINVAL; + } + if (strcmp(name, "") == 0) + return -EINVAL; + return f2fs_getxattr(dentry->d_inode, type, name, buffer, size, NULL); +} + +static int f2fs_xattr_generic_set(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + + switch (type) { + case F2FS_XATTR_INDEX_USER: + if (!test_opt(sbi, XATTR_USER)) + return -EOPNOTSUPP; + break; + case F2FS_XATTR_INDEX_TRUSTED: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + break; + case F2FS_XATTR_INDEX_SECURITY: + break; + default: + return -EINVAL; + } + if (strcmp(name, "") == 0) + return -EINVAL; + + return f2fs_setxattr(dentry->d_inode, type, name, + value, size, NULL, flags); +} + +static size_t f2fs_xattr_advise_list(struct dentry *dentry, char *list, + size_t list_size, const char *name, size_t len, int type) +{ + const char *xname = F2FS_SYSTEM_ADVISE_PREFIX; + size_t size; + + if (type != F2FS_XATTR_INDEX_ADVISE) + return 0; + + size = strlen(xname) + 1; + if (list && size <= list_size) + memcpy(list, xname, size); + return size; +} + +static int f2fs_xattr_advise_get(struct dentry *dentry, const char *name, + void *buffer, size_t size, int type) +{ + struct inode *inode = dentry->d_inode; + + if (strcmp(name, "") != 0) + return -EINVAL; + + if (buffer) + *((char *)buffer) = F2FS_I(inode)->i_advise; + return sizeof(char); +} + +static int f2fs_xattr_advise_set(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags, int type) +{ + struct inode *inode = dentry->d_inode; + + if (strcmp(name, "") != 0) + return -EINVAL; + if (!inode_owner_or_capable(inode)) + return -EPERM; + if (value == NULL) + return -EINVAL; + + F2FS_I(inode)->i_advise |= *(char *)value; + mark_inode_dirty(inode); + return 0; +} + +#ifdef CONFIG_F2FS_FS_SECURITY +static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *page) +{ + const struct xattr *xattr; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_SECURITY, + xattr->name, xattr->value, + xattr->value_len, (struct page *)page, 0); + if (err < 0) + break; + } + return err; +} + +int f2fs_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct page *ipage) +{ + return security_inode_init_security(inode, dir, qstr, + &f2fs_initxattrs, ipage); +} +#endif + +const struct xattr_handler f2fs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = F2FS_XATTR_INDEX_USER, + .list = f2fs_xattr_generic_list, + .get = f2fs_xattr_generic_get, + .set = f2fs_xattr_generic_set, +}; + +const struct xattr_handler f2fs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = F2FS_XATTR_INDEX_TRUSTED, + .list = f2fs_xattr_generic_list, + .get = f2fs_xattr_generic_get, + .set = f2fs_xattr_generic_set, +}; + +const struct xattr_handler f2fs_xattr_advise_handler = { + .prefix = F2FS_SYSTEM_ADVISE_PREFIX, + .flags = F2FS_XATTR_INDEX_ADVISE, + .list = f2fs_xattr_advise_list, + .get = f2fs_xattr_advise_get, + .set = f2fs_xattr_advise_set, +}; + +const struct xattr_handler f2fs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = F2FS_XATTR_INDEX_SECURITY, + .list = f2fs_xattr_generic_list, + .get = f2fs_xattr_generic_get, + .set = f2fs_xattr_generic_set, +}; + +static const struct xattr_handler *f2fs_xattr_handler_map[] = { + [F2FS_XATTR_INDEX_USER] = &f2fs_xattr_user_handler, +#ifdef CONFIG_F2FS_FS_POSIX_ACL + [F2FS_XATTR_INDEX_POSIX_ACL_ACCESS] = &f2fs_xattr_acl_access_handler, + [F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT] = &f2fs_xattr_acl_default_handler, +#endif + [F2FS_XATTR_INDEX_TRUSTED] = &f2fs_xattr_trusted_handler, +#ifdef CONFIG_F2FS_FS_SECURITY + [F2FS_XATTR_INDEX_SECURITY] = &f2fs_xattr_security_handler, +#endif + [F2FS_XATTR_INDEX_ADVISE] = &f2fs_xattr_advise_handler, +}; + +const struct xattr_handler *f2fs_xattr_handlers[] = { + &f2fs_xattr_user_handler, +#ifdef CONFIG_F2FS_FS_POSIX_ACL + &f2fs_xattr_acl_access_handler, + &f2fs_xattr_acl_default_handler, +#endif + &f2fs_xattr_trusted_handler, +#ifdef CONFIG_F2FS_FS_SECURITY + &f2fs_xattr_security_handler, +#endif + &f2fs_xattr_advise_handler, + NULL, +}; + +static inline const struct xattr_handler *f2fs_xattr_handler(int index) +{ + const struct xattr_handler *handler = NULL; + + if (index > 0 && index < ARRAY_SIZE(f2fs_xattr_handler_map)) + handler = f2fs_xattr_handler_map[index]; + return handler; +} + +static struct f2fs_xattr_entry *__find_xattr(void *base_addr, int index, + size_t len, const char *name) +{ + struct f2fs_xattr_entry *entry; + + list_for_each_xattr(entry, base_addr) { + if (entry->e_name_index != index) + continue; + if (entry->e_name_len != len) + continue; + if (!memcmp(entry->e_name, name, len)) + break; + } + return entry; +} + +static void *read_all_xattrs(struct inode *inode, struct page *ipage) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_xattr_header *header; + size_t size = PAGE_SIZE, inline_size = 0; + void *txattr_addr; + + inline_size = inline_xattr_size(inode); + + txattr_addr = kzalloc(inline_size + size, GFP_F2FS_ZERO); + if (!txattr_addr) + return NULL; + + /* read from inline xattr */ + if (inline_size) { + struct page *page = NULL; + void *inline_addr; + + if (ipage) { + inline_addr = inline_xattr_addr(ipage); + } else { + page = get_node_page(sbi, inode->i_ino); + if (IS_ERR(page)) + goto fail; + inline_addr = inline_xattr_addr(page); + } + memcpy(txattr_addr, inline_addr, inline_size); + f2fs_put_page(page, 1); + } + + /* read from xattr node block */ + if (F2FS_I(inode)->i_xattr_nid) { + struct page *xpage; + void *xattr_addr; + + /* The inode already has an extended attribute block. */ + xpage = get_node_page(sbi, F2FS_I(inode)->i_xattr_nid); + if (IS_ERR(xpage)) + goto fail; + + xattr_addr = page_address(xpage); + memcpy(txattr_addr + inline_size, xattr_addr, PAGE_SIZE); + f2fs_put_page(xpage, 1); + } + + header = XATTR_HDR(txattr_addr); + + /* never been allocated xattrs */ + if (le32_to_cpu(header->h_magic) != F2FS_XATTR_MAGIC) { + header->h_magic = cpu_to_le32(F2FS_XATTR_MAGIC); + header->h_refcount = cpu_to_le32(1); + } + return txattr_addr; +fail: + kzfree(txattr_addr); + return NULL; +} + +static inline int write_all_xattrs(struct inode *inode, __u32 hsize, + void *txattr_addr, struct page *ipage) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + size_t inline_size = 0; + void *xattr_addr; + struct page *xpage; + nid_t new_nid = 0; + int err; + + inline_size = inline_xattr_size(inode); + + if (hsize > inline_size && !F2FS_I(inode)->i_xattr_nid) + if (!alloc_nid(sbi, &new_nid)) + return -ENOSPC; + + /* write to inline xattr */ + if (inline_size) { + struct page *page = NULL; + void *inline_addr; + + if (ipage) { + inline_addr = inline_xattr_addr(ipage); + f2fs_wait_on_page_writeback(ipage, NODE); + } else { + page = get_node_page(sbi, inode->i_ino); + if (IS_ERR(page)) { + alloc_nid_failed(sbi, new_nid); + return PTR_ERR(page); + } + inline_addr = inline_xattr_addr(page); + f2fs_wait_on_page_writeback(page, NODE); + } + memcpy(inline_addr, txattr_addr, inline_size); + f2fs_put_page(page, 1); + + /* no need to use xattr node block */ + if (hsize <= inline_size) { + err = truncate_xattr_node(inode, ipage); + alloc_nid_failed(sbi, new_nid); + return err; + } + } + + /* write to xattr node block */ + if (F2FS_I(inode)->i_xattr_nid) { + xpage = get_node_page(sbi, F2FS_I(inode)->i_xattr_nid); + if (IS_ERR(xpage)) { + alloc_nid_failed(sbi, new_nid); + return PTR_ERR(xpage); + } + f2fs_bug_on(sbi, new_nid); + f2fs_wait_on_page_writeback(xpage, NODE); + } else { + struct dnode_of_data dn; + set_new_dnode(&dn, inode, NULL, NULL, new_nid); + xpage = new_node_page(&dn, XATTR_NODE_OFFSET, ipage); + if (IS_ERR(xpage)) { + alloc_nid_failed(sbi, new_nid); + return PTR_ERR(xpage); + } + alloc_nid_done(sbi, new_nid); + } + + xattr_addr = page_address(xpage); + memcpy(xattr_addr, txattr_addr + inline_size, PAGE_SIZE - + sizeof(struct node_footer)); + set_page_dirty(xpage); + f2fs_put_page(xpage, 1); + + /* need to checkpoint during fsync */ + F2FS_I(inode)->xattr_ver = cur_cp_version(F2FS_CKPT(sbi)); + return 0; +} + +int f2fs_getxattr(struct inode *inode, int index, const char *name, + void *buffer, size_t buffer_size, struct page *ipage) +{ + struct f2fs_xattr_entry *entry; + void *base_addr; + int error = 0; + size_t size, len; + + if (name == NULL) + return -EINVAL; + + len = strlen(name); + if (len > F2FS_NAME_LEN) + return -ERANGE; + + base_addr = read_all_xattrs(inode, ipage); + if (!base_addr) + return -ENOMEM; + + entry = __find_xattr(base_addr, index, len, name); + if (IS_XATTR_LAST_ENTRY(entry)) { + error = -ENODATA; + goto cleanup; + } + + size = le16_to_cpu(entry->e_value_size); + + if (buffer && size > buffer_size) { + error = -ERANGE; + goto cleanup; + } + + if (buffer) { + char *pval = entry->e_name + entry->e_name_len; + memcpy(buffer, pval, size); + } + error = size; + +cleanup: + kzfree(base_addr); + return error; +} + +ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + struct inode *inode = dentry->d_inode; + struct f2fs_xattr_entry *entry; + void *base_addr; + int error = 0; + size_t rest = buffer_size; + + base_addr = read_all_xattrs(inode, NULL); + if (!base_addr) + return -ENOMEM; + + list_for_each_xattr(entry, base_addr) { + const struct xattr_handler *handler = + f2fs_xattr_handler(entry->e_name_index); + size_t size; + + if (!handler) + continue; + + size = handler->list(dentry, buffer, rest, entry->e_name, + entry->e_name_len, handler->flags); + if (buffer && size > rest) { + error = -ERANGE; + goto cleanup; + } + + if (buffer) + buffer += size; + rest -= size; + } + error = buffer_size - rest; +cleanup: + kzfree(base_addr); + return error; +} + +static int __f2fs_setxattr(struct inode *inode, int index, + const char *name, const void *value, size_t size, + struct page *ipage, int flags) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_xattr_entry *here, *last; + void *base_addr; + int found, newsize; + size_t len; + __u32 new_hsize; + int error = -ENOMEM; + + if (name == NULL) + return -EINVAL; + + if (value == NULL) + size = 0; + + len = strlen(name); + + if (len > F2FS_NAME_LEN) + return -ERANGE; + + if (size > MAX_VALUE_LEN(inode)) + return -E2BIG; + + base_addr = read_all_xattrs(inode, ipage); + if (!base_addr) + goto exit; + + /* find entry with wanted name. */ + here = __find_xattr(base_addr, index, len, name); + + found = IS_XATTR_LAST_ENTRY(here) ? 0 : 1; + + if ((flags & XATTR_REPLACE) && !found) { + error = -ENODATA; + goto exit; + } else if ((flags & XATTR_CREATE) && found) { + error = -EEXIST; + goto exit; + } + + last = here; + while (!IS_XATTR_LAST_ENTRY(last)) + last = XATTR_NEXT_ENTRY(last); + + newsize = XATTR_ALIGN(sizeof(struct f2fs_xattr_entry) + len + size); + + /* 1. Check space */ + if (value) { + int free; + /* + * If value is NULL, it is remove operation. + * In case of update operation, we calculate free. + */ + free = MIN_OFFSET(inode) - ((char *)last - (char *)base_addr); + if (found) + free = free + ENTRY_SIZE(here); + + if (unlikely(free < newsize)) { + error = -ENOSPC; + goto exit; + } + } + + /* 2. Remove old entry */ + if (found) { + /* + * If entry is found, remove old entry. + * If not found, remove operation is not needed. + */ + struct f2fs_xattr_entry *next = XATTR_NEXT_ENTRY(here); + int oldsize = ENTRY_SIZE(here); + + memmove(here, next, (char *)last - (char *)next); + last = (struct f2fs_xattr_entry *)((char *)last - oldsize); + memset(last, 0, oldsize); + } + + new_hsize = (char *)last - (char *)base_addr; + + /* 3. Write new entry */ + if (value) { + char *pval; + /* + * Before we come here, old entry is removed. + * We just write new entry. + */ + memset(last, 0, newsize); + last->e_name_index = index; + last->e_name_len = len; + memcpy(last->e_name, name, len); + pval = last->e_name + len; + memcpy(pval, value, size); + last->e_value_size = cpu_to_le16(size); + new_hsize += newsize; + } + + error = write_all_xattrs(inode, new_hsize, base_addr, ipage); + if (error) + goto exit; + + if (is_inode_flag_set(fi, FI_ACL_MODE)) { + inode->i_mode = fi->i_acl_mode; + inode->i_ctime = CURRENT_TIME; + clear_inode_flag(fi, FI_ACL_MODE); + } + if (index == F2FS_XATTR_INDEX_ENCRYPTION && + !strcmp(name, F2FS_XATTR_NAME_ENCRYPTION_CONTEXT)) + f2fs_set_encrypted_inode(inode); + + if (ipage) + update_inode(inode, ipage); + else + update_inode_page(inode); +exit: + kzfree(base_addr); + return error; +} + +int f2fs_setxattr(struct inode *inode, int index, const char *name, + const void *value, size_t size, + struct page *ipage, int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int err; + + /* this case is only from init_inode_metadata */ + if (ipage) + return __f2fs_setxattr(inode, index, name, value, + size, ipage, flags); + f2fs_balance_fs(sbi); + + f2fs_lock_op(sbi); + /* protect xattr_ver */ + down_write(&F2FS_I(inode)->i_sem); + err = __f2fs_setxattr(inode, index, name, value, size, ipage, flags); + up_write(&F2FS_I(inode)->i_sem); + f2fs_unlock_op(sbi); + + return err; +} diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h new file mode 100644 index 00000000000..47cf0e58b0e --- /dev/null +++ b/fs/f2fs/xattr.h @@ -0,0 +1,158 @@ +/* + * fs/f2fs/xattr.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/xattr.h + * + * On-disk format of extended attributes for the ext2 filesystem. + * + * (C) 2001 Andreas Gruenbacher, + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __F2FS_XATTR_H__ +#define __F2FS_XATTR_H__ + +#include +#include + +/* Magic value in attribute blocks */ +#define F2FS_XATTR_MAGIC 0xF2F52011 + +/* Maximum number of references to one attribute block */ +#define F2FS_XATTR_REFCOUNT_MAX 1024 + +/* Name indexes */ +#define F2FS_SYSTEM_ADVISE_PREFIX "system.advise" +#define F2FS_XATTR_INDEX_USER 1 +#define F2FS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define F2FS_XATTR_INDEX_TRUSTED 4 +#define F2FS_XATTR_INDEX_LUSTRE 5 +#define F2FS_XATTR_INDEX_SECURITY 6 +#define F2FS_XATTR_INDEX_ADVISE 7 +/* Should be same as EXT4_XATTR_INDEX_ENCRYPTION */ +#define F2FS_XATTR_INDEX_ENCRYPTION 9 + +#define F2FS_XATTR_NAME_ENCRYPTION_CONTEXT "c" + +struct f2fs_xattr_header { + __le32 h_magic; /* magic number for identification */ + __le32 h_refcount; /* reference count */ + __u32 h_reserved[4]; /* zero right now */ +}; + +struct f2fs_xattr_entry { + __u8 e_name_index; + __u8 e_name_len; + __le16 e_value_size; /* size of attribute value */ + char e_name[0]; /* attribute name */ +}; + +#define XATTR_HDR(ptr) ((struct f2fs_xattr_header *)(ptr)) +#define XATTR_ENTRY(ptr) ((struct f2fs_xattr_entry *)(ptr)) +#define XATTR_FIRST_ENTRY(ptr) (XATTR_ENTRY(XATTR_HDR(ptr) + 1)) +#define XATTR_ROUND (3) + +#define XATTR_ALIGN(size) ((size + XATTR_ROUND) & ~XATTR_ROUND) + +#define ENTRY_SIZE(entry) (XATTR_ALIGN(sizeof(struct f2fs_xattr_entry) + \ + entry->e_name_len + le16_to_cpu(entry->e_value_size))) + +#define XATTR_NEXT_ENTRY(entry) ((struct f2fs_xattr_entry *)((char *)(entry) +\ + ENTRY_SIZE(entry))) + +#define IS_XATTR_LAST_ENTRY(entry) (*(__u32 *)(entry) == 0) + +#define list_for_each_xattr(entry, addr) \ + for (entry = XATTR_FIRST_ENTRY(addr);\ + !IS_XATTR_LAST_ENTRY(entry);\ + entry = XATTR_NEXT_ENTRY(entry)) + +#define MIN_OFFSET(i) XATTR_ALIGN(inline_xattr_size(i) + PAGE_SIZE - \ + sizeof(struct node_footer) - sizeof(__u32)) + +#define MAX_VALUE_LEN(i) (MIN_OFFSET(i) - \ + sizeof(struct f2fs_xattr_header) - \ + sizeof(struct f2fs_xattr_entry)) + +/* + * On-disk structure of f2fs_xattr + * We use inline xattrs space + 1 block for xattr. + * + * +--------------------+ + * | f2fs_xattr_header | + * | | + * +--------------------+ + * | f2fs_xattr_entry | + * | .e_name_index = 1 | + * | .e_name_len = 3 | + * | .e_value_size = 14 | + * | .e_name = "foo" | + * | "value_of_xattr" |<- value_offs = e_name + e_name_len + * +--------------------+ + * | f2fs_xattr_entry | + * | .e_name_index = 4 | + * | .e_name = "bar" | + * +--------------------+ + * | | + * | Free | + * | | + * +--------------------+<- MIN_OFFSET + * | node_footer | + * | (nid, ino, offset) | + * +--------------------+ + * + **/ + +#ifdef CONFIG_F2FS_FS_XATTR +extern const struct xattr_handler f2fs_xattr_user_handler; +extern const struct xattr_handler f2fs_xattr_trusted_handler; +extern const struct xattr_handler f2fs_xattr_acl_access_handler; +extern const struct xattr_handler f2fs_xattr_acl_default_handler; +extern const struct xattr_handler f2fs_xattr_advise_handler; +extern const struct xattr_handler f2fs_xattr_security_handler; + +extern const struct xattr_handler *f2fs_xattr_handlers[]; + +extern int f2fs_setxattr(struct inode *, int, const char *, + const void *, size_t, struct page *, int); +extern int f2fs_getxattr(struct inode *, int, const char *, void *, + size_t, struct page *); +extern ssize_t f2fs_listxattr(struct dentry *, char *, size_t); +#else + +#define f2fs_xattr_handlers NULL +static inline int f2fs_setxattr(struct inode *inode, int index, + const char *name, const void *value, size_t size, int flags) +{ + return -EOPNOTSUPP; +} +static inline int f2fs_getxattr(struct inode *inode, int index, + const char *name, void *buffer, + size_t buffer_size, struct page *dpage) +{ + return -EOPNOTSUPP; +} +static inline ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, + size_t buffer_size) +{ + return -EOPNOTSUPP; +} +#endif + +#ifdef CONFIG_F2FS_FS_SECURITY +extern int f2fs_init_security(struct inode *, struct inode *, + const struct qstr *, struct page *); +#else +static inline int f2fs_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct page *ipage) +{ + return 0; +} +#endif +#endif /* __F2FS_XATTR_H__ */ diff --git a/fs/file_table.c b/fs/file_table.c index 70f2a0fd6ae..a305d9e2d1b 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -34,7 +34,6 @@ struct files_stat_struct files_stat = { .max_files = NR_FILE }; -DECLARE_LGLOCK(files_lglock); DEFINE_LGLOCK(files_lglock); /* SLAB cache for file structures */ @@ -421,9 +420,9 @@ static inline void __file_sb_list_add(struct file *file, struct super_block *sb) */ void file_sb_list_add(struct file *file, struct super_block *sb) { - lg_local_lock(files_lglock); + lg_local_lock(&files_lglock); __file_sb_list_add(file, sb); - lg_local_unlock(files_lglock); + lg_local_unlock(&files_lglock); } /** @@ -436,9 +435,9 @@ void file_sb_list_add(struct file *file, struct super_block *sb) void file_sb_list_del(struct file *file) { if (!list_empty(&file->f_u.fu_list)) { - lg_local_lock_cpu(files_lglock, file_list_cpu(file)); + lg_local_lock_cpu(&files_lglock, file_list_cpu(file)); list_del_init(&file->f_u.fu_list); - lg_local_unlock_cpu(files_lglock, file_list_cpu(file)); + lg_local_unlock_cpu(&files_lglock, file_list_cpu(file)); } } @@ -485,7 +484,7 @@ void mark_files_ro(struct super_block *sb) struct file *f; retry: - lg_global_lock(files_lglock); + lg_global_lock(&files_lglock); do_file_list_for_each_entry(sb, f) { struct vfsmount *mnt; if (!S_ISREG(f->f_path.dentry->d_inode->i_mode)) @@ -502,12 +501,12 @@ void mark_files_ro(struct super_block *sb) file_release_write(f); mnt = mntget(f->f_path.mnt); /* This can sleep, so we can't hold the spinlock. */ - lg_global_unlock(files_lglock); + lg_global_unlock(&files_lglock); mnt_drop_write(mnt); mntput(mnt); goto retry; } while_file_list_for_each_entry; - lg_global_unlock(files_lglock); + lg_global_unlock(&files_lglock); } void __init files_init(unsigned long mempages) @@ -525,6 +524,6 @@ void __init files_init(unsigned long mempages) n = (mempages * (PAGE_SIZE / 1024)) / 10; files_stat.max_files = max_t(unsigned long, n, NR_FILE); files_defer_init(); - lg_lock_init(files_lglock); + lg_lock_init(&files_lglock, "files_lglock"); percpu_counter_init(&nr_files, 0); } diff --git a/fs/fs_struct.c b/fs/fs_struct.c index e159e682ad4..5df4775fea0 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c @@ -6,18 +6,6 @@ #include #include "internal.h" -static inline void path_get_longterm(struct path *path) -{ - path_get(path); - mnt_make_longterm(path->mnt); -} - -static inline void path_put_longterm(struct path *path) -{ - mnt_make_shortterm(path->mnt); - path_put(path); -} - /* * Replace the fs->{rootmnt,root} with {mnt,dentry}. Put the old values. * It can block. @@ -26,7 +14,7 @@ void set_fs_root(struct fs_struct *fs, struct path *path) { struct path old_root; - path_get_longterm(path); + path_get(path); spin_lock(&fs->lock); write_seqcount_begin(&fs->seq); old_root = fs->root; @@ -34,7 +22,7 @@ void set_fs_root(struct fs_struct *fs, struct path *path) write_seqcount_end(&fs->seq); spin_unlock(&fs->lock); if (old_root.dentry) - path_put_longterm(&old_root); + path_put(&old_root); } /* @@ -45,7 +33,7 @@ void set_fs_pwd(struct fs_struct *fs, struct path *path) { struct path old_pwd; - path_get_longterm(path); + path_get(path); spin_lock(&fs->lock); write_seqcount_begin(&fs->seq); old_pwd = fs->pwd; @@ -54,7 +42,7 @@ void set_fs_pwd(struct fs_struct *fs, struct path *path) spin_unlock(&fs->lock); if (old_pwd.dentry) - path_put_longterm(&old_pwd); + path_put(&old_pwd); } static inline int replace_path(struct path *p, const struct path *old, const struct path *new) @@ -84,7 +72,7 @@ void chroot_fs_refs(struct path *old_root, struct path *new_root) write_seqcount_end(&fs->seq); while (hits--) { count++; - path_get_longterm(new_root); + path_get(new_root); } spin_unlock(&fs->lock); } @@ -92,13 +80,13 @@ void chroot_fs_refs(struct path *old_root, struct path *new_root) } while_each_thread(g, p); read_unlock(&tasklist_lock); while (count--) - path_put_longterm(old_root); + path_put(old_root); } void free_fs_struct(struct fs_struct *fs) { - path_put_longterm(&fs->root); - path_put_longterm(&fs->pwd); + path_put(&fs->root); + path_put(&fs->pwd); kmem_cache_free(fs_cachep, fs); } @@ -132,9 +120,9 @@ struct fs_struct *copy_fs_struct(struct fs_struct *old) spin_lock(&old->lock); fs->root = old->root; - path_get_longterm(&fs->root); + path_get(&fs->root); fs->pwd = old->pwd; - path_get_longterm(&fs->pwd); + path_get(&fs->pwd); spin_unlock(&old->lock); } return fs; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 6b8322225a1..311aa5c9d6a 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -692,6 +692,8 @@ static int fuse_readpages_fill(void *_data, struct page *page) lock_page(newpage); put_page(newpage); + lru_cache_add_file(newpage); + /* finally release the old page and swap pointers */ unlock_page(oldpage); page_cache_release(oldpage); @@ -881,6 +883,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_req *req, mark_page_accessed(page); + iov_iter_advance(ii, tmp); if (!tmp) { unlock_page(page); page_cache_release(page); @@ -892,7 +895,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_req *req, req->pages[req->num_pages] = page; req->num_pages++; - iov_iter_advance(ii, tmp); count += tmp; pos += tmp; offset += tmp; @@ -998,7 +1000,9 @@ static ssize_t fuse_file_aio_write(struct kiocb *iocb, const struct iovec *iov, if (err) goto out; - file_update_time(file); + err = file_update_time(file); + if (err) + goto out; if (file->f_flags & O_DIRECT) { written = generic_file_direct_write(iocb, iov, &nr_segs, diff --git a/fs/inode.c b/fs/inode.c index 9f4f5fecc09..06d8bd40b29 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1480,6 +1480,27 @@ static int relatime_need_update(struct vfsmount *mnt, struct inode *inode, return 0; } +/* + * This does the actual work of updating an inodes time or version. Must have + * had called mnt_want_write() before calling this. + */ +static int update_time(struct inode *inode, struct timespec *time, int flags) +{ + if (inode->i_op->update_time) + return inode->i_op->update_time(inode, time, flags); + + if (flags & S_ATIME) + inode->i_atime = *time; + if (flags & S_VERSION) + inode_inc_iversion(inode); + if (flags & S_CTIME) + inode->i_ctime = *time; + if (flags & S_MTIME) + inode->i_mtime = *time; + mark_inode_dirty_sync(inode); + return 0; +} + /** * touch_atime - update the access time * @mnt: mount the inode is accessed on @@ -1518,8 +1539,14 @@ void touch_atime(struct path *path) if (mnt_want_write(mnt)) return; - inode->i_atime = now; - mark_inode_dirty_sync(inode); + /* + * File systems can error out when updating inodes if they need to + * allocate new space to modify an inode (such is the case for + * Btrfs), but since we touch atime while walking down the path we + * really don't care if we failed to update the atime of the file, + * so just ignore the return value. + */ + update_time(inode, &now, S_ATIME); mnt_drop_write(mnt); } EXPORT_SYMBOL(touch_atime); @@ -1533,18 +1560,20 @@ EXPORT_SYMBOL(touch_atime); * usage in the file write path of filesystems, and filesystems may * choose to explicitly ignore update via this function with the * S_NOCMTIME inode flag, e.g. for network filesystem where these - * timestamps are handled by the server. + * timestamps are handled by the server. This can return an error for + * file systems who need to allocate space in order to update an inode. */ -void file_update_time(struct file *file) +int file_update_time(struct file *file) { struct inode *inode = file->f_path.dentry->d_inode; struct timespec now; - enum { S_MTIME = 1, S_CTIME = 2, S_VERSION = 4 } sync_it = 0; + int sync_it = 0; + int ret; /* First try to exhaust all avenues to not sync */ if (IS_NOCMTIME(inode)) - return; + return 0; now = current_fs_time(inode->i_sb); if (!timespec_equal(&inode->i_mtime, &now)) @@ -1557,21 +1586,16 @@ void file_update_time(struct file *file) sync_it |= S_VERSION; if (!sync_it) - return; + return 0; /* Finally allowed to write? Takes lock. */ if (mnt_want_write_file(file)) - return; + return 0; - /* Only change inode inside the lock region */ - if (sync_it & S_VERSION) - inode_inc_iversion(inode); - if (sync_it & S_CTIME) - inode->i_ctime = now; - if (sync_it & S_MTIME) - inode->i_mtime = now; - mark_inode_dirty_sync(inode); + ret = update_time(inode, &now, sync_it); mnt_drop_write_file(file); + + return ret; } EXPORT_SYMBOL(file_update_time); diff --git a/fs/internal.h b/fs/internal.h index 9962c59ba28..c0f75b80425 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -50,13 +50,11 @@ extern int copy_mount_string(const void __user *, char **); extern struct vfsmount *lookup_mnt(struct path *); extern int finish_automount(struct vfsmount *, struct path *); -extern void mnt_make_longterm(struct vfsmount *); -extern void mnt_make_shortterm(struct vfsmount *); extern int sb_prepare_remount_readonly(struct super_block *); extern void __init mnt_init(void); -DECLARE_BRLOCK(vfsmount_lock); +extern struct lglock vfsmount_lock; /* diff --git a/fs/ioprio.c b/fs/ioprio.c index 0f1b9515213..d792f569907 100644 --- a/fs/ioprio.c +++ b/fs/ioprio.c @@ -145,8 +145,10 @@ static int get_task_ioprio(struct task_struct *p) if (ret) goto out; ret = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, IOPRIO_NORM); + task_lock(p); if (p->io_context) ret = p->io_context->ioprio; + task_unlock(p); out: return ret; } diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c index 70e79d0c756..e44dc6ad0d5 100644 --- a/fs/isofs/rock.c +++ b/fs/isofs/rock.c @@ -30,6 +30,7 @@ struct rock_state { int cont_size; int cont_extent; int cont_offset; + int cont_loops; struct inode *inode; }; @@ -73,6 +74,9 @@ static void init_rock_state(struct rock_state *rs, struct inode *inode) rs->inode = inode; } +/* Maximum number of Rock Ridge continuation entries */ +#define RR_MAX_CE_ENTRIES 32 + /* * Returns 0 if the caller should continue scanning, 1 if the scan must end * and -ve on error. @@ -105,6 +109,8 @@ static int rock_continue(struct rock_state *rs) goto out; } ret = -EIO; + if (++rs->cont_loops >= RR_MAX_CE_ENTRIES) + goto out; bh = sb_bread(rs->inode->i_sb, rs->cont_extent); if (bh) { memcpy(rs->buffer, bh->b_data + rs->cont_offset, diff --git a/fs/mount.h b/fs/mount.h index 4ef36d93e5a..cd500798040 100644 --- a/fs/mount.h +++ b/fs/mount.h @@ -4,8 +4,11 @@ struct mnt_namespace { atomic_t count; + unsigned int proc_inum; struct mount * root; struct list_head list; + struct user_namespace *user_ns; + u64 seq; /* Sequence number to prevent loops */ wait_queue_head_t poll; int event; }; @@ -22,7 +25,6 @@ struct mount { struct vfsmount mnt; #ifdef CONFIG_SMP struct mnt_pcp __percpu *mnt_pcp; - atomic_t mnt_longterm; /* how many of the refs are longterm */ #else int mnt_count; int mnt_writers; @@ -49,6 +51,8 @@ struct mount { int mnt_ghosts; }; +#define MNT_NS_INTERNAL ERR_PTR(-EINVAL) /* distinct from any mnt_namespace */ + static inline struct mount *real_mount(struct vfsmount *mnt) { return container_of(mnt, struct mount, mnt); @@ -59,6 +63,12 @@ static inline int mnt_has_parent(struct mount *mnt) return mnt != mnt->mnt_parent; } +static inline int is_mounted(struct vfsmount *mnt) +{ + /* neither detached nor internal? */ + return !IS_ERR_OR_NULL(real_mount(mnt)); +} + extern struct mount *__lookup_mnt(struct vfsmount *, struct dentry *, int); static inline void get_mnt_ns(struct mnt_namespace *ns) @@ -67,10 +77,12 @@ static inline void get_mnt_ns(struct mnt_namespace *ns) } struct proc_mounts { - struct seq_file m; /* must be the first element */ + struct seq_file m; struct mnt_namespace *ns; struct path root; int (*show)(struct seq_file *, struct vfsmount *); }; +#define proc_mounts(p) (container_of((p), struct proc_mounts, m)) + extern const struct seq_operations mounts_op; diff --git a/fs/namei.c b/fs/namei.c index c42791914f8..f4c5d195208 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -462,7 +462,7 @@ static int unlazy_walk(struct nameidata *nd, struct dentry *dentry) mntget(nd->path.mnt); rcu_read_unlock(); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); nd->flags &= ~LOOKUP_RCU; return 0; @@ -520,14 +520,14 @@ static int complete_walk(struct nameidata *nd) if (unlikely(!__d_rcu_to_refcount(dentry, nd->seq))) { spin_unlock(&dentry->d_lock); rcu_read_unlock(); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return -ECHILD; } BUG_ON(nd->inode != dentry->d_inode); spin_unlock(&dentry->d_lock); mntget(nd->path.mnt); rcu_read_unlock(); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); } if (likely(!(nd->flags & LOOKUP_JUMPED))) @@ -688,21 +688,31 @@ static int follow_up_rcu(struct path *path) return 1; } +/* + * follow_up - Find the mountpoint of path's vfsmount + * + * Given a path, find the mountpoint of its source file system. + * Replace @path with the path of the mountpoint in the parent mount. + * Up is towards /. + * + * Return 1 if we went up a level and 0 if we were already at the + * root. + */ int follow_up(struct path *path) { struct mount *mnt = real_mount(path->mnt); struct mount *parent; struct dentry *mountpoint; - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); parent = mnt->mnt_parent; if (&parent->mnt == path->mnt) { - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return 0; } mntget(&parent->mnt); mountpoint = dget(mnt->mnt_mountpoint); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); dput(path->dentry); path->dentry = mountpoint; mntput(path->mnt); @@ -960,7 +970,7 @@ static int follow_dotdot_rcu(struct nameidata *nd) if (!(nd->flags & LOOKUP_ROOT)) nd->root.mnt = NULL; rcu_read_unlock(); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return -ECHILD; } @@ -1265,7 +1275,7 @@ static void terminate_walk(struct nameidata *nd) if (!(nd->flags & LOOKUP_ROOT)) nd->root.mnt = NULL; rcu_read_unlock(); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); } } @@ -1618,7 +1628,7 @@ static int path_init(int dfd, const char *name, unsigned int flags, nd->path = nd->root; nd->inode = inode; if (flags & LOOKUP_RCU) { - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); rcu_read_lock(); nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq); } else { @@ -1631,7 +1641,7 @@ static int path_init(int dfd, const char *name, unsigned int flags, if (*name=='/') { if (flags & LOOKUP_RCU) { - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); rcu_read_lock(); set_root_rcu(nd); } else { @@ -1644,7 +1654,7 @@ static int path_init(int dfd, const char *name, unsigned int flags, struct fs_struct *fs = current->fs; unsigned seq; - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); rcu_read_lock(); do { @@ -1680,7 +1690,7 @@ static int path_init(int dfd, const char *name, unsigned int flags, if (fput_needed) *fp = file; nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq); - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); rcu_read_lock(); } else { path_get(&file->f_path); @@ -1760,6 +1770,16 @@ static int path_lookupat(int dfd, const char *name, } } + if (!err) { + struct super_block *sb = nd->inode->i_sb; + if (sb->s_flags & MS_RDONLY) { + if (d_is_su(nd->path.dentry) && !su_visible()) { + path_put(&nd->path); + err = -ENOENT; + } + } + } + if (base) fput(base); diff --git a/fs/namespace.c b/fs/namespace.c index e6081996c9a..4e89d3f0f0d 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include /* get_fs_root et.al. */ #include /* fsnotify_vfsmount_delete */ #include +#include #include "pnode.h" #include "internal.h" @@ -397,7 +399,7 @@ static int mnt_make_readonly(struct mount *mnt) { int ret = 0; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); mnt->mnt.mnt_flags |= MNT_WRITE_HOLD; /* * After storing MNT_WRITE_HOLD, we'll read the counters. This store @@ -431,15 +433,15 @@ static int mnt_make_readonly(struct mount *mnt) */ smp_wmb(); mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); return ret; } static void __mnt_unmake_readonly(struct mount *mnt) { - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); mnt->mnt.mnt_flags &= ~MNT_READONLY; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } int sb_prepare_remount_readonly(struct super_block *sb) @@ -451,7 +453,7 @@ int sb_prepare_remount_readonly(struct super_block *sb) if (atomic_long_read(&sb->s_remove_count)) return -EBUSY; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); list_for_each_entry(mnt, &sb->s_mounts, mnt_instance) { if (!(mnt->mnt.mnt_flags & MNT_READONLY)) { mnt->mnt.mnt_flags |= MNT_WRITE_HOLD; @@ -473,7 +475,7 @@ int sb_prepare_remount_readonly(struct super_block *sb) if (mnt->mnt.mnt_flags & MNT_WRITE_HOLD) mnt->mnt.mnt_flags &= ~MNT_WRITE_HOLD; } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); return err; } @@ -515,21 +517,33 @@ struct mount *__lookup_mnt(struct vfsmount *mnt, struct dentry *dentry, } /* - * lookup_mnt increments the ref count before returning - * the vfsmount struct. + * lookup_mnt - Return the first child mount mounted at path + * + * "First" means first mounted chronologically. If you create the + * following mounts: + * + * mount /dev/sda1 /mnt + * mount /dev/sda2 /mnt + * mount /dev/sda3 /mnt + * + * Then lookup_mnt() on the base /mnt dentry in the root mount will + * return successively the root dentry and vfsmount of /dev/sda1, then + * /dev/sda2, then /dev/sda3, then NULL. + * + * lookup_mnt takes a reference to the found vfsmount. */ struct vfsmount *lookup_mnt(struct path *path) { struct mount *child_mnt; - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); child_mnt = __lookup_mnt(path->mnt, path->dentry, 1); if (child_mnt) { mnt_add_count(child_mnt, 1); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return &child_mnt->mnt; } else { - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return NULL; } } @@ -621,21 +635,6 @@ static void attach_mnt(struct mount *mnt, struct path *path) list_add_tail(&mnt->mnt_child, &real_mount(path->mnt)->mnt_mounts); } -static inline void __mnt_make_longterm(struct mount *mnt) -{ -#ifdef CONFIG_SMP - atomic_inc(&mnt->mnt_longterm); -#endif -} - -/* needs vfsmount lock for write */ -static inline void __mnt_make_shortterm(struct mount *mnt) -{ -#ifdef CONFIG_SMP - atomic_dec(&mnt->mnt_longterm); -#endif -} - /* * vfsmount lock must be held for write */ @@ -649,10 +648,8 @@ static void commit_tree(struct mount *mnt) BUG_ON(parent == mnt); list_add_tail(&head, &mnt->mnt_list); - list_for_each_entry(m, &head, mnt_list) { + list_for_each_entry(m, &head, mnt_list) m->mnt_ns = n; - __mnt_make_longterm(m); - } list_splice(&head, n->list.prev); @@ -714,9 +711,9 @@ vfs_kern_mount(struct file_system_type *type, int flags, const char *name, void mnt->mnt.mnt_sb = root->d_sb; mnt->mnt_mountpoint = mnt->mnt.mnt_root; mnt->mnt_parent = mnt; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); list_add_tail(&mnt->mnt_instance, &root->d_sb->s_mounts); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); return &mnt->mnt; } EXPORT_SYMBOL_GPL(vfs_kern_mount); @@ -725,56 +722,61 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root, int flag) { struct super_block *sb = old->mnt.mnt_sb; - struct mount *mnt = alloc_vfsmnt(old->mnt_devname); + struct mount *mnt; + int err; - if (mnt) { - if (flag & (CL_SLAVE | CL_PRIVATE)) - mnt->mnt_group_id = 0; /* not a peer of original */ - else - mnt->mnt_group_id = old->mnt_group_id; - - if ((flag & CL_MAKE_SHARED) && !mnt->mnt_group_id) { - int err = mnt_alloc_group_id(mnt); - if (err) - goto out_free; - } + mnt = alloc_vfsmnt(old->mnt_devname); + if (!mnt) + return ERR_PTR(-ENOMEM); - mnt->mnt.mnt_flags = old->mnt.mnt_flags & ~MNT_WRITE_HOLD; - atomic_inc(&sb->s_active); - mnt->mnt.mnt_sb = sb; - mnt->mnt.mnt_root = dget(root); - mnt->mnt_mountpoint = mnt->mnt.mnt_root; - mnt->mnt_parent = mnt; - br_write_lock(vfsmount_lock); - list_add_tail(&mnt->mnt_instance, &sb->s_mounts); - br_write_unlock(vfsmount_lock); - - if (flag & CL_SLAVE) { - list_add(&mnt->mnt_slave, &old->mnt_slave_list); - mnt->mnt_master = old; - CLEAR_MNT_SHARED(mnt); - } else if (!(flag & CL_PRIVATE)) { - if ((flag & CL_MAKE_SHARED) || IS_MNT_SHARED(old)) - list_add(&mnt->mnt_share, &old->mnt_share); - if (IS_MNT_SLAVE(old)) - list_add(&mnt->mnt_slave, &old->mnt_slave); - mnt->mnt_master = old->mnt_master; - } - if (flag & CL_MAKE_SHARED) - set_mnt_shared(mnt); - - /* stick the duplicate mount on the same expiry list - * as the original if that was on one */ - if (flag & CL_EXPIRE) { - if (!list_empty(&old->mnt_expire)) - list_add(&mnt->mnt_expire, &old->mnt_expire); - } + if (flag & (CL_SLAVE | CL_PRIVATE | CL_SHARED_TO_SLAVE)) + mnt->mnt_group_id = 0; /* not a peer of original */ + else + mnt->mnt_group_id = old->mnt_group_id; + + if ((flag & CL_MAKE_SHARED) && !mnt->mnt_group_id) { + err = mnt_alloc_group_id(mnt); + if (err) + goto out_free; + } + + mnt->mnt.mnt_flags = old->mnt.mnt_flags & ~MNT_WRITE_HOLD; + atomic_inc(&sb->s_active); + mnt->mnt.mnt_sb = sb; + mnt->mnt.mnt_root = dget(root); + mnt->mnt_mountpoint = mnt->mnt.mnt_root; + mnt->mnt_parent = mnt; + br_write_lock(&vfsmount_lock); + list_add_tail(&mnt->mnt_instance, &sb->s_mounts); + br_write_unlock(&vfsmount_lock); + + if ((flag & CL_SLAVE) || + ((flag & CL_SHARED_TO_SLAVE) && IS_MNT_SHARED(old))) { + list_add(&mnt->mnt_slave, &old->mnt_slave_list); + mnt->mnt_master = old; + CLEAR_MNT_SHARED(mnt); + } else if (!(flag & CL_PRIVATE)) { + if ((flag & CL_MAKE_SHARED) || IS_MNT_SHARED(old)) + list_add(&mnt->mnt_share, &old->mnt_share); + if (IS_MNT_SLAVE(old)) + list_add(&mnt->mnt_slave, &old->mnt_slave); + mnt->mnt_master = old->mnt_master; + } + if (flag & CL_MAKE_SHARED) + set_mnt_shared(mnt); + + /* stick the duplicate mount on the same expiry list + * as the original if that was on one */ + if (flag & CL_EXPIRE) { + if (!list_empty(&old->mnt_expire)) + list_add(&mnt->mnt_expire, &old->mnt_expire); } + return mnt; out_free: free_vfsmnt(mnt); - return NULL; + return ERR_PTR(err); } static inline void mntfree(struct mount *mnt) @@ -803,35 +805,37 @@ static void mntput_no_expire(struct mount *mnt) { put_again: #ifdef CONFIG_SMP - br_read_lock(vfsmount_lock); - if (likely(atomic_read(&mnt->mnt_longterm))) { + br_read_lock(&vfsmount_lock); + if (likely(mnt->mnt_ns)) { + /* shouldn't be the last one */ mnt_add_count(mnt, -1); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return; } - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); mnt_add_count(mnt, -1); if (mnt_get_count(mnt)) { - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); return; } #else mnt_add_count(mnt, -1); if (likely(mnt_get_count(mnt))) return; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); #endif if (unlikely(mnt->mnt_pinned)) { mnt_add_count(mnt, mnt->mnt_pinned + 1); mnt->mnt_pinned = 0; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); acct_auto_close_mnt(&mnt->mnt); goto put_again; } + list_del(&mnt->mnt_instance); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); mntfree(mnt); } @@ -857,21 +861,21 @@ EXPORT_SYMBOL(mntget); void mnt_pin(struct vfsmount *mnt) { - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); real_mount(mnt)->mnt_pinned++; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } EXPORT_SYMBOL(mnt_pin); void mnt_unpin(struct vfsmount *m) { struct mount *mnt = real_mount(m); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); if (mnt->mnt_pinned) { mnt_add_count(mnt, 1); mnt->mnt_pinned--; } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } EXPORT_SYMBOL(mnt_unpin); @@ -938,7 +942,7 @@ EXPORT_SYMBOL(replace_mount_options); /* iterator; we want it to have access to namespace_sem, thus here... */ static void *m_start(struct seq_file *m, loff_t *pos) { - struct proc_mounts *p = container_of(m, struct proc_mounts, m); + struct proc_mounts *p = proc_mounts(m); down_read(&namespace_sem); return seq_list_start(&p->ns->list, *pos); @@ -946,7 +950,7 @@ static void *m_start(struct seq_file *m, loff_t *pos) static void *m_next(struct seq_file *m, void *v, loff_t *pos) { - struct proc_mounts *p = container_of(m, struct proc_mounts, m); + struct proc_mounts *p = proc_mounts(m); return seq_list_next(v, &p->ns->list, pos); } @@ -958,7 +962,7 @@ static void m_stop(struct seq_file *m, void *v) static int m_show(struct seq_file *m, void *v) { - struct proc_mounts *p = container_of(m, struct proc_mounts, m); + struct proc_mounts *p = proc_mounts(m); struct mount *r = list_entry(v, struct mount, mnt_list); return p->show(m, &r->mnt); } @@ -988,12 +992,12 @@ int may_umount_tree(struct vfsmount *m) BUG_ON(!m); /* write lock needed for mnt_get_count */ - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); for (p = mnt; p; p = next_mnt(p, mnt)) { actual_refs += mnt_get_count(p); minimum_refs += 2; } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); if (actual_refs > minimum_refs) return 0; @@ -1020,10 +1024,10 @@ int may_umount(struct vfsmount *mnt) { int ret = 1; down_read(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); if (propagate_mount_busy(real_mount(mnt), 2)) ret = 0; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_read(&namespace_sem); return ret; } @@ -1040,13 +1044,13 @@ void release_mounts(struct list_head *head) struct dentry *dentry; struct mount *m; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); dentry = mnt->mnt_mountpoint; m = mnt->mnt_parent; mnt->mnt_mountpoint = mnt->mnt.mnt_root; mnt->mnt_parent = mnt; m->mnt_ghosts--; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); dput(dentry); mntput(&m->mnt); } @@ -1074,7 +1078,6 @@ void umount_tree(struct mount *mnt, int propagate, struct list_head *kill) list_del_init(&p->mnt_list); __touch_mnt_namespace(p->mnt_ns); p->mnt_ns = NULL; - __mnt_make_shortterm(p); list_del_init(&p->mnt_child); if (mnt_has_parent(p)) { p->mnt_parent->mnt_ghosts++; @@ -1112,12 +1115,12 @@ static int do_umount(struct mount *mnt, int flags) * probably don't strictly need the lock here if we examined * all race cases, but it's a slowpath. */ - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); if (mnt_get_count(mnt) != 2) { - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); return -EBUSY; } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); if (!xchg(&mnt->mnt_expiry_mark, 1)) return -EAGAIN; @@ -1159,7 +1162,7 @@ static int do_umount(struct mount *mnt, int flags) } down_write(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); event++; if (!(flags & MNT_DETACH)) @@ -1171,7 +1174,7 @@ static int do_umount(struct mount *mnt, int flags) umount_tree(mnt, 1, &umount_list); retval = 0; } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_write(&namespace_sem); release_mounts(&umount_list); return retval; @@ -1209,7 +1212,7 @@ SYSCALL_DEFINE2(umount, char __user *, name, int, flags) goto dput_and_out; retval = -EPERM; - if (!capable(CAP_SYS_ADMIN)) + if (!ns_capable(mnt->mnt_ns->user_ns, CAP_SYS_ADMIN)) goto dput_and_out; retval = do_umount(mnt, flags); @@ -1235,7 +1238,7 @@ SYSCALL_DEFINE1(oldumount, char __user *, name) static int mount_is_safe(struct path *path) { - if (capable(CAP_SYS_ADMIN)) + if (ns_capable(real_mount(path->mnt)->mnt_ns->user_ns, CAP_SYS_ADMIN)) return 0; return -EPERM; #ifdef notyet @@ -1251,6 +1254,26 @@ static int mount_is_safe(struct path *path) #endif } +static bool mnt_ns_loop(struct path *path) +{ + /* Could bind mounting the mount namespace inode cause a + * mount namespace loop? + */ + struct inode *inode = path->dentry->d_inode; + struct proc_inode *ei; + struct mnt_namespace *mnt_ns; + + if (!proc_ns_inode(inode)) + return false; + + ei = PROC_I(inode); + if (ei->ns_ops != &mntns_operations) + return false; + + mnt_ns = ei->ns; + return current->nsproxy->mnt_ns->seq >= mnt_ns->seq; +} + struct mount *copy_tree(struct mount *mnt, struct dentry *dentry, int flag) { @@ -1258,11 +1281,12 @@ struct mount *copy_tree(struct mount *mnt, struct dentry *dentry, struct path path; if (!(flag & CL_COPY_ALL) && IS_MNT_UNBINDABLE(mnt)) - return NULL; + return ERR_PTR(-EINVAL); res = q = clone_mnt(mnt, dentry, flag); - if (!q) - goto Enomem; + if (IS_ERR(q)) + return q; + q->mnt_mountpoint = mnt->mnt_mountpoint; p = mnt; @@ -1284,43 +1308,50 @@ struct mount *copy_tree(struct mount *mnt, struct dentry *dentry, path.mnt = &q->mnt; path.dentry = p->mnt_mountpoint; q = clone_mnt(p, p->mnt.mnt_root, flag); - if (!q) - goto Enomem; - br_write_lock(vfsmount_lock); + if (IS_ERR(q)) + goto out; + br_write_lock(&vfsmount_lock); list_add_tail(&q->mnt_list, &res->mnt_list); attach_mnt(q, &path); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } } return res; -Enomem: +out: if (res) { LIST_HEAD(umount_list); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); umount_tree(res, 0, &umount_list); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); release_mounts(&umount_list); } - return NULL; + return q; } +/* Caller should check returned pointer for errors */ + struct vfsmount *collect_mounts(struct path *path) { struct mount *tree; down_write(&namespace_sem); - tree = copy_tree(real_mount(path->mnt), path->dentry, - CL_COPY_ALL | CL_PRIVATE); + if (!check_mnt(real_mount(path->mnt))) + tree = ERR_PTR(-EINVAL); + else + tree = copy_tree(real_mount(path->mnt), path->dentry, + CL_COPY_ALL | CL_PRIVATE); up_write(&namespace_sem); - return tree ? &tree->mnt : NULL; + if (IS_ERR(tree)) + return NULL; + return &tree->mnt; } void drop_collected_mounts(struct vfsmount *mnt) { LIST_HEAD(umount_list); down_write(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); umount_tree(real_mount(mnt), 0, &umount_list); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_write(&namespace_sem); release_mounts(&umount_list); } @@ -1448,7 +1479,7 @@ static int attach_recursive_mnt(struct mount *source_mnt, if (err) goto out_cleanup_ids; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); if (IS_MNT_SHARED(dest_mnt)) { for (p = source_mnt; p; p = next_mnt(p, source_mnt)) @@ -1467,7 +1498,7 @@ static int attach_recursive_mnt(struct mount *source_mnt, list_del_init(&child->mnt_hash); commit_tree(child); } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); return 0; @@ -1548,7 +1579,7 @@ static int do_change_type(struct path *path, int flag) int type; int err = 0; - if (!capable(CAP_SYS_ADMIN)) + if (!ns_capable(mnt->mnt_ns->user_ns, CAP_SYS_ADMIN)) return -EPERM; if (path->dentry != path->mnt->mnt_root) @@ -1565,10 +1596,10 @@ static int do_change_type(struct path *path, int flag) goto out_unlock; } - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); for (m = mnt; m; m = (recurse ? next_mnt(m, mnt) : NULL)) change_mnt_propagation(m, type); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); out_unlock: up_write(&namespace_sem); @@ -1578,7 +1609,7 @@ static int do_change_type(struct path *path, int flag) /* * do loopback mount. */ -static int do_loopback(struct path *path, char *old_name, +static int do_loopback(struct path *path, const char *old_name, int recurse) { LIST_HEAD(umount_list); @@ -1593,6 +1624,10 @@ static int do_loopback(struct path *path, char *old_name, if (err) return err; + err = -EINVAL; + if (mnt_ns_loop(&old_path)) + goto out; + err = lock_mount(path); if (err) goto out; @@ -1606,20 +1641,21 @@ static int do_loopback(struct path *path, char *old_name, if (!check_mnt(real_mount(path->mnt)) || !check_mnt(old)) goto out2; - err = -ENOMEM; if (recurse) mnt = copy_tree(old, old_path.dentry, 0); else mnt = clone_mnt(old, old_path.dentry, 0); - if (!mnt) - goto out2; + if (IS_ERR(mnt)) { + err = PTR_ERR(mnt); + goto out; + } err = graft_tree(mnt, path); if (err) { - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); umount_tree(mnt, 0, &umount_list); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } out2: unlock_mount(path); @@ -1677,16 +1713,16 @@ static int do_remount(struct path *path, int flags, int mnt_flags, else err = do_remount_sb(sb, flags, data, 0); if (!err) { - br_write_lock(vfsmount_lock); - mnt_flags |= mnt->mnt.mnt_flags & MNT_PROPAGATION_MASK; + br_write_lock(&vfsmount_lock); + mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK; mnt->mnt.mnt_flags = mnt_flags; - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } up_write(&sb->s_umount); if (!err) { - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); touch_mnt_namespace(mnt->mnt_ns); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); } return err; } @@ -1701,13 +1737,13 @@ static inline int tree_contains_unbindable(struct mount *mnt) return 0; } -static int do_move_mount(struct path *path, char *old_name) +static int do_move_mount(struct path *path, const char *old_name) { struct path old_path, parent_path; struct mount *p; struct mount *old; int err = 0; - if (!capable(CAP_SYS_ADMIN)) + if (!ns_capable(real_mount(path->mnt)->mnt_ns->user_ns, CAP_SYS_ADMIN)) return -EPERM; if (!old_name || !*old_name) return -EINVAL; @@ -1794,21 +1830,6 @@ static struct vfsmount *fs_set_subtype(struct vfsmount *mnt, const char *fstype) return ERR_PTR(err); } -static struct vfsmount * -do_kern_mount(const char *fstype, int flags, const char *name, void *data) -{ - struct file_system_type *type = get_fs_type(fstype); - struct vfsmount *mnt; - if (!type) - return ERR_PTR(-ENODEV); - mnt = vfs_kern_mount(type, flags, name, data); - if (!IS_ERR(mnt) && (type->fs_flags & FS_HAS_SUBTYPE) && - !mnt->mnt_sb->s_subtype) - mnt = fs_set_subtype(mnt, fstype); - put_filesystem(type); - return mnt; -} - /* * add a mount into a namespace's mount tree */ @@ -1823,8 +1844,14 @@ static int do_add_mount(struct mount *newmnt, struct path *path, int mnt_flags) return err; err = -EINVAL; - if (!(mnt_flags & MNT_SHRINKABLE) && !check_mnt(real_mount(path->mnt))) - goto unlock; + if (unlikely(!check_mnt(real_mount(path->mnt)))) { + /* that's acceptable only for automounts done in private ns */ + if (!(mnt_flags & MNT_SHRINKABLE)) + goto unlock; + /* ... and for those we'd better have mountpoint still alive */ + if (!real_mount(path->mnt)->mnt_ns) + goto unlock; + } /* Refuse the same filesystem on the same mount point */ err = -EBUSY; @@ -1848,20 +1875,46 @@ static int do_add_mount(struct mount *newmnt, struct path *path, int mnt_flags) * create a new mount for userspace and request it to be added into the * namespace's tree */ -static int do_new_mount(struct path *path, char *type, int flags, - int mnt_flags, char *name, void *data) +static int do_new_mount(struct path *path, const char *fstype, int flags, + int mnt_flags, const char *name, void *data) { + struct file_system_type *type; + struct user_namespace *user_ns; struct vfsmount *mnt; int err; - if (!type) + if (!fstype) return -EINVAL; /* we need capabilities... */ - if (!capable(CAP_SYS_ADMIN)) + user_ns = real_mount(path->mnt)->mnt_ns->user_ns; + if (!ns_capable(user_ns, CAP_SYS_ADMIN)) return -EPERM; - mnt = do_kern_mount(type, flags, name, data); + type = get_fs_type(fstype); + if (!type) + return -ENODEV; + + if (user_ns != &init_user_ns) { + if (!(type->fs_flags & FS_USERNS_MOUNT)) { + put_filesystem(type); + return -EPERM; + } + /* Only in special cases allow devices from mounts + * created outside the initial user namespace. + */ + if (!(type->fs_flags & FS_USERNS_DEV_MOUNT)) { + flags |= MS_NODEV; + mnt_flags |= MNT_NODEV; + } + } + + mnt = vfs_kern_mount(type, flags, name, data); + if (!IS_ERR(mnt) && (type->fs_flags & FS_HAS_SUBTYPE) && + !mnt->mnt_sb->s_subtype) + mnt = fs_set_subtype(mnt, fstype); + + put_filesystem(type); if (IS_ERR(mnt)) return PTR_ERR(mnt); @@ -1893,9 +1946,9 @@ int finish_automount(struct vfsmount *m, struct path *path) /* remove m from any expiration list it may be on */ if (!list_empty(&mnt->mnt_expire)) { down_write(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); list_del_init(&mnt->mnt_expire); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_write(&namespace_sem); } mntput(m); @@ -1911,11 +1964,11 @@ int finish_automount(struct vfsmount *m, struct path *path) void mnt_set_expiry(struct vfsmount *mnt, struct list_head *expiry_list) { down_write(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); list_add_tail(&real_mount(mnt)->mnt_expire, expiry_list); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_write(&namespace_sem); } EXPORT_SYMBOL(mnt_set_expiry); @@ -1935,7 +1988,7 @@ void mark_mounts_for_expiry(struct list_head *mounts) return; down_write(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); /* extract from the expiration list every vfsmount that matches the * following criteria: @@ -1954,7 +2007,7 @@ void mark_mounts_for_expiry(struct list_head *mounts) touch_mnt_namespace(mnt->mnt_ns); umount_tree(mnt, 1, &umounts); } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_write(&namespace_sem); release_mounts(&umounts); @@ -2122,8 +2175,8 @@ int copy_mount_string(const void __user *data, char **where) * Therefore, if this magic number is present, it carries no information * and must be discarded. */ -long do_mount(char *dev_name, char *dir_name, char *type_page, - unsigned long flags, void *data_page) +long do_mount(const char *dev_name, const char *dir_name, + const char *type_page, unsigned long flags, void *data_page) { struct path path; int retval = 0; @@ -2192,67 +2245,78 @@ long do_mount(char *dev_name, char *dir_name, char *type_page, return retval; } -static struct mnt_namespace *alloc_mnt_ns(void) +static void free_mnt_ns(struct mnt_namespace *ns) +{ + proc_free_inum(ns->proc_inum); + put_user_ns(ns->user_ns); + kfree(ns); +} + +/* + * Assign a sequence number so we can detect when we attempt to bind + * mount a reference to an older mount namespace into the current + * mount namespace, preventing reference counting loops. A 64bit + * number incrementing at 10Ghz will take 12,427 years to wrap which + * is effectively never, so we can ignore the possibility. + */ +static atomic64_t mnt_ns_seq = ATOMIC64_INIT(1); + +static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns) { struct mnt_namespace *new_ns; + int ret; new_ns = kmalloc(sizeof(struct mnt_namespace), GFP_KERNEL); if (!new_ns) return ERR_PTR(-ENOMEM); + ret = proc_alloc_inum(&new_ns->proc_inum); + if (ret) { + kfree(new_ns); + return ERR_PTR(ret); + } + new_ns->seq = atomic64_add_return(1, &mnt_ns_seq); atomic_set(&new_ns->count, 1); new_ns->root = NULL; INIT_LIST_HEAD(&new_ns->list); init_waitqueue_head(&new_ns->poll); new_ns->event = 0; + new_ns->user_ns = get_user_ns(user_ns); return new_ns; } -void mnt_make_longterm(struct vfsmount *mnt) -{ - __mnt_make_longterm(real_mount(mnt)); -} - -void mnt_make_shortterm(struct vfsmount *m) -{ -#ifdef CONFIG_SMP - struct mount *mnt = real_mount(m); - if (atomic_add_unless(&mnt->mnt_longterm, -1, 1)) - return; - br_write_lock(vfsmount_lock); - atomic_dec(&mnt->mnt_longterm); - br_write_unlock(vfsmount_lock); -#endif -} - /* * Allocate a new namespace structure and populate it with contents * copied from the namespace of the passed in task structure. */ static struct mnt_namespace *dup_mnt_ns(struct mnt_namespace *mnt_ns, - struct fs_struct *fs) + struct user_namespace *user_ns, struct fs_struct *fs) { struct mnt_namespace *new_ns; struct vfsmount *rootmnt = NULL, *pwdmnt = NULL; struct mount *p, *q; struct mount *old = mnt_ns->root; struct mount *new; + int copy_flags; - new_ns = alloc_mnt_ns(); + new_ns = alloc_mnt_ns(user_ns); if (IS_ERR(new_ns)) return new_ns; down_write(&namespace_sem); /* First pass: copy the tree topology */ - new = copy_tree(old, old->mnt.mnt_root, CL_COPY_ALL | CL_EXPIRE); - if (!new) { + copy_flags = CL_COPY_ALL | CL_EXPIRE; + if (user_ns != mnt_ns->user_ns) + copy_flags |= CL_SHARED_TO_SLAVE; + new = copy_tree(old, old->mnt.mnt_root, copy_flags); + if (IS_ERR(new)) { up_write(&namespace_sem); - kfree(new_ns); - return ERR_PTR(-ENOMEM); + free_mnt_ns(new_ns); + return ERR_CAST(new); } new_ns->root = new; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); list_add_tail(&new_ns->list, &new->mnt_list); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); /* * Second pass: switch the tsk->fs->* elements and mark new vfsmounts @@ -2263,18 +2327,13 @@ static struct mnt_namespace *dup_mnt_ns(struct mnt_namespace *mnt_ns, q = new; while (p) { q->mnt_ns = new_ns; - __mnt_make_longterm(q); if (fs) { if (&p->mnt == fs->root.mnt) { fs->root.mnt = mntget(&q->mnt); - __mnt_make_longterm(q); - mnt_make_shortterm(&p->mnt); rootmnt = &p->mnt; } if (&p->mnt == fs->pwd.mnt) { fs->pwd.mnt = mntget(&q->mnt); - __mnt_make_longterm(q); - mnt_make_shortterm(&p->mnt); pwdmnt = &p->mnt; } } @@ -2292,7 +2351,7 @@ static struct mnt_namespace *dup_mnt_ns(struct mnt_namespace *mnt_ns, } struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns, - struct fs_struct *new_fs) + struct user_namespace *user_ns, struct fs_struct *new_fs) { struct mnt_namespace *new_ns; @@ -2302,7 +2361,7 @@ struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns, if (!(flags & CLONE_NEWNS)) return ns; - new_ns = dup_mnt_ns(ns, new_fs); + new_ns = dup_mnt_ns(ns, user_ns, new_fs); put_mnt_ns(ns); return new_ns; @@ -2314,11 +2373,10 @@ struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns, */ static struct mnt_namespace *create_mnt_ns(struct vfsmount *m) { - struct mnt_namespace *new_ns = alloc_mnt_ns(); + struct mnt_namespace *new_ns = alloc_mnt_ns(&init_user_ns); if (!IS_ERR(new_ns)) { struct mount *mnt = real_mount(m); mnt->mnt_ns = new_ns; - __mnt_make_longterm(mnt); new_ns->root = mnt; list_add(&new_ns->list, &mnt->mnt_list); } else { @@ -2416,9 +2474,9 @@ bool is_path_reachable(struct mount *mnt, struct dentry *dentry, int path_is_under(struct path *path1, struct path *path2) { int res; - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); res = is_path_reachable(real_mount(path1->mnt), path1->dentry, path2); - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return res; } EXPORT_SYMBOL(path_is_under); @@ -2455,7 +2513,7 @@ SYSCALL_DEFINE2(pivot_root, const char __user *, new_root, struct mount *new_mnt, *root_mnt; int error; - if (!capable(CAP_SYS_ADMIN)) + if (!ns_capable(current->nsproxy->mnt_ns->user_ns, CAP_SYS_ADMIN)) return -EPERM; error = user_path_dir(new_root, &new); @@ -2505,7 +2563,7 @@ SYSCALL_DEFINE2(pivot_root, const char __user *, new_root, /* make sure we can reach put_old from new_root */ if (!is_path_reachable(real_mount(old.mnt), old.dentry, &new)) goto out4; - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); detach_mnt(new_mnt, &parent_path); detach_mnt(root_mnt, &root_parent); /* mount old root on put_old */ @@ -2513,7 +2571,7 @@ SYSCALL_DEFINE2(pivot_root, const char __user *, new_root, /* mount new_root on / */ attach_mnt(new_mnt, &root_parent); touch_mnt_namespace(current->nsproxy->mnt_ns); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); chroot_fs_refs(&root, &new); error = 0; out4: @@ -2537,8 +2595,13 @@ static void __init init_mount_tree(void) struct vfsmount *mnt; struct mnt_namespace *ns; struct path root; + struct file_system_type *type; - mnt = do_kern_mount("rootfs", 0, "rootfs", NULL); + type = get_fs_type("rootfs"); + if (!type) + panic("Can't find rootfs type"); + mnt = vfs_kern_mount(type, 0, "rootfs", NULL); + put_filesystem(type); if (IS_ERR(mnt)) panic("Can't create rootfs"); @@ -2576,7 +2639,7 @@ void __init mnt_init(void) for (u = 0; u < HASH_SIZE; u++) INIT_LIST_HEAD(&mount_hashtable[u]); - br_lock_init(vfsmount_lock); + br_lock_init(&vfsmount_lock); err = sysfs_init(); if (err) @@ -2596,12 +2659,12 @@ void put_mnt_ns(struct mnt_namespace *ns) if (!atomic_dec_and_test(&ns->count)) return; down_write(&namespace_sem); - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); umount_tree(ns->root, 0, &umount_list); - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); up_write(&namespace_sem); release_mounts(&umount_list); - kfree(ns); + free_mnt_ns(ns); } struct vfsmount *kern_mount_data(struct file_system_type *type, void *data) @@ -2613,7 +2676,7 @@ struct vfsmount *kern_mount_data(struct file_system_type *type, void *data) * it is a longterm mount, don't release mnt until * we unmount before file sys is unregistered */ - mnt_make_longterm(mnt); + real_mount(mnt)->mnt_ns = MNT_NS_INTERNAL; } return mnt; } @@ -2623,7 +2686,9 @@ void kern_unmount(struct vfsmount *mnt) { /* release long term mount so mount point can be released */ if (!IS_ERR_OR_NULL(mnt)) { - mnt_make_shortterm(mnt); + br_write_lock(&vfsmount_lock); + real_mount(mnt)->mnt_ns = NULL; + br_write_unlock(&vfsmount_lock); mntput(mnt); } } @@ -2633,3 +2698,71 @@ bool our_mnt(struct vfsmount *mnt) { return check_mnt(real_mount(mnt)); } + +static void *mntns_get(struct task_struct *task) +{ + struct mnt_namespace *ns = NULL; + struct nsproxy *nsproxy; + + rcu_read_lock(); + nsproxy = task_nsproxy(task); + if (nsproxy) { + ns = nsproxy->mnt_ns; + get_mnt_ns(ns); + } + rcu_read_unlock(); + + return ns; +} + +static void mntns_put(void *ns) +{ + put_mnt_ns(ns); +} + +static int mntns_install(struct nsproxy *nsproxy, void *ns) +{ + struct fs_struct *fs = current->fs; + struct mnt_namespace *mnt_ns = ns; + struct path root; + + if (!ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN) || + !nsown_capable(CAP_SYS_CHROOT)) + return -EINVAL; + + if (fs->users != 1) + return -EINVAL; + + get_mnt_ns(mnt_ns); + put_mnt_ns(nsproxy->mnt_ns); + nsproxy->mnt_ns = mnt_ns; + + /* Find the root */ + root.mnt = &mnt_ns->root->mnt; + root.dentry = mnt_ns->root->mnt.mnt_root; + path_get(&root); + while(d_mountpoint(root.dentry) && follow_down_one(&root)) + ; + + /* Update the pwd and root */ + set_fs_pwd(fs, &root); + set_fs_root(fs, &root); + + path_put(&root); + return 0; +} + +static unsigned int mntns_inum(void *ns) +{ + struct mnt_namespace *mnt_ns = ns; + return mnt_ns->proc_inum; +} + +const struct proc_ns_operations mntns_operations = { + .name = "mnt", + .type = CLONE_NEWNS, + .get = mntns_get, + .put = mntns_put, + .install = mntns_install, + .inum = mntns_inum, +}; diff --git a/fs/ncpfs/file.c b/fs/ncpfs/file.c index 3ff5fcc1528..122e260247f 100644 --- a/fs/ncpfs/file.c +++ b/fs/ncpfs/file.c @@ -221,6 +221,10 @@ ncp_file_write(struct file *file, const char __user *buf, size_t count, loff_t * already_written = 0; + errno = file_update_time(file); + if (errno) + goto outrel; + bouncebuffer = vmalloc(bufsize); if (!bouncebuffer) { errno = -EIO; /* -ENOMEM */ @@ -252,8 +256,6 @@ ncp_file_write(struct file *file, const char __user *buf, size_t count, loff_t * } vfree(bouncebuffer); - file_update_time(file); - *ppos = pos; if (pos > i_size_read(inode)) { diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 8639169221c..7389d2d5e51 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -2096,7 +2096,9 @@ static ssize_t ntfs_file_aio_write_nolock(struct kiocb *iocb, err = file_remove_suid(file); if (err) goto out; - file_update_time(file); + err = file_update_time(file); + if (err) + goto out; written = ntfs_file_buffered_write(iocb, iov, nr_segs, pos, ppos, count); out: diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 061591a3ab0..e87b7b45fde 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -2466,9 +2466,7 @@ static ssize_t ocfs2_file_splice_write(struct pipe_inode_info *pipe, struct address_space *mapping = out->f_mapping; struct inode *inode = mapping->host; struct splice_desc sd = { - .total_len = len, .flags = flags, - .pos = *ppos, .u.file = out, }; @@ -2478,6 +2476,12 @@ static ssize_t ocfs2_file_splice_write(struct pipe_inode_info *pipe, out->f_path.dentry->d_name.len, out->f_path.dentry->d_name.name, len); + ret = generic_write_checks(out, ppos, &len, 0); + if (ret) + return ret; + sd.total_len = len; + sd.pos = *ppos; + if (pipe->inode) mutex_lock_nested(&pipe->inode->i_mutex, I_MUTEX_PARENT); diff --git a/fs/open.c b/fs/open.c index 5720854156d..c4b8b523f0d 100644 --- a/fs/open.c +++ b/fs/open.c @@ -1041,9 +1041,12 @@ SYSCALL_DEFINE2(creat, const char __user *, pathname, umode_t, mode) int filp_close(struct file *filp, fl_owner_t id) { int retval = 0; + long ret; - if (!file_count(filp)) { - printk(KERN_ERR "VFS: Close: file count is 0\n"); + ret = file_count(filp); + if (ret <= 0) { + printk(KERN_ERR "VFS: Close: file count is %ld\n", ret); + WARN_ON(ret < 0); return 0; } diff --git a/fs/pipe.c b/fs/pipe.c index fec5e4ad071..c1ddb284357 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -36,6 +36,12 @@ unsigned int pipe_max_size = 1048576; */ unsigned int pipe_min_size = PAGE_SIZE; +/* Maximum allocatable pages per user. Hard limit is unset by default, soft + * matches default values. + */ +unsigned long pipe_user_pages_hard; +unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; + /* * We use a start+len construction, which provides full use of the * allocated memory. @@ -104,25 +110,27 @@ void pipe_wait(struct pipe_inode_info *pipe) } static int -pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, - int atomic) +pipe_iov_copy_from_user(void *addr, int *offset, struct iovec *iov, + size_t *remaining, int atomic) { unsigned long copy; - while (len > 0) { + while (*remaining > 0) { while (!iov->iov_len) iov++; - copy = min_t(unsigned long, len, iov->iov_len); + copy = min_t(unsigned long, *remaining, iov->iov_len); if (atomic) { - if (__copy_from_user_inatomic(to, iov->iov_base, copy)) + if (__copy_from_user_inatomic(addr + *offset, + iov->iov_base, copy)) return -EFAULT; } else { - if (copy_from_user(to, iov->iov_base, copy)) + if (copy_from_user(addr + *offset, + iov->iov_base, copy)) return -EFAULT; } - to += copy; - len -= copy; + *offset += copy; + *remaining -= copy; iov->iov_base += copy; iov->iov_len -= copy; } @@ -130,25 +138,27 @@ pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, } static int -pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, - int atomic) +pipe_iov_copy_to_user(struct iovec *iov, void *addr, int *offset, + size_t *remaining, int atomic) { unsigned long copy; - while (len > 0) { + while (*remaining > 0) { while (!iov->iov_len) iov++; - copy = min_t(unsigned long, len, iov->iov_len); + copy = min_t(unsigned long, *remaining, iov->iov_len); if (atomic) { - if (__copy_to_user_inatomic(iov->iov_base, from, copy)) + if (__copy_to_user_inatomic(iov->iov_base, + addr + *offset, copy)) return -EFAULT; } else { - if (copy_to_user(iov->iov_base, from, copy)) + if (copy_to_user(iov->iov_base, + addr + *offset, copy)) return -EFAULT; } - from += copy; - len -= copy; + *offset += copy; + *remaining -= copy; iov->iov_base += copy; iov->iov_len -= copy; } @@ -384,8 +394,8 @@ pipe_read(struct kiocb *iocb, const struct iovec *_iov, struct pipe_buffer *buf = pipe->bufs + curbuf; const struct pipe_buf_operations *ops = buf->ops; void *addr; - size_t chars = buf->len; - int error, atomic; + size_t chars = buf->len, remaining; + int error, atomic, offset; if (chars > total_len) chars = total_len; @@ -398,9 +408,12 @@ pipe_read(struct kiocb *iocb, const struct iovec *_iov, } atomic = !iov_fault_in_pages_write(iov, chars); + remaining = chars; + offset = buf->offset; redo: addr = ops->map(pipe, buf, atomic); - error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); + error = pipe_iov_copy_to_user(iov, addr, &offset, + &remaining, atomic); ops->unmap(pipe, buf, addr); if (unlikely(error)) { /* @@ -522,6 +535,7 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, if (ops->can_merge && offset + chars <= PAGE_SIZE) { int error, atomic = 1; void *addr; + size_t remaining = chars; error = ops->confirm(pipe, buf); if (error) @@ -530,8 +544,8 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, iov_fault_in_pages_read(iov, chars); redo1: addr = ops->map(pipe, buf, atomic); - error = pipe_iov_copy_from_user(offset + addr, iov, - chars, atomic); + error = pipe_iov_copy_from_user(addr, &offset, iov, + &remaining, atomic); ops->unmap(pipe, buf, addr); ret = error; do_wakeup = 1; @@ -566,6 +580,8 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, struct page *page = pipe->tmp_page; char *src; int error, atomic = 1; + int offset = 0; + size_t remaining; if (!page) { page = alloc_page(GFP_HIGHUSER); @@ -586,14 +602,15 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, chars = total_len; iov_fault_in_pages_read(iov, chars); + remaining = chars; redo2: if (atomic) src = kmap_atomic(page); else src = kmap(page); - error = pipe_iov_copy_from_user(src, iov, chars, - atomic); + error = pipe_iov_copy_from_user(src, &offset, iov, + &remaining, atomic); if (atomic) kunmap_atomic(src); else @@ -654,8 +671,11 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM); kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } - if (ret > 0) - file_update_time(filp); + if (ret > 0) { + int err = file_update_time(filp); + if (err) + ret = err; + } return ret; } @@ -918,20 +938,49 @@ const struct file_operations rdwr_pipefifo_fops = { .fasync = pipe_rdwr_fasync, }; +static void account_pipe_buffers(struct pipe_inode_info *pipe, + unsigned long old, unsigned long new) +{ + atomic_long_add(new - old, &pipe->user->pipe_bufs); +} + +static bool too_many_pipe_buffers_soft(struct user_struct *user) +{ + return pipe_user_pages_soft && + atomic_long_read(&user->pipe_bufs) >= pipe_user_pages_soft; +} + +static bool too_many_pipe_buffers_hard(struct user_struct *user) +{ + return pipe_user_pages_hard && + atomic_long_read(&user->pipe_bufs) >= pipe_user_pages_hard; +} + struct pipe_inode_info * alloc_pipe_info(struct inode *inode) { struct pipe_inode_info *pipe; pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); if (pipe) { - pipe->bufs = kzalloc(sizeof(struct pipe_buffer) * PIPE_DEF_BUFFERS, GFP_KERNEL); + unsigned long pipe_bufs = PIPE_DEF_BUFFERS; + struct user_struct *user = get_current_user(); + + if (!too_many_pipe_buffers_hard(user)) { + if (too_many_pipe_buffers_soft(user)) + pipe_bufs = 1; + pipe->bufs = kzalloc(sizeof(struct pipe_buffer) * pipe_bufs, GFP_KERNEL); + } + if (pipe->bufs) { init_waitqueue_head(&pipe->wait); pipe->r_counter = pipe->w_counter = 1; pipe->inode = inode; - pipe->buffers = PIPE_DEF_BUFFERS; + pipe->buffers = pipe_bufs; + pipe->user = user; + account_pipe_buffers(pipe, 0, pipe_bufs); return pipe; } + free_uid(user); kfree(pipe); } @@ -942,6 +991,8 @@ void __free_pipe_info(struct pipe_inode_info *pipe) { int i; + account_pipe_buffers(pipe, pipe->buffers, 0); + free_uid(pipe->user); for (i = 0; i < pipe->buffers; i++) { struct pipe_buffer *buf = pipe->bufs + i; if (buf->ops) @@ -1190,6 +1241,7 @@ static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long nr_pages) memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer)); } + account_pipe_buffers(pipe, pipe->buffers, nr_pages); pipe->curbuf = 0; kfree(pipe->bufs); pipe->bufs = bufs; @@ -1263,6 +1315,11 @@ long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg) if (!capable(CAP_SYS_RESOURCE) && size > pipe_max_size) { ret = -EPERM; goto out; + } else if ((too_many_pipe_buffers_hard(pipe->user) || + too_many_pipe_buffers_soft(pipe->user)) && + !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; } ret = pipe_set_size(pipe, nr_pages); break; diff --git a/fs/pnode.c b/fs/pnode.c index ab5fa9e1a79..3e000a51ac0 100644 --- a/fs/pnode.c +++ b/fs/pnode.c @@ -237,8 +237,9 @@ int propagate_mnt(struct mount *dest_mnt, struct dentry *dest_dentry, source = get_source(m, prev_dest_mnt, prev_src_mnt, &type); - if (!(child = copy_tree(source, source->mnt.mnt_root, type))) { - ret = -ENOMEM; + child = copy_tree(source, source->mnt.mnt_root, type); + if (IS_ERR(child)) { + ret = PTR_ERR(child); list_splice(tree_list, tmp_list.prev); goto out; } @@ -257,12 +258,12 @@ int propagate_mnt(struct mount *dest_mnt, struct dentry *dest_dentry, prev_src_mnt = child; } out: - br_write_lock(vfsmount_lock); + br_write_lock(&vfsmount_lock); while (!list_empty(&tmp_list)) { child = list_first_entry(&tmp_list, struct mount, mnt_hash); umount_tree(child, 0, &umount_list); } - br_write_unlock(vfsmount_lock); + br_write_unlock(&vfsmount_lock); release_mounts(&umount_list); return ret; } diff --git a/fs/pnode.h b/fs/pnode.h index 65c60979d54..19b853a3445 100644 --- a/fs/pnode.h +++ b/fs/pnode.h @@ -22,6 +22,7 @@ #define CL_COPY_ALL 0x04 #define CL_MAKE_SHARED 0x08 #define CL_PRIVATE 0x10 +#define CL_SHARED_TO_SLAVE 0x20 static inline void set_mnt_shared(struct mount *mnt) { diff --git a/fs/proc/array.c b/fs/proc/array.c index f9bd395b347..e0209a5a2c3 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -165,16 +165,16 @@ static inline void task_state(struct seq_file *m, struct pid_namespace *ns, int g; struct fdtable *fdt = NULL; const struct cred *cred; - pid_t ppid, tpid; + pid_t ppid = 0, tpid = 0; + struct task_struct *leader = NULL; rcu_read_lock(); - ppid = pid_alive(p) ? - task_tgid_nr_ns(rcu_dereference(p->real_parent), ns) : 0; - tpid = 0; if (pid_alive(p)) { struct task_struct *tracer = ptrace_parent(p); if (tracer) tpid = task_pid_nr_ns(tracer, ns); + ppid = task_tgid_nr_ns(rcu_dereference(p->real_parent), ns); + leader = p->group_leader; } cred = get_task_cred(p); seq_printf(m, @@ -186,7 +186,7 @@ static inline void task_state(struct seq_file *m, struct pid_namespace *ns, "Uid:\t%d\t%d\t%d\t%d\n" "Gid:\t%d\t%d\t%d\t%d\n", get_task_state(p), - task_tgid_nr_ns(p, ns), + leader ? task_pid_nr_ns(leader, ns) : 0, pid_nr_ns(pid, ns), ppid, tpid, cred->uid, cred->euid, cred->suid, cred->fsuid, diff --git a/fs/proc/base.c b/fs/proc/base.c index 2f198dad12c..91608102f59 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -137,12 +137,6 @@ struct pid_entry { static int proc_fd_permission(struct inode *inode, int mask); -/* ANDROID is for special files in /proc. */ -#define ANDROID(NAME, MODE, OTYPE) \ - NOD(NAME, (S_IFREG|(MODE)), \ - &proc_##OTYPE##_inode_operations, \ - &proc_##OTYPE##_operations, {}) - /* * Count the number of hardlinks for the pid_entry table, excluding the . * and .. links. @@ -827,7 +821,8 @@ static ssize_t environ_read(struct file *file, char __user *buf, mm = mm_for_maps(task); ret = PTR_ERR(mm); - if (!mm || IS_ERR(mm)) + /* Ensure the process spawned far enough to have an environment. */ + if (!mm || IS_ERR(mm) || !mm->env_end) goto out_free; ret = 0; @@ -975,35 +970,6 @@ static ssize_t oom_adjust_write(struct file *file, const char __user *buf, return err < 0 ? err : count; } -static int oom_adjust_permission(struct inode *inode, int mask) -{ - uid_t uid; - struct task_struct *p; - - p = get_proc_task(inode); - if(p) { - uid = task_uid(p); - put_task_struct(p); - } - - /* - * System Server (uid == 1000) is granted access to oom_adj of all - * android applications (uid > 10000) as and services (uid >= 1000) - */ - if (p && (current_fsuid() == 1000) && (uid >= 1000)) { - if (inode->i_mode >> 6 & mask) { - return 0; - } - } - - /* Fall back to default. */ - return generic_permission(inode, mask); -} - -static const struct inode_operations proc_oom_adjust_inode_operations = { - .permission = oom_adjust_permission, -}; - static const struct file_operations proc_oom_adjust_operations = { .read = oom_adjust_read, .write = oom_adjust_write, @@ -2370,11 +2336,19 @@ static const struct file_operations proc_map_files_operations = { */ static int proc_fd_permission(struct inode *inode, int mask) { - int rv = generic_permission(inode, mask); + struct task_struct *p; + int rv; + + rv = generic_permission(inode, mask); if (rv == 0) - return 0; - if (task_pid(current) == proc_pid(inode)) + return rv; + + rcu_read_lock(); + p = pid_task(proc_pid(inode), PIDTYPE_PID); + if (p && same_thread_group(p, current)) rv = 0; + rcu_read_unlock(); + return rv; } @@ -3043,8 +3017,8 @@ static const struct pid_entry tgid_base_stuff[] = { REG("cgroup", S_IRUGO, proc_cgroup_operations), #endif INF("oom_score", S_IRUGO, proc_oom_score), - ANDROID("oom_adj",S_IRUGO|S_IWUSR, oom_adjust), - REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations), + REG("oom_adj", S_IRUSR, proc_oom_adjust_operations), + REG("oom_score_adj", S_IRUSR, proc_oom_score_adj_operations), #ifdef CONFIG_AUDITSYSCALL REG("loginuid", S_IWUSR|S_IRUGO, proc_loginuid_operations), REG("sessionid", S_IRUGO, proc_sessionid_operations), @@ -3401,8 +3375,8 @@ static const struct pid_entry tid_base_stuff[] = { REG("cgroup", S_IRUGO, proc_cgroup_operations), #endif INF("oom_score", S_IRUGO, proc_oom_score), - REG("oom_adj", S_IRUGO|S_IWUSR, proc_oom_adjust_operations), - REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations), + REG("oom_adj", S_IRUSR, proc_oom_adjust_operations), + REG("oom_score_adj", S_IRUSR, proc_oom_score_adj_operations), #ifdef CONFIG_AUDITSYSCALL REG("loginuid", S_IWUSR|S_IRUGO, proc_loginuid_operations), REG("sessionid", S_IRUGO, proc_sessionid_operations), diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 2edf34f2eb6..a1487e5ab08 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -350,37 +350,39 @@ static DEFINE_SPINLOCK(proc_inum_lock); /* protects the above */ * Return an inode number between PROC_DYNAMIC_FIRST and * 0xffffffff, or zero on failure. */ -static unsigned int get_inode_number(void) +int proc_alloc_inum(unsigned int *inum) { unsigned int i; int error; retry: - if (ida_pre_get(&proc_inum_ida, GFP_KERNEL) == 0) - return 0; + if (!ida_pre_get(&proc_inum_ida, GFP_KERNEL)) + return -ENOMEM; - spin_lock(&proc_inum_lock); + spin_lock_irq(&proc_inum_lock); error = ida_get_new(&proc_inum_ida, &i); - spin_unlock(&proc_inum_lock); + spin_unlock_irq(&proc_inum_lock); if (error == -EAGAIN) goto retry; else if (error) - return 0; + return error; if (i > UINT_MAX - PROC_DYNAMIC_FIRST) { - spin_lock(&proc_inum_lock); + spin_lock_irq(&proc_inum_lock); ida_remove(&proc_inum_ida, i); - spin_unlock(&proc_inum_lock); - return 0; + spin_unlock_irq(&proc_inum_lock); + return -ENOSPC; } - return PROC_DYNAMIC_FIRST + i; + *inum = PROC_DYNAMIC_FIRST + i; + return 0; } -static void release_inode_number(unsigned int inum) +void proc_free_inum(unsigned int inum) { - spin_lock(&proc_inum_lock); + unsigned long flags; + spin_lock_irqsave(&proc_inum_lock, flags); ida_remove(&proc_inum_ida, inum - PROC_DYNAMIC_FIRST); - spin_unlock(&proc_inum_lock); + spin_unlock_irqrestore(&proc_inum_lock, flags); } static void *proc_follow_link(struct dentry *dentry, struct nameidata *nd) @@ -554,13 +556,12 @@ static const struct inode_operations proc_dir_inode_operations = { static int proc_register(struct proc_dir_entry * dir, struct proc_dir_entry * dp) { - unsigned int i; struct proc_dir_entry *tmp; + int ret; - i = get_inode_number(); - if (i == 0) - return -EAGAIN; - dp->low_ino = i; + ret = proc_alloc_inum(&dp->low_ino); + if (ret) + return ret; if (S_ISDIR(dp->mode)) { if (dp->proc_iops == NULL) { @@ -765,7 +766,7 @@ EXPORT_SYMBOL(proc_create_data); static void free_proc_entry(struct proc_dir_entry *de) { - release_inode_number(de->low_ino); + proc_free_inum(de->low_ino); if (S_ISLNK(de->mode)) kfree(de->data); diff --git a/fs/proc/inode.c b/fs/proc/inode.c index 205c9228083..b1f55aef44d 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -31,6 +31,7 @@ static void proc_evict_inode(struct inode *inode) struct proc_dir_entry *de; struct ctl_table_header *head; const struct proc_ns_operations *ns_ops; + void *ns; truncate_inode_pages(&inode->i_data, 0); end_writeback(inode); @@ -49,8 +50,9 @@ static void proc_evict_inode(struct inode *inode) } /* Release any associated namespace */ ns_ops = PROC_I(inode)->ns_ops; - if (ns_ops && ns_ops->put) - ns_ops->put(PROC_I(inode)->ns); + ns = PROC_I(inode)->ns; + if (ns_ops && ns) + ns_ops->put(ns); } static struct kmem_cache * proc_inode_cachep; diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c index 0d9e23a39e4..08dfd6ad8f3 100644 --- a/fs/proc/namespaces.c +++ b/fs/proc/namespaces.c @@ -24,12 +24,165 @@ static const struct proc_ns_operations *ns_entries[] = { #ifdef CONFIG_IPC_NS &ipcns_operations, #endif +#ifdef CONFIG_PID_NS + &pidns_operations, +#endif + &mntns_operations, }; static const struct file_operations ns_file_operations = { .llseek = no_llseek, }; +static const struct inode_operations ns_inode_operations = { + .setattr = proc_setattr, +}; + +static int ns_delete_dentry(const struct dentry *dentry) +{ + /* Don't cache namespace inodes when not in use */ + return 1; +} + +static char *ns_dname(struct dentry *dentry, char *buffer, int buflen) +{ + struct inode *inode = dentry->d_inode; + const struct proc_ns_operations *ns_ops = PROC_I(inode)->ns_ops; + + return dynamic_dname(dentry, buffer, buflen, "%s:[%lu]", + ns_ops->name, inode->i_ino); +} + +const struct dentry_operations ns_dentry_operations = +{ + .d_delete = ns_delete_dentry, + .d_dname = ns_dname, +}; + +static struct dentry *proc_ns_get_dentry(struct super_block *sb, + struct task_struct *task, const struct proc_ns_operations *ns_ops) +{ + struct dentry *dentry, *result; + struct inode *inode; + struct proc_inode *ei; + struct qstr qname = { .name = "", }; + void *ns; + + ns = ns_ops->get(task); + if (!ns) + return ERR_PTR(-ENOENT); + + dentry = d_alloc_pseudo(sb, &qname); + if (!dentry) { + ns_ops->put(ns); + return ERR_PTR(-ENOMEM); + } + + inode = iget_locked(sb, ns_ops->inum(ns)); + if (!inode) { + dput(dentry); + ns_ops->put(ns); + return ERR_PTR(-ENOMEM); + } + + ei = PROC_I(inode); + if (inode->i_state & I_NEW) { + inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME; + inode->i_op = &ns_inode_operations; + inode->i_mode = S_IFREG | S_IRUGO; + inode->i_fop = &ns_file_operations; + ei->ns_ops = ns_ops; + ei->ns = ns; + unlock_new_inode(inode); + } else { + ns_ops->put(ns); + } + + d_set_d_op(dentry, &ns_dentry_operations); + result = d_instantiate_unique(dentry, inode); + if (result) { + dput(dentry); + dentry = result; + } + + return dentry; +} + +static void *proc_ns_follow_link(struct dentry *dentry, struct nameidata *nd) +{ + struct inode *inode = dentry->d_inode; + struct super_block *sb = inode->i_sb; + struct proc_inode *ei = PROC_I(inode); + struct task_struct *task; + struct dentry *ns_dentry; + void *error = ERR_PTR(-EACCES); + + task = get_proc_task(inode); + if (!task) + goto out; + + if (!ptrace_may_access(task, PTRACE_MODE_READ)) + goto out_put_task; + + ns_dentry = proc_ns_get_dentry(sb, task, ei->ns_ops); + if (IS_ERR(ns_dentry)) { + error = ERR_CAST(ns_dentry); + goto out_put_task; + } + + dput(nd->path.dentry); + nd->path.dentry = ns_dentry; + error = NULL; + +out_put_task: + put_task_struct(task); +out: + return error; +} + +static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int buflen) +{ + struct inode *inode = dentry->d_inode; + struct proc_inode *ei = PROC_I(inode); + const struct proc_ns_operations *ns_ops = ei->ns_ops; + struct task_struct *task; + void *ns; + char name[50]; + int len = -EACCES; + + task = get_proc_task(inode); + if (!task) + goto out; + + if (!ptrace_may_access(task, PTRACE_MODE_READ)) + goto out_put_task; + + len = -ENOENT; + ns = ns_ops->get(task); + if (!ns) + goto out_put_task; + + snprintf(name, sizeof(name), "%s:[%u]", ns_ops->name, ns_ops->inum(ns)); + len = strlen(name); + + if (len > buflen) + len = buflen; + if (copy_to_user(buffer, name, len)) + len = -EFAULT; + + ns_ops->put(ns); +out_put_task: + put_task_struct(task); +out: + return len; +} + +static const struct inode_operations proc_ns_link_inode_operations = { + .readlink = proc_ns_readlink, + .follow_link = proc_ns_follow_link, + .setattr = proc_setattr, +}; + static struct dentry *proc_ns_instantiate(struct inode *dir, struct dentry *dentry, struct task_struct *task, const void *ptr) { @@ -37,21 +190,15 @@ static struct dentry *proc_ns_instantiate(struct inode *dir, struct inode *inode; struct proc_inode *ei; struct dentry *error = ERR_PTR(-ENOENT); - void *ns; inode = proc_pid_make_inode(dir->i_sb, task); if (!inode) goto out; - ns = ns_ops->get(task); - if (!ns) - goto out_iput; - ei = PROC_I(inode); - inode->i_mode = S_IFREG|S_IRUSR; - inode->i_fop = &ns_file_operations; - ei->ns_ops = ns_ops; - ei->ns = ns; + inode->i_mode = S_IFLNK|S_IRWXUGO; + inode->i_op = &proc_ns_link_inode_operations; + ei->ns_ops = ns_ops; d_set_d_op(dentry, &pid_dentry_operations); d_add(dentry, inode); @@ -60,9 +207,6 @@ static struct dentry *proc_ns_instantiate(struct inode *dir, error = NULL; out: return error; -out_iput: - iput(inode); - goto out; } static int proc_ns_fill_cache(struct file *filp, void *dirent, @@ -89,10 +233,6 @@ static int proc_ns_dir_readdir(struct file *filp, void *dirent, if (!task) goto out_no_task; - ret = -EPERM; - if (!ptrace_may_access(task, PTRACE_MODE_READ)) - goto out; - ret = 0; i = filp->f_pos; switch (i) { @@ -152,10 +292,6 @@ static struct dentry *proc_ns_dir_lookup(struct inode *dir, if (!task) goto out_no_task; - error = ERR_PTR(-EPERM); - if (!ptrace_may_access(task, PTRACE_MODE_READ)) - goto out; - last = &ns_entries[ARRAY_SIZE(ns_entries)]; for (entry = ns_entries; entry < last; entry++) { if (strlen((*entry)->name) != len) @@ -163,7 +299,6 @@ static struct dentry *proc_ns_dir_lookup(struct inode *dir, if (!memcmp(dentry->d_name.name, (*entry)->name, len)) break; } - error = ERR_PTR(-ENOENT); if (entry == last) goto out; @@ -198,3 +333,7 @@ struct file *proc_ns_fget(int fd) return ERR_PTR(-EINVAL); } +bool proc_ns_inode(struct inode *inode) +{ + return inode->i_fop == &ns_file_operations; +} diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index a789934737e..ef5c7e2f77b 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -671,6 +671,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, #define CLEAR_REFS_ALL 1 #define CLEAR_REFS_ANON 2 #define CLEAR_REFS_MAPPED 3 +#define CLEAR_REFS_MM_HIWATER_RSS 5 static ssize_t clear_refs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) @@ -690,7 +691,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, rv = kstrtoint(strstrip(buffer), 10, &type); if (rv < 0) return rv; - if (type < CLEAR_REFS_ALL || type > CLEAR_REFS_MAPPED) + if ((type < CLEAR_REFS_ALL || type > CLEAR_REFS_MAPPED) && + type != CLEAR_REFS_MM_HIWATER_RSS) return -EINVAL; task = get_proc_task(file->f_path.dentry->d_inode); if (!task) @@ -701,6 +703,18 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, .pmd_entry = clear_refs_pte_range, .mm = mm, }; + + if (type == CLEAR_REFS_MM_HIWATER_RSS) { + /* + * Writing 5 to /proc/pid/clear_refs resets the peak + * resident set size to this mm's current rss value. + */ + down_write(&mm->mmap_sem); + reset_mm_hiwater_rss(mm); + up_write(&mm->mmap_sem); + goto out_mm; + } + down_read(&mm->mmap_sem); for (vma = mm->mmap; vma; vma = vma->vm_next) { clear_refs_walk.private = vma; @@ -724,6 +738,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, } flush_tlb_mm(mm); up_read(&mm->mmap_sem); +out_mm: mmput(mm); } put_task_struct(task); @@ -1047,9 +1062,18 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, return ret; } +static int pagemap_open(struct inode *inode, struct file *file) +{ + /* do not disclose physical addresses: attack vector */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + return 0; +} + const struct file_operations proc_pagemap_operations = { .llseek = mem_lseek, /* borrow this */ .read = pagemap_read, + .open = pagemap_open, }; #endif /* CONFIG_PROC_PAGE_MONITOR */ diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c index 12412852d88..5fe34c355e8 100644 --- a/fs/proc_namespace.c +++ b/fs/proc_namespace.c @@ -17,18 +17,18 @@ static unsigned mounts_poll(struct file *file, poll_table *wait) { - struct proc_mounts *p = file->private_data; + struct proc_mounts *p = proc_mounts(file->private_data); struct mnt_namespace *ns = p->ns; unsigned res = POLLIN | POLLRDNORM; poll_wait(file, &p->ns->poll, wait); - br_read_lock(vfsmount_lock); + br_read_lock(&vfsmount_lock); if (p->m.poll_event != ns->event) { p->m.poll_event = ns->event; res |= POLLERR | POLLPRI; } - br_read_unlock(vfsmount_lock); + br_read_unlock(&vfsmount_lock); return res; } @@ -121,7 +121,7 @@ static int show_vfsmnt(struct seq_file *m, struct vfsmount *mnt) static int show_mountinfo(struct seq_file *m, struct vfsmount *mnt) { - struct proc_mounts *p = m->private; + struct proc_mounts *p = proc_mounts(m); struct mount *r = real_mount(mnt); struct super_block *sb = mnt->mnt_sb; struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt }; @@ -268,7 +268,6 @@ static int mounts_open_common(struct inode *inode, struct file *file, if (ret) goto err_free; - p->m.private = p; p->ns = ns; p->root = root; p->m.poll_event = ns->event; @@ -288,7 +287,7 @@ static int mounts_open_common(struct inode *inode, struct file *file, static int mounts_release(struct inode *inode, struct file *file) { - struct proc_mounts *p = file->private_data; + struct proc_mounts *p = proc_mounts(file->private_data); path_put(&p->root); put_mnt_ns(p->ns); return seq_release(inode, file); diff --git a/fs/readdir.c b/fs/readdir.c index cc0a8227cdd..536054080bc 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -47,6 +47,14 @@ int vfs_readdir(struct file *file, filldir_t filler, void *buf) EXPORT_SYMBOL(vfs_readdir); +static bool hide_name(const char *name, int namlen) +{ + if (namlen == 2 && !memcmp(name, "su", 2)) + if (!su_visible()) + return true; + return false; +} + /* * Traditional linux readdir() handling.. * @@ -68,6 +76,7 @@ struct old_linux_dirent { struct readdir_callback { struct old_linux_dirent __user * dirent; int result; + bool romnt; }; static int fillonedir(void * __buf, const char * name, int namlen, loff_t offset, @@ -84,6 +93,8 @@ static int fillonedir(void * __buf, const char * name, int namlen, loff_t offset buf->result = -EOVERFLOW; return -EOVERFLOW; } + if (hide_name(name, namlen) && buf->romnt) + return 0; buf->result++; dirent = buf->dirent; if (!access_ok(VERIFY_WRITE, dirent, @@ -116,6 +127,7 @@ SYSCALL_DEFINE3(old_readdir, unsigned int, fd, buf.result = 0; buf.dirent = dirent; + buf.romnt = (file->f_path.dentry->d_sb->s_flags & MS_RDONLY); error = vfs_readdir(file, fillonedir, &buf); if (buf.result) @@ -144,6 +156,7 @@ struct getdents_callback { struct linux_dirent __user * previous; int count; int error; + bool romnt; }; static int filldir(void * __buf, const char * name, int namlen, loff_t offset, @@ -163,6 +176,8 @@ static int filldir(void * __buf, const char * name, int namlen, loff_t offset, buf->error = -EOVERFLOW; return -EOVERFLOW; } + if (hide_name(name, namlen) && buf->romnt) + return 0; dirent = buf->previous; if (dirent) { if (__put_user(offset, &dirent->d_off)) @@ -210,6 +225,7 @@ SYSCALL_DEFINE3(getdents, unsigned int, fd, buf.previous = NULL; buf.count = count; buf.error = 0; + buf.romnt = (file->f_path.dentry->d_sb->s_flags & MS_RDONLY); error = vfs_readdir(file, filldir, &buf); if (error >= 0) @@ -231,6 +247,7 @@ struct getdents_callback64 { struct linux_dirent64 __user * previous; int count; int error; + bool romnt; }; static int filldir64(void * __buf, const char * name, int namlen, loff_t offset, @@ -244,6 +261,8 @@ static int filldir64(void * __buf, const char * name, int namlen, loff_t offset, buf->error = -EINVAL; /* only used if we fail.. */ if (reclen > buf->count) return -EINVAL; + if (hide_name(name, namlen) && buf->romnt) + return 0; dirent = buf->previous; if (dirent) { if (__put_user(offset, &dirent->d_off)) @@ -293,6 +312,7 @@ SYSCALL_DEFINE3(getdents64, unsigned int, fd, buf.previous = NULL; buf.count = count; buf.error = 0; + buf.romnt = (file->f_path.dentry->d_sb->s_flags & MS_RDONLY); error = vfs_readdir(file, filldir64, &buf); if (error >= 0) diff --git a/fs/seq_file.c b/fs/seq_file.c index b857a379616..03f46abf893 100644 --- a/fs/seq_file.c +++ b/fs/seq_file.c @@ -8,7 +8,10 @@ #include #include #include +#include #include +#include +#include #include #include @@ -29,6 +32,18 @@ static void seq_set_overflow(struct seq_file *m) m->count = m->size; } +static void *seq_buf_alloc(unsigned long size) +{ + void *buf; + + if (size > PAGE_SIZE) + buf = vmalloc(size); + else + buf = kmalloc(size, GFP_KERNEL | __GFP_NOWARN); + + return buf; +} + /** * seq_open - initialize sequential file * @file: file we initialize @@ -92,7 +107,7 @@ static int traverse(struct seq_file *m, loff_t offset) return 0; } if (!m->buf) { - m->buf = kmalloc(m->size = PAGE_SIZE, GFP_KERNEL); + m->buf = seq_buf_alloc(m->size = PAGE_SIZE); if (!m->buf) return -ENOMEM; } @@ -131,9 +146,9 @@ static int traverse(struct seq_file *m, loff_t offset) Eoverflow: m->op->stop(m, p); - kfree(m->buf); + kvfree(m->buf); m->count = 0; - m->buf = kmalloc(m->size <<= 1, GFP_KERNEL); + m->buf = seq_buf_alloc(m->size <<= 1); return !m->buf ? -ENOMEM : -EAGAIN; } @@ -188,7 +203,7 @@ ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos) /* grab buffer if we didn't have one */ if (!m->buf) { - m->buf = kmalloc(m->size = PAGE_SIZE, GFP_KERNEL); + m->buf = seq_buf_alloc(m->size = PAGE_SIZE); if (!m->buf) goto Enomem; } @@ -228,9 +243,9 @@ ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos) if (m->count < m->size) goto Fill; m->op->stop(m, p); - kfree(m->buf); + kvfree(m->buf); m->count = 0; - m->buf = kmalloc(m->size <<= 1, GFP_KERNEL); + m->buf = seq_buf_alloc(m->size <<= 1); if (!m->buf) goto Enomem; m->version = 0; @@ -346,7 +361,7 @@ EXPORT_SYMBOL(seq_lseek); int seq_release(struct inode *inode, struct file *file) { struct seq_file *m = file->private_data; - kfree(m->buf); + kvfree(m->buf); kfree(m); return 0; } @@ -588,6 +603,24 @@ int single_open(struct file *file, int (*show)(struct seq_file *, void *), } EXPORT_SYMBOL(single_open); +int single_open_size(struct file *file, int (*show)(struct seq_file *, void *), + void *data, size_t size) +{ + char *buf = seq_buf_alloc(size); + int ret; + if (!buf) + return -ENOMEM; + ret = single_open(file, show, data); + if (ret) { + kvfree(buf); + return ret; + } + ((struct seq_file *)file->private_data)->buf = buf; + ((struct seq_file *)file->private_data)->size = size; + return 0; +} +EXPORT_SYMBOL(single_open_size); + int single_release(struct inode *inode, struct file *file) { const struct seq_operations *op = ((struct seq_file *)file->private_data)->op; diff --git a/fs/splice.c b/fs/splice.c index f8476841eb0..ea85353273c 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -985,13 +985,17 @@ generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, struct address_space *mapping = out->f_mapping; struct inode *inode = mapping->host; struct splice_desc sd = { - .total_len = len, .flags = flags, - .pos = *ppos, .u.file = out, }; ssize_t ret; + ret = generic_write_checks(out, ppos, &len, S_ISBLK(inode->i_mode)); + if (ret) + return ret; + sd.total_len = len; + sd.pos = *ppos; + pipe_lock(pipe); splice_from_pipe_begin(&sd); @@ -1003,8 +1007,10 @@ generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD); ret = file_remove_suid(out); if (!ret) { - file_update_time(out); - ret = splice_from_pipe_feed(pipe, &sd, pipe_to_file); + ret = file_update_time(out); + if (!ret) + ret = splice_from_pipe_feed(pipe, &sd, + pipe_to_file); } mutex_unlock(&inode->i_mutex); } while (ret > 0); diff --git a/fs/udf/dir.c b/fs/udf/dir.c index eb8bfe2b89a..56341af63ac 100644 --- a/fs/udf/dir.c +++ b/fs/udf/dir.c @@ -163,7 +163,8 @@ static int do_udf_readdir(struct inode *dir, struct file *filp, struct kernel_lb_addr tloc = lelb_to_cpu(cfi.icb.extLocation); iblock = udf_get_lb_pblock(dir->i_sb, &tloc, 0); - flen = udf_get_filename(dir->i_sb, nameptr, fname, lfi); + flen = udf_get_filename(dir->i_sb, nameptr, lfi, fname, + UDF_NAME_LEN); dt_type = DT_UNKNOWN; } diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 38de8f234b9..99cc37bfcfa 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -233,7 +233,8 @@ static struct fileIdentDesc *udf_find_entry(struct inode *dir, if (!lfi) continue; - flen = udf_get_filename(dir->i_sb, nameptr, fname, lfi); + flen = udf_get_filename(dir->i_sb, nameptr, lfi, fname, + UDF_NAME_LEN); if (flen && udf_match(flen, fname, child->len, child->name)) goto out_ok; } diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index d7c6dbe4194..ab94e667184 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -30,13 +30,16 @@ #include #include "udf_i.h" -static void udf_pc_to_char(struct super_block *sb, unsigned char *from, - int fromlen, unsigned char *to) +static int udf_pc_to_char(struct super_block *sb, unsigned char *from, + int fromlen, unsigned char *to, int tolen) { struct pathComponent *pc; int elen = 0; + int comp_len; unsigned char *p = to; + /* Reserve one byte for terminating \0 */ + tolen--; while (elen < fromlen) { pc = (struct pathComponent *)(from + elen); switch (pc->componentType) { @@ -49,22 +52,37 @@ static void udf_pc_to_char(struct super_block *sb, unsigned char *from, break; /* Fall through */ case 2: + if (tolen == 0) + return -ENAMETOOLONG; p = to; *p++ = '/'; + tolen--; break; case 3: + if (tolen < 3) + return -ENAMETOOLONG; memcpy(p, "../", 3); p += 3; + tolen -= 3; break; case 4: + if (tolen < 2) + return -ENAMETOOLONG; memcpy(p, "./", 2); p += 2; + tolen -= 2; /* that would be . - just ignore */ break; case 5: - p += udf_get_filename(sb, pc->componentIdent, p, - pc->lengthComponentIdent); + comp_len = udf_get_filename(sb, pc->componentIdent, + pc->lengthComponentIdent, + p, tolen); + p += comp_len; + tolen -= comp_len; + if (tolen == 0) + return -ENAMETOOLONG; *p++ = '/'; + tolen--; break; } elen += sizeof(struct pathComponent) + pc->lengthComponentIdent; @@ -73,6 +91,7 @@ static void udf_pc_to_char(struct super_block *sb, unsigned char *from, p[-1] = '\0'; else p[0] = '\0'; + return 0; } static int udf_symlink_filler(struct file *file, struct page *page) @@ -100,8 +119,10 @@ static int udf_symlink_filler(struct file *file, struct page *page) symlink = bh->b_data; } - udf_pc_to_char(inode->i_sb, symlink, inode->i_size, p); + err = udf_pc_to_char(inode->i_sb, symlink, inode->i_size, p, PAGE_SIZE); brelse(bh); + if (err) + goto out_unlock_inode; up_read(&iinfo->i_data_sem); SetPageUptodate(page); diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index ebe10314e51..375f64884c7 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -207,7 +207,8 @@ udf_get_lb_pblock(struct super_block *sb, struct kernel_lb_addr *loc, } /* unicode.c */ -extern int udf_get_filename(struct super_block *, uint8_t *, uint8_t *, int); +extern int udf_get_filename(struct super_block *, uint8_t *, int, uint8_t *, + int); extern int udf_put_filename(struct super_block *, const uint8_t *, uint8_t *, int); extern int udf_build_ustr(struct ustr *, dstring *, int); diff --git a/fs/udf/unicode.c b/fs/udf/unicode.c index 44b815e57f9..d29c06fbf4c 100644 --- a/fs/udf/unicode.c +++ b/fs/udf/unicode.c @@ -28,7 +28,8 @@ #include "udf_sb.h" -static int udf_translate_to_linux(uint8_t *, uint8_t *, int, uint8_t *, int); +static int udf_translate_to_linux(uint8_t *, int, uint8_t *, int, uint8_t *, + int); static int udf_char_to_ustr(struct ustr *dest, const uint8_t *src, int strlen) { @@ -333,8 +334,8 @@ static int udf_NLStoCS0(struct nls_table *nls, dstring *ocu, struct ustr *uni, return u_len + 1; } -int udf_get_filename(struct super_block *sb, uint8_t *sname, uint8_t *dname, - int flen) +int udf_get_filename(struct super_block *sb, uint8_t *sname, int slen, + uint8_t *dname, int dlen) { struct ustr *filename, *unifilename; int len = 0; @@ -347,7 +348,7 @@ int udf_get_filename(struct super_block *sb, uint8_t *sname, uint8_t *dname, if (!unifilename) goto out1; - if (udf_build_ustr_exact(unifilename, sname, flen)) + if (udf_build_ustr_exact(unifilename, sname, slen)) goto out2; if (UDF_QUERY_FLAG(sb, UDF_FLAG_UTF8)) { @@ -366,7 +367,8 @@ int udf_get_filename(struct super_block *sb, uint8_t *sname, uint8_t *dname, } else goto out2; - len = udf_translate_to_linux(dname, filename->u_name, filename->u_len, + len = udf_translate_to_linux(dname, dlen, + filename->u_name, filename->u_len, unifilename->u_name, unifilename->u_len); out2: kfree(unifilename); @@ -403,10 +405,12 @@ int udf_put_filename(struct super_block *sb, const uint8_t *sname, #define EXT_MARK '.' #define CRC_MARK '#' #define EXT_SIZE 5 +/* Number of chars we need to store generated CRC to make filename unique */ +#define CRC_LEN 5 -static int udf_translate_to_linux(uint8_t *newName, uint8_t *udfName, - int udfLen, uint8_t *fidName, - int fidNameLen) +static int udf_translate_to_linux(uint8_t *newName, int newLen, + uint8_t *udfName, int udfLen, + uint8_t *fidName, int fidNameLen) { int index, newIndex = 0, needsCRC = 0; int extIndex = 0, newExtIndex = 0, hasExt = 0; @@ -440,7 +444,7 @@ static int udf_translate_to_linux(uint8_t *newName, uint8_t *udfName, newExtIndex = newIndex; } } - if (newIndex < 256) + if (newIndex < newLen) newName[newIndex++] = curr; else needsCRC = 1; @@ -468,13 +472,13 @@ static int udf_translate_to_linux(uint8_t *newName, uint8_t *udfName, } ext[localExtIndex++] = curr; } - maxFilenameLen = 250 - localExtIndex; + maxFilenameLen = newLen - CRC_LEN - localExtIndex; if (newIndex > maxFilenameLen) newIndex = maxFilenameLen; else newIndex = newExtIndex; - } else if (newIndex > 250) - newIndex = 250; + } else if (newIndex > newLen - CRC_LEN) + newIndex = newLen - CRC_LEN; newName[newIndex++] = CRC_MARK; valueCRC = crc_itu_t(0, fidName, fidNameLen); newName[newIndex++] = hexChar[(valueCRC & 0xf000) >> 12]; diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 54a67dd9ac0..a27969494b8 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -629,8 +629,11 @@ xfs_file_aio_write_checks( * lock above. Eventually we should look into a way to avoid * the pointless lock roundtrip. */ - if (likely(!(file->f_mode & FMODE_NOCMTIME))) - file_update_time(file); + if (likely(!(file->f_mode & FMODE_NOCMTIME))) { + error = file_update_time(file); + if (error) + return error; + } /* * If we're writing the file then make sure to clear the setuid and diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 125c54e9851..6c449f203b9 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -471,11 +471,10 @@ static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) #ifdef CONFIG_TRANSPARENT_HUGEPAGE barrier(); #endif - if (pmd_none(pmdval)) + if (pmd_none(pmdval) || pmd_trans_huge(pmdval)) return 1; if (unlikely(pmd_bad(pmdval))) { - if (!pmd_trans_huge(pmdval)) - pmd_clear_bad(pmd); + pmd_clear_bad(pmd); return 1; } return 0; diff --git a/include/asm-generic/siginfo.h b/include/asm-generic/siginfo.h index 5e5e3865f1e..8ed67779fc0 100644 --- a/include/asm-generic/siginfo.h +++ b/include/asm-generic/siginfo.h @@ -98,9 +98,18 @@ typedef struct siginfo { __ARCH_SI_BAND_T _band; /* POLL_IN, POLL_OUT, POLL_MSG */ int _fd; } _sigpoll; + + /* SIGSYS */ + struct { + void __user *_call_addr; /* calling user insn */ + int _syscall; /* triggering system call number */ + unsigned int _arch; /* AUDIT_ARCH_* of syscall */ + } _sigsys; } _sifields; } __ARCH_SI_ATTRIBUTES siginfo_t; +/* If the arch shares siginfo, then it has SIGSYS. */ +#define __ARCH_SIGSYS #endif /* @@ -124,6 +133,11 @@ typedef struct siginfo { #define si_addr_lsb _sifields._sigfault._addr_lsb #define si_band _sifields._sigpoll._band #define si_fd _sifields._sigpoll._fd +#ifdef __ARCH_SIGSYS +#define si_call_addr _sifields._sigsys._call_addr +#define si_syscall _sifields._sigsys._syscall +#define si_arch _sifields._sigsys._arch +#endif #ifdef __KERNEL__ #define __SI_MASK 0xffff0000u @@ -134,6 +148,7 @@ typedef struct siginfo { #define __SI_CHLD (4 << 16) #define __SI_RT (5 << 16) #define __SI_MESGQ (6 << 16) +#define __SI_SYS (7 << 16) #define __SI_CODE(T,N) ((T) | ((N) & 0xffff)) #else #define __SI_KILL 0 @@ -143,6 +158,7 @@ typedef struct siginfo { #define __SI_CHLD 0 #define __SI_RT 0 #define __SI_MESGQ 0 +#define __SI_SYS 0 #define __SI_CODE(T,N) (N) #endif @@ -239,6 +255,12 @@ typedef struct siginfo { #define POLL_HUP (__SI_POLL|6) /* device disconnected */ #define NSIGPOLL 6 +/* + * SIGSYS si_codes + */ +#define SYS_SECCOMP (__SI_SYS|1) /* seccomp triggered */ +#define NSIGSYS 1 + /* * sigevent definitions * diff --git a/include/asm-generic/syscall.h b/include/asm-generic/syscall.h index 5c122ae6bfa..5b09392db67 100644 --- a/include/asm-generic/syscall.h +++ b/include/asm-generic/syscall.h @@ -142,4 +142,18 @@ void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, unsigned int i, unsigned int n, const unsigned long *args); +/** + * syscall_get_arch - return the AUDIT_ARCH for the current system call + * @task: task of interest, must be in system call entry tracing + * @regs: task_pt_regs() of @task + * + * Returns the AUDIT_ARCH_* based on the system call convention in use. + * + * It's only valid to call this when @task is stopped on entry to a system + * call, due to %TIF_SYSCALL_TRACE, %TIF_SYSCALL_AUDIT, or %TIF_SECCOMP. + * + * Architectures which permit CONFIG_HAVE_ARCH_SECCOMP_FILTER must + * provide an implementation of this. + */ +int syscall_get_arch(struct task_struct *task, struct pt_regs *regs); #endif /* _ASM_SYSCALL_H */ diff --git a/include/asm-generic/unistd.h b/include/asm-generic/unistd.h index 991ef01cd77..ae8513b32af 100644 --- a/include/asm-generic/unistd.h +++ b/include/asm-generic/unistd.h @@ -691,9 +691,11 @@ __SC_COMP(__NR_process_vm_readv, sys_process_vm_readv, \ #define __NR_process_vm_writev 271 __SC_COMP(__NR_process_vm_writev, sys_process_vm_writev, \ compat_sys_process_vm_writev) +#define __NR_seccomp 277 +__SYSCALL(__NR_seccomp, sys_seccomp) #undef __NR_syscalls -#define __NR_syscalls 272 +#define __NR_syscalls 278 /* * All syscalls below here should go away really, diff --git a/include/linux/Kbuild b/include/linux/Kbuild index 2042224474f..17013360078 100644 --- a/include/linux/Kbuild +++ b/include/linux/Kbuild @@ -343,6 +343,7 @@ header-y += scc.h header-y += sched.h header-y += screen_info.h header-y += sdla.h +header-y += seccomp.h header-y += securebits.h header-y += selinux_netlink.h header-y += sem.h diff --git a/include/linux/audit.h b/include/linux/audit.h index ed3ef197249..22f292a917a 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -463,7 +463,7 @@ extern void audit_putname(const char *name); extern void __audit_inode(const char *name, const struct dentry *dentry); extern void __audit_inode_child(const struct dentry *dentry, const struct inode *parent); -extern void __audit_seccomp(unsigned long syscall); +extern void __audit_seccomp(unsigned long syscall, long signr, int code); extern void __audit_ptrace(struct task_struct *t); static inline int audit_dummy_context(void) @@ -508,10 +508,10 @@ static inline void audit_inode_child(const struct dentry *dentry, } void audit_core_dumps(long signr); -static inline void audit_seccomp(unsigned long syscall) +static inline void audit_seccomp(unsigned long syscall, long signr, int code) { if (unlikely(!audit_dummy_context())) - __audit_seccomp(syscall); + __audit_seccomp(syscall, signr, code); } static inline void audit_ptrace(struct task_struct *t) @@ -634,7 +634,7 @@ extern int audit_signals; #define audit_inode(n,d) do { (void)(d); } while (0) #define audit_inode_child(i,p) do { ; } while (0) #define audit_core_dumps(i) do { ; } while (0) -#define audit_seccomp(i) do { ; } while (0) +#define audit_seccomp(i,s,c) do { ; } while (0) #define auditsc_get_stamp(c,t,s) (0) #define audit_get_loginuid(t) (-1) #define audit_get_sessionid(t) (-1) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 6723f48f2d4..58442f6b277 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -20,6 +20,7 @@ #include #include #include +#include #define CPUFREQ_NAME_LEN 16 @@ -417,4 +418,10 @@ void cpufreq_frequency_table_get_attr(struct cpufreq_frequency_table *table, void cpufreq_frequency_table_put_attr(unsigned int cpu); +/********************************************************************* + * CPUFREQ STATS * + *********************************************************************/ + +void acct_update_power(struct task_struct *p, cputime_t cputime); + #endif /* _LINUX_CPUFREQ_H */ diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 7e11f141820..fcac0df71e2 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -398,6 +398,13 @@ static inline bool d_need_lookup(struct dentry *dentry) extern void d_clear_need_lookup(struct dentry *dentry); +static inline bool d_is_su(const struct dentry *dentry) +{ + return dentry && + dentry->d_name.len == 2 && + !memcmp(dentry->d_name.name, "su", 2); +} + extern int sysctl_vfs_cache_pressure; #endif /* __LINUX_DCACHE_H */ diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h new file mode 100644 index 00000000000..25c6324a0dd --- /dev/null +++ b/include/linux/f2fs_fs.h @@ -0,0 +1,494 @@ +/** + * include/linux/f2fs_fs.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _LINUX_F2FS_FS_H +#define _LINUX_F2FS_FS_H + +#include +#include + +#define F2FS_SUPER_OFFSET 1024 /* byte-size offset */ +#define F2FS_MIN_LOG_SECTOR_SIZE 9 /* 9 bits for 512 bytes */ +#define F2FS_MAX_LOG_SECTOR_SIZE 12 /* 12 bits for 4096 bytes */ +#define F2FS_LOG_SECTORS_PER_BLOCK 3 /* log number for sector/blk */ +#define F2FS_BLKSIZE 4096 /* support only 4KB block */ +#define F2FS_BLKSIZE_BITS 12 /* bits for F2FS_BLKSIZE */ +#define F2FS_MAX_EXTENSION 64 /* # of extension entries */ +#define F2FS_BLK_ALIGN(x) (((x) + F2FS_BLKSIZE - 1) / F2FS_BLKSIZE) + +#define NULL_ADDR ((block_t)0) /* used as block_t addresses */ +#define NEW_ADDR ((block_t)-1) /* used as block_t addresses */ + +#define F2FS_BYTES_TO_BLK(bytes) ((bytes) >> F2FS_BLKSIZE_BITS) +#define F2FS_BLK_TO_BYTES(blk) ((blk) << F2FS_BLKSIZE_BITS) + +/* 0, 1(node nid), 2(meta nid) are reserved node id */ +#define F2FS_RESERVED_NODE_NUM 3 + +#define F2FS_ROOT_INO(sbi) (sbi->root_ino_num) +#define F2FS_NODE_INO(sbi) (sbi->node_ino_num) +#define F2FS_META_INO(sbi) (sbi->meta_ino_num) + +/* This flag is used by node and meta inodes, and by recovery */ +#define GFP_F2FS_ZERO (GFP_NOFS | __GFP_ZERO) +#define GFP_F2FS_HIGH_ZERO (GFP_NOFS | __GFP_ZERO | __GFP_HIGHMEM) + +/* + * For further optimization on multi-head logs, on-disk layout supports maximum + * 16 logs by default. The number, 16, is expected to cover all the cases + * enoughly. The implementaion currently uses no more than 6 logs. + * Half the logs are used for nodes, and the other half are used for data. + */ +#define MAX_ACTIVE_LOGS 16 +#define MAX_ACTIVE_NODE_LOGS 8 +#define MAX_ACTIVE_DATA_LOGS 8 + +#define VERSION_LEN 256 + +/* + * For superblock + */ +struct f2fs_super_block { + __le32 magic; /* Magic Number */ + __le16 major_ver; /* Major Version */ + __le16 minor_ver; /* Minor Version */ + __le32 log_sectorsize; /* log2 sector size in bytes */ + __le32 log_sectors_per_block; /* log2 # of sectors per block */ + __le32 log_blocksize; /* log2 block size in bytes */ + __le32 log_blocks_per_seg; /* log2 # of blocks per segment */ + __le32 segs_per_sec; /* # of segments per section */ + __le32 secs_per_zone; /* # of sections per zone */ + __le32 checksum_offset; /* checksum offset inside super block */ + __le64 block_count; /* total # of user blocks */ + __le32 section_count; /* total # of sections */ + __le32 segment_count; /* total # of segments */ + __le32 segment_count_ckpt; /* # of segments for checkpoint */ + __le32 segment_count_sit; /* # of segments for SIT */ + __le32 segment_count_nat; /* # of segments for NAT */ + __le32 segment_count_ssa; /* # of segments for SSA */ + __le32 segment_count_main; /* # of segments for main area */ + __le32 segment0_blkaddr; /* start block address of segment 0 */ + __le32 cp_blkaddr; /* start block address of checkpoint */ + __le32 sit_blkaddr; /* start block address of SIT */ + __le32 nat_blkaddr; /* start block address of NAT */ + __le32 ssa_blkaddr; /* start block address of SSA */ + __le32 main_blkaddr; /* start block address of main area */ + __le32 root_ino; /* root inode number */ + __le32 node_ino; /* node inode number */ + __le32 meta_ino; /* meta inode number */ + __u8 uuid[16]; /* 128-bit uuid for volume */ + __le16 volume_name[512]; /* volume name */ + __le32 extension_count; /* # of extensions below */ + __u8 extension_list[F2FS_MAX_EXTENSION][8]; /* extension array */ + __le32 cp_payload; + __u8 version[VERSION_LEN]; /* the kernel version */ + __u8 init_version[VERSION_LEN]; /* the initial kernel version */ + __le32 feature; /* defined features */ + __u8 encryption_level; /* versioning level for encryption */ + __u8 encrypt_pw_salt[16]; /* Salt used for string2key algorithm */ + __u8 reserved[871]; /* valid reserved region */ +} __packed; + +/* + * For checkpoint + */ +#define CP_FASTBOOT_FLAG 0x00000020 +#define CP_FSCK_FLAG 0x00000010 +#define CP_ERROR_FLAG 0x00000008 +#define CP_COMPACT_SUM_FLAG 0x00000004 +#define CP_ORPHAN_PRESENT_FLAG 0x00000002 +#define CP_UMOUNT_FLAG 0x00000001 + +#define F2FS_CP_PACKS 2 /* # of checkpoint packs */ + +struct f2fs_checkpoint { + __le64 checkpoint_ver; /* checkpoint block version number */ + __le64 user_block_count; /* # of user blocks */ + __le64 valid_block_count; /* # of valid blocks in main area */ + __le32 rsvd_segment_count; /* # of reserved segments for gc */ + __le32 overprov_segment_count; /* # of overprovision segments */ + __le32 free_segment_count; /* # of free segments in main area */ + + /* information of current node segments */ + __le32 cur_node_segno[MAX_ACTIVE_NODE_LOGS]; + __le16 cur_node_blkoff[MAX_ACTIVE_NODE_LOGS]; + /* information of current data segments */ + __le32 cur_data_segno[MAX_ACTIVE_DATA_LOGS]; + __le16 cur_data_blkoff[MAX_ACTIVE_DATA_LOGS]; + __le32 ckpt_flags; /* Flags : umount and journal_present */ + __le32 cp_pack_total_block_count; /* total # of one cp pack */ + __le32 cp_pack_start_sum; /* start block number of data summary */ + __le32 valid_node_count; /* Total number of valid nodes */ + __le32 valid_inode_count; /* Total number of valid inodes */ + __le32 next_free_nid; /* Next free node number */ + __le32 sit_ver_bitmap_bytesize; /* Default value 64 */ + __le32 nat_ver_bitmap_bytesize; /* Default value 256 */ + __le32 checksum_offset; /* checksum offset inside cp block */ + __le64 elapsed_time; /* mounted time */ + /* allocation type of current segment */ + unsigned char alloc_type[MAX_ACTIVE_LOGS]; + + /* SIT and NAT version bitmap */ + unsigned char sit_nat_version_bitmap[1]; +} __packed; + +/* + * For orphan inode management + */ +#define F2FS_ORPHANS_PER_BLOCK 1020 + +#define GET_ORPHAN_BLOCKS(n) ((n + F2FS_ORPHANS_PER_BLOCK - 1) / \ + F2FS_ORPHANS_PER_BLOCK) + +struct f2fs_orphan_block { + __le32 ino[F2FS_ORPHANS_PER_BLOCK]; /* inode numbers */ + __le32 reserved; /* reserved */ + __le16 blk_addr; /* block index in current CP */ + __le16 blk_count; /* Number of orphan inode blocks in CP */ + __le32 entry_count; /* Total number of orphan nodes in current CP */ + __le32 check_sum; /* CRC32 for orphan inode block */ +} __packed; + +/* + * For NODE structure + */ +struct f2fs_extent { + __le32 fofs; /* start file offset of the extent */ + __le32 blk; /* start block address of the extent */ + __le32 len; /* lengh of the extent */ +} __packed; + +#define F2FS_NAME_LEN 255 +#define F2FS_INLINE_XATTR_ADDRS 50 /* 200 bytes for inline xattrs */ +#define DEF_ADDRS_PER_INODE 923 /* Address Pointers in an Inode */ +#define DEF_NIDS_PER_INODE 5 /* Node IDs in an Inode */ +#define ADDRS_PER_INODE(fi) addrs_per_inode(fi) +#define ADDRS_PER_BLOCK 1018 /* Address Pointers in a Direct Block */ +#define NIDS_PER_BLOCK 1018 /* Node IDs in an Indirect Block */ + +#define ADDRS_PER_PAGE(page, fi) \ + (IS_INODE(page) ? ADDRS_PER_INODE(fi) : ADDRS_PER_BLOCK) + +#define NODE_DIR1_BLOCK (DEF_ADDRS_PER_INODE + 1) +#define NODE_DIR2_BLOCK (DEF_ADDRS_PER_INODE + 2) +#define NODE_IND1_BLOCK (DEF_ADDRS_PER_INODE + 3) +#define NODE_IND2_BLOCK (DEF_ADDRS_PER_INODE + 4) +#define NODE_DIND_BLOCK (DEF_ADDRS_PER_INODE + 5) + +#define F2FS_INLINE_XATTR 0x01 /* file inline xattr flag */ +#define F2FS_INLINE_DATA 0x02 /* file inline data flag */ +#define F2FS_INLINE_DENTRY 0x04 /* file inline dentry flag */ +#define F2FS_DATA_EXIST 0x08 /* file inline data exist flag */ +#define F2FS_INLINE_DOTS 0x10 /* file having implicit dot dentries */ + +#define MAX_INLINE_DATA (sizeof(__le32) * (DEF_ADDRS_PER_INODE - \ + F2FS_INLINE_XATTR_ADDRS - 1)) + +struct f2fs_inode { + __le16 i_mode; /* file mode */ + __u8 i_advise; /* file hints */ + __u8 i_inline; /* file inline flags */ + __le32 i_uid; /* user ID */ + __le32 i_gid; /* group ID */ + __le32 i_links; /* links count */ + __le64 i_size; /* file size in bytes */ + __le64 i_blocks; /* file size in blocks */ + __le64 i_atime; /* access time */ + __le64 i_ctime; /* change time */ + __le64 i_mtime; /* modification time */ + __le32 i_atime_nsec; /* access time in nano scale */ + __le32 i_ctime_nsec; /* change time in nano scale */ + __le32 i_mtime_nsec; /* modification time in nano scale */ + __le32 i_generation; /* file version (for NFS) */ + __le32 i_current_depth; /* only for directory depth */ + __le32 i_xattr_nid; /* nid to save xattr */ + __le32 i_flags; /* file attributes */ + __le32 i_pino; /* parent inode number */ + __le32 i_namelen; /* file name length */ + __u8 i_name[F2FS_NAME_LEN]; /* file name for SPOR */ + __u8 i_dir_level; /* dentry_level for large dir */ + + struct f2fs_extent i_ext; /* caching a largest extent */ + + __le32 i_addr[DEF_ADDRS_PER_INODE]; /* Pointers to data blocks */ + + __le32 i_nid[DEF_NIDS_PER_INODE]; /* direct(2), indirect(2), + double_indirect(1) node id */ +} __packed; + +struct direct_node { + __le32 addr[ADDRS_PER_BLOCK]; /* array of data block address */ +} __packed; + +struct indirect_node { + __le32 nid[NIDS_PER_BLOCK]; /* array of data block address */ +} __packed; + +enum { + COLD_BIT_SHIFT = 0, + FSYNC_BIT_SHIFT, + DENT_BIT_SHIFT, + OFFSET_BIT_SHIFT +}; + +#define OFFSET_BIT_MASK (0x07) /* (0x01 << OFFSET_BIT_SHIFT) - 1 */ + +struct node_footer { + __le32 nid; /* node id */ + __le32 ino; /* inode nunmber */ + __le32 flag; /* include cold/fsync/dentry marks and offset */ + __le64 cp_ver; /* checkpoint version */ + __le32 next_blkaddr; /* next node page block address */ +} __packed; + +struct f2fs_node { + /* can be one of three types: inode, direct, and indirect types */ + union { + struct f2fs_inode i; + struct direct_node dn; + struct indirect_node in; + }; + struct node_footer footer; +} __packed; + +/* + * For NAT entries + */ +#define NAT_ENTRY_PER_BLOCK (PAGE_CACHE_SIZE / sizeof(struct f2fs_nat_entry)) + +struct f2fs_nat_entry { + __u8 version; /* latest version of cached nat entry */ + __le32 ino; /* inode number */ + __le32 block_addr; /* block address */ +} __packed; + +struct f2fs_nat_block { + struct f2fs_nat_entry entries[NAT_ENTRY_PER_BLOCK]; +} __packed; + +/* + * For SIT entries + * + * Each segment is 2MB in size by default so that a bitmap for validity of + * there-in blocks should occupy 64 bytes, 512 bits. + * Not allow to change this. + */ +#define SIT_VBLOCK_MAP_SIZE 64 +#define SIT_ENTRY_PER_BLOCK (PAGE_CACHE_SIZE / sizeof(struct f2fs_sit_entry)) + +/* + * Note that f2fs_sit_entry->vblocks has the following bit-field information. + * [15:10] : allocation type such as CURSEG_XXXX_TYPE + * [9:0] : valid block count + */ +#define SIT_VBLOCKS_SHIFT 10 +#define SIT_VBLOCKS_MASK ((1 << SIT_VBLOCKS_SHIFT) - 1) +#define GET_SIT_VBLOCKS(raw_sit) \ + (le16_to_cpu((raw_sit)->vblocks) & SIT_VBLOCKS_MASK) +#define GET_SIT_TYPE(raw_sit) \ + ((le16_to_cpu((raw_sit)->vblocks) & ~SIT_VBLOCKS_MASK) \ + >> SIT_VBLOCKS_SHIFT) + +struct f2fs_sit_entry { + __le16 vblocks; /* reference above */ + __u8 valid_map[SIT_VBLOCK_MAP_SIZE]; /* bitmap for valid blocks */ + __le64 mtime; /* segment age for cleaning */ +} __packed; + +struct f2fs_sit_block { + struct f2fs_sit_entry entries[SIT_ENTRY_PER_BLOCK]; +} __packed; + +/* + * For segment summary + * + * One summary block contains exactly 512 summary entries, which represents + * exactly 2MB segment by default. Not allow to change the basic units. + * + * NOTE: For initializing fields, you must use set_summary + * + * - If data page, nid represents dnode's nid + * - If node page, nid represents the node page's nid. + * + * The ofs_in_node is used by only data page. It represents offset + * from node's page's beginning to get a data block address. + * ex) data_blkaddr = (block_t)(nodepage_start_address + ofs_in_node) + */ +#define ENTRIES_IN_SUM 512 +#define SUMMARY_SIZE (7) /* sizeof(struct summary) */ +#define SUM_FOOTER_SIZE (5) /* sizeof(struct summary_footer) */ +#define SUM_ENTRY_SIZE (SUMMARY_SIZE * ENTRIES_IN_SUM) + +/* a summary entry for a 4KB-sized block in a segment */ +struct f2fs_summary { + __le32 nid; /* parent node id */ + union { + __u8 reserved[3]; + struct { + __u8 version; /* node version number */ + __le16 ofs_in_node; /* block index in parent node */ + } __packed; + }; +} __packed; + +/* summary block type, node or data, is stored to the summary_footer */ +#define SUM_TYPE_NODE (1) +#define SUM_TYPE_DATA (0) + +struct summary_footer { + unsigned char entry_type; /* SUM_TYPE_XXX */ + __u32 check_sum; /* summary checksum */ +} __packed; + +#define SUM_JOURNAL_SIZE (F2FS_BLKSIZE - SUM_FOOTER_SIZE -\ + SUM_ENTRY_SIZE) +#define NAT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ + sizeof(struct nat_journal_entry)) +#define NAT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ + sizeof(struct nat_journal_entry)) +#define SIT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ + sizeof(struct sit_journal_entry)) +#define SIT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ + sizeof(struct sit_journal_entry)) +/* + * frequently updated NAT/SIT entries can be stored in the spare area in + * summary blocks + */ +enum { + NAT_JOURNAL = 0, + SIT_JOURNAL +}; + +struct nat_journal_entry { + __le32 nid; + struct f2fs_nat_entry ne; +} __packed; + +struct nat_journal { + struct nat_journal_entry entries[NAT_JOURNAL_ENTRIES]; + __u8 reserved[NAT_JOURNAL_RESERVED]; +} __packed; + +struct sit_journal_entry { + __le32 segno; + struct f2fs_sit_entry se; +} __packed; + +struct sit_journal { + struct sit_journal_entry entries[SIT_JOURNAL_ENTRIES]; + __u8 reserved[SIT_JOURNAL_RESERVED]; +} __packed; + +/* 4KB-sized summary block structure */ +struct f2fs_summary_block { + struct f2fs_summary entries[ENTRIES_IN_SUM]; + union { + __le16 n_nats; + __le16 n_sits; + }; + /* spare area is used by NAT or SIT journals */ + union { + struct nat_journal nat_j; + struct sit_journal sit_j; + }; + struct summary_footer footer; +} __packed; + +/* + * For directory operations + */ +#define F2FS_DOT_HASH 0 +#define F2FS_DDOT_HASH F2FS_DOT_HASH +#define F2FS_MAX_HASH (~((0x3ULL) << 62)) +#define F2FS_HASH_COL_BIT ((0x1ULL) << 63) + +typedef __le32 f2fs_hash_t; + +/* One directory entry slot covers 8bytes-long file name */ +#define F2FS_SLOT_LEN 8 +#define F2FS_SLOT_LEN_BITS 3 + +#define GET_DENTRY_SLOTS(x) ((x + F2FS_SLOT_LEN - 1) >> F2FS_SLOT_LEN_BITS) + +/* MAX level for dir lookup */ +#define MAX_DIR_HASH_DEPTH 63 + +/* MAX buckets in one level of dir */ +#define MAX_DIR_BUCKETS (1 << ((MAX_DIR_HASH_DEPTH / 2) - 1)) + +/* + * space utilization of regular dentry and inline dentry + * regular dentry inline dentry + * bitmap 1 * 27 = 27 1 * 23 = 23 + * reserved 1 * 3 = 3 1 * 7 = 7 + * dentry 11 * 214 = 2354 11 * 182 = 2002 + * filename 8 * 214 = 1712 8 * 182 = 1456 + * total 4096 3488 + * + * Note: there are more reserved space in inline dentry than in regular + * dentry, when converting inline dentry we should handle this carefully. + */ +#define NR_DENTRY_IN_BLOCK 214 /* the number of dentry in a block */ +#define SIZE_OF_DIR_ENTRY 11 /* by byte */ +#define SIZE_OF_DENTRY_BITMAP ((NR_DENTRY_IN_BLOCK + BITS_PER_BYTE - 1) / \ + BITS_PER_BYTE) +#define SIZE_OF_RESERVED (PAGE_SIZE - ((SIZE_OF_DIR_ENTRY + \ + F2FS_SLOT_LEN) * \ + NR_DENTRY_IN_BLOCK + SIZE_OF_DENTRY_BITMAP)) + +/* One directory entry slot representing F2FS_SLOT_LEN-sized file name */ +struct f2fs_dir_entry { + __le32 hash_code; /* hash code of file name */ + __le32 ino; /* inode number */ + __le16 name_len; /* lengh of file name */ + __u8 file_type; /* file type */ +} __packed; + +/* 4KB-sized directory entry block */ +struct f2fs_dentry_block { + /* validity bitmap for directory entries in each block */ + __u8 dentry_bitmap[SIZE_OF_DENTRY_BITMAP]; + __u8 reserved[SIZE_OF_RESERVED]; + struct f2fs_dir_entry dentry[NR_DENTRY_IN_BLOCK]; + __u8 filename[NR_DENTRY_IN_BLOCK][F2FS_SLOT_LEN]; +} __packed; + +/* for inline dir */ +#define NR_INLINE_DENTRY (MAX_INLINE_DATA * BITS_PER_BYTE / \ + ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \ + BITS_PER_BYTE + 1)) +#define INLINE_DENTRY_BITMAP_SIZE ((NR_INLINE_DENTRY + \ + BITS_PER_BYTE - 1) / BITS_PER_BYTE) +#define INLINE_RESERVED_SIZE (MAX_INLINE_DATA - \ + ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \ + NR_INLINE_DENTRY + INLINE_DENTRY_BITMAP_SIZE)) + +/* inline directory entry structure */ +struct f2fs_inline_dentry { + __u8 dentry_bitmap[INLINE_DENTRY_BITMAP_SIZE]; + __u8 reserved[INLINE_RESERVED_SIZE]; + struct f2fs_dir_entry dentry[NR_INLINE_DENTRY]; + __u8 filename[NR_INLINE_DENTRY][F2FS_SLOT_LEN]; +} __packed; + +/* file types used in inode_info->flags */ +enum { + F2FS_FT_UNKNOWN, + F2FS_FT_REG_FILE, + F2FS_FT_DIR, + F2FS_FT_CHRDEV, + F2FS_FT_BLKDEV, + F2FS_FT_FIFO, + F2FS_FT_SOCK, + F2FS_FT_SYMLINK, + F2FS_FT_MAX +}; + +#endif /* _LINUX_F2FS_FS_H */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 8eeb205f298..f2e53152e83 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -10,6 +10,7 @@ #ifdef __KERNEL__ #include +#include #endif /* @@ -132,6 +133,16 @@ struct sock_fprog { /* Required for SO_ATTACH_FILTER. */ #ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +/* + * A struct sock_filter is architecture independent. + */ +struct compat_sock_fprog { + u16 len; + compat_uptr_t filter; /* struct sock_filter * */ +}; +#endif + struct sk_buff; struct sock; @@ -228,6 +239,7 @@ enum { BPF_S_ANC_HATYPE, BPF_S_ANC_RXHASH, BPF_S_ANC_CPU, + BPF_S_ANC_SECCOMP_LD_W, }; #endif /* __KERNEL__ */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 093f0b8878c..0b75d83371d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -490,6 +490,12 @@ struct iattr { */ #include +/* + * Maximum number of layers of fs stack. Needs to be limited to + * prevent kernel stack overflow + */ +#define FILESYSTEM_MAX_STACK_DEPTH 2 + /** * enum positive_aop_returns - aop return codes with specific semantics * @@ -1508,6 +1514,11 @@ struct super_block { /* Being remounted read-only */ int s_readonly_remount; + + /* + * Indicates how deep in a filesystem stack this SB is + */ + int s_stack_depth; }; /* superblock cache pruning functions */ @@ -1665,6 +1676,7 @@ struct inode_operations { void (*truncate_range)(struct inode *, loff_t, loff_t); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len); + int (*update_time)(struct inode *, struct timespec *, int); } ____cacheline_aligned; struct seq_file; @@ -1823,6 +1835,13 @@ static inline void inode_inc_iversion(struct inode *inode) spin_unlock(&inode->i_lock); } +enum file_time_flags { + S_ATIME = 1, + S_MTIME = 2, + S_CTIME = 4, + S_VERSION = 8, +}; + extern void touch_atime(struct path *); static inline void file_accessed(struct file *file) { @@ -1836,6 +1855,13 @@ int sync_inode_metadata(struct inode *inode, int wait); struct file_system_type { const char *name; int fs_flags; +#define FS_REQUIRES_DEV 1 +#define FS_BINARY_MOUNTDATA 2 +#define FS_HAS_SUBTYPE 4 +#define FS_USERNS_MOUNT 8 /* Can be mounted by userns root */ +#define FS_USERNS_DEV_MOUNT 16 /* A userns mount does not imply MNT_NODEV */ +#define FS_REVAL_DOT 16384 /* Check the paths ".", ".." for staleness */ +#define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ struct dentry *(*mount) (struct file_system_type *, int, const char *, void *); void (*kill_sb) (struct super_block *); @@ -1895,7 +1921,7 @@ extern struct vfsmount *kern_mount_data(struct file_system_type *, void *data); extern void kern_unmount(struct vfsmount *mnt); extern int may_umount_tree(struct vfsmount *); extern int may_umount(struct vfsmount *); -extern long do_mount(char *, char *, char *, unsigned long, void *); +extern long do_mount(const char *, const char *, const char *, unsigned long, void *); extern struct vfsmount *collect_mounts(struct path *); extern void drop_collected_mounts(struct vfsmount *); extern int iterate_mounts(int (*)(struct vfsmount *, void *), void *, @@ -2559,7 +2585,7 @@ extern int inode_change_ok(const struct inode *, struct iattr *); extern int inode_newsize_ok(const struct inode *, loff_t offset); extern void setattr_copy(struct inode *inode, const struct iattr *attr); -extern void file_update_time(struct file *file); +extern int file_update_time(struct file *file); extern int generic_show_options(struct seq_file *m, struct dentry *root); extern void save_mount_options(struct super_block *sb, char *options); diff --git a/include/linux/hashtable.h b/include/linux/hashtable.h new file mode 100644 index 00000000000..227c62424f3 --- /dev/null +++ b/include/linux/hashtable.h @@ -0,0 +1,192 @@ +/* + * Statically sized hash table implementation + * (C) 2012 Sasha Levin + */ + +#ifndef _LINUX_HASHTABLE_H +#define _LINUX_HASHTABLE_H + +#include +#include +#include +#include +#include + +#define DEFINE_HASHTABLE(name, bits) \ + struct hlist_head name[1 << (bits)] = \ + { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT } + +#define DECLARE_HASHTABLE(name, bits) \ + struct hlist_head name[1 << (bits)] + +#define HASH_SIZE(name) (ARRAY_SIZE(name)) +#define HASH_BITS(name) ilog2(HASH_SIZE(name)) + +/* Use hash_32 when possible to allow for fast 32bit hashing in 64bit kernels. */ +#define hash_min(val, bits) \ + (sizeof(val) <= 4 ? hash_32(val, bits) : hash_long(val, bits)) + +static inline void __hash_init(struct hlist_head *ht, unsigned int sz) +{ + unsigned int i; + + for (i = 0; i < sz; i++) + INIT_HLIST_HEAD(&ht[i]); +} + +/** + * hash_init - initialize a hash table + * @hashtable: hashtable to be initialized + * + * Calculates the size of the hashtable from the given parameter, otherwise + * same as hash_init_size. + * + * This has to be a macro since HASH_BITS() will not work on pointers since + * it calculates the size during preprocessing. + */ +#define hash_init(hashtable) __hash_init(hashtable, HASH_SIZE(hashtable)) + +/** + * hash_add - add an object to a hashtable + * @hashtable: hashtable to add to + * @node: the &struct hlist_node of the object to be added + * @key: the key of the object to be added + */ +#define hash_add(hashtable, node, key) \ + hlist_add_head(node, &hashtable[hash_min(key, HASH_BITS(hashtable))]) + +/** + * hash_add_rcu - add an object to a rcu enabled hashtable + * @hashtable: hashtable to add to + * @node: the &struct hlist_node of the object to be added + * @key: the key of the object to be added + */ +#define hash_add_rcu(hashtable, node, key) \ + hlist_add_head_rcu(node, &hashtable[hash_min(key, HASH_BITS(hashtable))]) + +/** + * hash_hashed - check whether an object is in any hashtable + * @node: the &struct hlist_node of the object to be checked + */ +static inline bool hash_hashed(struct hlist_node *node) +{ + return !hlist_unhashed(node); +} + +static inline bool __hash_empty(struct hlist_head *ht, unsigned int sz) +{ + unsigned int i; + + for (i = 0; i < sz; i++) + if (!hlist_empty(&ht[i])) + return false; + + return true; +} + +/** + * hash_empty - check whether a hashtable is empty + * @hashtable: hashtable to check + * + * This has to be a macro since HASH_BITS() will not work on pointers since + * it calculates the size during preprocessing. + */ +#define hash_empty(hashtable) __hash_empty(hashtable, HASH_SIZE(hashtable)) + +/** + * hash_del - remove an object from a hashtable + * @node: &struct hlist_node of the object to remove + */ +static inline void hash_del(struct hlist_node *node) +{ + hlist_del_init(node); +} + +/** + * hash_del_rcu - remove an object from a rcu enabled hashtable + * @node: &struct hlist_node of the object to remove + */ +static inline void hash_del_rcu(struct hlist_node *node) +{ + hlist_del_init_rcu(node); +} + +/** + * hash_for_each - iterate over a hashtable + * @name: hashtable to iterate + * @bkt: integer to use as bucket loop cursor + * @node: the &struct list_head to use as a loop cursor for each entry + * @obj: the type * to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + */ +#define hash_for_each(name, bkt, node, obj, member) \ + for ((bkt) = 0, node = NULL; node == NULL && (bkt) < HASH_SIZE(name); (bkt)++)\ + hlist_for_each_entry(obj, node, &name[bkt], member) + +/** + * hash_for_each_rcu - iterate over a rcu enabled hashtable + * @name: hashtable to iterate + * @bkt: integer to use as bucket loop cursor + * @node: the &struct list_head to use as a loop cursor for each entry + * @obj: the type * to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + */ +#define hash_for_each_rcu(name, bkt, node, obj, member) \ + for ((bkt) = 0, node = NULL; node == NULL && (bkt) < HASH_SIZE(name); (bkt)++)\ + hlist_for_each_entry_rcu(obj, node, &name[bkt], member) + +/** + * hash_for_each_safe - iterate over a hashtable safe against removal of + * hash entry + * @name: hashtable to iterate + * @bkt: integer to use as bucket loop cursor + * @node: the &struct list_head to use as a loop cursor for each entry + * @tmp: a &struct used for temporary storage + * @obj: the type * to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + */ +#define hash_for_each_safe(name, bkt, node, tmp, obj, member) \ + for ((bkt) = 0, node = NULL; node == NULL && (bkt) < HASH_SIZE(name); (bkt)++)\ + hlist_for_each_entry_safe(obj, node, tmp, &name[bkt], member) + +/** + * hash_for_each_possible - iterate over all possible objects hashing to the + * same bucket + * @name: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @node: the &struct list_head to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + * @key: the key of the objects to iterate over + */ +#define hash_for_each_possible(name, obj, node, member, key) \ + hlist_for_each_entry(obj, node, &name[hash_min(key, HASH_BITS(name))], member) + +/** + * hash_for_each_possible_rcu - iterate over all possible objects hashing to the + * same bucket in an rcu enabled hashtable + * in a rcu enabled hashtable + * @name: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @node: the &struct list_head to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + * @key: the key of the objects to iterate over + */ +#define hash_for_each_possible_rcu(name, obj, node, member, key) \ + hlist_for_each_entry_rcu(obj, node, &name[hash_min(key, HASH_BITS(name))], member) + +/** + * hash_for_each_possible_safe - iterate over all possible objects hashing to the + * same bucket safe against removals + * @name: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @node: the &struct list_head to use as a loop cursor for each entry + * @tmp: a &struct used for temporary storage + * @member: the name of the hlist_node within the struct + * @key: the key of the objects to iterate over + */ +#define hash_for_each_possible_safe(name, obj, node, tmp, member, key) \ + hlist_for_each_entry_safe(obj, node, tmp, \ + &name[hash_min(key, HASH_BITS(name))], member) + + +#endif diff --git a/include/linux/init_task.h b/include/linux/init_task.h index e4baff5f7ff..58eb47115ed 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -38,6 +38,7 @@ extern struct fs_struct init_fs; #define INIT_SIGNALS(sig) { \ .nr_threads = 1, \ + .thread_head = LIST_HEAD_INIT(init_task.thread_node), \ .wait_chldexit = __WAIT_QUEUE_HEAD_INITIALIZER(sig.wait_chldexit),\ .shared_pending = { \ .list = LIST_HEAD_INIT(sig.shared_pending.list), \ @@ -192,6 +193,7 @@ extern struct cred init_cred; [PIDTYPE_SID] = INIT_PID_LINK(PIDTYPE_SID), \ }, \ .thread_group = LIST_HEAD_INIT(tsk.thread_group), \ + .thread_node = LIST_HEAD_INIT(init_signals.thread_head), \ INIT_IDS \ INIT_PERF_EVENTS(tsk) \ INIT_TRACE_IRQFLAGS \ diff --git a/include/linux/ion.h b/include/linux/ion.h index 498331660c5..e3ace18d314 100644 --- a/include/linux/ion.h +++ b/include/linux/ion.h @@ -22,6 +22,7 @@ #include struct ion_handle; +typedef struct ion_handle *ion_user_handle_t; /** * enum ion_heap_types - list of all possible types of heaps * @ION_HEAP_TYPE_SYSTEM: memory allocated via vmalloc @@ -340,9 +341,13 @@ static inline int ion_handle_get_flags(struct ion_client *client, struct ion_allocation_data { size_t len; size_t align; +#ifdef __KERNEL__ unsigned int heap_mask; +#else + unsigned int heap_id_mask; +#endif unsigned int flags; - struct ion_handle *handle; + ion_user_handle_t handle; }; /** @@ -356,7 +361,7 @@ struct ion_allocation_data { * provides the file descriptor and the kernel returns the handle. */ struct ion_fd_data { - struct ion_handle *handle; + ion_user_handle_t handle; int fd; }; @@ -365,7 +370,7 @@ struct ion_fd_data { * @handle: a handle */ struct ion_handle_data { - struct ion_handle *handle; + ion_user_handle_t handle; }; /** diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h index 8a297a5e794..0bd5a4e75ee 100644 --- a/include/linux/ipc_namespace.h +++ b/include/linux/ipc_namespace.h @@ -65,6 +65,8 @@ struct ipc_namespace { /* user_ns which owns the ipc ns */ struct user_namespace *user_ns; + + unsigned int proc_inum; }; extern struct ipc_namespace init_ipc_ns; diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 07f506310ab..8e3f2cb7d7c 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -166,6 +166,7 @@ struct ipv6_devconf { __s32 accept_source_route; #ifdef CONFIG_IPV6_OPTIMISTIC_DAD __s32 optimistic_dad; + __s32 use_optimistic; #endif #ifdef CONFIG_IPV6_MROUTE __s32 mc_forwarding; @@ -174,6 +175,7 @@ struct ipv6_devconf { __s32 accept_dad; __s32 force_tllao; __s32 accept_ra_prefix_route; + __s32 use_oif_addrs_only; void *sysctl; }; @@ -217,6 +219,8 @@ enum { DEVCONF_FORCE_TLLAO, DEVCONF_ACCEPT_RA_PREFIX_ROUTE, DEVCONF_ACCEPT_RA_RT_TABLE, + DEVCONF_USE_OPTIMISTIC, + DEVCONF_USE_OIF_ADDRS_ONLY, DEVCONF_MAX }; @@ -378,7 +382,7 @@ struct ipv6_pinfo { struct ipv6_ac_socklist *ipv6_ac_list; struct ipv6_fl_socklist *ipv6_fl_list; - struct ipv6_txoptions *opt; + struct ipv6_txoptions __rcu *opt; struct sk_buff *pktoptions; struct sk_buff *rxpmtu; struct { diff --git a/include/linux/irq.h b/include/linux/irq.h index 2a67ab2c666..6f9e90b3270 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -32,7 +32,7 @@ struct seq_file; struct module; struct irq_desc; struct irq_data; -typedef void (*irq_flow_handler_t)(unsigned int irq, +typedef bool (*irq_flow_handler_t)(unsigned int irq, struct irq_desc *desc); typedef void (*irq_preflow_handler_t)(struct irq_data *data); @@ -406,15 +406,15 @@ extern int no_irq_affinity; * Built-in IRQ handlers for various IRQ types, * callable via desc->handle_irq() */ -extern void handle_level_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_edge_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_simple_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_percpu_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_bad_irq(unsigned int irq, struct irq_desc *desc); -extern void handle_nested_irq(unsigned int irq); +extern bool handle_level_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_edge_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_simple_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_percpu_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_bad_irq(unsigned int irq, struct irq_desc *desc); +extern bool handle_nested_irq(unsigned int irq); /* Handling of unhandled and spurious interrupts: */ extern void note_interrupt(unsigned int irq, struct irq_desc *desc, diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index 4d22be25b48..6ae21bcc0ce 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -108,9 +108,10 @@ static inline struct msi_desc *irq_desc_get_msi_desc(struct irq_desc *desc) * irqchip-style controller then we call the ->handle_irq() handler, * and it calls __do_IRQ() if it's attached to an irqtype-style controller. */ -static inline void generic_handle_irq_desc(unsigned int irq, struct irq_desc *desc) + +static inline bool generic_handle_irq_desc(unsigned int irq, struct irq_desc *desc) { - desc->handle_irq(irq, desc); + return desc->handle_irq(irq, desc); } int generic_handle_irq(unsigned int irq); diff --git a/include/linux/lglock.h b/include/linux/lglock.h index 87f402ccec5..f01e5f6d1f0 100644 --- a/include/linux/lglock.h +++ b/include/linux/lglock.h @@ -23,28 +23,17 @@ #include #include #include +#include /* can make br locks by using local lock for read side, global lock for write */ -#define br_lock_init(name) name##_lock_init() -#define br_read_lock(name) name##_local_lock() -#define br_read_unlock(name) name##_local_unlock() -#define br_write_lock(name) name##_global_lock_online() -#define br_write_unlock(name) name##_global_unlock_online() +#define br_lock_init(name) lg_lock_init(name, #name) +#define br_read_lock(name) lg_local_lock(name) +#define br_read_unlock(name) lg_local_unlock(name) +#define br_write_lock(name) lg_global_lock(name) +#define br_write_unlock(name) lg_global_unlock(name) -#define DECLARE_BRLOCK(name) DECLARE_LGLOCK(name) #define DEFINE_BRLOCK(name) DEFINE_LGLOCK(name) - -#define lg_lock_init(name) name##_lock_init() -#define lg_local_lock(name) name##_local_lock() -#define lg_local_unlock(name) name##_local_unlock() -#define lg_local_lock_cpu(name, cpu) name##_local_lock_cpu(cpu) -#define lg_local_unlock_cpu(name, cpu) name##_local_unlock_cpu(cpu) -#define lg_global_lock(name) name##_global_lock() -#define lg_global_unlock(name) name##_global_unlock() -#define lg_global_lock_online(name) name##_global_lock_online() -#define lg_global_unlock_online(name) name##_global_unlock_online() - #ifdef CONFIG_DEBUG_LOCK_ALLOC #define LOCKDEP_INIT_MAP lockdep_init_map @@ -59,142 +48,26 @@ #define DEFINE_LGLOCK_LOCKDEP(name) #endif - -#define DECLARE_LGLOCK(name) \ - extern void name##_lock_init(void); \ - extern void name##_local_lock(void); \ - extern void name##_local_unlock(void); \ - extern void name##_local_lock_cpu(int cpu); \ - extern void name##_local_unlock_cpu(int cpu); \ - extern void name##_global_lock(void); \ - extern void name##_global_unlock(void); \ - extern void name##_global_lock_online(void); \ - extern void name##_global_unlock_online(void); \ +struct lglock { + arch_spinlock_t __percpu *lock; +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lock_class_key lock_key; + struct lockdep_map lock_dep_map; +#endif +}; #define DEFINE_LGLOCK(name) \ - \ - DEFINE_SPINLOCK(name##_cpu_lock); \ - cpumask_t name##_cpus __read_mostly; \ - DEFINE_PER_CPU(arch_spinlock_t, name##_lock); \ - DEFINE_LGLOCK_LOCKDEP(name); \ - \ - static int \ - name##_lg_cpu_callback(struct notifier_block *nb, \ - unsigned long action, void *hcpu) \ - { \ - switch (action & ~CPU_TASKS_FROZEN) { \ - case CPU_UP_PREPARE: \ - spin_lock(&name##_cpu_lock); \ - cpu_set((unsigned long)hcpu, name##_cpus); \ - spin_unlock(&name##_cpu_lock); \ - break; \ - case CPU_UP_CANCELED: case CPU_DEAD: \ - spin_lock(&name##_cpu_lock); \ - cpu_clear((unsigned long)hcpu, name##_cpus); \ - spin_unlock(&name##_cpu_lock); \ - } \ - return NOTIFY_OK; \ - } \ - static struct notifier_block name##_lg_cpu_notifier = { \ - .notifier_call = name##_lg_cpu_callback, \ - }; \ - void name##_lock_init(void) { \ - int i; \ - LOCKDEP_INIT_MAP(&name##_lock_dep_map, #name, &name##_lock_key, 0); \ - for_each_possible_cpu(i) { \ - arch_spinlock_t *lock; \ - lock = &per_cpu(name##_lock, i); \ - *lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; \ - } \ - register_hotcpu_notifier(&name##_lg_cpu_notifier); \ - get_online_cpus(); \ - for_each_online_cpu(i) \ - cpu_set(i, name##_cpus); \ - put_online_cpus(); \ - } \ - EXPORT_SYMBOL(name##_lock_init); \ - \ - void name##_local_lock(void) { \ - arch_spinlock_t *lock; \ - preempt_disable(); \ - rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \ - lock = &__get_cpu_var(name##_lock); \ - arch_spin_lock(lock); \ - } \ - EXPORT_SYMBOL(name##_local_lock); \ - \ - void name##_local_unlock(void) { \ - arch_spinlock_t *lock; \ - rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \ - lock = &__get_cpu_var(name##_lock); \ - arch_spin_unlock(lock); \ - preempt_enable(); \ - } \ - EXPORT_SYMBOL(name##_local_unlock); \ - \ - void name##_local_lock_cpu(int cpu) { \ - arch_spinlock_t *lock; \ - preempt_disable(); \ - rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \ - lock = &per_cpu(name##_lock, cpu); \ - arch_spin_lock(lock); \ - } \ - EXPORT_SYMBOL(name##_local_lock_cpu); \ - \ - void name##_local_unlock_cpu(int cpu) { \ - arch_spinlock_t *lock; \ - rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \ - lock = &per_cpu(name##_lock, cpu); \ - arch_spin_unlock(lock); \ - preempt_enable(); \ - } \ - EXPORT_SYMBOL(name##_local_unlock_cpu); \ - \ - void name##_global_lock_online(void) { \ - int i; \ - spin_lock(&name##_cpu_lock); \ - rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \ - for_each_cpu(i, &name##_cpus) { \ - arch_spinlock_t *lock; \ - lock = &per_cpu(name##_lock, i); \ - arch_spin_lock(lock); \ - } \ - } \ - EXPORT_SYMBOL(name##_global_lock_online); \ - \ - void name##_global_unlock_online(void) { \ - int i; \ - rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \ - for_each_cpu(i, &name##_cpus) { \ - arch_spinlock_t *lock; \ - lock = &per_cpu(name##_lock, i); \ - arch_spin_unlock(lock); \ - } \ - spin_unlock(&name##_cpu_lock); \ - } \ - EXPORT_SYMBOL(name##_global_unlock_online); \ - \ - void name##_global_lock(void) { \ - int i; \ - preempt_disable(); \ - rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \ - for_each_possible_cpu(i) { \ - arch_spinlock_t *lock; \ - lock = &per_cpu(name##_lock, i); \ - arch_spin_lock(lock); \ - } \ - } \ - EXPORT_SYMBOL(name##_global_lock); \ - \ - void name##_global_unlock(void) { \ - int i; \ - rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \ - for_each_possible_cpu(i) { \ - arch_spinlock_t *lock; \ - lock = &per_cpu(name##_lock, i); \ - arch_spin_unlock(lock); \ - } \ - preempt_enable(); \ - } \ - EXPORT_SYMBOL(name##_global_unlock); + DEFINE_LGLOCK_LOCKDEP(name); \ + DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ + = __ARCH_SPIN_LOCK_UNLOCKED; \ + struct lglock name = { .lock = &name ## _lock } + +void lg_lock_init(struct lglock *lg, char *name); +void lg_local_lock(struct lglock *lg); +void lg_local_unlock(struct lglock *lg); +void lg_local_lock_cpu(struct lglock *lg, int cpu); +void lg_local_unlock_cpu(struct lglock *lg, int cpu); +void lg_global_lock(struct lglock *lg); +void lg_global_unlock(struct lglock *lg); + #endif diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h index fad48aab893..9323a9861d2 100644 --- a/include/linux/lsm_audit.h +++ b/include/linux/lsm_audit.h @@ -40,6 +40,11 @@ struct lsm_network_audit { } fam; }; +struct lsm_ioctlop_audit { + struct path path; + u16 cmd; +}; + /* Auxiliary data to use in generating the audit record. */ struct common_audit_data { char type; @@ -53,6 +58,7 @@ struct common_audit_data { #define LSM_AUDIT_DATA_KMOD 8 #define LSM_AUDIT_DATA_INODE 9 #define LSM_AUDIT_DATA_DENTRY 10 +#define LSM_AUDIT_DATA_IOCTL_OP 11 struct task_struct *tsk; union { struct path path; @@ -69,6 +75,7 @@ struct common_audit_data { } key_struct; #endif char *kmod_name; + struct lsm_ioctlop_audit *op; } u; /* this union contains LSM specific data */ union { diff --git a/include/linux/magic.h b/include/linux/magic.h index e15192cb9cf..66353ffd06a 100644 --- a/include/linux/magic.h +++ b/include/linux/magic.h @@ -23,6 +23,7 @@ #define EXT4_SUPER_MAGIC 0xEF53 #define BTRFS_SUPER_MAGIC 0x9123683E #define NILFS_SUPER_MAGIC 0x3434 +#define F2FS_SUPER_MAGIC 0xF2F52010 #define HPFS_SUPER_MAGIC 0xf995e849 #define ISOFS_SUPER_MAGIC 0x9660 #define JFFS2_SUPER_MAGIC 0x72b6 diff --git a/include/linux/mm.h b/include/linux/mm.h index 6eba591f0a9..c86d8390724 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -43,6 +43,17 @@ extern int sysctl_legacy_va_layout; #define sysctl_legacy_va_layout 0 #endif +#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS +extern const int mmap_rnd_bits_min; +extern const int mmap_rnd_bits_max; +extern int mmap_rnd_bits __read_mostly; +#endif +#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS +extern const int mmap_rnd_compat_bits_min; +extern const int mmap_rnd_compat_bits_max; +extern int mmap_rnd_compat_bits __read_mostly; +#endif + #include #include #include @@ -321,6 +332,8 @@ static inline int is_vmalloc_or_module_addr(const void *x) } #endif +extern void kvfree(const void *addr); + static inline void compound_lock(struct page *page) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1126,6 +1139,11 @@ static inline void update_hiwater_vm(struct mm_struct *mm) mm->hiwater_vm = mm->total_vm; } +static inline void reset_mm_hiwater_rss(struct mm_struct *mm) +{ + mm->hiwater_rss = get_mm_rss(mm); +} + static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { @@ -1524,6 +1542,7 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address, #define FOLL_MLOCK 0x40 /* mark page as mlocked */ #define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */ #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */ +#define FOLL_COW 0x4000 /* internal GUP flag */ typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, void *data); diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index 2cb297e2b1e..13d925915ec 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -346,7 +346,8 @@ struct mmc_card { #define MMC_QUIRK_LONG_READ_TIME (1<<9) /* Data read time > CSD says */ #define MMC_QUIRK_SEC_ERASE_TRIM_BROKEN (1<<10) /* Skip secure for erase/trim */ /* byte mode */ -#define MMC_QUIRK_INAND_DATA_TIMEOUT (1<<8) /* For incorrect data timeout */ +#define MMC_QUIRK_INAND_DATA_TIMEOUT (1<<11) /* For incorrect data timeout */ +#define MMC_QUIRK_BLK_NEED_DUMMY_READ (1<<12) /* Issue dummy read in some cases */ unsigned int erase_size; /* erase size in sectors */ unsigned int erase_shift; /* if erase unit is power 2 */ diff --git a/include/linux/mnt_namespace.h b/include/linux/mnt_namespace.h index 5a8e3903d77..12b2ab51032 100644 --- a/include/linux/mnt_namespace.h +++ b/include/linux/mnt_namespace.h @@ -4,9 +4,10 @@ struct mnt_namespace; struct fs_struct; +struct user_namespace; extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *, - struct fs_struct *); + struct user_namespace *, struct fs_struct *); extern void put_mnt_ns(struct mnt_namespace *ns); extern const struct file_operations proc_mounts_operations; diff --git a/include/linux/mount.h b/include/linux/mount.h index d7029f4a191..2044aacc0d2 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -42,7 +42,9 @@ struct mnt_namespace; * flag, consider how it interacts with shared mounts. */ #define MNT_SHARED_MASK (MNT_UNBINDABLE) -#define MNT_PROPAGATION_MASK (MNT_SHARED | MNT_UNBINDABLE) +#define MNT_USER_SETTABLE_MASK (MNT_NOSUID | MNT_NODEV | MNT_NOEXEC \ + | MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME \ + | MNT_READONLY) #define MNT_INTERNAL 0x4000 diff --git a/include/linux/msm_vidc_dec.h b/include/linux/msm_vidc_dec.h index 6a1c3cb02bf..801da978bab 100644 --- a/include/linux/msm_vidc_dec.h +++ b/include/linux/msm_vidc_dec.h @@ -58,6 +58,9 @@ #define VDEC_MSG_EVT_HW_ERROR (VDEC_MSG_BASE + 14) #define VDEC_MSG_EVT_INFO_CONFIG_CHANGED (VDEC_MSG_BASE + 15) #define VDEC_MSG_EVT_INFO_FIELD_DROPPED (VDEC_MSG_BASE + 16) +#define VDEC_MSG_EVT_HW_OVERLOAD (VDEC_MSG_BASE + 17) +#define VDEC_MSG_EVT_MAX_CLIENTS (VDEC_MSG_BASE + 18) +#define VDEC_MSG_EVT_HW_UNSUPPORTED (VDEC_MSG_BASE + 19) /*Buffer flags bits masks.*/ #define VDEC_BUFFERFLAG_EOS 0x00000001 @@ -550,6 +553,7 @@ struct vdec_output_frameinfo { enum vdec_picture pic_type; void *client_data; void *input_frame_clientdata; + struct vdec_picsize picsize; struct vdec_framesize framesize; enum vdec_interlaced_format interlaced_format; struct vdec_aspectratioinfo aspect_ratio_info; diff --git a/include/linux/msm_vidc_enc.h b/include/linux/msm_vidc_enc.h index 4ce3db18839..d28bd9eab87 100644 --- a/include/linux/msm_vidc_enc.h +++ b/include/linux/msm_vidc_enc.h @@ -45,6 +45,8 @@ #define VEN_MSG_RESUME 9 #define VEN_MSG_STOP_READING_MSG 10 #define VEN_MSG_LTRUSE_FAILED 11 +#define VEN_MSG_HW_OVERLOAD 12 +#define VEN_MSG_MAX_CLIENTS 13 /*Buffer flags bits masks*/ diff --git a/include/linux/oom.h b/include/linux/oom.h index 3d7647536b0..f047a543211 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -67,6 +67,9 @@ static inline void oom_killer_enable(void) extern struct task_struct *find_lock_task_mm(struct task_struct *p); +extern void dump_tasks(const struct mem_cgroup *memcg, + const nodemask_t *nodemask); + /* sysctls */ extern int sysctl_oom_dump_tasks; extern int sysctl_oom_kill_allocating_task; diff --git a/include/linux/partialresume.h b/include/linux/partialresume.h new file mode 100644 index 00000000000..f52e612a1ce --- /dev/null +++ b/include/linux/partialresume.h @@ -0,0 +1,55 @@ +/* include/linux/partialresume.h + * + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_PARTIALRESUME_H +#define _LINUX_PARTIALRESUME_H + +#ifdef CONFIG_PARTIALRESUME + +#include + +struct partial_resume_stats { + unsigned total; + unsigned total_yes; +}; + +struct partial_resume { + struct list_head next_handler; + struct list_head next_match; + int irq; + struct partial_resume_stats stats; + void *private; + bool (*partial_resume)(struct partial_resume *); +}; + +int register_partial_resume(struct partial_resume *handler); +void unregister_partial_resume(struct partial_resume *handler); + +bool suspend_again_match(const struct list_head *irqs, + const struct list_head *unfinished); +bool suspend_again_consensus(void); + +#else /* !CONFIG_PARTIALRESUME */ + +struct partial_resume; +static inline int register_partial_resume(struct partial_resume *handler) { return 0; } +static inline void unregister_partial_resume(struct partial_resume *handler) {} +static inline bool suspend_again_match(const struct list_head *irqs) { return false; } +static inline bool suspend_again_consensus(void) { return false; } + +#endif + +#endif + diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 289754eb4f3..e2d526977ab 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1251,6 +1251,11 @@ extern int perf_proc_update_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +static inline bool perf_paranoid_any(void) +{ + return sysctl_perf_event_paranoid > 2; +} + static inline bool perf_paranoid_tracepoint_raw(void) { return sysctl_perf_event_paranoid > -1; diff --git a/include/linux/pid_namespace.h b/include/linux/pid_namespace.h index b067bd8c49d..375527bab2c 100644 --- a/include/linux/pid_namespace.h +++ b/include/linux/pid_namespace.h @@ -34,6 +34,7 @@ struct pid_namespace { gid_t pid_gid; int hide_pid; int reboot; /* group exit code if this pidns was rebooted */ + unsigned int proc_inum; }; extern struct pid_namespace init_pid_ns; diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index e1ac1ce16fb..a900a320d1e 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -41,6 +41,7 @@ struct pipe_buffer { * @fasync_writers: writer side fasync * @inode: inode this pipe is attached to * @bufs: the circular array of pipe buffers + * @user: the user who created this pipe **/ struct pipe_inode_info { wait_queue_head_t wait; @@ -55,6 +56,7 @@ struct pipe_inode_info { struct fasync_struct *fasync_writers; struct inode *inode; struct pipe_buffer *bufs; + struct user_struct *user; }; /* @@ -140,6 +142,8 @@ void pipe_unlock(struct pipe_inode_info *); void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *); extern unsigned int pipe_max_size, pipe_min_size; +extern unsigned long pipe_user_pages_hard; +extern unsigned long pipe_user_pages_soft; int pipe_proc_fn(struct ctl_table *, int, void __user *, size_t *, loff_t *); diff --git a/include/linux/poison.h b/include/linux/poison.h index 2110a81c5e2..253c9b4198e 100644 --- a/include/linux/poison.h +++ b/include/linux/poison.h @@ -19,8 +19,8 @@ * under normal circumstances, used to verify that nobody uses * non-initialized list entries. */ -#define LIST_POISON1 ((void *) 0x00100100 + POISON_POINTER_DELTA) -#define LIST_POISON2 ((void *) 0x00200200 + POISON_POINTER_DELTA) +#define LIST_POISON1 ((void *) 0x100 + POISON_POINTER_DELTA) +#define LIST_POISON2 ((void *) 0x200 + POISON_POINTER_DELTA) /********** include/linux/timer.h **********/ /* diff --git a/include/linux/prctl.h b/include/linux/prctl.h index 23728f03184..ffb364a5c4a 100644 --- a/include/linux/prctl.h +++ b/include/linux/prctl.h @@ -133,4 +133,19 @@ #define PR_SET_VMA 0x53564d41 # define PR_SET_VMA_ANON_NAME 0 +/* + * If no_new_privs is set, then operations that grant new privileges (i.e. + * execve) will either fail or not grant them. This affects suid/sgid, + * file capabilities, and LSMs. + * + * Operations that merely manipulate or drop existing privileges (setresuid, + * capset, etc.) will still work. Drop those privileges if you want them gone. + * + * Changing LSM security domain is considered a new privilege. So, for example, + * asking selinux for a specific new context (e.g. with runcon) will result + * in execve returning -EPERM. + */ +#define PR_SET_NO_NEW_PRIVS 38 +#define PR_GET_NO_NEW_PRIVS 39 + #endif /* _LINUX_PRCTL_H */ diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 85c50730623..6c890170af8 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -28,7 +28,11 @@ struct mm_struct; */ enum { - PROC_ROOT_INO = 1, + PROC_ROOT_INO = 1, + PROC_IPC_INIT_INO = 0xEFFFFFFFU, + PROC_UTS_INIT_INO = 0xEFFFFFFEU, + PROC_USER_INIT_INO = 0xEFFFFFFDU, + PROC_PID_INIT_INO = 0xEFFFFFFCU, }; /* @@ -174,7 +178,10 @@ extern struct proc_dir_entry *proc_net_mkdir(struct net *net, const char *name, struct proc_dir_entry *parent); extern struct file *proc_ns_fget(int fd); +extern bool proc_ns_inode(struct inode *inode); +extern int proc_alloc_inum(unsigned int *pino); +extern void proc_free_inum(unsigned int inum); #else #define proc_net_fops_create(net, name, mode, fops) ({ (void)(mode), NULL; }) @@ -229,6 +236,19 @@ static inline struct file *proc_ns_fget(int fd) return ERR_PTR(-EINVAL); } +static inline bool proc_ns_inode(struct inode *inode) +{ + return false; +} + +static inline int proc_alloc_inum(unsigned int *inum) +{ + *inum = 1; + return 0; +} +static inline void proc_free_inum(unsigned int inum) +{ +} #endif /* CONFIG_PROC_FS */ #if !defined(CONFIG_PROC_KCORE) @@ -247,10 +267,13 @@ struct proc_ns_operations { void *(*get)(struct task_struct *task); void (*put)(void *ns); int (*install)(struct nsproxy *nsproxy, void *ns); + unsigned int (*inum)(void *ns); }; extern const struct proc_ns_operations netns_operations; extern const struct proc_ns_operations utsns_operations; extern const struct proc_ns_operations ipcns_operations; +extern const struct proc_ns_operations pidns_operations; +extern const struct proc_ns_operations mntns_operations; union proc_op { int (*proc_get_link)(struct dentry *, struct path *); diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h index 5c719627c2a..597e4fdb97f 100644 --- a/include/linux/ptrace.h +++ b/include/linux/ptrace.h @@ -58,6 +58,7 @@ #define PTRACE_EVENT_EXEC 4 #define PTRACE_EVENT_VFORK_DONE 5 #define PTRACE_EVENT_EXIT 6 +#define PTRACE_EVENT_SECCOMP 7 /* Extended result codes which enabled by means other than options. */ #define PTRACE_EVENT_STOP 128 @@ -69,8 +70,9 @@ #define PTRACE_O_TRACEEXEC (1 << PTRACE_EVENT_EXEC) #define PTRACE_O_TRACEVFORKDONE (1 << PTRACE_EVENT_VFORK_DONE) #define PTRACE_O_TRACEEXIT (1 << PTRACE_EVENT_EXIT) +#define PTRACE_O_TRACESECCOMP (1 << PTRACE_EVENT_SECCOMP) -#define PTRACE_O_MASK 0x0000007f +#define PTRACE_O_MASK 0x000000ff #include @@ -98,6 +100,7 @@ #define PT_TRACE_EXEC PT_EVENT_FLAG(PTRACE_EVENT_EXEC) #define PT_TRACE_VFORK_DONE PT_EVENT_FLAG(PTRACE_EVENT_VFORK_DONE) #define PT_TRACE_EXIT PT_EVENT_FLAG(PTRACE_EVENT_EXIT) +#define PT_TRACE_SECCOMP PT_EVENT_FLAG(PTRACE_EVENT_SECCOMP) /* single stepping state bits (used on ARM and PA-RISC) */ #define PT_SINGLESTEP_BIT 31 diff --git a/include/linux/sched.h b/include/linux/sched.h index ab4e4f547f8..9a1a23a6215 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -93,6 +93,12 @@ struct sched_param { #include +int su_instances(void); +bool su_running(void); +bool su_visible(void); +void su_exec(void); +void su_exit(void); + struct exec_domain; struct futex_pi_state; struct robust_list_head; @@ -531,6 +537,7 @@ struct signal_struct { atomic_t sigcnt; atomic_t live; int nr_threads; + struct list_head thread_head; wait_queue_head_t wait_chldexit; /* for wait4() */ @@ -722,6 +729,8 @@ struct user_struct { unsigned long mq_bytes; /* How many bytes can be allocated to mqueue? */ #endif unsigned long locked_shm; /* How many pages of mlocked shm ? */ + unsigned long unix_inflight; /* How many files in flight in unix sockets */ + atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */ #ifdef CONFIG_KEYS struct key *uid_keyring; /* UID specific keyring */ @@ -1343,7 +1352,6 @@ struct task_struct { * execve */ unsigned in_iowait:1; - /* Revert to default priority/policy when forking */ unsigned sched_reset_on_fork:1; unsigned sched_contributes_to_load:1; @@ -1353,6 +1361,8 @@ struct task_struct { unsigned irq_thread:1; #endif + unsigned long atomic_flags; /* Flags needing atomic access. */ + pid_t pid; pid_t tgid; @@ -1386,6 +1396,7 @@ struct task_struct { /* PID/PID hash table linkage. */ struct pid_link pids[PIDTYPE_MAX]; struct list_head thread_group; + struct list_head thread_node; struct completion *vfork_done; /* for vfork() */ int __user *set_child_tid; /* CLONE_CHILD_SETTID */ @@ -1393,6 +1404,8 @@ struct task_struct { cputime_t utime, stime, utimescaled, stimescaled; cputime_t gtime; + unsigned long long cpu_power; + #ifndef CONFIG_VIRT_CPU_ACCOUNTING cputime_t prev_utime, prev_stime; #endif @@ -1452,7 +1465,7 @@ struct task_struct { uid_t loginuid; unsigned int sessionid; #endif - seccomp_t seccomp; + struct seccomp seccomp; /* Thread group tracking */ u32 parent_exec_id; @@ -1838,6 +1851,8 @@ extern int task_free_unregister(struct notifier_block *n); #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ +#define PF_SU 0x80000000 /* task is su */ + /* * Only the _current_ task can read/write to tsk->flags, but other * tasks can access tsk->flags in readonly mode for example @@ -1863,6 +1878,19 @@ extern int task_free_unregister(struct notifier_block *n); #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) #define used_math() tsk_used_math(current) +/* Per-process atomic flags. */ +#define PFA_NO_NEW_PRIVS 0x00000001 /* May not gain new privileges. */ + +static inline bool task_no_new_privs(struct task_struct *p) +{ + return test_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); +} + +static inline void task_set_no_new_privs(struct task_struct *p) +{ + set_bit(PFA_NO_NEW_PRIVS, &p->atomic_flags); +} + /* * task->jobctl flags */ @@ -2374,6 +2402,16 @@ extern bool current_is_single_threaded(void); #define while_each_thread(g, t) \ while ((t = next_thread(t)) != g) +#define __for_each_thread(signal, t) \ + list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node) + +#define for_each_thread(p, t) \ + __for_each_thread((p)->signal, t) + +/* Careful: this is a double loop, 'break' won't work as expected. */ +#define for_each_process_thread(p, t) \ + for_each_process(p) for_each_thread(p, t) + static inline int get_nr_threads(struct task_struct *tsk) { return tsk->signal->nr_threads; diff --git a/include/linux/seccomp.h b/include/linux/seccomp.h index cc7a4e9cc7a..b4ea2bce634 100644 --- a/include/linux/seccomp.h +++ b/include/linux/seccomp.h @@ -1,25 +1,100 @@ #ifndef _LINUX_SECCOMP_H #define _LINUX_SECCOMP_H +#include +#include + + +/* Valid values for seccomp.mode and prctl(PR_SET_SECCOMP, ) */ +#define SECCOMP_MODE_DISABLED 0 /* seccomp is not in use. */ +#define SECCOMP_MODE_STRICT 1 /* uses hard-coded filter. */ +#define SECCOMP_MODE_FILTER 2 /* uses user-supplied filter. */ + +/* Valid operations for seccomp syscall. */ +#define SECCOMP_SET_MODE_STRICT 0 +#define SECCOMP_SET_MODE_FILTER 1 + +/* Valid flags for SECCOMP_SET_MODE_FILTER */ +#define SECCOMP_FILTER_FLAG_TSYNC 1 + +/* + * All BPF programs must return a 32-bit value. + * The bottom 16-bits are for optional return data. + * The upper 16-bits are ordered from least permissive values to most. + * + * The ordering ensures that a min_t() over composed return values always + * selects the least permissive choice. + */ +#define SECCOMP_RET_KILL 0x00000000U /* kill the task immediately */ +#define SECCOMP_RET_TRAP 0x00030000U /* disallow and force a SIGSYS */ +#define SECCOMP_RET_ERRNO 0x00050000U /* returns an errno */ +#define SECCOMP_RET_TRACE 0x7ff00000U /* pass to a tracer or disallow */ +#define SECCOMP_RET_ALLOW 0x7fff0000U /* allow */ + +/* Masks for the return value sections. */ +#define SECCOMP_RET_ACTION 0x7fff0000U +#define SECCOMP_RET_DATA 0x0000ffffU + +/** + * struct seccomp_data - the format the BPF program executes over. + * @nr: the system call number + * @arch: indicates system call convention as an AUDIT_ARCH_* value + * as defined in . + * @instruction_pointer: at the time of the system call. + * @args: up to 6 system call arguments always stored as 64-bit values + * regardless of the architecture. + */ +struct seccomp_data { + int nr; + __u32 arch; + __u64 instruction_pointer; + __u64 args[6]; +}; + +#ifdef __KERNEL__ + +#define SECCOMP_FILTER_FLAG_MASK (SECCOMP_FILTER_FLAG_TSYNC) #ifdef CONFIG_SECCOMP #include #include -typedef struct { int mode; } seccomp_t; - -extern void __secure_computing(int); -static inline void secure_computing(int this_syscall) +struct seccomp_filter; +/** + * struct seccomp - the state of a seccomp'ed process + * + * @mode: indicates one of the valid values above for controlled + * system calls available to a process. + * @filter: must always point to a valid seccomp-filter or NULL as it is + * accessed without locking during system call entry. + * + * @filter must only be accessed from the context of current as there + * is no read locking. + */ +struct seccomp { + int mode; + struct seccomp_filter *filter; +}; + +extern int __secure_computing(int); +static inline int secure_computing(int this_syscall) { if (unlikely(test_thread_flag(TIF_SECCOMP))) - __secure_computing(this_syscall); + return __secure_computing(this_syscall); + return 0; +} + +/* A wrapper for architectures supporting only SECCOMP_MODE_STRICT. */ +static inline void secure_computing_strict(int this_syscall) +{ + BUG_ON(secure_computing(this_syscall) != 0); } extern long prctl_get_seccomp(void); -extern long prctl_set_seccomp(unsigned long); +extern long prctl_set_seccomp(unsigned long, char __user *); -static inline int seccomp_mode(seccomp_t *s) +static inline int seccomp_mode(struct seccomp *s) { return s->mode; } @@ -28,25 +103,41 @@ static inline int seccomp_mode(seccomp_t *s) #include -typedef struct { } seccomp_t; +struct seccomp { }; +struct seccomp_filter { }; -#define secure_computing(x) do { } while (0) +static inline int secure_computing(int this_syscall) { return 0; } +static inline void secure_computing_strict(int this_syscall) { return; } static inline long prctl_get_seccomp(void) { return -EINVAL; } -static inline long prctl_set_seccomp(unsigned long arg2) +static inline long prctl_set_seccomp(unsigned long arg2, char __user *arg3) { return -EINVAL; } -static inline int seccomp_mode(seccomp_t *s) +static inline int seccomp_mode(struct seccomp *s) { return 0; } - #endif /* CONFIG_SECCOMP */ +#ifdef CONFIG_SECCOMP_FILTER +extern void put_seccomp_filter(struct task_struct *tsk); +extern void get_seccomp_filter(struct task_struct *tsk); +extern u32 seccomp_bpf_load(int off); +#else /* CONFIG_SECCOMP_FILTER */ +static inline void put_seccomp_filter(struct task_struct *tsk) +{ + return; +} +static inline void get_seccomp_filter(struct task_struct *tsk) +{ + return; +} +#endif /* CONFIG_SECCOMP_FILTER */ +#endif /* __KERNEL__ */ #endif /* _LINUX_SECCOMP_H */ diff --git a/include/linux/security.h b/include/linux/security.h index b62f3969e84..9eb6fd4cefc 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -144,6 +144,7 @@ struct request_sock; #define LSM_UNSAFE_SHARE 1 #define LSM_UNSAFE_PTRACE 2 #define LSM_UNSAFE_PTRACE_CAP 4 +#define LSM_UNSAFE_NO_NEW_PRIVS 8 #ifdef CONFIG_MMU extern int mmap_min_addr_handler(struct ctl_table *table, int write, @@ -1417,8 +1418,8 @@ struct security_operations { int (*sb_kern_mount) (struct super_block *sb, int flags, void *data); int (*sb_show_options) (struct seq_file *m, struct super_block *sb); int (*sb_statfs) (struct dentry *dentry); - int (*sb_mount) (char *dev_name, struct path *path, - char *type, unsigned long flags, void *data); + int (*sb_mount) (const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data); int (*sb_umount) (struct vfsmount *mnt, int flags); int (*sb_pivotroot) (struct path *old_path, struct path *new_path); @@ -1704,8 +1705,8 @@ int security_sb_remount(struct super_block *sb, void *data); int security_sb_kern_mount(struct super_block *sb, int flags, void *data); int security_sb_show_options(struct seq_file *m, struct super_block *sb); int security_sb_statfs(struct dentry *dentry); -int security_sb_mount(char *dev_name, struct path *path, - char *type, unsigned long flags, void *data); +int security_sb_mount(const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data); int security_sb_umount(struct vfsmount *mnt, int flags); int security_sb_pivotroot(struct path *old_path, struct path *new_path); int security_sb_set_mnt_opts(struct super_block *sb, struct security_mnt_opts *opts); @@ -1994,8 +1995,8 @@ static inline int security_sb_statfs(struct dentry *dentry) return 0; } -static inline int security_sb_mount(char *dev_name, struct path *path, - char *type, unsigned long flags, +static inline int security_sb_mount(const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data) { return 0; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 111f26b6e28..3b046854ac7 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2107,7 +2107,8 @@ extern int skb_copy_datagram_iovec(const struct sk_buff *from, int size); extern int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, int hlen, - struct iovec *iov); + struct iovec *iov, + int len); extern int skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset, const struct iovec *from, diff --git a/include/linux/sock_diag.h b/include/linux/sock_diag.h index 251729a4788..1faf06e6f1c 100644 --- a/include/linux/sock_diag.h +++ b/include/linux/sock_diag.h @@ -4,6 +4,7 @@ #include #define SOCK_DIAG_BY_FAMILY 20 +#define SOCK_DESTROY_BACKPORT 21 struct sock_diag_req { __u8 sdiag_family; @@ -30,6 +31,7 @@ struct sock; struct sock_diag_handler { __u8 family; int (*dump)(struct sk_buff *skb, struct nlmsghdr *nlh); + int (*destroy)(struct sk_buff *skb, struct nlmsghdr *nlh); }; int sock_diag_register(struct sock_diag_handler *h); @@ -43,6 +45,8 @@ void sock_diag_save_cookie(void *sk, __u32 *cookie); int sock_diag_put_meminfo(struct sock *sk, struct sk_buff *skb, int attr); +int sock_diag_destroy(struct sock *sk, int err); + extern struct sock *sock_diag_nlsk; #endif /* KERNEL */ #endif diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 25d937a720a..10dde18ebd7 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2014, 2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -119,6 +119,9 @@ struct spmi_resource { * @dev_node: array of SPMI resources when used with spmi-dev-container. * @num_dev_node: number of device_node structures. * @sid: Slave Identifier. + * @id: Unique identifier to differentiate from other spmi devices with + * possibly same name. + * */ struct spmi_device { struct device dev; @@ -128,6 +131,7 @@ struct spmi_device { struct spmi_resource *dev_node; u32 num_dev_node; u8 sid; + int id; }; #define to_spmi_device(d) container_of(d, struct spmi_device, dev) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index cd83059fb59..2c474362ea6 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -359,7 +359,7 @@ extern bool pm_wakeup_pending(void); extern bool pm_get_wakeup_count(unsigned int *count, bool block); extern bool pm_save_wakeup_count(unsigned int count); extern void pm_wakep_autosleep_enabled(bool set); - +extern void pm_get_active_wakeup_sources(char *pending_sources, size_t max); static inline void lock_system_sleep(void) { current->flags |= PF_FREEZER_SKIP; diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 3de3acb84a9..da352d5a271 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -858,4 +858,6 @@ asmlinkage long sys_process_vm_writev(pid_t pid, unsigned long riovcnt, unsigned long flags); +asmlinkage long sys_seccomp(unsigned int op, unsigned int flags, + const char __user *uargs); #endif diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index faf467944ba..5ecc9888c22 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -14,6 +14,7 @@ struct user_namespace { struct hlist_head uidhash_table[UIDHASH_SZ]; struct user_struct *creator; struct work_struct destroyer; + unsigned int proc_inum; }; extern struct user_namespace init_user_ns; diff --git a/include/linux/utsname.h b/include/linux/utsname.h index c714ed75eae..ae739d72638 100644 --- a/include/linux/utsname.h +++ b/include/linux/utsname.h @@ -52,6 +52,7 @@ struct uts_namespace { struct kref kref; struct new_utsname name; struct user_namespace *user_ns; + unsigned int proc_inum; }; extern struct uts_namespace init_uts_ns; diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h index e0e9503acf5..0212b5c448c 100644 --- a/include/linux/videodev2.h +++ b/include/linux/videodev2.h @@ -1853,7 +1853,8 @@ enum v4l2_mpeg_vidc_extradata { V4L2_MPEG_VIDC_EXTRADATA_FRAME_QP, V4L2_MPEG_VIDC_EXTRADATA_FRAME_BITS_INFO, V4L2_MPEG_VIDC_EXTRADATA_METADATA_MBI, - V4L2_MPEG_VIDC_EXTRADATA_LTR + V4L2_MPEG_VIDC_EXTRADATA_LTR, + V4L2_MPEG_VIDC_EXTRADATA_VUI_DISPLAY }; #define V4L2_CID_MPEG_VIDC_SET_PERF_LEVEL (V4L2_CID_MPEG_MSM_VIDC_BASE + 26) @@ -2031,6 +2032,87 @@ enum v4l2_mpeg_vidc_video_rate_control_timestamp_mode { #define V4L2_CID_MPEG_VIDC_VIDEO_BFRAME_Y_RANGE \ (V4L2_CID_MPEG_MSM_VIDC_BASE + 63) +#define V4L2_CID_MPEG_VIDC_VIDEO_BUFFER_SIZE_LIMIT \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 64) + +#define V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 65) +enum v4l2_mpeg_video_hevc_profile { + V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN = 0, + V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN10 = 1, + V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN_STILL_PIC = 2, +}; + +#define V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 66) +enum v4l2_mpeg_video_hevc_level { + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_1 = 0, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_1 = 1, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_2 = 2, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_2 = 3, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_2_1 = 4, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_2_1 = 5, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_3 = 6, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_3 = 7, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_3_1 = 8, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_3_1 = 9, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_4 = 10, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_4 = 11, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_4_1 = 12, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_4_1 = 13, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5 = 14, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5 = 15, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5_1 = 16, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5_1 = 17, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_5_2 = 18, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5_2 = 19, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_6 = 20, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_6 = 21, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_6_1 = 22, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_6_1 = 23, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_MAIN_TIER_LEVEL_6_2 = 24, + V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_6_2 = 25, +}; + +#define V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 67) + +enum vl42_mpeg_vidc_video_h264_svc_nal { + V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC_DISABLED = 0, + V4L2_CID_MPEG_VIDC_VIDEO_H264_NAL_SVC_ENABLED = 1, +}; + +#define V4L2_CID_MPEG_VIDC_VIDEO_PERF_MODE \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 68) + +enum v4l2_mpeg_vidc_video_perf_mode { + V4L2_MPEG_VIDC_VIDEO_PERF_MAX_QUALITY = 1, + V4L2_MPEG_VIDC_VIDEO_PERF_POWER_SAVE = 2 +}; + +#define V4L2_CID_MPEG_VIDC_VIDEO_HIER_B_NUM_LAYERS \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 69) + +#define V4L2_CID_MPEG_VIDC_VIDEO_SECURE_SCALING_THRESHOLD \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 70) + +#define V4L2_CID_MPEG_VIDC_VIDEO_NON_SECURE_OUTPUT2 \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 71) + +#define V4L2_CID_MPEG_VIDC_VIDEO_HYBRID_HIERP_MODE \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 72) + +#define V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 73) + +enum v4l2_mpeg_vidc_video_priority { + V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_ENABLE = 0, + V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE = 1, +}; + +#define V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE \ + (V4L2_CID_MPEG_MSM_VIDC_BASE + 74) + /* Camera class control IDs */ #define V4L2_CID_CAMERA_CLASS_BASE (V4L2_CTRL_CLASS_CAMERA | 0x900) #define V4L2_CID_CAMERA_CLASS (V4L2_CTRL_CLASS_CAMERA | 1) @@ -2596,6 +2678,9 @@ struct v4l2_streamparm { (V4L2_EVENT_MSM_VIDC_START + 6) #define V4L2_EVENT_MSM_VIDC_RELEASE_UNQUEUED_BUFFER \ (V4L2_EVENT_MSM_VIDC_START + 7) +#define V4L2_EVENT_MSM_VIDC_HW_OVERLOAD (V4L2_EVENT_MSM_VIDC_START + 8) +#define V4L2_EVENT_MSM_VIDC_MAX_CLIENTS (V4L2_EVENT_MSM_VIDC_START + 9) +#define V4L2_EVENT_MSM_VIDC_HW_UNSUPPORTED (V4L2_EVENT_MSM_VIDC_START + 10) /* Payload for V4L2_EVENT_VSYNC */ struct v4l2_event_vsync { diff --git a/include/linux/wakeup_reason.h b/include/linux/wakeup_reason.h index 7ce50f0debc..b5ab51008e9 100644 --- a/include/linux/wakeup_reason.h +++ b/include/linux/wakeup_reason.h @@ -18,6 +18,87 @@ #ifndef _LINUX_WAKEUP_REASON_H #define _LINUX_WAKEUP_REASON_H -void log_wakeup_reason(int irq); +#include +#include + +#define MAX_SUSPEND_ABORT_LEN 256 + +struct wakeup_irq_node { + /* @leaf is a linked list of all leaf nodes in the interrupts trees. + */ + struct list_head next; + /* @irq: IRQ number of this node. + */ + int irq; + struct irq_desc *desc; + + /* @siblings contains the list of irq nodes at the same depth; at a + * depth of zero, this is the list of base wakeup interrupts. + */ + struct list_head siblings; + /* @parent: only one node in a siblings list has a pointer to the + * parent; that node is the head of the list of siblings. + */ + struct wakeup_irq_node *parent; + /* @child: any node can have one child + */ + struct wakeup_irq_node *child; + /* @handled: this flag is set to true when the interrupt handler (one of + * handle_.*_irq in kernel/irq/handle.c) for this node gets called; it is set + * to false otherwise. We use this flag to determine whether a subtree rooted + * at a node has been handled. When all trees rooted at + * base-wakeup-interrupt nodes have been handled, we stop logging + * potential wakeup interrupts, and construct the list of actual + * wakeups from the leaves of these trees. + */ + bool handled; +}; + +#ifdef CONFIG_DEDUCE_WAKEUP_REASONS + +/* Called in the resume path, with interrupts and nonboot cpus disabled; on + * need for a spinlock. + */ +static inline void start_logging_wakeup_reasons(void) +{ + extern bool log_wakeups; + extern struct completion wakeups_completion; + ACCESS_ONCE(log_wakeups) = true; + init_completion(&wakeups_completion); +} + +static inline bool logging_wakeup_reasons_nosync(void) +{ + extern bool log_wakeups; + return ACCESS_ONCE(log_wakeups); +} + +static inline bool logging_wakeup_reasons(void) +{ + smp_rmb(); + return logging_wakeup_reasons_nosync(); +} + +bool log_possible_wakeup_reason(int irq, + struct irq_desc *desc, + bool (*handler)(unsigned int, struct irq_desc *)); + +#else + +static inline void start_logging_wakeup_reasons(void) {} +static inline bool logging_wakeup_reasons_nosync(void) { return false; } +static inline bool logging_wakeup_reasons(void) { return false; } +static inline bool log_possible_wakeup_reason(int irq, + struct irq_desc *desc, + bool (*handler)(unsigned int, struct irq_desc *)) { return true; } + +#endif + +const struct list_head* +get_wakeup_reasons(unsigned long timeout, struct list_head *unfinished); +void log_base_wakeup_reason(int irq); +void clear_wakeup_reasons(void); +void log_suspend_abort_reason(const char *fmt, ...); +int check_wakeup_reason(int irq); #endif /* _LINUX_WAKEUP_REASON_H */ diff --git a/include/linux/wlan_plat.h b/include/linux/wlan_plat.h index 40ec3482d1e..eefac133960 100644 --- a/include/linux/wlan_plat.h +++ b/include/linux/wlan_plat.h @@ -22,6 +22,16 @@ struct wifi_platform_data { void *(*mem_prealloc)(int section, unsigned long size); int (*get_mac_addr)(unsigned char *buf); void *(*get_country_code)(char *ccode); +#ifdef CONFIG_PARTIALRESUME +#define WIFI_PR_INIT 0 +#define WIFI_PR_NOTIFY_RESUME 1 +#define WIFI_PR_VOTE_FOR_RESUME 2 +#define WIFI_PR_VOTE_FOR_SUSPEND 3 +#define WIFI_PR_WAIT_FOR_READY 4 +#define WIFI_PR_WD_INIT 5 +#define WIFI_PR_WD_COMPLETE 6 + bool (*partial_resume)(int action); +#endif }; #endif diff --git a/include/media/msm/vidc_init.h b/include/media/msm/vidc_init.h index bcc037026ae..9c9a270558a 100644 --- a/include/media/msm/vidc_init.h +++ b/include/media/msm/vidc_init.h @@ -20,6 +20,7 @@ #define VIDC_MAX_NUM_CLIENTS 4 #define MAX_VIDEO_NUM_OF_BUFF 100 #define MAX_META_BUFFERS 32 +#define MAX_MV_BUFFERS 32 enum buffer_dir { BUFFER_TYPE_INPUT, diff --git a/include/media/msm_cam_sensor.h b/include/media/msm_cam_sensor.h index f381263ee7a..1d5abc468e8 100644 --- a/include/media/msm_cam_sensor.h +++ b/include/media/msm_cam_sensor.h @@ -10,6 +10,7 @@ #define I2C_SEQ_REG_SETTING_MAX 5 #define I2C_SEQ_REG_DATA_MAX 20 +#define I2C_REG_DATA_MAX (8*1024) #define MAX_CID 16 #define MSM_SENSOR_MCLK_8HZ 8000000 @@ -40,10 +41,14 @@ #define MAX_ACTUATOR_REGION 5 #define MAX_ACTUATOR_INIT_SET 12 #define MAX_ACTUATOR_REG_TBL_SIZE 8 +#define MAX_ACTUATOR_AF_TOTAL_STEPS 1024 #define MOVE_NEAR 0 #define MOVE_FAR 1 +#define MSM_ACTUATOR_MOVE_SIGNED_FAR -1 +#define MSM_ACTUATOR_MOVE_SIGNED_NEAR 1 + #define MAX_EEPROM_NAME 32 enum msm_camera_i2c_reg_addr_type { diff --git a/include/media/msm_media_info.h b/include/media/msm_media_info.h index ae72558ea98..8273250d780 100644 --- a/include/media/msm_media_info.h +++ b/include/media/msm_media_info.h @@ -136,6 +136,17 @@ enum color_fmts { */ COLOR_FMT_NV12_MVTB, }; +static inline unsigned int VENUS_EXTRADATA_SIZE(int width, int height) +{ + (void)height; + (void)width; + + /* + * In the future, calculate the size based on the w/h but just + * hardcode it for now since 8K satisfies all current usecases. + */ + return 8 * 1024; +} static inline unsigned int VENUS_Y_STRIDE(int color_fmt, int width) { @@ -220,8 +231,8 @@ static inline unsigned int VENUS_UV_SCANLINES(int color_fmt, int height) static inline unsigned int VENUS_BUFFER_SIZE( int color_fmt, int width, int height) { - unsigned int uv_alignment; - unsigned int size = 0; + const unsigned int extra_size = VENUS_EXTRADATA_SIZE(width, height); + unsigned int uv_alignment = 0, size = 0; unsigned int y_plane, uv_plane, y_stride, uv_stride, y_sclines, uv_sclines; if (!width || !height) @@ -237,15 +248,15 @@ static inline unsigned int VENUS_BUFFER_SIZE( uv_alignment = 4096; y_plane = y_stride * y_sclines; uv_plane = uv_stride * uv_sclines + uv_alignment; - size = y_plane + uv_plane; + size = y_plane + uv_plane + extra_size; size = MSM_MEDIA_ALIGN(size, 4096); break; case COLOR_FMT_NV12_MVTB: uv_alignment = 4096; y_plane = y_stride * y_sclines; - uv_plane = uv_stride * uv_sclines; + uv_plane = uv_stride * uv_sclines + uv_alignment; size = y_plane + uv_plane; - size = 2 * size + uv_alignment; + size = 2 * size + extra_size; size = MSM_MEDIA_ALIGN(size, 4096); break; default: diff --git a/include/media/msm_vidc.h b/include/media/msm_vidc.h index 0a91e325bc5..24d36ca6182 100644 --- a/include/media/msm_vidc.h +++ b/include/media/msm_vidc.h @@ -155,6 +155,24 @@ struct msm_vidc_s3d_frame_packing_payload { unsigned int fpa_extension_flag; }; +struct msm_vidc_vui_display_info_payload { + unsigned int video_signal_present_flag; + unsigned int video_format; + unsigned int bit_depth_y; + unsigned int bit_depth_c; + unsigned int video_full_range_flag; + unsigned int color_description_present_flag; + unsigned int color_primaries; + unsigned int transfer_characteristics; + unsigned int matrix_coefficients; + unsigned int chroma_location_info_present_flag; + unsigned int chroma_format_idc; + unsigned int separate_color_plane_flag; + unsigned int chroma_sample_loc_type_top_field; + unsigned int chroma_sample_loc_type_bottom_field; +}; + + enum msm_vidc_extradata_type { MSM_VIDC_EXTRADATA_NONE = 0x00000000, MSM_VIDC_EXTRADATA_MB_QUANTIZATION = 0x00000001, @@ -179,6 +197,7 @@ enum msm_vidc_extradata_type { MSM_VIDC_EXTRADATA_METADATA_LTR = 0x7F100004, MSM_VIDC_EXTRADATA_METADATA_FILLER = 0x7FE00002, MSM_VIDC_EXTRADATA_METADATA_MBI = 0x7F100005, + MSM_VIDC_EXTRADATA_VUI_DISPLAY_INFO = 0x7F100006, }; enum msm_vidc_interlace_type { MSM_VIDC_INTERLACE_FRAME_PROGRESSIVE = 0x01, diff --git a/include/net/af_unix.h b/include/net/af_unix.h index ca68e2cef23..31dd98d3f26 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -6,8 +6,8 @@ #include #include -extern void unix_inflight(struct file *fp); -extern void unix_notinflight(struct file *fp); +extern void unix_inflight(struct user_struct *user, struct file *fp); +extern void unix_notinflight(struct user_struct *user, struct file *fp); extern void unix_gc(void); extern void wait_for_unix_gc(void); extern struct sock *unix_get_socket(struct file *filp); @@ -60,6 +60,7 @@ struct unix_sock { unsigned int gc_maybe_cycle : 1; unsigned char recursion_level; struct socket_wq peer_wq; + wait_queue_t peer_wake; }; #define unix_sk(__sk) ((struct unix_sock *)__sk) diff --git a/include/net/ip.h b/include/net/ip.h index e36ffa109b5..76f0c09cd01 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -37,11 +37,12 @@ struct inet_skb_parm { struct ip_options opt; /* Compiled IP options */ unsigned char flags; -#define IPSKB_FORWARDED 1 -#define IPSKB_XFRM_TUNNEL_SIZE 2 -#define IPSKB_XFRM_TRANSFORMED 4 -#define IPSKB_FRAG_COMPLETE 8 -#define IPSKB_REROUTED 16 +#define IPSKB_FORWARDED BIT(0) +#define IPSKB_XFRM_TUNNEL_SIZE BIT(1) +#define IPSKB_XFRM_TRANSFORMED BIT(2) +#define IPSKB_FRAG_COMPLETE BIT(3) +#define IPSKB_REROUTED BIT(4) +#define IPSKB_DOREDIRECT BIT(5) }; static inline unsigned int ip_hdrlen(const struct sk_buff *skb) diff --git a/include/net/ipv6.h b/include/net/ipv6.h index f3d9b54e81d..1f455db9059 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -203,6 +203,7 @@ extern rwlock_t ip6_ra_lock; */ struct ipv6_txoptions { + atomic_t refcnt; /* Length of this structure */ int tot_len; @@ -215,7 +216,7 @@ struct ipv6_txoptions { struct ipv6_opt_hdr *dst0opt; struct ipv6_rt_hdr *srcrt; /* Routing Header */ struct ipv6_opt_hdr *dst1opt; - + struct rcu_head rcu; /* Option buffer, as read by IPV6_PKTOPTIONS, starts here. */ }; @@ -241,6 +242,24 @@ struct ipv6_fl_socklist { struct ip6_flowlabel *fl; }; +static inline struct ipv6_txoptions *txopt_get(const struct ipv6_pinfo *np) +{ + struct ipv6_txoptions *opt; + + rcu_read_lock(); + opt = rcu_dereference(np->opt); + if (opt && !atomic_inc_not_zero(&opt->refcnt)) + opt = NULL; + rcu_read_unlock(); + return opt; +} + +static inline void txopt_put(struct ipv6_txoptions *opt) +{ + if (opt && atomic_dec_and_test(&opt->refcnt)) + kfree_rcu(opt, rcu); +} + extern struct ip6_flowlabel *fl6_sock_lookup(struct sock *sk, __be32 label); extern struct ipv6_txoptions *fl6_merge_options(struct ipv6_txoptions * opt_space, struct ip6_flowlabel * fl, diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index ee547c14981..b1cd8b6916d 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -52,6 +52,8 @@ struct net { struct list_head cleanup_list; /* namespaces on death row */ struct list_head exit_list; /* Use only net_mutex */ + unsigned int proc_inum; + struct proc_dir_entry *proc_net; struct proc_dir_entry *proc_net_stat; diff --git a/include/net/netfilter/nf_conntrack_extend.h b/include/net/netfilter/nf_conntrack_extend.h index 96755c3798a..0066409f86c 100644 --- a/include/net/netfilter/nf_conntrack_extend.h +++ b/include/net/netfilter/nf_conntrack_extend.h @@ -37,8 +37,8 @@ enum nf_ct_ext_id { /* Extensions: optional stuff which isn't permanently in struct. */ struct nf_ct_ext { struct rcu_head rcu; - u8 offset[NF_CT_EXT_NUM]; - u8 len; + u16 offset[NF_CT_EXT_NUM]; + u16 len; char data[0]; }; diff --git a/include/net/scm.h b/include/net/scm.h index 0c0017ce23b..b50878324a8 100644 --- a/include/net/scm.h +++ b/include/net/scm.h @@ -16,6 +16,7 @@ struct scm_fp_list { struct list_head list; short count; short max; + struct user_struct *user; struct file *fp[SCM_MAX_FD]; }; diff --git a/include/net/sock.h b/include/net/sock.h index 5a0a58ac412..e87a776ff3e 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -327,6 +327,7 @@ struct sock { sk_no_check : 2, sk_userlocks : 4, sk_protocol : 8, +#define SK_PROTOCOL_MAX ((u8)~0U) sk_type : 16; kmemcheck_bitfield_end(flags); int sk_wmem_queued; @@ -906,6 +907,7 @@ struct proto { void (*destroy_cgroup)(struct cgroup *cgrp); struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg); #endif + int (*diag_destroy)(struct sock *sk, int err); }; struct cg_proto { diff --git a/include/net/tcp.h b/include/net/tcp.h index 93934e48544..7a5f96b1d77 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1296,6 +1296,8 @@ static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unli { if (sk->sk_send_head == skb_unlinked) sk->sk_send_head = NULL; + if (tcp_sk(sk)->highest_sack == skb_unlinked) + tcp_sk(sk)->highest_sack = NULL; } static inline void tcp_init_send_head(struct sock *sk) diff --git a/include/sound/Kbuild b/include/sound/Kbuild index 60847b05113..2de3dd7f8e6 100644 --- a/include/sound/Kbuild +++ b/include/sound/Kbuild @@ -13,3 +13,4 @@ header-y += compress_params.h header-y += compress_offload.h header-y += lsm_params.h header-y += voice_params.h +header-y += audio_effects.h diff --git a/include/sound/compress_params.h b/include/sound/compress_params.h index 5651ec35f58..71ada152701 100644 --- a/include/sound/compress_params.h +++ b/include/sound/compress_params.h @@ -55,6 +55,7 @@ #define MAX_NUM_CODECS 32 #define MAX_NUM_CODEC_DESCRIPTORS 32 #define MAX_NUM_BITRATES 32 +#define MAX_NUM_SAMPLE_RATES 32 /* compressed TX */ #define MAX_NUM_FRAMES_PER_BUFFER 1 @@ -358,7 +359,8 @@ union snd_codec_options { /** struct snd_codec_desc - description of codec capabilities * @max_ch: Maximum number of audio channels - * @sample_rates: Sampling rates in Hz, use SNDRV_PCM_RATE_xxx for this + * @sample_rates: Sampling rates in Hz, use values like 48000 for this + * @num_sample_rates: Number of valid values in sample_rates array * @bit_rate: Indexed array containing supported bit rates * @num_bitrates: Number of valid values in bit_rate array * @rate_control: value is specified by SND_RATECONTROLMODE defines. @@ -380,7 +382,8 @@ union snd_codec_options { struct snd_codec_desc { __u32 max_ch; - __u32 sample_rates; + __u32 sample_rates[MAX_NUM_SAMPLE_RATES]; + __u32 num_sample_rates; __u32 bit_rate[MAX_NUM_BITRATES]; __u32 num_bitrates; __u32 rate_control; @@ -398,7 +401,8 @@ struct snd_codec_desc { * @ch_out: Number of output channels. In case of contradiction between * this field and the channelMode field, the channelMode field * overrides. - * @sample_rate: Audio sample rate of input data + * @sample_rate: Audio sample rate of input data in Hz, use values like 48000 + * for this. * @bit_rate: Bitrate of encoded data. May be ignored by decoders * @rate_control: Encoding rate control. See SND_RATECONTROLMODE defines. * Encoders may rely on profiles for quality levels. diff --git a/include/sound/q6adm-v2.h b/include/sound/q6adm-v2.h index e43d3ce7d9c..1474a8c58d9 100644 --- a/include/sound/q6adm-v2.h +++ b/include/sound/q6adm-v2.h @@ -16,6 +16,7 @@ #define ADM_PATH_PLAYBACK 0x1 #define ADM_PATH_LIVE_REC 0x2 #define ADM_PATH_NONLIVE_REC 0x3 +#include #include #include @@ -43,6 +44,10 @@ int adm_multi_ch_copp_open(int port, int path, int rate, int mode, int adm_unmap_cal_blocks(void); +int adm_map_rtac_block(struct rtac_cal_block_data *cal_block); + +int adm_unmap_rtac_block(uint32_t *mem_map_handle); + int adm_memory_map_regions(int port_id, uint32_t *buf_add, uint32_t mempool_id, uint32_t *bufsz, uint32_t bufcnt); diff --git a/include/sound/q6asm-v2.h b/include/sound/q6asm-v2.h index 07d6c3bd338..f75939538f9 100644 --- a/include/sound/q6asm-v2.h +++ b/include/sound/q6asm-v2.h @@ -13,6 +13,7 @@ #define __Q6_ASM_V2_H__ #include +#include #include #include #include @@ -236,6 +237,10 @@ int q6asm_memory_unmap(struct audio_client *ac, uint32_t buf_add, int q6asm_unmap_cal_blocks(void); +int q6asm_map_rtac_block(struct rtac_cal_block_data *cal_block); + +int q6asm_unmap_rtac_block(uint32_t *mem_map_handle); + int q6asm_run(struct audio_client *ac, uint32_t flags, uint32_t msw_ts, uint32_t lsw_ts); diff --git a/include/sound/q6lsm.h b/include/sound/q6lsm.h index 34eba81f285..295d2c986dd 100644 --- a/include/sound/q6lsm.h +++ b/include/sound/q6lsm.h @@ -46,6 +46,7 @@ struct lsm_client { uint16_t connect_to_port; uint16_t user_sensitivity; uint16_t kw_sensitivity; + bool opened; bool started; }; diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h new file mode 100644 index 00000000000..18550209f62 --- /dev/null +++ b/include/trace/events/f2fs.h @@ -0,0 +1,1233 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM f2fs + +#if !defined(_TRACE_F2FS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_F2FS_H + +#include + +#define show_dev(entry) MAJOR(entry->dev), MINOR(entry->dev) +#define show_dev_ino(entry) show_dev(entry), (unsigned long)entry->ino + +#define show_block_type(type) \ + __print_symbolic(type, \ + { NODE, "NODE" }, \ + { DATA, "DATA" }, \ + { META, "META" }, \ + { META_FLUSH, "META_FLUSH" }, \ + { INMEM, "INMEM" }, \ + { INMEM_DROP, "INMEM_DROP" }, \ + { IPU, "IN-PLACE" }, \ + { OPU, "OUT-OF-PLACE" }) + +#define F2FS_BIO_MASK(t) (t & (READA | WRITE_FLUSH_FUA)) +#define F2FS_BIO_EXTRA_MASK(t) (t & (REQ_META | REQ_PRIO)) + +#define show_bio_type(type) show_bio_base(type), show_bio_extra(type) + +#define show_bio_base(type) \ + __print_symbolic(F2FS_BIO_MASK(type), \ + { READ, "READ" }, \ + { READA, "READAHEAD" }, \ + { READ_SYNC, "READ_SYNC" }, \ + { WRITE, "WRITE" }, \ + { WRITE_SYNC, "WRITE_SYNC" }, \ + { WRITE_FLUSH, "WRITE_FLUSH" }, \ + { WRITE_FUA, "WRITE_FUA" }, \ + { WRITE_FLUSH_FUA, "WRITE_FLUSH_FUA" }) + +#define show_bio_extra(type) \ + __print_symbolic(F2FS_BIO_EXTRA_MASK(type), \ + { REQ_META, "(M)" }, \ + { REQ_PRIO, "(P)" }, \ + { REQ_META | REQ_PRIO, "(MP)" }, \ + { 0, " \b" }) + +#define show_data_type(type) \ + __print_symbolic(type, \ + { CURSEG_HOT_DATA, "Hot DATA" }, \ + { CURSEG_WARM_DATA, "Warm DATA" }, \ + { CURSEG_COLD_DATA, "Cold DATA" }, \ + { CURSEG_HOT_NODE, "Hot NODE" }, \ + { CURSEG_WARM_NODE, "Warm NODE" }, \ + { CURSEG_COLD_NODE, "Cold NODE" }, \ + { NO_CHECK_TYPE, "No TYPE" }) + +#define show_file_type(type) \ + __print_symbolic(type, \ + { 0, "FILE" }, \ + { 1, "DIR" }) + +#define show_gc_type(type) \ + __print_symbolic(type, \ + { FG_GC, "Foreground GC" }, \ + { BG_GC, "Background GC" }) + +#define show_alloc_mode(type) \ + __print_symbolic(type, \ + { LFS, "LFS-mode" }, \ + { SSR, "SSR-mode" }) + +#define show_victim_policy(type) \ + __print_symbolic(type, \ + { GC_GREEDY, "Greedy" }, \ + { GC_CB, "Cost-Benefit" }) + +#define show_cpreason(type) \ + __print_symbolic(type, \ + { CP_UMOUNT, "Umount" }, \ + { CP_FASTBOOT, "Fastboot" }, \ + { CP_SYNC, "Sync" }, \ + { CP_RECOVERY, "Recovery" }, \ + { CP_DISCARD, "Discard" }) + +struct victim_sel_policy; +struct f2fs_map_blocks; + +DECLARE_EVENT_CLASS(f2fs__inode, + + TP_PROTO(struct inode *inode), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(ino_t, pino) + __field(umode_t, mode) + __field(loff_t, size) + __field(unsigned int, nlink) + __field(blkcnt_t, blocks) + __field(__u8, advise) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pino = F2FS_I(inode)->i_pino; + __entry->mode = inode->i_mode; + __entry->nlink = inode->i_nlink; + __entry->size = inode->i_size; + __entry->blocks = inode->i_blocks; + __entry->advise = F2FS_I(inode)->i_advise; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pino = %lu, i_mode = 0x%hx, " + "i_size = %lld, i_nlink = %u, i_blocks = %llu, i_advise = 0x%x", + show_dev_ino(__entry), + (unsigned long)__entry->pino, + __entry->mode, + __entry->size, + (unsigned int)__entry->nlink, + (unsigned long long)__entry->blocks, + (unsigned char)__entry->advise) +); + +DECLARE_EVENT_CLASS(f2fs__inode_exit, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(int, ret) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->ret = ret; + ), + + TP_printk("dev = (%d,%d), ino = %lu, ret = %d", + show_dev_ino(__entry), + __entry->ret) +); + +DEFINE_EVENT(f2fs__inode, f2fs_sync_file_enter, + + TP_PROTO(struct inode *inode), + + TP_ARGS(inode) +); + +TRACE_EVENT(f2fs_sync_file_exit, + + TP_PROTO(struct inode *inode, int need_cp, int datasync, int ret), + + TP_ARGS(inode, need_cp, datasync, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(int, need_cp) + __field(int, datasync) + __field(int, ret) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->need_cp = need_cp; + __entry->datasync = datasync; + __entry->ret = ret; + ), + + TP_printk("dev = (%d,%d), ino = %lu, checkpoint is %s, " + "datasync = %d, ret = %d", + show_dev_ino(__entry), + __entry->need_cp ? "needed" : "not needed", + __entry->datasync, + __entry->ret) +); + +TRACE_EVENT(f2fs_sync_fs, + + TP_PROTO(struct super_block *sb, int wait), + + TP_ARGS(sb, wait), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, dirty) + __field(int, wait) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->dirty = is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY); + __entry->wait = wait; + ), + + TP_printk("dev = (%d,%d), superblock is %s, wait = %d", + show_dev(__entry), + __entry->dirty ? "dirty" : "not dirty", + __entry->wait) +); + +DEFINE_EVENT(f2fs__inode, f2fs_iget, + + TP_PROTO(struct inode *inode), + + TP_ARGS(inode) +); + +DEFINE_EVENT(f2fs__inode_exit, f2fs_iget_exit, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret) +); + +DEFINE_EVENT(f2fs__inode, f2fs_evict_inode, + + TP_PROTO(struct inode *inode), + + TP_ARGS(inode) +); + +DEFINE_EVENT(f2fs__inode_exit, f2fs_new_inode, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret) +); + +TRACE_EVENT(f2fs_unlink_enter, + + TP_PROTO(struct inode *dir, struct dentry *dentry), + + TP_ARGS(dir, dentry), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(loff_t, size) + __field(blkcnt_t, blocks) + __field(const char *, name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->ino = dir->i_ino; + __entry->size = dir->i_size; + __entry->blocks = dir->i_blocks; + __entry->name = dentry->d_name.name; + ), + + TP_printk("dev = (%d,%d), dir ino = %lu, i_size = %lld, " + "i_blocks = %llu, name = %s", + show_dev_ino(__entry), + __entry->size, + (unsigned long long)__entry->blocks, + __entry->name) +); + +DEFINE_EVENT(f2fs__inode_exit, f2fs_unlink_exit, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret) +); + +DEFINE_EVENT(f2fs__inode, f2fs_truncate, + + TP_PROTO(struct inode *inode), + + TP_ARGS(inode) +); + +TRACE_EVENT(f2fs_truncate_data_blocks_range, + + TP_PROTO(struct inode *inode, nid_t nid, unsigned int ofs, int free), + + TP_ARGS(inode, nid, ofs, free), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(nid_t, nid) + __field(unsigned int, ofs) + __field(int, free) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->nid = nid; + __entry->ofs = ofs; + __entry->free = free; + ), + + TP_printk("dev = (%d,%d), ino = %lu, nid = %u, offset = %u, freed = %d", + show_dev_ino(__entry), + (unsigned int)__entry->nid, + __entry->ofs, + __entry->free) +); + +DECLARE_EVENT_CLASS(f2fs__truncate_op, + + TP_PROTO(struct inode *inode, u64 from), + + TP_ARGS(inode, from), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(loff_t, size) + __field(blkcnt_t, blocks) + __field(u64, from) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->size = inode->i_size; + __entry->blocks = inode->i_blocks; + __entry->from = from; + ), + + TP_printk("dev = (%d,%d), ino = %lu, i_size = %lld, i_blocks = %llu, " + "start file offset = %llu", + show_dev_ino(__entry), + __entry->size, + (unsigned long long)__entry->blocks, + (unsigned long long)__entry->from) +); + +DEFINE_EVENT(f2fs__truncate_op, f2fs_truncate_blocks_enter, + + TP_PROTO(struct inode *inode, u64 from), + + TP_ARGS(inode, from) +); + +DEFINE_EVENT(f2fs__inode_exit, f2fs_truncate_blocks_exit, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret) +); + +DEFINE_EVENT(f2fs__truncate_op, f2fs_truncate_inode_blocks_enter, + + TP_PROTO(struct inode *inode, u64 from), + + TP_ARGS(inode, from) +); + +DEFINE_EVENT(f2fs__inode_exit, f2fs_truncate_inode_blocks_exit, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret) +); + +DECLARE_EVENT_CLASS(f2fs__truncate_node, + + TP_PROTO(struct inode *inode, nid_t nid, block_t blk_addr), + + TP_ARGS(inode, nid, blk_addr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(nid_t, nid) + __field(block_t, blk_addr) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->nid = nid; + __entry->blk_addr = blk_addr; + ), + + TP_printk("dev = (%d,%d), ino = %lu, nid = %u, block_address = 0x%llx", + show_dev_ino(__entry), + (unsigned int)__entry->nid, + (unsigned long long)__entry->blk_addr) +); + +DEFINE_EVENT(f2fs__truncate_node, f2fs_truncate_nodes_enter, + + TP_PROTO(struct inode *inode, nid_t nid, block_t blk_addr), + + TP_ARGS(inode, nid, blk_addr) +); + +DEFINE_EVENT(f2fs__inode_exit, f2fs_truncate_nodes_exit, + + TP_PROTO(struct inode *inode, int ret), + + TP_ARGS(inode, ret) +); + +DEFINE_EVENT(f2fs__truncate_node, f2fs_truncate_node, + + TP_PROTO(struct inode *inode, nid_t nid, block_t blk_addr), + + TP_ARGS(inode, nid, blk_addr) +); + +TRACE_EVENT(f2fs_truncate_partial_nodes, + + TP_PROTO(struct inode *inode, nid_t nid[], int depth, int err), + + TP_ARGS(inode, nid, depth, err), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(nid_t, nid[3]) + __field(int, depth) + __field(int, err) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->nid[0] = nid[0]; + __entry->nid[1] = nid[1]; + __entry->nid[2] = nid[2]; + __entry->depth = depth; + __entry->err = err; + ), + + TP_printk("dev = (%d,%d), ino = %lu, " + "nid[0] = %u, nid[1] = %u, nid[2] = %u, depth = %d, err = %d", + show_dev_ino(__entry), + (unsigned int)__entry->nid[0], + (unsigned int)__entry->nid[1], + (unsigned int)__entry->nid[2], + __entry->depth, + __entry->err) +); + +TRACE_EVENT(f2fs_map_blocks, + TP_PROTO(struct inode *inode, struct f2fs_map_blocks *map, int ret), + + TP_ARGS(inode, map, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(block_t, m_lblk) + __field(block_t, m_pblk) + __field(unsigned int, m_len) + __field(int, ret) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->m_lblk = map->m_lblk; + __entry->m_pblk = map->m_pblk; + __entry->m_len = map->m_len; + __entry->ret = ret; + ), + + TP_printk("dev = (%d,%d), ino = %lu, file offset = %llu, " + "start blkaddr = 0x%llx, len = 0x%llx, err = %d", + show_dev_ino(__entry), + (unsigned long long)__entry->m_lblk, + (unsigned long long)__entry->m_pblk, + (unsigned long long)__entry->m_len, + __entry->ret) +); + +TRACE_EVENT(f2fs_background_gc, + + TP_PROTO(struct super_block *sb, long wait_ms, + unsigned int prefree, unsigned int free), + + TP_ARGS(sb, wait_ms, prefree, free), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(long, wait_ms) + __field(unsigned int, prefree) + __field(unsigned int, free) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->wait_ms = wait_ms; + __entry->prefree = prefree; + __entry->free = free; + ), + + TP_printk("dev = (%d,%d), wait_ms = %ld, prefree = %u, free = %u", + show_dev(__entry), + __entry->wait_ms, + __entry->prefree, + __entry->free) +); + +TRACE_EVENT(f2fs_get_victim, + + TP_PROTO(struct super_block *sb, int type, int gc_type, + struct victim_sel_policy *p, unsigned int pre_victim, + unsigned int prefree, unsigned int free), + + TP_ARGS(sb, type, gc_type, p, pre_victim, prefree, free), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, type) + __field(int, gc_type) + __field(int, alloc_mode) + __field(int, gc_mode) + __field(unsigned int, victim) + __field(unsigned int, ofs_unit) + __field(unsigned int, pre_victim) + __field(unsigned int, prefree) + __field(unsigned int, free) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->type = type; + __entry->gc_type = gc_type; + __entry->alloc_mode = p->alloc_mode; + __entry->gc_mode = p->gc_mode; + __entry->victim = p->min_segno; + __entry->ofs_unit = p->ofs_unit; + __entry->pre_victim = pre_victim; + __entry->prefree = prefree; + __entry->free = free; + ), + + TP_printk("dev = (%d,%d), type = %s, policy = (%s, %s, %s), victim = %u " + "ofs_unit = %u, pre_victim_secno = %d, prefree = %u, free = %u", + show_dev(__entry), + show_data_type(__entry->type), + show_gc_type(__entry->gc_type), + show_alloc_mode(__entry->alloc_mode), + show_victim_policy(__entry->gc_mode), + __entry->victim, + __entry->ofs_unit, + (int)__entry->pre_victim, + __entry->prefree, + __entry->free) +); + +TRACE_EVENT(f2fs_fallocate, + + TP_PROTO(struct inode *inode, int mode, + loff_t offset, loff_t len, int ret), + + TP_ARGS(inode, mode, offset, len, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(int, mode) + __field(loff_t, offset) + __field(loff_t, len) + __field(loff_t, size) + __field(blkcnt_t, blocks) + __field(int, ret) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->mode = mode; + __entry->offset = offset; + __entry->len = len; + __entry->size = inode->i_size; + __entry->blocks = inode->i_blocks; + __entry->ret = ret; + ), + + TP_printk("dev = (%d,%d), ino = %lu, mode = %x, offset = %lld, " + "len = %lld, i_size = %lld, i_blocks = %llu, ret = %d", + show_dev_ino(__entry), + __entry->mode, + (unsigned long long)__entry->offset, + (unsigned long long)__entry->len, + (unsigned long long)__entry->size, + (unsigned long long)__entry->blocks, + __entry->ret) +); + +TRACE_EVENT(f2fs_direct_IO_enter, + + TP_PROTO(struct inode *inode, loff_t offset, unsigned long len, int rw), + + TP_ARGS(inode, offset, len, rw), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(loff_t, pos) + __field(unsigned long, len) + __field(int, rw) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pos = offset; + __entry->len = len; + __entry->rw = rw; + ), + + TP_printk("dev = (%d,%d), ino = %lu pos = %lld len = %lu rw = %d", + show_dev_ino(__entry), + __entry->pos, + __entry->len, + __entry->rw) +); + +TRACE_EVENT(f2fs_direct_IO_exit, + + TP_PROTO(struct inode *inode, loff_t offset, unsigned long len, + int rw, int ret), + + TP_ARGS(inode, offset, len, rw, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(loff_t, pos) + __field(unsigned long, len) + __field(int, rw) + __field(int, ret) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pos = offset; + __entry->len = len; + __entry->rw = rw; + __entry->ret = ret; + ), + + TP_printk("dev = (%d,%d), ino = %lu pos = %lld len = %lu " + "rw = %d ret = %d", + show_dev_ino(__entry), + __entry->pos, + __entry->len, + __entry->rw, + __entry->ret) +); + +TRACE_EVENT(f2fs_reserve_new_block, + + TP_PROTO(struct inode *inode, nid_t nid, unsigned int ofs_in_node), + + TP_ARGS(inode, nid, ofs_in_node), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(nid_t, nid) + __field(unsigned int, ofs_in_node) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->nid = nid; + __entry->ofs_in_node = ofs_in_node; + ), + + TP_printk("dev = (%d,%d), nid = %u, ofs_in_node = %u", + show_dev(__entry), + (unsigned int)__entry->nid, + __entry->ofs_in_node) +); + +DECLARE_EVENT_CLASS(f2fs__submit_page_bio, + + TP_PROTO(struct page *page, struct f2fs_io_info *fio), + + TP_ARGS(page, fio), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(pgoff_t, index) + __field(block_t, blkaddr) + __field(int, rw) + __field(int, type) + ), + + TP_fast_assign( + __entry->dev = page->mapping->host->i_sb->s_dev; + __entry->ino = page->mapping->host->i_ino; + __entry->index = page->index; + __entry->blkaddr = fio->blk_addr; + __entry->rw = fio->rw; + __entry->type = fio->type; + ), + + TP_printk("dev = (%d,%d), ino = %lu, page_index = 0x%lx, " + "blkaddr = 0x%llx, rw = %s%s, type = %s", + show_dev_ino(__entry), + (unsigned long)__entry->index, + (unsigned long long)__entry->blkaddr, + show_bio_type(__entry->rw), + show_block_type(__entry->type)) +); + +DEFINE_EVENT_CONDITION(f2fs__submit_page_bio, f2fs_submit_page_bio, + + TP_PROTO(struct page *page, struct f2fs_io_info *fio), + + TP_ARGS(page, fio), + + TP_CONDITION(page->mapping) +); + +DEFINE_EVENT_CONDITION(f2fs__submit_page_bio, f2fs_submit_page_mbio, + + TP_PROTO(struct page *page, struct f2fs_io_info *fio), + + TP_ARGS(page, fio), + + TP_CONDITION(page->mapping) +); + +DECLARE_EVENT_CLASS(f2fs__submit_bio, + + TP_PROTO(struct super_block *sb, struct f2fs_io_info *fio, + struct bio *bio), + + TP_ARGS(sb, fio, bio), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, rw) + __field(int, type) + __field(sector_t, sector) + __field(unsigned int, size) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->rw = fio->rw; + __entry->type = fio->type; + __entry->sector = bio->bi_sector; + __entry->size = bio->bi_size; + ), + + TP_printk("dev = (%d,%d), %s%s, %s, sector = %lld, size = %u", + show_dev(__entry), + show_bio_type(__entry->rw), + show_block_type(__entry->type), + (unsigned long long)__entry->sector, + __entry->size) +); + +DEFINE_EVENT_CONDITION(f2fs__submit_bio, f2fs_submit_write_bio, + + TP_PROTO(struct super_block *sb, struct f2fs_io_info *fio, + struct bio *bio), + + TP_ARGS(sb, fio, bio), + + TP_CONDITION(bio) +); + +DEFINE_EVENT_CONDITION(f2fs__submit_bio, f2fs_submit_read_bio, + + TP_PROTO(struct super_block *sb, struct f2fs_io_info *fio, + struct bio *bio), + + TP_ARGS(sb, fio, bio), + + TP_CONDITION(bio) +); + +TRACE_EVENT(f2fs_write_begin, + + TP_PROTO(struct inode *inode, loff_t pos, unsigned int len, + unsigned int flags), + + TP_ARGS(inode, pos, len, flags), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(loff_t, pos) + __field(unsigned int, len) + __field(unsigned int, flags) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pos = pos; + __entry->len = len; + __entry->flags = flags; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pos = %llu, len = %u, flags = %u", + show_dev_ino(__entry), + (unsigned long long)__entry->pos, + __entry->len, + __entry->flags) +); + +TRACE_EVENT(f2fs_write_end, + + TP_PROTO(struct inode *inode, loff_t pos, unsigned int len, + unsigned int copied), + + TP_ARGS(inode, pos, len, copied), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(loff_t, pos) + __field(unsigned int, len) + __field(unsigned int, copied) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pos = pos; + __entry->len = len; + __entry->copied = copied; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pos = %llu, len = %u, copied = %u", + show_dev_ino(__entry), + (unsigned long long)__entry->pos, + __entry->len, + __entry->copied) +); + +DECLARE_EVENT_CLASS(f2fs__page, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(int, type) + __field(int, dir) + __field(pgoff_t, index) + __field(int, dirty) + __field(int, uptodate) + ), + + TP_fast_assign( + __entry->dev = page->mapping->host->i_sb->s_dev; + __entry->ino = page->mapping->host->i_ino; + __entry->type = type; + __entry->dir = S_ISDIR(page->mapping->host->i_mode); + __entry->index = page->index; + __entry->dirty = PageDirty(page); + __entry->uptodate = PageUptodate(page); + ), + + TP_printk("dev = (%d,%d), ino = %lu, %s, %s, index = %lu, " + "dirty = %d, uptodate = %d", + show_dev_ino(__entry), + show_block_type(__entry->type), + show_file_type(__entry->dir), + (unsigned long)__entry->index, + __entry->dirty, + __entry->uptodate) +); + +DEFINE_EVENT(f2fs__page, f2fs_writepage, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +DEFINE_EVENT(f2fs__page, f2fs_do_write_data_page, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +DEFINE_EVENT(f2fs__page, f2fs_readpage, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +DEFINE_EVENT(f2fs__page, f2fs_set_page_dirty, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +DEFINE_EVENT(f2fs__page, f2fs_vm_page_mkwrite, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +DEFINE_EVENT(f2fs__page, f2fs_register_inmem_page, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +DEFINE_EVENT(f2fs__page, f2fs_commit_inmem_page, + + TP_PROTO(struct page *page, int type), + + TP_ARGS(page, type) +); + +TRACE_EVENT(f2fs_writepages, + + TP_PROTO(struct inode *inode, struct writeback_control *wbc, int type), + + TP_ARGS(inode, wbc, type), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(int, type) + __field(int, dir) + __field(long, nr_to_write) + __field(long, pages_skipped) + __field(loff_t, range_start) + __field(loff_t, range_end) + __field(pgoff_t, writeback_index) + __field(int, sync_mode) + __field(char, for_kupdate) + __field(char, for_background) + __field(char, tagged_writepages) + __field(char, for_reclaim) + __field(char, range_cyclic) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->type = type; + __entry->dir = S_ISDIR(inode->i_mode); + __entry->nr_to_write = wbc->nr_to_write; + __entry->pages_skipped = wbc->pages_skipped; + __entry->range_start = wbc->range_start; + __entry->range_end = wbc->range_end; + __entry->writeback_index = inode->i_mapping->writeback_index; + __entry->sync_mode = wbc->sync_mode; + __entry->for_kupdate = wbc->for_kupdate; + __entry->for_background = wbc->for_background; + __entry->tagged_writepages = wbc->tagged_writepages; + __entry->for_reclaim = wbc->for_reclaim; + __entry->range_cyclic = wbc->range_cyclic; + ), + + TP_printk("dev = (%d,%d), ino = %lu, %s, %s, nr_to_write %ld, " + "skipped %ld, start %lld, end %lld, wb_idx %lu, sync_mode %d, " + "kupdate %u background %u tagged %u reclaim %u cyclic %u", + show_dev_ino(__entry), + show_block_type(__entry->type), + show_file_type(__entry->dir), + __entry->nr_to_write, + __entry->pages_skipped, + __entry->range_start, + __entry->range_end, + (unsigned long)__entry->writeback_index, + __entry->sync_mode, + __entry->for_kupdate, + __entry->for_background, + __entry->tagged_writepages, + __entry->for_reclaim, + __entry->range_cyclic) +); + +TRACE_EVENT(f2fs_readpages, + + TP_PROTO(struct inode *inode, struct page *page, unsigned int nrpage), + + TP_ARGS(inode, page, nrpage), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(pgoff_t, start) + __field(unsigned int, nrpage) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->start = page->index; + __entry->nrpage = nrpage; + ), + + TP_printk("dev = (%d,%d), ino = %lu, start = %lu nrpage = %u", + show_dev_ino(__entry), + (unsigned long)__entry->start, + __entry->nrpage) +); + +TRACE_EVENT(f2fs_write_checkpoint, + + TP_PROTO(struct super_block *sb, int reason, char *msg), + + TP_ARGS(sb, reason, msg), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, reason) + __field(char *, msg) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->reason = reason; + __entry->msg = msg; + ), + + TP_printk("dev = (%d,%d), checkpoint for %s, state = %s", + show_dev(__entry), + show_cpreason(__entry->reason), + __entry->msg) +); + +TRACE_EVENT(f2fs_issue_discard, + + TP_PROTO(struct super_block *sb, block_t blkstart, block_t blklen), + + TP_ARGS(sb, blkstart, blklen), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(block_t, blkstart) + __field(block_t, blklen) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->blkstart = blkstart; + __entry->blklen = blklen; + ), + + TP_printk("dev = (%d,%d), blkstart = 0x%llx, blklen = 0x%llx", + show_dev(__entry), + (unsigned long long)__entry->blkstart, + (unsigned long long)__entry->blklen) +); + +TRACE_EVENT(f2fs_issue_flush, + + TP_PROTO(struct super_block *sb, unsigned int nobarrier, + unsigned int flush_merge), + + TP_ARGS(sb, nobarrier, flush_merge), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, nobarrier) + __field(unsigned int, flush_merge) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->nobarrier = nobarrier; + __entry->flush_merge = flush_merge; + ), + + TP_printk("dev = (%d,%d), %s %s", + show_dev(__entry), + __entry->nobarrier ? "skip (nobarrier)" : "issue", + __entry->flush_merge ? " with flush_merge" : "") +); + +TRACE_EVENT(f2fs_lookup_extent_tree_start, + + TP_PROTO(struct inode *inode, unsigned int pgofs), + + TP_ARGS(inode, pgofs), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(unsigned int, pgofs) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pgofs = pgofs; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u", + show_dev_ino(__entry), + __entry->pgofs) +); + +TRACE_EVENT_CONDITION(f2fs_lookup_extent_tree_end, + + TP_PROTO(struct inode *inode, unsigned int pgofs, + struct extent_info *ei), + + TP_ARGS(inode, pgofs, ei), + + TP_CONDITION(ei), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(unsigned int, pgofs) + __field(unsigned int, fofs) + __field(u32, blk) + __field(unsigned int, len) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pgofs = pgofs; + __entry->fofs = ei->fofs; + __entry->blk = ei->blk; + __entry->len = ei->len; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " + "ext_info(fofs: %u, blk: %u, len: %u)", + show_dev_ino(__entry), + __entry->pgofs, + __entry->fofs, + __entry->blk, + __entry->len) +); + +TRACE_EVENT(f2fs_update_extent_tree_range, + + TP_PROTO(struct inode *inode, unsigned int pgofs, block_t blkaddr, + unsigned int len), + + TP_ARGS(inode, pgofs, blkaddr, len), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(unsigned int, pgofs) + __field(u32, blk) + __field(unsigned int, len) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pgofs = pgofs; + __entry->blk = blkaddr; + __entry->len = len; + ), + + TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " + "blkaddr = %u, len = %u", + show_dev_ino(__entry), + __entry->pgofs, + __entry->blk, + __entry->len) +); + +TRACE_EVENT(f2fs_shrink_extent_tree, + + TP_PROTO(struct f2fs_sb_info *sbi, unsigned int node_cnt, + unsigned int tree_cnt), + + TP_ARGS(sbi, node_cnt, tree_cnt), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, node_cnt) + __field(unsigned int, tree_cnt) + ), + + TP_fast_assign( + __entry->dev = sbi->sb->s_dev; + __entry->node_cnt = node_cnt; + __entry->tree_cnt = tree_cnt; + ), + + TP_printk("dev = (%d,%d), shrunk: node_cnt = %u, tree_cnt = %u", + show_dev(__entry), + __entry->node_cnt, + __entry->tree_cnt) +); + +TRACE_EVENT(f2fs_destroy_extent_tree, + + TP_PROTO(struct inode *inode, unsigned int node_cnt), + + TP_ARGS(inode, node_cnt), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(ino_t, ino) + __field(unsigned int, node_cnt) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->node_cnt = node_cnt; + ), + + TP_printk("dev = (%d,%d), ino = %lu, destroyed: node_cnt = %u", + show_dev_ino(__entry), + __entry->node_cnt) +); + +#endif /* _TRACE_F2FS_H */ + + /* This part must be outside protection */ +#include diff --git a/init/version.c b/init/version.c index 86fe0ccb997..58170f18912 100644 --- a/init/version.c +++ b/init/version.c @@ -12,6 +12,7 @@ #include #include #include +#include #ifndef CONFIG_KALLSYMS #define version(a) Version_ ## a @@ -34,6 +35,7 @@ struct uts_namespace init_uts_ns = { .domainname = UTS_DOMAINNAME, }, .user_ns = &init_user_ns, + .proc_inum = PROC_UTS_INIT_INO, }; EXPORT_SYMBOL_GPL(init_uts_ns); diff --git a/ipc/msgutil.c b/ipc/msgutil.c index 26143d377c9..6471f1bdae9 100644 --- a/ipc/msgutil.c +++ b/ipc/msgutil.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "util.h" @@ -30,6 +31,7 @@ DEFINE_SPINLOCK(mq_lock); struct ipc_namespace init_ipc_ns = { .count = ATOMIC_INIT(1), .user_ns = &init_user_ns, + .proc_inum = PROC_IPC_INIT_INO, }; atomic_t nr_ipc_ns = ATOMIC_INIT(1); diff --git a/ipc/namespace.c b/ipc/namespace.c index ce0a647869b..cd7f7330d68 100644 --- a/ipc/namespace.c +++ b/ipc/namespace.c @@ -26,9 +26,16 @@ static struct ipc_namespace *create_ipc_ns(struct task_struct *tsk, if (ns == NULL) return ERR_PTR(-ENOMEM); + err = proc_alloc_inum(&ns->proc_inum); + if (err) { + kfree(ns); + return ERR_PTR(err); + } + atomic_set(&ns->count, 1); err = mq_init_ns(ns); if (err) { + proc_free_inum(ns->proc_inum); kfree(ns); return ERR_PTR(err); } @@ -113,6 +120,7 @@ static void free_ipc_ns(struct ipc_namespace *ns) */ ipcns_notify(IPCNS_REMOVED); put_user_ns(ns->user_ns); + proc_free_inum(ns->proc_inum); kfree(ns); } @@ -170,10 +178,18 @@ static int ipcns_install(struct nsproxy *nsproxy, void *ns) return 0; } +static unsigned int ipcns_inum(void *vp) +{ + struct ipc_namespace *ns = vp; + + return ns->proc_inum; +} + const struct proc_ns_operations ipcns_operations = { .name = "ipc", .type = CLONE_NEWIPC, .get = ipcns_get, .put = ipcns_put, .install = ipcns_install, + .inum = ipcns_inum, }; diff --git a/kernel/.gitignore b/kernel/.gitignore index ab4f1090f43..b3097bde4e9 100644 --- a/kernel/.gitignore +++ b/kernel/.gitignore @@ -4,3 +4,4 @@ config_data.h config_data.gz timeconst.h +hz.bc diff --git a/kernel/Makefile b/kernel/Makefile index cb41b9547c9..3d0dafa8361 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -10,7 +10,7 @@ obj-y = fork.o exec_domain.o panic.o printk.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ notifier.o ksysfs.o cred.o \ - async.o range.o groups.o + async.o range.o groups.o lglock.o ifdef CONFIG_FUNCTION_TRACER # Do not trace debug files and internal ftrace files @@ -122,8 +122,16 @@ $(obj)/config_data.h: $(obj)/config_data.gz FORCE $(obj)/time.o: $(obj)/timeconst.h -quiet_cmd_timeconst = TIMEC $@ - cmd_timeconst = $(PERL) $< $(CONFIG_HZ) > $@ +quiet_cmd_hzfile = HZFILE $@ + cmd_hzfile = echo "hz=$(CONFIG_HZ)" > $@ + +targets += hz.bc +$(obj)/hz.bc: $(objtree)/include/config/hz.h FORCE + $(call if_changed,hzfile) + +quiet_cmd_bc = BC $@ + cmd_bc = bc -q $(filter-out FORCE,$^) > $@ + targets += timeconst.h -$(obj)/timeconst.h: $(src)/timeconst.pl FORCE - $(call if_changed,timeconst) +$(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE + $(call if_changed,bc) diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 5bf0790497e..3a5ca582ba1 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -595,7 +595,7 @@ void audit_trim_trees(void) root_mnt = collect_mounts(&path); path_put(&path); - if (!root_mnt) + if (IS_ERR(root_mnt)) goto skip_it; spin_lock(&hash_lock); @@ -669,8 +669,8 @@ int audit_add_tree_rule(struct audit_krule *rule) goto Err; mnt = collect_mounts(&path); path_put(&path); - if (!mnt) { - err = -ENOMEM; + if (IS_ERR(mnt)) { + err = PTR_ERR(mnt); goto Err; } @@ -719,8 +719,8 @@ int audit_tag_tree(char *old, char *new) return err; tagged = collect_mounts(&path2); path_put(&path2); - if (!tagged) - return -ENOMEM; + if (IS_ERR(tagged)) + return PTR_ERR(tagged); err = kern_path(old, 0, &path1); if (err) { diff --git a/kernel/auditsc.c b/kernel/auditsc.c index af1de0f34ea..2d2f19b477b 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -67,6 +67,8 @@ #include #include #include +#include +#include #include "audit.h" @@ -83,7 +85,8 @@ /* Indicates that audit should log the full pathname. */ #define AUDIT_NAME_FULL -1 -/* no execve audit message should be longer than this (userspace limits) */ +/* no execve audit message should be longer than this (userspace limits), + * see the note near the top of audit_log_execve_info() about this value */ #define MAX_EXECVE_AUDIT_LEN 7500 /* number of audit rules */ @@ -1211,189 +1214,184 @@ static int audit_log_pid_context(struct audit_context *context, pid_t pid, return rc; } -/* - * to_send and len_sent accounting are very loose estimates. We aren't - * really worried about a hard cap to MAX_EXECVE_AUDIT_LEN so much as being - * within about 500 bytes (next page boundary) - * - * why snprintf? an int is up to 12 digits long. if we just assumed when - * logging that a[%d]= was going to be 16 characters long we would be wasting - * space in every audit message. In one 7500 byte message we can log up to - * about 1000 min size arguments. That comes down to about 50% waste of space - * if we didn't do the snprintf to find out how long arg_num_len was. - */ -static int audit_log_single_execve_arg(struct audit_context *context, - struct audit_buffer **ab, - int arg_num, - size_t *len_sent, - const char __user *p, - char *buf) -{ - char arg_num_len_buf[12]; - const char __user *tmp_p = p; - /* how many digits are in arg_num? 5 is the length of ' a=""' */ - size_t arg_num_len = snprintf(arg_num_len_buf, 12, "%d", arg_num) + 5; - size_t len, len_left, to_send; - size_t max_execve_audit_len = MAX_EXECVE_AUDIT_LEN; - unsigned int i, has_cntl = 0, too_long = 0; - int ret; - - /* strnlen_user includes the null we don't want to send */ - len_left = len = strnlen_user(p, MAX_ARG_STRLEN) - 1; - - /* - * We just created this mm, if we can't find the strings - * we just copied into it something is _very_ wrong. Similar - * for strings that are too long, we should not have created - * any. - */ - if (unlikely((len == -1) || len > MAX_ARG_STRLEN - 1)) { - WARN_ON(1); - send_sig(SIGKILL, current, 0); - return -1; - } - - /* walk the whole argument looking for non-ascii chars */ - do { - if (len_left > MAX_EXECVE_AUDIT_LEN) - to_send = MAX_EXECVE_AUDIT_LEN; - else - to_send = len_left; - ret = copy_from_user(buf, tmp_p, to_send); - /* - * There is no reason for this copy to be short. We just - * copied them here, and the mm hasn't been exposed to user- - * space yet. - */ - if (ret) { - WARN_ON(1); - send_sig(SIGKILL, current, 0); - return -1; - } - buf[to_send] = '\0'; - has_cntl = audit_string_contains_control(buf, to_send); - if (has_cntl) { - /* - * hex messages get logged as 2 bytes, so we can only - * send half as much in each message - */ - max_execve_audit_len = MAX_EXECVE_AUDIT_LEN / 2; - break; - } - len_left -= to_send; - tmp_p += to_send; - } while (len_left > 0); - - len_left = len; - - if (len > max_execve_audit_len) - too_long = 1; - - /* rewalk the argument actually logging the message */ - for (i = 0; len_left > 0; i++) { - int room_left; - - if (len_left > max_execve_audit_len) - to_send = max_execve_audit_len; - else - to_send = len_left; - - /* do we have space left to send this argument in this ab? */ - room_left = MAX_EXECVE_AUDIT_LEN - arg_num_len - *len_sent; - if (has_cntl) - room_left -= (to_send * 2); - else - room_left -= to_send; - if (room_left < 0) { - *len_sent = 0; - audit_log_end(*ab); - *ab = audit_log_start(context, GFP_KERNEL, AUDIT_EXECVE); - if (!*ab) - return 0; - } - - /* - * first record needs to say how long the original string was - * so we can be sure nothing was lost. - */ - if ((i == 0) && (too_long)) - audit_log_format(*ab, " a%d_len=%zu", arg_num, - has_cntl ? 2*len : len); - - /* - * normally arguments are small enough to fit and we already - * filled buf above when we checked for control characters - * so don't bother with another copy_from_user - */ - if (len >= max_execve_audit_len) - ret = copy_from_user(buf, p, to_send); - else - ret = 0; - if (ret) { - WARN_ON(1); - send_sig(SIGKILL, current, 0); - return -1; - } - buf[to_send] = '\0'; - - /* actually log it */ - audit_log_format(*ab, " a%d", arg_num); - if (too_long) - audit_log_format(*ab, "[%d]", i); - audit_log_format(*ab, "="); - if (has_cntl) - audit_log_n_hex(*ab, buf, to_send); - else - audit_log_string(*ab, buf); - - p += to_send; - len_left -= to_send; - *len_sent += arg_num_len; - if (has_cntl) - *len_sent += to_send * 2; - else - *len_sent += to_send; - } - /* include the null we didn't log */ - return len + 1; -} - static void audit_log_execve_info(struct audit_context *context, struct audit_buffer **ab, struct audit_aux_data_execve *axi) { - int i, len; - size_t len_sent = 0; - const char __user *p; + long len_max; + long len_rem; + long len_full; + long len_buf; + long len_abuf; + long len_tmp; + bool require_data; + bool encode; + unsigned int iter; + unsigned int arg; + char *buf_head; char *buf; + const char __user *p; + + /* NOTE: this buffer needs to be large enough to hold all the non-arg + * data we put in the audit record for this argument (see the + * code below) ... at this point in time 96 is plenty */ + char abuf[96]; if (axi->mm != current->mm) return; /* execve failed, no additional info */ p = (const char __user *)axi->mm->arg_start; - audit_log_format(*ab, "argc=%d", axi->argc); - - /* - * we need some kernel buffer to hold the userspace args. Just - * allocate one big one rather than allocating one of the right size - * for every single argument inside audit_log_single_execve_arg() - * should be <8k allocation so should be pretty safe. - */ - buf = kmalloc(MAX_EXECVE_AUDIT_LEN + 1, GFP_KERNEL); - if (!buf) { - audit_panic("out of memory for argv string\n"); + /* NOTE: we set MAX_EXECVE_AUDIT_LEN to a rather arbitrary limit, the + * current value of 7500 is not as important as the fact that it + * is less than 8k, a setting of 7500 gives us plenty of wiggle + * room if we go over a little bit in the logging below */ + WARN_ON_ONCE(MAX_EXECVE_AUDIT_LEN > 7500); + len_max = MAX_EXECVE_AUDIT_LEN; + + /* scratch buffer to hold the userspace args */ + buf_head = kmalloc(MAX_EXECVE_AUDIT_LEN + 1, GFP_KERNEL); + if (!buf_head) { + audit_panic("out of memory for argv string"); return; } + buf = buf_head; - for (i = 0; i < axi->argc; i++) { - len = audit_log_single_execve_arg(context, ab, i, - &len_sent, p, buf); - if (len <= 0) - break; - p += len; - } - kfree(buf); + audit_log_format(*ab, "argc=%d", axi->argc); + + len_rem = len_max; + len_buf = 0; + len_full = 0; + require_data = true; + encode = false; + iter = 0; + arg = 0; + do { + /* NOTE: we don't ever want to trust this value for anything + * serious, but the audit record format insists we + * provide an argument length for really long arguments, + * e.g. > MAX_EXECVE_AUDIT_LEN, so we have no choice but + * to use strncpy_from_user() to obtain this value for + * recording in the log, although we don't use it + * anywhere here to avoid a double-fetch problem */ + if (len_full == 0) + len_full = strnlen_user(p, MAX_ARG_STRLEN) - 1; + + /* read more data from userspace */ + if (require_data) { + /* can we make more room in the buffer? */ + if (buf != buf_head) { + memmove(buf_head, buf, len_buf); + buf = buf_head; + } + + /* fetch as much as we can of the argument */ + len_tmp = strncpy_from_user(&buf_head[len_buf], p, + len_max - len_buf); + if (len_tmp == -EFAULT) { + /* unable to copy from userspace */ + send_sig(SIGKILL, current, 0); + goto out; + } else if (len_tmp == (len_max - len_buf)) { + /* buffer is not large enough */ + require_data = true; + /* NOTE: if we are going to span multiple + * buffers force the encoding so we stand + * a chance at a sane len_full value and + * consistent record encoding */ + encode = true; + len_full = len_full * 2; + p += len_tmp; + } else { + require_data = false; + if (!encode) + encode = audit_string_contains_control( + buf, len_tmp); + /* try to use a trusted value for len_full */ + if (len_full < len_max) + len_full = (encode ? + len_tmp * 2 : len_tmp); + p += len_tmp + 1; + } + len_buf += len_tmp; + buf_head[len_buf] = '\0'; + + /* length of the buffer in the audit record? */ + len_abuf = (encode ? len_buf * 2 : len_buf + 2); + } + + /* write as much as we can to the audit log */ + if (len_buf > 0) { + /* NOTE: some magic numbers here - basically if we + * can't fit a reasonable amount of data into the + * existing audit buffer, flush it and start with + * a new buffer */ + if ((sizeof(abuf) + 8) > len_rem) { + len_rem = len_max; + audit_log_end(*ab); + *ab = audit_log_start(context, + GFP_KERNEL, AUDIT_EXECVE); + if (!*ab) + goto out; + } + + /* create the non-arg portion of the arg record */ + len_tmp = 0; + if (require_data || (iter > 0) || + ((len_abuf + sizeof(abuf)) > len_rem)) { + if (iter == 0) { + len_tmp += snprintf(&abuf[len_tmp], + sizeof(abuf) - len_tmp, + " a%d_len=%lu", + arg, len_full); + } + len_tmp += snprintf(&abuf[len_tmp], + sizeof(abuf) - len_tmp, + " a%d[%d]=", arg, iter++); + } else + len_tmp += snprintf(&abuf[len_tmp], + sizeof(abuf) - len_tmp, + " a%d=", arg); + WARN_ON(len_tmp >= sizeof(abuf)); + abuf[sizeof(abuf) - 1] = '\0'; + + /* log the arg in the audit record */ + audit_log_format(*ab, "%s", abuf); + len_rem -= len_tmp; + len_tmp = len_buf; + if (encode) { + if (len_abuf > len_rem) + len_tmp = len_rem / 2; /* encoding */ + audit_log_n_hex(*ab, buf, len_tmp); + len_rem -= len_tmp * 2; + len_abuf -= len_tmp * 2; + } else { + if (len_abuf > len_rem) + len_tmp = len_rem - 2; /* quotes */ + audit_log_n_string(*ab, buf, len_tmp); + len_rem -= len_tmp + 2; + /* don't subtract the "2" because we still need + * to add quotes to the remaining string */ + len_abuf -= len_tmp; + } + len_buf -= len_tmp; + buf += len_tmp; + } + + /* ready to move to the next argument? */ + if ((len_buf == 0) && !require_data) { + arg++; + iter = 0; + len_full = 0; + require_data = true; + encode = false; + } + } while (arg < axi->argc); + + /* NOTE: the caller handles the final audit_log_end() call */ + +out: + kfree(buf_head); } static void audit_log_cap(struct audit_buffer *ab, char *prefix, kernel_cap_t *cap) @@ -2710,13 +2708,16 @@ void audit_core_dumps(long signr) audit_log_end(ab); } -void __audit_seccomp(unsigned long syscall) +void __audit_seccomp(unsigned long syscall, long signr, int code) { struct audit_buffer *ab; ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_ANOM_ABEND); - audit_log_abend(ab, "seccomp", SIGKILL); + audit_log_abend(ab, "seccomp", signr); audit_log_format(ab, " syscall=%ld", syscall); + audit_log_format(ab, " compat=%d", is_compat_task()); + audit_log_format(ab, " ip=0x%lx", KSTK_EIP(current)); + audit_log_format(ab, " code=0x%x", code); audit_log_end(ab); } diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 5d631677c18..fd8a7440531 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c @@ -2116,7 +2116,6 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader) /* * step 5: success! and cleanup */ - synchronize_rcu(); cgroup_wakeup_rmdir_waiter(cgrp); retval = 0; out_put_css_set_refs: @@ -5223,7 +5222,7 @@ static int cgroup_css_links_read(struct cgroup *cont, struct css_set *cg = link->cg; struct task_struct *task; int count = 0; - seq_printf(seq, "css_set %p\n", cg); + seq_printf(seq, "css_set %pK\n", cg); list_for_each_entry(task, &cg->tasks, cg_list) { if (count++ > MAX_TASKS_SHOWN_PER_CSS) { seq_puts(seq, " ...\n"); diff --git a/kernel/events/core.c b/kernel/events/core.c index 7dd822b5e39..101bf3a6814 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -153,8 +153,13 @@ static struct srcu_struct pmus_srcu; * 0 - disallow raw tracepoint access for unpriv * 1 - disallow cpu events for unpriv * 2 - disallow kernel profiling for unpriv + * 3 - disallow all unpriv perf event use */ +#ifdef CONFIG_SECURITY_PERF_EVENTS_RESTRICT +int sysctl_perf_event_paranoid __read_mostly = 3; +#else int sysctl_perf_event_paranoid __read_mostly = 1; +#endif /* Minimum for 512 kiB + 1 user control page */ int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ @@ -1085,10 +1090,17 @@ static void perf_group_detach(struct perf_event *event) * If this was a group event with sibling events then * upgrade the siblings to singleton events by adding them * to whatever list we are on. + * If this isn't on a list, make sure we still remove the sibling's + * group_entry from this sibling_list; otherwise, when that sibling + * is later deallocated, it will try to remove itself from this + * sibling_list, which may well have been deallocated already, + * resulting in a use-after-free. */ list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { if (list) list_move_tail(&sibling->group_entry, list); + else + list_del_init(&sibling->group_entry); sibling->group_leader = sibling; /* Inherit group flags from the previous leader */ @@ -5026,7 +5038,8 @@ static int perf_swevent_add(struct perf_event *event, int flags) static void perf_swevent_del(struct perf_event *event, int flags) { - hlist_del_rcu(&event->hlist_entry); + if(!hlist_unhashed(&event->hlist_entry)) + hlist_del_rcu(&event->hlist_entry); } static void perf_swevent_start(struct perf_event *event, int flags) @@ -5089,7 +5102,6 @@ static int swevent_hlist_get_cpu(struct perf_event *event, int cpu) int err = 0; mutex_lock(&swhash->hlist_mutex); - if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { struct swevent_hlist *hlist; @@ -6239,10 +6251,16 @@ SYSCALL_DEFINE5(perf_event_open, if (flags & ~PERF_FLAG_ALL) return -EINVAL; + if (perf_paranoid_any() && !capable(CAP_SYS_ADMIN)) + return -EACCES; + err = perf_copy_attr(attr_uptr, &attr); if (err) return err; + if (attr.constraint_duplicate || attr.__reserved_1) + return -EINVAL; + if (!attr.exclude_kernel) { if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) return -EACCES; @@ -7086,12 +7104,6 @@ static void perf_event_exit_cpu_context(int cpu) static void perf_event_exit_cpu(int cpu) { - struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); - - mutex_lock(&swhash->hlist_mutex); - swevent_hlist_release(swhash); - mutex_unlock(&swhash->hlist_mutex); - perf_event_exit_cpu_context(cpu); } #else diff --git a/kernel/exit.c b/kernel/exit.c index bc7d69084fd..f46e9bf8ae4 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -74,6 +74,7 @@ static void __unhash_process(struct task_struct *p, bool group_dead) __this_cpu_dec(process_counts); } list_del_rcu(&p->thread_group); + list_del_rcu(&p->thread_node); } /* @@ -639,7 +640,9 @@ static void exit_mm(struct task_struct * tsk) { struct mm_struct *mm = tsk->mm; struct core_state *core_state; +#ifndef CONFIG_UML int mm_released; +#endif mm_release(tsk, mm); if (!mm) @@ -685,9 +688,11 @@ static void exit_mm(struct task_struct * tsk) task_unlock(tsk); mm_update_next_owner(mm); +#ifndef CONFIG_UML mm_released = mmput(mm); if (mm_released) set_tsk_thread_flag(tsk, TIF_MM_RELEASED); +#endif } /* @@ -948,6 +953,11 @@ void do_exit(long code) } exit_signals(tsk); /* sets PF_EXITING */ + + if (tsk->flags & PF_SU) { + su_exit(); + } + /* * tsk->flags are checked in the futex code to protect against * an exiting task cleaning up the robust pi futexes. diff --git a/kernel/fork.c b/kernel/fork.c index 0de735c9b6b..542a1180020 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -174,6 +175,7 @@ void free_task(struct task_struct *tsk) free_thread_info(tsk->stack); rt_mutex_debug_task_free(tsk); ftrace_graph_exit_task(tsk); + put_seccomp_filter(tsk); free_task_struct(tsk); } EXPORT_SYMBOL(free_task); @@ -293,7 +295,18 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) if (err) goto out; + tsk->flags &= ~PF_SU; + tsk->stack = ti; +#ifdef CONFIG_SECCOMP + /* + * We must handle setting up seccomp filters once we're under + * the sighand lock in case orig has changed between now and + * then. Until then, filter must be NULL to avoid messing up + * the usage counts on the error path calling free_task. + */ + tsk->seccomp.filter = NULL; +#endif setup_thread_stack(tsk, orig); clear_user_return_notifier(tsk); @@ -1045,6 +1058,11 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) sig->nr_threads = 1; atomic_set(&sig->live, 1); atomic_set(&sig->sigcnt, 1); + + /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */ + sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); + tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); + init_waitqueue_head(&sig->wait_chldexit); if (clone_flags & CLONE_NEWPID) sig->flags |= SIGNAL_UNKILLABLE; @@ -1089,6 +1107,39 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p) p->flags = new_flags; } +static void copy_seccomp(struct task_struct *p) +{ +#ifdef CONFIG_SECCOMP + /* + * Must be called with sighand->lock held, which is common to + * all threads in the group. Holding cred_guard_mutex is not + * needed because this new task is not yet running and cannot + * be racing exec. + */ + BUG_ON(!spin_is_locked(¤t->sighand->siglock)); + + /* Ref-count the new filter user, and assign it. */ + get_seccomp_filter(current); + p->seccomp = current->seccomp; + + /* + * Explicitly enable no_new_privs here in case it got set + * between the task_struct being duplicated and holding the + * sighand lock. The seccomp state and nnp must be in sync. + */ + if (task_no_new_privs(current)) + task_set_no_new_privs(p); + + /* + * If the parent gained a seccomp mode after copying thread + * flags and between before we held the sighand lock, we have + * to manually enable the seccomp thread flag here. + */ + if (p->seccomp.mode != SECCOMP_MODE_DISABLED) + set_tsk_thread_flag(p, TIF_SECCOMP); +#endif +} + SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) { current->clear_child_tid = tidptr; @@ -1228,6 +1279,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->utime = p->stime = p->gtime = 0; p->utimescaled = p->stimescaled = 0; + p->cpu_power = 0; #ifndef CONFIG_VIRT_CPU_ACCOUNTING p->prev_utime = p->prev_stime = 0; #endif @@ -1422,6 +1474,12 @@ static struct task_struct *copy_process(unsigned long clone_flags, spin_lock(¤t->sighand->siglock); + /* + * Copy seccomp details explicitly here, in case they were changed + * before holding sighand lock. + */ + copy_seccomp(p); + /* * Process group and session signals need to be delivered to just the * parent before the fork or both the parent and the child after the @@ -1460,6 +1518,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, list_add_tail(&p->sibling, &p->real_parent->children); list_add_tail_rcu(&p->tasks, &init_task.tasks); __this_cpu_inc(process_counts); + } else { + list_add_tail_rcu(&p->thread_node, + &p->signal->thread_head); } attach_pid(p, PIDTYPE_PID, pid); nr_threads++; diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 7a391df4bce..16aba7dc864 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -262,12 +262,13 @@ void unmask_irq(struct irq_desc *desc) * handler. The handler function is called inside the calling * threads context. */ -void handle_nested_irq(unsigned int irq) +bool handle_nested_irq(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); struct irqaction *action; int mask_this_irq = 0; irqreturn_t action_ret; + bool handled = false; might_sleep(); @@ -291,6 +292,8 @@ void handle_nested_irq(unsigned int irq) raw_spin_lock_irq(&desc->lock); irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS); + handled = true; + out_unlock: raw_spin_unlock_irq(&desc->lock); if (unlikely(mask_this_irq)) { @@ -298,6 +301,8 @@ void handle_nested_irq(unsigned int irq) mask_irq(desc); chip_bus_sync_unlock(desc); } + + return handled; } EXPORT_SYMBOL_GPL(handle_nested_irq); @@ -320,9 +325,11 @@ static bool irq_check_poll(struct irq_desc *desc) * Note: The caller is expected to handle the ack, clear, mask and * unmask issues if necessary. */ -void +bool handle_simple_irq(unsigned int irq, struct irq_desc *desc) { + bool handled = false; + raw_spin_lock(&desc->lock); if (unlikely(irqd_irq_inprogress(&desc->irq_data))) @@ -337,8 +344,11 @@ handle_simple_irq(unsigned int irq, struct irq_desc *desc) handle_irq_event(desc); + handled = true; + out_unlock: raw_spin_unlock(&desc->lock); + return handled; } EXPORT_SYMBOL_GPL(handle_simple_irq); @@ -370,9 +380,11 @@ static void cond_unmask_irq(struct irq_desc *desc) * it after the associated handler has acknowledged the device, so the * interrupt line is back to inactive. */ -void +bool handle_level_irq(unsigned int irq, struct irq_desc *desc) { + bool handled = false; + raw_spin_lock(&desc->lock); mask_ack_irq(desc); @@ -394,8 +406,11 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) cond_unmask_irq(desc); + handled = true; + out_unlock: raw_spin_unlock(&desc->lock); + return handled; } EXPORT_SYMBOL_GPL(handle_level_irq); @@ -419,9 +434,11 @@ static inline void preflow_handler(struct irq_desc *desc) { } * for modern forms of interrupt handlers, which handle the flow * details in hardware, transparently. */ -void +bool handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) { + bool handled = false; + raw_spin_lock(&desc->lock); if (unlikely(irqd_irq_inprogress(&desc->irq_data))) @@ -451,11 +468,13 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) if (desc->istate & IRQS_ONESHOT) cond_unmask_irq(desc); + handled = true; + out_eoi: desc->irq_data.chip->irq_eoi(&desc->irq_data); out_unlock: raw_spin_unlock(&desc->lock); - return; + return handled; out: if (!(desc->irq_data.chip->flags & IRQCHIP_EOI_IF_HANDLED)) goto out_eoi; @@ -478,9 +497,11 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) * the handler was running. If all pending interrupts are handled, the * loop is left. */ -void +bool handle_edge_irq(unsigned int irq, struct irq_desc *desc) { + bool handled = false; + raw_spin_lock(&desc->lock); desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING); @@ -520,12 +541,14 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc) } handle_irq_event(desc); + handled = true; } while ((desc->istate & IRQS_PENDING) && !irqd_irq_disabled(&desc->irq_data)); out_unlock: raw_spin_unlock(&desc->lock); + return handled; } EXPORT_SYMBOL(handle_edge_irq); @@ -538,8 +561,9 @@ EXPORT_SYMBOL(handle_edge_irq); * Similar as the above handle_edge_irq, but using eoi and w/o the * mask/unmask logic. */ -void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc) +bool handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc) { + bool handled = false; struct irq_chip *chip = irq_desc_get_chip(desc); raw_spin_lock(&desc->lock); @@ -564,6 +588,7 @@ void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc) goto out_eoi; handle_irq_event(desc); + handled = true; } while ((desc->istate & IRQS_PENDING) && !irqd_irq_disabled(&desc->irq_data)); @@ -571,6 +596,7 @@ void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc) out_eoi: chip->irq_eoi(&desc->irq_data); raw_spin_unlock(&desc->lock); + return handled; } #endif @@ -581,7 +607,7 @@ void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc) * * Per CPU interrupts on SMP machines without locking requirements */ -void +bool handle_percpu_irq(unsigned int irq, struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); @@ -595,6 +621,8 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) if (chip->irq_eoi) chip->irq_eoi(&desc->irq_data); + + return true; } /** @@ -609,7 +637,7 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) * contain the real device id for the cpu on which this handler is * called */ -void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc) +bool handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); struct irqaction *action = desc->action; @@ -627,6 +655,8 @@ void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc) if (chip->irq_eoi) chip->irq_eoi(&desc->irq_data); + + return true; } void diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 131ca176b49..b87bf8cd9df 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -27,11 +27,12 @@ * * Handles spurious and unhandled IRQ's. It also prints a debugmessage. */ -void handle_bad_irq(unsigned int irq, struct irq_desc *desc) +bool handle_bad_irq(unsigned int irq, struct irq_desc *desc) { print_irq_desc(irq, desc); kstat_incr_irqs_this_cpu(irq, desc); ack_bad_irq(irq); + return true; } /* diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 192a302d6cf..e0999509e2d 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -14,6 +14,9 @@ #include #include #include +#ifndef CONFIG_UML +#include +#endif #include "internals.h" @@ -303,16 +306,27 @@ static int irq_expand_nr_irqs(unsigned int nr) /** * generic_handle_irq - Invoke the handler for a particular irq * @irq: The irq number to handle - * + * returns: + * negative on error + * 0 when the interrupt handler was not called + * 1 when the interrupt handler was called */ + int generic_handle_irq(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); if (!desc) return -EINVAL; - generic_handle_irq_desc(irq, desc); - return 0; + +#ifndef CONFIG_UML + if (unlikely(logging_wakeup_reasons_nosync())) + return log_possible_wakeup_reason(irq, + desc, + generic_handle_irq_desc); +#endif + + return generic_handle_irq_desc(irq, desc); } EXPORT_SYMBOL_GPL(generic_handle_irq); diff --git a/kernel/irq/pm.c b/kernel/irq/pm.c index 0e3b91916c7..e01feefb2c7 100644 --- a/kernel/irq/pm.c +++ b/kernel/irq/pm.c @@ -10,7 +10,7 @@ #include #include #include - +#include #include "internals.h" /** @@ -105,6 +105,10 @@ int check_wakeup_irqs(void) for_each_irq_desc(irq, desc) { if (irqd_is_wakeup_set(&desc->irq_data)) { if (desc->istate & IRQS_PENDING) { + log_suspend_abort_reason("Wakeup IRQ %d %s pending", + irq, + desc->action && desc->action->name ? + desc->action->name : ""); pr_info("Wakeup IRQ %d %s pending, suspend aborted\n", irq, desc->action && desc->action->name ? diff --git a/kernel/lglock.c b/kernel/lglock.c new file mode 100644 index 00000000000..6535a667a5a --- /dev/null +++ b/kernel/lglock.c @@ -0,0 +1,89 @@ +/* See include/linux/lglock.h for description */ +#include +#include +#include +#include + +/* + * Note there is no uninit, so lglocks cannot be defined in + * modules (but it's fine to use them from there) + * Could be added though, just undo lg_lock_init + */ + +void lg_lock_init(struct lglock *lg, char *name) +{ + LOCKDEP_INIT_MAP(&lg->lock_dep_map, name, &lg->lock_key, 0); +} +EXPORT_SYMBOL(lg_lock_init); + +void lg_local_lock(struct lglock *lg) +{ + arch_spinlock_t *lock; + + preempt_disable(); + rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_); + lock = this_cpu_ptr(lg->lock); + arch_spin_lock(lock); +} +EXPORT_SYMBOL(lg_local_lock); + +void lg_local_unlock(struct lglock *lg) +{ + arch_spinlock_t *lock; + + rwlock_release(&lg->lock_dep_map, 1, _RET_IP_); + lock = this_cpu_ptr(lg->lock); + arch_spin_unlock(lock); + preempt_enable(); +} +EXPORT_SYMBOL(lg_local_unlock); + +void lg_local_lock_cpu(struct lglock *lg, int cpu) +{ + arch_spinlock_t *lock; + + preempt_disable(); + rwlock_acquire_read(&lg->lock_dep_map, 0, 0, _RET_IP_); + lock = per_cpu_ptr(lg->lock, cpu); + arch_spin_lock(lock); +} +EXPORT_SYMBOL(lg_local_lock_cpu); + +void lg_local_unlock_cpu(struct lglock *lg, int cpu) +{ + arch_spinlock_t *lock; + + rwlock_release(&lg->lock_dep_map, 1, _RET_IP_); + lock = per_cpu_ptr(lg->lock, cpu); + arch_spin_unlock(lock); + preempt_enable(); +} +EXPORT_SYMBOL(lg_local_unlock_cpu); + +void lg_global_lock(struct lglock *lg) +{ + int i; + + preempt_disable(); + rwlock_acquire(&lg->lock_dep_map, 0, 0, _RET_IP_); + for_each_possible_cpu(i) { + arch_spinlock_t *lock; + lock = per_cpu_ptr(lg->lock, i); + arch_spin_lock(lock); + } +} +EXPORT_SYMBOL(lg_global_lock); + +void lg_global_unlock(struct lglock *lg) +{ + int i; + + rwlock_release(&lg->lock_dep_map, 1, _RET_IP_); + for_each_possible_cpu(i) { + arch_spinlock_t *lock; + lock = per_cpu_ptr(lg->lock, i); + arch_spin_unlock(lock); + } + preempt_enable(); +} +EXPORT_SYMBOL(lg_global_unlock); diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index b576f7f14bc..5b6ce19064a 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -66,7 +66,7 @@ static struct nsproxy *create_new_namespaces(unsigned long flags, if (!new_nsp) return ERR_PTR(-ENOMEM); - new_nsp->mnt_ns = copy_mnt_ns(flags, tsk->nsproxy->mnt_ns, new_fs); + new_nsp->mnt_ns = copy_mnt_ns(flags, tsk->nsproxy->mnt_ns, task_cred_xxx(tsk, user_ns), new_fs); if (IS_ERR(new_nsp->mnt_ns)) { err = PTR_ERR(new_nsp->mnt_ns); goto out_ns; diff --git a/kernel/pid.c b/kernel/pid.c index 7acf5909415..b1e701b1cda 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -78,6 +78,7 @@ struct pid_namespace init_pid_ns = { .last_pid = 0, .level = 0, .child_reaper = &init_task, + .proc_inum = PROC_PID_INIT_INO, }; EXPORT_SYMBOL_GPL(init_pid_ns); diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index 57bc1fd35b3..4f15befb090 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -88,6 +88,10 @@ static struct pid_namespace *create_pid_namespace(struct pid_namespace *parent_p if (ns->pid_cachep == NULL) goto out_free_map; + err = proc_alloc_inum(&ns->proc_inum); + if (err) + goto out_free_map; + kref_init(&ns->kref); ns->level = level; ns->parent = get_pid_ns(parent_pid_ns); @@ -118,6 +122,7 @@ static void destroy_pid_namespace(struct pid_namespace *ns) { int i; + proc_free_inum(ns->proc_inum); for (i = 0; i < PIDMAP_ENTRIES; i++) kfree(ns->pidmap[i].page); kmem_cache_free(pid_ns_cachep, ns); diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 904fc660339..4215cd4cf91 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -285,3 +285,20 @@ config SUSPEND_TIME Prints the time spent in suspend in the kernel log, and keeps statistics on the time spent in suspend in /sys/kernel/debug/suspend_time + +config DEDUCE_WAKEUP_REASONS + bool + default n + +config PARTIALRESUME + bool "Partial-resume framework" + select DEDUCE_WAKEUP_REASONS + ---help--- + Provides hooks for drivers to register partial-resume handlers. + Similar to suspend_again support already in kernel, except that it + operates with kernel threads unfrozen and userspace still frozen, + allowing callbacks to block on work queues and kernel threads. + Partial resume will occur only if all wakeup sources have + partial-resume handlers associated with them, and they all return + true. + diff --git a/kernel/power/Makefile b/kernel/power/Makefile index 299f8a4d42f..460f0b21a2e 100644 --- a/kernel/power/Makefile +++ b/kernel/power/Makefile @@ -15,3 +15,4 @@ obj-$(CONFIG_PM_WAKELOCKS) += wakelock.o obj-$(CONFIG_MAGIC_SYSRQ) += poweroff.o obj-$(CONFIG_SUSPEND) += wakeup_reason.o +obj-$(CONFIG_PARTIALRESUME) += partialresume.o diff --git a/kernel/power/partialresume.c b/kernel/power/partialresume.c new file mode 100644 index 00000000000..844955218bc --- /dev/null +++ b/kernel/power/partialresume.c @@ -0,0 +1,191 @@ +/* kernel/power/partialresume.c + * + * Copyright (C) 2015 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static DEFINE_MUTEX(pr_handlers_lock); +static LIST_HEAD(pr_handlers); +static LIST_HEAD(pr_matches); +static struct partial_resume_stats match_stats; +static struct partial_resume_stats consensus_stats; +static struct kobject *partialresume; + +bool suspend_again_match(const struct list_head *irqs, + const struct list_head *unfinished) +{ + const struct wakeup_irq_node *i; + struct partial_resume *h, *match; + + INIT_LIST_HEAD(&pr_matches); + + match_stats.total++; + + if (!irqs || list_empty(irqs)) + return false; + + list_for_each_entry(i, irqs, next) { + match = NULL; + list_for_each_entry(h, &pr_handlers, next_handler) { + if (i->irq == h->irq) { + match = h; + break; + } + } + if (!match) { + pr_debug("%s: wakeup irq %d does not have a handler\n", __func__, i->irq); + return false; + } + list_add(&match->next_match, &pr_matches); + } + + match_stats.total_yes++; + + return true; +} + + +bool suspend_again_consensus(void) +{ + struct partial_resume *h; + + BUG_ON(list_empty(&pr_matches)); + list_for_each_entry(h, &pr_matches, next_match) { + h->stats.total++; + if (!h->partial_resume(h)) { + pr_debug("%s: partial-resume for %d: false\n", __func__, h->irq); + return false; + } + h->stats.total_yes++; + pr_debug("%s: partial-resume for %d: true\n", __func__, h->irq); + } + + consensus_stats.total_yes++; + + return true; +} + + +int register_partial_resume(struct partial_resume *handler) +{ + struct partial_resume *e; + + if (!handler || !handler->irq || !handler->partial_resume) + return -EINVAL; + + mutex_lock(&pr_handlers_lock); + list_for_each_entry(e, &pr_handlers, next_handler) { + if (e->irq == handler->irq) { + if (e->partial_resume == handler->partial_resume) { + mutex_unlock(&pr_handlers_lock); + return 0; + } + pr_err("%s: error registering %pF for irq %d: "\ + "%pF already registered\n", + __func__, + handler->partial_resume, + e->irq, + e->partial_resume); + mutex_unlock(&pr_handlers_lock); + return -EIO; + } + } + + list_add(&handler->next_handler, &pr_handlers); + mutex_unlock(&pr_handlers_lock); + return 0; +} + +void unregister_partial_resume(struct partial_resume *handler) +{ + mutex_lock(&pr_handlers_lock); + list_del(&handler->next_handler); + mutex_unlock(&pr_handlers_lock); +} + + +static ssize_t partialresume_stats_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + ssize_t offset = 0; + struct partial_resume *h; + + offset += sprintf(buf + offset, "global: %d %d %d \n", + match_stats.total, + match_stats.total_yes, + consensus_stats.total_yes); + + mutex_lock(&pr_handlers_lock); + + list_for_each_entry(h, &pr_handlers, next_handler) { + offset += sprintf(buf + offset, "%d: %d %d\n", + h->irq, + h->stats.total, + h->stats.total_yes); + } + + mutex_unlock(&pr_handlers_lock); + + return offset; +} + +static struct kobj_attribute partialresume_stats = __ATTR_RO(partialresume_stats); + +static struct attribute *partialresume_stats_attrs[] = { + &partialresume_stats.attr, + NULL, +}; +static struct attribute_group partialresume_stats_attr_group = { + .attrs = partialresume_stats_attrs, +}; + +int __init partial_resume_init(void) +{ + int rc = -EIO; + + partialresume = kobject_create_and_add("partialresume", kernel_kobj); + if (!partialresume) { + pr_warning("%s: failed to create a sysfs kobject\n", __func__); + goto fail; + } + + rc = sysfs_create_group(partialresume, &partialresume_stats_attr_group); + if (rc) { + pr_warning("%s: failed to create a sysfs group\n", __func__); + goto fail_kobject_put; + } + + return 0; + +#if 0 +fail_remove_group: + sysfs_remove_group(partialresume, &partialresume_stats_attr_group); +#endif +fail_kobject_put: + kobject_put(partialresume); +fail: + return rc; +} + +subsys_initcall(partial_resume_init); diff --git a/kernel/power/process.c b/kernel/power/process.c index d98fa18542e..6b220f9c3ad 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "power.h" /* @@ -36,6 +37,7 @@ static int try_to_freeze_tasks(bool user_only) unsigned int elapsed_msecs; bool wakeup = false; int sleep_usecs = USEC_PER_MSEC; + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; do_gettimeofday(&start); @@ -76,6 +78,11 @@ static int try_to_freeze_tasks(bool user_only) break; if (pm_wakeup_pending()) { +#ifndef CONFIG_UML + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); +#endif + log_suspend_abort_reason(suspend_abort); wakeup = true; break; } @@ -95,23 +102,24 @@ static int try_to_freeze_tasks(bool user_only) do_div(elapsed_msecs64, NSEC_PER_MSEC); elapsed_msecs = elapsed_msecs64; - if (todo) { + if (wakeup) { printk("\n"); - printk(KERN_ERR "Freezing of tasks %s after %d.%03d seconds " - "(%d tasks refusing to freeze, wq_busy=%d):\n", - wakeup ? "aborted" : "failed", + printk(KERN_ERR "Freezing of tasks aborted after %d.%03d seconds", + elapsed_msecs / 1000, elapsed_msecs % 1000); + } else if (todo) { + printk("\n"); + printk(KERN_ERR "Freezing of tasks failed after %d.%03d seconds" + " (%d tasks refusing to freeze, wq_busy=%d):\n", elapsed_msecs / 1000, elapsed_msecs % 1000, todo - wq_busy, wq_busy); - if (!wakeup) { - read_lock(&tasklist_lock); - do_each_thread(g, p) { - if (p != current && !freezer_should_skip(p) - && freezing(p) && !frozen(p)) - sched_show_task(p); - } while_each_thread(g, p); - read_unlock(&tasklist_lock); - } + read_lock(&tasklist_lock); + do_each_thread(g, p) { + if (p != current && !freezer_should_skip(p) + && freezing(p) && !frozen(p)) + sched_show_task(p); + } while_each_thread(g, p); + read_unlock(&tasklist_lock); } else { printk("(elapsed %d.%03d seconds) ", elapsed_msecs / 1000, elapsed_msecs % 1000); diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 172e4155b7f..eb7696200a7 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -25,7 +25,10 @@ #include #include #include +#include +#include #include +#include #include "power.h" @@ -108,7 +111,7 @@ static int suspend_prepare(void) error = suspend_freeze_processes(); if (!error) return 0; - + log_suspend_abort_reason("One or more tasks refusing to freeze"); suspend_stats.failed_freeze++; dpm_save_failed_step(SUSPEND_FREEZE); Finish: @@ -138,7 +141,8 @@ void __attribute__ ((weak)) arch_suspend_enable_irqs(void) */ static int suspend_enter(suspend_state_t state, bool *wakeup) { - int error; + char suspend_abort[MAX_SUSPEND_ABORT_LEN]; + int error, last_dev; if (suspend_ops->prepare) { error = suspend_ops->prepare(); @@ -148,7 +152,11 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) error = dpm_suspend_end(PMSG_SUSPEND); if (error) { + last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1; + last_dev %= REC_FAILED_NUM; printk(KERN_ERR "PM: Some devices failed to power down\n"); + log_suspend_abort_reason("%s device failed to power down", + suspend_stats.failed_devs[last_dev]); goto Platform_finish; } @@ -162,8 +170,10 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) goto Platform_wake; error = disable_nonboot_cpus(); - if (error || suspend_test(TEST_CPUS)) + if (error || suspend_test(TEST_CPUS)) { + log_suspend_abort_reason("Disabling non-boot cpus failed"); goto Enable_cpus; + } arch_suspend_disable_irqs(); BUG_ON(!irqs_disabled()); @@ -174,7 +184,14 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) if (!(suspend_test(TEST_CORE) || *wakeup)) { error = suspend_ops->enter(state); events_check_enabled = false; + } else if (*wakeup) { + pm_get_active_wakeup_sources(suspend_abort, + MAX_SUSPEND_ABORT_LEN); + log_suspend_abort_reason(suspend_abort); + error = -EBUSY; } + + start_logging_wakeup_reasons(); syscore_resume(); } @@ -197,6 +214,65 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) return error; } +#ifdef CONFIG_PARTIALRESUME +static bool suspend_again(bool *drivers_resumed) +{ + const struct list_head *irqs; + struct list_head unfinished; + + *drivers_resumed = false; + + /* If a platform suspend_again handler is defined, when it decides to + * not suspend again, this takes precedence over drivers. If a + * platform's suspend_again callback returns true, then we proceed to + * check the drivers as well. + */ + if (suspend_ops->suspend_again && !suspend_ops->suspend_again()) + return false; + + /* TODO: resume only the drivers associated with the wakeup interrupts! + */ + dpm_resume_end(PMSG_RESUME); + *drivers_resumed = true; + + /* Thaw kernel threads opportunistically, to allow get_wakeup_reasons + * to block while the wakeup interrupt list is being assembled. Calls + * schedule() internally. + */ + thaw_kernel_threads(); + + /* Look for a match between the wakeup reasons and the registered + * callbacks. Don't bother thawing the kernel threads if a match is + * not found. + */ + irqs = get_wakeup_reasons(msecs_to_jiffies(100), &unfinished); + if (!suspend_again_match(irqs, &unfinished)) + return false; + + if (suspend_again_consensus() && + !freeze_kernel_threads()) { + clear_wakeup_reasons(); + *drivers_resumed = false; + if (dpm_suspend_start(PMSG_SUSPEND)) { + printk(KERN_ERR "PM: Some devices failed to suspend\n"); + log_suspend_abort_reason("Some devices failed to suspend"); + if (suspend_ops->recover) + suspend_ops->recover(); + return false; + } + return true; + } + + return false; +} +#else +static __always_inline bool +suspend_again(bool *drivers_resumed __attribute__((unused))) +{ + return suspend_ops->suspend_again && suspend_ops->suspend_again(); +} +#endif /* CONFIG_PARTIALRESUME */ + /** * suspend_devices_and_enter - Suspend devices and enter system sleep state. * @state: System sleep state to enter. @@ -205,6 +281,7 @@ int suspend_devices_and_enter(suspend_state_t state) { int error; bool wakeup = false; + bool resumed = false; if (!suspend_ops) return -ENOSYS; @@ -220,6 +297,7 @@ int suspend_devices_and_enter(suspend_state_t state) error = dpm_suspend_start(PMSG_SUSPEND); if (error) { printk(KERN_ERR "PM: Some devices failed to suspend\n"); + log_suspend_abort_reason("Some devices failed to suspend"); goto Recover_platform; } suspend_test_finish("suspend devices"); @@ -228,12 +306,12 @@ int suspend_devices_and_enter(suspend_state_t state) do { error = suspend_enter(state, &wakeup); - } while (!error && !wakeup - && suspend_ops->suspend_again && suspend_ops->suspend_again()); + } while (!error && !wakeup && suspend_again(&resumed)); Resume_devices: suspend_test_start(); - dpm_resume_end(PMSG_RESUME); + if (!resumed) + dpm_resume_end(PMSG_RESUME); suspend_test_finish("resume devices"); resume_console(); Close: diff --git a/kernel/power/wakeup_reason.c b/kernel/power/wakeup_reason.c index 1e1277369f1..ecc8632769b 100644 --- a/kernel/power/wakeup_reason.c +++ b/kernel/power/wakeup_reason.c @@ -26,82 +26,598 @@ #include #include #include +#include +static bool suspend_abort; +static char abort_reason[MAX_SUSPEND_ABORT_LEN]; -#define MAX_WAKEUP_REASON_IRQS 32 -static int irq_list[MAX_WAKEUP_REASON_IRQS]; -static int irq_count; +static struct wakeup_irq_node *base_irq_nodes; +static struct wakeup_irq_node *cur_irq_tree; +static int cur_irq_tree_depth; +static LIST_HEAD(wakeup_irqs); + +static struct kmem_cache *wakeup_irq_nodes_cache; static struct kobject *wakeup_reason; static spinlock_t resume_reason_lock; +bool log_wakeups __read_mostly; +struct completion wakeups_completion; + +static unsigned long wakeup_ready_timeout; +static unsigned long wakeup_ready_wait; +static unsigned long wakeup_ready_nowait; + +static struct timespec last_xtime; /* wall time before last suspend */ +static struct timespec curr_xtime; /* wall time after last suspend */ +static struct timespec last_stime; /* sleep time before last suspend */ +static struct timespec curr_stime; /* sleep time after last suspend */ -static ssize_t last_resume_reason_show(struct kobject *kobj, struct kobj_attribute *attr, - char *buf) +static struct timespec total_xtime; /* total suspend time since boot */ +static struct timespec total_stime; /* total sleep time since boot */ +static struct timespec total_atime; /* total suspend abort time since boot */ +static unsigned long suspend_count; /* total amount of resumes */ +static unsigned long abort_count; /* total amount of suspend abort */ + +static void init_wakeup_irq_node(struct wakeup_irq_node *p, int irq) { - int irq_no, buf_offset = 0; - struct irq_desc *desc; - spin_lock(&resume_reason_lock); - for (irq_no = 0; irq_no < irq_count; irq_no++) { - desc = irq_to_desc(irq_list[irq_no]); - if (desc && desc->action && desc->action->name) - buf_offset += sprintf(buf + buf_offset, "%d %s\n", - irq_list[irq_no], desc->action->name); + p->irq = irq; + p->desc = irq_to_desc(irq); + p->child = NULL; + p->parent = NULL; + p->handled = false; + INIT_LIST_HEAD(&p->siblings); + INIT_LIST_HEAD(&p->next); +} + +static struct wakeup_irq_node* alloc_irq_node(int irq) +{ + struct wakeup_irq_node *n; + + n = kmem_cache_alloc(wakeup_irq_nodes_cache, GFP_ATOMIC); + if (!n) { + pr_warning("Failed to log chained wakeup IRQ %d\n", + irq); + return NULL; + } + + init_wakeup_irq_node(n, irq); + return n; +} + +static struct wakeup_irq_node * +search_siblings(struct wakeup_irq_node *root, int irq) +{ + bool found = false; + struct wakeup_irq_node *n = NULL; + BUG_ON(!root); + + if (root->irq == irq) + return root; + + list_for_each_entry(n, &root->siblings, siblings) { + if (n->irq == irq) { + found = true; + break; + } + } + + return found ? n : NULL; +} + +static struct wakeup_irq_node * +add_to_siblings(struct wakeup_irq_node *root, int irq) +{ + struct wakeup_irq_node *n; + if (root) { + n = search_siblings(root, irq); + if (n) + return n; + } + n = alloc_irq_node(irq); + + if (n && root) + list_add(&n->siblings, &root->siblings); + return n; +} + +#ifdef CONFIG_DEDUCE_WAKEUP_REASONS +static struct wakeup_irq_node* add_child(struct wakeup_irq_node *root, int irq) +{ + if (!root->child) { + root->child = alloc_irq_node(irq); + if (!root->child) + return NULL; + root->child->parent = root; + return root->child; + } + + return add_to_siblings(root->child, irq); +} + +static struct wakeup_irq_node *find_first_sibling(struct wakeup_irq_node *node) +{ + struct wakeup_irq_node *n; + if (node->parent) + return node; + list_for_each_entry(n, &node->siblings, siblings) { + if (n->parent) + return n; + } + return NULL; +} + +static struct wakeup_irq_node * +get_base_node(struct wakeup_irq_node *node, unsigned depth) +{ + if (!node) + return NULL; + + while (depth) { + node = find_first_sibling(node); + BUG_ON(!node); + node = node->parent; + depth--; + } + + return node; +} +#endif /* CONFIG_DEDUCE_WAKEUP_REASONS */ + +static const struct list_head* get_wakeup_reasons_nosync(void); + +static void print_wakeup_sources(void) +{ + struct wakeup_irq_node *n; + const struct list_head *wakeups; + + if (suspend_abort) { + pr_info("Abort: %s", abort_reason); + return; + } + + wakeups = get_wakeup_reasons_nosync(); + list_for_each_entry(n, wakeups, next) { + if (n->desc && n->desc->action && n->desc->action->name) + pr_info("Resume caused by IRQ %d, %s\n", n->irq, + n->desc->action->name); else - buf_offset += sprintf(buf + buf_offset, "%d\n", - irq_list[irq_no]); + pr_info("Resume caused by IRQ %d\n", n->irq); } - spin_unlock(&resume_reason_lock); - return buf_offset; +} + +static bool walk_irq_node_tree(struct wakeup_irq_node *root, + bool (*visit)(struct wakeup_irq_node *, void *), + void *cookie) +{ + struct wakeup_irq_node *n, *t; + + if (!root) + return true; + + list_for_each_entry_safe(n, t, &root->siblings, siblings) { + if (!walk_irq_node_tree(n->child, visit, cookie)) + return false; + if (!visit(n, cookie)) + return false; + } + + if (!walk_irq_node_tree(root->child, visit, cookie)) + return false; + return visit(root, cookie); +} + +#ifdef CONFIG_DEDUCE_WAKEUP_REASONS +static bool is_node_handled(struct wakeup_irq_node *n, void *_p) +{ + return n->handled; +} + +static bool base_irq_nodes_done(void) +{ + return walk_irq_node_tree(base_irq_nodes, is_node_handled, NULL); +} +#endif + +struct buf_cookie { + char *buf; + int buf_offset; +}; + +static bool print_leaf_node(struct wakeup_irq_node *n, void *_p) +{ + struct buf_cookie *b = _p; + if (!n->child) { + if (n->desc && n->desc->action && n->desc->action->name) + b->buf_offset += + snprintf(b->buf + b->buf_offset, + PAGE_SIZE - b->buf_offset, + "%d %s\n", + n->irq, n->desc->action->name); + else + b->buf_offset += + snprintf(b->buf + b->buf_offset, + PAGE_SIZE - b->buf_offset, + "%d\n", + n->irq); + } + return true; +} + +static ssize_t last_resume_reason_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + unsigned long flags; + + struct buf_cookie b = { + .buf = buf, + .buf_offset = 0 + }; + + spin_lock_irqsave(&resume_reason_lock, flags); + if (suspend_abort) + b.buf_offset = snprintf(buf, PAGE_SIZE, "Abort: %s", abort_reason); + else + walk_irq_node_tree(base_irq_nodes, print_leaf_node, &b); + spin_unlock_irqrestore(&resume_reason_lock, flags); + + return b.buf_offset; +} + +static ssize_t last_suspend_time_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct timespec sleep_time; + struct timespec total_time; + struct timespec suspend_resume_time; + + sleep_time = timespec_sub(curr_stime, last_stime); + total_time = timespec_sub(curr_xtime, last_xtime); + suspend_resume_time = timespec_sub(total_time, sleep_time); + + /* + * suspend_resume_time is calculated from sleep_time. Userspace would + * always need both. Export them in pair here. + */ + return sprintf(buf, "%lu.%09lu %lu.%09lu\n", + suspend_resume_time.tv_sec, suspend_resume_time.tv_nsec, + sleep_time.tv_sec, sleep_time.tv_nsec); +} + + +static ssize_t suspend_since_boot_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct timespec xtime; + + xtime = timespec_sub(total_xtime, total_stime); + return sprintf(buf, "%lu %lu %lu.%09lu %lu.%09lu %lu.%09lu\n" + "%lu %lu %u\n", + suspend_count, abort_count, + xtime.tv_sec, xtime.tv_nsec, + total_atime.tv_sec, total_atime.tv_nsec, + total_stime.tv_sec, total_stime.tv_nsec, + wakeup_ready_nowait, wakeup_ready_timeout, + jiffies_to_msecs(wakeup_ready_wait)); } static struct kobj_attribute resume_reason = __ATTR_RO(last_resume_reason); +static struct kobj_attribute suspend_time = __ATTR_RO(last_suspend_time); +static struct kobj_attribute suspend_since_boot = __ATTR_RO(suspend_since_boot); static struct attribute *attrs[] = { &resume_reason.attr, + &suspend_time.attr, + &suspend_since_boot.attr, NULL, }; static struct attribute_group attr_group = { .attrs = attrs, }; +static inline void stop_logging_wakeup_reasons(void) +{ + ACCESS_ONCE(log_wakeups) = false; + smp_wmb(); +} + /* - * logs all the wake up reasons to the kernel - * stores the irqs to expose them to the userspace via sysfs + * stores the immediate wakeup irqs; these often aren't the ones seen by + * the drivers that registered them, due to chained interrupt controllers, + * and multiple-interrupt dispatch. */ -void log_wakeup_reason(int irq) +void log_base_wakeup_reason(int irq) { - struct irq_desc *desc; - desc = irq_to_desc(irq); - if (desc && desc->action && desc->action->name) - printk(KERN_INFO "Resume caused by IRQ %d, %s\n", irq, - desc->action->name); - else - printk(KERN_INFO "Resume caused by IRQ %d\n", irq); + /* No locking is needed, since this function is called within + * syscore_resume, with both nonboot CPUs and interrupts disabled. + */ + base_irq_nodes = add_to_siblings(base_irq_nodes, irq); + BUG_ON(!base_irq_nodes); +#ifndef CONFIG_DEDUCE_WAKEUP_REASONS + base_irq_nodes->handled = true; +#endif +} + +#ifdef CONFIG_DEDUCE_WAKEUP_REASONS + +/* This function is called by generic_handle_irq, which may call itself + * recursively. This happens with interrupts disabled. Using + * log_possible_wakeup_reason, we build a tree of interrupts, tracing the call + * stack of generic_handle_irq, for each wakeup source containing the + * interrupts actually handled. + * + * Most of these "trees" would either have a single node (in the event that the + * wakeup source is the final interrupt), or consist of a list of two + * interrupts, with the wakeup source at the root, and the final dispatched + * interrupt at the leaf. + * + * When *all* wakeup sources have been thusly spoken for, this function will + * clear the log_wakeups flag, and print the wakeup reasons. + + TODO: percpu + + */ + +static struct wakeup_irq_node * +log_possible_wakeup_reason_start(int irq, struct irq_desc *desc, unsigned depth) +{ + BUG_ON(!irqs_disabled()); + BUG_ON((signed)depth < 0); + + /* This function can race with a call to stop_logging_wakeup_reasons() + * from a thread context. If this happens, just exit silently, as we are no + * longer interested in logging interrupts. + */ + if (!logging_wakeup_reasons()) + return NULL; + + /* If suspend was aborted, the base IRQ nodes are missing, and we stop + * logging interrupts immediately. + */ + if (!base_irq_nodes) { + stop_logging_wakeup_reasons(); + return NULL; + } + + /* We assume wakeup interrupts are handlerd only by the first core. */ + /* TODO: relax this by having percpu versions of the irq tree */ + if (smp_processor_id() != 0) { + return NULL; + } + + if (depth == 0) { + cur_irq_tree_depth = 0; + cur_irq_tree = search_siblings(base_irq_nodes, irq); + } + else if (cur_irq_tree) { + if (depth > cur_irq_tree_depth) { + BUG_ON(depth - cur_irq_tree_depth > 1); + cur_irq_tree = add_child(cur_irq_tree, irq); + if (cur_irq_tree) + cur_irq_tree_depth++; + } + else { + cur_irq_tree = get_base_node(cur_irq_tree, + cur_irq_tree_depth - depth); + cur_irq_tree_depth = depth; + cur_irq_tree = add_to_siblings(cur_irq_tree, irq); + } + } + + return cur_irq_tree; +} + +static void log_possible_wakeup_reason_complete(struct wakeup_irq_node *n, + unsigned depth, + bool handled) +{ + if (!n) + return; + n->handled = handled; + if (depth == 0) { + if (base_irq_nodes_done()) { + stop_logging_wakeup_reasons(); + complete(&wakeups_completion); + print_wakeup_sources(); + } + } +} + +bool log_possible_wakeup_reason(int irq, + struct irq_desc *desc, + bool (*handler)(unsigned int, struct irq_desc *)) +{ + static DEFINE_PER_CPU(unsigned int, depth); + + struct wakeup_irq_node *n; + bool handled; + unsigned d; + + d = get_cpu_var(depth)++; + put_cpu_var(depth); + + n = log_possible_wakeup_reason_start(irq, desc, d); + + handled = handler(irq, desc); + + d = --get_cpu_var(depth); + put_cpu_var(depth); + + if (!handled && desc && desc->action) + pr_debug("%s: irq %d action %pF not handled\n", __func__, + irq, desc->action->handler); + + log_possible_wakeup_reason_complete(n, d, handled); + + return handled; +} + +#endif /* CONFIG_DEDUCE_WAKEUP_REASONS */ + +void log_suspend_abort_reason(const char *fmt, ...) +{ + va_list args; spin_lock(&resume_reason_lock); - if (irq_count == MAX_WAKEUP_REASON_IRQS) { + + //Suspend abort reason has already been logged. + if (suspend_abort) { spin_unlock(&resume_reason_lock); - printk(KERN_WARNING "Resume caused by more than %d IRQs\n", - MAX_WAKEUP_REASON_IRQS); return; } - irq_list[irq_count++] = irq; + suspend_abort = true; + va_start(args, fmt); + vsnprintf(abort_reason, MAX_SUSPEND_ABORT_LEN, fmt, args); + va_end(args); + + spin_unlock(&resume_reason_lock); +} + +static bool match_node(struct wakeup_irq_node *n, void *_p) +{ + int irq = *((int *)_p); + return n->irq != irq; +} + +int check_wakeup_reason(int irq) +{ + bool found; + spin_lock(&resume_reason_lock); + found = !walk_irq_node_tree(base_irq_nodes, match_node, &irq); spin_unlock(&resume_reason_lock); + return found; +} + +static bool build_leaf_nodes(struct wakeup_irq_node *n, void *_p) +{ + struct list_head *wakeups = _p; + if (!n->child) + list_add(&n->next, wakeups); + return true; +} + +static const struct list_head* get_wakeup_reasons_nosync(void) +{ + BUG_ON(logging_wakeup_reasons()); + INIT_LIST_HEAD(&wakeup_irqs); + walk_irq_node_tree(base_irq_nodes, build_leaf_nodes, &wakeup_irqs); + return &wakeup_irqs; +} + +static bool build_unfinished_nodes(struct wakeup_irq_node *n, void *_p) +{ + struct list_head *unfinished = _p; + if (!n->handled) { + pr_warning("%s: wakeup irq %d was not handled\n", + __func__, n->irq); + list_add(&n->next, unfinished); + } + return true; +} + +const struct list_head* get_wakeup_reasons(unsigned long timeout, + struct list_head *unfinished) +{ + INIT_LIST_HEAD(unfinished); + + if (logging_wakeup_reasons()) { + unsigned long signalled = 0; + unsigned long time_waited; + + if (timeout) + signalled = wait_for_completion_timeout(&wakeups_completion, timeout); + if (!signalled) { + pr_warn("%s: completion timeout\n", __func__); + wakeup_ready_timeout++; + stop_logging_wakeup_reasons(); + walk_irq_node_tree(base_irq_nodes, build_unfinished_nodes, unfinished); + return NULL; + } + time_waited = timeout - signalled; + pr_info("%s: waited for %u ms\n", + __func__, + jiffies_to_msecs(time_waited)); + if (time_waited > wakeup_ready_wait) + wakeup_ready_wait = time_waited; + } else { + wakeup_ready_nowait++; + } + + return get_wakeup_reasons_nosync(); +} + +static bool delete_node(struct wakeup_irq_node *n, void *unused) +{ + list_del(&n->siblings); + kmem_cache_free(wakeup_irq_nodes_cache, n); + return true; +} + +static void clear_wakeup_reasons_nolock(void) +{ + BUG_ON(logging_wakeup_reasons()); + walk_irq_node_tree(base_irq_nodes, delete_node, NULL); + base_irq_nodes = NULL; + cur_irq_tree = NULL; + cur_irq_tree_depth = 0; + INIT_LIST_HEAD(&wakeup_irqs); + suspend_abort = false; +} + +void clear_wakeup_reasons(void) +{ + unsigned long flags; + + spin_lock_irqsave(&resume_reason_lock, flags); + clear_wakeup_reasons_nolock(); + spin_unlock_irqrestore(&resume_reason_lock, flags); } /* Detects a suspend and clears all the previous wake up reasons*/ static int wakeup_reason_pm_event(struct notifier_block *notifier, unsigned long pm_event, void *unused) { + struct timespec xtom; /* wall_to_monotonic, ignored */ + unsigned long flags; + + spin_lock_irqsave(&resume_reason_lock, flags); switch (pm_event) { case PM_SUSPEND_PREPARE: - spin_lock(&resume_reason_lock); - irq_count = 0; - spin_unlock(&resume_reason_lock); + clear_wakeup_reasons_nolock(); + + get_xtime_and_monotonic_and_sleep_offset(&last_xtime, &xtom, + &last_stime); + break; + case PM_POST_SUSPEND: + get_xtime_and_monotonic_and_sleep_offset(&curr_xtime, &xtom, + &curr_stime); + + if (!suspend_abort) { + suspend_count++; + total_xtime = timespec_add(total_xtime, + timespec_sub(curr_xtime, last_xtime)); + total_stime = timespec_add(total_stime, + timespec_sub(curr_stime, last_stime)); + } else { + abort_count++; + total_atime = timespec_add(total_atime, + timespec_sub(curr_xtime, last_xtime)); + } + +#ifdef CONFIG_DEDUCE_WAKEUP_REASONS + /* log_wakeups should have been cleared by now. */ + if (WARN_ON(logging_wakeup_reasons())) { + stop_logging_wakeup_reasons(); + print_wakeup_sources(); + } +#else + print_wakeup_sources(); +#endif break; default: break; } + spin_unlock_irqrestore(&resume_reason_lock, flags); return NOTIFY_DONE; } @@ -109,31 +625,46 @@ static struct notifier_block wakeup_reason_pm_notifier_block = { .notifier_call = wakeup_reason_pm_event, }; -/* Initializes the sysfs parameter - * registers the pm_event notifier - */ int __init wakeup_reason_init(void) { - int retval; spin_lock_init(&resume_reason_lock); - retval = register_pm_notifier(&wakeup_reason_pm_notifier_block); - if (retval) - printk(KERN_WARNING "[%s] failed to register PM notifier %d\n", - __func__, retval); + + if (register_pm_notifier(&wakeup_reason_pm_notifier_block)) { + pr_warning("[%s] failed to register PM notifier\n", + __func__); + goto fail; + } wakeup_reason = kobject_create_and_add("wakeup_reasons", kernel_kobj); if (!wakeup_reason) { - printk(KERN_WARNING "[%s] failed to create a sysfs kobject\n", + pr_warning("[%s] failed to create a sysfs kobject\n", __func__); - return 1; + goto fail_unregister_pm_notifier; } - retval = sysfs_create_group(wakeup_reason, &attr_group); - if (retval) { - kobject_put(wakeup_reason); - printk(KERN_WARNING "[%s] failed to create a sysfs group %d\n", - __func__, retval); + + if (sysfs_create_group(wakeup_reason, &attr_group)) { + pr_warning("[%s] failed to create a sysfs group\n", + __func__); + goto fail_kobject_put; } + + wakeup_irq_nodes_cache = + kmem_cache_create("wakeup_irq_node_cache", + sizeof(struct wakeup_irq_node), 0, + 0, NULL); + if (!wakeup_irq_nodes_cache) + goto fail_remove_group; + return 0; + +fail_remove_group: + sysfs_remove_group(wakeup_reason, &attr_group); +fail_kobject_put: + kobject_put(wakeup_reason); +fail_unregister_pm_notifier: + unregister_pm_notifier(&wakeup_reason_pm_notifier_block); +fail: + return 1; } late_initcall(wakeup_reason_init); diff --git a/kernel/printk.c b/kernel/printk.c index d80ffaf0a80..a3efe8c51ef 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -44,7 +44,10 @@ #include +#ifdef CONFIG_MSM_RTB #include +#endif + #define CREATE_TRACE_POINTS #include diff --git a/kernel/ptrace.c b/kernel/ptrace.c index daf4394d1ab..a6ed0c8926d 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -233,7 +233,7 @@ int __ptrace_may_access(struct task_struct *task, unsigned int mode) */ int dumpable = 0; /* Don't let security modules deny introspection */ - if (task == current) + if (same_thread_group(task, current)) return 0; rcu_read_lock(); tcred = __task_cred(task); diff --git a/kernel/resource.c b/kernel/resource.c index 7203c06273a..e9ba0770ec3 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -142,7 +142,7 @@ static const struct file_operations proc_iomem_operations = { static int __init ioresources_init(void) { proc_create("ioports", 0, NULL, &proc_ioports_operations); - proc_create("iomem", 0, NULL, &proc_iomem_operations); + proc_create("iomem", S_IRUSR, NULL, &proc_iomem_operations); return 0; } __initcall(ioresources_init); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7236a1f48c2..cd495e6b198 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,38 @@ #define CREATE_TRACE_POINTS #include +static atomic_t __su_instances; + +int su_instances(void) +{ + return atomic_read(&__su_instances); +} + +bool su_running(void) +{ + return su_instances() > 0; +} + +bool su_visible(void) +{ + uid_t uid = current_uid(); + if (su_running()) + return true; + if (uid == 0 || uid == 1000) + return true; + return false; +} + +void su_exec(void) +{ + atomic_inc(&__su_instances); +} + +void su_exit(void) +{ + atomic_dec(&__su_instances); +} + ATOMIC_NOTIFIER_HEAD(migration_notifier_head); void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) @@ -1650,9 +1683,11 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) out: raw_spin_unlock_irqrestore(&p->pi_lock, flags); +#ifndef CONFIG_UML if (src_cpu != cpu && task_notify_on_migrate(p)) atomic_notifier_call_chain(&migration_notifier_head, cpu, (void *)src_cpu); +#endif return success; } @@ -2696,6 +2731,9 @@ void account_user_time(struct task_struct *p, cputime_t cputime, /* Account for user time used */ acct_update_integrals(p); + + /* Account power usage for user time */ + acct_update_power(p, cputime); } /* @@ -2746,6 +2784,9 @@ void __account_system_time(struct task_struct *p, cputime_t cputime, /* Account for system time used */ acct_update_integrals(p); + + /* Account power usage for system time */ + acct_update_power(p, cputime); } /* @@ -6933,6 +6974,7 @@ int in_sched_functions(unsigned long addr) #ifdef CONFIG_CGROUP_SCHED struct task_group root_task_group; +LIST_HEAD(task_groups); #endif DECLARE_PER_CPU(cpumask_var_t, load_balance_tmpmask); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 55f6d9cac86..c31106a2e1c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -80,7 +80,7 @@ extern struct mutex sched_domains_mutex; struct cfs_rq; struct rt_rq; -static LIST_HEAD(task_groups); +extern struct list_head task_groups; struct cfs_bandwidth { #ifdef CONFIG_CFS_BANDWIDTH diff --git a/kernel/seccomp.c b/kernel/seccomp.c index e8d76c5895e..829a2cae276 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -3,16 +3,547 @@ * * Copyright 2004-2005 Andrea Arcangeli * - * This defines a simple but solid secure-computing mode. + * Copyright (C) 2012 Google, Inc. + * Will Drewry + * + * This defines a simple but solid secure-computing facility. + * + * Mode 1 uses a fixed list of allowed system calls. + * Mode 2 allows user-defined system call filters in the form + * of Berkeley Packet Filters/Linux Socket Filters. */ +#include #include -#include -#include #include +#include +#include +#include +#include /* #define SECCOMP_DEBUG 1 */ -#define NR_SECCOMP_MODES 1 + +#ifdef CONFIG_SECCOMP_FILTER +#include +#include +#include +#include +#include +#include +#include + +/** + * struct seccomp_filter - container for seccomp BPF programs + * + * @usage: reference count to manage the object lifetime. + * get/put helpers should be used when accessing an instance + * outside of a lifetime-guarded section. In general, this + * is only needed for handling filters shared across tasks. + * @prev: points to a previously installed, or inherited, filter + * @len: the number of instructions in the program + * @insns: the BPF program instructions to evaluate + * + * seccomp_filter objects are organized in a tree linked via the @prev + * pointer. For any task, it appears to be a singly-linked list starting + * with current->seccomp.filter, the most recently attached or inherited filter. + * However, multiple filters may share a @prev node, by way of fork(), which + * results in a unidirectional tree existing in memory. This is similar to + * how namespaces work. + * + * seccomp_filter objects should never be modified after being attached + * to a task_struct (other than @usage). + */ +struct seccomp_filter { + atomic_t usage; + struct seccomp_filter *prev; + unsigned short len; /* Instruction count */ + struct sock_filter insns[]; +}; + +/* Limit any path through the tree to 256KB worth of instructions. */ +#define MAX_INSNS_PER_PATH ((1 << 18) / sizeof(struct sock_filter)) + +/** + * get_u32 - returns a u32 offset into data + * @data: a unsigned 64 bit value + * @index: 0 or 1 to return the first or second 32-bits + * + * This inline exists to hide the length of unsigned long. If a 32-bit + * unsigned long is passed in, it will be extended and the top 32-bits will be + * 0. If it is a 64-bit unsigned long, then whatever data is resident will be + * properly returned. + * + * Endianness is explicitly ignored and left for BPF program authors to manage + * as per the specific architecture. + */ +static inline u32 get_u32(u64 data, int index) +{ + return ((u32 *)&data)[index]; +} + +/* Helper for bpf_load below. */ +#define BPF_DATA(_name) offsetof(struct seccomp_data, _name) +/** + * bpf_load: checks and returns a pointer to the requested offset + * @off: offset into struct seccomp_data to load from + * + * Returns the requested 32-bits of data. + * seccomp_check_filter() should assure that @off is 32-bit aligned + * and not out of bounds. Failure to do so is a BUG. + */ +u32 seccomp_bpf_load(int off) +{ + struct pt_regs *regs = task_pt_regs(current); + if (off == BPF_DATA(nr)) + return syscall_get_nr(current, regs); + if (off == BPF_DATA(arch)) + return syscall_get_arch(current, regs); + if (off >= BPF_DATA(args[0]) && off < BPF_DATA(args[6])) { + unsigned long value; + int arg = (off - BPF_DATA(args[0])) / sizeof(u64); + int index = !!(off % sizeof(u64)); + syscall_get_arguments(current, regs, arg, 1, &value); + return get_u32(value, index); + } + if (off == BPF_DATA(instruction_pointer)) + return get_u32(KSTK_EIP(current), 0); + if (off == BPF_DATA(instruction_pointer) + sizeof(u32)) + return get_u32(KSTK_EIP(current), 1); + /* seccomp_check_filter should make this impossible. */ + BUG(); +} + +/** + * seccomp_check_filter - verify seccomp filter code + * @filter: filter to verify + * @flen: length of filter + * + * Takes a previously checked filter (by sk_chk_filter) and + * redirects all filter code that loads struct sk_buff data + * and related data through seccomp_bpf_load. It also + * enforces length and alignment checking of those loads. + * + * Returns 0 if the rule set is legal or -EINVAL if not. + */ +static int seccomp_check_filter(struct sock_filter *filter, unsigned int flen) +{ + int pc; + for (pc = 0; pc < flen; pc++) { + struct sock_filter *ftest = &filter[pc]; + u16 code = ftest->code; + u32 k = ftest->k; + + switch (code) { + case BPF_S_LD_W_ABS: + ftest->code = BPF_S_ANC_SECCOMP_LD_W; + /* 32-bit aligned and not out of bounds. */ + if (k >= sizeof(struct seccomp_data) || k & 3) + return -EINVAL; + continue; + case BPF_S_LD_W_LEN: + ftest->code = BPF_S_LD_IMM; + ftest->k = sizeof(struct seccomp_data); + continue; + case BPF_S_LDX_W_LEN: + ftest->code = BPF_S_LDX_IMM; + ftest->k = sizeof(struct seccomp_data); + continue; + /* Explicitly include allowed calls. */ + case BPF_S_RET_K: + case BPF_S_RET_A: + case BPF_S_ALU_ADD_K: + case BPF_S_ALU_ADD_X: + case BPF_S_ALU_SUB_K: + case BPF_S_ALU_SUB_X: + case BPF_S_ALU_MUL_K: + case BPF_S_ALU_MUL_X: + case BPF_S_ALU_DIV_X: + case BPF_S_ALU_AND_K: + case BPF_S_ALU_AND_X: + case BPF_S_ALU_OR_K: + case BPF_S_ALU_OR_X: + case BPF_S_ALU_LSH_K: + case BPF_S_ALU_LSH_X: + case BPF_S_ALU_RSH_K: + case BPF_S_ALU_RSH_X: + case BPF_S_ALU_NEG: + case BPF_S_LD_IMM: + case BPF_S_LDX_IMM: + case BPF_S_MISC_TAX: + case BPF_S_MISC_TXA: + case BPF_S_ALU_DIV_K: + case BPF_S_LD_MEM: + case BPF_S_LDX_MEM: + case BPF_S_ST: + case BPF_S_STX: + case BPF_S_JMP_JA: + case BPF_S_JMP_JEQ_K: + case BPF_S_JMP_JEQ_X: + case BPF_S_JMP_JGE_K: + case BPF_S_JMP_JGE_X: + case BPF_S_JMP_JGT_K: + case BPF_S_JMP_JGT_X: + case BPF_S_JMP_JSET_K: + case BPF_S_JMP_JSET_X: + continue; + default: + return -EINVAL; + } + } + return 0; +} + +/** + * seccomp_run_filters - evaluates all seccomp filters against @syscall + * @syscall: number of the current system call + * + * Returns valid seccomp BPF response codes. + */ +static u32 seccomp_run_filters(int syscall) +{ + struct seccomp_filter *f = ACCESS_ONCE(current->seccomp.filter); + u32 ret = SECCOMP_RET_ALLOW; + + /* Ensure unexpected behavior doesn't result in failing open. */ + if (unlikely(WARN_ON(f == NULL))) + return SECCOMP_RET_KILL; + + /* Make sure cross-thread synced filter points somewhere sane. */ + smp_read_barrier_depends(); + + /* + * All filters in the list are evaluated and the lowest BPF return + * value always takes priority (ignoring the DATA). + */ + for (; f; f = f->prev) { + u32 cur_ret = sk_run_filter(NULL, f->insns); + + if ((cur_ret & SECCOMP_RET_ACTION) < (ret & SECCOMP_RET_ACTION)) + ret = cur_ret; + } + return ret; +} +#endif /* CONFIG_SECCOMP_FILTER */ + +static inline bool seccomp_may_assign_mode(unsigned long seccomp_mode) +{ + BUG_ON(!spin_is_locked(¤t->sighand->siglock)); + + if (current->seccomp.mode && current->seccomp.mode != seccomp_mode) + return false; + + return true; +} + +static inline void seccomp_assign_mode(struct task_struct *task, + unsigned long seccomp_mode) +{ + BUG_ON(!spin_is_locked(&task->sighand->siglock)); + + task->seccomp.mode = seccomp_mode; + /* + * Make sure TIF_SECCOMP cannot be set before the mode (and + * filter) is set. + */ + smp_mb(); + set_tsk_thread_flag(task, TIF_SECCOMP); +} + +#ifdef CONFIG_SECCOMP_FILTER +/* Returns 1 if the parent is an ancestor of the child. */ +static int is_ancestor(struct seccomp_filter *parent, + struct seccomp_filter *child) +{ + /* NULL is the root ancestor. */ + if (parent == NULL) + return 1; + for (; child; child = child->prev) + if (child == parent) + return 1; + return 0; +} + +/** + * seccomp_can_sync_threads: checks if all threads can be synchronized + * + * Expects sighand and cred_guard_mutex locks to be held. + * + * Returns 0 on success, -ve on error, or the pid of a thread which was + * either not in the correct seccomp mode or it did not have an ancestral + * seccomp filter. + */ +static inline pid_t seccomp_can_sync_threads(void) +{ + struct task_struct *thread, *caller; + + BUG_ON(!mutex_is_locked(¤t->signal->cred_guard_mutex)); + BUG_ON(!spin_is_locked(¤t->sighand->siglock)); + + /* Validate all threads being eligible for synchronization. */ + caller = current; + for_each_thread(caller, thread) { + pid_t failed; + + /* Skip current, since it is initiating the sync. */ + if (thread == caller) + continue; + + if (thread->seccomp.mode == SECCOMP_MODE_DISABLED || + (thread->seccomp.mode == SECCOMP_MODE_FILTER && + is_ancestor(thread->seccomp.filter, + caller->seccomp.filter))) + continue; + + /* Return the first thread that cannot be synchronized. */ + failed = task_pid_vnr(thread); + /* If the pid cannot be resolved, then return -ESRCH */ + if (unlikely(WARN_ON(failed == 0))) + failed = -ESRCH; + return failed; + } + + return 0; +} + +/** + * seccomp_sync_threads: sets all threads to use current's filter + * + * Expects sighand and cred_guard_mutex locks to be held, and for + * seccomp_can_sync_threads() to have returned success already + * without dropping the locks. + * + */ +static inline void seccomp_sync_threads(void) +{ + struct task_struct *thread, *caller; + + BUG_ON(!mutex_is_locked(¤t->signal->cred_guard_mutex)); + BUG_ON(!spin_is_locked(¤t->sighand->siglock)); + + /* Synchronize all threads. */ + caller = current; + for_each_thread(caller, thread) { + /* Skip current, since it needs no changes. */ + if (thread == caller) + continue; + + /* Get a task reference for the new leaf node. */ + get_seccomp_filter(caller); + /* + * Drop the task reference to the shared ancestor since + * current's path will hold a reference. (This also + * allows a put before the assignment.) + */ + put_seccomp_filter(thread); + smp_mb(); + ACCESS_ONCE(thread->seccomp.filter) = caller->seccomp.filter; + /* + * Opt the other thread into seccomp if needed. + * As threads are considered to be trust-realm + * equivalent (see ptrace_may_access), it is safe to + * allow one thread to transition the other. + */ + if (thread->seccomp.mode == SECCOMP_MODE_DISABLED) { + /* + * Don't let an unprivileged task work around + * the no_new_privs restriction by creating + * a thread that sets it up, enters seccomp, + * then dies. + */ + if (task_no_new_privs(caller)) + task_set_no_new_privs(thread); + + seccomp_assign_mode(thread, SECCOMP_MODE_FILTER); + } + } +} + +/** + * seccomp_prepare_filter: Prepares a seccomp filter for use. + * @fprog: BPF program to install + * + * Returns filter on success or an ERR_PTR on failure. + */ +static struct seccomp_filter *seccomp_prepare_filter(struct sock_fprog *fprog) +{ + struct seccomp_filter *filter; + unsigned long fp_size = fprog->len * sizeof(struct sock_filter); + unsigned long total_insns = fprog->len; + long ret; + + if (fprog->len == 0 || fprog->len > BPF_MAXINSNS) + return ERR_PTR(-EINVAL); + BUG_ON(INT_MAX / fprog->len < sizeof(struct sock_filter)); + + for (filter = current->seccomp.filter; filter; filter = filter->prev) + total_insns += filter->len + 4; /* include a 4 instr penalty */ + if (total_insns > MAX_INSNS_PER_PATH) + return ERR_PTR(-ENOMEM); + + /* + * Installing a seccomp filter requires that the task have + * CAP_SYS_ADMIN in its namespace or be running with no_new_privs. + * This avoids scenarios where unprivileged tasks can affect the + * behavior of privileged children. + */ + if (!task_no_new_privs(current) && + security_capable_noaudit(current_cred(), current_user_ns(), + CAP_SYS_ADMIN) != 0) + return ERR_PTR(-EACCES); + + /* Allocate a new seccomp_filter */ + filter = kzalloc(sizeof(struct seccomp_filter) + fp_size, + GFP_KERNEL|__GFP_NOWARN); + if (!filter) + return ERR_PTR(-ENOMEM); + atomic_set(&filter->usage, 1); + filter->len = fprog->len; + + /* Copy the instructions from fprog. */ + ret = -EFAULT; + if (copy_from_user(filter->insns, fprog->filter, fp_size)) + goto fail; + + /* Check and rewrite the fprog via the skb checker */ + ret = sk_chk_filter(filter->insns, filter->len); + if (ret) + goto fail; + + /* Check and rewrite the fprog for seccomp use */ + ret = seccomp_check_filter(filter->insns, filter->len); + if (ret) + goto fail; + + return filter; +fail: + kfree(filter); + return ERR_PTR(ret); +} + +/** + * seccomp_prepare_user_filter - prepares a user-supplied sock_fprog + * @user_filter: pointer to the user data containing a sock_fprog. + * + * Returns 0 on success and non-zero otherwise. + */ +static struct seccomp_filter * +seccomp_prepare_user_filter(const char __user *user_filter) +{ + struct sock_fprog fprog; + struct seccomp_filter *filter = ERR_PTR(-EFAULT); + +#ifdef CONFIG_COMPAT + if (is_compat_task()) { + struct compat_sock_fprog fprog32; + if (copy_from_user(&fprog32, user_filter, sizeof(fprog32))) + goto out; + fprog.len = fprog32.len; + fprog.filter = compat_ptr(fprog32.filter); + } else /* falls through to the if below. */ +#endif + if (copy_from_user(&fprog, user_filter, sizeof(fprog))) + goto out; + filter = seccomp_prepare_filter(&fprog); +out: + return filter; +} + +/** + * seccomp_attach_filter: validate and attach filter + * @flags: flags to change filter behavior + * @filter: seccomp filter to add to the current process + * + * Caller must be holding current->sighand->siglock lock. + * + * Returns 0 on success, -ve on error. + */ +static long seccomp_attach_filter(unsigned int flags, + struct seccomp_filter *filter) +{ + unsigned long total_insns; + struct seccomp_filter *walker; + + BUG_ON(!spin_is_locked(¤t->sighand->siglock)); + + /* Validate resulting filter length. */ + total_insns = filter->len; + for (walker = current->seccomp.filter; walker; walker = walker->prev) + total_insns += walker->len + 4; /* 4 instr penalty */ + if (total_insns > MAX_INSNS_PER_PATH) + return -ENOMEM; + + /* If thread sync has been requested, check that it is possible. */ + if (flags & SECCOMP_FILTER_FLAG_TSYNC) { + int ret; + + ret = seccomp_can_sync_threads(); + if (ret) + return ret; + } + + /* + * If there is an existing filter, make it the prev and don't drop its + * task reference. + */ + filter->prev = current->seccomp.filter; + current->seccomp.filter = filter; + + /* Now that the new filter is in place, synchronize to all threads. */ + if (flags & SECCOMP_FILTER_FLAG_TSYNC) + seccomp_sync_threads(); + + return 0; +} + +/* get_seccomp_filter - increments the reference count of the filter on @tsk */ +void get_seccomp_filter(struct task_struct *tsk) +{ + struct seccomp_filter *orig = tsk->seccomp.filter; + if (!orig) + return; + /* Reference count is bounded by the number of total processes. */ + atomic_inc(&orig->usage); +} + +static inline void seccomp_filter_free(struct seccomp_filter *filter) +{ + if (filter) { + kfree(filter); + } +} + +/* put_seccomp_filter - decrements the ref count of tsk->seccomp.filter */ +void put_seccomp_filter(struct task_struct *tsk) +{ + struct seccomp_filter *orig = tsk->seccomp.filter; + /* Clean up single-reference branches iteratively. */ + while (orig && atomic_dec_and_test(&orig->usage)) { + struct seccomp_filter *freeme = orig; + orig = orig->prev; + seccomp_filter_free(freeme); + } +} + +/** + * seccomp_send_sigsys - signals the task to allow in-process syscall emulation + * @syscall: syscall number to send to userland + * @reason: filter-supplied reason code to send to userland (via si_errno) + * + * Forces a SIGSYS with a code of SYS_SECCOMP and related sigsys info. + */ +static void seccomp_send_sigsys(int syscall, int reason) +{ + struct siginfo info; + memset(&info, 0, sizeof(info)); + info.si_signo = SIGSYS; + info.si_code = SYS_SECCOMP; + info.si_call_addr = (void __user *)KSTK_EIP(current); + info.si_errno = reason; + info.si_arch = syscall_get_arch(current, task_pt_regs(current)); + info.si_syscall = syscall; + force_sig_info(SIGSYS, &info, current); +} +#endif /* CONFIG_SECCOMP_FILTER */ /* * Secure computing mode 1 allows only read/write/exit/sigreturn. @@ -31,13 +562,20 @@ static int mode1_syscalls_32[] = { }; #endif -void __secure_computing(int this_syscall) +int __secure_computing(int this_syscall) { - int mode = current->seccomp.mode; - int * syscall; + int exit_sig = 0; + int *syscall; + u32 ret; - switch (mode) { - case 1: + /* + * Make sure that any changes to mode from another thread have + * been seen after TIF_SECCOMP was seen. + */ + rmb(); + + switch (current->seccomp.mode) { + case SECCOMP_MODE_STRICT: syscall = mode1_syscalls; #ifdef CONFIG_COMPAT if (is_compat_task()) @@ -45,9 +583,58 @@ void __secure_computing(int this_syscall) #endif do { if (*syscall == this_syscall) - return; + return 0; } while (*++syscall); + exit_sig = SIGKILL; + ret = SECCOMP_RET_KILL; + break; +#ifdef CONFIG_SECCOMP_FILTER + case SECCOMP_MODE_FILTER: { + int data; + ret = seccomp_run_filters(this_syscall); + data = ret & SECCOMP_RET_DATA; + ret &= SECCOMP_RET_ACTION; + switch (ret) { + case SECCOMP_RET_ERRNO: + /* Set the low-order 16-bits as a errno. */ + syscall_set_return_value(current, task_pt_regs(current), + -data, 0); + goto skip; + case SECCOMP_RET_TRAP: + /* Show the handler the original registers. */ + syscall_rollback(current, task_pt_regs(current)); + /* Let the filter pass back 16 bits of data. */ + seccomp_send_sigsys(this_syscall, data); + goto skip; + case SECCOMP_RET_TRACE: + /* Skip these calls if there is no tracer. */ + if (!ptrace_event_enabled(current, PTRACE_EVENT_SECCOMP)) { + /* Make sure userspace sees an ENOSYS. */ + syscall_set_return_value(current, + task_pt_regs(current), -ENOSYS, 0); + goto skip; + } + /* Allow the BPF to provide the event message */ + ptrace_event(PTRACE_EVENT_SECCOMP, data); + /* + * The delivery of a fatal signal during event + * notification may silently skip tracer notification. + * Terminating the task now avoids executing a system + * call that may not be intended. + */ + if (fatal_signal_pending(current)) + break; + return 0; + case SECCOMP_RET_ALLOW: + return 0; + case SECCOMP_RET_KILL: + default: + break; + } + exit_sig = SIGSYS; break; + } +#endif default: BUG(); } @@ -55,8 +642,13 @@ void __secure_computing(int this_syscall) #ifdef SECCOMP_DEBUG dump_stack(); #endif - audit_seccomp(this_syscall); - do_exit(SIGKILL); + audit_seccomp(this_syscall, exit_sig, ret); + do_exit(exit_sig); +#ifdef CONFIG_SECCOMP_FILTER +skip: + audit_seccomp(this_syscall, exit_sig, ret); +#endif + return -1; } long prctl_get_seccomp(void) @@ -64,25 +656,153 @@ long prctl_get_seccomp(void) return current->seccomp.mode; } -long prctl_set_seccomp(unsigned long seccomp_mode) +/** + * seccomp_set_mode_strict: internal function for setting strict seccomp + * + * Once current->seccomp.mode is non-zero, it may not be changed. + * + * Returns 0 on success or -EINVAL on failure. + */ +static long seccomp_set_mode_strict(void) { - long ret; + const unsigned long seccomp_mode = SECCOMP_MODE_STRICT; + long ret = -EINVAL; + + spin_lock_irq(¤t->sighand->siglock); - /* can set it only once to be even more secure */ - ret = -EPERM; - if (unlikely(current->seccomp.mode)) + if (!seccomp_may_assign_mode(seccomp_mode)) goto out; - ret = -EINVAL; - if (seccomp_mode && seccomp_mode <= NR_SECCOMP_MODES) { - current->seccomp.mode = seccomp_mode; - set_thread_flag(TIF_SECCOMP); #ifdef TIF_NOTSC - disable_TSC(); + disable_TSC(); #endif - ret = 0; - } + seccomp_assign_mode(current, seccomp_mode); + ret = 0; + +out: + spin_unlock_irq(¤t->sighand->siglock); + + return ret; +} + +#ifdef CONFIG_SECCOMP_FILTER +/** + * seccomp_set_mode_filter: internal function for setting seccomp filter + * @flags: flags to change filter behavior + * @filter: struct sock_fprog containing filter + * + * This function may be called repeatedly to install additional filters. + * Every filter successfully installed will be evaluated (in reverse order) + * for each system call the task makes. + * + * Once current->seccomp.mode is non-zero, it may not be changed. + * + * Returns 0 on success or -EINVAL on failure. + */ +static long seccomp_set_mode_filter(unsigned int flags, + const char __user *filter) +{ + const unsigned long seccomp_mode = SECCOMP_MODE_FILTER; + struct seccomp_filter *prepared = NULL; + long ret = -EINVAL; + + /* Validate flags. */ + if (flags & ~SECCOMP_FILTER_FLAG_MASK) + return -EINVAL; + + /* Prepare the new filter before holding any locks. */ + prepared = seccomp_prepare_user_filter(filter); + if (IS_ERR(prepared)) + return PTR_ERR(prepared); + + /* + * Make sure we cannot change seccomp or nnp state via TSYNC + * while another thread is in the middle of calling exec. + */ + if (flags & SECCOMP_FILTER_FLAG_TSYNC && + mutex_lock_killable(¤t->signal->cred_guard_mutex)) + goto out_free; + + spin_lock_irq(¤t->sighand->siglock); + + if (!seccomp_may_assign_mode(seccomp_mode)) + goto out; + + ret = seccomp_attach_filter(flags, prepared); + if (ret) + goto out; + /* Do not free the successfully attached filter. */ + prepared = NULL; - out: + seccomp_assign_mode(current, seccomp_mode); +out: + spin_unlock_irq(¤t->sighand->siglock); + if (flags & SECCOMP_FILTER_FLAG_TSYNC) + mutex_unlock(¤t->signal->cred_guard_mutex); +out_free: + seccomp_filter_free(prepared); return ret; } +#else +static inline long seccomp_set_mode_filter(unsigned int flags, + const char __user *filter) +{ + return -EINVAL; +} +#endif + +/* Common entry point for both prctl and syscall. */ +static long do_seccomp(unsigned int op, unsigned int flags, + const char __user *uargs) +{ + switch (op) { + case SECCOMP_SET_MODE_STRICT: + if (flags != 0 || uargs != NULL) + return -EINVAL; + return seccomp_set_mode_strict(); + case SECCOMP_SET_MODE_FILTER: + return seccomp_set_mode_filter(flags, uargs); + default: + return -EINVAL; + } +} + +SYSCALL_DEFINE3(seccomp, unsigned int, op, unsigned int, flags, + const char __user *, uargs) +{ + return do_seccomp(op, flags, uargs); +} + +/** + * prctl_set_seccomp: configures current->seccomp.mode + * @seccomp_mode: requested mode to use + * @filter: optional struct sock_fprog for use with SECCOMP_MODE_FILTER + * + * Returns 0 on success or -EINVAL on failure. + */ +long prctl_set_seccomp(unsigned long seccomp_mode, char __user *filter) +{ + unsigned int op; + char __user *uargs; + + switch (seccomp_mode) { + case SECCOMP_MODE_STRICT: + op = SECCOMP_SET_MODE_STRICT; + /* + * Setting strict mode through prctl always ignored filter, + * so make sure it is always NULL here to pass the internal + * check in do_seccomp(). + */ + uargs = NULL; + break; + case SECCOMP_MODE_FILTER: + op = SECCOMP_SET_MODE_FILTER; + uargs = filter; + break; + default: + return -EINVAL; + } + + /* prctl interface doesn't have flags, so they are always zero. */ + return do_seccomp(op, 0, uargs); +} diff --git a/kernel/signal.c b/kernel/signal.c index cd0e615b558..18476d43b22 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -160,7 +160,7 @@ void recalc_sigpending(void) #define SYNCHRONOUS_MASK \ (sigmask(SIGSEGV) | sigmask(SIGBUS) | sigmask(SIGILL) | \ - sigmask(SIGTRAP) | sigmask(SIGFPE)) + sigmask(SIGTRAP) | sigmask(SIGFPE) | sigmask(SIGSYS)) int next_signal(struct sigpending *pending, sigset_t *mask) { @@ -2708,6 +2708,13 @@ int copy_siginfo_to_user(siginfo_t __user *to, siginfo_t *from) err |= __put_user(from->si_uid, &to->si_uid); err |= __put_user(from->si_ptr, &to->si_ptr); break; +#ifdef __ARCH_SIGSYS + case __SI_SYS: + err |= __put_user(from->si_call_addr, &to->si_call_addr); + err |= __put_user(from->si_syscall, &to->si_syscall); + err |= __put_user(from->si_arch, &to->si_arch); + break; +#endif default: /* this is just in case for now ... */ err |= __put_user(from->si_pid, &to->si_pid); err |= __put_user(from->si_uid, &to->si_uid); @@ -2930,7 +2937,8 @@ SYSCALL_DEFINE3(rt_sigqueueinfo, pid_t, pid, int, sig, /* Not even root can pretend to send signals from the kernel. * Nor can they impersonate a kill()/tgkill(), which adds source info. */ - if (info.si_code >= 0 || info.si_code == SI_TKILL) { + if ((info.si_code >= 0 || info.si_code == SI_TKILL) && + (task_pid_vnr(current) != pid)) { /* We used to allow any < 0 si_code */ WARN_ON_ONCE(info.si_code < 0); return -EPERM; @@ -2950,7 +2958,8 @@ long do_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, siginfo_t *info) /* Not even root can pretend to send signals from the kernel. * Nor can they impersonate a kill()/tgkill(), which adds source info. */ - if (info->si_code >= 0 || info->si_code == SI_TKILL) { + if ((info->si_code >= 0 || info->si_code == SI_TKILL) && + (task_pid_vnr(current) != pid)) { /* We used to allow any < 0 si_code */ WARN_ON_ONCE(info->si_code < 0); return -EPERM; diff --git a/kernel/sys.c b/kernel/sys.c index 2f951580edc..0dfe95883aa 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1909,7 +1909,7 @@ static int prctl_set_vma_anon_name(unsigned long start, unsigned long end, tmp = end; /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ - error = prctl_update_vma_anon_name(vma, &prev, start, end, + error = prctl_update_vma_anon_name(vma, &prev, start, tmp, (const char __user *)arg); if (error) return error; @@ -2054,7 +2054,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = prctl_get_seccomp(); break; case PR_SET_SECCOMP: - error = prctl_set_seccomp(arg2); + error = prctl_set_seccomp(arg2, (char __user *)arg3); break; case PR_GET_TSC: error = GET_TSC_CTL(arg2); @@ -2148,6 +2148,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, put_task_struct(tsk); error = 0; break; + case PR_SET_NO_NEW_PRIVS: + if (arg2 != 1 || arg3 || arg4 || arg5) + return -EINVAL; + + task_set_no_new_privs(current); + break; + case PR_GET_NO_NEW_PRIVS: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return task_no_new_privs(current) ? 1 : 0; default: error = -EINVAL; break; diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 47bfa16430d..026f30a8985 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -203,3 +203,6 @@ cond_syscall(sys_fanotify_mark); cond_syscall(sys_name_to_handle_at); cond_syscall(sys_open_by_handle_at); cond_syscall(compat_sys_open_by_handle_at); + +/* operate on Secure Computing state */ +cond_syscall(sys_seccomp); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b390dadc52c..5f807c7fc17 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1389,6 +1389,28 @@ static struct ctl_table vm_table[] = { .extra1 = &zero, .extra2 = &one, }, +#endif +#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS + { + .procname = "mmap_rnd_bits", + .data = &mmap_rnd_bits, + .maxlen = sizeof(mmap_rnd_bits), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = (void *)&mmap_rnd_bits_min, + .extra2 = (void *)&mmap_rnd_bits_max, + }, +#endif +#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS + { + .procname = "mmap_rnd_compat_bits", + .data = &mmap_rnd_compat_bits, + .maxlen = sizeof(mmap_rnd_compat_bits), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = (void *)&mmap_rnd_compat_bits_min, + .extra2 = (void *)&mmap_rnd_compat_bits_max, + }, #endif { } }; @@ -1545,6 +1567,20 @@ static struct ctl_table fs_table[] = { .proc_handler = &pipe_proc_fn, .extra1 = &pipe_min_size, }, + { + .procname = "pipe-user-pages-hard", + .data = &pipe_user_pages_hard, + .maxlen = sizeof(pipe_user_pages_hard), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "pipe-user-pages-soft", + .data = &pipe_user_pages_soft, + .maxlen = sizeof(pipe_user_pages_soft), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, { } }; diff --git a/kernel/timeconst.bc b/kernel/timeconst.bc new file mode 100644 index 00000000000..511bdf2cafd --- /dev/null +++ b/kernel/timeconst.bc @@ -0,0 +1,108 @@ +scale=0 + +define gcd(a,b) { + auto t; + while (b) { + t = b; + b = a % b; + a = t; + } + return a; +} + +/* Division by reciprocal multiplication. */ +define fmul(b,n,d) { + return (2^b*n+d-1)/d; +} + +/* Adjustment factor when a ceiling value is used. Use as: + (imul * n) + (fmulxx * n + fadjxx) >> xx) */ +define fadj(b,n,d) { + auto v; + d = d/gcd(n,d); + v = 2^b*(d-1)/d; + return v; +} + +/* Compute the appropriate mul/adj values as well as a shift count, + which brings the mul value into the range 2^b-1 <= x < 2^b. Such + a shift value will be correct in the signed integer range and off + by at most one in the upper half of the unsigned range. */ +define fmuls(b,n,d) { + auto s, m; + for (s = 0; 1; s++) { + m = fmul(s,n,d); + if (m >= 2^(b-1)) + return s; + } + return 0; +} + +define timeconst(hz) { + print "/* Automatically generated by kernel/timeconst.bc */\n" + print "/* Time conversion constants for HZ == ", hz, " */\n" + print "\n" + + print "#ifndef KERNEL_TIMECONST_H\n" + print "#define KERNEL_TIMECONST_H\n\n" + + print "#include \n" + print "#include \n\n" + + print "#if HZ != ", hz, "\n" + print "#error \qkernel/timeconst.h has the wrong HZ value!\q\n" + print "#endif\n\n" + + if (hz < 2) { + print "#error Totally bogus HZ value!\n" + } else { + s=fmuls(32,1000,hz) + obase=16 + print "#define HZ_TO_MSEC_MUL32\tU64_C(0x", fmul(s,1000,hz), ")\n" + print "#define HZ_TO_MSEC_ADJ32\tU64_C(0x", fadj(s,1000,hz), ")\n" + obase=10 + print "#define HZ_TO_MSEC_SHR32\t", s, "\n" + + s=fmuls(32,hz,1000) + obase=16 + print "#define MSEC_TO_HZ_MUL32\tU64_C(0x", fmul(s,hz,1000), ")\n" + print "#define MSEC_TO_HZ_ADJ32\tU64_C(0x", fadj(s,hz,1000), ")\n" + obase=10 + print "#define MSEC_TO_HZ_SHR32\t", s, "\n" + + obase=10 + cd=gcd(hz,1000) + print "#define HZ_TO_MSEC_NUM\t\t", 1000/cd, "\n" + print "#define HZ_TO_MSEC_DEN\t\t", hz/cd, "\n" + print "#define MSEC_TO_HZ_NUM\t\t", hz/cd, "\n" + print "#define MSEC_TO_HZ_DEN\t\t", 1000/cd, "\n" + print "\n" + + s=fmuls(32,1000000,hz) + obase=16 + print "#define HZ_TO_USEC_MUL32\tU64_C(0x", fmul(s,1000000,hz), ")\n" + print "#define HZ_TO_USEC_ADJ32\tU64_C(0x", fadj(s,1000000,hz), ")\n" + obase=10 + print "#define HZ_TO_USEC_SHR32\t", s, "\n" + + s=fmuls(32,hz,1000000) + obase=16 + print "#define USEC_TO_HZ_MUL32\tU64_C(0x", fmul(s,hz,1000000), ")\n" + print "#define USEC_TO_HZ_ADJ32\tU64_C(0x", fadj(s,hz,1000000), ")\n" + obase=10 + print "#define USEC_TO_HZ_SHR32\t", s, "\n" + + obase=10 + cd=gcd(hz,1000000) + print "#define HZ_TO_USEC_NUM\t\t", 1000000/cd, "\n" + print "#define HZ_TO_USEC_DEN\t\t", hz/cd, "\n" + print "#define USEC_TO_HZ_NUM\t\t", hz/cd, "\n" + print "#define USEC_TO_HZ_DEN\t\t", 1000000/cd, "\n" + print "\n" + + print "#endif /* KERNEL_TIMECONST_H */\n" + } + halt +} + +timeconst(hz) diff --git a/kernel/timeconst.pl b/kernel/timeconst.pl deleted file mode 100644 index eb51d76e058..00000000000 --- a/kernel/timeconst.pl +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/perl -# ----------------------------------------------------------------------- -# -# Copyright 2007-2008 rPath, Inc. - All Rights Reserved -# -# This file is part of the Linux kernel, and is made available under -# the terms of the GNU General Public License version 2 or (at your -# option) any later version; incorporated herein by reference. -# -# ----------------------------------------------------------------------- -# - -# -# Usage: timeconst.pl HZ > timeconst.h -# - -# Precomputed values for systems without Math::BigInt -# Generated by: -# timeconst.pl --can 24 32 48 64 100 122 128 200 250 256 300 512 1000 1024 1200 -%canned_values = ( - 24 => [ - '0xa6aaaaab','0x2aaaaaa',26, - 125,3, - '0xc49ba5e4','0x1fbe76c8b4',37, - 3,125, - '0xa2c2aaab','0xaaaa',16, - 125000,3, - '0xc9539b89','0x7fffbce4217d',47, - 3,125000, - ], 32 => [ - '0xfa000000','0x6000000',27, - 125,4, - '0x83126e98','0xfdf3b645a',36, - 4,125, - '0xf4240000','0x0',17, - 31250,1, - '0x8637bd06','0x3fff79c842fa',46, - 1,31250, - ], 48 => [ - '0xa6aaaaab','0x6aaaaaa',27, - 125,6, - '0xc49ba5e4','0xfdf3b645a',36, - 6,125, - '0xa2c2aaab','0x15555',17, - 62500,3, - '0xc9539b89','0x3fffbce4217d',46, - 3,62500, - ], 64 => [ - '0xfa000000','0xe000000',28, - 125,8, - '0x83126e98','0x7ef9db22d',35, - 8,125, - '0xf4240000','0x0',18, - 15625,1, - '0x8637bd06','0x1fff79c842fa',45, - 1,15625, - ], 100 => [ - '0xa0000000','0x0',28, - 10,1, - '0xcccccccd','0x733333333',35, - 1,10, - '0x9c400000','0x0',18, - 10000,1, - '0xd1b71759','0x1fff2e48e8a7',45, - 1,10000, - ], 122 => [ - '0x8325c53f','0xfbcda3a',28, - 500,61, - '0xf9db22d1','0x7fbe76c8b',35, - 61,500, - '0x8012e2a0','0x3ef36',18, - 500000,61, - '0xffda4053','0x1ffffbce4217',45, - 61,500000, - ], 128 => [ - '0xfa000000','0x1e000000',29, - 125,16, - '0x83126e98','0x3f7ced916',34, - 16,125, - '0xf4240000','0x40000',19, - 15625,2, - '0x8637bd06','0xfffbce4217d',44, - 2,15625, - ], 200 => [ - '0xa0000000','0x0',29, - 5,1, - '0xcccccccd','0x333333333',34, - 1,5, - '0x9c400000','0x0',19, - 5000,1, - '0xd1b71759','0xfff2e48e8a7',44, - 1,5000, - ], 250 => [ - '0x80000000','0x0',29, - 4,1, - '0x80000000','0x180000000',33, - 1,4, - '0xfa000000','0x0',20, - 4000,1, - '0x83126e98','0x7ff7ced9168',43, - 1,4000, - ], 256 => [ - '0xfa000000','0x3e000000',30, - 125,32, - '0x83126e98','0x1fbe76c8b',33, - 32,125, - '0xf4240000','0xc0000',20, - 15625,4, - '0x8637bd06','0x7ffde7210be',43, - 4,15625, - ], 300 => [ - '0xd5555556','0x2aaaaaaa',30, - 10,3, - '0x9999999a','0x1cccccccc',33, - 3,10, - '0xd0555556','0xaaaaa',20, - 10000,3, - '0x9d495183','0x7ffcb923a29',43, - 3,10000, - ], 512 => [ - '0xfa000000','0x7e000000',31, - 125,64, - '0x83126e98','0xfdf3b645',32, - 64,125, - '0xf4240000','0x1c0000',21, - 15625,8, - '0x8637bd06','0x3ffef39085f',42, - 8,15625, - ], 1000 => [ - '0x80000000','0x0',31, - 1,1, - '0x80000000','0x0',31, - 1,1, - '0xfa000000','0x0',22, - 1000,1, - '0x83126e98','0x1ff7ced9168',41, - 1,1000, - ], 1024 => [ - '0xfa000000','0xfe000000',32, - 125,128, - '0x83126e98','0x7ef9db22',31, - 128,125, - '0xf4240000','0x3c0000',22, - 15625,16, - '0x8637bd06','0x1fff79c842f',41, - 16,15625, - ], 1200 => [ - '0xd5555556','0xd5555555',32, - 5,6, - '0x9999999a','0x66666666',31, - 6,5, - '0xd0555556','0x2aaaaa',22, - 2500,3, - '0x9d495183','0x1ffcb923a29',41, - 3,2500, - ] -); - -$has_bigint = eval 'use Math::BigInt qw(bgcd); 1;'; - -sub bint($) -{ - my($x) = @_; - return Math::BigInt->new($x); -} - -# -# Constants for division by reciprocal multiplication. -# (bits, numerator, denominator) -# -sub fmul($$$) -{ - my ($b,$n,$d) = @_; - - $n = bint($n); - $d = bint($d); - - return scalar (($n << $b)+$d-bint(1))/$d; -} - -sub fadj($$$) -{ - my($b,$n,$d) = @_; - - $n = bint($n); - $d = bint($d); - - $d = $d/bgcd($n, $d); - return scalar (($d-bint(1)) << $b)/$d; -} - -sub fmuls($$$) { - my($b,$n,$d) = @_; - my($s,$m); - my($thres) = bint(1) << ($b-1); - - $n = bint($n); - $d = bint($d); - - for ($s = 0; 1; $s++) { - $m = fmul($s,$n,$d); - return $s if ($m >= $thres); - } - return 0; -} - -# Generate a hex value if the result fits in 64 bits; -# otherwise skip. -sub bignum_hex($) { - my($x) = @_; - my $s = $x->as_hex(); - - return (length($s) > 18) ? undef : $s; -} - -# Provides mul, adj, and shr factors for a specific -# (bit, time, hz) combination -sub muladj($$$) { - my($b, $t, $hz) = @_; - my $s = fmuls($b, $t, $hz); - my $m = fmul($s, $t, $hz); - my $a = fadj($s, $t, $hz); - return (bignum_hex($m), bignum_hex($a), $s); -} - -# Provides numerator, denominator values -sub numden($$) { - my($n, $d) = @_; - my $g = bgcd($n, $d); - return ($n/$g, $d/$g); -} - -# All values for a specific (time, hz) combo -sub conversions($$) { - my ($t, $hz) = @_; - my @val = (); - - # HZ_TO_xx - push(@val, muladj(32, $t, $hz)); - push(@val, numden($t, $hz)); - - # xx_TO_HZ - push(@val, muladj(32, $hz, $t)); - push(@val, numden($hz, $t)); - - return @val; -} - -sub compute_values($) { - my($hz) = @_; - my @val = (); - my $s, $m, $a, $g; - - if (!$has_bigint) { - die "$0: HZ == $hz not canned and ". - "Math::BigInt not available\n"; - } - - # MSEC conversions - push(@val, conversions(1000, $hz)); - - # USEC conversions - push(@val, conversions(1000000, $hz)); - - return @val; -} - -sub outputval($$) -{ - my($name, $val) = @_; - my $csuf; - - if (defined($val)) { - if ($name !~ /SHR/) { - $val = "U64_C($val)"; - } - printf "#define %-23s %s\n", $name.$csuf, $val.$csuf; - } -} - -sub output($@) -{ - my($hz, @val) = @_; - my $pfx, $bit, $suf, $s, $m, $a; - - print "/* Automatically generated by kernel/timeconst.pl */\n"; - print "/* Conversion constants for HZ == $hz */\n"; - print "\n"; - print "#ifndef KERNEL_TIMECONST_H\n"; - print "#define KERNEL_TIMECONST_H\n"; - print "\n"; - - print "#include \n"; - print "#include \n"; - - print "\n"; - print "#if HZ != $hz\n"; - print "#error \"kernel/timeconst.h has the wrong HZ value!\"\n"; - print "#endif\n"; - print "\n"; - - foreach $pfx ('HZ_TO_MSEC','MSEC_TO_HZ', - 'HZ_TO_USEC','USEC_TO_HZ') { - foreach $bit (32) { - foreach $suf ('MUL', 'ADJ', 'SHR') { - outputval("${pfx}_$suf$bit", shift(@val)); - } - } - foreach $suf ('NUM', 'DEN') { - outputval("${pfx}_$suf", shift(@val)); - } - } - - print "\n"; - print "#endif /* KERNEL_TIMECONST_H */\n"; -} - -# Pretty-print Perl values -sub perlvals(@) { - my $v; - my @l = (); - - foreach $v (@_) { - if (!defined($v)) { - push(@l, 'undef'); - } elsif ($v =~ /^0x/) { - push(@l, "\'".$v."\'"); - } else { - push(@l, $v.''); - } - } - return join(',', @l); -} - -($hz) = @ARGV; - -# Use this to generate the %canned_values structure -if ($hz eq '--can') { - shift(@ARGV); - @hzlist = sort {$a <=> $b} (@ARGV); - - print "# Precomputed values for systems without Math::BigInt\n"; - print "# Generated by:\n"; - print "# timeconst.pl --can ", join(' ', @hzlist), "\n"; - print "\%canned_values = (\n"; - my $pf = "\t"; - foreach $hz (@hzlist) { - my @values = compute_values($hz); - print "$pf$hz => [\n"; - while (scalar(@values)) { - my $bit; - foreach $bit (32) { - my $m = shift(@values); - my $a = shift(@values); - my $s = shift(@values); - print "\t\t", perlvals($m,$a,$s), ",\n"; - } - my $n = shift(@values); - my $d = shift(@values); - print "\t\t", perlvals($n,$d), ",\n"; - } - print "\t]"; - $pf = ', '; - } - print "\n);\n"; -} else { - $hz += 0; # Force to number - if ($hz < 1) { - die "Usage: $0 HZ\n"; - } - - @val = @{$canned_values{$hz}}; - if (!defined(@val)) { - @val = compute_values($hz); - } - output($hz, @val); -} -exit 0; diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index cf8d11e91ef..3924b130985 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1300,13 +1300,14 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) if (!buffer) return size; - size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); - size *= BUF_PAGE_SIZE; + nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); buffer_size = buffer->pages * BUF_PAGE_SIZE; /* we need a minimum of two pages */ - if (size < BUF_PAGE_SIZE * 2) - size = BUF_PAGE_SIZE * 2; + if (nr_pages < 2) + nr_pages = 2; + + size = nr_pages * BUF_PAGE_SIZE; if (size == buffer_size) return size; @@ -1319,8 +1320,6 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) mutex_lock(&buffer->mutex); get_online_cpus(); - nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); - if (size < buffer_size) { /* easy case, just free pages */ diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index fee3752ae8f..a2db136faa5 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -222,7 +222,10 @@ int perf_trace_add(struct perf_event *p_event, int flags) void perf_trace_del(struct perf_event *p_event, int flags) { struct ftrace_event_call *tp_event = p_event->tp_event; - hlist_del_rcu(&p_event->hlist_entry); + if(!hlist_unhashed(&p_event->hlist_entry)) + hlist_del_rcu(&p_event->hlist_entry); + else + return; tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event); } diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 96fc7336909..277ee7f8407 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -307,7 +307,7 @@ void ftrace_syscall_enter(void *ignore, struct pt_regs *regs, long id) int syscall_nr; syscall_nr = syscall_get_nr(current, regs); - if (syscall_nr < 0) + if (syscall_nr < 0 || syscall_nr >= NR_syscalls) return; if (!test_bit(syscall_nr, enabled_enter_syscalls)) return; @@ -341,7 +341,7 @@ void ftrace_syscall_exit(void *ignore, struct pt_regs *regs, long ret) int syscall_nr; syscall_nr = syscall_get_nr(current, regs); - if (syscall_nr < 0) + if (syscall_nr < 0 || syscall_nr >= NR_syscalls) return; if (!test_bit(syscall_nr, enabled_exit_syscalls)) return; @@ -506,6 +506,8 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) int size; syscall_nr = syscall_get_nr(current, regs); + if (syscall_nr < 0 || syscall_nr >= NR_syscalls) + return; if (!test_bit(syscall_nr, enabled_perf_enter_syscalls)) return; @@ -580,6 +582,8 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) int size; syscall_nr = syscall_get_nr(current, regs); + if (syscall_nr < 0 || syscall_nr >= NR_syscalls) + return; if (!test_bit(syscall_nr, enabled_perf_exit_syscalls)) return; diff --git a/kernel/user.c b/kernel/user.c index 71dd2363ab0..9013a4fe0b1 100644 --- a/kernel/user.c +++ b/kernel/user.c @@ -16,6 +16,7 @@ #include #include #include +#include /* * userns count is 1 for root user, 1 for init_uts_ns, @@ -26,6 +27,7 @@ struct user_namespace init_user_ns = { .refcount = ATOMIC_INIT(3), }, .creator = &root_user, + .proc_inum = PROC_USER_INIT_INO, }; EXPORT_SYMBOL_GPL(init_user_ns); diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 3b906e98b1d..c14b7b9fe41 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -27,11 +27,18 @@ int create_user_ns(struct cred *new) struct user_namespace *ns; struct user_struct *root_user; int n; + int ret; ns = kmem_cache_alloc(user_ns_cachep, GFP_KERNEL); if (!ns) return -ENOMEM; + ret = proc_alloc_inum(&ns->proc_inum); + if (ret) { + kmem_cache_free(user_ns_cachep, ns); + return ret; + } + kref_init(&ns->kref); for (n = 0; n < UIDHASH_SZ; ++n) @@ -73,6 +80,7 @@ static void free_user_ns_work(struct work_struct *work) struct user_namespace *ns = container_of(work, struct user_namespace, destroyer); free_uid(ns->creator); + proc_free_inum(ns->proc_inum); kmem_cache_free(user_ns_cachep, ns); } diff --git a/kernel/utsname.c b/kernel/utsname.c index 405caf91aad..ce3d44b4187 100644 --- a/kernel/utsname.c +++ b/kernel/utsname.c @@ -36,11 +36,18 @@ static struct uts_namespace *clone_uts_ns(struct task_struct *tsk, struct uts_namespace *old_ns) { struct uts_namespace *ns; + int err; ns = create_uts_ns(); if (!ns) return ERR_PTR(-ENOMEM); + err = proc_alloc_inum(&ns->proc_inum); + if (err) { + kfree(ns); + return ERR_PTR(err); + } + down_read(&uts_sem); memcpy(&ns->name, &old_ns->name, sizeof(ns->name)); ns->user_ns = get_user_ns(task_cred_xxx(tsk, user)->user_ns); @@ -78,6 +85,7 @@ void free_uts_ns(struct kref *kref) ns = container_of(kref, struct uts_namespace, kref); put_user_ns(ns->user_ns); + proc_free_inum(ns->proc_inum); kfree(ns); } @@ -110,11 +118,18 @@ static int utsns_install(struct nsproxy *nsproxy, void *ns) return 0; } +static unsigned int utsns_inum(void *vp) +{ + struct uts_namespace *ns = vp; + + return ns->proc_inum; +} + const struct proc_ns_operations utsns_operations = { .name = "uts", .type = CLONE_NEWUTS, .get = utsns_get, .put = utsns_put, .install = utsns_install, + .inum = utsns_inum, }; - diff --git a/lib/idr.c b/lib/idr.c index 4046e29c0a9..e90d2d05d78 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -625,7 +625,14 @@ void *idr_get_next(struct idr *idp, int *nextidp) return p; } - id += 1 << n; + /* + * Proceed to the next layer at the current level. Unlike + * idr_for_each(), @id isn't guaranteed to be aligned to + * layer boundary at this point and adding 1 << n may + * incorrectly skip IDs. Make sure we jump to the + * beginning of the next layer using round_up(). + */ + id = round_up(id + 1, 1 << n); while (n < fls(id)) { n += IDR_BITS; p = *--paa; diff --git a/lib/memory_alloc.c b/lib/memory_alloc.c index 03f19448984..7f9ebe0f2d6 100644 --- a/lib/memory_alloc.c +++ b/lib/memory_alloc.c @@ -186,10 +186,12 @@ static void *__alloc(struct mem_pool *mpool, unsigned long size, if (!node) goto out; +#ifndef CONFIG_UML if (cached) vaddr = ioremap_cached(paddr, aligned_size); else vaddr = ioremap(paddr, aligned_size); +#endif if (!vaddr) goto out_kfree; @@ -211,8 +213,10 @@ static void *__alloc(struct mem_pool *mpool, unsigned long size, return vaddr; out_kfree: +#ifndef CONFIG_UML if (vaddr) iounmap(vaddr); +#endif kfree(node); out: gen_pool_free(mpool->gpool, paddr, aligned_size); @@ -226,6 +230,7 @@ static void __free(void *vaddr, bool unmap) if (!node) return; +#ifndef CONFIG_UML if (unmap) /* * We need the double cast because otherwise gcc complains about @@ -234,6 +239,8 @@ static void __free(void *vaddr, bool unmap) * actual 32-bit pointer anyway. */ iounmap((void *)(unsigned long)node->vaddr); +#endif + gen_pool_free(node->mpool->gpool, node->paddr, node->len); node->mpool->free += node->len; diff --git a/mm/filemap.c b/mm/filemap.c index 8ed5c5c0be5..5d601cb852b 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2536,7 +2536,9 @@ ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, if (err) goto out; - file_update_time(file); + err = file_update_time(file); + if (err) + goto out; /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ if (unlikely(file->f_flags & O_DIRECT)) { diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index a4eb3113222..213ca1f5340 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c @@ -426,7 +426,9 @@ xip_file_write(struct file *filp, const char __user *buf, size_t len, if (ret) goto out_backing; - file_update_time(filp); + ret = file_update_time(filp); + if (ret) + goto out_backing; ret = __xip_file_write (filp, buf, count, pos, ppos); diff --git a/mm/memory.c b/mm/memory.c index 55a21eb8478..0b6406e7189 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1445,6 +1445,24 @@ int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, } EXPORT_SYMBOL_GPL(zap_vma_ptes); +static inline bool can_follow_write_pte(pte_t pte, struct page *page, + unsigned int flags) +{ + if (pte_write(pte)) + return true; + + /* + * Make sure that we are really following CoWed page. We do not really + * have to care about exclusiveness of the page because we only want + * to ensure that once COWed page hasn't disappeared in the meantime + * or it hasn't been merged to a KSM page. + */ + if ((flags & FOLL_FORCE) && (flags & FOLL_COW)) + return page && PageAnon(page) && !PageKsm(page); + + return false; +} + /** * follow_page - look up a page descriptor from a user-virtual address * @vma: vm_area_struct mapping @address @@ -1527,10 +1545,13 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, pte = *ptep; if (!pte_present(pte)) goto no_page; - if ((flags & FOLL_WRITE) && !pte_write(pte)) - goto unlock; page = vm_normal_page(vma, address, pte); + if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, page, flags)) { + pte_unmap_unlock(ptep, ptl); + return NULL; + } + if (unlikely(!page)) { if ((flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(pte))) @@ -1573,7 +1594,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, unlock_page(page); } } -unlock: + pte_unmap_unlock(ptep, ptl); out: return page; @@ -1820,17 +1841,13 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, * The VM_FAULT_WRITE bit tells us that * do_wp_page has broken COW when necessary, * even if maybe_mkwrite decided not to set - * pte_write. We can thus safely do subsequent - * page lookups as if they were reads. But only - * do so when looping for pte_write is futile: - * in some cases userspace may also be wanting - * to write to the gotten user page, which a - * read fault here might prevent (a readonly - * page might get reCOWed by userspace write). + * pte_write. We cannot simply drop FOLL_WRITE + * here because the COWed page might be gone by + * the time we do the subsequent page lookups. */ if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) - foll_flags &= ~FOLL_WRITE; + foll_flags |= FOLL_COW; cond_resched(); } @@ -3135,6 +3152,10 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, pte_unmap(page_table); + /* File mapping without ->vm_ops ? */ + if (vma->vm_flags & VM_SHARED) + return VM_FAULT_SIGBUS; + /* Check if we need to add a guard page to the stack */ if (check_stack_guard_page(vma, address) < 0) return VM_FAULT_SIGBUS; @@ -3394,6 +3415,11 @@ static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; pte_unmap(page_table); + + /* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */ + if (!vma->vm_ops->fault) + return VM_FAULT_SIGBUS; + return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); } @@ -3452,13 +3478,12 @@ int handle_pte_fault(struct mm_struct *mm, entry = *pte; if (!pte_present(entry)) { if (pte_none(entry)) { - if (vma->vm_ops) { - if (likely(vma->vm_ops->fault)) - return do_linear_fault(mm, vma, address, + if (vma->vm_ops) + return do_linear_fault(mm, vma, address, pte, pmd, flags, entry); - } + return do_anonymous_page(mm, vma, address, - pte, pmd, flags); + pte, pmd, flags); } if (pte_file(entry)) return do_nonlinear_fault(mm, vma, address, diff --git a/mm/mlock.c b/mm/mlock.c index 029f234eb6d..f38698cd7c9 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -23,10 +23,10 @@ int can_do_mlock(void) { - if (capable(CAP_IPC_LOCK)) - return 1; if (rlimit(RLIMIT_MEMLOCK) != 0) return 1; + if (capable(CAP_IPC_LOCK)) + return 1; return 0; } EXPORT_SYMBOL(can_do_mlock); diff --git a/mm/mmap.c b/mm/mmap.c index ded6fe9f144..e495a844156 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -46,6 +46,18 @@ #define arch_rebalance_pgtables(addr, len) (addr) #endif +#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS +const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN; +const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX; +int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS; +#endif +#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS +const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN; +const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX; +int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; +#endif + + static void unmap_region(struct mm_struct *mm, struct vm_area_struct *vma, struct vm_area_struct *prev, unsigned long start, unsigned long end); diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 46bf2ed5594..b9615b0dcca 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -389,7 +389,7 @@ static struct task_struct *select_bad_process(unsigned int *ppoints, * * Call with tasklist_lock read-locked. */ -static void dump_tasks(const struct mem_cgroup *memcg, const nodemask_t *nodemask) +void dump_tasks(const struct mem_cgroup *memcg, const nodemask_t *nodemask) { struct task_struct *p; struct task_struct *task; diff --git a/mm/util.c b/mm/util.c index ae962b31de8..6d4bc4e253d 100644 --- a/mm/util.c +++ b/mm/util.c @@ -4,6 +4,10 @@ #include #include #include +#include +#include +#include +#include #include #include "internal.h" @@ -341,6 +345,15 @@ int __attribute__((weak)) get_user_pages_fast(unsigned long start, } EXPORT_SYMBOL_GPL(get_user_pages_fast); +void kvfree(const void *addr) +{ + if (is_vmalloc_addr(addr)) + vfree(addr); + else + kfree(addr); +} +EXPORT_SYMBOL(kvfree); + /* Tracepoints definitions. */ EXPORT_TRACEPOINT_SYMBOL(kmalloc); EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c index 9d9a6a3edbd..6ab35a29ed2 100644 --- a/net/ax25/af_ax25.c +++ b/net/ax25/af_ax25.c @@ -811,6 +811,9 @@ static int ax25_create(struct net *net, struct socket *sock, int protocol, struct sock *sk; ax25_cb *ax25; + if (protocol < 0 || protocol > SK_PROTOCOL_MAX) + return -EINVAL; + if (!net_eq(net, &init_net)) return -EAFNOSUPPORT; diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index 216068f4af1..ef5e3a2bd4c 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -346,15 +346,19 @@ static int rfcomm_sock_create(struct net *net, struct socket *sock, static int rfcomm_sock_bind(struct socket *sock, struct sockaddr *addr, int addr_len) { - struct sockaddr_rc *sa = (struct sockaddr_rc *) addr; + struct sockaddr_rc sa; struct sock *sk = sock->sk; - int err = 0; - - BT_DBG("sk %p %s", sk, batostr(&sa->rc_bdaddr)); + int len, err = 0; if (!addr || addr->sa_family != AF_BLUETOOTH) return -EINVAL; + memset(&sa, 0, sizeof(sa)); + len = min_t(unsigned int, sizeof(sa), addr_len); + memcpy(&sa, addr, len); + + BT_DBG("sk %p %s", sk, batostr(&sa.rc_bdaddr)); + lock_sock(sk); if (sk->sk_state != BT_OPEN) { @@ -369,12 +373,12 @@ static int rfcomm_sock_bind(struct socket *sock, struct sockaddr *addr, int addr write_lock_bh(&rfcomm_sk_list.lock); - if (sa->rc_channel && __rfcomm_get_sock_by_addr(sa->rc_channel, &sa->rc_bdaddr)) { + if (sa.rc_channel && __rfcomm_get_sock_by_addr(sa.rc_channel, &sa.rc_bdaddr)) { err = -EADDRINUSE; } else { /* Save source address */ - bacpy(&bt_sk(sk)->src, &sa->rc_bdaddr); - rfcomm_pi(sk)->channel = sa->rc_channel; + bacpy(&bt_sk(sk)->src, &sa.rc_bdaddr); + rfcomm_pi(sk)->channel = sa.rc_channel; sk->sk_state = BT_BOUND; } diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 3170190f83c..d214aa4a876 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -499,6 +499,9 @@ static int sco_sock_bind(struct socket *sock, struct sockaddr *addr, int alen) if (!addr || addr->sa_family != AF_BLUETOOTH) return -EINVAL; + if (alen < sizeof(struct sockaddr_sco)) + return -EINVAL; + memset(&sa, 0, sizeof(sa)); len = min_t(unsigned int, sizeof(sa), alen); memcpy(&sa, addr, len); diff --git a/net/compat.c b/net/compat.c index e055708b8ec..242c828810f 100644 --- a/net/compat.c +++ b/net/compat.c @@ -328,14 +328,6 @@ void scm_detach_fds_compat(struct msghdr *kmsg, struct scm_cookie *scm) __scm_destroy(scm); } -/* - * A struct sock_filter is architecture independent. - */ -struct compat_sock_fprog { - u16 len; - compat_uptr_t filter; /* struct sock_filter * */ -}; - static int do_set_attach_filter(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen) { diff --git a/net/core/datagram.c b/net/core/datagram.c index e4fbfd6e2bd..f98b38f8cbe 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -677,6 +677,7 @@ EXPORT_SYMBOL(__skb_checksum_complete); * @skb: skbuff * @hlen: hardware length * @iov: io vector + * @len: amount of data to copy from skb to iov * * Caller _must_ check that skb will fit to this iovec. * @@ -686,11 +687,14 @@ EXPORT_SYMBOL(__skb_checksum_complete); * can be modified! */ int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, - int hlen, struct iovec *iov) + int hlen, struct iovec *iov, int len) { __wsum csum; int chunk = skb->len - hlen; + if (chunk > len) + chunk = len; + if (!chunk) return 0; diff --git a/net/core/ethtool.c b/net/core/ethtool.c index 6d6d7d25caa..452fa869026 100644 --- a/net/core/ethtool.c +++ b/net/core/ethtool.c @@ -691,11 +691,13 @@ static int ethtool_reset(struct net_device *dev, char __user *useraddr) static int ethtool_get_wol(struct net_device *dev, char __user *useraddr) { - struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; + struct ethtool_wolinfo wol; if (!dev->ethtool_ops->get_wol) return -EOPNOTSUPP; + memset(&wol, 0, sizeof(struct ethtool_wolinfo)); + wol.cmd = ETHTOOL_GWOL; dev->ethtool_ops->get_wol(dev, &wol); if (copy_to_user(useraddr, &wol, sizeof(wol))) diff --git a/net/core/filter.c b/net/core/filter.c index 6f755cca452..21237cec2df 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -38,6 +38,7 @@ #include #include #include +#include /* No hurry in this branch * @@ -322,6 +323,8 @@ unsigned int sk_run_filter(const struct sk_buff *skb, if (skb_is_nonlinear(skb)) return 0; + if (skb->len < sizeof(struct nlattr)) + return 0; if (A > skb->len - sizeof(struct nlattr)) return 0; @@ -338,11 +341,13 @@ unsigned int sk_run_filter(const struct sk_buff *skb, if (skb_is_nonlinear(skb)) return 0; + if (skb->len < sizeof(struct nlattr)) + return 0; if (A > skb->len - sizeof(struct nlattr)) return 0; nla = (struct nlattr *)&skb->data[A]; - if (nla->nla_len > A - skb->len) + if (nla->nla_len > skb->len - A) return 0; nla = nla_find_nested(nla, X); @@ -352,6 +357,11 @@ unsigned int sk_run_filter(const struct sk_buff *skb, A = 0; continue; } +#ifdef CONFIG_SECCOMP_FILTER + case BPF_S_ANC_SECCOMP_LD_W: + A = seccomp_bpf_load(fentry->k); + continue; +#endif default: WARN_RATELIMIT(1, "Unknown code:%u jt:%u tf:%u k:%u\n", fentry->code, fentry->jt, diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 0a68045782d..603b4036ff0 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -935,6 +935,7 @@ static void neigh_timer_handler(unsigned long arg) neigh->nud_state = NUD_PROBE; neigh->updated = jiffies; atomic_set(&neigh->probes, 0); + notify = 1; next = now + neigh->parms->retrans_time; } } else { @@ -1162,6 +1163,8 @@ int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new, if (new != old) { neigh_del_timer(neigh); + if (new & NUD_PROBE) + atomic_set(&neigh->probes, 0); if (new & NUD_IN_TIMER) neigh_add_timer(neigh, (jiffies + ((new & NUD_REACHABLE) ? diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 31a5ae51a45..335ab8875bf 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -376,6 +376,21 @@ struct net *get_net_ns_by_pid(pid_t pid) } EXPORT_SYMBOL_GPL(get_net_ns_by_pid); +static __net_init int net_ns_net_init(struct net *net) +{ + return proc_alloc_inum(&net->proc_inum); +} + +static __net_exit void net_ns_net_exit(struct net *net) +{ + proc_free_inum(net->proc_inum); +} + +static struct pernet_operations __net_initdata net_ns_ops = { + .init = net_ns_net_init, + .exit = net_ns_net_exit, +}; + static int __init net_ns_init(void) { struct net_generic *ng; @@ -407,6 +422,8 @@ static int __init net_ns_init(void) mutex_unlock(&net_mutex); + register_pernet_subsys(&net_ns_ops); + return 0; } @@ -630,11 +647,18 @@ static int netns_install(struct nsproxy *nsproxy, void *ns) return 0; } +static unsigned int netns_inum(void *ns) +{ + struct net *net = ns; + return net->proc_inum; +} + const struct proc_ns_operations netns_operations = { .name = "net", .type = CLONE_NEWNET, .get = netns_get, .put = netns_put, .install = netns_install, + .inum = netns_inum, }; #endif diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index c897d512f2c..b0640fdbf0b 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -912,14 +912,14 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev, NLA_PUT_STRING(skb, IFLA_IFALIAS, dev->ifalias); if (1) { - struct rtnl_link_ifmap map = { - .mem_start = dev->mem_start, - .mem_end = dev->mem_end, - .base_addr = dev->base_addr, - .irq = dev->irq, - .dma = dev->dma, - .port = dev->if_port, - }; + struct rtnl_link_ifmap map; + memset(&map, 0, sizeof(map)); + map.mem_start = dev->mem_start; + map.mem_end = dev->mem_end; + map.base_addr = dev->base_addr; + map.irq = dev->irq; + map.dma = dev->dma; + map.port = dev->if_port; NLA_PUT(skb, IFLA_MAP, sizeof(map), &map); } diff --git a/net/core/scm.c b/net/core/scm.c index 611c5efd4cb..70faa4b9dd1 100644 --- a/net/core/scm.c +++ b/net/core/scm.c @@ -79,6 +79,7 @@ static int scm_fp_copy(struct cmsghdr *cmsg, struct scm_fp_list **fplp) *fplp = fpl; fpl->count = 0; fpl->max = SCM_MAX_FD; + fpl->user = NULL; } fpp = &fpl->fp[fpl->count]; @@ -99,6 +100,10 @@ static int scm_fp_copy(struct cmsghdr *cmsg, struct scm_fp_list **fplp) *fpp++ = file; fpl->count++; } + + if (!fpl->user) + fpl->user = get_uid(current_user()); + return num; } @@ -123,6 +128,7 @@ void __scm_destroy(struct scm_cookie *scm) list_del(&fpl->list); for (i=fpl->count-1; i>=0; i--) fput(fpl->fp[i]); + free_uid(fpl->user); kfree(fpl); } @@ -339,6 +345,7 @@ struct scm_fp_list *scm_fp_dup(struct scm_fp_list *fpl) for (i = 0; i < fpl->count; i++) get_file(fpl->fp[i]); new_fpl->max = new_fpl->count; + new_fpl->user = get_uid(fpl->user); } return new_fpl; } diff --git a/net/core/sock.c b/net/core/sock.c index b2e14c07d92..56c4570a93e 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -577,23 +577,15 @@ int sock_setsockopt(struct socket *sock, int level, int optname, break; case SO_SNDBUF: /* Don't error on this BSD doesn't and if you think - about it this is right. Otherwise apps have to - play 'guess the biggest size' games. RCVBUF/SNDBUF - are treated in BSD as hints */ - - if (val > sysctl_wmem_max) - val = sysctl_wmem_max; + * about it this is right. Otherwise apps have to + * play 'guess the biggest size' games. RCVBUF/SNDBUF + * are treated in BSD as hints + */ + val = min_t(u32, val, sysctl_wmem_max); set_sndbuf: sk->sk_userlocks |= SOCK_SNDBUF_LOCK; - if ((val * 2) < SOCK_MIN_SNDBUF) - sk->sk_sndbuf = SOCK_MIN_SNDBUF; - else - sk->sk_sndbuf = val * 2; - - /* - * Wake up sending tasks if we - * upped the value. - */ + sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); + /* Wake up sending tasks if we upped the value. */ sk->sk_write_space(sk); break; @@ -606,12 +598,11 @@ int sock_setsockopt(struct socket *sock, int level, int optname, case SO_RCVBUF: /* Don't error on this BSD doesn't and if you think - about it this is right. Otherwise apps have to - play 'guess the biggest size' games. RCVBUF/SNDBUF - are treated in BSD as hints */ - - if (val > sysctl_rmem_max) - val = sysctl_rmem_max; + * about it this is right. Otherwise apps have to + * play 'guess the biggest size' games. RCVBUF/SNDBUF + * are treated in BSD as hints + */ + val = min_t(u32, val, sysctl_rmem_max); set_rcvbuf: sk->sk_userlocks |= SOCK_RCVBUF_LOCK; /* @@ -629,10 +620,7 @@ int sock_setsockopt(struct socket *sock, int level, int optname, * returning the value we actually used in getsockopt * is the most desirable behavior. */ - if ((val * 2) < SOCK_MIN_RCVBUF) - sk->sk_rcvbuf = SOCK_MIN_RCVBUF; - else - sk->sk_rcvbuf = val * 2; + sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); break; case SO_RCVBUFFORCE: @@ -644,7 +632,8 @@ int sock_setsockopt(struct socket *sock, int level, int optname, case SO_KEEPALIVE: #ifdef CONFIG_INET - if (sk->sk_protocol == IPPROTO_TCP) + if (sk->sk_protocol == IPPROTO_TCP && + sk->sk_type == SOCK_STREAM) tcp_set_keepalive(sk, valbool); #endif sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); @@ -975,7 +964,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, break; case SO_PASSCRED: - v.val = test_bit(SOCK_PASSCRED, &sock->flags) ? 1 : 0; + v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); break; case SO_PEERCRED: @@ -1010,7 +999,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, break; case SO_PASSSEC: - v.val = test_bit(SOCK_PASSSEC, &sock->flags) ? 1 : 0; + v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); break; case SO_PEERSEC: diff --git a/net/core/sock_diag.c b/net/core/sock_diag.c index aa74be442df..21f8042e72f 100644 --- a/net/core/sock_diag.c +++ b/net/core/sock_diag.c @@ -117,7 +117,7 @@ static inline void sock_diag_unlock_handler(struct sock_diag_handler *h) mutex_unlock(&sock_diag_table_mutex); } -static int __sock_diag_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh) +static int __sock_diag_cmd(struct sk_buff *skb, struct nlmsghdr *nlh) { int err; struct sock_diag_req *req = NLMSG_DATA(nlh); @@ -132,8 +132,12 @@ static int __sock_diag_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh) hndl = sock_diag_lock_handler(req->sdiag_family); if (hndl == NULL) err = -ENOENT; - else + else if (nlh->nlmsg_type == SOCK_DIAG_BY_FAMILY) err = hndl->dump(skb, nlh); + else if (nlh->nlmsg_type == SOCK_DESTROY_BACKPORT && hndl->destroy) + err = hndl->destroy(skb, nlh); + else + err = -EOPNOTSUPP; sock_diag_unlock_handler(hndl); return err; @@ -159,7 +163,8 @@ static int sock_diag_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh) return ret; case SOCK_DIAG_BY_FAMILY: - return __sock_diag_rcv_msg(skb, nlh); + case SOCK_DESTROY_BACKPORT: + return __sock_diag_cmd(skb, nlh); default: return -EINVAL; } @@ -174,6 +179,18 @@ static void sock_diag_rcv(struct sk_buff *skb) mutex_unlock(&sock_diag_mutex); } +int sock_diag_destroy(struct sock *sk, int err) +{ + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + + if (!sk->sk_prot->diag_destroy) + return -EOPNOTSUPP; + + return sk->sk_prot->diag_destroy(sk, err); +} +EXPORT_SYMBOL_GPL(sock_diag_destroy); + struct sock *sock_diag_nlsk; EXPORT_SYMBOL_GPL(sock_diag_nlsk); diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c index 4dc588f520e..95fd5ec945f 100644 --- a/net/dccp/ipv6.c +++ b/net/dccp/ipv6.c @@ -253,9 +253,9 @@ static int dccp_v6_send_response(struct sock *sk, struct request_sock *req, fl6.fl6_sport = inet_rsk(req)->loc_port; security_req_classify_flow(req, flowi6_to_flowi(&fl6)); - opt = np->opt; - - final_p = fl6_update_dst(&fl6, opt, &final); + rcu_read_lock(); + final_p = fl6_update_dst(&fl6, rcu_dereference(np->opt), &final); + rcu_read_unlock(); dst = ip6_dst_lookup_flow(sk, &fl6, final_p, false); if (IS_ERR(dst)) { @@ -272,13 +272,14 @@ static int dccp_v6_send_response(struct sock *sk, struct request_sock *req, &ireq6->loc_addr, &ireq6->rmt_addr); fl6.daddr = ireq6->rmt_addr; - err = ip6_xmit(sk, skb, &fl6, opt, np->tclass); + rcu_read_lock(); + err = ip6_xmit(sk, skb, &fl6, rcu_dereference(np->opt), + np->tclass); + rcu_read_unlock(); err = net_xmit_eval(err); } done: - if (opt != NULL && opt != np->opt) - sock_kfree_s(sk, opt, opt->tot_len); dst_release(dst); return err; } @@ -469,6 +470,7 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk, { struct inet6_request_sock *ireq6 = inet6_rsk(req); struct ipv6_pinfo *newnp, *np = inet6_sk(sk); + struct ipv6_txoptions *opt; struct inet_sock *newinet; struct dccp6_sock *newdp6; struct sock *newsk; @@ -594,16 +596,16 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk, * Yes, keeping reference count would be much more clever, but we make * one more one thing there: reattach optmem to newsk. */ - if (opt != NULL) { - newnp->opt = ipv6_dup_options(newsk, opt); - if (opt != np->opt) - sock_kfree_s(sk, opt, opt->tot_len); - } + opt = rcu_dereference(np->opt); + if (opt) { + opt = ipv6_dup_options(newsk, opt); + RCU_INIT_POINTER(newnp->opt, opt); + } inet_csk(newsk)->icsk_ext_hdr_len = 0; - if (newnp->opt != NULL) - inet_csk(newsk)->icsk_ext_hdr_len = (newnp->opt->opt_nflen + - newnp->opt->opt_flen); + if (opt) + inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen + + opt->opt_flen; dccp_sync_mss(newsk, dst_mtu(dst)); @@ -856,6 +858,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr, struct ipv6_pinfo *np = inet6_sk(sk); struct dccp_sock *dp = dccp_sk(sk); struct in6_addr *saddr = NULL, *final_p, final; + struct ipv6_txoptions *opt; struct flowi6 fl6; struct dst_entry *dst; int addr_type; @@ -958,7 +961,8 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr, fl6.fl6_sport = inet->inet_sport; security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); - final_p = fl6_update_dst(&fl6, np->opt, &final); + opt = rcu_dereference_protected(np->opt, sock_owned_by_user(sk)); + final_p = fl6_update_dst(&fl6, opt, &final); dst = ip6_dst_lookup_flow(sk, &fl6, final_p, true); if (IS_ERR(dst)) { @@ -978,9 +982,8 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr, __ip6_dst_store(sk, dst, NULL, NULL); icsk->icsk_ext_hdr_len = 0; - if (np->opt != NULL) - icsk->icsk_ext_hdr_len = (np->opt->opt_flen + - np->opt->opt_nflen); + if (opt) + icsk->icsk_ext_hdr_len = opt->opt_flen + opt->opt_nflen; inet->inet_dport = usin->sin6_port; diff --git a/net/decnet/af_decnet.c b/net/decnet/af_decnet.c index 4136987d94d..4fa941ea4d6 100644 --- a/net/decnet/af_decnet.c +++ b/net/decnet/af_decnet.c @@ -680,6 +680,9 @@ static int dn_create(struct net *net, struct socket *sock, int protocol, { struct sock *sk; + if (protocol < 0 || protocol > SK_PROTOCOL_MAX) + return -EINVAL; + if (!net_eq(net, &init_net)) return -EAFNOSUPPORT; diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 690948330d5..9b731c030b6 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -297,6 +297,9 @@ static int inet_create(struct net *net, struct socket *sock, int protocol, int try_loading_module = 0; int err; + if (protocol < 0 || protocol >= IPPROTO_MAX) + return -EINVAL; + if (!current_has_network()) return -EACCES; diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index 8a9aab37f0a..2f274b8a35a 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -326,6 +326,9 @@ static void __inet_del_ifa(struct in_device *in_dev, struct in_ifaddr **ifap, ASSERT_RTNL(); + if (in_dev->dead) + goto no_promotions; + /* 1. Deleting primary ifaddr forces deletion all secondaries * unless alias promotion is set **/ @@ -372,6 +375,7 @@ static void __inet_del_ifa(struct in_device *in_dev, struct in_ifaddr **ifap, fib_del_ifaddr(ifa, ifa1); } +no_promotions: /* 2. Unlink it */ *ifap = ifa1->ifa_next; diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 0a24199ff84..6ca74ff0dd8 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -764,6 +764,9 @@ void fib_del_ifaddr(struct in_ifaddr *ifa, struct in_ifaddr *iprim) subnet = 1; } + if (in_dev->dead) + goto no_promotions; + /* Deletion is more complicated than add. * We should take care of not to delete too much :-) * @@ -839,6 +842,7 @@ void fib_del_ifaddr(struct in_ifaddr *ifa, struct in_ifaddr *iprim) } } +no_promotions: if (!(ok & BRD_OK)) fib_magic(RTM_DELROUTE, RTN_BROADCAST, ifa->ifa_broadcast, 32, prim); if (subnet && ifa->ifa_prefixlen < 31) { diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c index 29a07b6c716..0ee2d50c634 100644 --- a/net/ipv4/ip_forward.c +++ b/net/ipv4/ip_forward.c @@ -107,7 +107,8 @@ int ip_forward(struct sk_buff *skb) * We now generate an ICMP HOST REDIRECT giving the route * we calculated. */ - if (rt->rt_flags&RTCF_DOREDIRECT && !opt->srr && !skb_sec_path(skb)) + if (IPCB(skb)->flags & IPSKB_DOREDIRECT && !opt->srr && + !skb_sec_path(skb)) ip_rt_send_redirect(skb); skb->priority = rt_tos2priority(iph->tos); diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index fd7a3f68917..f4865d2958f 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -350,11 +350,12 @@ unsigned int arpt_do_table(struct sk_buff *skb, } /* All zeroes == unconditional rule. */ -static inline bool unconditional(const struct arpt_arp *arp) +static inline bool unconditional(const struct arpt_entry *e) { static const struct arpt_arp uncond; - return memcmp(arp, &uncond, sizeof(uncond)) == 0; + return e->target_offset == sizeof(struct arpt_entry) && + memcmp(&e->arp, &uncond, sizeof(uncond)) == 0; } /* Figures out from what hook each rule can be called: returns 0 if @@ -393,11 +394,10 @@ static int mark_source_chains(const struct xt_table_info *newinfo, |= ((1 << hook) | (1 << NF_ARP_NUMHOOKS)); /* Unconditional return/END. */ - if ((e->target_offset == sizeof(struct arpt_entry) && + if ((unconditional(e) && (strcmp(t->target.u.user.name, XT_STANDARD_TARGET) == 0) && - t->verdict < 0 && unconditional(&e->arp)) || - visited) { + t->verdict < 0) || visited) { unsigned int oldpos, size; if ((strcmp(t->target.u.user.name, @@ -465,14 +465,12 @@ static int mark_source_chains(const struct xt_table_info *newinfo, return 1; } -static inline int check_entry(const struct arpt_entry *e, const char *name) +static inline int check_entry(const struct arpt_entry *e) { const struct xt_entry_target *t; - if (!arp_checkentry(&e->arp)) { - duprintf("arp_tables: arp check failed %p %s.\n", e, name); + if (!arp_checkentry(&e->arp)) return -EINVAL; - } if (e->target_offset + sizeof(struct xt_entry_target) > e->next_offset) return -EINVAL; @@ -513,10 +511,6 @@ find_check_entry(struct arpt_entry *e, const char *name, unsigned int size) struct xt_target *target; int ret; - ret = check_entry(e, name); - if (ret) - return ret; - t = arpt_get_target(e); target = xt_request_find_target(NFPROTO_ARP, t->u.user.name, t->u.user.revision); @@ -542,7 +536,7 @@ static bool check_underflow(const struct arpt_entry *e) const struct xt_entry_target *t; unsigned int verdict; - if (!unconditional(&e->arp)) + if (!unconditional(e)) return false; t = arpt_get_target_c(e); if (strcmp(t->u.user.name, XT_STANDARD_TARGET) != 0) @@ -561,9 +555,11 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, unsigned int valid_hooks) { unsigned int h; + int err; if ((unsigned long)e % __alignof__(struct arpt_entry) != 0 || - (unsigned char *)e + sizeof(struct arpt_entry) >= limit) { + (unsigned char *)e + sizeof(struct arpt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p\n", e); return -EINVAL; } @@ -575,6 +571,10 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, return -EINVAL; } + err = check_entry(e); + if (err) + return err; + /* Check hooks & underflows */ for (h = 0; h < NF_ARP_NUMHOOKS; h++) { if (!(valid_hooks & (1 << h))) @@ -583,9 +583,9 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, newinfo->hook_entry[h] = hook_entries[h]; if ((unsigned char *)e - base == underflows[h]) { if (!check_underflow(e)) { - pr_err("Underflows must be unconditional and " - "use the STANDARD target with " - "ACCEPT/DROP\n"); + pr_debug("Underflows must be unconditional and " + "use the STANDARD target with " + "ACCEPT/DROP\n"); return -EINVAL; } newinfo->underflow[h] = underflows[h]; @@ -1217,7 +1217,8 @@ check_compat_entry_size_and_hooks(struct compat_arpt_entry *e, duprintf("check_compat_entry_size_and_hooks %p\n", e); if ((unsigned long)e % __alignof__(struct compat_arpt_entry) != 0 || - (unsigned char *)e + sizeof(struct compat_arpt_entry) >= limit) { + (unsigned char *)e + sizeof(struct compat_arpt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p, limit = %p\n", e, limit); return -EINVAL; } @@ -1230,7 +1231,7 @@ check_compat_entry_size_and_hooks(struct compat_arpt_entry *e, } /* For purposes of check_entry casting the compat entry is fine */ - ret = check_entry((struct arpt_entry *)e, name); + ret = check_entry((struct arpt_entry *)e); if (ret) return ret; diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c index 24e556e83a3..d0f907567e1 100644 --- a/net/ipv4/netfilter/ip_tables.c +++ b/net/ipv4/netfilter/ip_tables.c @@ -168,11 +168,12 @@ get_entry(const void *base, unsigned int offset) /* All zeroes == unconditional rule. */ /* Mildly perf critical (only if packet tracing is on) */ -static inline bool unconditional(const struct ipt_ip *ip) +static inline bool unconditional(const struct ipt_entry *e) { static const struct ipt_ip uncond; - return memcmp(ip, &uncond, sizeof(uncond)) == 0; + return e->target_offset == sizeof(struct ipt_entry) && + memcmp(&e->ip, &uncond, sizeof(uncond)) == 0; #undef FWINV } @@ -230,11 +231,10 @@ get_chainname_rulenum(const struct ipt_entry *s, const struct ipt_entry *e, } else if (s == e) { (*rulenum)++; - if (s->target_offset == sizeof(struct ipt_entry) && + if (unconditional(s) && strcmp(t->target.u.kernel.target->name, XT_STANDARD_TARGET) == 0 && - t->verdict < 0 && - unconditional(&s->ip)) { + t->verdict < 0) { /* Tail of chains: STANDARD target (return/policy) */ *comment = *chainname == hookname ? comments[NF_IP_TRACE_COMMENT_POLICY] @@ -468,11 +468,10 @@ mark_source_chains(const struct xt_table_info *newinfo, e->comefrom |= ((1 << hook) | (1 << NF_INET_NUMHOOKS)); /* Unconditional return/END. */ - if ((e->target_offset == sizeof(struct ipt_entry) && + if ((unconditional(e) && (strcmp(t->target.u.user.name, XT_STANDARD_TARGET) == 0) && - t->verdict < 0 && unconditional(&e->ip)) || - visited) { + t->verdict < 0) || visited) { unsigned int oldpos, size; if ((strcmp(t->target.u.user.name, @@ -561,14 +560,12 @@ static void cleanup_match(struct xt_entry_match *m, struct net *net) } static int -check_entry(const struct ipt_entry *e, const char *name) +check_entry(const struct ipt_entry *e) { const struct xt_entry_target *t; - if (!ip_checkentry(&e->ip)) { - duprintf("ip check failed %p %s.\n", e, name); + if (!ip_checkentry(&e->ip)) return -EINVAL; - } if (e->target_offset + sizeof(struct xt_entry_target) > e->next_offset) @@ -658,10 +655,6 @@ find_check_entry(struct ipt_entry *e, struct net *net, const char *name, struct xt_mtchk_param mtpar; struct xt_entry_match *ematch; - ret = check_entry(e, name); - if (ret) - return ret; - j = 0; mtpar.net = net; mtpar.table = name; @@ -705,7 +698,7 @@ static bool check_underflow(const struct ipt_entry *e) const struct xt_entry_target *t; unsigned int verdict; - if (!unconditional(&e->ip)) + if (!unconditional(e)) return false; t = ipt_get_target_c(e); if (strcmp(t->u.user.name, XT_STANDARD_TARGET) != 0) @@ -725,9 +718,11 @@ check_entry_size_and_hooks(struct ipt_entry *e, unsigned int valid_hooks) { unsigned int h; + int err; if ((unsigned long)e % __alignof__(struct ipt_entry) != 0 || - (unsigned char *)e + sizeof(struct ipt_entry) >= limit) { + (unsigned char *)e + sizeof(struct ipt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p\n", e); return -EINVAL; } @@ -739,6 +734,10 @@ check_entry_size_and_hooks(struct ipt_entry *e, return -EINVAL; } + err = check_entry(e); + if (err) + return err; + /* Check hooks & underflows */ for (h = 0; h < NF_INET_NUMHOOKS; h++) { if (!(valid_hooks & (1 << h))) @@ -747,9 +746,9 @@ check_entry_size_and_hooks(struct ipt_entry *e, newinfo->hook_entry[h] = hook_entries[h]; if ((unsigned char *)e - base == underflows[h]) { if (!check_underflow(e)) { - pr_err("Underflows must be unconditional and " - "use the STANDARD target with " - "ACCEPT/DROP\n"); + pr_debug("Underflows must be unconditional and " + "use the STANDARD target with " + "ACCEPT/DROP\n"); return -EINVAL; } newinfo->underflow[h] = underflows[h]; @@ -1484,7 +1483,8 @@ check_compat_entry_size_and_hooks(struct compat_ipt_entry *e, duprintf("check_compat_entry_size_and_hooks %p\n", e); if ((unsigned long)e % __alignof__(struct compat_ipt_entry) != 0 || - (unsigned char *)e + sizeof(struct compat_ipt_entry) >= limit) { + (unsigned char *)e + sizeof(struct compat_ipt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p, limit = %p\n", e, limit); return -EINVAL; } @@ -1497,7 +1497,7 @@ check_compat_entry_size_and_hooks(struct compat_ipt_entry *e, } /* For purposes of check_entry casting the compat entry is fine */ - ret = check_entry((struct ipt_entry *)e, name); + ret = check_entry((struct ipt_entry *)e); if (ret) return ret; diff --git a/net/ipv4/netfilter/ipt_MASQUERADE.c b/net/ipv4/netfilter/ipt_MASQUERADE.c index 2f210c79dc8..dcd90860bee 100644 --- a/net/ipv4/netfilter/ipt_MASQUERADE.c +++ b/net/ipv4/netfilter/ipt_MASQUERADE.c @@ -123,8 +123,16 @@ static int masq_inet_event(struct notifier_block *this, unsigned long event, void *ptr) { - struct net_device *dev = ((struct in_ifaddr *)ptr)->ifa_dev->dev; - return masq_device_event(this, event, dev); + struct in_device *idev = ((struct in_ifaddr *)ptr)->ifa_dev; + /* The masq_dev_notifier will catch the case of the device going + * down. So if the inetdev is dead and being destroyed we have + * no work to do. Otherwise this is an individual address removal + * and we have to perform the flush. + */ + if (idev->dead) + return NOTIFY_DONE; + + return masq_device_event(this, event, idev->dev); } static struct notifier_block masq_dev_notifier = { diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 4faaea9b4db..ac8a03cb6f2 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -153,6 +153,7 @@ void ping_unhash(struct sock *sk) if (sk_hashed(sk)) { write_lock_bh(&ping_table.lock); hlist_nulls_del(&sk->sk_nulls_node); + sk_nulls_node_init(&sk->sk_nulls_node); sock_put(sk); isk->inet_num = 0; isk->inet_sport = 0; @@ -213,6 +214,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) &ipv6_hdr(skb)->daddr)) continue; #endif + } else { + continue; } if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif) @@ -252,6 +255,9 @@ int ping_init_sock(struct sock *sk) int i, j, count; int ret = 0; + if (sk->sk_family == AF_INET6) + inet6_sk(sk)->ipv6only = 1; + inet_get_ping_group_range_net(net, range, range+1); if (range[0] <= group && group <= range[1]) return 0; @@ -299,6 +305,11 @@ int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk, if (addr_len < sizeof(*addr)) return -EINVAL; + if (addr->sin_family != AF_INET && + !(addr->sin_family == AF_UNSPEC && + addr->sin_addr.s_addr == htonl(INADDR_ANY))) + return -EAFNOSUPPORT; + pr_debug("ping_check_bind_addr(sk=%p,addr=%pI4,port=%d)\n", sk, &addr->sin_addr.s_addr, ntohs(addr->sin_port)); @@ -323,6 +334,9 @@ int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk, if (addr_len < sizeof(*addr)) return -EINVAL; + if (addr->sin6_family != AF_INET6) + return -EAFNOSUPPORT; + pr_debug("ping_check_bind_addr(sk=%p,addr=%pI6c,port=%d)\n", sk, addr->sin6_addr.s6_addr, ntohs(addr->sin6_port)); @@ -640,7 +654,7 @@ int ping_common_sendmsg(int family, struct msghdr *msg, size_t len, void *user_icmph, size_t icmph_len) { u8 type, code; - if (len > 0xFFFF) + if (len > 0xFFFF || len < icmph_len) return -EMSGSIZE; /* @@ -709,7 +723,7 @@ int ping_v4_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, if (msg->msg_namelen < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) - return -EINVAL; + return -EAFNOSUPPORT; daddr = usin->sin_addr.s_addr; /* no remote port */ } else { diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 3097eca70fd..12e7474d6e3 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -2156,9 +2156,10 @@ static int __mkroute_input(struct sk_buff *skb, flags |= RTCF_DIRECTSRC; if (out_dev == in_dev && err && + skb->protocol == htons(ETH_P_IP) && (IN_DEV_SHARED_MEDIA(out_dev) || inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) - flags |= RTCF_DOREDIRECT; + IPCB(skb)->flags |= IPSKB_DOREDIRECT; if (skb->protocol != htons(ETH_P_IP)) { /* Not IP (i.e. ARP). Do not create route, if it is @@ -2987,6 +2988,8 @@ static int rt_fill_info(struct net *net, r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; if (rt->rt_flags & RTCF_NOTIFY) r->rtm_flags |= RTM_F_NOTIFY; + if (IPCB(skb)->flags & IPSKB_DOREDIRECT) + r->rtm_flags |= RTCF_DOREDIRECT; NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 74a286c2c43..a409a83b7dc 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -2523,7 +2523,7 @@ void tcp_get_info(const struct sock *sk, struct tcp_info *info) if (sk->sk_socket) { struct file *filep = sk->sk_socket->file; if (filep) - info->tcpi_count = atomic_read(&filep->f_count); + info->tcpi_count = file_count(filep); } } EXPORT_SYMBOL_GPL(tcp_get_info); diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 14a227e8141..0078ce73c23 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -5186,7 +5186,7 @@ static int tcp_copy_to_iovec(struct sock *sk, struct sk_buff *skb, int hlen) err = skb_copy_datagram_iovec(skb, hlen, tp->ucopy.iov, chunk); else err = skb_copy_and_csum_datagram_iovec(skb, hlen, - tp->ucopy.iov); + tp->ucopy.iov, chunk); if (!err) { tp->ucopy.len -= chunk; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 4e9044391b4..c8a621e1364 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -678,10 +678,11 @@ static void tcp_v4_send_reset(struct sock *sk, struct sk_buff *skb) arg.csumoffset = offsetof(struct tcphdr, check) / 2; arg.flags = (sk && inet_sk(sk)->transparent) ? IP_REPLY_ARG_NOSRCCHECK : 0; /* When socket is gone, all binding information is lost. - * routing might fail in this case. using iif for oif to - * make sure we can deliver it + * routing might fail in this case. No choice here, if we choose to force + * input interface, we will misroute in case of asymmetric route. */ - arg.bound_dev_if = sk ? sk->sk_bound_dev_if : inet_iif(skb); + if (sk) + arg.bound_dev_if = sk->sk_bound_dev_if; net = dev_net(skb_dst(skb)->dev); arg.tos = ip_hdr(skb)->tos; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 4d844344f26..3dc63757ce6 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1213,7 +1213,7 @@ int udp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, else { err = skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr), - msg->msg_iov); + msg->msg_iov, copied); if (err == -EINVAL) goto csum_copy_err; @@ -1253,10 +1253,8 @@ int udp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, UDP_INC_STATS_USER(sock_net(sk), UDP_MIB_INERRORS, is_udplite); unlock_sock_fast(sk, slow); - if (noblock) - return -EAGAIN; - - /* starting over for a new packet */ + /* starting over for a new packet, but check if we need to yield */ + cond_resched(); msg->msg_flags &= ~MSG_TRUNC; goto try_again; } diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 21765829086..3c0f9fed1d0 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -201,6 +201,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .disable_ipv6 = 0, .accept_dad = 1, .accept_ra_prefix_route = 1, + .use_oif_addrs_only = 0, }; static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { @@ -237,6 +238,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .disable_ipv6 = 0, .accept_dad = 1, .accept_ra_prefix_route = 1, + .use_oif_addrs_only = 0, }; /* IPv6 Wildcard Address and Loopback Address defined by RFC2553 */ @@ -961,6 +963,9 @@ enum { #endif IPV6_SADDR_RULE_ORCHID, IPV6_SADDR_RULE_PREFIX, +#ifdef CONFIG_IPV6_OPTIMISTIC_DAD + IPV6_SADDR_RULE_NOT_OPTIMISTIC, +#endif IPV6_SADDR_RULE_MAX }; @@ -988,6 +993,15 @@ static inline int ipv6_saddr_preferred(int type) return 0; } +static inline bool ipv6_use_optimistic_addr(struct inet6_dev *idev) +{ +#ifdef CONFIG_IPV6_OPTIMISTIC_DAD + return idev && idev->cnf.optimistic_dad && idev->cnf.use_optimistic; +#else + return false; +#endif +} + static int ipv6_get_saddr_eval(struct net *net, struct ipv6_saddr_score *score, struct ipv6_saddr_dst *dst, @@ -1048,10 +1062,16 @@ static int ipv6_get_saddr_eval(struct net *net, score->scopedist = ret; break; case IPV6_SADDR_RULE_PREFERRED: + { /* Rule 3: Avoid deprecated and optimistic addresses */ + u8 avoid = IFA_F_DEPRECATED; + + if (!ipv6_use_optimistic_addr(score->ifa->idev)) + avoid |= IFA_F_OPTIMISTIC; ret = ipv6_saddr_preferred(score->addr_type) || - !(score->ifa->flags & (IFA_F_DEPRECATED|IFA_F_OPTIMISTIC)); + !(score->ifa->flags & avoid); break; + } #ifdef CONFIG_IPV6_MIP6 case IPV6_SADDR_RULE_HOA: { @@ -1097,6 +1117,14 @@ static int ipv6_get_saddr_eval(struct net *net, score->matchlen = ret = ipv6_addr_diff(&score->ifa->addr, dst->addr); break; +#ifdef CONFIG_IPV6_OPTIMISTIC_DAD + case IPV6_SADDR_RULE_NOT_OPTIMISTIC: + /* Optimistic addresses still have lower precedence than other + * preferred addresses. + */ + ret = !(score->ifa->flags & IFA_F_OPTIMISTIC); + break; +#endif default: ret = 0; } @@ -1144,9 +1172,15 @@ int ipv6_dev_get_saddr(struct net *net, struct net_device *dst_dev, * include addresses assigned to interfaces * belonging to the same site as the outgoing * interface.) + * - "It is RECOMMENDED that the candidate source addresses + * be the set of unicast addresses assigned to the + * interface that will be used to send to the destination + * (the 'outgoing' interface)." (RFC 6724) */ + idev = dst_dev ? __in6_dev_get(dst_dev) : NULL; if (((dst_type & IPV6_ADDR_MULTICAST) || - dst.scope <= IPV6_ADDR_SCOPE_LINKLOCAL) && + dst.scope <= IPV6_ADDR_SCOPE_LINKLOCAL || + (idev && idev->cnf.use_oif_addrs_only)) && dst.ifindex && dev->ifindex != dst.ifindex) continue; @@ -1285,7 +1319,9 @@ int ipv6_chk_addr(struct net *net, const struct in6_addr *addr, if (!net_eq(dev_net(ifp->idev->dev), net)) continue; if (ipv6_addr_equal(&ifp->addr, addr) && - !(ifp->flags&IFA_F_TENTATIVE) && + (!(ifp->flags&IFA_F_TENTATIVE) || + (ipv6_use_optimistic_addr(ifp->idev) && + ifp->flags&IFA_F_OPTIMISTIC)) && (dev == NULL || ifp->idev->dev == dev || !(ifp->scope&(IFA_LINK|IFA_HOST) || strict))) { rcu_read_unlock_bh(); @@ -3006,8 +3042,15 @@ static void addrconf_dad_start(struct inet6_ifaddr *ifp, u32 flags) * Optimistic nodes can start receiving * Frames right away */ - if (ifp->flags & IFA_F_OPTIMISTIC) + if (ifp->flags & IFA_F_OPTIMISTIC) { ip6_ins_rt(ifp->rt); + if (ipv6_use_optimistic_addr(idev)) { + /* Because optimistic nodes can use this address, + * notify listeners. If DAD fails, RTM_DELADDR is sent. + */ + ipv6_ifa_notify(RTM_NEWADDR, ifp); + } + } addrconf_dad_kick(ifp); out: @@ -3955,6 +3998,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf, array[DEVCONF_ACCEPT_SOURCE_ROUTE] = cnf->accept_source_route; #ifdef CONFIG_IPV6_OPTIMISTIC_DAD array[DEVCONF_OPTIMISTIC_DAD] = cnf->optimistic_dad; + array[DEVCONF_USE_OPTIMISTIC] = cnf->use_optimistic; #endif #ifdef CONFIG_IPV6_MROUTE array[DEVCONF_MC_FORWARDING] = cnf->mc_forwarding; @@ -3962,6 +4006,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf, array[DEVCONF_DISABLE_IPV6] = cnf->disable_ipv6; array[DEVCONF_ACCEPT_DAD] = cnf->accept_dad; array[DEVCONF_FORCE_TLLAO] = cnf->force_tllao; + array[DEVCONF_USE_OIF_ADDRS_ONLY] = cnf->use_oif_addrs_only; } static inline size_t inet6_ifla6_size(void) @@ -4328,6 +4373,21 @@ int addrconf_sysctl_forward(ctl_table *ctl, int write, return ret; } +static +int addrconf_sysctl_mtu(struct ctl_table *ctl, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct inet6_dev *idev = ctl->extra1; + int min_mtu = IPV6_MIN_MTU; + struct ctl_table lctl; + + lctl = *ctl; + lctl.extra1 = &min_mtu; + lctl.extra2 = idev ? &idev->dev->mtu : NULL; + + return proc_dointvec_minmax(&lctl, write, buffer, lenp, ppos); +} + static void dev_disable_change(struct inet6_dev *idev) { if (!idev || !idev->dev) @@ -4437,7 +4497,7 @@ static struct addrconf_sysctl_table .data = &ipv6_devconf.mtu6, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = addrconf_sysctl_mtu, }, { .procname = "accept_ra", @@ -4608,6 +4668,14 @@ static struct addrconf_sysctl_table .proc_handler = proc_dointvec, }, + { + .procname = "use_optimistic", + .data = &ipv6_devconf.use_optimistic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + + }, #endif #ifdef CONFIG_IPV6_MROUTE { @@ -4646,6 +4714,14 @@ static struct addrconf_sysctl_table .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "use_oif_addrs_only", + .data = &ipv6_devconf.use_oif_addrs_only, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + + }, { /* sentinel */ } diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 296886bff73..b7ed0abd6ea 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -123,6 +123,9 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, int try_loading_module = 0; int err; + if (protocol < 0 || protocol >= IPPROTO_MAX) + return -EINVAL; + if (!current_has_network()) return -EACCES; @@ -448,8 +451,11 @@ void inet6_destroy_sock(struct sock *sk) /* Free tx options */ - if ((opt = xchg(&np->opt, NULL)) != NULL) - sock_kfree_s(sk, opt, opt->tot_len); + opt = xchg((__force struct ipv6_txoptions **)&np->opt, NULL); + if (opt) { + atomic_sub(opt->tot_len, &sk->sk_omem_alloc); + txopt_put(opt); + } } EXPORT_SYMBOL_GPL(inet6_destroy_sock); @@ -705,7 +711,10 @@ int inet6_sk_rebuild_header(struct sock *sk) fl6.flowi6_uid = sock_i_uid(sk); security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); - final_p = fl6_update_dst(&fl6, np->opt, &final); + rcu_read_lock(); + final_p = fl6_update_dst(&fl6, rcu_dereference(np->opt), + &final); + rcu_read_unlock(); dst = ip6_dst_lookup_flow(sk, &fl6, final_p, false); if (IS_ERR(dst)) { diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c index ae4d713ac88..2659d0028bb 100644 --- a/net/ipv6/datagram.c +++ b/net/ipv6/datagram.c @@ -167,8 +167,10 @@ int ip6_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); - opt = flowlabel ? flowlabel->opt : np->opt; + rcu_read_lock(); + opt = flowlabel ? flowlabel->opt : rcu_dereference(np->opt); final_p = fl6_update_dst(&fl6, opt, &final); + rcu_read_unlock(); dst = ip6_dst_lookup_flow(sk, &fl6, final_p, true); err = 0; diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index 3d641b6e9b0..e66773850e5 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -748,6 +748,7 @@ ipv6_dup_options(struct sock *sk, struct ipv6_txoptions *opt) *((char**)&opt2->dst1opt) += dif; if (opt2->srcrt) *((char**)&opt2->srcrt) += dif; + atomic_set(&opt2->refcnt, 1); } return opt2; } @@ -812,7 +813,7 @@ ipv6_renew_options(struct sock *sk, struct ipv6_txoptions *opt, return ERR_PTR(-ENOBUFS); memset(opt2, 0, tot_len); - + atomic_set(&opt2->refcnt, 1); opt2->tot_len = tot_len; p = (char *)(opt2 + 1); diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index b595aa69d01..4b575f0583b 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -68,6 +68,10 @@ #include +#ifdef CONFIG_PARTIALRESUME +bool wlan_vote_for_suspend(void); +#endif + /* * The ICMP socket(s). This is the most convenient way to flow control * our ICMP output as well as maintain a clean interface throughout @@ -761,6 +765,12 @@ static int icmpv6_rcv(struct sk_buff *skb) case NDISC_NEIGHBOUR_ADVERTISEMENT: case NDISC_REDIRECT: ndisc_rcv(skb); +#ifdef CONFIG_PARTIALRESUME + if ((type == NDISC_NEIGHBOUR_ADVERTISEMENT) && + (ipv6_addr_equal(&in6addr_linklocal_allnodes, + &ipv6_hdr(skb)->daddr))) + wlan_vote_for_suspend(); +#endif break; case ICMPV6_MGM_QUERY: diff --git a/net/ipv6/inet6_connection_sock.c b/net/ipv6/inet6_connection_sock.c index aefc8b71809..67aa2c2b502 100644 --- a/net/ipv6/inet6_connection_sock.c +++ b/net/ipv6/inet6_connection_sock.c @@ -66,7 +66,9 @@ struct dst_entry *inet6_csk_route_req(struct sock *sk, memset(&fl6, 0, sizeof(fl6)); fl6.flowi6_proto = IPPROTO_TCP; fl6.daddr = treq->rmt_addr; - final_p = fl6_update_dst(&fl6, np->opt, &final); + rcu_read_lock(); + final_p = fl6_update_dst(&fl6, rcu_dereference(np->opt), &final); + rcu_read_unlock(); fl6.saddr = treq->loc_addr; fl6.flowi6_oif = sk->sk_bound_dev_if; fl6.flowi6_mark = inet_rsk(req)->ir_mark; @@ -227,7 +229,9 @@ int inet6_csk_xmit(struct sk_buff *skb, struct flowi *fl_unused) fl6.flowi6_uid = sock_i_uid(sk); security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); - final_p = fl6_update_dst(&fl6, np->opt, &final); + rcu_read_lock(); + final_p = fl6_update_dst(&fl6, rcu_dereference(np->opt), &final); + rcu_read_unlock(); dst = __inet6_csk_dst_check(sk, np->dst_cookie); @@ -250,7 +254,8 @@ int inet6_csk_xmit(struct sk_buff *skb, struct flowi *fl_unused) /* Restore final destination back after routing done */ fl6.daddr = np->daddr; - res = ip6_xmit(sk, skb, &fl6, np->opt, np->tclass); + res = ip6_xmit(sk, skb, &fl6, rcu_dereference(np->opt), + np->tclass); rcu_read_unlock(); return res; } diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index 63dd1f89ed7..601360e6bb8 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -110,10 +110,12 @@ struct ipv6_txoptions *ipv6_update_options(struct sock *sk, icsk->icsk_ext_hdr_len = opt->opt_flen + opt->opt_nflen; icsk->icsk_sync_mss(sk, icsk->icsk_pmtu_cookie); } - opt = xchg(&inet6_sk(sk)->opt, opt); + opt = xchg((__force struct ipv6_txoptions **)&inet6_sk(sk)->opt, + opt); } else { spin_lock(&sk->sk_dst_lock); - opt = xchg(&inet6_sk(sk)->opt, opt); + opt = xchg((__force struct ipv6_txoptions **)&inet6_sk(sk)->opt, + opt); spin_unlock(&sk->sk_dst_lock); } sk_dst_reset(sk); @@ -213,9 +215,12 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, sk->sk_socket->ops = &inet_dgram_ops; sk->sk_family = PF_INET; } - opt = xchg(&np->opt, NULL); - if (opt) - sock_kfree_s(sk, opt, opt->tot_len); + opt = xchg((__force struct ipv6_txoptions **)&np->opt, + NULL); + if (opt) { + atomic_sub(opt->tot_len, &sk->sk_omem_alloc); + txopt_put(opt); + } pktopt = xchg(&np->pktoptions, NULL); kfree_skb(pktopt); @@ -384,7 +389,8 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (optname != IPV6_RTHDR && !capable(CAP_NET_RAW)) break; - opt = ipv6_renew_options(sk, np->opt, optname, + opt = rcu_dereference_protected(np->opt, sock_owned_by_user(sk)); + opt = ipv6_renew_options(sk, opt, optname, (struct ipv6_opt_hdr __user *)optval, optlen); if (IS_ERR(opt)) { @@ -413,8 +419,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, retv = 0; opt = ipv6_update_options(sk, opt); sticky_done: - if (opt) - sock_kfree_s(sk, opt, opt->tot_len); + if (opt) { + atomic_sub(opt->tot_len, &sk->sk_omem_alloc); + txopt_put(opt); + } break; } @@ -467,6 +475,7 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, break; memset(opt, 0, sizeof(*opt)); + atomic_set(&opt->refcnt, 1); opt->tot_len = sizeof(*opt) + optlen; retv = -EFAULT; if (copy_from_user(opt+1, optval, optlen)) @@ -483,8 +492,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, retv = 0; opt = ipv6_update_options(sk, opt); done: - if (opt) - sock_kfree_s(sk, opt, opt->tot_len); + if (opt) { + atomic_sub(opt->tot_len, &sk->sk_omem_alloc); + txopt_put(opt); + } break; } case IPV6_UNICAST_HOPS: @@ -1084,10 +1095,11 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, case IPV6_RTHDR: case IPV6_DSTOPTS: { + struct ipv6_txoptions *opt; lock_sock(sk); - len = ipv6_getsockopt_sticky(sk, np->opt, - optname, optval, len); + opt = rcu_dereference_protected(np->opt, sock_owned_by_user(sk)); + len = ipv6_getsockopt_sticky(sk, opt, optname, optval, len); release_sock(sk); /* check if ipv6_getsockopt_sticky() returns err code */ if (len < 0) diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index 176b469322a..882525464a0 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1266,7 +1266,14 @@ static void ndisc_router_discovery(struct sk_buff *skb) if (rt) rt6_set_expires(rt, jiffies + (HZ * lifetime)); if (ra_msg->icmph.icmp6_hop_limit) { - in6_dev->cnf.hop_limit = ra_msg->icmph.icmp6_hop_limit; + /* Only set hop_limit on the interface if it is higher than + * the current hop_limit. + */ + if (in6_dev->cnf.hop_limit < ra_msg->icmph.icmp6_hop_limit) { + in6_dev->cnf.hop_limit = ra_msg->icmph.icmp6_hop_limit; + } else { + ND_PRINTK2(KERN_WARNING, "RA: Got route advertisement with lower hop_limit than current\n"); + } if (rt) dst_metric_set(&rt->dst, RTAX_HOPLIMIT, ra_msg->icmph.icmp6_hop_limit); diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index e641f8fa748..4b570b06fdb 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -195,11 +195,12 @@ get_entry(const void *base, unsigned int offset) /* All zeroes == unconditional rule. */ /* Mildly perf critical (only if packet tracing is on) */ -static inline bool unconditional(const struct ip6t_ip6 *ipv6) +static inline bool unconditional(const struct ip6t_entry *e) { static const struct ip6t_ip6 uncond; - return memcmp(ipv6, &uncond, sizeof(uncond)) == 0; + return e->target_offset == sizeof(struct ip6t_entry) && + memcmp(&e->ipv6, &uncond, sizeof(uncond)) == 0; } static inline const struct xt_entry_target * @@ -256,11 +257,10 @@ get_chainname_rulenum(const struct ip6t_entry *s, const struct ip6t_entry *e, } else if (s == e) { (*rulenum)++; - if (s->target_offset == sizeof(struct ip6t_entry) && + if (unconditional(s) && strcmp(t->target.u.kernel.target->name, XT_STANDARD_TARGET) == 0 && - t->verdict < 0 && - unconditional(&s->ipv6)) { + t->verdict < 0) { /* Tail of chains: STANDARD target (return/policy) */ *comment = *chainname == hookname ? comments[NF_IP6_TRACE_COMMENT_POLICY] @@ -477,11 +477,10 @@ mark_source_chains(const struct xt_table_info *newinfo, e->comefrom |= ((1 << hook) | (1 << NF_INET_NUMHOOKS)); /* Unconditional return/END. */ - if ((e->target_offset == sizeof(struct ip6t_entry) && + if ((unconditional(e) && (strcmp(t->target.u.user.name, XT_STANDARD_TARGET) == 0) && - t->verdict < 0 && - unconditional(&e->ipv6)) || visited) { + t->verdict < 0) || visited) { unsigned int oldpos, size; if ((strcmp(t->target.u.user.name, @@ -570,14 +569,12 @@ static void cleanup_match(struct xt_entry_match *m, struct net *net) } static int -check_entry(const struct ip6t_entry *e, const char *name) +check_entry(const struct ip6t_entry *e) { const struct xt_entry_target *t; - if (!ip6_checkentry(&e->ipv6)) { - duprintf("ip_tables: ip check failed %p %s.\n", e, name); + if (!ip6_checkentry(&e->ipv6)) return -EINVAL; - } if (e->target_offset + sizeof(struct xt_entry_target) > e->next_offset) @@ -668,10 +665,6 @@ find_check_entry(struct ip6t_entry *e, struct net *net, const char *name, struct xt_mtchk_param mtpar; struct xt_entry_match *ematch; - ret = check_entry(e, name); - if (ret) - return ret; - j = 0; mtpar.net = net; mtpar.table = name; @@ -715,7 +708,7 @@ static bool check_underflow(const struct ip6t_entry *e) const struct xt_entry_target *t; unsigned int verdict; - if (!unconditional(&e->ipv6)) + if (!unconditional(e)) return false; t = ip6t_get_target_c(e); if (strcmp(t->u.user.name, XT_STANDARD_TARGET) != 0) @@ -735,9 +728,11 @@ check_entry_size_and_hooks(struct ip6t_entry *e, unsigned int valid_hooks) { unsigned int h; + int err; if ((unsigned long)e % __alignof__(struct ip6t_entry) != 0 || - (unsigned char *)e + sizeof(struct ip6t_entry) >= limit) { + (unsigned char *)e + sizeof(struct ip6t_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p\n", e); return -EINVAL; } @@ -749,6 +744,10 @@ check_entry_size_and_hooks(struct ip6t_entry *e, return -EINVAL; } + err = check_entry(e); + if (err) + return err; + /* Check hooks & underflows */ for (h = 0; h < NF_INET_NUMHOOKS; h++) { if (!(valid_hooks & (1 << h))) @@ -757,9 +756,9 @@ check_entry_size_and_hooks(struct ip6t_entry *e, newinfo->hook_entry[h] = hook_entries[h]; if ((unsigned char *)e - base == underflows[h]) { if (!check_underflow(e)) { - pr_err("Underflows must be unconditional and " - "use the STANDARD target with " - "ACCEPT/DROP\n"); + pr_debug("Underflows must be unconditional and " + "use the STANDARD target with " + "ACCEPT/DROP\n"); return -EINVAL; } newinfo->underflow[h] = underflows[h]; @@ -1495,7 +1494,8 @@ check_compat_entry_size_and_hooks(struct compat_ip6t_entry *e, duprintf("check_compat_entry_size_and_hooks %p\n", e); if ((unsigned long)e % __alignof__(struct compat_ip6t_entry) != 0 || - (unsigned char *)e + sizeof(struct compat_ip6t_entry) >= limit) { + (unsigned char *)e + sizeof(struct compat_ip6t_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p, limit = %p\n", e, limit); return -EINVAL; } @@ -1508,7 +1508,7 @@ check_compat_entry_size_and_hooks(struct compat_ip6t_entry *e, } /* For purposes of check_entry casting the compat entry is fine */ - ret = check_entry((struct ip6t_entry *)e, name); + ret = check_entry((struct ip6t_entry *)e); if (ret) return ret; diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index f1c42c8eee5..045eb06a43f 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -127,9 +127,10 @@ int ping_v6_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, if (msg->msg_name) { struct sockaddr_in6 *u = (struct sockaddr_in6 *) msg->msg_name; - if (msg->msg_namelen < sizeof(struct sockaddr_in6) || - u->sin6_family != AF_INET6) { + if (msg->msg_namelen < sizeof(*u)) return -EINVAL; + if (u->sin6_family != AF_INET6) { + return -EAFNOSUPPORT; } if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != u->sin6_scope_id) { diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 49ec3f8e7ce..8ef29a1f2dc 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -484,7 +484,7 @@ static int rawv6_recvmsg(struct kiocb *iocb, struct sock *sk, goto csum_copy_err; err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied); } else { - err = skb_copy_and_csum_datagram_iovec(skb, 0, msg->msg_iov); + err = skb_copy_and_csum_datagram_iovec(skb, 0, msg->msg_iov, copied); if (err == -EINVAL) goto csum_copy_err; } @@ -728,6 +728,7 @@ static int rawv6_probe_proto_opt(struct flowi6 *fl6, struct msghdr *msg) static int rawv6_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, size_t len) { + struct ipv6_txoptions *opt_to_free = NULL; struct ipv6_txoptions opt_space; struct sockaddr_in6 * sin6 = (struct sockaddr_in6 *) msg->msg_name; struct in6_addr *daddr, *final_p, final; @@ -835,8 +836,10 @@ static int rawv6_sendmsg(struct kiocb *iocb, struct sock *sk, if (!(opt->opt_nflen|opt->opt_flen)) opt = NULL; } - if (opt == NULL) - opt = np->opt; + if (!opt) { + opt = txopt_get(np); + opt_to_free = opt; + } if (flowlabel) opt = fl6_merge_options(&opt_space, flowlabel, opt); opt = ipv6_fixup_options(&opt_space, opt); @@ -903,6 +906,7 @@ static int rawv6_sendmsg(struct kiocb *iocb, struct sock *sk, dst_release(dst); out: fl6_sock_release(flowlabel); + txopt_put(opt_to_free); return err<0?err:len; do_confirm: dst_confirm(dst); diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 3e6b53b981a..ab71b41e139 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -928,6 +928,8 @@ struct dst_entry * ip6_route_output(struct net *net, const struct sock *sk, { int flags = 0; + fl6->flowi6_iif = net->loopback_dev->ifindex; + if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr)) flags |= RT6_LOOKUP_F_IFACE; diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index af939afeae2..b57996a4fd6 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -240,7 +240,7 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) memset(&fl6, 0, sizeof(fl6)); fl6.flowi6_proto = IPPROTO_TCP; fl6.daddr = ireq6->rmt_addr; - final_p = fl6_update_dst(&fl6, np->opt, &final); + final_p = fl6_update_dst(&fl6, rcu_dereference(np->opt), &final); fl6.saddr = ireq6->loc_addr; fl6.flowi6_oif = sk->sk_bound_dev_if; fl6.flowi6_mark = inet_rsk(req)->ir_mark; diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index a62ce592889..c39a2f47dd8 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -132,6 +132,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, struct ipv6_pinfo *np = inet6_sk(sk); struct tcp_sock *tp = tcp_sk(sk); struct in6_addr *saddr = NULL, *final_p, final; + struct ipv6_txoptions *opt; struct rt6_info *rt; struct flowi6 fl6; struct dst_entry *dst; @@ -253,7 +254,8 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, fl6.fl6_sport = inet->inet_sport; fl6.flowi6_uid = sock_i_uid(sk); - final_p = fl6_update_dst(&fl6, np->opt, &final); + opt = rcu_dereference_protected(np->opt, sock_owned_by_user(sk)); + final_p = fl6_update_dst(&fl6, opt, &final); security_sk_classify_flow(sk, flowi6_to_flowi(&fl6)); @@ -296,9 +298,9 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, } icsk->icsk_ext_hdr_len = 0; - if (np->opt) - icsk->icsk_ext_hdr_len = (np->opt->opt_flen + - np->opt->opt_nflen); + if (opt) + icsk->icsk_ext_hdr_len = opt->opt_flen + + opt->opt_nflen; tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr); @@ -516,7 +518,8 @@ static int tcp_v6_send_synack(struct sock *sk, struct request_sock *req, __tcp_v6_send_check(skb, &treq->loc_addr, &treq->rmt_addr); fl6.daddr = treq->rmt_addr; - err = ip6_xmit(sk, skb, &fl6, opt, np->tclass); + err = ip6_xmit(sk, skb, &fl6, rcu_dereference(np->opt), + np->tclass); err = net_xmit_eval(err); } @@ -899,7 +902,8 @@ static void tcp_v6_send_response(struct sk_buff *skb, u32 seq, u32 ack, u32 win, __tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr); fl6.flowi6_proto = IPPROTO_TCP; - fl6.flowi6_oif = inet6_iif(skb); + if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) + fl6.flowi6_oif = inet6_iif(skb); fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark); fl6.fl6_dport = t1->dest; fl6.fl6_sport = t1->source; @@ -1242,10 +1246,10 @@ static struct sock * tcp_v6_syn_recv_sock(struct sock *sk, struct sk_buff *skb, struct inet6_request_sock *treq; struct ipv6_pinfo *newnp, *np = inet6_sk(sk); struct tcp6_sock *newtcp6sk; + struct ipv6_txoptions *opt; struct inet_sock *newinet; struct tcp_sock *newtp; struct sock *newsk; - struct ipv6_txoptions *opt; #ifdef CONFIG_TCP_MD5SIG struct tcp_md5sig_key *key; #endif @@ -1374,16 +1378,15 @@ static struct sock * tcp_v6_syn_recv_sock(struct sock *sk, struct sk_buff *skb, but we make one more one thing there: reattach optmem to newsk. */ + opt = rcu_dereference(np->opt); if (opt) { - newnp->opt = ipv6_dup_options(newsk, opt); - if (opt != np->opt) - sock_kfree_s(sk, opt, opt->tot_len); + opt = ipv6_dup_options(newsk, opt); + RCU_INIT_POINTER(newnp->opt, opt); } - inet_csk(newsk)->icsk_ext_hdr_len = 0; - if (newnp->opt) - inet_csk(newsk)->icsk_ext_hdr_len = (newnp->opt->opt_nflen + - newnp->opt->opt_flen); + if (opt) + inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen + + opt->opt_flen; tcp_mtup_init(newsk); tcp_sync_mss(newsk, dst_mtu(dst)); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index a20d55dc9c2..c2e658b560e 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -387,7 +387,8 @@ int udpv6_recvmsg(struct kiocb *iocb, struct sock *sk, err = skb_copy_datagram_iovec(skb, sizeof(struct udphdr), msg->msg_iov, copied ); else { - err = skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr), msg->msg_iov); + err = skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr), + msg->msg_iov, copied); if (err == -EINVAL) goto csum_copy_err; } @@ -454,10 +455,8 @@ int udpv6_recvmsg(struct kiocb *iocb, struct sock *sk, } unlock_sock_fast(sk, slow); - if (noblock) - return -EAGAIN; - - /* starting over for a new packet */ + /* starting over for a new packet, but check if we need to yield */ + cond_resched(); msg->msg_flags &= ~MSG_TRUNC; goto try_again; } @@ -955,6 +954,7 @@ int udpv6_sendmsg(struct kiocb *iocb, struct sock *sk, struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) msg->msg_name; struct in6_addr *daddr, *final_p, final; struct ipv6_txoptions *opt = NULL; + struct ipv6_txoptions *opt_to_free = NULL; struct ip6_flowlabel *flowlabel = NULL; struct flowi6 fl6; struct dst_entry *dst; @@ -1109,8 +1109,10 @@ int udpv6_sendmsg(struct kiocb *iocb, struct sock *sk, opt = NULL; connected = 0; } - if (opt == NULL) - opt = np->opt; + if (!opt) { + opt = txopt_get(np); + opt_to_free = opt; + } if (flowlabel) opt = fl6_merge_options(&opt_space, flowlabel, opt); opt = ipv6_fixup_options(&opt_space, opt); @@ -1211,6 +1213,7 @@ int udpv6_sendmsg(struct kiocb *iocb, struct sock *sk, out: dst_release(dst); fl6_sock_release(flowlabel); + txopt_put(opt_to_free); if (!err) return len; /* diff --git a/net/irda/af_irda.c b/net/irda/af_irda.c index bb14c347768..f98f832d0bb 100644 --- a/net/irda/af_irda.c +++ b/net/irda/af_irda.c @@ -1106,6 +1106,9 @@ static int irda_create(struct net *net, struct socket *sock, int protocol, IRDA_DEBUG(2, "%s()\n", __func__); + if (protocol < 0 || protocol > SK_PROTOCOL_MAX) + return -EINVAL; + if (net != &init_net) return -EAFNOSUPPORT; diff --git a/net/llc/sysctl_net_llc.c b/net/llc/sysctl_net_llc.c index e2ebe358626..be078ec43d7 100644 --- a/net/llc/sysctl_net_llc.c +++ b/net/llc/sysctl_net_llc.c @@ -17,28 +17,28 @@ static struct ctl_table llc2_timeout_table[] = { { .procname = "ack", .data = &sysctl_llc2_ack_timeout, - .maxlen = sizeof(long), + .maxlen = sizeof(sysctl_llc2_ack_timeout), .mode = 0644, .proc_handler = proc_dointvec_jiffies, }, { .procname = "busy", .data = &sysctl_llc2_busy_timeout, - .maxlen = sizeof(long), + .maxlen = sizeof(sysctl_llc2_busy_timeout), .mode = 0644, .proc_handler = proc_dointvec_jiffies, }, { .procname = "p", .data = &sysctl_llc2_p_timeout, - .maxlen = sizeof(long), + .maxlen = sizeof(sysctl_llc2_p_timeout), .mode = 0644, .proc_handler = proc_dointvec_jiffies, }, { .procname = "rej", .data = &sysctl_llc2_rej_timeout, - .maxlen = sizeof(long), + .maxlen = sizeof(sysctl_llc2_rej_timeout), .mode = 0644, .proc_handler = proc_dointvec_jiffies, }, diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 38137cb5f6f..8c5b4508d41 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -242,6 +242,7 @@ struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata, return NULL; spin_lock_init(&sta->lock); + spin_lock_init(&sta->ps_lock); INIT_WORK(&sta->drv_unblock_wk, sta_unblock); INIT_WORK(&sta->ampdu_mlme.work, ieee80211_ba_session_work); mutex_init(&sta->ampdu_mlme.mtx); @@ -970,6 +971,8 @@ void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta) skb_queue_head_init(&pending); + /* sync with ieee80211_tx_h_unicast_ps_buf */ + spin_lock(&sta->ps_lock); /* Send all buffered frames to the station */ for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { int count = skb_queue_len(&pending), tmp; @@ -985,6 +988,7 @@ void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta) } ieee80211_add_pending_skbs_fn(local, &pending, clear_sta_ps_flags, sta); + spin_unlock(&sta->ps_lock); local->total_ps_buffered -= buffered; diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h index ab0576827ba..249f4d08793 100644 --- a/net/mac80211/sta_info.h +++ b/net/mac80211/sta_info.h @@ -227,6 +227,7 @@ struct sta_ampdu_mlme { * @drv_unblock_wk: used for driver PS unblocking * @listen_interval: listen interval of this station, when we're acting as AP * @_flags: STA flags, see &enum ieee80211_sta_info_flags, do not use directly + * @ps_lock: used for powersave (when mac80211 is the AP) related locking * @ps_tx_buf: buffers (per AC) of frames to transmit to this station * when it leaves power saving state or polls * @tx_filtered: buffers (per AC) of frames we already tried to @@ -297,10 +298,8 @@ struct sta_info { /* use the accessors defined below */ unsigned long _flags; - /* - * STA powersave frame queues, no more than the internal - * locking required. - */ + /* STA powersave lock and frame queues */ + spinlock_t ps_lock; struct sk_buff_head ps_tx_buf[IEEE80211_NUM_ACS]; struct sk_buff_head tx_filtered[IEEE80211_NUM_ACS]; unsigned long driver_buffered_tids; diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index e76facc69e9..6614a60800d 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -471,6 +471,20 @@ ieee80211_tx_h_unicast_ps_buf(struct ieee80211_tx_data *tx) #endif /* CONFIG_MAC80211_VERBOSE_PS_DEBUG */ if (tx->local->total_ps_buffered >= TOTAL_MAX_TX_BUFFER) purge_old_ps_buffers(tx->local); + + /* sync with ieee80211_sta_ps_deliver_wakeup */ + spin_lock(&sta->ps_lock); + /* + * STA woke up the meantime and all the frames on ps_tx_buf have + * been queued to pending queue. No reordering can happen, go + * ahead and Tx the packet. + */ + if (!test_sta_flag(sta, WLAN_STA_PS_STA) && + !test_sta_flag(sta, WLAN_STA_PS_DRIVER)) { + spin_unlock(&sta->ps_lock); + return TX_CONTINUE; + } + if (skb_queue_len(&sta->ps_tx_buf[ac]) >= STA_MAX_TX_BUFFER) { struct sk_buff *old = skb_dequeue(&sta->ps_tx_buf[ac]); #ifdef CONFIG_MAC80211_VERBOSE_PS_DEBUG @@ -487,6 +501,7 @@ ieee80211_tx_h_unicast_ps_buf(struct ieee80211_tx_data *tx) info->control.vif = &tx->sdata->vif; info->flags |= IEEE80211_TX_INTFL_NEED_TXPROCESSING; skb_queue_tail(&sta->ps_tx_buf[ac], tx->skb); + spin_unlock(&sta->ps_lock); if (!timer_pending(&local->sta_cleanup)) mod_timer(&local->sta_cleanup, @@ -801,11 +816,19 @@ ieee80211_tx_h_sequence(struct ieee80211_tx_data *tx) /* * Packet injection may want to control the sequence - * number, if we have no matching interface then we - * neither assign one ourselves nor ask the driver to. + * number, so if an injected packet is found, skip + * renumbering it. Also make the packet NO_ACK to avoid + * excessive retries (ACKing and retrying should be + * handled by the injecting application). + * FIXME This may break hostapd and some other injectors. + * This should be done using a radiotap flag. */ - if (unlikely(info->control.vif->type == NL80211_IFTYPE_MONITOR)) + if (unlikely((info->flags & IEEE80211_TX_CTL_INJECTED) && + !(tx->sdata->u.mntr_flags & MONITOR_FLAG_COOK_FRAMES))) { + if (!ieee80211_has_morefrags(hdr->frame_control)) + info->flags |= IEEE80211_TX_CTL_NO_ACK; return TX_CONTINUE; + } if (unlikely(ieee80211_is_ctl(hdr->frame_control))) return TX_CONTINUE; @@ -907,7 +930,7 @@ static int ieee80211_fragment(struct ieee80211_tx_data *tx, } /* adjust first fragment's length */ - skb->len = hdrlen + per_fragm; + skb_trim(skb, hdrlen + per_fragm); return 0; } diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index d7eafd5ba94..95a70d9bed5 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -179,7 +179,7 @@ EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); static void clean_from_lists(struct nf_conn *ct) { - pr_debug("clean_from_lists(%p)\n", ct); + pr_debug("clean_from_lists(%pK)\n", ct); hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode); @@ -194,7 +194,7 @@ destroy_conntrack(struct nf_conntrack *nfct) struct net *net = nf_ct_net(ct); struct nf_conntrack_l4proto *l4proto; - pr_debug("destroy_conntrack(%p)\n", ct); + pr_debug("destroy_conntrack(%pK)\n", ct); NF_CT_ASSERT(atomic_read(&nfct->use) == 0); NF_CT_ASSERT(!timer_pending(&ct->timeout)); @@ -227,7 +227,7 @@ destroy_conntrack(struct nf_conntrack *nfct) if (ct->master) nf_ct_put(ct->master); - pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct); + pr_debug("destroy_conntrack: returning ct=%pK to slab\n", ct); nf_conntrack_free(ct); } @@ -488,7 +488,7 @@ __nf_conntrack_confirm(struct sk_buff *skb) /* No external references means no one else could have confirmed us. */ NF_CT_ASSERT(!nf_ct_is_confirmed(ct)); - pr_debug("Confirming conntrack %p\n", ct); + pr_debug("Confirming conntrack %pK\n", ct); spin_lock_bh(&nf_conntrack_lock); @@ -816,7 +816,7 @@ init_conntrack(struct net *net, struct nf_conn *tmpl, spin_lock_bh(&nf_conntrack_lock); exp = nf_ct_find_expectation(net, zone, tuple); if (exp) { - pr_debug("conntrack: expectation arrives ct=%p exp=%p\n", + pr_debug("conntrack: expectation arrives ct=%pK exp=%pK\n", ct, exp); /* Welcome, Mr. Bond. We've been expecting you... */ __set_bit(IPS_EXPECTED_BIT, &ct->status); @@ -905,14 +905,14 @@ resolve_normal_ct(struct net *net, struct nf_conn *tmpl, } else { /* Once we've had two way comms, always ESTABLISHED. */ if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) { - pr_debug("nf_conntrack_in: normal packet for %p\n", ct); + pr_debug("nf_conntrack_in: normal packet for %pK\n", ct); *ctinfo = IP_CT_ESTABLISHED; } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) { - pr_debug("nf_conntrack_in: related packet for %p\n", + pr_debug("nf_conntrack_in: related packet for %pK\n", ct); *ctinfo = IP_CT_RELATED; } else { - pr_debug("nf_conntrack_in: new packet for %p\n", ct); + pr_debug("nf_conntrack_in: new packet for %pK\n", ct); *ctinfo = IP_CT_NEW; } *set_reply = 0; @@ -1059,7 +1059,7 @@ void nf_conntrack_alter_reply(struct nf_conn *ct, /* Should be unconfirmed, so not in hash table yet */ NF_CT_ASSERT(!nf_ct_is_confirmed(ct)); - pr_debug("Altering reply tuple of %p to ", ct); + pr_debug("Altering reply tuple of %pK to ", ct); nf_ct_dump_tuple(newreply); ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply; @@ -1560,7 +1560,7 @@ static int nf_conntrack_init_net(struct net *net) goto err_stat; } - net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net); + net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%pK", net); if (!net->ct.slabname) { ret = -ENOMEM; goto err_slabname; diff --git a/net/netfilter/nf_conntrack_proto_dccp.c b/net/netfilter/nf_conntrack_proto_dccp.c index 24fdce256cb..31dbf005076 100644 --- a/net/netfilter/nf_conntrack_proto_dccp.c +++ b/net/netfilter/nf_conntrack_proto_dccp.c @@ -431,7 +431,7 @@ static bool dccp_new(struct nf_conn *ct, const struct sk_buff *skb, const char *msg; u_int8_t state; - dh = skb_header_pointer(skb, dataoff, sizeof(_dh), &dh); + dh = skb_header_pointer(skb, dataoff, sizeof(_dh), &_dh); BUG_ON(dh == NULL); state = dccp_state_table[CT_DCCP_ROLE_CLIENT][dh->dccph_type][CT_DCCP_NONE]; @@ -488,7 +488,7 @@ static int dccp_packet(struct nf_conn *ct, const struct sk_buff *skb, u_int8_t type, old_state, new_state; enum ct_dccp_roles role; - dh = skb_header_pointer(skb, dataoff, sizeof(_dh), &dh); + dh = skb_header_pointer(skb, dataoff, sizeof(_dh), &_dh); BUG_ON(dh == NULL); type = dh->dccph_type; @@ -579,7 +579,7 @@ static int dccp_error(struct net *net, struct nf_conn *tmpl, unsigned int cscov; const char *msg; - dh = skb_header_pointer(skb, dataoff, sizeof(_dh), &dh); + dh = skb_header_pointer(skb, dataoff, sizeof(_dh), &_dh); if (dh == NULL) { msg = "nf_ct_dccp: short packet "; goto out_invalid; diff --git a/net/netfilter/nf_conntrack_proto_generic.c b/net/netfilter/nf_conntrack_proto_generic.c index 835e24c58f0..54e3900a49d 100644 --- a/net/netfilter/nf_conntrack_proto_generic.c +++ b/net/netfilter/nf_conntrack_proto_generic.c @@ -14,6 +14,30 @@ static unsigned int nf_ct_generic_timeout __read_mostly = 600*HZ; +static bool nf_generic_should_process(u8 proto) +{ + switch (proto) { +#ifdef CONFIG_NF_CT_PROTO_SCTP_MODULE + case IPPROTO_SCTP: + return false; +#endif +#ifdef CONFIG_NF_CT_PROTO_DCCP_MODULE + case IPPROTO_DCCP: + return false; +#endif +#ifdef CONFIG_NF_CT_PROTO_GRE_MODULE + case IPPROTO_GRE: + return false; +#endif +#ifdef CONFIG_NF_CT_PROTO_UDPLITE_MODULE + case IPPROTO_UDPLITE: + return false; +#endif + default: + return true; + } +} + static bool generic_pkt_to_tuple(const struct sk_buff *skb, unsigned int dataoff, struct nf_conntrack_tuple *tuple) @@ -62,7 +86,7 @@ static int generic_packet(struct nf_conn *ct, static bool generic_new(struct nf_conn *ct, const struct sk_buff *skb, unsigned int dataoff, unsigned int *timeouts) { - return true; + return nf_generic_should_process(nf_ct_protonum(ct)); } #if IS_ENABLED(CONFIG_NF_CT_NETLINK_TIMEOUT) diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c index cce326bdfaf..5c552593ef5 100644 --- a/net/netfilter/xt_IDLETIMER.c +++ b/net/netfilter/xt_IDLETIMER.c @@ -48,6 +48,7 @@ #include #include #include +#include struct idletimer_tg_attr { struct attribute attr; @@ -73,6 +74,7 @@ struct idletimer_tg { bool work_pending; bool send_nl_msg; bool active; + uid_t uid; }; static LIST_HEAD(idletimer_tg_list); @@ -117,7 +119,8 @@ static void notify_netlink_uevent(const char *iface, struct idletimer_tg *timer) char iface_msg[NLMSG_MAX_SIZE]; char state_msg[NLMSG_MAX_SIZE]; char timestamp_msg[NLMSG_MAX_SIZE]; - char *envp[] = { iface_msg, state_msg, timestamp_msg, NULL }; + char uid_msg[NLMSG_MAX_SIZE]; + char *envp[] = { iface_msg, state_msg, timestamp_msg, uid_msg, NULL }; int res; struct timespec ts; uint64_t time_ns; @@ -140,6 +143,16 @@ static void notify_netlink_uevent(const char *iface, struct idletimer_tg *timer) return; } + if (state) { + res = snprintf(uid_msg, NLMSG_MAX_SIZE, "UID=%u", timer->uid); + if (NLMSG_MAX_SIZE <= res) + pr_err("message too long (%d)", res); + } else { + res = snprintf(uid_msg, NLMSG_MAX_SIZE, "UID="); + if (NLMSG_MAX_SIZE <= res) + pr_err("message too long (%d)", res); + } + time_ns = timespec_to_ns(&ts); res = snprintf(timestamp_msg, NLMSG_MAX_SIZE, "TIME_NS=%llu", time_ns); if (NLMSG_MAX_SIZE <= res) { @@ -147,7 +160,8 @@ static void notify_netlink_uevent(const char *iface, struct idletimer_tg *timer) pr_err("message too long (%d)", res); } - pr_debug("putting nlmsg: <%s> <%s>\n", iface_msg, state_msg); + pr_debug("putting nlmsg: <%s> <%s> <%s> <%s>\n", iface_msg, state_msg, + timestamp_msg, uid_msg); kobject_uevent_env(idletimer_tg_kobj, KOBJ_CHANGE, envp); return; } @@ -296,6 +310,7 @@ static int idletimer_tg_create(struct idletimer_tg_info *info) info->timer->delayed_timer_trigger.tv_sec = 0; info->timer->delayed_timer_trigger.tv_nsec = 0; info->timer->work_pending = false; + info->timer->uid = 0; get_monotonic_boottime(&info->timer->last_modified_timer); info->timer->pm_nb.notifier_call = idletimer_resume; @@ -319,7 +334,8 @@ static int idletimer_tg_create(struct idletimer_tg_info *info) return ret; } -static void reset_timer(const struct idletimer_tg_info *info) +static void reset_timer(const struct idletimer_tg_info *info, + struct sk_buff *skb) { unsigned long now = jiffies; struct idletimer_tg *timer = info->timer; @@ -332,6 +348,17 @@ static void reset_timer(const struct idletimer_tg_info *info) if (!timer_prev || time_before(timer->timer.expires, now)) { pr_debug("Starting Checkentry timer (Expired, Jiffies): %lu, %lu\n", timer->timer.expires, now); + + /* Stores the uid resposible for waking up the radio */ + if (skb && (skb->sk)) { + struct sock *sk = skb->sk; + read_lock_bh(&sk->sk_callback_lock); + if ((sk->sk_socket) && (sk->sk_socket->file) && + (sk->sk_socket->file->f_cred)) + timer->uid = sk->sk_socket->file->f_cred->uid; + read_unlock_bh(&sk->sk_callback_lock); + } + /* checks if there is a pending inactive notification*/ if (timer->work_pending) timer->delayed_timer_trigger = timer->last_modified_timer; @@ -360,7 +387,7 @@ static unsigned int idletimer_tg_target(struct sk_buff *skb, BUG_ON(!info->timer); /* TODO: Avoid modifying timers on each packet */ - reset_timer(info); + reset_timer(info, skb); return XT_CONTINUE; } @@ -387,7 +414,7 @@ static int idletimer_tg_checkentry(const struct xt_tgchk_param *par) info->timer = __idletimer_tg_find_by_label(info->label); if (info->timer) { info->timer->refcnt++; - reset_timer(info); + reset_timer(info, NULL); pr_debug("increased refcnt of timer %s to %u\n", info->label, info->timer->refcnt); } else { diff --git a/net/netfilter/xt_qtaguid.c b/net/netfilter/xt_qtaguid.c index f6de15ae1f5..5c491f8fc70 100644 --- a/net/netfilter/xt_qtaguid.c +++ b/net/netfilter/xt_qtaguid.c @@ -1972,7 +1972,7 @@ static int qtaguid_ctrl_proc_read(char *page, char **num_items_returned, f_count = atomic_long_read( &sock_tag_entry->socket->file->f_count); len = snprintf(outp, char_count, - "sock=%p tag=0x%llx (uid=%u) pid=%u " + "sock=%pK tag=0x%llx (uid=%u) pid=%u " "f_count=%lu\n", sock_tag_entry->sk, sock_tag_entry->tag, uid, @@ -2592,8 +2592,7 @@ static int pp_stats_line(struct proc_print_info *ppi, int cnt_set) tag_t tag = ppi->ts_entry->tn.tag; uid_t stat_uid = get_uid_from_tag(tag); /* Detailed tags are not available to everybody */ - if (get_atag_from_tag(tag) - && !can_read_other_uid_stats(stat_uid)) { + if (!can_read_other_uid_stats(stat_uid)) { CT_DEBUG("qtaguid: stats line: " "%s 0x%llx %u: insufficient priv " "from pid=%u tgid=%u uid=%u stats.gid=%u\n", diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 4f2c0df7956..f15a3eb25ce 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3112,19 +3112,25 @@ packet_setsockopt(struct socket *sock, int level, int optname, char __user *optv if (optlen != sizeof(val)) return -EINVAL; - if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) - return -EBUSY; if (copy_from_user(&val, optval, sizeof(val))) return -EFAULT; switch (val) { case TPACKET_V1: case TPACKET_V2: case TPACKET_V3: - po->tp_version = val; - return 0; + break; default: return -EINVAL; } + lock_sock(sk); + if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { + ret = -EBUSY; + } else { + po->tp_version = val; + ret = 0; + } + release_sock(sk); + return ret; } case PACKET_RESERVE: { @@ -3595,6 +3601,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, /* Added to avoid minimal code churn */ struct tpacket_req *req = &req_u->req; + lock_sock(sk); /* Opening a Tx-ring is NOT supported in TPACKET_V3 */ if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) { WARN(1, "Tx-ring is not supported.\n"); @@ -3672,7 +3679,6 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, goto out; } - lock_sock(sk); /* Detach socket from network */ spin_lock(&po->bind_lock); @@ -3721,11 +3727,11 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, if (!tx_ring) prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue); } - release_sock(sk); if (pg_vec) free_pg_vec(pg_vec, order, req->tp_block_nr); out: + release_sock(sk); return err; } diff --git a/net/rxrpc/ar-recvmsg.c b/net/rxrpc/ar-recvmsg.c index 4b48687c389..c581029c1a9 100644 --- a/net/rxrpc/ar-recvmsg.c +++ b/net/rxrpc/ar-recvmsg.c @@ -182,7 +182,8 @@ int rxrpc_recvmsg(struct kiocb *iocb, struct socket *sock, msg->msg_iov, copy); } else { ret = skb_copy_and_csum_datagram_iovec(skb, offset, - msg->msg_iov); + msg->msg_iov, + copy); if (ret == -EINVAL) goto csum_copy_error; } diff --git a/net/socket.c b/net/socket.c index 851edcd6b09..b3e305287bf 100644 --- a/net/socket.c +++ b/net/socket.c @@ -1703,6 +1703,8 @@ SYSCALL_DEFINE6(sendto, int, fd, void __user *, buff, size_t, len, if (len > INT_MAX) len = INT_MAX; + if (unlikely(!access_ok(VERIFY_READ, buff, len))) + return -EFAULT; sock = sockfd_lookup_light(fd, &err, &fput_needed); if (!sock) goto out; @@ -1762,6 +1764,8 @@ SYSCALL_DEFINE6(recvfrom, int, fd, void __user *, ubuf, size_t, size, if (size > INT_MAX) size = INT_MAX; + if (unlikely(!access_ok(VERIFY_WRITE, ubuf, size))) + return -EFAULT; sock = sockfd_lookup_light(fd, &err, &fput_needed); if (!sock) goto out; @@ -2286,31 +2290,31 @@ int __sys_recvmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen, break; } -out_put: - fput_light(sock->file, fput_needed); - if (err == 0) - return datagrams; + goto out_put; + + if (datagrams == 0) { + datagrams = err; + goto out_put; + } - if (datagrams != 0) { + /* + * We may return less entries than requested (vlen) if the + * sock is non block and there aren't enough datagrams... + */ + if (err != -EAGAIN) { /* - * We may return less entries than requested (vlen) if the - * sock is non block and there aren't enough datagrams... + * ... or if recvmsg returns an error after we + * received some datagrams, where we record the + * error to return on the next call or if the + * app asks about it using getsockopt(SO_ERROR). */ - if (err != -EAGAIN) { - /* - * ... or if recvmsg returns an error after we - * received some datagrams, where we record the - * error to return on the next call or if the - * app asks about it using getsockopt(SO_ERROR). - */ - sock->sk->sk_err = -err; - } - - return datagrams; + sock->sk->sk_err = -err; } +out_put: + fput_light(sock->file, fput_needed); - return err; + return datagrams; } SYSCALL_DEFINE5(recvmmsg, int, fd, struct mmsghdr __user *, mmsg, diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 8d932a5c534..43d56d01f52 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -307,6 +307,118 @@ static struct sock *unix_find_socket_byinode(struct inode *i) return s; } +/* Support code for asymmetrically connected dgram sockets + * + * If a datagram socket is connected to a socket not itself connected + * to the first socket (eg, /dev/log), clients may only enqueue more + * messages if the present receive queue of the server socket is not + * "too large". This means there's a second writeability condition + * poll and sendmsg need to test. The dgram recv code will do a wake + * up on the peer_wait wait queue of a socket upon reception of a + * datagram which needs to be propagated to sleeping would-be writers + * since these might not have sent anything so far. This can't be + * accomplished via poll_wait because the lifetime of the server + * socket might be less than that of its clients if these break their + * association with it or if the server socket is closed while clients + * are still connected to it and there's no way to inform "a polling + * implementation" that it should let go of a certain wait queue + * + * In order to propagate a wake up, a wait_queue_t of the client + * socket is enqueued on the peer_wait queue of the server socket + * whose wake function does a wake_up on the ordinary client socket + * wait queue. This connection is established whenever a write (or + * poll for write) hit the flow control condition and broken when the + * association to the server socket is dissolved or after a wake up + * was relayed. + */ + +static int unix_dgram_peer_wake_relay(wait_queue_t *q, unsigned mode, int flags, + void *key) +{ + struct unix_sock *u; + wait_queue_head_t *u_sleep; + + u = container_of(q, struct unix_sock, peer_wake); + + __remove_wait_queue(&unix_sk(u->peer_wake.private)->peer_wait, + q); + u->peer_wake.private = NULL; + + /* relaying can only happen while the wq still exists */ + u_sleep = sk_sleep(&u->sk); + if (u_sleep) + wake_up_interruptible_poll(u_sleep, key); + + return 0; +} + +static int unix_dgram_peer_wake_connect(struct sock *sk, struct sock *other) +{ + struct unix_sock *u, *u_other; + int rc; + + u = unix_sk(sk); + u_other = unix_sk(other); + rc = 0; + spin_lock(&u_other->peer_wait.lock); + + if (!u->peer_wake.private) { + u->peer_wake.private = other; + __add_wait_queue(&u_other->peer_wait, &u->peer_wake); + + rc = 1; + } + + spin_unlock(&u_other->peer_wait.lock); + return rc; +} + +static void unix_dgram_peer_wake_disconnect(struct sock *sk, + struct sock *other) +{ + struct unix_sock *u, *u_other; + + u = unix_sk(sk); + u_other = unix_sk(other); + spin_lock(&u_other->peer_wait.lock); + + if (u->peer_wake.private == other) { + __remove_wait_queue(&u_other->peer_wait, &u->peer_wake); + u->peer_wake.private = NULL; + } + + spin_unlock(&u_other->peer_wait.lock); +} + +static void unix_dgram_peer_wake_disconnect_wakeup(struct sock *sk, + struct sock *other) +{ + unix_dgram_peer_wake_disconnect(sk, other); + wake_up_interruptible_poll(sk_sleep(sk), + POLLOUT | + POLLWRNORM | + POLLWRBAND); +} + +/* preconditions: + * - unix_peer(sk) == other + * - association is stable + */ +static int unix_dgram_peer_wake_me(struct sock *sk, struct sock *other) +{ + int connected; + + connected = unix_dgram_peer_wake_connect(sk, other); + + if (unix_recvq_full(other)) + return 1; + + if (connected) + unix_dgram_peer_wake_disconnect(sk, other); + + return 0; +} + static inline int unix_writable(struct sock *sk) { return (atomic_read(&sk->sk_wmem_alloc) << 2) <= sk->sk_sndbuf; @@ -411,6 +523,8 @@ static int unix_release_sock(struct sock *sk, int embrion) skpair->sk_state_change(skpair); sk_wake_async(skpair, SOCK_WAKE_WAITD, POLL_HUP); } + + unix_dgram_peer_wake_disconnect(sk, skpair); sock_put(skpair); /* It may now die */ unix_peer(sk) = NULL; } @@ -645,6 +759,7 @@ static struct sock *unix_create1(struct net *net, struct socket *sock) INIT_LIST_HEAD(&u->link); mutex_init(&u->readlock); /* single task reading lock */ init_waitqueue_head(&u->peer_wait); + init_waitqueue_func_entry(&u->peer_wake, unix_dgram_peer_wake_relay); unix_insert_socket(unix_sockets_unbound, sk); out: if (sk == NULL) @@ -1014,6 +1129,8 @@ static int unix_dgram_connect(struct socket *sock, struct sockaddr *addr, if (unix_peer(sk)) { struct sock *old_peer = unix_peer(sk); unix_peer(sk) = other; + unix_dgram_peer_wake_disconnect_wakeup(sk, old_peer); + unix_state_double_unlock(sk, other); if (other != old_peer) @@ -1336,7 +1453,7 @@ static void unix_detach_fds(struct scm_cookie *scm, struct sk_buff *skb) UNIXCB(skb).fp = NULL; for (i = scm->fp->count-1; i >= 0; i--) - unix_notinflight(scm->fp->fp[i]); + unix_notinflight(scm->fp->user, scm->fp->fp[i]); } static void unix_destruct_scm(struct sk_buff *skb) @@ -1354,6 +1471,21 @@ static void unix_destruct_scm(struct sk_buff *skb) sock_wfree(skb); } +/* + * The "user->unix_inflight" variable is protected by the garbage + * collection lock, and we just read it locklessly here. If you go + * over the limit, there might be a tiny race in actually noticing + * it across threads. Tough. + */ +static inline bool too_many_unix_fds(struct task_struct *p) +{ + struct user_struct *user = current_user(); + + if (unlikely(user->unix_inflight > task_rlimit(p, RLIMIT_NOFILE))) + return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN); + return false; +} + #define MAX_RECURSION_LEVEL 4 static int unix_attach_fds(struct scm_cookie *scm, struct sk_buff *skb) @@ -1362,6 +1494,9 @@ static int unix_attach_fds(struct scm_cookie *scm, struct sk_buff *skb) unsigned char max_level = 0; int unix_sock_count = 0; + if (too_many_unix_fds(current)) + return -ETOOMANYREFS; + for (i = scm->fp->count - 1; i >= 0; i--) { struct sock *sk = unix_get_socket(scm->fp->fp[i]); @@ -1383,10 +1518,8 @@ static int unix_attach_fds(struct scm_cookie *scm, struct sk_buff *skb) if (!UNIXCB(skb).fp) return -ENOMEM; - if (unix_sock_count) { - for (i = scm->fp->count - 1; i >= 0; i--) - unix_inflight(scm->fp->fp[i]); - } + for (i = scm->fp->count - 1; i >= 0; i--) + unix_inflight(scm->fp->user, scm->fp->fp[i]); return max_level; } @@ -1443,6 +1576,7 @@ static int unix_dgram_sendmsg(struct kiocb *kiocb, struct socket *sock, long timeo; struct scm_cookie tmp_scm; int max_level; + int sk_locked; if (NULL == siocb->scm) siocb->scm = &tmp_scm; @@ -1511,12 +1645,14 @@ static int unix_dgram_sendmsg(struct kiocb *kiocb, struct socket *sock, goto out_free; } + sk_locked = 0; unix_state_lock(other); +restart_locked: err = -EPERM; if (!unix_may_send(sk, other)) goto out_unlock; - if (sock_flag(other, SOCK_DEAD)) { + if (unlikely(sock_flag(other, SOCK_DEAD))) { /* * Check with 1003.1g - what should * datagram error @@ -1524,10 +1660,14 @@ static int unix_dgram_sendmsg(struct kiocb *kiocb, struct socket *sock, unix_state_unlock(other); sock_put(other); + if (!sk_locked) + unix_state_lock(sk); + err = 0; - unix_state_lock(sk); if (unix_peer(sk) == other) { unix_peer(sk) = NULL; + unix_dgram_peer_wake_disconnect_wakeup(sk, other); + unix_state_unlock(sk); unix_dgram_disconnected(sk, other); @@ -1553,21 +1693,38 @@ static int unix_dgram_sendmsg(struct kiocb *kiocb, struct socket *sock, goto out_unlock; } - if (unix_peer(other) != sk && unix_recvq_full(other)) { - if (!timeo) { - err = -EAGAIN; - goto out_unlock; + if (unlikely(unix_peer(other) != sk && unix_recvq_full(other))) { + if (timeo) { + timeo = unix_wait_for_peer(other, timeo); + + err = sock_intr_errno(timeo); + if (signal_pending(current)) + goto out_free; + + goto restart; } - timeo = unix_wait_for_peer(other, timeo); + if (!sk_locked) { + unix_state_unlock(other); + unix_state_double_lock(sk, other); + } - err = sock_intr_errno(timeo); - if (signal_pending(current)) - goto out_free; + if (unix_peer(sk) != other || + unix_dgram_peer_wake_me(sk, other)) { + err = -EAGAIN; + sk_locked = 1; + goto out_unlock; + } - goto restart; + if (!sk_locked) { + sk_locked = 1; + goto restart_locked; + } } + if (unlikely(sk_locked)) + unix_state_unlock(sk); + if (sock_flag(other, SOCK_RCVTSTAMP)) __net_timestamp(skb); maybe_add_creds(skb, sock, other); @@ -1581,6 +1738,8 @@ static int unix_dgram_sendmsg(struct kiocb *kiocb, struct socket *sock, return len; out_unlock: + if (sk_locked) + unix_state_unlock(sk); unix_state_unlock(other); out_free: kfree_skb(skb); @@ -2211,14 +2370,16 @@ static unsigned int unix_dgram_poll(struct file *file, struct socket *sock, return mask; writable = unix_writable(sk); - other = unix_peer_get(sk); - if (other) { - if (unix_peer(other) != sk) { - sock_poll_wait(file, &unix_sk(other)->peer_wait, wait); - if (unix_recvq_full(other)) - writable = 0; - } - sock_put(other); + if (writable) { + unix_state_lock(sk); + + other = unix_peer(sk); + if (other && unix_peer(other) != sk && + unix_recvq_full(other) && + unix_dgram_peer_wake_me(sk, other)) + writable = 0; + + unix_state_unlock(sk); } if (writable) diff --git a/net/unix/garbage.c b/net/unix/garbage.c index b6f4b994eb3..d04a2eb8270 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -122,12 +122,14 @@ struct sock *unix_get_socket(struct file *filp) * descriptor if it is for an AF_UNIX socket. */ -void unix_inflight(struct file *fp) +void unix_inflight(struct user_struct *user, struct file *fp) { struct sock *s = unix_get_socket(fp); + + spin_lock(&unix_gc_lock); + if (s) { struct unix_sock *u = unix_sk(s); - spin_lock(&unix_gc_lock); if (atomic_long_inc_return(&u->inflight) == 1) { BUG_ON(!list_empty(&u->link)); list_add_tail(&u->link, &gc_inflight_list); @@ -135,22 +137,26 @@ void unix_inflight(struct file *fp) BUG_ON(list_empty(&u->link)); } unix_tot_inflight++; - spin_unlock(&unix_gc_lock); } + user->unix_inflight++; + spin_unlock(&unix_gc_lock); } -void unix_notinflight(struct file *fp) +void unix_notinflight(struct user_struct *user, struct file *fp) { struct sock *s = unix_get_socket(fp); + + spin_lock(&unix_gc_lock); + if (s) { struct unix_sock *u = unix_sk(s); - spin_lock(&unix_gc_lock); BUG_ON(list_empty(&u->link)); if (atomic_long_dec_and_test(&u->inflight)) list_del_init(&u->link); unix_tot_inflight--; - spin_unlock(&unix_gc_lock); } + user->unix_inflight--; + spin_unlock(&unix_gc_lock); } static void scan_inflight(struct sock *x, void (*func)(struct unix_sock *), diff --git a/samples/Makefile b/samples/Makefile index 2f75851ec62..5ef08bba96c 100644 --- a/samples/Makefile +++ b/samples/Makefile @@ -1,4 +1,4 @@ # Makefile for Linux samples code obj-$(CONFIG_SAMPLES) += kobject/ kprobes/ tracepoints/ trace_events/ \ - hw_breakpoint/ kfifo/ kdb/ hidraw/ rpmsg/ + hw_breakpoint/ kfifo/ kdb/ hidraw/ rpmsg/ seccomp/ diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile new file mode 100644 index 00000000000..16aa2d42498 --- /dev/null +++ b/samples/seccomp/Makefile @@ -0,0 +1,32 @@ +# kbuild trick to avoid linker error. Can be omitted if a module is built. +obj- := dummy.o + +hostprogs-$(CONFIG_SECCOMP_FILTER) := bpf-fancy dropper bpf-direct + +HOSTCFLAGS_bpf-fancy.o += -I$(objtree)/usr/include +HOSTCFLAGS_bpf-fancy.o += -idirafter $(objtree)/include +HOSTCFLAGS_bpf-helper.o += -I$(objtree)/usr/include +HOSTCFLAGS_bpf-helper.o += -idirafter $(objtree)/include +bpf-fancy-objs := bpf-fancy.o bpf-helper.o + +HOSTCFLAGS_dropper.o += -I$(objtree)/usr/include +HOSTCFLAGS_dropper.o += -idirafter $(objtree)/include +dropper-objs := dropper.o + +HOSTCFLAGS_bpf-direct.o += -I$(objtree)/usr/include +HOSTCFLAGS_bpf-direct.o += -idirafter $(objtree)/include +bpf-direct-objs := bpf-direct.o + +# Try to match the kernel target. +ifeq ($(CONFIG_64BIT),) +HOSTCFLAGS_bpf-direct.o += -m32 +HOSTCFLAGS_dropper.o += -m32 +HOSTCFLAGS_bpf-helper.o += -m32 +HOSTCFLAGS_bpf-fancy.o += -m32 +HOSTLOADLIBES_bpf-direct += -m32 +HOSTLOADLIBES_bpf-fancy += -m32 +HOSTLOADLIBES_dropper += -m32 +endif + +# Tell kbuild to always build the programs +always := $(hostprogs-y) diff --git a/samples/seccomp/bpf-direct.c b/samples/seccomp/bpf-direct.c new file mode 100644 index 00000000000..151ec3f5218 --- /dev/null +++ b/samples/seccomp/bpf-direct.c @@ -0,0 +1,190 @@ +/* + * Seccomp filter example for x86 (32-bit and 64-bit) with BPF macros + * + * Copyright (c) 2012 The Chromium OS Authors + * Author: Will Drewry + * + * The code may be used by anyone for any purpose, + * and can serve as a starting point for developing + * applications using prctl(PR_SET_SECCOMP, 2, ...). + */ +#if defined(__i386__) || defined(__x86_64__) +#define SUPPORTED_ARCH 1 +#endif + +#if defined(SUPPORTED_ARCH) +#define __USE_GNU 1 +#define _GNU_SOURCE 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define syscall_arg(_n) (offsetof(struct seccomp_data, args[_n])) +#define syscall_nr (offsetof(struct seccomp_data, nr)) + +#if defined(__i386__) +#define REG_RESULT REG_EAX +#define REG_SYSCALL REG_EAX +#define REG_ARG0 REG_EBX +#define REG_ARG1 REG_ECX +#define REG_ARG2 REG_EDX +#define REG_ARG3 REG_ESI +#define REG_ARG4 REG_EDI +#define REG_ARG5 REG_EBP +#elif defined(__x86_64__) +#define REG_RESULT REG_RAX +#define REG_SYSCALL REG_RAX +#define REG_ARG0 REG_RDI +#define REG_ARG1 REG_RSI +#define REG_ARG2 REG_RDX +#define REG_ARG3 REG_R10 +#define REG_ARG4 REG_R8 +#define REG_ARG5 REG_R9 +#endif + +#ifndef PR_SET_NO_NEW_PRIVS +#define PR_SET_NO_NEW_PRIVS 38 +#endif + +#ifndef SYS_SECCOMP +#define SYS_SECCOMP 1 +#endif + +static void emulator(int nr, siginfo_t *info, void *void_context) +{ + ucontext_t *ctx = (ucontext_t *)(void_context); + int syscall; + char *buf; + ssize_t bytes; + size_t len; + if (info->si_code != SYS_SECCOMP) + return; + if (!ctx) + return; + syscall = ctx->uc_mcontext.gregs[REG_SYSCALL]; + buf = (char *) ctx->uc_mcontext.gregs[REG_ARG1]; + len = (size_t) ctx->uc_mcontext.gregs[REG_ARG2]; + + if (syscall != __NR_write) + return; + if (ctx->uc_mcontext.gregs[REG_ARG0] != STDERR_FILENO) + return; + /* Redirect stderr messages to stdout. Doesn't handle EINTR, etc */ + ctx->uc_mcontext.gregs[REG_RESULT] = -1; + if (write(STDOUT_FILENO, "[ERR] ", 6) > 0) { + bytes = write(STDOUT_FILENO, buf, len); + ctx->uc_mcontext.gregs[REG_RESULT] = bytes; + } + return; +} + +static int install_emulator(void) +{ + struct sigaction act; + sigset_t mask; + memset(&act, 0, sizeof(act)); + sigemptyset(&mask); + sigaddset(&mask, SIGSYS); + + act.sa_sigaction = &emulator; + act.sa_flags = SA_SIGINFO; + if (sigaction(SIGSYS, &act, NULL) < 0) { + perror("sigaction"); + return -1; + } + if (sigprocmask(SIG_UNBLOCK, &mask, NULL)) { + perror("sigprocmask"); + return -1; + } + return 0; +} + +static int install_filter(void) +{ + struct sock_filter filter[] = { + /* Grab the system call number */ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, syscall_nr), + /* Jump table for the allowed syscalls */ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, __NR_rt_sigreturn, 0, 1), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), +#ifdef __NR_sigreturn + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, __NR_sigreturn, 0, 1), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), +#endif + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, __NR_exit_group, 0, 1), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, __NR_exit, 0, 1), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, __NR_read, 1, 0), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, __NR_write, 3, 2), + + /* Check that read is only using stdin. */ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, syscall_arg(0)), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, STDIN_FILENO, 4, 0), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_KILL), + + /* Check that write is only using stdout */ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, syscall_arg(0)), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, STDOUT_FILENO, 1, 0), + /* Trap attempts to write to stderr */ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, STDERR_FILENO, 1, 2), + + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_TRAP), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_KILL), + }; + struct sock_fprog prog = { + .len = (unsigned short)(sizeof(filter)/sizeof(filter[0])), + .filter = filter, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { + perror("prctl(NO_NEW_PRIVS)"); + return 1; + } + + + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) { + perror("prctl"); + return 1; + } + return 0; +} + +#define payload(_c) (_c), sizeof((_c)) +int main(int argc, char **argv) +{ + char buf[4096]; + ssize_t bytes = 0; + if (install_emulator()) + return 1; + if (install_filter()) + return 1; + syscall(__NR_write, STDOUT_FILENO, + payload("OHAI! WHAT IS YOUR NAME? ")); + bytes = syscall(__NR_read, STDIN_FILENO, buf, sizeof(buf)); + syscall(__NR_write, STDOUT_FILENO, payload("HELLO, ")); + syscall(__NR_write, STDOUT_FILENO, buf, bytes); + syscall(__NR_write, STDERR_FILENO, + payload("Error message going to STDERR\n")); + return 0; +} +#else /* SUPPORTED_ARCH */ +/* + * This sample is x86-only. Since kernel samples are compiled with the + * host toolchain, a non-x86 host will result in using only the main() + * below. + */ +int main(void) +{ + return 1; +} +#endif /* SUPPORTED_ARCH */ diff --git a/samples/seccomp/bpf-fancy.c b/samples/seccomp/bpf-fancy.c new file mode 100644 index 00000000000..8eb483aaec4 --- /dev/null +++ b/samples/seccomp/bpf-fancy.c @@ -0,0 +1,102 @@ +/* + * Seccomp BPF example using a macro-based generator. + * + * Copyright (c) 2012 The Chromium OS Authors + * Author: Will Drewry + * + * The code may be used by anyone for any purpose, + * and can serve as a starting point for developing + * applications using prctl(PR_ATTACH_SECCOMP_FILTER). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "bpf-helper.h" + +#ifndef PR_SET_NO_NEW_PRIVS +#define PR_SET_NO_NEW_PRIVS 38 +#endif + +int main(int argc, char **argv) +{ + struct bpf_labels l; + static const char msg1[] = "Please type something: "; + static const char msg2[] = "You typed: "; + char buf[256]; + struct sock_filter filter[] = { + /* TODO: LOAD_SYSCALL_NR(arch) and enforce an arch */ + LOAD_SYSCALL_NR, + SYSCALL(__NR_exit, ALLOW), + SYSCALL(__NR_exit_group, ALLOW), + SYSCALL(__NR_write, JUMP(&l, write_fd)), + SYSCALL(__NR_read, JUMP(&l, read)), + DENY, /* Don't passthrough into a label */ + + LABEL(&l, read), + ARG(0), + JNE(STDIN_FILENO, DENY), + ARG(1), + JNE((unsigned long)buf, DENY), + ARG(2), + JGE(sizeof(buf), DENY), + ALLOW, + + LABEL(&l, write_fd), + ARG(0), + JEQ(STDOUT_FILENO, JUMP(&l, write_buf)), + JEQ(STDERR_FILENO, JUMP(&l, write_buf)), + DENY, + + LABEL(&l, write_buf), + ARG(1), + JEQ((unsigned long)msg1, JUMP(&l, msg1_len)), + JEQ((unsigned long)msg2, JUMP(&l, msg2_len)), + JEQ((unsigned long)buf, JUMP(&l, buf_len)), + DENY, + + LABEL(&l, msg1_len), + ARG(2), + JLT(sizeof(msg1), ALLOW), + DENY, + + LABEL(&l, msg2_len), + ARG(2), + JLT(sizeof(msg2), ALLOW), + DENY, + + LABEL(&l, buf_len), + ARG(2), + JLT(sizeof(buf), ALLOW), + DENY, + }; + struct sock_fprog prog = { + .filter = filter, + .len = (unsigned short)(sizeof(filter)/sizeof(filter[0])), + }; + ssize_t bytes; + bpf_resolve_jumps(&l, filter, sizeof(filter)/sizeof(*filter)); + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { + perror("prctl(NO_NEW_PRIVS)"); + return 1; + } + + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) { + perror("prctl(SECCOMP)"); + return 1; + } + syscall(__NR_write, STDOUT_FILENO, msg1, strlen(msg1)); + bytes = syscall(__NR_read, STDIN_FILENO, buf, sizeof(buf)-1); + bytes = (bytes > 0 ? bytes : 0); + syscall(__NR_write, STDERR_FILENO, msg2, strlen(msg2)); + syscall(__NR_write, STDERR_FILENO, buf, bytes); + /* Now get killed */ + syscall(__NR_write, STDERR_FILENO, msg2, strlen(msg2)+2); + return 0; +} diff --git a/samples/seccomp/bpf-helper.c b/samples/seccomp/bpf-helper.c new file mode 100644 index 00000000000..579cfe33188 --- /dev/null +++ b/samples/seccomp/bpf-helper.c @@ -0,0 +1,89 @@ +/* + * Seccomp BPF helper functions + * + * Copyright (c) 2012 The Chromium OS Authors + * Author: Will Drewry + * + * The code may be used by anyone for any purpose, + * and can serve as a starting point for developing + * applications using prctl(PR_ATTACH_SECCOMP_FILTER). + */ + +#include +#include + +#include "bpf-helper.h" + +int bpf_resolve_jumps(struct bpf_labels *labels, + struct sock_filter *filter, size_t count) +{ + struct sock_filter *begin = filter; + __u8 insn = count - 1; + + if (count < 1) + return -1; + /* + * Walk it once, backwards, to build the label table and do fixups. + * Since backward jumps are disallowed by BPF, this is easy. + */ + filter += insn; + for (; filter >= begin; --insn, --filter) { + if (filter->code != (BPF_JMP+BPF_JA)) + continue; + switch ((filter->jt<<8)|filter->jf) { + case (JUMP_JT<<8)|JUMP_JF: + if (labels->labels[filter->k].location == 0xffffffff) { + fprintf(stderr, "Unresolved label: '%s'\n", + labels->labels[filter->k].label); + return 1; + } + filter->k = labels->labels[filter->k].location - + (insn + 1); + filter->jt = 0; + filter->jf = 0; + continue; + case (LABEL_JT<<8)|LABEL_JF: + if (labels->labels[filter->k].location != 0xffffffff) { + fprintf(stderr, "Duplicate label use: '%s'\n", + labels->labels[filter->k].label); + return 1; + } + labels->labels[filter->k].location = insn; + filter->k = 0; /* fall through */ + filter->jt = 0; + filter->jf = 0; + continue; + } + } + return 0; +} + +/* Simple lookup table for labels. */ +__u32 seccomp_bpf_label(struct bpf_labels *labels, const char *label) +{ + struct __bpf_label *begin = labels->labels, *end; + int id; + if (labels->count == 0) { + begin->label = label; + begin->location = 0xffffffff; + labels->count++; + return 0; + } + end = begin + labels->count; + for (id = 0; begin < end; ++begin, ++id) { + if (!strcmp(label, begin->label)) + return id; + } + begin->label = label; + begin->location = 0xffffffff; + labels->count++; + return id; +} + +void seccomp_bpf_print(struct sock_filter *filter, size_t count) +{ + struct sock_filter *end = filter + count; + for ( ; filter < end; ++filter) + printf("{ code=%u,jt=%u,jf=%u,k=%u },\n", + filter->code, filter->jt, filter->jf, filter->k); +} diff --git a/samples/seccomp/bpf-helper.h b/samples/seccomp/bpf-helper.h new file mode 100644 index 00000000000..643279dd30f --- /dev/null +++ b/samples/seccomp/bpf-helper.h @@ -0,0 +1,238 @@ +/* + * Example wrapper around BPF macros. + * + * Copyright (c) 2012 The Chromium OS Authors + * Author: Will Drewry + * + * The code may be used by anyone for any purpose, + * and can serve as a starting point for developing + * applications using prctl(PR_SET_SECCOMP, 2, ...). + * + * No guarantees are provided with respect to the correctness + * or functionality of this code. + */ +#ifndef __BPF_HELPER_H__ +#define __BPF_HELPER_H__ + +#include /* for __BITS_PER_LONG */ +#include +#include +#include /* for seccomp_data */ +#include +#include +#include + +#define BPF_LABELS_MAX 256 +struct bpf_labels { + int count; + struct __bpf_label { + const char *label; + __u32 location; + } labels[BPF_LABELS_MAX]; +}; + +int bpf_resolve_jumps(struct bpf_labels *labels, + struct sock_filter *filter, size_t count); +__u32 seccomp_bpf_label(struct bpf_labels *labels, const char *label); +void seccomp_bpf_print(struct sock_filter *filter, size_t count); + +#define JUMP_JT 0xff +#define JUMP_JF 0xff +#define LABEL_JT 0xfe +#define LABEL_JF 0xfe + +#define ALLOW \ + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW) +#define DENY \ + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_KILL) +#define JUMP(labels, label) \ + BPF_JUMP(BPF_JMP+BPF_JA, FIND_LABEL((labels), (label)), \ + JUMP_JT, JUMP_JF) +#define LABEL(labels, label) \ + BPF_JUMP(BPF_JMP+BPF_JA, FIND_LABEL((labels), (label)), \ + LABEL_JT, LABEL_JF) +#define SYSCALL(nr, jt) \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (nr), 0, 1), \ + jt + +/* Lame, but just an example */ +#define FIND_LABEL(labels, label) seccomp_bpf_label((labels), #label) + +#define EXPAND(...) __VA_ARGS__ +/* Map all width-sensitive operations */ +#if __BITS_PER_LONG == 32 + +#define JEQ(x, jt) JEQ32(x, EXPAND(jt)) +#define JNE(x, jt) JNE32(x, EXPAND(jt)) +#define JGT(x, jt) JGT32(x, EXPAND(jt)) +#define JLT(x, jt) JLT32(x, EXPAND(jt)) +#define JGE(x, jt) JGE32(x, EXPAND(jt)) +#define JLE(x, jt) JLE32(x, EXPAND(jt)) +#define JA(x, jt) JA32(x, EXPAND(jt)) +#define ARG(i) ARG_32(i) +#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) + +#elif __BITS_PER_LONG == 64 + +/* Ensure that we load the logically correct offset. */ +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define ENDIAN(_lo, _hi) _lo, _hi +#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) +#define HI_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) + sizeof(__u32) +#elif __BYTE_ORDER == __BIG_ENDIAN +#define ENDIAN(_lo, _hi) _hi, _lo +#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) + sizeof(__u32) +#define HI_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) +#else +#error "Unknown endianness" +#endif + +union arg64 { + struct { + __u32 ENDIAN(lo32, hi32); + }; + __u64 u64; +}; + +#define JEQ(x, jt) \ + JEQ64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) +#define JGT(x, jt) \ + JGT64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) +#define JGE(x, jt) \ + JGE64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) +#define JNE(x, jt) \ + JNE64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) +#define JLT(x, jt) \ + JLT64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) +#define JLE(x, jt) \ + JLE64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) + +#define JA(x, jt) \ + JA64(((union arg64){.u64 = (x)}).lo32, \ + ((union arg64){.u64 = (x)}).hi32, \ + EXPAND(jt)) +#define ARG(i) ARG_64(i) + +#else +#error __BITS_PER_LONG value unusable. +#endif + +/* Loads the arg into A */ +#define ARG_32(idx) \ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, LO_ARG(idx)) + +/* Loads hi into A and lo in X */ +#define ARG_64(idx) \ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, LO_ARG(idx)), \ + BPF_STMT(BPF_ST, 0), /* lo -> M[0] */ \ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, HI_ARG(idx)), \ + BPF_STMT(BPF_ST, 1) /* hi -> M[1] */ + +#define JEQ32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (value), 0, 1), \ + jt + +#define JNE32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (value), 1, 0), \ + jt + +/* Checks the lo, then swaps to check the hi. A=lo,X=hi */ +#define JEQ64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (hi), 0, 5), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (lo), 0, 2), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define JNE64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (hi), 5, 0), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (lo), 2, 0), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define JA32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JSET+BPF_K, (value), 0, 1), \ + jt + +#define JA64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JSET+BPF_K, (hi), 3, 0), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JSET+BPF_K, (lo), 0, 2), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define JGE32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGE+BPF_K, (value), 0, 1), \ + jt + +#define JLT32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGE+BPF_K, (value), 1, 0), \ + jt + +/* Shortcut checking if hi > arg.hi. */ +#define JGE64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (hi), 4, 0), \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (hi), 0, 5), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JGE+BPF_K, (lo), 0, 2), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define JLT64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGE+BPF_K, (hi), 0, 4), \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (hi), 0, 5), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (lo), 2, 0), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define JGT32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (value), 0, 1), \ + jt + +#define JLE32(value, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (value), 1, 0), \ + jt + +/* Check hi > args.hi first, then do the GE checking */ +#define JGT64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (hi), 4, 0), \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (hi), 0, 5), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (lo), 0, 2), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define JLE64(lo, hi, jt) \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (hi), 6, 0), \ + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, (hi), 0, 3), \ + BPF_STMT(BPF_LD+BPF_MEM, 0), /* swap in lo */ \ + BPF_JUMP(BPF_JMP+BPF_JGT+BPF_K, (lo), 2, 0), \ + BPF_STMT(BPF_LD+BPF_MEM, 1), /* passed: swap hi back in */ \ + jt, \ + BPF_STMT(BPF_LD+BPF_MEM, 1) /* failed: swap hi back in */ + +#define LOAD_SYSCALL_NR \ + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, \ + offsetof(struct seccomp_data, nr)) + +#endif /* __BPF_HELPER_H__ */ diff --git a/samples/seccomp/dropper.c b/samples/seccomp/dropper.c new file mode 100644 index 00000000000..c69c347c701 --- /dev/null +++ b/samples/seccomp/dropper.c @@ -0,0 +1,68 @@ +/* + * Naive system call dropper built on seccomp_filter. + * + * Copyright (c) 2012 The Chromium OS Authors + * Author: Will Drewry + * + * The code may be used by anyone for any purpose, + * and can serve as a starting point for developing + * applications using prctl(PR_SET_SECCOMP, 2, ...). + * + * When run, returns the specified errno for the specified + * system call number against the given architecture. + * + * Run this one as root as PR_SET_NO_NEW_PRIVS is not called. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int install_filter(int nr, int arch, int error) +{ + struct sock_filter filter[] = { + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, + (offsetof(struct seccomp_data, arch))), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, arch, 0, 3), + BPF_STMT(BPF_LD+BPF_W+BPF_ABS, + (offsetof(struct seccomp_data, nr))), + BPF_JUMP(BPF_JMP+BPF_JEQ+BPF_K, nr, 0, 1), + BPF_STMT(BPF_RET+BPF_K, + SECCOMP_RET_ERRNO|(error & SECCOMP_RET_DATA)), + BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog prog = { + .len = (unsigned short)(sizeof(filter)/sizeof(filter[0])), + .filter = filter, + }; + if (prctl(PR_SET_SECCOMP, 2, &prog)) { + perror("prctl"); + return 1; + } + return 0; +} + +int main(int argc, char **argv) +{ + if (argc < 5) { + fprintf(stderr, "Usage:\n" + "dropper []\n" + "Hint: AUDIT_ARCH_I386: 0x%X\n" + " AUDIT_ARCH_X86_64: 0x%X\n" + "\n", AUDIT_ARCH_I386, AUDIT_ARCH_X86_64); + return 1; + } + if (install_filter(strtol(argv[1], NULL, 0), strtol(argv[2], NULL, 0), + strtol(argv[3], NULL, 0))) + return 1; + execv(argv[4], &argv[4]); + printf("Failed to execv\n"); + return 255; +} diff --git a/scripts/dtc/libfdt/fdt_rw.c b/scripts/dtc/libfdt/fdt_rw.c index 24437dfc32b..768422646fd 100644 --- a/scripts/dtc/libfdt/fdt_rw.c +++ b/scripts/dtc/libfdt/fdt_rw.c @@ -415,20 +415,31 @@ static void _fdt_packblocks(const char *old, char *new, int fdt_open_into(const void *fdt, void *buf, int bufsize) { - int err; - int mem_rsv_size, struct_size; - int newsize; + int err = -1; + uint32_t mem_rsv_size; + int struct_size; + uint32_t newsize; const char *fdtstart = fdt; - const char *fdtend = fdtstart + fdt_totalsize(fdt); + const char *fdtend = NULL; char *tmp; + if (fdtstart + fdt_totalsize(fdt) < fdtstart) { + return err; + } + fdtend = fdtstart + fdt_totalsize(fdt); FDT_CHECK_HEADER(fdt); + if ((fdt_num_mem_rsv(fdt)+1) > (UINT_MAX / sizeof(struct fdt_reserve_entry))) { + return err; + } + mem_rsv_size = (fdt_num_mem_rsv(fdt)+1) * sizeof(struct fdt_reserve_entry); if (fdt_version(fdt) >= 17) { struct_size = fdt_size_dt_struct(fdt); + if (struct_size < 0) + return struct_size; } else { struct_size = 0; while (fdt_next_tag(fdt, struct_size, &struct_size) != FDT_END) @@ -447,16 +458,22 @@ int fdt_open_into(const void *fdt, void *buf, int bufsize) fdt_set_totalsize(buf, bufsize); return 0; } + if (((uint64_t)FDT_ALIGN(sizeof(struct fdt_header), 8) + (uint64_t)mem_rsv_size \ + + (uint64_t)struct_size + (uint64_t)fdt_size_dt_strings(fdt)) > UINT_MAX) { + return (err = -1); + } /* Need to reorder */ newsize = FDT_ALIGN(sizeof(struct fdt_header), 8) + mem_rsv_size + struct_size + fdt_size_dt_strings(fdt); - if (bufsize < newsize) return -FDT_ERR_NOSPACE; /* First attempt to build converted tree at beginning of buffer */ tmp = buf; + if (((tmp + newsize) < tmp) || ((buf + bufsize) < buf)) { + return (err = -1); + } /* But if that overlaps with the old tree... */ if (((tmp + newsize) > fdtstart) && (tmp < fdtend)) { /* Try right after the old tree instead */ diff --git a/security/Kconfig b/security/Kconfig index ccc61f8006b..f9e267c2523 100644 --- a/security/Kconfig +++ b/security/Kconfig @@ -84,6 +84,15 @@ config SECURITY_DMESG_RESTRICT If you are unsure how to answer this question, answer N. +config SECURITY_PERF_EVENTS_RESTRICT + bool "Restrict unprivileged use of performance events" + depends on PERF_EVENTS + help + If you say Y here, the kernel.perf_event_paranoid sysctl + will be set to 3 by default, and no unprivileged use of the + perf_event_open syscall will be permitted unless it is + changed. + config SECURITY bool "Enable different security models" depends on SYSFS diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index 6327685c101..e9d7fa5788a 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -394,6 +394,11 @@ int apparmor_bprm_set_creds(struct linux_binprm *bprm) new_profile = find_attach(ns, &ns->base.profiles, name); if (!new_profile) goto cleanup; + /* + * NOTE: Domain transitions from unconfined are allowed + * even when no_new_privs is set because this aways results + * in a further reduction of permissions. + */ goto apply; } @@ -455,6 +460,16 @@ int apparmor_bprm_set_creds(struct linux_binprm *bprm) /* fail exec */ error = -EACCES; + /* + * Policy has specified a domain transition, if no_new_privs then + * fail the exec. + */ + if (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS) { + aa_put_profile(new_profile); + error = -EPERM; + goto cleanup; + } + if (!new_profile) goto audit; @@ -609,6 +624,14 @@ int aa_change_hat(const char *hats[], int count, u64 token, bool permtest) const char *target = NULL, *info = NULL; int error = 0; + /* + * Fail explicitly requested domain transitions if no_new_privs. + * There is no exception for unconfined as change_hat is not + * available. + */ + if (task_no_new_privs(current)) + return -EPERM; + /* released below */ cred = get_current_cred(); cxt = cred->security; @@ -750,6 +773,18 @@ int aa_change_profile(const char *ns_name, const char *hname, bool onexec, cxt = cred->security; profile = aa_cred_profile(cred); + /* + * Fail explicitly requested domain transitions if no_new_privs + * and not unconfined. + * Domain transitions from unconfined are allowed even when + * no_new_privs is set because this aways results in a reduction + * of permissions. + */ + if (task_no_new_privs(current) && !unconfined(profile)) { + put_cred(cred); + return -EPERM; + } + if (ns_name) { /* released below */ ns = aa_find_namespace(profile->ns, ns_name); diff --git a/security/apparmor/include/apparmor.h b/security/apparmor/include/apparmor.h index 40aedd9f73e..4a8cbfeef8b 100644 --- a/security/apparmor/include/apparmor.h +++ b/security/apparmor/include/apparmor.h @@ -65,7 +65,6 @@ extern int apparmor_initialized __initdata; char *aa_split_fqname(char *args, char **ns_name); void aa_info_message(const char *str); void *kvmalloc(size_t size); -void kvfree(void *buffer); /** diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index e75829ba0ff..2b038e9628a 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -104,34 +104,3 @@ void *kvmalloc(size_t size) return buffer; } -/** - * do_vfree - workqueue routine for freeing vmalloced memory - * @work: data to be freed - * - * The work_struct is overlaid to the data being freed, as at the point - * the work is scheduled the data is no longer valid, be its freeing - * needs to be delayed until safe. - */ -static void do_vfree(struct work_struct *work) -{ - vfree(work); -} - -/** - * kvfree - free an allocation do by kvmalloc - * @buffer: buffer to free (MAYBE_NULL) - * - * Free a buffer allocated by kvmalloc - */ -void kvfree(void *buffer) -{ - if (is_vmalloc_addr(buffer)) { - /* Data is no longer valid so just use the allocated space - * as the work_struct - */ - struct work_struct *work = (struct work_struct *) buffer; - INIT_WORK(work, do_vfree); - schedule_work(work); - } else - kfree(buffer); -} diff --git a/security/capability.c b/security/capability.c index fc991bcab24..d0a49eeca59 100644 --- a/security/capability.c +++ b/security/capability.c @@ -94,8 +94,8 @@ static int cap_sb_statfs(struct dentry *dentry) return 0; } -static int cap_sb_mount(char *dev_name, struct path *path, char *type, - unsigned long flags, void *data) +static int cap_sb_mount(const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data) { return 0; } diff --git a/security/commoncap.c b/security/commoncap.c index 0051ac2d058..98ff4630f9f 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -523,14 +523,17 @@ int cap_bprm_set_creds(struct linux_binprm *bprm) /* Don't let someone trace a set[ug]id/setpcap binary with the revised - * credentials unless they have the appropriate permit + * credentials unless they have the appropriate permit. + * + * In addition, if NO_NEW_PRIVS, then ensure we get no new privs. */ if ((new->euid != old->uid || new->egid != old->gid || !cap_issubset(new->cap_permitted, old->cap_permitted)) && bprm->unsafe & ~LSM_UNSAFE_PTRACE_CAP) { /* downgrade; they get no more than they had, and maybe less */ - if (!capable(CAP_SETUID)) { + if (!capable(CAP_SETUID) || + (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) { new->euid = new->uid; new->egid = new->gid; } diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 2d1bb8af769..988a7e227c4 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -843,6 +843,8 @@ static int encrypted_update(struct key *key, const void *data, size_t datalen) const char *format = NULL; int ret = 0; + if (test_bit(KEY_FLAG_NEGATIVE, &key->flags)) + return -ENOKEY; if (datalen <= 0 || datalen > 32767 || !data) return -EINVAL; diff --git a/security/keys/gc.c b/security/keys/gc.c index a42b45531aa..ec035deda63 100644 --- a/security/keys/gc.c +++ b/security/keys/gc.c @@ -188,11 +188,13 @@ static noinline void key_gc_unused_key(struct key *key) if (test_bit(KEY_FLAG_INSTANTIATED, &key->flags)) atomic_dec(&key->user->nikeys); - key_user_put(key->user); + /* Throw away the key data if the key is instantiated */ + if (test_bit(KEY_FLAG_INSTANTIATED, &key->flags) && + !test_bit(KEY_FLAG_NEGATIVE, &key->flags) && + key->type->destroy) + key->type->destroy(key); - /* now throw away the key memory */ - if (key->type->destroy) - key->type->destroy(key); + key_user_put(key->user); kfree(key->description); diff --git a/security/keys/key.c b/security/keys/key.c index 06783cffb3a..5f9902e3882 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -573,7 +573,7 @@ int key_reject_and_link(struct key *key, mutex_unlock(&key_construction_mutex); - if (keyring) + if (keyring && link_ret == 0) __key_link_end(keyring, key->type, prealloc); /* wake up anyone waiting for a key to be constructed */ diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index fb767c6cd99..a83f2c9bea6 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -702,16 +702,16 @@ long keyctl_read_key(key_serial_t keyid, char __user *buffer, size_t buflen) /* the key is probably readable - now try to read it */ can_read_key: - ret = key_validate(key); - if (ret == 0) { - ret = -EOPNOTSUPP; - if (key->type->read) { - /* read the data with the semaphore held (since we - * might sleep) */ - down_read(&key->sem); + ret = -EOPNOTSUPP; + if (key->type->read) { + /* Read the data with the semaphore held (since we might sleep) + * to protect against the key being updated or revoked. + */ + down_read(&key->sem); + ret = key_validate(key); + if (ret == 0) ret = key->type->read(key, buffer, buflen); - up_read(&key->sem); - } + up_read(&key->sem); } error2: diff --git a/security/keys/proc.c b/security/keys/proc.c index 49bbc97943a..3f7b4102a35 100644 --- a/security/keys/proc.c +++ b/security/keys/proc.c @@ -188,7 +188,7 @@ static int proc_keys_show(struct seq_file *m, void *v) struct timespec now; unsigned long timo; key_ref_t key_ref, skey_ref; - char xbuf[12]; + char xbuf[16]; int rc; key_ref = make_key_ref(key, 0); diff --git a/security/keys/trusted.c b/security/keys/trusted.c index 2d5d041f204..9614dbcae8a 100644 --- a/security/keys/trusted.c +++ b/security/keys/trusted.c @@ -1013,12 +1013,15 @@ static void trusted_rcu_free(struct rcu_head *rcu) */ static int trusted_update(struct key *key, const void *data, size_t datalen) { - struct trusted_key_payload *p = key->payload.data; + struct trusted_key_payload *p; struct trusted_key_payload *new_p; struct trusted_key_options *new_o; char *datablob; int ret = 0; + if (test_bit(KEY_FLAG_NEGATIVE, &key->flags)) + return -ENOKEY; + p = key->payload.data; if (!p->migratable) return -EPERM; if (datalen <= 0 || datalen > 32767 || !data) diff --git a/security/keys/user_defined.c b/security/keys/user_defined.c index c7660a25a3e..f89846f6bd3 100644 --- a/security/keys/user_defined.c +++ b/security/keys/user_defined.c @@ -117,7 +117,10 @@ int user_update(struct key *key, const void *data, size_t datalen) if (ret == 0) { /* attach the new data, displacing the old */ - zap = key->payload.data; + if (!test_bit(KEY_FLAG_NEGATIVE, &key->flags)) + zap = key->payload.data; + else + zap = NULL; rcu_assign_keypointer(key, upayload); key->expiry = 0; } diff --git a/security/lsm_audit.c b/security/lsm_audit.c index 90c129b0102..abacc49f111 100644 --- a/security/lsm_audit.c +++ b/security/lsm_audit.c @@ -242,6 +242,21 @@ static void dump_common_audit_data(struct audit_buffer *ab, } break; } + case LSM_AUDIT_DATA_IOCTL_OP: { + struct inode *inode; + + audit_log_d_path(ab, " path=", &a->u.op->path); + + inode = a->u.op->path.dentry->d_inode; + if (inode) { + audit_log_format(ab, " dev="); + audit_log_untrustedstring(ab, inode->i_sb->s_id); + audit_log_format(ab, " ino=%lu", inode->i_ino); + } + + audit_log_format(ab, " ioctlcmd=%hx", a->u.op->cmd); + break; + } case LSM_AUDIT_DATA_DENTRY: { struct inode *inode; diff --git a/security/security.c b/security/security.c index cecd55e581c..0e1916e91bd 100644 --- a/security/security.c +++ b/security/security.c @@ -280,8 +280,8 @@ int security_sb_statfs(struct dentry *dentry) return security_ops->sb_statfs(dentry); } -int security_sb_mount(char *dev_name, struct path *path, - char *type, unsigned long flags, void *data) +int security_sb_mount(const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data) { return security_ops->sb_mount(dev_name, path, type, flags, data); } diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 8ee42b2a5f1..87a4af34f37 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,7 @@ struct avc_entry { u32 tsid; u16 tclass; struct av_decision avd; + struct avc_xperms_node *xp_node; }; struct avc_node { @@ -56,6 +58,16 @@ struct avc_node { struct rcu_head rhead; }; +struct avc_xperms_decision_node { + struct extended_perms_decision xpd; + struct list_head xpd_list; /* list of extended_perms_decision */ +}; + +struct avc_xperms_node { + struct extended_perms xp; + struct list_head xpd_head; /* list head of extended_perms_decision */ +}; + struct avc_cache { struct hlist_head slots[AVC_CACHE_SLOTS]; /* head for avc_node->list */ spinlock_t slots_lock[AVC_CACHE_SLOTS]; /* lock for writes */ @@ -86,6 +98,9 @@ DEFINE_PER_CPU(struct avc_cache_stats, avc_cache_stats) = { 0 }; static struct avc_cache avc_cache; static struct avc_callback_node *avc_callbacks; static struct kmem_cache *avc_node_cachep; +static struct kmem_cache *avc_xperms_data_cachep; +static struct kmem_cache *avc_xperms_decision_cachep; +static struct kmem_cache *avc_xperms_cachep; static inline int avc_hash(u32 ssid, u32 tsid, u16 tclass) { @@ -176,7 +191,17 @@ void __init avc_init(void) atomic_set(&avc_cache.lru_hint, 0); avc_node_cachep = kmem_cache_create("avc_node", sizeof(struct avc_node), - 0, SLAB_PANIC, NULL); + 0, SLAB_PANIC, NULL); + avc_xperms_cachep = kmem_cache_create("avc_xperms_node", + sizeof(struct avc_xperms_node), + 0, SLAB_PANIC, NULL); + avc_xperms_decision_cachep = kmem_cache_create( + "avc_xperms_decision_node", + sizeof(struct avc_xperms_decision_node), + 0, SLAB_PANIC, NULL); + avc_xperms_data_cachep = kmem_cache_create("avc_xperms_data", + sizeof(struct extended_perms_data), + 0, SLAB_PANIC, NULL); audit_log(current->audit_context, GFP_KERNEL, AUDIT_KERNEL, "AVC INITIALIZED\n"); } @@ -213,9 +238,245 @@ int avc_get_hash_stats(char *page) slots_used, AVC_CACHE_SLOTS, max_chain_len); } +/* + * using a linked list for extended_perms_decision lookup because the list is + * always small. i.e. less than 5, typically 1 + */ +static struct extended_perms_decision *avc_xperms_decision_lookup(u8 driver, + struct avc_xperms_node *xp_node) +{ + struct avc_xperms_decision_node *xpd_node; + + list_for_each_entry(xpd_node, &xp_node->xpd_head, xpd_list) { + if (xpd_node->xpd.driver == driver) + return &xpd_node->xpd; + } + return NULL; +} + +static inline unsigned int +avc_xperms_has_perm(struct extended_perms_decision *xpd, + u8 perm, u8 which) +{ + unsigned int rc = 0; + + if ((which == XPERMS_ALLOWED) && + (xpd->used & XPERMS_ALLOWED)) + rc = security_xperm_test(xpd->allowed->p, perm); + else if ((which == XPERMS_AUDITALLOW) && + (xpd->used & XPERMS_AUDITALLOW)) + rc = security_xperm_test(xpd->auditallow->p, perm); + else if ((which == XPERMS_DONTAUDIT) && + (xpd->used & XPERMS_DONTAUDIT)) + rc = security_xperm_test(xpd->dontaudit->p, perm); + return rc; +} + +static void avc_xperms_allow_perm(struct avc_xperms_node *xp_node, + u8 driver, u8 perm) +{ + struct extended_perms_decision *xpd; + security_xperm_set(xp_node->xp.drivers.p, driver); + xpd = avc_xperms_decision_lookup(driver, xp_node); + if (xpd && xpd->allowed) + security_xperm_set(xpd->allowed->p, perm); +} + +static void avc_xperms_decision_free(struct avc_xperms_decision_node *xpd_node) +{ + struct extended_perms_decision *xpd; + + xpd = &xpd_node->xpd; + if (xpd->allowed) + kmem_cache_free(avc_xperms_data_cachep, xpd->allowed); + if (xpd->auditallow) + kmem_cache_free(avc_xperms_data_cachep, xpd->auditallow); + if (xpd->dontaudit) + kmem_cache_free(avc_xperms_data_cachep, xpd->dontaudit); + kmem_cache_free(avc_xperms_decision_cachep, xpd_node); +} + +static void avc_xperms_free(struct avc_xperms_node *xp_node) +{ + struct avc_xperms_decision_node *xpd_node, *tmp; + + if (!xp_node) + return; + + list_for_each_entry_safe(xpd_node, tmp, &xp_node->xpd_head, xpd_list) { + list_del(&xpd_node->xpd_list); + avc_xperms_decision_free(xpd_node); + } + kmem_cache_free(avc_xperms_cachep, xp_node); +} + +static void avc_copy_xperms_decision(struct extended_perms_decision *dest, + struct extended_perms_decision *src) +{ + dest->driver = src->driver; + dest->used = src->used; + if (dest->used & XPERMS_ALLOWED) + memcpy(dest->allowed->p, src->allowed->p, + sizeof(src->allowed->p)); + if (dest->used & XPERMS_AUDITALLOW) + memcpy(dest->auditallow->p, src->auditallow->p, + sizeof(src->auditallow->p)); + if (dest->used & XPERMS_DONTAUDIT) + memcpy(dest->dontaudit->p, src->dontaudit->p, + sizeof(src->dontaudit->p)); +} + +/* + * similar to avc_copy_xperms_decision, but only copy decision + * information relevant to this perm + */ +static inline void avc_quick_copy_xperms_decision(u8 perm, + struct extended_perms_decision *dest, + struct extended_perms_decision *src) +{ + /* + * compute index of the u32 of the 256 bits (8 u32s) that contain this + * command permission + */ + u8 i = perm >> 5; + + dest->used = src->used; + if (dest->used & XPERMS_ALLOWED) + dest->allowed->p[i] = src->allowed->p[i]; + if (dest->used & XPERMS_AUDITALLOW) + dest->auditallow->p[i] = src->auditallow->p[i]; + if (dest->used & XPERMS_DONTAUDIT) + dest->dontaudit->p[i] = src->dontaudit->p[i]; +} + +static struct avc_xperms_decision_node + *avc_xperms_decision_alloc(u8 which) +{ + struct avc_xperms_decision_node *xpd_node; + struct extended_perms_decision *xpd; + + xpd_node = kmem_cache_zalloc(avc_xperms_decision_cachep, + GFP_ATOMIC | __GFP_NOMEMALLOC); + if (!xpd_node) + return NULL; + + xpd = &xpd_node->xpd; + if (which & XPERMS_ALLOWED) { + xpd->allowed = kmem_cache_zalloc(avc_xperms_data_cachep, + GFP_ATOMIC | __GFP_NOMEMALLOC); + if (!xpd->allowed) + goto error; + } + if (which & XPERMS_AUDITALLOW) { + xpd->auditallow = kmem_cache_zalloc(avc_xperms_data_cachep, + GFP_ATOMIC | __GFP_NOMEMALLOC); + if (!xpd->auditallow) + goto error; + } + if (which & XPERMS_DONTAUDIT) { + xpd->dontaudit = kmem_cache_zalloc(avc_xperms_data_cachep, + GFP_ATOMIC | __GFP_NOMEMALLOC); + if (!xpd->dontaudit) + goto error; + } + return xpd_node; +error: + avc_xperms_decision_free(xpd_node); + return NULL; +} + +static int avc_add_xperms_decision(struct avc_node *node, + struct extended_perms_decision *src) +{ + struct avc_xperms_decision_node *dest_xpd; + + node->ae.xp_node->xp.len++; + dest_xpd = avc_xperms_decision_alloc(src->used); + if (!dest_xpd) + return -ENOMEM; + avc_copy_xperms_decision(&dest_xpd->xpd, src); + list_add(&dest_xpd->xpd_list, &node->ae.xp_node->xpd_head); + return 0; +} + +static struct avc_xperms_node *avc_xperms_alloc(void) +{ + struct avc_xperms_node *xp_node; + + xp_node = kmem_cache_zalloc(avc_xperms_cachep, + GFP_ATOMIC|__GFP_NOMEMALLOC); + if (!xp_node) + return xp_node; + INIT_LIST_HEAD(&xp_node->xpd_head); + return xp_node; +} + +static int avc_xperms_populate(struct avc_node *node, + struct avc_xperms_node *src) +{ + struct avc_xperms_node *dest; + struct avc_xperms_decision_node *dest_xpd; + struct avc_xperms_decision_node *src_xpd; + + if (src->xp.len == 0) + return 0; + dest = avc_xperms_alloc(); + if (!dest) + return -ENOMEM; + + memcpy(dest->xp.drivers.p, src->xp.drivers.p, sizeof(dest->xp.drivers.p)); + dest->xp.len = src->xp.len; + + /* for each source xpd allocate a destination xpd and copy */ + list_for_each_entry(src_xpd, &src->xpd_head, xpd_list) { + dest_xpd = avc_xperms_decision_alloc(src_xpd->xpd.used); + if (!dest_xpd) + goto error; + avc_copy_xperms_decision(&dest_xpd->xpd, &src_xpd->xpd); + list_add(&dest_xpd->xpd_list, &dest->xpd_head); + } + node->ae.xp_node = dest; + return 0; +error: + avc_xperms_free(dest); + return -ENOMEM; + +} + +static inline u32 avc_xperms_audit_required(u32 requested, + struct av_decision *avd, + struct extended_perms_decision *xpd, + u8 perm, + int result, + u32 *deniedp) +{ + u32 denied, audited; + + denied = requested & ~avd->allowed; + if (unlikely(denied)) { + audited = denied & avd->auditdeny; + if (audited && xpd) { + if (avc_xperms_has_perm(xpd, perm, XPERMS_DONTAUDIT)) + audited &= ~requested; + } + } else if (result) { + audited = denied = requested; + } else { + audited = requested & avd->auditallow; + if (audited && xpd) { + if (!avc_xperms_has_perm(xpd, perm, XPERMS_AUDITALLOW)) + audited &= ~requested; + } + } + + *deniedp = denied; + return audited; +} + static void avc_node_free(struct rcu_head *rhead) { struct avc_node *node = container_of(rhead, struct avc_node, rhead); + avc_xperms_free(node->ae.xp_node); kmem_cache_free(avc_node_cachep, node); avc_cache_stats_incr(frees); } @@ -229,6 +490,7 @@ static void avc_node_delete(struct avc_node *node) static void avc_node_kill(struct avc_node *node) { + avc_xperms_free(node->ae.xp_node); kmem_cache_free(avc_node_cachep, node); avc_cache_stats_incr(frees); atomic_dec(&avc_cache.active_nodes); @@ -377,6 +639,7 @@ static int avc_latest_notif_update(int seqno, int is_insert) * @tsid: target security identifier * @tclass: target security class * @avd: resulting av decision + * @xp_node: resulting extended permissions * * Insert an AVC entry for the SID pair * (@ssid, @tsid) and class @tclass. @@ -388,7 +651,9 @@ static int avc_latest_notif_update(int seqno, int is_insert) * the access vectors into a cache entry, returns * avc_node inserted. Otherwise, this function returns NULL. */ -static struct avc_node *avc_insert(u32 ssid, u32 tsid, u16 tclass, struct av_decision *avd) +static struct avc_node *avc_insert(u32 ssid, u32 tsid, u16 tclass, + struct av_decision *avd, + struct avc_xperms_node *xp_node) { struct avc_node *pos, *node = NULL; int hvalue; @@ -402,10 +667,15 @@ static struct avc_node *avc_insert(u32 ssid, u32 tsid, u16 tclass, struct av_dec struct hlist_head *head; struct hlist_node *next; spinlock_t *lock; + int rc = 0; hvalue = avc_hash(ssid, tsid, tclass); avc_node_populate(node, ssid, tsid, tclass, avd); - + rc = avc_xperms_populate(node, xp_node); + if (rc) { + kmem_cache_free(avc_node_cachep, node); + return NULL; + } head = &avc_cache.slots[hvalue]; lock = &avc_cache.slots_lock[hvalue]; @@ -455,11 +725,15 @@ static void avc_audit_post_callback(struct audit_buffer *ab, void *a) avc_dump_query(ab, ad->selinux_audit_data->slad->ssid, ad->selinux_audit_data->slad->tsid, ad->selinux_audit_data->slad->tclass); + if (ad->selinux_audit_data->slad->denied) { + audit_log_format(ab, " permissive=%u", + ad->selinux_audit_data->slad->result ? 0 : 1); + } } /* This is the slow part of avc audit with big stack footprint */ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, - u32 requested, u32 audited, u32 denied, + u32 requested, u32 audited, u32 denied, int result, struct common_audit_data *a, unsigned flags) { @@ -490,12 +764,29 @@ static noinline int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, slad.tsid = tsid; slad.audited = audited; slad.denied = denied; + slad.result = result; a->selinux_audit_data->slad = &slad; common_lsm_audit(a, avc_audit_pre_callback, avc_audit_post_callback); return 0; } +static inline int avc_xperms_audit(u32 ssid, u32 tsid, u16 tclass, + u32 requested, struct av_decision *avd, + struct extended_perms_decision *xpd, + u8 perm, int result, + struct common_audit_data *ad) +{ + u32 audited, denied; + + audited = avc_xperms_audit_required( + requested, avd, xpd, perm, result, &denied); + if (likely(!audited)) + return 0; + return slow_avc_audit(ssid, tsid, tclass, requested, + audited, denied, result, ad, 0); +} + /** * avc_audit - Audit the granting or denial of permissions. * @ssid: source security identifier @@ -553,7 +844,7 @@ inline int avc_audit(u32 ssid, u32 tsid, return 0; return slow_avc_audit(ssid, tsid, tclass, - requested, audited, denied, + requested, audited, denied, result, a, flags); } @@ -609,14 +900,17 @@ static inline int avc_sidcmp(u32 x, u32 y) * @perms : Permission mask bits * @ssid,@tsid,@tclass : identifier of an AVC entry * @seqno : sequence number when decision was made + * @xpd: extended_perms_decision to be added to the node * * if a valid AVC entry doesn't exist,this function returns -ENOENT. * if kmalloc() called internal returns NULL, this function returns -ENOMEM. * otherwise, this function updates the AVC entry. The original AVC-entry object * will release later by RCU. */ -static int avc_update_node(u32 event, u32 perms, u32 ssid, u32 tsid, u16 tclass, - u32 seqno) +static int avc_update_node(u32 event, u32 perms, u8 driver, u8 xperm, u32 ssid, + u32 tsid, u16 tclass, u32 seqno, + struct extended_perms_decision *xpd, + u32 flags) { int hvalue, rc = 0; unsigned long flag; @@ -661,9 +955,19 @@ static int avc_update_node(u32 event, u32 perms, u32 ssid, u32 tsid, u16 tclass, avc_node_populate(node, ssid, tsid, tclass, &orig->ae.avd); + if (orig->ae.xp_node) { + rc = avc_xperms_populate(node, orig->ae.xp_node); + if (rc) { + kmem_cache_free(avc_node_cachep, node); + goto out_unlock; + } + } + switch (event) { case AVC_CALLBACK_GRANT: node->ae.avd.allowed |= perms; + if (node->ae.xp_node && (flags & AVC_EXTENDED_PERMS)) + avc_xperms_allow_perm(node->ae.xp_node, driver, xperm); break; case AVC_CALLBACK_TRY_REVOKE: case AVC_CALLBACK_REVOKE: @@ -681,6 +985,9 @@ static int avc_update_node(u32 event, u32 perms, u32 ssid, u32 tsid, u16 tclass, case AVC_CALLBACK_AUDITDENY_DISABLE: node->ae.avd.auditdeny &= ~perms; break; + case AVC_CALLBACK_ADD_XPERMS: + avc_add_xperms_decision(node, xpd); + break; } avc_node_replace(node, orig); out_unlock: @@ -754,18 +1061,20 @@ int avc_ss_reset(u32 seqno) * results in a bigger stack frame. */ static noinline struct avc_node *avc_compute_av(u32 ssid, u32 tsid, - u16 tclass, struct av_decision *avd) + u16 tclass, struct av_decision *avd, + struct avc_xperms_node *xp_node) { rcu_read_unlock(); - security_compute_av(ssid, tsid, tclass, avd); + INIT_LIST_HEAD(&xp_node->xpd_head); + security_compute_av(ssid, tsid, tclass, avd, &xp_node->xp); rcu_read_lock(); - return avc_insert(ssid, tsid, tclass, avd); + return avc_insert(ssid, tsid, tclass, avd, xp_node); } static noinline int avc_denied(u32 ssid, u32 tsid, - u16 tclass, u32 requested, - unsigned flags, - struct av_decision *avd) + u16 tclass, u32 requested, + u8 driver, u8 xperm, unsigned flags, + struct av_decision *avd) { if (flags & AVC_STRICT) return -EACCES; @@ -773,11 +1082,91 @@ static noinline int avc_denied(u32 ssid, u32 tsid, if (selinux_enforcing && !(avd->flags & AVD_FLAGS_PERMISSIVE)) return -EACCES; - avc_update_node(AVC_CALLBACK_GRANT, requested, ssid, - tsid, tclass, avd->seqno); + avc_update_node(AVC_CALLBACK_GRANT, requested, driver, xperm, ssid, + tsid, tclass, avd->seqno, NULL, flags); return 0; } +/* + * The avc extended permissions logic adds an additional 256 bits of + * permissions to an avc node when extended permissions for that node are + * specified in the avtab. If the additional 256 permissions is not adequate, + * as-is the case with ioctls, then multiple may be chained together and the + * driver field is used to specify which set contains the permission. + */ +int avc_has_extended_perms(u32 ssid, u32 tsid, u16 tclass, u32 requested, + u8 driver, u8 xperm, struct common_audit_data *ad) +{ + struct avc_node *node; + struct av_decision avd; + u32 denied; + struct extended_perms_decision local_xpd; + struct extended_perms_decision *xpd = NULL; + struct extended_perms_data allowed; + struct extended_perms_data auditallow; + struct extended_perms_data dontaudit; + struct avc_xperms_node local_xp_node; + struct avc_xperms_node *xp_node; + int rc = 0, rc2; + + xp_node = &local_xp_node; + BUG_ON(!requested); + + rcu_read_lock(); + + node = avc_lookup(ssid, tsid, tclass); + if (unlikely(!node)) { + node = avc_compute_av(ssid, tsid, tclass, &avd, xp_node); + } else { + memcpy(&avd, &node->ae.avd, sizeof(avd)); + xp_node = node->ae.xp_node; + } + /* if extended permissions are not defined, only consider av_decision */ + if (!xp_node || !xp_node->xp.len) + goto decision; + + local_xpd.allowed = &allowed; + local_xpd.auditallow = &auditallow; + local_xpd.dontaudit = &dontaudit; + + xpd = avc_xperms_decision_lookup(driver, xp_node); + if (unlikely(!xpd)) { + /* + * Compute the extended_perms_decision only if the driver + * is flagged + */ + if (!security_xperm_test(xp_node->xp.drivers.p, driver)) { + avd.allowed &= ~requested; + goto decision; + } + rcu_read_unlock(); + security_compute_xperms_decision(ssid, tsid, tclass, driver, + &local_xpd); + rcu_read_lock(); + avc_update_node(AVC_CALLBACK_ADD_XPERMS, requested, driver, xperm, + ssid, tsid, tclass, avd.seqno, &local_xpd, 0); + } else { + avc_quick_copy_xperms_decision(xperm, &local_xpd, xpd); + } + xpd = &local_xpd; + + if (!avc_xperms_has_perm(xpd, xperm, XPERMS_ALLOWED)) + avd.allowed &= ~requested; + +decision: + denied = requested & ~(avd.allowed); + if (unlikely(denied)) + rc = avc_denied(ssid, tsid, tclass, requested, driver, xperm, + AVC_EXTENDED_PERMS, &avd); + + rcu_read_unlock(); + + rc2 = avc_xperms_audit(ssid, tsid, tclass, requested, + &avd, xpd, xperm, rc, ad); + if (rc2) + return rc2; + return rc; +} /** * avc_has_perm_noaudit - Check permissions but perform no auditing. @@ -805,6 +1194,7 @@ inline int avc_has_perm_noaudit(u32 ssid, u32 tsid, struct av_decision *avd) { struct avc_node *node; + struct avc_xperms_node xp_node; int rc = 0; u32 denied; @@ -813,16 +1203,14 @@ inline int avc_has_perm_noaudit(u32 ssid, u32 tsid, rcu_read_lock(); node = avc_lookup(ssid, tsid, tclass); - if (unlikely(!node)) { - node = avc_compute_av(ssid, tsid, tclass, avd); - } else { + if (unlikely(!node)) + node = avc_compute_av(ssid, tsid, tclass, avd, &xp_node); + else memcpy(avd, &node->ae.avd, sizeof(*avd)); - avd = &node->ae.avd; - } denied = requested & ~(avd->allowed); if (unlikely(denied)) - rc = avc_denied(ssid, tsid, tclass, requested, flags, avd); + rc = avc_denied(ssid, tsid, tclass, requested, 0, 0, flags, avd); rcu_read_unlock(); return rc; diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 393e59a6738..0ac3c918849 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -417,15 +417,11 @@ static int sb_finish_set_opts(struct super_block *sb) sbsec->behavior > ARRAY_SIZE(labeling_behaviors)) sbsec->flags &= ~SE_SBLABELSUPP; - /* Special handling for sysfs. Is genfs but also has setxattr handler*/ - if (strncmp(sb->s_type->name, "sysfs", sizeof("sysfs")) == 0) - sbsec->flags |= SE_SBLABELSUPP; - - /* - * Special handling for rootfs. Is genfs but supports - * setting SELinux context on in-core inodes. - */ - if (strncmp(sb->s_type->name, "rootfs", sizeof("rootfs")) == 0) + /* Special handling. Is genfs but also has in-core setxattr handler*/ + if (!strcmp(sb->s_type->name, "sysfs") || + !strcmp(sb->s_type->name, "pstore") || + !strcmp(sb->s_type->name, "debugfs") || + !strcmp(sb->s_type->name, "rootfs")) sbsec->flags |= SE_SBLABELSUPP; /* Initialize the root inode. */ @@ -1335,15 +1331,33 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent isec->sid = sbsec->sid; if ((sbsec->flags & SE_SBPROC) && !S_ISLNK(inode->i_mode)) { - if (opt_dentry) { - isec->sclass = inode_mode_to_security_class(inode->i_mode); - rc = selinux_proc_get_sid(opt_dentry, - isec->sclass, - &sid); - if (rc) - goto out_unlock; - isec->sid = sid; - } + /* We must have a dentry to determine the label on + * procfs inodes */ + if (opt_dentry) + /* Called from d_instantiate or + * d_splice_alias. */ + dentry = dget(opt_dentry); + else + /* Called from selinux_complete_init, try to + * find a dentry. */ + dentry = d_find_alias(inode); + /* + * This can be hit on boot when a file is accessed + * before the policy is loaded. When we load policy we + * may find inodes that have no dentry on the + * sbsec->isec_head list. No reason to complain as + * these will get fixed up the next time we go through + * inode_doinit() with a dentry, before these inodes + * could be used again by userspace. + */ + if (!dentry) + goto out_unlock; + isec->sclass = inode_mode_to_security_class(inode->i_mode); + rc = selinux_proc_get_sid(dentry, isec->sclass, &sid); + dput(dentry); + if (rc) + goto out_unlock; + isec->sid = sid; } break; } @@ -2100,6 +2114,13 @@ static int selinux_bprm_set_creds(struct linux_binprm *bprm) new_tsec->sid = old_tsec->exec_sid; /* Reset exec SID on execve. */ new_tsec->exec_sid = 0; + + /* + * Minimize confusion: if no_new_privs and a transition is + * explicitly requested, then fail the exec. + */ + if (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS) + return -EPERM; } else { /* Check for a default transition on this program. */ rc = security_transition_sid(old_tsec->sid, isec->sid, @@ -2113,7 +2134,8 @@ static int selinux_bprm_set_creds(struct linux_binprm *bprm) ad.selinux_audit_data = &sad; ad.u.path = bprm->file->f_path; - if (bprm->file->f_path.mnt->mnt_flags & MNT_NOSUID) + if ((bprm->file->f_path.mnt->mnt_flags & MNT_NOSUID) || + (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) new_tsec->sid = old_tsec->sid; if (new_tsec->sid == old_tsec->sid) { @@ -2605,9 +2627,9 @@ static int selinux_sb_statfs(struct dentry *dentry) return superblock_has_perm(cred, dentry->d_sb, FILESYSTEM__GETATTR, &ad); } -static int selinux_mount(char *dev_name, +static int selinux_mount(const char *dev_name, struct path *path, - char *type, + const char *type, unsigned long flags, void *data) { @@ -3069,6 +3091,48 @@ static void selinux_file_free_security(struct file *file) file_free_security(file); } +/* + * Check whether a task has the ioctl permission and cmd + * operation to an inode. + */ +int ioctl_has_perm(const struct cred *cred, struct file *file, + u32 requested, u16 cmd) +{ + struct common_audit_data ad; + struct file_security_struct *fsec = file->f_security; + struct inode *inode = file->f_path.dentry->d_inode; + struct inode_security_struct *isec = inode->i_security; + struct lsm_ioctlop_audit ioctl; + u32 ssid = cred_sid(cred); + struct selinux_audit_data sad = {0,}; + int rc; + u8 driver = cmd >> 8; + u8 xperm = cmd & 0xff; + + COMMON_AUDIT_DATA_INIT(&ad, IOCTL_OP); + ad.u.op = &ioctl; + ad.u.op->cmd = cmd; + ad.selinux_audit_data = &sad; + ad.u.op->path = file->f_path; + + if (ssid != fsec->sid) { + rc = avc_has_perm(ssid, fsec->sid, + SECCLASS_FD, + FD__USE, + &ad); + if (rc) + goto out; + } + + if (unlikely(IS_PRIVATE(inode))) + return 0; + + rc = avc_has_extended_perms(ssid, isec->sid, isec->sclass, + requested, driver, xperm, &ad); +out: + return rc; +} + static int selinux_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -3111,7 +3175,7 @@ static int selinux_file_ioctl(struct file *file, unsigned int cmd, * to the file's ioctl() function. */ default: - error = file_has_perm(cred, file, FILE__IOCTL); + error = ioctl_has_perm(cred, file, FILE__IOCTL, (u16) cmd); } return error; } diff --git a/security/selinux/include/avc.h b/security/selinux/include/avc.h index 1931370233d..6c681e8d3d3 100644 --- a/security/selinux/include/avc.h +++ b/security/selinux/include/avc.h @@ -84,6 +84,7 @@ int avc_audit(u32 ssid, u32 tsid, struct common_audit_data *a, unsigned flags); #define AVC_STRICT 1 /* Ignore permissive mode. */ +#define AVC_EXTENDED_PERMS 2 /* update extended permissions */ int avc_has_perm_noaudit(u32 ssid, u32 tsid, u16 tclass, u32 requested, unsigned flags, @@ -101,6 +102,9 @@ static inline int avc_has_perm(u32 ssid, u32 tsid, return avc_has_perm_flags(ssid, tsid, tclass, requested, auditdata, 0); } +int avc_has_extended_perms(u32 ssid, u32 tsid, u16 tclass, u32 requested, + u8 driver, u8 perm, struct common_audit_data *ad); + u32 avc_policy_seqno(void); #define AVC_CALLBACK_GRANT 1 @@ -111,6 +115,7 @@ u32 avc_policy_seqno(void); #define AVC_CALLBACK_AUDITALLOW_DISABLE 32 #define AVC_CALLBACK_AUDITDENY_ENABLE 64 #define AVC_CALLBACK_AUDITDENY_DISABLE 128 +#define AVC_CALLBACK_ADD_XPERMS 256 int avc_add_callback(int (*callback)(u32 event, u32 ssid, u32 tsid, u16 tclass, u32 perms, diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index d871e8ad210..0bca13e4933 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -31,13 +31,17 @@ #define POLICYDB_VERSION_BOUNDARY 24 #define POLICYDB_VERSION_FILENAME_TRANS 25 #define POLICYDB_VERSION_ROLETRANS 26 +#define POLICYDB_VERSION_NEW_OBJECT_DEFAULTS 27 +#define POLICYDB_VERSION_DEFAULT_TYPE 28 +#define POLICYDB_VERSION_CONSTRAINT_NAMES 29 +#define POLICYDB_VERSION_XPERMS_IOCTL 30 /* Range of policy versions we understand*/ #define POLICYDB_VERSION_MIN POLICYDB_VERSION_BASE #ifdef CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX #define POLICYDB_VERSION_MAX CONFIG_SECURITY_SELINUX_POLICYDB_VERSION_MAX_VALUE #else -#define POLICYDB_VERSION_MAX POLICYDB_VERSION_ROLETRANS +#define POLICYDB_VERSION_MAX POLICYDB_VERSION_XPERMS_IOCTL #endif /* Mask for just the mount related flags */ @@ -100,11 +104,38 @@ struct av_decision { u32 flags; }; +#define XPERMS_ALLOWED 1 +#define XPERMS_AUDITALLOW 2 +#define XPERMS_DONTAUDIT 4 + +#define security_xperm_set(perms, x) (perms[x >> 5] |= 1 << (x & 0x1f)) +#define security_xperm_test(perms, x) (1 & (perms[x >> 5] >> (x & 0x1f))) +struct extended_perms_data { + u32 p[8]; +}; + +struct extended_perms_decision { + u8 used; + u8 driver; + struct extended_perms_data *allowed; + struct extended_perms_data *auditallow; + struct extended_perms_data *dontaudit; +}; + +struct extended_perms { + u16 len; /* length associated decision chain */ + struct extended_perms_data drivers; /* flag drivers that are used */ +}; + /* definitions of av_decision.flags */ #define AVD_FLAGS_PERMISSIVE 0x0001 void security_compute_av(u32 ssid, u32 tsid, - u16 tclass, struct av_decision *avd); + u16 tclass, struct av_decision *avd, + struct extended_perms *xperms); + +void security_compute_xperms_decision(u32 ssid, u32 tsid, u16 tclass, + u8 driver, struct extended_perms_decision *xpermd); void security_compute_av_user(u32 ssid, u32 tsid, u16 tclass, struct av_decision *avd); diff --git a/security/selinux/nlmsgtab.c b/security/selinux/nlmsgtab.c index 0920ea3bf59..eec10b5d912 100644 --- a/security/selinux/nlmsgtab.c +++ b/security/selinux/nlmsgtab.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "flask.h" #include "av_permissions.h" @@ -78,8 +79,10 @@ static struct nlmsg_perm nlmsg_firewall_perms[] = static struct nlmsg_perm nlmsg_tcpdiag_perms[] = { - { TCPDIAG_GETSOCK, NETLINK_TCPDIAG_SOCKET__NLMSG_READ }, - { DCCPDIAG_GETSOCK, NETLINK_TCPDIAG_SOCKET__NLMSG_READ }, + { TCPDIAG_GETSOCK, NETLINK_TCPDIAG_SOCKET__NLMSG_READ }, + { DCCPDIAG_GETSOCK, NETLINK_TCPDIAG_SOCKET__NLMSG_READ }, + { SOCK_DIAG_BY_FAMILY, NETLINK_TCPDIAG_SOCKET__NLMSG_READ }, + { SOCK_DESTROY_BACKPORT, NETLINK_TCPDIAG_SOCKET__NLMSG_WRITE }, }; static struct nlmsg_perm nlmsg_xfrm_perms[] = @@ -100,6 +103,13 @@ static struct nlmsg_perm nlmsg_xfrm_perms[] = { XFRM_MSG_FLUSHPOLICY, NETLINK_XFRM_SOCKET__NLMSG_WRITE }, { XFRM_MSG_NEWAE, NETLINK_XFRM_SOCKET__NLMSG_WRITE }, { XFRM_MSG_GETAE, NETLINK_XFRM_SOCKET__NLMSG_READ }, + { XFRM_MSG_REPORT, NETLINK_XFRM_SOCKET__NLMSG_READ }, + { XFRM_MSG_MIGRATE, NETLINK_XFRM_SOCKET__NLMSG_WRITE }, + { XFRM_MSG_NEWSADINFO, NETLINK_XFRM_SOCKET__NLMSG_READ }, + { XFRM_MSG_GETSADINFO, NETLINK_XFRM_SOCKET__NLMSG_READ }, + { XFRM_MSG_NEWSPDINFO, NETLINK_XFRM_SOCKET__NLMSG_WRITE }, + { XFRM_MSG_GETSPDINFO, NETLINK_XFRM_SOCKET__NLMSG_READ }, + { XFRM_MSG_MAPPING, NETLINK_XFRM_SOCKET__NLMSG_READ }, }; static struct nlmsg_perm nlmsg_audit_perms[] = diff --git a/security/selinux/ss/avtab.c b/security/selinux/ss/avtab.c index a3dd9faa19c..640c16b9d3f 100644 --- a/security/selinux/ss/avtab.c +++ b/security/selinux/ss/avtab.c @@ -24,6 +24,7 @@ #include "policydb.h" static struct kmem_cache *avtab_node_cachep; +static struct kmem_cache *avtab_xperms_cachep; static inline int avtab_hash(struct avtab_key *keyp, u16 mask) { @@ -37,11 +38,24 @@ avtab_insert_node(struct avtab *h, int hvalue, struct avtab_key *key, struct avtab_datum *datum) { struct avtab_node *newnode; + struct avtab_extended_perms *xperms; newnode = kmem_cache_zalloc(avtab_node_cachep, GFP_KERNEL); if (newnode == NULL) return NULL; newnode->key = *key; - newnode->datum = *datum; + + if (key->specified & AVTAB_XPERMS) { + xperms = kmem_cache_zalloc(avtab_xperms_cachep, GFP_KERNEL); + if (xperms == NULL) { + kmem_cache_free(avtab_node_cachep, newnode); + return NULL; + } + *xperms = *(datum->u.xperms); + newnode->datum.u.xperms = xperms; + } else { + newnode->datum.u.data = datum->u.data; + } + if (prev) { newnode->next = prev->next; prev->next = newnode; @@ -70,8 +84,12 @@ static int avtab_insert(struct avtab *h, struct avtab_key *key, struct avtab_dat if (key->source_type == cur->key.source_type && key->target_type == cur->key.target_type && key->target_class == cur->key.target_class && - (specified & cur->key.specified)) + (specified & cur->key.specified)) { + /* extended perms may not be unique */ + if (specified & AVTAB_XPERMS) + break; return -EEXIST; + } if (key->source_type < cur->key.source_type) break; if (key->source_type == cur->key.source_type && @@ -232,6 +250,9 @@ void avtab_destroy(struct avtab *h) while (cur) { temp = cur; cur = cur->next; + if (temp->key.specified & AVTAB_XPERMS) + kmem_cache_free(avtab_xperms_cachep, + temp->datum.u.xperms); kmem_cache_free(avtab_node_cachep, temp); } h->htable[i] = NULL; @@ -314,13 +335,42 @@ void avtab_hash_eval(struct avtab *h, char *tag) chain2_len_sum); } +/* + * extended permissions compatibility. Make ToT Android kernels compatible + * with Android M releases + */ +#define AVTAB_OPTYPE_ALLOWED 0x1000 +#define AVTAB_OPTYPE_AUDITALLOW 0x2000 +#define AVTAB_OPTYPE_DONTAUDIT 0x4000 +#define AVTAB_OPTYPE (AVTAB_OPTYPE_ALLOWED | \ + AVTAB_OPTYPE_AUDITALLOW | \ + AVTAB_OPTYPE_DONTAUDIT) +#define AVTAB_XPERMS_OPTYPE 4 + +#define avtab_xperms_to_optype(x) (x << AVTAB_XPERMS_OPTYPE) +#define avtab_optype_to_xperms(x) (x >> AVTAB_XPERMS_OPTYPE) + +static unsigned int avtab_android_m_compat; + +static void avtab_android_m_compat_set(void) +{ + if (!avtab_android_m_compat) { + pr_info("SELinux: Android master kernel running Android" + " M policy in compatibility mode.\n"); + avtab_android_m_compat = 1; + } +} + static uint16_t spec_order[] = { AVTAB_ALLOWED, AVTAB_AUDITDENY, AVTAB_AUDITALLOW, AVTAB_TRANSITION, AVTAB_CHANGE, - AVTAB_MEMBER + AVTAB_MEMBER, + AVTAB_XPERMS_ALLOWED, + AVTAB_XPERMS_AUDITALLOW, + AVTAB_XPERMS_DONTAUDIT }; int avtab_read_item(struct avtab *a, void *fp, struct policydb *pol, @@ -330,10 +380,12 @@ int avtab_read_item(struct avtab *a, void *fp, struct policydb *pol, { __le16 buf16[4]; u16 enabled; - __le32 buf32[7]; u32 items, items2, val, vers = pol->policyvers; struct avtab_key key; struct avtab_datum datum; + struct avtab_extended_perms xperms; + __le32 buf32[ARRAY_SIZE(xperms.perms.p)]; + unsigned int android_m_compat_optype = 0; int i, rc; unsigned set; @@ -390,11 +442,15 @@ int avtab_read_item(struct avtab *a, void *fp, struct policydb *pol, printk(KERN_ERR "SELinux: avtab: entry has both access vectors and types\n"); return -EINVAL; } + if (val & AVTAB_XPERMS) { + printk(KERN_ERR "SELinux: avtab: entry has extended permissions\n"); + return -EINVAL; + } for (i = 0; i < ARRAY_SIZE(spec_order); i++) { if (val & spec_order[i]) { key.specified = spec_order[i] | enabled; - datum.data = le32_to_cpu(buf32[items++]); + datum.u.data = le32_to_cpu(buf32[items++]); rc = insertf(a, &key, &datum, p); if (rc) return rc; @@ -420,6 +476,13 @@ int avtab_read_item(struct avtab *a, void *fp, struct policydb *pol, key.target_class = le16_to_cpu(buf16[items++]); key.specified = le16_to_cpu(buf16[items++]); + if ((key.specified & AVTAB_OPTYPE) && + (vers == POLICYDB_VERSION_XPERMS_IOCTL)) { + key.specified = avtab_optype_to_xperms(key.specified); + android_m_compat_optype = 1; + avtab_android_m_compat_set(); + } + if (!policydb_type_isvalid(pol, key.source_type) || !policydb_type_isvalid(pol, key.target_type) || !policydb_class_isvalid(pol, key.target_class)) { @@ -437,14 +500,54 @@ int avtab_read_item(struct avtab *a, void *fp, struct policydb *pol, return -EINVAL; } - rc = next_entry(buf32, fp, sizeof(u32)); - if (rc) { - printk(KERN_ERR "SELinux: avtab: truncated entry\n"); - return rc; + if ((vers < POLICYDB_VERSION_XPERMS_IOCTL) && + (key.specified & AVTAB_XPERMS)) { + printk(KERN_ERR "SELinux: avtab: policy version %u does not " + "support extended permissions rules and one " + "was specified\n", vers); + return -EINVAL; + } else if (key.specified & AVTAB_XPERMS) { + memset(&xperms, 0, sizeof(struct avtab_extended_perms)); + rc = next_entry(&xperms.specified, fp, sizeof(u8)); + if (rc) { + printk(KERN_ERR "SELinux: avtab: truncated entry\n"); + return rc; + } + if (avtab_android_m_compat || + ((xperms.specified != AVTAB_XPERMS_IOCTLFUNCTION) && + (xperms.specified != AVTAB_XPERMS_IOCTLDRIVER) && + (vers == POLICYDB_VERSION_XPERMS_IOCTL))) { + xperms.driver = xperms.specified; + if (android_m_compat_optype) + xperms.specified = AVTAB_XPERMS_IOCTLDRIVER; + else + xperms.specified = AVTAB_XPERMS_IOCTLFUNCTION; + avtab_android_m_compat_set(); + } else { + rc = next_entry(&xperms.driver, fp, sizeof(u8)); + if (rc) { + printk(KERN_ERR "SELinux: avtab: truncated entry\n"); + return rc; + } + } + rc = next_entry(buf32, fp, sizeof(u32)*ARRAY_SIZE(xperms.perms.p)); + if (rc) { + printk(KERN_ERR "SELinux: avtab: truncated entry\n"); + return rc; + } + for (i = 0; i < ARRAY_SIZE(xperms.perms.p); i++) + xperms.perms.p[i] = le32_to_cpu(buf32[i]); + datum.u.xperms = &xperms; + } else { + rc = next_entry(buf32, fp, sizeof(u32)); + if (rc) { + printk(KERN_ERR "SELinux: avtab: truncated entry\n"); + return rc; + } + datum.u.data = le32_to_cpu(*buf32); } - datum.data = le32_to_cpu(*buf32); if ((key.specified & AVTAB_TYPE) && - !policydb_type_isvalid(pol, datum.data)) { + !policydb_type_isvalid(pol, datum.u.data)) { printk(KERN_ERR "SELinux: avtab: invalid type\n"); return -EINVAL; } @@ -504,18 +607,40 @@ int avtab_read(struct avtab *a, void *fp, struct policydb *pol) int avtab_write_item(struct policydb *p, struct avtab_node *cur, void *fp) { __le16 buf16[4]; - __le32 buf32[1]; + __le32 buf32[ARRAY_SIZE(cur->datum.u.xperms->perms.p)]; int rc; + unsigned int i; buf16[0] = cpu_to_le16(cur->key.source_type); buf16[1] = cpu_to_le16(cur->key.target_type); buf16[2] = cpu_to_le16(cur->key.target_class); - buf16[3] = cpu_to_le16(cur->key.specified); + if (avtab_android_m_compat && (cur->key.specified & AVTAB_XPERMS) && + (cur->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLDRIVER)) + buf16[3] = cpu_to_le16(avtab_xperms_to_optype(cur->key.specified)); + else + buf16[3] = cpu_to_le16(cur->key.specified); rc = put_entry(buf16, sizeof(u16), 4, fp); if (rc) return rc; - buf32[0] = cpu_to_le32(cur->datum.data); - rc = put_entry(buf32, sizeof(u32), 1, fp); + + if (cur->key.specified & AVTAB_XPERMS) { + if (avtab_android_m_compat == 0) { + rc = put_entry(&cur->datum.u.xperms->specified, + sizeof(u8), 1, fp); + if (rc) + return rc; + } + rc = put_entry(&cur->datum.u.xperms->driver, sizeof(u8), 1, fp); + if (rc) + return rc; + for (i = 0; i < ARRAY_SIZE(cur->datum.u.xperms->perms.p); i++) + buf32[i] = cpu_to_le32(cur->datum.u.xperms->perms.p[i]); + rc = put_entry(buf32, sizeof(u32), + ARRAY_SIZE(cur->datum.u.xperms->perms.p), fp); + } else { + buf32[0] = cpu_to_le32(cur->datum.u.data); + rc = put_entry(buf32, sizeof(u32), 1, fp); + } if (rc) return rc; return 0; @@ -548,9 +673,13 @@ void avtab_cache_init(void) avtab_node_cachep = kmem_cache_create("avtab_node", sizeof(struct avtab_node), 0, SLAB_PANIC, NULL); + avtab_xperms_cachep = kmem_cache_create("avtab_extended_perms", + sizeof(struct avtab_extended_perms), + 0, SLAB_PANIC, NULL); } void avtab_cache_destroy(void) { kmem_cache_destroy(avtab_node_cachep); + kmem_cache_destroy(avtab_xperms_cachep); } diff --git a/security/selinux/ss/avtab.h b/security/selinux/ss/avtab.h index 63ce2f9e441..8133523ca67 100644 --- a/security/selinux/ss/avtab.h +++ b/security/selinux/ss/avtab.h @@ -23,6 +23,8 @@ #ifndef _SS_AVTAB_H_ #define _SS_AVTAB_H_ +#include "security.h" + struct avtab_key { u16 source_type; /* source type */ u16 target_type; /* target type */ @@ -35,13 +37,43 @@ struct avtab_key { #define AVTAB_MEMBER 0x0020 #define AVTAB_CHANGE 0x0040 #define AVTAB_TYPE (AVTAB_TRANSITION | AVTAB_MEMBER | AVTAB_CHANGE) +/* extended permissions */ +#define AVTAB_XPERMS_ALLOWED 0x0100 +#define AVTAB_XPERMS_AUDITALLOW 0x0200 +#define AVTAB_XPERMS_DONTAUDIT 0x0400 +#define AVTAB_XPERMS (AVTAB_XPERMS_ALLOWED | \ + AVTAB_XPERMS_AUDITALLOW | \ + AVTAB_XPERMS_DONTAUDIT) #define AVTAB_ENABLED_OLD 0x80000000 /* reserved for used in cond_avtab */ #define AVTAB_ENABLED 0x8000 /* reserved for used in cond_avtab */ u16 specified; /* what field is specified */ }; +/* + * For operations that require more than the 32 permissions provided by the avc + * extended permissions may be used to provide 256 bits of permissions. + */ +struct avtab_extended_perms { +/* These are not flags. All 256 values may be used */ +#define AVTAB_XPERMS_IOCTLFUNCTION 0x01 +#define AVTAB_XPERMS_IOCTLDRIVER 0x02 + /* extension of the avtab_key specified */ + u8 specified; /* ioctl, netfilter, ... */ + /* + * if 256 bits is not adequate as is often the case with ioctls, then + * multiple extended perms may be used and the driver field + * specifies which permissions are included. + */ + u8 driver; + /* 256 bits of permissions */ + struct extended_perms_data perms; +}; + struct avtab_datum { - u32 data; /* access vector or type value */ + union { + u32 data; /* access vector or type value */ + struct avtab_extended_perms *xperms; + } u; }; struct avtab_node { diff --git a/security/selinux/ss/conditional.c b/security/selinux/ss/conditional.c index 377d148e715..ba7dd936610 100644 --- a/security/selinux/ss/conditional.c +++ b/security/selinux/ss/conditional.c @@ -15,6 +15,7 @@ #include "security.h" #include "conditional.h" +#include "services.h" /* * cond_evaluate_expr evaluates a conditional expr @@ -617,10 +618,28 @@ int cond_write_list(struct policydb *p, struct cond_node *list, void *fp) return 0; } + +void cond_compute_xperms(struct avtab *ctab, struct avtab_key *key, + struct extended_perms_decision *xpermd) +{ + struct avtab_node *node; + + if (!ctab || !key || !xpermd) + return; + + for (node = avtab_search_node(ctab, key); node; + node = avtab_search_node_next(node, key->specified)) { + if (node->key.specified & AVTAB_ENABLED) + services_compute_xperms_decision(xpermd, node); + } + return; + +} /* Determine whether additional permissions are granted by the conditional * av table, and if so, add them to the result */ -void cond_compute_av(struct avtab *ctab, struct avtab_key *key, struct av_decision *avd) +void cond_compute_av(struct avtab *ctab, struct avtab_key *key, + struct av_decision *avd, struct extended_perms *xperms) { struct avtab_node *node; @@ -631,7 +650,7 @@ void cond_compute_av(struct avtab *ctab, struct avtab_key *key, struct av_decisi node = avtab_search_node_next(node, key->specified)) { if ((u16)(AVTAB_ALLOWED|AVTAB_ENABLED) == (node->key.specified & (AVTAB_ALLOWED|AVTAB_ENABLED))) - avd->allowed |= node->datum.data; + avd->allowed |= node->datum.u.data; if ((u16)(AVTAB_AUDITDENY|AVTAB_ENABLED) == (node->key.specified & (AVTAB_AUDITDENY|AVTAB_ENABLED))) /* Since a '0' in an auditdeny mask represents a @@ -639,10 +658,13 @@ void cond_compute_av(struct avtab *ctab, struct avtab_key *key, struct av_decisi * the '&' operand to ensure that all '0's in the mask * are retained (much unlike the allow and auditallow cases). */ - avd->auditdeny &= node->datum.data; + avd->auditdeny &= node->datum.u.data; if ((u16)(AVTAB_AUDITALLOW|AVTAB_ENABLED) == (node->key.specified & (AVTAB_AUDITALLOW|AVTAB_ENABLED))) - avd->auditallow |= node->datum.data; + avd->auditallow |= node->datum.u.data; + if (xperms && (node->key.specified & AVTAB_ENABLED) && + (node->key.specified & AVTAB_XPERMS)) + services_compute_xperms_drivers(xperms, node); } return; } diff --git a/security/selinux/ss/conditional.h b/security/selinux/ss/conditional.h index 4d1f8746650..ddb43e7e1c7 100644 --- a/security/selinux/ss/conditional.h +++ b/security/selinux/ss/conditional.h @@ -73,8 +73,10 @@ int cond_read_list(struct policydb *p, void *fp); int cond_write_bool(void *key, void *datum, void *ptr); int cond_write_list(struct policydb *p, struct cond_node *list, void *fp); -void cond_compute_av(struct avtab *ctab, struct avtab_key *key, struct av_decision *avd); - +void cond_compute_av(struct avtab *ctab, struct avtab_key *key, + struct av_decision *avd, struct extended_perms *xperms); +void cond_compute_xperms(struct avtab *ctab, struct avtab_key *key, + struct extended_perms_decision *xpermd); int evaluate_cond_node(struct policydb *p, struct cond_node *node); #endif /* _CONDITIONAL_H_ */ diff --git a/security/selinux/ss/constraint.h b/security/selinux/ss/constraint.h index 149dda731fd..96fd947c494 100644 --- a/security/selinux/ss/constraint.h +++ b/security/selinux/ss/constraint.h @@ -48,6 +48,7 @@ struct constraint_expr { u32 op; /* operator */ struct ebitmap names; /* names */ + struct type_set *type_names; struct constraint_expr *next; /* next expression */ }; diff --git a/security/selinux/ss/context.h b/security/selinux/ss/context.h index 45e8fb0515f..212e3479a0d 100644 --- a/security/selinux/ss/context.h +++ b/security/selinux/ss/context.h @@ -74,6 +74,26 @@ static inline int mls_context_cpy_low(struct context *dst, struct context *src) return rc; } +/* + * Sets both levels in the MLS range of 'dst' to the high level of 'src'. + */ +static inline int mls_context_cpy_high(struct context *dst, struct context *src) +{ + int rc; + + dst->range.level[0].sens = src->range.level[1].sens; + rc = ebitmap_cpy(&dst->range.level[0].cat, &src->range.level[1].cat); + if (rc) + goto out; + + dst->range.level[1].sens = src->range.level[1].sens; + rc = ebitmap_cpy(&dst->range.level[1].cat, &src->range.level[1].cat); + if (rc) + ebitmap_destroy(&dst->range.level[0].cat); +out: + return rc; +} + static inline int mls_context_cmp(struct context *c1, struct context *c2) { return ((c1->range.level[0].sens == c2->range.level[0].sens) && diff --git a/security/selinux/ss/mls.c b/security/selinux/ss/mls.c index fbf9c5816c7..40de8d3f208 100644 --- a/security/selinux/ss/mls.c +++ b/security/selinux/ss/mls.c @@ -517,6 +517,8 @@ int mls_compute_sid(struct context *scontext, { struct range_trans rtr; struct mls_range *r; + struct class_datum *cladatum; + int default_range = 0; if (!policydb.mls_enabled) return 0; @@ -530,6 +532,28 @@ int mls_compute_sid(struct context *scontext, r = hashtab_search(policydb.range_tr, &rtr); if (r) return mls_range_set(newcontext, r); + + if (tclass && tclass <= policydb.p_classes.nprim) { + cladatum = policydb.class_val_to_struct[tclass - 1]; + if (cladatum) + default_range = cladatum->default_range; + } + + switch (default_range) { + case DEFAULT_SOURCE_LOW: + return mls_context_cpy_low(newcontext, scontext); + case DEFAULT_SOURCE_HIGH: + return mls_context_cpy_high(newcontext, scontext); + case DEFAULT_SOURCE_LOW_HIGH: + return mls_context_cpy(newcontext, scontext); + case DEFAULT_TARGET_LOW: + return mls_context_cpy_low(newcontext, tcontext); + case DEFAULT_TARGET_HIGH: + return mls_context_cpy_high(newcontext, tcontext); + case DEFAULT_TARGET_LOW_HIGH: + return mls_context_cpy(newcontext, tcontext); + } + /* Fallthrough */ case AVTAB_CHANGE: if ((tclass == policydb.process_class) || (sock == true)) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index a7f61d52f05..1b3307c62e9 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -133,6 +133,26 @@ static struct policydb_compat_info policydb_compat[] = { .sym_num = SYM_NUM, .ocon_num = OCON_NUM, }, + { + .version = POLICYDB_VERSION_NEW_OBJECT_DEFAULTS, + .sym_num = SYM_NUM, + .ocon_num = OCON_NUM, + }, + { + .version = POLICYDB_VERSION_DEFAULT_TYPE, + .sym_num = SYM_NUM, + .ocon_num = OCON_NUM, + }, + { + .version = POLICYDB_VERSION_CONSTRAINT_NAMES, + .sym_num = SYM_NUM, + .ocon_num = OCON_NUM, + }, + { + .version = POLICYDB_VERSION_XPERMS_IOCTL, + .sym_num = SYM_NUM, + .ocon_num = OCON_NUM, + }, }; static struct policydb_compat_info *policydb_lookup_compat(int version) @@ -603,6 +623,19 @@ static int common_destroy(void *key, void *datum, void *p) return 0; } +static void constraint_expr_destroy(struct constraint_expr *expr) +{ + if (expr) { + ebitmap_destroy(&expr->names); + if (expr->type_names) { + ebitmap_destroy(&expr->type_names->types); + ebitmap_destroy(&expr->type_names->negset); + kfree(expr->type_names); + } + kfree(expr); + } +} + static int cls_destroy(void *key, void *datum, void *p) { struct class_datum *cladatum; @@ -618,10 +651,9 @@ static int cls_destroy(void *key, void *datum, void *p) while (constraint) { e = constraint->expr; while (e) { - ebitmap_destroy(&e->names); etmp = e; e = e->next; - kfree(etmp); + constraint_expr_destroy(etmp); } ctemp = constraint; constraint = constraint->next; @@ -632,16 +664,14 @@ static int cls_destroy(void *key, void *datum, void *p) while (constraint) { e = constraint->expr; while (e) { - ebitmap_destroy(&e->names); etmp = e; e = e->next; - kfree(etmp); + constraint_expr_destroy(etmp); } ctemp = constraint; constraint = constraint->next; kfree(ctemp); } - kfree(cladatum->comkey); } kfree(datum); @@ -1146,8 +1176,34 @@ static int common_read(struct policydb *p, struct hashtab *h, void *fp) return rc; } -static int read_cons_helper(struct constraint_node **nodep, int ncons, - int allowxtarget, void *fp) +static void type_set_init(struct type_set *t) +{ + ebitmap_init(&t->types); + ebitmap_init(&t->negset); +} + +static int type_set_read(struct type_set *t, void *fp) +{ + __le32 buf[1]; + int rc; + + if (ebitmap_read(&t->types, fp)) + return -EINVAL; + if (ebitmap_read(&t->negset, fp)) + return -EINVAL; + + rc = next_entry(buf, fp, sizeof(u32)); + if (rc < 0) + return -EINVAL; + t->flags = le32_to_cpu(buf[0]); + + return 0; +} + + +static int read_cons_helper(struct policydb *p, + struct constraint_node **nodep, + int ncons, int allowxtarget, void *fp) { struct constraint_node *c, *lc; struct constraint_expr *e, *le; @@ -1215,6 +1271,18 @@ static int read_cons_helper(struct constraint_node **nodep, int ncons, rc = ebitmap_read(&e->names, fp); if (rc) return rc; + if (p->policyvers >= + POLICYDB_VERSION_CONSTRAINT_NAMES) { + e->type_names = kzalloc(sizeof + (*e->type_names), + GFP_KERNEL); + if (!e->type_names) + return -ENOMEM; + type_set_init(e->type_names); + rc = type_set_read(e->type_names, fp); + if (rc) + return rc; + } break; default: return -EINVAL; @@ -1291,7 +1359,7 @@ static int class_read(struct policydb *p, struct hashtab *h, void *fp) goto bad; } - rc = read_cons_helper(&cladatum->constraints, ncons, 0, fp); + rc = read_cons_helper(p, &cladatum->constraints, ncons, 0, fp); if (rc) goto bad; @@ -1301,9 +1369,27 @@ static int class_read(struct policydb *p, struct hashtab *h, void *fp) if (rc) goto bad; ncons = le32_to_cpu(buf[0]); - rc = read_cons_helper(&cladatum->validatetrans, ncons, 1, fp); + rc = read_cons_helper(p, &cladatum->validatetrans, + ncons, 1, fp); + if (rc) + goto bad; + } + + if (p->policyvers >= POLICYDB_VERSION_NEW_OBJECT_DEFAULTS) { + rc = next_entry(buf, fp, sizeof(u32) * 3); + if (rc) + goto bad; + + cladatum->default_user = le32_to_cpu(buf[0]); + cladatum->default_role = le32_to_cpu(buf[1]); + cladatum->default_range = le32_to_cpu(buf[2]); + } + + if (p->policyvers >= POLICYDB_VERSION_DEFAULT_TYPE) { + rc = next_entry(buf, fp, sizeof(u32) * 1); if (rc) goto bad; + cladatum->default_type = le32_to_cpu(buf[0]); } rc = hashtab_insert(h, key, cladatum); @@ -2723,6 +2809,24 @@ static int common_write(void *vkey, void *datum, void *ptr) return 0; } +static int type_set_write(struct type_set *t, void *fp) +{ + int rc; + __le32 buf[1]; + + if (ebitmap_write(&t->types, fp)) + return -EINVAL; + if (ebitmap_write(&t->negset, fp)) + return -EINVAL; + + buf[0] = cpu_to_le32(t->flags); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return -EINVAL; + + return 0; +} + static int write_cons_helper(struct policydb *p, struct constraint_node *node, void *fp) { @@ -2754,6 +2858,12 @@ static int write_cons_helper(struct policydb *p, struct constraint_node *node, rc = ebitmap_write(&e->names, fp); if (rc) return rc; + if (p->policyvers >= + POLICYDB_VERSION_CONSTRAINT_NAMES) { + rc = type_set_write(e->type_names, fp); + if (rc) + return rc; + } break; default: break; @@ -2832,6 +2942,23 @@ static int class_write(void *vkey, void *datum, void *ptr) if (rc) return rc; + if (p->policyvers >= POLICYDB_VERSION_NEW_OBJECT_DEFAULTS) { + buf[0] = cpu_to_le32(cladatum->default_user); + buf[1] = cpu_to_le32(cladatum->default_role); + buf[2] = cpu_to_le32(cladatum->default_range); + + rc = put_entry(buf, sizeof(uint32_t), 3, fp); + if (rc) + return rc; + } + + if (p->policyvers >= POLICYDB_VERSION_DEFAULT_TYPE) { + buf[0] = cpu_to_le32(cladatum->default_type); + rc = put_entry(buf, sizeof(uint32_t), 1, fp); + if (rc) + return rc; + } + return 0; } diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index b846c038718..725d5945a97 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -60,6 +60,20 @@ struct class_datum { struct symtab permissions; /* class-specific permission symbol table */ struct constraint_node *constraints; /* constraints on class permissions */ struct constraint_node *validatetrans; /* special transition rules */ +/* Options how a new object user, role, and type should be decided */ +#define DEFAULT_SOURCE 1 +#define DEFAULT_TARGET 2 + char default_user; + char default_role; + char default_type; +/* Options how a new object range should be decided */ +#define DEFAULT_SOURCE_LOW 1 +#define DEFAULT_SOURCE_HIGH 2 +#define DEFAULT_SOURCE_LOW_HIGH 3 +#define DEFAULT_TARGET_LOW 4 +#define DEFAULT_TARGET_HIGH 5 +#define DEFAULT_TARGET_LOW_HIGH 6 + char default_range; }; /* Role attributes */ @@ -139,6 +153,17 @@ struct cond_bool_datum { struct cond_node; +/* + * type set preserves data needed to determine constraint info from + * policy source. This is not used by the kernel policy but allows + * utilities such as audit2allow to determine constraint denials. + */ +struct type_set { + struct ebitmap types; + struct ebitmap negset; + u32 flags; +}; + /* * The configuration data includes security contexts for * initial SIDs, unlabeled file systems, TCP and UDP port numbers, diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 72b20b1089d..4432e5e6037 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -92,9 +92,10 @@ static int context_struct_to_string(struct context *context, char **scontext, u32 *scontext_len); static void context_struct_compute_av(struct context *scontext, - struct context *tcontext, - u16 tclass, - struct av_decision *avd); + struct context *tcontext, + u16 tclass, + struct av_decision *avd, + struct extended_perms *xperms); struct selinux_mapping { u16 value; /* policy value */ @@ -564,7 +565,8 @@ static void type_attribute_bounds_av(struct context *scontext, context_struct_compute_av(&lo_scontext, tcontext, tclass, - &lo_avd); + &lo_avd, + NULL); if ((lo_avd.allowed & avd->allowed) == avd->allowed) return; /* no masked permission */ masked = ~lo_avd.allowed & avd->allowed; @@ -579,7 +581,8 @@ static void type_attribute_bounds_av(struct context *scontext, context_struct_compute_av(scontext, &lo_tcontext, tclass, - &lo_avd); + &lo_avd, + NULL); if ((lo_avd.allowed & avd->allowed) == avd->allowed) return; /* no masked permission */ masked = ~lo_avd.allowed & avd->allowed; @@ -595,7 +598,8 @@ static void type_attribute_bounds_av(struct context *scontext, context_struct_compute_av(&lo_scontext, &lo_tcontext, tclass, - &lo_avd); + &lo_avd, + NULL); if ((lo_avd.allowed & avd->allowed) == avd->allowed) return; /* no masked permission */ masked = ~lo_avd.allowed & avd->allowed; @@ -612,13 +616,39 @@ static void type_attribute_bounds_av(struct context *scontext, } /* - * Compute access vectors based on a context structure pair for - * the permissions in a particular class. + * flag which drivers have permissions + * only looking for ioctl based extended permssions + */ +void services_compute_xperms_drivers( + struct extended_perms *xperms, + struct avtab_node *node) +{ + unsigned int i; + + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLDRIVER) { + /* if one or more driver has all permissions allowed */ + for (i = 0; i < ARRAY_SIZE(xperms->drivers.p); i++) + xperms->drivers.p[i] |= node->datum.u.xperms->perms.p[i]; + } else if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLFUNCTION) { + /* if allowing permissions within a driver */ + security_xperm_set(xperms->drivers.p, + node->datum.u.xperms->driver); + } + + /* If no ioctl commands are allowed, ignore auditallow and auditdeny */ + if (node->key.specified & AVTAB_XPERMS_ALLOWED) + xperms->len = 1; +} + +/* + * Compute access vectors and extended permissions based on a context + * structure pair for the permissions in a particular class. */ static void context_struct_compute_av(struct context *scontext, - struct context *tcontext, - u16 tclass, - struct av_decision *avd) + struct context *tcontext, + u16 tclass, + struct av_decision *avd, + struct extended_perms *xperms) { struct constraint_node *constraint; struct role_allow *ra; @@ -632,6 +662,10 @@ static void context_struct_compute_av(struct context *scontext, avd->allowed = 0; avd->auditallow = 0; avd->auditdeny = 0xffffffff; + if (xperms) { + memset(&xperms->drivers, 0, sizeof(xperms->drivers)); + xperms->len = 0; + } if (unlikely(!tclass || tclass > policydb.p_classes.nprim)) { if (printk_ratelimit()) @@ -646,7 +680,7 @@ static void context_struct_compute_av(struct context *scontext, * this permission check, then use it. */ avkey.target_class = tclass; - avkey.specified = AVTAB_AV; + avkey.specified = AVTAB_AV | AVTAB_XPERMS; sattr = flex_array_get(policydb.type_attr_map_array, scontext->type - 1); BUG_ON(!sattr); tattr = flex_array_get(policydb.type_attr_map_array, tcontext->type - 1); @@ -659,15 +693,18 @@ static void context_struct_compute_av(struct context *scontext, node; node = avtab_search_node_next(node, avkey.specified)) { if (node->key.specified == AVTAB_ALLOWED) - avd->allowed |= node->datum.data; + avd->allowed |= node->datum.u.data; else if (node->key.specified == AVTAB_AUDITALLOW) - avd->auditallow |= node->datum.data; + avd->auditallow |= node->datum.u.data; else if (node->key.specified == AVTAB_AUDITDENY) - avd->auditdeny &= node->datum.data; + avd->auditdeny &= node->datum.u.data; + else if (xperms && (node->key.specified & AVTAB_XPERMS)) + services_compute_xperms_drivers(xperms, node); } /* Check conditional av table for additional permissions */ - cond_compute_av(&policydb.te_cond_avtab, &avkey, avd); + cond_compute_av(&policydb.te_cond_avtab, &avkey, + avd, xperms); } } @@ -898,6 +935,140 @@ static void avd_init(struct av_decision *avd) avd->flags = 0; } +void services_compute_xperms_decision(struct extended_perms_decision *xpermd, + struct avtab_node *node) +{ + unsigned int i; + + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLFUNCTION) { + if (xpermd->driver != node->datum.u.xperms->driver) + return; + } else if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLDRIVER) { + if (!security_xperm_test(node->datum.u.xperms->perms.p, + xpermd->driver)) + return; + } else { + BUG(); + } + + if (node->key.specified == AVTAB_XPERMS_ALLOWED) { + xpermd->used |= XPERMS_ALLOWED; + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLDRIVER) { + memset(xpermd->allowed->p, 0xff, + sizeof(xpermd->allowed->p)); + } + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLFUNCTION) { + for (i = 0; i < ARRAY_SIZE(xpermd->allowed->p); i++) + xpermd->allowed->p[i] |= + node->datum.u.xperms->perms.p[i]; + } + } else if (node->key.specified == AVTAB_XPERMS_AUDITALLOW) { + xpermd->used |= XPERMS_AUDITALLOW; + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLDRIVER) { + memset(xpermd->auditallow->p, 0xff, + sizeof(xpermd->auditallow->p)); + } + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLFUNCTION) { + for (i = 0; i < ARRAY_SIZE(xpermd->auditallow->p); i++) + xpermd->auditallow->p[i] |= + node->datum.u.xperms->perms.p[i]; + } + } else if (node->key.specified == AVTAB_XPERMS_DONTAUDIT) { + xpermd->used |= XPERMS_DONTAUDIT; + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLDRIVER) { + memset(xpermd->dontaudit->p, 0xff, + sizeof(xpermd->dontaudit->p)); + } + if (node->datum.u.xperms->specified == AVTAB_XPERMS_IOCTLFUNCTION) { + for (i = 0; i < ARRAY_SIZE(xpermd->dontaudit->p); i++) + xpermd->dontaudit->p[i] |= + node->datum.u.xperms->perms.p[i]; + } + } else { + BUG(); + } +} + +void security_compute_xperms_decision(u32 ssid, + u32 tsid, + u16 orig_tclass, + u8 driver, + struct extended_perms_decision *xpermd) +{ + u16 tclass; + struct context *scontext, *tcontext; + struct avtab_key avkey; + struct avtab_node *node; + struct ebitmap *sattr, *tattr; + struct ebitmap_node *snode, *tnode; + unsigned int i, j; + + xpermd->driver = driver; + xpermd->used = 0; + memset(xpermd->allowed->p, 0, sizeof(xpermd->allowed->p)); + memset(xpermd->auditallow->p, 0, sizeof(xpermd->auditallow->p)); + memset(xpermd->dontaudit->p, 0, sizeof(xpermd->dontaudit->p)); + + read_lock(&policy_rwlock); + if (!ss_initialized) + goto allow; + + scontext = sidtab_search(&sidtab, ssid); + if (!scontext) { + printk(KERN_ERR "SELinux: %s: unrecognized SID %d\n", + __func__, ssid); + goto out; + } + + tcontext = sidtab_search(&sidtab, tsid); + if (!tcontext) { + printk(KERN_ERR "SELinux: %s: unrecognized SID %d\n", + __func__, tsid); + goto out; + } + + tclass = unmap_class(orig_tclass); + if (unlikely(orig_tclass && !tclass)) { + if (policydb.allow_unknown) + goto allow; + goto out; + } + + + if (unlikely(!tclass || tclass > policydb.p_classes.nprim)) { + if (printk_ratelimit()) + printk(KERN_WARNING "SELinux: Invalid class %hu\n", tclass); + goto out; + } + + avkey.target_class = tclass; + avkey.specified = AVTAB_XPERMS; + sattr = flex_array_get(policydb.type_attr_map_array, + scontext->type - 1); + BUG_ON(!sattr); + tattr = flex_array_get(policydb.type_attr_map_array, + tcontext->type - 1); + BUG_ON(!tattr); + ebitmap_for_each_positive_bit(sattr, snode, i) { + ebitmap_for_each_positive_bit(tattr, tnode, j) { + avkey.source_type = i + 1; + avkey.target_type = j + 1; + for (node = avtab_search_node(&policydb.te_avtab, &avkey); + node; + node = avtab_search_node_next(node, avkey.specified)) + services_compute_xperms_decision(xpermd, node); + + cond_compute_xperms(&policydb.te_cond_avtab, + &avkey, xpermd); + } + } +out: + read_unlock(&policy_rwlock); + return; +allow: + memset(xpermd->allowed->p, 0xff, sizeof(xpermd->allowed->p)); + goto out; +} /** * security_compute_av - Compute access vector decisions. @@ -905,6 +1076,7 @@ static void avd_init(struct av_decision *avd) * @tsid: target security identifier * @tclass: target security class * @avd: access vector decisions + * @xperms: extended permissions * * Compute a set of access vector decisions based on the * SID pair (@ssid, @tsid) for the permissions in @tclass. @@ -912,13 +1084,15 @@ static void avd_init(struct av_decision *avd) void security_compute_av(u32 ssid, u32 tsid, u16 orig_tclass, - struct av_decision *avd) + struct av_decision *avd, + struct extended_perms *xperms) { u16 tclass; struct context *scontext = NULL, *tcontext = NULL; read_lock(&policy_rwlock); avd_init(avd); + xperms->len = 0; if (!ss_initialized) goto allow; @@ -946,7 +1120,7 @@ void security_compute_av(u32 ssid, goto allow; goto out; } - context_struct_compute_av(scontext, tcontext, tclass, avd); + context_struct_compute_av(scontext, tcontext, tclass, avd, xperms); map_decision(orig_tclass, avd, policydb.allow_unknown); out: read_unlock(&policy_rwlock); @@ -992,7 +1166,7 @@ void security_compute_av_user(u32 ssid, goto out; } - context_struct_compute_av(scontext, tcontext, tclass, avd); + context_struct_compute_av(scontext, tcontext, tclass, avd, NULL); out: read_unlock(&policy_rwlock); return; @@ -1393,6 +1567,7 @@ static int security_compute_sid(u32 ssid, u32 *out_sid, bool kern) { + struct class_datum *cladatum = NULL; struct context *scontext = NULL, *tcontext = NULL, newcontext; struct role_trans *roletr = NULL; struct avtab_key avkey; @@ -1441,12 +1616,20 @@ static int security_compute_sid(u32 ssid, goto out_unlock; } + if (tclass && tclass <= policydb.p_classes.nprim) + cladatum = policydb.class_val_to_struct[tclass - 1]; + /* Set the user identity. */ switch (specified) { case AVTAB_TRANSITION: case AVTAB_CHANGE: - /* Use the process user identity. */ - newcontext.user = scontext->user; + if (cladatum && cladatum->default_user == DEFAULT_TARGET) { + newcontext.user = tcontext->user; + } else { + /* notice this gets both DEFAULT_SOURCE and unset */ + /* Use the process user identity. */ + newcontext.user = scontext->user; + } break; case AVTAB_MEMBER: /* Use the related object owner. */ @@ -1454,16 +1637,31 @@ static int security_compute_sid(u32 ssid, break; } - /* Set the role and type to default values. */ - if ((tclass == policydb.process_class) || (sock == true)) { - /* Use the current role and type of process. */ + /* Set the role to default values. */ + if (cladatum && cladatum->default_role == DEFAULT_SOURCE) { newcontext.role = scontext->role; - newcontext.type = scontext->type; + } else if (cladatum && cladatum->default_role == DEFAULT_TARGET) { + newcontext.role = tcontext->role; } else { - /* Use the well-defined object role. */ - newcontext.role = OBJECT_R_VAL; - /* Use the type of the related object. */ + if ((tclass == policydb.process_class) || (sock == true)) + newcontext.role = scontext->role; + else + newcontext.role = OBJECT_R_VAL; + } + + /* Set the type to default values. */ + if (cladatum && cladatum->default_type == DEFAULT_SOURCE) { + newcontext.type = scontext->type; + } else if (cladatum && cladatum->default_type == DEFAULT_TARGET) { newcontext.type = tcontext->type; + } else { + if ((tclass == policydb.process_class) || (sock == true)) { + /* Use the type of process. */ + newcontext.type = scontext->type; + } else { + /* Use the type of the related object. */ + newcontext.type = tcontext->type; + } } /* Look for a type transition/member/change rule. */ @@ -1486,7 +1684,7 @@ static int security_compute_sid(u32 ssid, if (avdatum) { /* Use the type from the type transition/member/change rule. */ - newcontext.type = avdatum->data; + newcontext.type = avdatum->u.data; } /* if we have a objname this is a file trans check so check those rules */ diff --git a/security/selinux/ss/services.h b/security/selinux/ss/services.h index e8d907e903c..6abcd8729ec 100644 --- a/security/selinux/ss/services.h +++ b/security/selinux/ss/services.h @@ -11,5 +11,11 @@ extern struct policydb policydb; +void services_compute_xperms_drivers(struct extended_perms *xperms, + struct avtab_node *node); + +void services_compute_xperms_decision(struct extended_perms_decision *xpermd, + struct avtab_node *node); + #endif /* _SS_SERVICES_H_ */ diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 45c32f07416..d1e6879b962 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -406,8 +406,8 @@ static int smack_sb_statfs(struct dentry *dentry) * Returns 0 if current can write the floor of the filesystem * being mounted on, an error code otherwise. */ -static int smack_sb_mount(char *dev_name, struct path *path, - char *type, unsigned long flags, void *data) +static int smack_sb_mount(const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data) { struct superblock_smack *sbp = path->dentry->d_sb->s_security; struct smk_audit_info ad; diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index 30fd9836970..997aa5bfaf6 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -971,7 +971,7 @@ int tomoyo_init_request_info(struct tomoyo_request_info *r, const u8 index); int tomoyo_mkdev_perm(const u8 operation, struct path *path, const unsigned int mode, unsigned int dev); -int tomoyo_mount_permission(char *dev_name, struct path *path, +int tomoyo_mount_permission(const char *dev_name, struct path *path, const char *type, unsigned long flags, void *data_page); int tomoyo_open_control(const u8 type, struct file *file); diff --git a/security/tomoyo/mount.c b/security/tomoyo/mount.c index fe00cdfd026..390c646013c 100644 --- a/security/tomoyo/mount.c +++ b/security/tomoyo/mount.c @@ -71,7 +71,8 @@ static bool tomoyo_check_mount_acl(struct tomoyo_request_info *r, * * Caller holds tomoyo_read_lock(). */ -static int tomoyo_mount_acl(struct tomoyo_request_info *r, char *dev_name, +static int tomoyo_mount_acl(struct tomoyo_request_info *r, + const char *dev_name, struct path *dir, const char *type, unsigned long flags) { @@ -183,7 +184,7 @@ static int tomoyo_mount_acl(struct tomoyo_request_info *r, char *dev_name, * * Returns 0 on success, negative value otherwise. */ -int tomoyo_mount_permission(char *dev_name, struct path *path, +int tomoyo_mount_permission(const char *dev_name, struct path *path, const char *type, unsigned long flags, void *data_page) { diff --git a/security/tomoyo/tomoyo.c b/security/tomoyo/tomoyo.c index 620d37c159a..43d3f97af46 100644 --- a/security/tomoyo/tomoyo.c +++ b/security/tomoyo/tomoyo.c @@ -406,8 +406,8 @@ static int tomoyo_path_chroot(struct path *path) * * Returns 0 on success, negative value otherwise. */ -static int tomoyo_sb_mount(char *dev_name, struct path *path, - char *type, unsigned long flags, void *data) +static int tomoyo_sb_mount(const char *dev_name, struct path *path, + const char *type, unsigned long flags, void *data) { return tomoyo_mount_permission(dev_name, path, type, flags, data); } diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index 8daa015a878..962601d6bf0 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -479,7 +479,7 @@ static int snd_compress_check_input(struct snd_compr_params *params) { /* first let's check the buffer parameter's */ if (params->buffer.fragment_size == 0 || - params->buffer.fragments > UINT_MAX / params->buffer.fragment_size) + params->buffer.fragments > INT_MAX / params->buffer.fragment_size) return -EINVAL; /* now codec parameters */ @@ -489,9 +489,6 @@ static int snd_compress_check_input(struct snd_compr_params *params) if (params->codec.ch_in == 0 || params->codec.ch_out == 0) return -EINVAL; - if (!(params->codec.sample_rate & SNDRV_PCM_RATE_8000_192000)) - return -EINVAL; - return 0; } @@ -612,9 +609,10 @@ snd_compr_set_metadata(struct snd_compr_stream *stream, unsigned long arg) static inline int snd_compr_tstamp(struct snd_compr_stream *stream, unsigned long arg) { - struct snd_compr_tstamp tstamp = {0}; + struct snd_compr_tstamp tstamp; int ret; + memset(&tstamp, 0, sizeof(tstamp)); ret = snd_compr_update_tstamp(stream, &tstamp); if (ret == 0) ret = copy_to_user((struct snd_compr_tstamp __user *)arg, diff --git a/sound/core/control.c b/sound/core/control.c index 2487a6bb1c5..29b8e4a5295 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -333,6 +333,9 @@ int snd_ctl_add(struct snd_card *card, struct snd_kcontrol *kcontrol) if (snd_BUG_ON(!card || !kcontrol->info)) goto error; id = kcontrol->id; + if (id.index > UINT_MAX - kcontrol->count) + goto error; + down_write(&card->controls_rwsem); if (snd_ctl_find_id(card, &id)) { up_write(&card->controls_rwsem); @@ -1130,8 +1133,6 @@ static int snd_ctl_elem_add(struct snd_ctl_file *file, struct user_element *ue; int idx, err; - if (!replace && card->user_ctl_count >= MAX_USER_CONTROLS) - return -ENOMEM; if (info->count < 1) return -EINVAL; access = info->access == 0 ? SNDRV_CTL_ELEM_ACCESS_READWRITE : @@ -1140,21 +1141,16 @@ static int snd_ctl_elem_add(struct snd_ctl_file *file, SNDRV_CTL_ELEM_ACCESS_TLV_READWRITE)); info->id.numid = 0; memset(&kctl, 0, sizeof(kctl)); - down_write(&card->controls_rwsem); - _kctl = snd_ctl_find_id(card, &info->id); - err = 0; - if (_kctl) { - if (replace) - err = snd_ctl_remove(card, _kctl); - else - err = -EBUSY; - } else { - if (replace) - err = -ENOENT; + + if (replace) { + err = snd_ctl_remove_user_ctl(file, &info->id); + if (err) + return err; } - up_write(&card->controls_rwsem); - if (err < 0) - return err; + + if (card->user_ctl_count >= MAX_USER_CONTROLS) + return -ENOMEM; + memcpy(&kctl.id, &info->id, sizeof(info->id)); kctl.count = info->owner ? info->owner : 1; access |= SNDRV_CTL_ELEM_ACCESS_USER; diff --git a/sound/core/hrtimer.c b/sound/core/hrtimer.c index b8b31c433d6..14d483d6b3b 100644 --- a/sound/core/hrtimer.c +++ b/sound/core/hrtimer.c @@ -90,7 +90,7 @@ static int snd_hrtimer_start(struct snd_timer *t) struct snd_hrtimer *stime = t->private_data; atomic_set(&stime->running, 0); - hrtimer_cancel(&stime->hrt); + hrtimer_try_to_cancel(&stime->hrt); hrtimer_start(&stime->hrt, ns_to_ktime(t->sticks * resolution), HRTIMER_MODE_REL); atomic_set(&stime->running, 1); @@ -101,6 +101,7 @@ static int snd_hrtimer_stop(struct snd_timer *t) { struct snd_hrtimer *stime = t->private_data; atomic_set(&stime->running, 0); + hrtimer_try_to_cancel(&stime->hrt); return 0; } diff --git a/sound/core/info.c b/sound/core/info.c index c1e611c65c8..99198e1cf97 100644 --- a/sound/core/info.c +++ b/sound/core/info.c @@ -260,6 +260,7 @@ static ssize_t snd_info_entry_write(struct file *file, const char __user *buffer struct snd_info_buffer *buf; ssize_t size = 0; loff_t pos; + unsigned long realloc_size; data = file->private_data; if (snd_BUG_ON(!data)) @@ -268,7 +269,8 @@ static ssize_t snd_info_entry_write(struct file *file, const char __user *buffer pos = *offset; if (pos < 0 || (long) pos != pos || (ssize_t) count < 0) return -EIO; - if ((unsigned long) pos + (unsigned long) count < (unsigned long) pos) + realloc_size = (unsigned long) pos + (unsigned long) count; + if (realloc_size < (unsigned long) pos || realloc_size > UINT_MAX) return -EIO; switch (entry->content) { case SNDRV_INFO_CONTENT_TEXT: diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c index 06d617b9e59..063f0701915 100644 --- a/sound/core/pcm_lib.c +++ b/sound/core/pcm_lib.c @@ -1767,10 +1767,10 @@ void snd_pcm_period_elapsed(struct snd_pcm_substream *substream) if (substream->timer_running) snd_timer_interrupt(substream->timer, 1); _end: - snd_pcm_stream_unlock_irqrestore(substream, flags); if (runtime->transfer_ack_end) runtime->transfer_ack_end(substream); kill_fasync(&runtime->fasync, SIGIO, POLL_IN); + snd_pcm_stream_unlock_irqrestore(substream, flags); } EXPORT_SYMBOL(snd_pcm_period_elapsed); diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c index f9077361c11..4c9aa462de9 100644 --- a/sound/core/seq/seq_queue.c +++ b/sound/core/seq/seq_queue.c @@ -144,8 +144,10 @@ static struct snd_seq_queue *queue_new(int owner, int locked) static void queue_delete(struct snd_seq_queue *q) { /* stop and release the timer */ + mutex_lock(&q->timer_mutex); snd_seq_timer_stop(q->timer); snd_seq_timer_close(q); + mutex_unlock(&q->timer_mutex); /* wait until access free */ snd_use_lock_sync(&q->use_lock); /* release resources... */ diff --git a/sound/core/timer.c b/sound/core/timer.c index 6ddcf06f52f..b9cbf2010ac 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -73,7 +73,7 @@ struct snd_timer_user { struct timespec tstamp; /* trigger tstamp */ wait_queue_head_t qchange_sleep; struct fasync_struct *fasync; - struct mutex tread_sem; + struct mutex ioctl_lock; }; /* list of timers */ @@ -215,11 +215,13 @@ static void snd_timer_check_master(struct snd_timer_instance *master) slave->slave_id == master->slave_id) { list_move_tail(&slave->open_list, &master->slave_list_head); spin_lock_irq(&slave_active_lock); + spin_lock(&master->timer->lock); slave->master = master; slave->timer = master->timer; if (slave->flags & SNDRV_TIMER_IFLG_RUNNING) list_add_tail(&slave->active_list, &master->slave_active_head); + spin_unlock(&master->timer->lock); spin_unlock_irq(&slave_active_lock); } } @@ -345,15 +347,18 @@ int snd_timer_close(struct snd_timer_instance *timeri) timer->hw.close) timer->hw.close(timer); /* remove slave links */ + spin_lock_irq(&slave_active_lock); + spin_lock(&timer->lock); list_for_each_entry_safe(slave, tmp, &timeri->slave_list_head, open_list) { - spin_lock_irq(&slave_active_lock); - _snd_timer_stop(slave, 1, SNDRV_TIMER_EVENT_RESOLUTION); list_move_tail(&slave->open_list, &snd_timer_slave_list); slave->master = NULL; slave->timer = NULL; - spin_unlock_irq(&slave_active_lock); + list_del_init(&slave->ack_list); + list_del_init(&slave->active_list); } + spin_unlock(&timer->lock); + spin_unlock_irq(&slave_active_lock); mutex_unlock(®ister_mutex); } out: @@ -440,9 +445,12 @@ static int snd_timer_start_slave(struct snd_timer_instance *timeri) spin_lock_irqsave(&slave_active_lock, flags); timeri->flags |= SNDRV_TIMER_IFLG_RUNNING; - if (timeri->master) + if (timeri->master && timeri->timer) { + spin_lock(&timeri->timer->lock); list_add_tail(&timeri->active_list, &timeri->master->slave_active_head); + spin_unlock(&timeri->timer->lock); + } spin_unlock_irqrestore(&slave_active_lock, flags); return 1; /* delayed start */ } @@ -488,6 +496,8 @@ static int _snd_timer_stop(struct snd_timer_instance * timeri, if (!keep_flag) { spin_lock_irqsave(&slave_active_lock, flags); timeri->flags &= ~SNDRV_TIMER_IFLG_RUNNING; + list_del_init(&timeri->ack_list); + list_del_init(&timeri->active_list); spin_unlock_irqrestore(&slave_active_lock, flags); } goto __end; @@ -693,7 +703,7 @@ void snd_timer_interrupt(struct snd_timer * timer, unsigned long ticks_left) } else { ti->flags &= ~SNDRV_TIMER_IFLG_RUNNING; if (--timer->running) - list_del(&ti->active_list); + list_del_init(&ti->active_list); } if ((timer->hw.flags & SNDRV_TIMER_HW_TASKLET) || (ti->flags & SNDRV_TIMER_IFLG_FAST)) @@ -1174,6 +1184,7 @@ static void snd_timer_user_ccallback(struct snd_timer_instance *timeri, tu->tstamp = *tstamp; if ((tu->filter & (1 << event)) == 0 || !tu->tread) return; + memset(&r1, 0, sizeof(r1)); r1.event = event; r1.tstamp = *tstamp; r1.val = resolution; @@ -1208,6 +1219,7 @@ static void snd_timer_user_tinterrupt(struct snd_timer_instance *timeri, } if ((tu->filter & (1 << SNDRV_TIMER_EVENT_RESOLUTION)) && tu->last_resolution != resolution) { + memset(&r1, 0, sizeof(r1)); r1.event = SNDRV_TIMER_EVENT_RESOLUTION; r1.tstamp = tstamp; r1.val = resolution; @@ -1256,7 +1268,7 @@ static int snd_timer_user_open(struct inode *inode, struct file *file) return -ENOMEM; spin_lock_init(&tu->qlock); init_waitqueue_head(&tu->qchange_sleep); - mutex_init(&tu->tread_sem); + mutex_init(&tu->ioctl_lock); tu->ticks = 1; tu->queue_size = 128; tu->queue = kmalloc(tu->queue_size * sizeof(struct snd_timer_read), @@ -1276,8 +1288,10 @@ static int snd_timer_user_release(struct inode *inode, struct file *file) if (file->private_data) { tu = file->private_data; file->private_data = NULL; + mutex_lock(&tu->ioctl_lock); if (tu->timeri) snd_timer_close(tu->timeri); + mutex_unlock(&tu->ioctl_lock); kfree(tu->queue); kfree(tu->tqueue); kfree(tu); @@ -1515,7 +1529,6 @@ static int snd_timer_user_tselect(struct file *file, int err = 0; tu = file->private_data; - mutex_lock(&tu->tread_sem); if (tu->timeri) { snd_timer_close(tu->timeri); tu->timeri = NULL; @@ -1559,7 +1572,6 @@ static int snd_timer_user_tselect(struct file *file, } __err: - mutex_unlock(&tu->tread_sem); return err; } @@ -1673,6 +1685,7 @@ static int snd_timer_user_params(struct file *file, if (tu->timeri->flags & SNDRV_TIMER_IFLG_EARLY_EVENT) { if (tu->tread) { struct snd_timer_tread tread; + memset(&tread, 0, sizeof(tread)); tread.event = SNDRV_TIMER_EVENT_EARLY; tread.tstamp.tv_sec = 0; tread.tstamp.tv_nsec = 0; @@ -1772,7 +1785,7 @@ enum { SNDRV_TIMER_IOCTL_PAUSE_OLD = _IO('T', 0x23), }; -static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, +static long __snd_timer_user_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct snd_timer_user *tu; @@ -1789,17 +1802,11 @@ static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, { int xarg; - mutex_lock(&tu->tread_sem); - if (tu->timeri) { /* too late */ - mutex_unlock(&tu->tread_sem); + if (tu->timeri) /* too late */ return -EBUSY; - } - if (get_user(xarg, p)) { - mutex_unlock(&tu->tread_sem); + if (get_user(xarg, p)) return -EFAULT; - } tu->tread = xarg ? 1 : 0; - mutex_unlock(&tu->tread_sem); return 0; } case SNDRV_TIMER_IOCTL_GINFO: @@ -1832,6 +1839,18 @@ static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, return -ENOTTY; } +static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct snd_timer_user *tu = file->private_data; + long ret; + + mutex_lock(&tu->ioctl_lock); + ret = __snd_timer_user_ioctl(file, cmd, arg); + mutex_unlock(&tu->ioctl_lock); + return ret; +} + static int snd_timer_user_fasync(int fd, struct file * file, int on) { struct snd_timer_user *tu; diff --git a/sound/soc/msm/qdsp6v2/audio_acdb.c b/sound/soc/msm/qdsp6v2/audio_acdb.c index 5da82983548..f8e1b7a41e1 100644 --- a/sound/soc/msm/qdsp6v2/audio_acdb.c +++ b/sound/soc/msm/qdsp6v2/audio_acdb.c @@ -37,126 +37,123 @@ #define ACDB_TOTAL_VOICE_ALLOCATION (ACDB_BLOCK_SIZE * NUM_VOCPROC_BLOCKS) -struct sidetone_atomic_cal { - atomic_t enable; - atomic_t gain; -}; - - struct acdb_data { + uint32_t usage_count; + struct mutex acdb_mutex; /* ANC Cal */ - struct acdb_atomic_cal_block anc_cal; + struct acdb_cal_block anc_cal; /* AANC Cal */ - struct acdb_atomic_cal_block aanc_cal; + struct acdb_cal_block aanc_cal; /* LSM Cal */ - struct acdb_atomic_cal_block lsm_cal; + struct acdb_cal_block lsm_cal; /* AudProc Cal */ - atomic_t asm_topology; - atomic_t adm_topology[MAX_AUDPROC_TYPES]; - struct acdb_atomic_cal_block audproc_cal[MAX_AUDPROC_TYPES]; - struct acdb_atomic_cal_block audstrm_cal[MAX_AUDPROC_TYPES]; - struct acdb_atomic_cal_block audvol_cal[MAX_AUDPROC_TYPES]; + uint32_t asm_topology; + uint32_t adm_topology[MAX_AUDPROC_TYPES]; + struct acdb_cal_block audproc_cal[MAX_AUDPROC_TYPES]; + struct acdb_cal_block audstrm_cal[MAX_AUDPROC_TYPES]; + struct acdb_cal_block audvol_cal[MAX_AUDPROC_TYPES]; /* VocProc Cal */ - atomic_t voice_rx_topology; - atomic_t voice_tx_topology; - struct acdb_atomic_cal_block vocproc_cal; - struct acdb_atomic_cal_block vocstrm_cal; - struct acdb_atomic_cal_block vocvol_cal; + uint32_t voice_rx_topology; + uint32_t voice_tx_topology; + struct acdb_cal_block vocproc_cal; + struct acdb_cal_block vocstrm_cal; + struct acdb_cal_block vocvol_cal; /* Voice Column data */ - struct acdb_atomic_cal_block vocproc_col_cal[MAX_VOCPROC_TYPES]; + struct acdb_cal_block vocproc_col_cal[MAX_VOCPROC_TYPES]; uint32_t *col_data[MAX_VOCPROC_TYPES]; /* VocProc dev cfg cal*/ - struct acdb_atomic_cal_block vocproc_dev_cal; + struct acdb_cal_block vocproc_dev_cal; /* Custom topology */ - struct acdb_atomic_cal_block adm_custom_topology; - struct acdb_atomic_cal_block asm_custom_topology; - atomic_t valid_adm_custom_top; - atomic_t valid_asm_custom_top; + struct acdb_cal_block adm_custom_topology; + struct acdb_cal_block asm_custom_topology; + uint32_t valid_adm_custom_top; + uint32_t valid_asm_custom_top; /* AFE cal */ - struct acdb_atomic_cal_block afe_cal[MAX_AUDPROC_TYPES]; + struct acdb_cal_block afe_cal[MAX_AUDPROC_TYPES]; /* Sidetone Cal */ - struct sidetone_atomic_cal sidetone_cal; + struct sidetone_cal sidetone_cal; /* Allocation information */ struct ion_client *ion_client; struct ion_handle *ion_handle; - atomic_t map_handle; - atomic64_t paddr; - atomic64_t kvaddr; - atomic64_t mem_len; + uint32_t map_handle; + uint64_t paddr; + uint64_t kvaddr; + uint64_t mem_len; /* Speaker protection */ struct msm_spk_prot_cfg spk_prot_cfg; }; static struct acdb_data acdb_data; -static atomic_t usage_count; uint32_t get_voice_rx_topology(void) { - return atomic_read(&acdb_data.voice_rx_topology); + return acdb_data.voice_rx_topology; } void store_voice_rx_topology(uint32_t topology) { - atomic_set(&acdb_data.voice_rx_topology, topology); + acdb_data.voice_rx_topology = topology; } uint32_t get_voice_tx_topology(void) { - return atomic_read(&acdb_data.voice_tx_topology); + return acdb_data.voice_tx_topology; } void store_voice_tx_topology(uint32_t topology) { - atomic_set(&acdb_data.voice_tx_topology, topology); + acdb_data.voice_tx_topology = topology; } uint32_t get_adm_rx_topology(void) { - return atomic_read(&acdb_data.adm_topology[RX_CAL]); + return acdb_data.adm_topology[RX_CAL]; } void store_adm_rx_topology(uint32_t topology) { - atomic_set(&acdb_data.adm_topology[RX_CAL], topology); + acdb_data.adm_topology[RX_CAL] = topology; } uint32_t get_adm_tx_topology(void) { - return atomic_read(&acdb_data.adm_topology[TX_CAL]); + return acdb_data.adm_topology[TX_CAL]; } void store_adm_tx_topology(uint32_t topology) { - atomic_set(&acdb_data.adm_topology[TX_CAL], topology); + acdb_data.adm_topology[TX_CAL] = topology; } uint32_t get_asm_topology(void) { - return atomic_read(&acdb_data.asm_topology); + return acdb_data.asm_topology; } void store_asm_topology(uint32_t topology) { - atomic_set(&acdb_data.asm_topology, topology); + acdb_data.asm_topology = topology; } void reset_custom_topology_flags(void) { - atomic_set(&acdb_data.valid_adm_custom_top, 1); - atomic_set(&acdb_data.valid_asm_custom_top, 1); + mutex_lock(&acdb_data.acdb_mutex); + acdb_data.valid_adm_custom_top = 1; + acdb_data.valid_asm_custom_top = 1; + mutex_unlock(&acdb_data.acdb_mutex); } int get_adm_custom_topology(struct acdb_cal_block *cal_block) @@ -170,19 +167,19 @@ int get_adm_custom_topology(struct acdb_cal_block *cal_block) goto done; } + mutex_lock(&acdb_data.acdb_mutex); /* Only return allow one access after memory registered */ - if (atomic_read(&acdb_data.valid_adm_custom_top) == 0) { + if (acdb_data.valid_adm_custom_top == 0) { cal_block->cal_size = 0; - goto done; + goto unlock; } - atomic_set(&acdb_data.valid_adm_custom_top, 0); + acdb_data.valid_adm_custom_top = 0; - cal_block->cal_size = - atomic_read(&acdb_data.adm_custom_topology.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.adm_custom_topology.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.adm_custom_topology.cal_kvaddr); + cal_block->cal_size = acdb_data.adm_custom_topology.cal_size; + cal_block->cal_paddr = acdb_data.adm_custom_topology.cal_paddr; + cal_block->cal_kvaddr = acdb_data.adm_custom_topology.cal_kvaddr; +unlock: + mutex_unlock(&acdb_data.acdb_mutex); done: return result; } @@ -192,21 +189,18 @@ int store_adm_custom_topology(struct cal_block *cal_block) int result = 0; pr_debug("%s,\n", __func__); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } - atomic_set(&acdb_data.adm_custom_topology.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.adm_custom_topology.cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.adm_custom_topology.cal_kvaddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.kvaddr)); + acdb_data.adm_custom_topology.cal_size = cal_block->cal_size; + acdb_data.adm_custom_topology.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.adm_custom_topology.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -222,19 +216,19 @@ int get_asm_custom_topology(struct acdb_cal_block *cal_block) goto done; } + mutex_lock(&acdb_data.acdb_mutex); /* Only return allow one access after memory registered */ - if (atomic_read(&acdb_data.valid_asm_custom_top) == 0) { + if (acdb_data.valid_asm_custom_top == 0) { cal_block->cal_size = 0; - goto done; + goto unlock; } - atomic_set(&acdb_data.valid_asm_custom_top, 0); + acdb_data.valid_asm_custom_top = 0; - cal_block->cal_size = - atomic_read(&acdb_data.asm_custom_topology.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.asm_custom_topology.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.asm_custom_topology.cal_kvaddr); + cal_block->cal_size = acdb_data.asm_custom_topology.cal_size; + cal_block->cal_paddr = acdb_data.asm_custom_topology.cal_paddr; + cal_block->cal_kvaddr = acdb_data.asm_custom_topology.cal_kvaddr; +unlock: + mutex_unlock(&acdb_data.acdb_mutex); done: return result; } @@ -244,21 +238,18 @@ int store_asm_custom_topology(struct cal_block *cal_block) int result = 0; pr_debug("%s,\n", __func__); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } - atomic_set(&acdb_data.asm_custom_topology.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.asm_custom_topology.cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.asm_custom_topology.cal_kvaddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.kvaddr)); + acdb_data.asm_custom_topology.cal_size = cal_block->cal_size; + acdb_data.asm_custom_topology.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.asm_custom_topology.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -275,10 +266,8 @@ int get_voice_cal_allocation(struct acdb_cal_block *cal_block) } cal_block->cal_size = ACDB_TOTAL_VOICE_ALLOCATION; - cal_block->cal_paddr = - atomic_read(&acdb_data.vocproc_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.vocproc_cal.cal_kvaddr); + cal_block->cal_paddr = acdb_data.vocproc_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.vocproc_cal.cal_kvaddr; done: return result; } @@ -294,12 +283,9 @@ int get_aanc_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.aanc_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.aanc_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.aanc_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.aanc_cal.cal_size; + cal_block->cal_paddr = acdb_data.aanc_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.aanc_cal.cal_kvaddr; done: return result; } @@ -309,20 +295,18 @@ int store_aanc_cal(struct cal_block *cal_block) int result = 0; pr_debug("%s,\n", __func__); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } - atomic_set(&acdb_data.aanc_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.aanc_cal.cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.aanc_cal.cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.aanc_cal.cal_size = cal_block->cal_size; + acdb_data.aanc_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.aanc_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -338,12 +322,9 @@ int get_lsm_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.lsm_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.lsm_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.lsm_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.lsm_cal.cal_size; + cal_block->cal_paddr = acdb_data.lsm_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.lsm_cal.cal_kvaddr; done: return result; } @@ -353,20 +334,18 @@ int store_lsm_cal(struct cal_block *cal_block) int result = 0; pr_debug("%s,\n", __func__); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } - atomic_set(&acdb_data.lsm_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.lsm_cal.cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.lsm_cal.cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.lsm_cal.cal_size = cal_block->cal_size; + acdb_data.lsm_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.lsm_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -382,12 +361,9 @@ int get_anc_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.anc_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.anc_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.anc_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.anc_cal.cal_size; + cal_block->cal_paddr = acdb_data.anc_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.anc_cal.cal_kvaddr; done: return result; } @@ -397,20 +373,18 @@ int store_anc_cal(struct cal_block *cal_block) int result = 0; pr_debug("%s,\n", __func__); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } - atomic_set(&acdb_data.anc_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.anc_cal.cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.anc_cal.cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.anc_cal.cal_size = cal_block->cal_size; + acdb_data.anc_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.anc_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -420,10 +394,9 @@ int store_afe_cal(int32_t path, struct cal_block *cal_block) int result = 0; pr_debug("%s, path = %d\n", __func__, path); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } @@ -434,12 +407,11 @@ int store_afe_cal(int32_t path, struct cal_block *cal_block) goto done; } - atomic_set(&acdb_data.afe_cal[path].cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.afe_cal[path].cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.afe_cal[path].cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.afe_cal[path].cal_size = cal_block->cal_size; + acdb_data.afe_cal[path].cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.afe_cal[path].cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -461,12 +433,9 @@ int get_afe_cal(int32_t path, struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.afe_cal[path].cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.afe_cal[path].cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.afe_cal[path].cal_kvaddr); + cal_block->cal_size = acdb_data.afe_cal[path].cal_size; + cal_block->cal_paddr = acdb_data.afe_cal[path].cal_paddr; + cal_block->cal_kvaddr = acdb_data.afe_cal[path].cal_kvaddr; done: return result; } @@ -476,10 +445,9 @@ int store_audproc_cal(int32_t path, struct cal_block *cal_block) int result = 0; pr_debug("%s, path = %d\n", __func__, path); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } @@ -490,12 +458,11 @@ int store_audproc_cal(int32_t path, struct cal_block *cal_block) goto done; } - atomic_set(&acdb_data.audproc_cal[path].cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.audproc_cal[path].cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.audproc_cal[path].cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.audproc_cal[path].cal_size = cal_block->cal_size; + acdb_data.audproc_cal[path].cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.audproc_cal[path].cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -517,12 +484,9 @@ int get_audproc_cal(int32_t path, struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.audproc_cal[path].cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.audproc_cal[path].cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.audproc_cal[path].cal_kvaddr); + cal_block->cal_size = acdb_data.audproc_cal[path].cal_size; + cal_block->cal_paddr = acdb_data.audproc_cal[path].cal_paddr; + cal_block->cal_kvaddr = acdb_data.audproc_cal[path].cal_kvaddr; done: return result; } @@ -532,10 +496,9 @@ int store_audstrm_cal(int32_t path, struct cal_block *cal_block) int result = 0; pr_debug("%s, path = %d\n", __func__, path); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } @@ -546,12 +509,11 @@ int store_audstrm_cal(int32_t path, struct cal_block *cal_block) goto done; } - atomic_set(&acdb_data.audstrm_cal[path].cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.audstrm_cal[path].cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.audstrm_cal[path].cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.audstrm_cal[path].cal_size = cal_block->cal_size; + acdb_data.audstrm_cal[path].cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.audstrm_cal[path].cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -573,12 +535,9 @@ int get_audstrm_cal(int32_t path, struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.audstrm_cal[path].cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.audstrm_cal[path].cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.audstrm_cal[path].cal_kvaddr); + cal_block->cal_size = acdb_data.audstrm_cal[path].cal_size; + cal_block->cal_paddr = acdb_data.audstrm_cal[path].cal_paddr; + cal_block->cal_kvaddr = acdb_data.audstrm_cal[path].cal_kvaddr; done: return result; } @@ -588,10 +547,9 @@ int store_audvol_cal(int32_t path, struct cal_block *cal_block) int result = 0; pr_debug("%s, path = %d\n", __func__, path); - if (cal_block->cal_offset > atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); result = -EINVAL; goto done; } @@ -602,12 +560,11 @@ int store_audvol_cal(int32_t path, struct cal_block *cal_block) goto done; } - atomic_set(&acdb_data.audvol_cal[path].cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.audvol_cal[path].cal_paddr, - cal_block->cal_offset + atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.audvol_cal[path].cal_kvaddr, - cal_block->cal_offset + atomic64_read(&acdb_data.kvaddr)); + acdb_data.audvol_cal[path].cal_size = cal_block->cal_size; + acdb_data.audvol_cal[path].cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.audvol_cal[path].cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -629,12 +586,9 @@ int get_audvol_cal(int32_t path, struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.audvol_cal[path].cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.audvol_cal[path].cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.audvol_cal[path].cal_kvaddr); + cal_block->cal_size = acdb_data.audvol_cal[path].cal_size; + cal_block->cal_paddr = acdb_data.audvol_cal[path].cal_paddr; + cal_block->cal_kvaddr = acdb_data.audvol_cal[path].cal_kvaddr; done: return result; } @@ -645,22 +599,30 @@ int store_voice_col_data(uint32_t vocproc_type, uint32_t cal_size, int result = 0; pr_debug("%s,\n", __func__); + if (cal_block == NULL) { + pr_err("ACDB=> NULL pointer sent to %s\n", __func__); + result = -EINVAL; + goto done; + } if (cal_size > MAX_COL_SIZE) { - pr_err("%s: col size is to big %d\n", __func__, - cal_size); + pr_err("%s: col size is to big %d\n", __func__, cal_size); + result = -EINVAL; + goto done; + } + if (acdb_data.col_data[vocproc_type] == NULL) { + pr_err("%s: vocproc_type %d data not allocated!\n", + __func__, vocproc_type); result = -EINVAL; goto done; } if (copy_from_user(acdb_data.col_data[vocproc_type], (void *)((uint8_t *)cal_block + sizeof(cal_size)), cal_size)) { - pr_err("%s: fail to copy col size %d\n", - __func__, cal_size); + pr_err("%s: fail to copy col size %d\n", __func__, cal_size); result = -EINVAL; goto done; } - atomic_set(&acdb_data.vocproc_col_cal[vocproc_type].cal_size, - cal_size); + acdb_data.vocproc_col_cal[vocproc_type].cal_size = cal_size; done: return result; } @@ -676,13 +638,19 @@ int get_voice_col_data(uint32_t vocproc_type, result = -EINVAL; goto done; } + if (acdb_data.col_data[vocproc_type] == NULL) { + pr_err("%s: vocproc_type %d data not allocated!\n", + __func__, vocproc_type); + result = -EINVAL; + goto done; + } - cal_block->cal_size = atomic_read(&acdb_data. - vocproc_col_cal[vocproc_type].cal_size); - cal_block->cal_paddr = atomic_read(&acdb_data. - vocproc_col_cal[vocproc_type].cal_paddr); - cal_block->cal_kvaddr = atomic_read(&acdb_data. - vocproc_col_cal[vocproc_type].cal_kvaddr); + cal_block->cal_size = acdb_data. + vocproc_col_cal[vocproc_type].cal_size; + cal_block->cal_paddr = acdb_data. + vocproc_col_cal[vocproc_type].cal_paddr; + cal_block->cal_kvaddr = acdb_data. + vocproc_col_cal[vocproc_type].cal_kvaddr; done: return result; } @@ -694,24 +662,19 @@ int store_vocproc_dev_cfg_cal(struct cal_block *cal_block) if (cal_block->cal_offset > - atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); - atomic_set(&acdb_data.vocproc_dev_cal.cal_size, 0); + acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); + acdb_data.vocproc_dev_cal.cal_size = 0; result = -EINVAL; goto done; } - atomic_set(&acdb_data.vocproc_dev_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.vocproc_dev_cal.cal_paddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.vocproc_dev_cal.cal_kvaddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.kvaddr)); - + acdb_data.vocproc_dev_cal.cal_size = cal_block->cal_size; + acdb_data.vocproc_dev_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.vocproc_dev_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -727,12 +690,9 @@ int get_vocproc_dev_cfg_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.vocproc_dev_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.vocproc_dev_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.vocproc_dev_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.vocproc_dev_cal.cal_size; + cal_block->cal_paddr = acdb_data.vocproc_dev_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.vocproc_dev_cal.cal_kvaddr; done: return result; } @@ -745,24 +705,19 @@ int store_vocproc_cal(struct cal_block *cal_block) pr_debug("%s,\n", __func__); if (cal_block->cal_offset > - atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); - atomic_set(&acdb_data.vocproc_cal.cal_size, 0); + acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); + acdb_data.vocproc_cal.cal_size = 0; result = -EINVAL; goto done; } - atomic_set(&acdb_data.vocproc_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.vocproc_cal.cal_paddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.vocproc_cal.cal_kvaddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.kvaddr)); - + acdb_data.vocproc_cal.cal_size = cal_block->cal_size; + acdb_data.vocproc_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.vocproc_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -778,12 +733,9 @@ int get_vocproc_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.vocproc_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.vocproc_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.vocproc_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.vocproc_cal.cal_size; + cal_block->cal_paddr = acdb_data.vocproc_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.vocproc_cal.cal_kvaddr; done: return result; } @@ -794,24 +746,19 @@ int store_vocstrm_cal(struct cal_block *cal_block) pr_debug("%s,\n", __func__); if (cal_block->cal_offset > - atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); - atomic_set(&acdb_data.vocstrm_cal.cal_size, 0); + acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); + acdb_data.vocstrm_cal.cal_size = 0; result = -EINVAL; goto done; } - atomic_set(&acdb_data.vocstrm_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.vocstrm_cal.cal_paddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.vocstrm_cal.cal_kvaddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.kvaddr)); - + acdb_data.vocstrm_cal.cal_size = cal_block->cal_size; + acdb_data.vocstrm_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.vocstrm_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -827,12 +774,9 @@ int get_vocstrm_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.vocstrm_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.vocstrm_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.vocstrm_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.vocstrm_cal.cal_size; + cal_block->cal_paddr = acdb_data.vocstrm_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.vocstrm_cal.cal_kvaddr; done: return result; } @@ -842,25 +786,19 @@ int store_vocvol_cal(struct cal_block *cal_block) int result = 0; pr_debug("%s,\n", __func__); - if (cal_block->cal_offset > - atomic64_read(&acdb_data.mem_len)) { - pr_err("%s: offset %d is > mem_len %ld\n", - __func__, cal_block->cal_offset, - (long)atomic64_read(&acdb_data.mem_len)); - atomic_set(&acdb_data.vocvol_cal.cal_size, 0); + if (cal_block->cal_offset > acdb_data.mem_len) { + pr_err("%s: offset %d is > mem_len %llu\n", + __func__, cal_block->cal_offset, acdb_data.mem_len); + acdb_data.vocvol_cal.cal_size = 0; result = -EINVAL; goto done; } - atomic_set(&acdb_data.vocvol_cal.cal_size, - cal_block->cal_size); - atomic_set(&acdb_data.vocvol_cal.cal_paddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.paddr)); - atomic_set(&acdb_data.vocvol_cal.cal_kvaddr, - cal_block->cal_offset + - atomic64_read(&acdb_data.kvaddr)); - + acdb_data.vocvol_cal.cal_size = cal_block->cal_size; + acdb_data.vocvol_cal.cal_paddr = + cal_block->cal_offset + acdb_data.paddr; + acdb_data.vocvol_cal.cal_kvaddr = + cal_block->cal_offset + acdb_data.kvaddr; done: return result; } @@ -876,12 +814,9 @@ int get_vocvol_cal(struct acdb_cal_block *cal_block) goto done; } - cal_block->cal_size = - atomic_read(&acdb_data.vocvol_cal.cal_size); - cal_block->cal_paddr = - atomic_read(&acdb_data.vocvol_cal.cal_paddr); - cal_block->cal_kvaddr = - atomic_read(&acdb_data.vocvol_cal.cal_kvaddr); + cal_block->cal_size = acdb_data.vocvol_cal.cal_size; + cal_block->cal_paddr = acdb_data.vocvol_cal.cal_paddr; + cal_block->cal_kvaddr = acdb_data.vocvol_cal.cal_kvaddr; done: return result; } @@ -890,8 +825,8 @@ void store_sidetone_cal(struct sidetone_cal *cal_data) { pr_debug("%s,\n", __func__); - atomic_set(&acdb_data.sidetone_cal.enable, cal_data->enable); - atomic_set(&acdb_data.sidetone_cal.gain, cal_data->gain); + acdb_data.sidetone_cal.enable = cal_data->enable; + acdb_data.sidetone_cal.gain = cal_data->gain; } int get_sidetone_cal(struct sidetone_cal *cal_data) @@ -905,8 +840,10 @@ int get_sidetone_cal(struct sidetone_cal *cal_data) goto done; } - cal_data->enable = atomic_read(&acdb_data.sidetone_cal.enable); - cal_data->gain = atomic_read(&acdb_data.sidetone_cal.gain); + mutex_lock(&acdb_data.acdb_mutex); + cal_data->enable = acdb_data.sidetone_cal.enable; + cal_data->gain = acdb_data.sidetone_cal.gain; + mutex_unlock(&acdb_data.acdb_mutex); done: return result; } @@ -990,14 +927,48 @@ static int acdb_open(struct inode *inode, struct file *f) s32 result = 0; pr_debug("%s\n", __func__); - if (atomic64_read(&acdb_data.mem_len)) { + mutex_lock(&acdb_data.acdb_mutex); + if (acdb_data.mem_len) pr_debug("%s: ACDB opened but memory allocated, using existing allocation!\n", __func__); + + acdb_data.valid_adm_custom_top = 1; + acdb_data.valid_asm_custom_top = 1; + acdb_data.usage_count++; + mutex_unlock(&acdb_data.acdb_mutex); + + return result; +} + +static void deallocate_col_data(void) +{ + int i; + + for (i = 0; i < MAX_VOCPROC_TYPES; i++) { + kfree(acdb_data.col_data[i]); + acdb_data.col_data[i] = NULL; + } +} + +static int allocate_col_data(void) +{ + int result = 0; + int i; + + for (i = 0; i < MAX_VOCPROC_TYPES; i++) { + acdb_data.col_data[i] = kmalloc(MAX_COL_SIZE, GFP_KERNEL); + if (acdb_data.col_data[i] == NULL) { + pr_err("%s: kmalloc column data failed, type = %d\n", + __func__, i); + deallocate_col_data(); + result = -ENOMEM; + goto done; + } + acdb_data.vocproc_col_cal[i].cal_kvaddr = + (uint32_t)acdb_data.col_data[i]; } - atomic_set(&acdb_data.valid_adm_custom_top, 1); - atomic_set(&acdb_data.valid_asm_custom_top, 1); - atomic_inc(&usage_count); +done: return result; } @@ -1047,77 +1018,71 @@ static int unmap_cal_tables(void) static int deregister_memory(void) { int result = 0; - int i; pr_debug("%s\n", __func__); - if (atomic64_read(&acdb_data.mem_len)) { - mutex_lock(&acdb_data.acdb_mutex); - /* unmap all cal data */ - result = unmap_cal_tables(); - if (result < 0) - pr_err("%s: unmap_cal_tables failed, err = %d\n", - __func__, result); + if (acdb_data.mem_len == 0) + goto done; + pr_debug("Remove existing memory\n"); + acdb_data.mem_len = 0; - atomic64_set(&acdb_data.mem_len, 0); + /* unmap all cal data */ + result = unmap_cal_tables(); + if (result < 0) + pr_err("%s: unmap_cal_tables failed, err = %d\n", + __func__, result); - for (i = 0; i < MAX_VOCPROC_TYPES; i++) { - kfree(acdb_data.col_data[i]); - acdb_data.col_data[i] = NULL; - } - msm_audio_ion_free(acdb_data.ion_client, acdb_data.ion_handle); - mutex_unlock(&acdb_data.acdb_mutex); - } + msm_audio_ion_free(acdb_data.ion_client, acdb_data.ion_handle); + acdb_data.ion_client = NULL; + acdb_data.ion_handle = NULL; + + deallocate_col_data(); +done: return result; } static int register_memory(void) { int result; - int i; ion_phys_addr_t paddr; void *kvptr; unsigned long kvaddr; unsigned long mem_len; pr_debug("%s\n", __func__); - mutex_lock(&acdb_data.acdb_mutex); - for (i = 0; i < MAX_VOCPROC_TYPES; i++) { - acdb_data.col_data[i] = kmalloc(MAX_COL_SIZE, GFP_KERNEL); - atomic_set(&acdb_data.vocproc_col_cal[i].cal_kvaddr, - (uint32_t)acdb_data.col_data[i]); + result = allocate_col_data(); + if (result) { + pr_err("%s: allocate_hw_delay_entries failed, rc = %d\n", + __func__, result); + goto err_done; } result = msm_audio_ion_import("audio_acdb_client", &acdb_data.ion_client, &acdb_data.ion_handle, - atomic_read(&acdb_data.map_handle), + acdb_data.map_handle, NULL, 0, &paddr, (size_t *)&mem_len, &kvptr); if (result) { pr_err("%s: audio ION alloc failed, rc = %d\n", __func__, result); - result = PTR_ERR(acdb_data.ion_client); - goto err_ion_handle; + goto err_col; } + kvaddr = (unsigned long)kvptr; - atomic64_set(&acdb_data.paddr, paddr); - atomic64_set(&acdb_data.kvaddr, kvaddr); - atomic64_set(&acdb_data.mem_len, mem_len); - mutex_unlock(&acdb_data.acdb_mutex); + acdb_data.paddr = paddr; + acdb_data.kvaddr = kvaddr; + acdb_data.mem_len = mem_len; - pr_debug("%s done! paddr = 0x%lx, kvaddr = 0x%lx, len = x%lx\n", - __func__, - (long)atomic64_read(&acdb_data.paddr), - (long)atomic64_read(&acdb_data.kvaddr), - (long)atomic64_read(&acdb_data.mem_len)); + pr_debug("%s done! paddr = 0x%llx, kvaddr = 0x%llx, len = 0x%llx\n", + __func__, acdb_data.paddr, acdb_data.kvaddr, + acdb_data.mem_len); return result; -err_ion_handle: - msm_audio_ion_free(acdb_data.ion_client, acdb_data.ion_handle); - - atomic64_set(&acdb_data.mem_len, 0); - mutex_unlock(&acdb_data.acdb_mutex); +err_col: + deallocate_col_data(); +err_done: + acdb_data.mem_len = 0; return result; } static long acdb_ioctl(struct file *f, @@ -1132,26 +1097,26 @@ static long acdb_ioctl(struct file *f, struct msm_spk_prot_status acdb_spk_status; pr_debug("%s\n", __func__); + mutex_lock(&acdb_data.acdb_mutex); switch (cmd) { case AUDIO_REGISTER_PMEM: pr_debug("AUDIO_REGISTER_PMEM\n"); - if (atomic_read(&acdb_data.mem_len)) { - deregister_memory(); - pr_debug("Remove the existing memory\n"); - } + result = deregister_memory(); + if (result < 0) + pr_err("%s: deregister_memory failed returned %d!\n", + __func__, result); if (copy_from_user(&map_fd, (void *)arg, sizeof(map_fd))) { pr_err("%s: fail to copy memory handle!\n", __func__); result = -EFAULT; } else { - atomic_set(&acdb_data.map_handle, map_fd); + acdb_data.map_handle = map_fd; result = register_memory(); } goto done; - case AUDIO_DEREGISTER_PMEM: pr_debug("AUDIO_DEREGISTER_PMEM\n"); - deregister_memory(); + result = deregister_memory(); goto done; case AUDIO_SET_VOICE_RX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, @@ -1194,16 +1159,13 @@ static long acdb_ioctl(struct file *f, store_asm_topology(topology); goto done; case AUDIO_SET_SPEAKER_PROT: - mutex_lock(&acdb_data.acdb_mutex); if (copy_from_user(&acdb_data.spk_prot_cfg, (void *)arg, sizeof(acdb_data.spk_prot_cfg))) { pr_err("%s fail to copy spk_prot_cfg\n", __func__); result = -EFAULT; } - mutex_unlock(&acdb_data.acdb_mutex); goto done; case AUDIO_GET_SPEAKER_PROT: - mutex_lock(&acdb_data.acdb_mutex); /*Indicates calibration was succesfull*/ if (acdb_data.spk_prot_cfg.mode == MSM_SPKR_PROT_CALIBRATED) { prot_status.r0 = acdb_data.spk_prot_cfg.r0; @@ -1230,7 +1192,6 @@ static long acdb_ioctl(struct file *f, sizeof(prot_status))) { pr_err("%s: Failed to update prot_status\n", __func__); } - mutex_unlock(&acdb_data.acdb_mutex); goto done; case AUDIO_REGISTER_VOCPROC_VOL_TABLE: result = register_vocvol_table(); @@ -1347,23 +1308,25 @@ static long acdb_ioctl(struct file *f, } done: + mutex_unlock(&acdb_data.acdb_mutex); return result; } static int acdb_mmap(struct file *file, struct vm_area_struct *vma) { int result = 0; - int size = vma->vm_end - vma->vm_start; + size_t size = vma->vm_end - vma->vm_start; pr_debug("%s\n", __func__); - if (atomic64_read(&acdb_data.mem_len)) { - if (size <= atomic64_read(&acdb_data.mem_len)) { + mutex_lock(&acdb_data.acdb_mutex); + if (acdb_data.mem_len) { + if (size <= acdb_data.mem_len) { vma->vm_page_prot = pgprot_noncached( vma->vm_page_prot); result = remap_pfn_range(vma, vma->vm_start, - atomic64_read(&acdb_data.paddr) >> PAGE_SHIFT, + acdb_data.paddr >> PAGE_SHIFT, size, vma->vm_page_prot); } else { @@ -1374,25 +1337,29 @@ static int acdb_mmap(struct file *file, struct vm_area_struct *vma) pr_err("%s: memory is not allocated, yet!\n", __func__); result = -ENODEV; } + mutex_unlock(&acdb_data.acdb_mutex); return result; } static int acdb_release(struct inode *inode, struct file *f) { - s32 result = 0; + int result = 0; + pr_debug("%s\n", __func__); - atomic_dec(&usage_count); - atomic_read(&usage_count); + mutex_lock(&acdb_data.acdb_mutex); + acdb_data.usage_count--; - pr_debug("%s: ref count %d!\n", __func__, - atomic_read(&usage_count)); + pr_debug("%s: ref count %d!\n", __func__, acdb_data.usage_count); - if (atomic_read(&usage_count) >= 1) + if (acdb_data.usage_count > 0) { result = -EBUSY; - else - result = deregister_memory(); + goto done; + } + result = deregister_memory(); +done: + mutex_unlock(&acdb_data.acdb_mutex); return result; } @@ -1416,9 +1383,9 @@ static int __init acdb_init(void) /*Speaker protection disabled*/ acdb_data.spk_prot_cfg.mode = MSM_SPKR_PROT_DISABLED; mutex_init(&acdb_data.acdb_mutex); - atomic_set(&usage_count, 0); - atomic_set(&acdb_data.valid_adm_custom_top, 1); - atomic_set(&acdb_data.valid_asm_custom_top, 1); + acdb_data.usage_count = 0; + acdb_data.valid_adm_custom_top = 1; + acdb_data.valid_asm_custom_top = 1; return misc_register(&acdb_misc); } diff --git a/sound/soc/msm/qdsp6v2/audio_acdb.h b/sound/soc/msm/qdsp6v2/audio_acdb.h index d9c12104509..12653fc3bc3 100644 --- a/sound/soc/msm/qdsp6v2/audio_acdb.h +++ b/sound/soc/msm/qdsp6v2/audio_acdb.h @@ -35,12 +35,6 @@ struct acdb_cal_block { uint32_t cal_paddr; }; -struct acdb_atomic_cal_block { - atomic_t cal_size; - atomic_t cal_kvaddr; - atomic_t cal_paddr; -}; - uint32_t get_voice_rx_topology(void); uint32_t get_voice_tx_topology(void); uint32_t get_adm_rx_topology(void); diff --git a/sound/soc/msm/qdsp6v2/audio_ocmem.c b/sound/soc/msm/qdsp6v2/audio_ocmem.c index 711f8500730..3b87893459e 100644 --- a/sound/soc/msm/qdsp6v2/audio_ocmem.c +++ b/sound/soc/msm/qdsp6v2/audio_ocmem.c @@ -243,7 +243,7 @@ int audio_ocmem_enable(int cid) struct ocmem_buf *buf = NULL; struct avcs_cmd_rsp_get_low_power_segments_info_t *lp_segptr; - pr_debug("%s, %p\n", __func__, &audio_ocmem_lcl); + pr_debug("%s, %pK\n", __func__, &audio_ocmem_lcl); atomic_set(&audio_ocmem_lcl.audio_state, OCMEM_STATE_DEFAULT); if (audio_ocmem_lcl.lp_memseg_ptr == NULL) { /* Retrieve low power segments */ diff --git a/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.c index 5e4d9d3209c..173582e3eab 100644 --- a/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -16,19 +16,38 @@ #include #include "msm-audio-effects-q6-v2.h" +#define GET_NEXT(ptr, upper_limit, rc) \ +({ \ + if (((ptr) + 1) > (upper_limit)) { \ + pr_err("%s: param list out of boundary\n", __func__); \ + (rc) = -EINVAL; \ + } \ + ((rc) == 0) ? *(ptr)++ : -EINVAL; \ +}) + +#define CHECK_PARAM_LEN(len, max_len, tag, rc) \ +do { \ + if ((len) > (max_len)) { \ + pr_err("%s: params length overflows\n", (tag)); \ + (rc) = -EINVAL; \ + } \ +} while (0) + + int msm_audio_effects_virtualizer_handler(struct audio_client *ac, struct virtualizer_params *virtualizer, long *values) { - int devices = *values++; - int num_commands = *values++; - char *params; + long *param_max_offset = values + MAX_PP_PARAMS_SZ - 1; + char *params = NULL; + int rc = 0; + int devices = GET_NEXT(values, param_max_offset, rc); + int num_commands = GET_NEXT(values, param_max_offset, rc); int *updt_params, i, prev_enable_flag; uint32_t params_length = (MAX_INBAND_PARAM_SZ); - int rc = 0; pr_debug("%s\n", __func__); - if (!ac) { + if (!ac || (devices == -EINVAL) || (num_commands == -EINVAL)) { pr_err("%s: cannot set audio effects\n", __func__); return -EINVAL; } @@ -41,83 +60,126 @@ int msm_audio_effects_virtualizer_handler(struct audio_client *ac, updt_params = (int *)params; params_length = 0; for (i = 0; i < num_commands; i++) { - uint32_t command_id = *values++; - uint32_t command_config_state = *values++; - uint32_t index_offset = *values++; - uint32_t length = *values++; + uint32_t command_id = + GET_NEXT(values, param_max_offset, rc); + uint32_t command_config_state = + GET_NEXT(values, param_max_offset, rc); + uint32_t index_offset = + GET_NEXT(values, param_max_offset, rc); + uint32_t length = + GET_NEXT(values, param_max_offset, rc); switch (command_id) { case VIRTUALIZER_ENABLE: - pr_debug("%s: VIRTUALIZER_ENABLE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("VIRT ENABLE:invalid params\n"); rc = -EINVAL; goto invalid_config; } prev_enable_flag = virtualizer->enable_flag; - virtualizer->enable_flag = *values++; + virtualizer->enable_flag = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s:VIRT ENABLE prev:%d, new:%d\n", __func__, + prev_enable_flag, virtualizer->enable_flag); if (prev_enable_flag != virtualizer->enable_flag) { - *updt_params++ = AUDPROC_MODULE_ID_VIRTUALIZER; - *updt_params++ = - AUDPROC_PARAM_ID_VIRTUALIZER_ENABLE; - *updt_params++ = VIRTUALIZER_ENABLE_PARAM_SZ; - *updt_params++ = virtualizer->enable_flag; params_length += COMMAND_PAYLOAD_SZ + VIRTUALIZER_ENABLE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "VIRT ENABLE", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_VIRTUALIZER; + *updt_params++ = + AUDPROC_PARAM_ID_VIRTUALIZER_ENABLE; + *updt_params++ = + VIRTUALIZER_ENABLE_PARAM_SZ; + *updt_params++ = + virtualizer->enable_flag; } break; case VIRTUALIZER_STRENGTH: - pr_debug("%s: VIRTUALIZER_STRENGTH\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("VIRT STRENGTH:invalid params\n"); rc = -EINVAL; goto invalid_config; } - virtualizer->strength = *values++; + virtualizer->strength = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: VIRT STRENGTH val: %d\n", + __func__, virtualizer->strength); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_VIRTUALIZER; + params_length += COMMAND_PAYLOAD_SZ + + VIRTUALIZER_STRENGTH_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "VIRT STRENGTH", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_VIRTUALIZER; *updt_params++ = AUDPROC_PARAM_ID_VIRTUALIZER_STRENGTH; - *updt_params++ = VIRTUALIZER_STRENGTH_PARAM_SZ; - *updt_params++ = virtualizer->strength; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = VIRTUALIZER_STRENGTH_PARAM_SZ; + *updt_params++ = + virtualizer->strength; } break; case VIRTUALIZER_OUT_TYPE: - pr_debug("%s: VIRTUALIZER_OUT_TYPE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("VIRT OUT_TYPE:invalid params\n"); rc = -EINVAL; goto invalid_config; } - virtualizer->out_type = *values++; + virtualizer->out_type = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: VIRT OUT_TYPE val:%d\n", + __func__, virtualizer->out_type); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_VIRTUALIZER; + params_length += COMMAND_PAYLOAD_SZ + + VIRTUALIZER_OUT_TYPE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "VIRT OUT_TYPE", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_VIRTUALIZER; *updt_params++ = AUDPROC_PARAM_ID_VIRTUALIZER_OUT_TYPE; - *updt_params++ = VIRTUALIZER_OUT_TYPE_PARAM_SZ; - *updt_params++ = virtualizer->out_type; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = VIRTUALIZER_OUT_TYPE_PARAM_SZ; + *updt_params++ = + virtualizer->out_type; } break; case VIRTUALIZER_GAIN_ADJUST: - pr_debug("%s: VIRTUALIZER_GAIN_ADJUST\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("VIRT GAIN_ADJUST: invalid params\n"); rc = -EINVAL; goto invalid_config; } - virtualizer->gain_adjust = *values++; + virtualizer->gain_adjust = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: VIRT GAIN_ADJUST val:%d\n", + __func__, virtualizer->gain_adjust); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_VIRTUALIZER; - *updt_params++ = - AUDPROC_PARAM_ID_VIRTUALIZER_GAIN_ADJUST; - *updt_params++ = - VIRTUALIZER_GAIN_ADJUST_PARAM_SZ; - *updt_params++ = virtualizer->gain_adjust; params_length += COMMAND_PAYLOAD_SZ + VIRTUALIZER_GAIN_ADJUST_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "VIRT GAIN_ADJUST", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_VIRTUALIZER; + *updt_params++ = + AUDPROC_PARAM_ID_VIRTUALIZER_GAIN_ADJUST; + *updt_params++ = + VIRTUALIZER_GAIN_ADJUST_PARAM_SZ; + *updt_params++ = + virtualizer->gain_adjust; } break; default: @@ -125,9 +187,11 @@ int msm_audio_effects_virtualizer_handler(struct audio_client *ac, break; } } - if (params_length) + if (params_length && (rc == 0)) q6asm_send_audio_effects_params(ac, params, params_length); + else + pr_debug("%s: did not send pp params\n", __func__); invalid_config: kfree(params); return rc; @@ -137,15 +201,16 @@ int msm_audio_effects_reverb_handler(struct audio_client *ac, struct reverb_params *reverb, long *values) { - int devices = *values++; - int num_commands = *values++; - char *params; + long *param_max_offset = values + MAX_PP_PARAMS_SZ - 1; + char *params = NULL; + int rc = 0; + int devices = GET_NEXT(values, param_max_offset, rc); + int num_commands = GET_NEXT(values, param_max_offset, rc); int *updt_params, i, prev_enable_flag; uint32_t params_length = (MAX_INBAND_PARAM_SZ); - int rc = 0; pr_debug("%s\n", __func__); - if (!ac) { + if (!ac || (devices == -EINVAL) || (num_commands == -EINVAL)) { pr_err("%s: cannot set audio effects\n", __func__); return -EINVAL; } @@ -158,277 +223,434 @@ int msm_audio_effects_reverb_handler(struct audio_client *ac, updt_params = (int *)params; params_length = 0; for (i = 0; i < num_commands; i++) { - uint32_t command_id = *values++; - uint32_t command_config_state = *values++; - uint32_t index_offset = *values++; - uint32_t length = *values++; + uint32_t command_id = + GET_NEXT(values, param_max_offset, rc); + uint32_t command_config_state = + GET_NEXT(values, param_max_offset, rc); + uint32_t index_offset = + GET_NEXT(values, param_max_offset, rc); + uint32_t length = + GET_NEXT(values, param_max_offset, rc); switch (command_id) { case REVERB_ENABLE: - pr_debug("%s: REVERB_ENABLE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_ENABLE:invalid params\n"); rc = -EINVAL; goto invalid_config; } prev_enable_flag = reverb->enable_flag; - reverb->enable_flag = *values++; + reverb->enable_flag = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s:REVERB_ENABLE prev:%d,new:%d\n", __func__, + prev_enable_flag, reverb->enable_flag); if (prev_enable_flag != reverb->enable_flag) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = AUDPROC_PARAM_ID_REVERB_ENABLE; - *updt_params++ = REVERB_ENABLE_PARAM_SZ; - *updt_params++ = reverb->enable_flag; params_length += COMMAND_PAYLOAD_SZ + REVERB_ENABLE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_ENABLE", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_ENABLE; + *updt_params++ = + REVERB_ENABLE_PARAM_SZ; + *updt_params++ = + reverb->enable_flag; } break; case REVERB_MODE: - pr_debug("%s: REVERB_MODE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_MODE:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->mode = *values++; + reverb->mode = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_MODE val:%d\n", + __func__, reverb->mode); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = AUDPROC_PARAM_ID_REVERB_MODE; - *updt_params++ = REVERB_MODE_PARAM_SZ; - *updt_params++ = reverb->mode; params_length += COMMAND_PAYLOAD_SZ + REVERB_MODE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_MODE", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_MODE; + *updt_params++ = + REVERB_MODE_PARAM_SZ; + *updt_params++ = + reverb->mode; } break; case REVERB_PRESET: - pr_debug("%s: REVERB_PRESET\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_PRESET:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->preset = *values++; + reverb->preset = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_PRESET val:%d\n", + __func__, reverb->preset); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = AUDPROC_PARAM_ID_REVERB_PRESET; - *updt_params++ = REVERB_PRESET_PARAM_SZ; - *updt_params++ = reverb->preset; params_length += COMMAND_PAYLOAD_SZ + REVERB_PRESET_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_PRESET", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_PRESET; + *updt_params++ = + REVERB_PRESET_PARAM_SZ; + *updt_params++ = + reverb->preset; } break; case REVERB_WET_MIX: - pr_debug("%s: REVERB_WET_MIX\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_WET_MIX:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->wet_mix = *values++; + reverb->wet_mix = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_WET_MIX val:%d\n", + __func__, reverb->wet_mix); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_WET_MIX_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_WET_MIX", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_WET_MIX; - *updt_params++ = REVERB_WET_MIX_PARAM_SZ; - *updt_params++ = reverb->wet_mix; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_WET_MIX_PARAM_SZ; + *updt_params++ = + reverb->wet_mix; } break; case REVERB_GAIN_ADJUST: - pr_debug("%s: REVERB_GAIN_ADJUST\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_GAIN_ADJUST:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->gain_adjust = *values++; + reverb->gain_adjust = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_GAIN_ADJUST val:%d\n", + __func__, reverb->gain_adjust); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_GAIN_ADJUST_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_GAIN_ADJUST", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_GAIN_ADJUST; - *updt_params++ = REVERB_GAIN_ADJUST_PARAM_SZ; - *updt_params++ = reverb->gain_adjust; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_GAIN_ADJUST_PARAM_SZ; + *updt_params++ = + reverb->gain_adjust; } break; case REVERB_ROOM_LEVEL: - pr_debug("%s: REVERB_ROOM_LEVEL\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_ROOM_LEVEL:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->room_level = *values++; + reverb->room_level = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_ROOM_LEVEL val:%d\n", + __func__, reverb->room_level); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_ROOM_LEVEL_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_ROOM_LEVEL", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_ROOM_LEVEL; - *updt_params++ = REVERB_ROOM_LEVEL_PARAM_SZ; - *updt_params++ = reverb->room_level; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_ROOM_LEVEL_PARAM_SZ; + *updt_params++ = + reverb->room_level; } break; case REVERB_ROOM_HF_LEVEL: - pr_debug("%s: REVERB_ROOM_HF_LEVEL\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_ROOM_HF_LEVEL:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->room_hf_level = *values++; + reverb->room_hf_level = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_ROOM_HF_LEVEL val%d\n", + __func__, reverb->room_hf_level); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_ROOM_HF_LEVEL_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_ROOM_HF_LEVEL", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_ROOM_HF_LEVEL; - *updt_params++ = REVERB_ROOM_HF_LEVEL_PARAM_SZ; - *updt_params++ = reverb->room_hf_level; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_ROOM_HF_LEVEL_PARAM_SZ; + *updt_params++ = + reverb->room_hf_level; } break; case REVERB_DECAY_TIME: - pr_debug("%s: REVERB_DECAY_TIME\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_DECAY_TIME:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->decay_time = *values++; + reverb->decay_time = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_DECAY_TIME val:%d\n", + __func__, reverb->decay_time); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_DECAY_TIME_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_DECAY_TIME", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_DECAY_TIME; - *updt_params++ = REVERB_DECAY_TIME_PARAM_SZ; - *updt_params++ = reverb->decay_time; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_DECAY_TIME_PARAM_SZ; + *updt_params++ = + reverb->decay_time; } break; case REVERB_DECAY_HF_RATIO: - pr_debug("%s: REVERB_DECAY_HF_RATIO\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_DECAY_HF_RATIOinvalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->decay_hf_ratio = *values++; + reverb->decay_hf_ratio = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_DECAY_HF_RATIO val%d\n", + __func__, reverb->decay_hf_ratio); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_DECAY_HF_RATIO_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_DECAY_HF_RATIO", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_DECAY_HF_RATIO; - *updt_params++ = REVERB_DECAY_HF_RATIO_PARAM_SZ; - *updt_params++ = reverb->decay_hf_ratio; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_DECAY_HF_RATIO_PARAM_SZ; + *updt_params++ = + reverb->decay_hf_ratio; } break; case REVERB_REFLECTIONS_LEVEL: - pr_debug("%s: REVERB_REFLECTIONS_LEVEL\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_REFLECTION_LVLinvalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->reflections_level = *values++; + reverb->reflections_level = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_REFLECTIONS_LEVEL val:%d\n", + __func__, reverb->reflections_level); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = - AUDPROC_PARAM_ID_REVERB_REFLECTIONS_LEVEL; - *updt_params++ = - REVERB_REFLECTIONS_LEVEL_PARAM_SZ; - *updt_params++ = reverb->reflections_level; params_length += COMMAND_PAYLOAD_SZ + REVERB_REFLECTIONS_LEVEL_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_REFLECTIONS_LEVEL", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_REFLECTIONS_LEVEL; + *updt_params++ = + REVERB_REFLECTIONS_LEVEL_PARAM_SZ; + *updt_params++ = + reverb->reflections_level; } break; case REVERB_REFLECTIONS_DELAY: - pr_debug("%s: REVERB_REFLECTIONS_DELAY\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_REFLECTION_DLYinvalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->reflections_delay = *values++; + reverb->reflections_delay = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_REFLECTIONS_DELAY val:%d\n", + __func__, reverb->reflections_delay); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = - AUDPROC_PARAM_ID_REVERB_REFLECTIONS_DELAY; - *updt_params++ = - REVERB_REFLECTIONS_DELAY_PARAM_SZ; - *updt_params++ = reverb->reflections_delay; params_length += COMMAND_PAYLOAD_SZ + REVERB_REFLECTIONS_DELAY_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_REFLECTIONS_DELAY", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_REFLECTIONS_DELAY; + *updt_params++ = + REVERB_REFLECTIONS_DELAY_PARAM_SZ; + *updt_params++ = + reverb->reflections_delay; } break; case REVERB_LEVEL: - pr_debug("%s: REVERB_LEVEL\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_LEVEL:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->level = *values++; + reverb->level = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_LEVEL val:%d\n", + __func__, reverb->level); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = AUDPROC_PARAM_ID_REVERB_LEVEL; - *updt_params++ = REVERB_LEVEL_PARAM_SZ; - *updt_params++ = reverb->level; params_length += COMMAND_PAYLOAD_SZ + REVERB_LEVEL_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_LEVEL", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_LEVEL; + *updt_params++ = + REVERB_LEVEL_PARAM_SZ; + *updt_params++ = + reverb->level; } break; case REVERB_DELAY: - pr_debug("%s: REVERB_DELAY\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_DELAY:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->delay = *values++; + reverb->delay = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s:REVERB_DELAY val:%d\n", + __func__, reverb->delay); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; - *updt_params++ = AUDPROC_PARAM_ID_REVERB_DELAY; - *updt_params++ = REVERB_DELAY_PARAM_SZ; - *updt_params++ = reverb->delay; params_length += COMMAND_PAYLOAD_SZ + REVERB_DELAY_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_DELAY", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; + *updt_params++ = + AUDPROC_PARAM_ID_REVERB_DELAY; + *updt_params++ = + REVERB_DELAY_PARAM_SZ; + *updt_params++ = + reverb->delay; } break; case REVERB_DIFFUSION: - pr_debug("%s: REVERB_DIFFUSION\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_DIFFUSION:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->diffusion = *values++; + reverb->diffusion = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_DIFFUSION val:%d\n", + __func__, reverb->diffusion); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_DIFFUSION_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_DIFFUSION", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_DIFFUSION; - *updt_params++ = REVERB_DIFFUSION_PARAM_SZ; - *updt_params++ = reverb->diffusion; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_DIFFUSION_PARAM_SZ; + *updt_params++ = + reverb->diffusion; } break; case REVERB_DENSITY: - pr_debug("%s: REVERB_DENSITY\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("REVERB_DENSITY:invalid params\n"); rc = -EINVAL; goto invalid_config; } - reverb->density = *values++; + reverb->density = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: REVERB_DENSITY val:%d\n", + __func__, reverb->density); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_REVERB; + params_length += COMMAND_PAYLOAD_SZ + + REVERB_DENSITY_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "REVERB_DENSITY", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_REVERB; *updt_params++ = AUDPROC_PARAM_ID_REVERB_DENSITY; - *updt_params++ = REVERB_DENSITY_PARAM_SZ; - *updt_params++ = reverb->density; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = REVERB_DENSITY_PARAM_SZ; + *updt_params++ = + reverb->density; } break; default: @@ -436,9 +658,11 @@ int msm_audio_effects_reverb_handler(struct audio_client *ac, break; } } - if (params_length) + if (params_length && (rc == 0)) q6asm_send_audio_effects_params(ac, params, params_length); + else + pr_debug("%s: did not send pp params\n", __func__); invalid_config: kfree(params); return rc; @@ -448,15 +672,16 @@ int msm_audio_effects_bass_boost_handler(struct audio_client *ac, struct bass_boost_params *bass_boost, long *values) { - int devices = *values++; - int num_commands = *values++; - char *params; + long *param_max_offset = values + MAX_PP_PARAMS_SZ - 1; + char *params = NULL; + int rc = 0; + int devices = GET_NEXT(values, param_max_offset, rc); + int num_commands = GET_NEXT(values, param_max_offset, rc); int *updt_params, i, prev_enable_flag; uint32_t params_length = (MAX_INBAND_PARAM_SZ); - int rc = 0; pr_debug("%s\n", __func__); - if (!ac) { + if (!ac || (devices == -EINVAL) || (num_commands == -EINVAL)) { pr_err("%s: cannot set audio effects\n", __func__); return -EINVAL; } @@ -469,64 +694,99 @@ int msm_audio_effects_bass_boost_handler(struct audio_client *ac, updt_params = (int *)params; params_length = 0; for (i = 0; i < num_commands; i++) { - uint32_t command_id = *values++; - uint32_t command_config_state = *values++; - uint32_t index_offset = *values++; - uint32_t length = *values++; + uint32_t command_id = + GET_NEXT(values, param_max_offset, rc); + uint32_t command_config_state = + GET_NEXT(values, param_max_offset, rc); + uint32_t index_offset = + GET_NEXT(values, param_max_offset, rc); + uint32_t length = + GET_NEXT(values, param_max_offset, rc); switch (command_id) { case BASS_BOOST_ENABLE: - pr_debug("%s: BASS_BOOST_ENABLE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("BASS_BOOST_ENABLE:invalid params\n"); rc = -EINVAL; goto invalid_config; } prev_enable_flag = bass_boost->enable_flag; - bass_boost->enable_flag = *values++; + bass_boost->enable_flag = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: BASS_BOOST_ENABLE prev:%d new:%d\n", + __func__, prev_enable_flag, + bass_boost->enable_flag); if (prev_enable_flag != bass_boost->enable_flag) { - *updt_params++ = AUDPROC_MODULE_ID_BASS_BOOST; + params_length += COMMAND_PAYLOAD_SZ + + BASS_BOOST_ENABLE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "BASS_BOOST_ENABLE", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_BASS_BOOST; *updt_params++ = AUDPROC_PARAM_ID_BASS_BOOST_ENABLE; - *updt_params++ = BASS_BOOST_ENABLE_PARAM_SZ; - *updt_params++ = bass_boost->enable_flag; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = BASS_BOOST_ENABLE_PARAM_SZ; + *updt_params++ = + bass_boost->enable_flag; } break; case BASS_BOOST_MODE: - pr_debug("%s: BASS_BOOST_MODE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("BASS_BOOST_MODE:invalid params\n"); rc = -EINVAL; goto invalid_config; } - bass_boost->mode = *values++; + bass_boost->mode = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: BASS_BOOST_MODE val:%d\n", + __func__, bass_boost->mode); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_BASS_BOOST; + params_length += COMMAND_PAYLOAD_SZ + + BASS_BOOST_MODE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "BASS_BOOST_MODE", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_BASS_BOOST; *updt_params++ = AUDPROC_PARAM_ID_BASS_BOOST_MODE; - *updt_params++ = BASS_BOOST_MODE_PARAM_SZ; - *updt_params++ = bass_boost->mode; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = BASS_BOOST_MODE_PARAM_SZ; + *updt_params++ = + bass_boost->mode; } break; case BASS_BOOST_STRENGTH: - pr_debug("%s: BASS_BOOST_STRENGTH\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("BASS_BOOST_STRENGTH:invalid params\n"); rc = -EINVAL; goto invalid_config; } - bass_boost->strength = *values++; + bass_boost->strength = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: BASS_BOOST_STRENGTH val:%d\n", + __func__, bass_boost->strength); if (command_config_state == CONFIG_SET) { - *updt_params++ = AUDPROC_MODULE_ID_BASS_BOOST; + params_length += COMMAND_PAYLOAD_SZ + + BASS_BOOST_STRENGTH_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "BASS_BOOST_STRENGTH", rc); + if (rc != 0) + break; + *updt_params++ = + AUDPROC_MODULE_ID_BASS_BOOST; *updt_params++ = AUDPROC_PARAM_ID_BASS_BOOST_STRENGTH; - *updt_params++ = BASS_BOOST_STRENGTH_PARAM_SZ; - *updt_params++ = bass_boost->strength; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = BASS_BOOST_STRENGTH_PARAM_SZ; + *updt_params++ = + bass_boost->strength; } break; default: @@ -534,9 +794,11 @@ int msm_audio_effects_bass_boost_handler(struct audio_client *ac, break; } } - if (params_length) + if (params_length && (rc == 0)) q6asm_send_audio_effects_params(ac, params, params_length); + else + pr_debug("%s: did not send pp params\n", __func__); invalid_config: kfree(params); return rc; @@ -546,15 +808,16 @@ int msm_audio_effects_popless_eq_handler(struct audio_client *ac, struct eq_params *eq, long *values) { - int devices = *values++; - int num_commands = *values++; - char *params; + long *param_max_offset = values + MAX_PP_PARAMS_SZ - 1; + char *params = NULL; + int rc = 0; + int devices = GET_NEXT(values, param_max_offset, rc); + int num_commands = GET_NEXT(values, param_max_offset, rc); int *updt_params, i, prev_enable_flag; uint32_t params_length = (MAX_INBAND_PARAM_SZ); - int rc = 0; pr_debug("%s\n", __func__); - if (!ac) { + if (!ac || (devices == -EINVAL) || (num_commands == -EINVAL)) { pr_err("%s: cannot set audio effects\n", __func__); return -EINVAL; } @@ -567,47 +830,65 @@ int msm_audio_effects_popless_eq_handler(struct audio_client *ac, updt_params = (int *)params; params_length = 0; for (i = 0; i < num_commands; i++) { - uint32_t command_id = *values++; - uint32_t command_config_state = *values++; - uint32_t index_offset = *values++; - uint32_t length = *values++; - int idx, j; + uint32_t command_id = + GET_NEXT(values, param_max_offset, rc); + uint32_t command_config_state = + GET_NEXT(values, param_max_offset, rc); + uint32_t index_offset = + GET_NEXT(values, param_max_offset, rc); + uint32_t length = + GET_NEXT(values, param_max_offset, rc); + uint32_t idx; + int j; switch (command_id) { case EQ_ENABLE: - pr_debug("%s: EQ_ENABLE\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("EQ_ENABLE:invalid params\n"); rc = -EINVAL; goto invalid_config; } prev_enable_flag = eq->enable_flag; - eq->enable_flag = *values++; - pr_debug("%s: prev_enable_flag : %d, eq.enable_flag : %d", - __func__, prev_enable_flag, eq->enable_flag); + eq->enable_flag = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: EQ_ENABLE prev:%d new:%d\n", __func__, + prev_enable_flag, eq->enable_flag); if (prev_enable_flag != eq->enable_flag) { + params_length += COMMAND_PAYLOAD_SZ + + EQ_ENABLE_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "EQ_ENABLE", rc); + if (rc != 0) + break; *updt_params++ = AUDPROC_MODULE_ID_POPLESS_EQUALIZER; - *updt_params++ = AUDPROC_PARAM_ID_EQ_ENABLE; - *updt_params++ = EQ_ENABLE_PARAM_SZ; - *updt_params++ = eq->enable_flag; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = + AUDPROC_PARAM_ID_EQ_ENABLE; + *updt_params++ = EQ_ENABLE_PARAM_SZ; + *updt_params++ = + eq->enable_flag; } break; case EQ_CONFIG: - pr_debug("%s: EQ_CONFIG\n", __func__); if (length < EQ_CONFIG_PARAM_LEN || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("EQ_CONFIG:invalid params\n"); rc = -EINVAL; goto invalid_config; } + pr_debug("%s: EQ_CONFIG bands:%d, pgain:%d, pset:%d\n", + __func__, eq->config.num_bands, + eq->config.eq_pregain, eq->config.preset_id); for (idx = 0; idx < MAX_EQ_BANDS; idx++) eq->per_band_cfg[idx].band_idx = -1; - eq->config.eq_pregain = *values++; - eq->config.preset_id = *values++; - eq->config.num_bands = *values++; + eq->config.eq_pregain = + GET_NEXT(values, param_max_offset, rc); + eq->config.preset_id = + GET_NEXT(values, param_max_offset, rc); + eq->config.num_bands = + GET_NEXT(values, param_max_offset, rc); if (eq->config.num_bands > MAX_EQ_BANDS) { - pr_err("invalid num of bands\n"); + pr_err("EQ_CONFIG:invalid num of bands\n"); rc = -EINVAL; goto invalid_config; } @@ -615,96 +896,129 @@ int msm_audio_effects_popless_eq_handler(struct audio_client *ac, (((length - EQ_CONFIG_PARAM_LEN)/ EQ_CONFIG_PER_BAND_PARAM_LEN) != eq->config.num_bands)) { - pr_err("invalid length to set config per band\n"); + pr_err("EQ_CONFIG:invalid length per band\n"); rc = -EINVAL; goto invalid_config; } for (j = 0; j < eq->config.num_bands; j++) { - idx = *values++; + idx = GET_NEXT(values, param_max_offset, rc); + if (idx >= MAX_EQ_BANDS) { + pr_err("EQ_CONFIG:invalid band index\n"); + rc = -EINVAL; + goto invalid_config; + } eq->per_band_cfg[idx].band_idx = idx; - eq->per_band_cfg[idx].filter_type = *values++; + eq->per_band_cfg[idx].filter_type = + GET_NEXT(values, param_max_offset, rc); eq->per_band_cfg[idx].freq_millihertz = - *values++; + GET_NEXT(values, param_max_offset, rc); eq->per_band_cfg[idx].gain_millibels = - *values++; + GET_NEXT(values, param_max_offset, rc); eq->per_band_cfg[idx].quality_factor = - *values++; + GET_NEXT(values, param_max_offset, rc); } if (command_config_state == CONFIG_SET) { int config_param_length = EQ_CONFIG_PARAM_SZ + (EQ_CONFIG_PER_BAND_PARAM_SZ* eq->config.num_bands); + params_length += COMMAND_PAYLOAD_SZ + + config_param_length; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "EQ_CONFIG", rc); + if (rc != 0) + break; *updt_params++ = AUDPROC_MODULE_ID_POPLESS_EQUALIZER; - *updt_params++ = AUDPROC_PARAM_ID_EQ_CONFIG; - *updt_params++ = config_param_length; - *updt_params++ = eq->config.eq_pregain; - *updt_params++ = eq->config.preset_id; - *updt_params++ = eq->config.num_bands; + *updt_params++ = + AUDPROC_PARAM_ID_EQ_CONFIG; + *updt_params++ = + config_param_length; + *updt_params++ = + eq->config.eq_pregain; + *updt_params++ = + eq->config.preset_id; + *updt_params++ = + eq->config.num_bands; for (idx = 0; idx < MAX_EQ_BANDS; idx++) { if (eq->per_band_cfg[idx].band_idx < 0) continue; *updt_params++ = - eq->per_band_cfg[idx].filter_type; + eq->per_band_cfg[idx].filter_type; *updt_params++ = - eq->per_band_cfg[idx].freq_millihertz; + eq->per_band_cfg[idx].freq_millihertz; *updt_params++ = - eq->per_band_cfg[idx].gain_millibels; + eq->per_band_cfg[idx].gain_millibels; *updt_params++ = - eq->per_band_cfg[idx].quality_factor; + eq->per_band_cfg[idx].quality_factor; *updt_params++ = - eq->per_band_cfg[idx].band_idx; + eq->per_band_cfg[idx].band_idx; } - params_length += COMMAND_PAYLOAD_SZ + - config_param_length; } break; case EQ_BAND_INDEX: - pr_debug("%s: EQ_BAND_INDEX\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("EQ_BAND_INDEX:invalid params\n"); rc = -EINVAL; goto invalid_config; } - idx = *values++; + idx = GET_NEXT(values, param_max_offset, rc); if (idx > MAX_EQ_BANDS) { - pr_err("invalid band index\n"); + pr_err("EQ_BAND_INDEX:invalid band index\n"); rc = -EINVAL; goto invalid_config; } eq->band_index = idx; + pr_debug("%s: EQ_BAND_INDEX val:%d\n", + __func__, eq->band_index); if (command_config_state == CONFIG_SET) { + params_length += COMMAND_PAYLOAD_SZ + + EQ_BAND_INDEX_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "EQ_BAND_INDEX", rc); + if (rc != 0) + break; *updt_params++ = AUDPROC_MODULE_ID_POPLESS_EQUALIZER; *updt_params++ = AUDPROC_PARAM_ID_EQ_BAND_INDEX; - *updt_params++ = EQ_BAND_INDEX_PARAM_SZ; - *updt_params++ = eq->band_index; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = EQ_BAND_INDEX_PARAM_SZ; + *updt_params++ = + eq->band_index; } break; case EQ_SINGLE_BAND_FREQ: - pr_debug("%s: EQ_SINGLE_BAND_FREQ\n", __func__); if (length != 1 || index_offset != 0) { - pr_err("no valid params\n"); + pr_err("EQ_SINGLE_BAND_FREQ:invalid params\n"); rc = -EINVAL; goto invalid_config; } if (eq->band_index > MAX_EQ_BANDS) { - pr_err("invalid band index to set frequency\n"); + pr_err("EQ_SINGLE_BAND_FREQ:invalid index\n"); break; } - eq->freq_millihertz = *values++; + eq->freq_millihertz = + GET_NEXT(values, param_max_offset, rc); + pr_debug("%s: EQ_SINGLE_BAND_FREQ idx:%d, val:%d\n", + __func__, eq->band_index, eq->freq_millihertz); if (command_config_state == CONFIG_SET) { + params_length += COMMAND_PAYLOAD_SZ + + EQ_SINGLE_BAND_FREQ_PARAM_SZ; + CHECK_PARAM_LEN(params_length, + MAX_INBAND_PARAM_SZ, + "EQ_SINGLE_BAND_FREQ", rc); + if (rc != 0) + break; *updt_params++ = AUDPROC_MODULE_ID_POPLESS_EQUALIZER; *updt_params++ = AUDPROC_PARAM_ID_EQ_SINGLE_BAND_FREQ; - *updt_params++ = EQ_SINGLE_BAND_FREQ_PARAM_SZ; - *updt_params++ = eq->freq_millihertz; - params_length += COMMAND_PAYLOAD_SZ + + *updt_params++ = EQ_SINGLE_BAND_FREQ_PARAM_SZ; + *updt_params++ = + eq->freq_millihertz; } break; default: @@ -712,9 +1026,11 @@ int msm_audio_effects_popless_eq_handler(struct audio_client *ac, break; } } - if (params_length) + if (params_length && (rc == 0)) q6asm_send_audio_effects_params(ac, params, params_length); + else + pr_debug("%s: did not send pp params\n", __func__); invalid_config: kfree(params); return rc; diff --git a/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.h b/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.h index 3d2e6d4065c..2fb5083d4f0 100644 --- a/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.h +++ b/sound/soc/msm/qdsp6v2/msm-audio-effects-q6-v2.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, The Linux Foundation. All rights reserved. + * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -16,6 +16,8 @@ #include +#define MAX_PP_PARAMS_SZ 128 + int msm_audio_effects_reverb_handler(struct audio_client *ac, struct reverb_params *reverb, long *values); diff --git a/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c index d58485c3f77..cb204b56de5 100644 --- a/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2013, 2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -51,6 +51,8 @@ #define COMPRE_OUTPUT_METADATA_SIZE (sizeof(struct output_meta_data_st)) #define COMPRESSED_LR_VOL_MAX_STEPS 0x20002000 +#define MAX_AC3_PARAM_SIZE (18*2*sizeof(int)) + const DECLARE_TLV_DB_LINEAR(compr_rx_vol_gain, 0, COMPRESSED_LR_VOL_MAX_STEPS); struct snd_msm { @@ -217,7 +219,7 @@ static void compr_event_handler(uint32_t opcode, break; case ASM_DATA_EVENT_READ_DONE_V2: { pr_debug("ASM_DATA_EVENT_READ_DONE\n"); - pr_debug("buf = %p, data = 0x%X, *data = %p,\n" + pr_debug("buf = %pK, data = 0x%X, *data = %pK,\n" "prtd->pcm_irq_pos = %d\n", prtd->audio_client->port[OUT].buf, *(uint32_t *)prtd->audio_client->port[OUT].buf->data, @@ -227,7 +229,7 @@ static void compr_event_handler(uint32_t opcode, memcpy(prtd->audio_client->port[OUT].buf->data + prtd->pcm_irq_pos, (ptrmem + READDONE_IDX_SIZE), COMPRE_CAPTURE_HEADER_SIZE); - pr_debug("buf = %p, updated data = 0x%X, *data = %p\n", + pr_debug("buf = %pK, updated data = 0x%X, *data = %pK\n", prtd->audio_client->port[OUT].buf, *(uint32_t *)(prtd->audio_client->port[OUT].buf->data + prtd->pcm_irq_pos), @@ -455,7 +457,7 @@ static int msm_compr_capture_prepare(struct snd_pcm_substream *substream) read_param.paddr = (unsigned long)(buf[i].phys) + COMPRE_CAPTURE_HEADER_SIZE; pr_debug("Push buffer [%d] to DSP, "\ - "paddr: %p, vaddr: %p\n", + "paddr: %pK, vaddr: %pK\n", i, (void *) read_param.paddr, buf[i].data); q6asm_async_read(prtd->audio_client, &read_param); @@ -895,7 +897,7 @@ static int msm_compr_hw_params(struct snd_pcm_substream *substream, dma_buf->addr = buf[0].phys; dma_buf->bytes = runtime->hw.buffer_bytes_max; - pr_debug("%s: buf[%p]dma_buf->area[%p]dma_buf->addr[%p]\n" + pr_debug("%s: buf[%pK]dma_buf->area[%pK]dma_buf->addr[%pK]\n" "dma_buf->bytes[%d]\n", __func__, (void *)buf, (void *)dma_buf->area, (void *)dma_buf->addr, dma_buf->bytes); @@ -969,19 +971,33 @@ static int msm_compr_ioctl(struct snd_pcm_substream *substream, compr->codec = FORMAT_MPEG4_AAC; break; case SND_AUDIOCODEC_AC3: { - char params_value[18*2*sizeof(int)]; + char params_value[MAX_AC3_PARAM_SIZE]; int *params_value_data = (int *)params_value; /* 36 is the max param length for ddp */ int i; struct snd_dec_ddp *ddp = &compr->info.codec_param.codec.options.ddp; - int params_length = ddp->params_length*sizeof(int); + uint32_t params_length = 0; + memset(params_value, 0, MAX_AC3_PARAM_SIZE); + /* check integer overflow */ + if (ddp->params_length > UINT_MAX/sizeof(int)) { + pr_err("%s: Integer overflow ddp->params_length %d\n", + __func__, ddp->params_length); + return -EINVAL; + } + params_length = ddp->params_length*sizeof(int); + if (params_length > MAX_AC3_PARAM_SIZE) { + /*MAX is 36*sizeof(int) this should not happen*/ + pr_err("%s: params_length(%d) is greater than %d\n", + __func__, params_length, MAX_AC3_PARAM_SIZE); + return -EINVAL; + } pr_debug("SND_AUDIOCODEC_AC3\n"); compr->codec = FORMAT_AC3; if (copy_from_user(params_value, (void *)ddp->params, params_length)) - pr_err("%s: ERROR: copy ddp params value\n", - __func__); + pr_err("%s: copy ddp params value, size=%d\n", + __func__, params_length); pr_debug("params_length: %d\n", ddp->params_length); for (i = 0; i < params_length; i++) pr_debug("params_value[%d]: %x\n", i, @@ -1000,19 +1016,33 @@ static int msm_compr_ioctl(struct snd_pcm_substream *substream, break; } case SND_AUDIOCODEC_EAC3: { - char params_value[18*2*sizeof(int)]; + char params_value[MAX_AC3_PARAM_SIZE]; int *params_value_data = (int *)params_value; /* 36 is the max param length for ddp */ int i; struct snd_dec_ddp *ddp = &compr->info.codec_param.codec.options.ddp; - int params_length = ddp->params_length*sizeof(int); + uint32_t params_length = 0; + memset(params_value, 0, MAX_AC3_PARAM_SIZE); + /* check integer overflow */ + if (ddp->params_length > UINT_MAX/sizeof(int)) { + pr_err("%s: Integer overflow ddp->params_length %d\n", + __func__, ddp->params_length); + return -EINVAL; + } + params_length = ddp->params_length*sizeof(int); + if (params_length > MAX_AC3_PARAM_SIZE) { + /*MAX is 36*sizeof(int) this should not happen*/ + pr_err("%s: params_length(%d) is greater than %d\n", + __func__, params_length, MAX_AC3_PARAM_SIZE); + return -EINVAL; + } pr_debug("SND_AUDIOCODEC_EAC3\n"); compr->codec = FORMAT_EAC3; if (copy_from_user(params_value, (void *)ddp->params, params_length)) - pr_err("%s: ERROR: copy ddp params value\n", - __func__); + pr_err("%s: copy ddp params value, size=%d\n", + __func__, params_length); pr_debug("params_length: %d\n", ddp->params_length); for (i = 0; i < ddp->params_length; i++) pr_debug("params_value[%d]: %x\n", i, diff --git a/sound/soc/msm/qdsp6v2/msm-compress-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-compress-q6-v2.c index 628f04c2ca6..bfccf626b9f 100644 --- a/sound/soc/msm/qdsp6v2/msm-compress-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-compress-q6-v2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -83,6 +83,10 @@ struct msm_compr_gapless_state { bool use_dsp_gapless_mode; }; +static unsigned int supported_sample_rates[] = { + 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000 +}; + struct msm_compr_pdata { atomic_t audio_ocmem_req; struct snd_compr_stream *cstream[MSM_FRONTEND_DAI_MAX]; @@ -750,40 +754,16 @@ static int msm_compr_set_params(struct snd_compr_stream *cstream, struct snd_compr_runtime *runtime = cstream->runtime; struct msm_compr_audio *prtd = runtime->private_data; int ret = 0, frame_sz = 0, delay_time_ms = 0; + int i, num_rates; pr_debug("%s\n", __func__); - memcpy(&prtd->codec_param, params, sizeof(struct snd_compr_params)); - - /* ToDo: remove duplicates */ - prtd->num_channels = prtd->codec_param.codec.ch_in; - - switch (prtd->codec_param.codec.sample_rate) { - case SNDRV_PCM_RATE_8000: - prtd->sample_rate = 8000; - break; - case SNDRV_PCM_RATE_11025: - prtd->sample_rate = 11025; - break; - /* ToDo: What about 12K and 24K sample rates ? */ - case SNDRV_PCM_RATE_16000: - prtd->sample_rate = 16000; - break; - case SNDRV_PCM_RATE_22050: - prtd->sample_rate = 22050; - break; - case SNDRV_PCM_RATE_32000: - prtd->sample_rate = 32000; - break; - case SNDRV_PCM_RATE_44100: - prtd->sample_rate = 44100; - break; - case SNDRV_PCM_RATE_48000: - prtd->sample_rate = 48000; - break; - } - - pr_debug("%s: sample_rate %d\n", __func__, prtd->sample_rate); + num_rates = sizeof(supported_sample_rates)/sizeof(unsigned int); + for (i = 0; i < num_rates; i++) + if (params->codec.sample_rate == supported_sample_rates[i]) + break; + if (i == num_rates) + return -EINVAL; switch (params->codec.id) { case SND_AUDIOCODEC_MP3: { @@ -811,6 +791,12 @@ static int msm_compr_set_params(struct snd_compr_stream *cstream, delay_time_ms - PARTIAL_DRAIN_ACK_EARLY_BY_MSEC : 0; prtd->partial_drain_delay = delay_time_ms; + memcpy(&prtd->codec_param, params, sizeof(struct snd_compr_params)); + + /* ToDo: remove duplicates */ + prtd->num_channels = prtd->codec_param.codec.ch_in; + prtd->sample_rate = prtd->codec_param.codec.sample_rate; + pr_debug("%s: sample_rate %d\n", __func__, prtd->sample_rate); ret = msm_compr_configure_dsp(cstream); return ret; @@ -1283,6 +1269,7 @@ static int msm_compr_pointer(struct snd_compr_stream *cstream, uint64_t timestamp = 0; int rc = 0, first_buffer; unsigned long flags; + uint32_t gapless_transition; pr_debug("%s\n", __func__); memset(&tstamp, 0x0, sizeof(struct snd_compr_tstamp)); @@ -1302,13 +1289,17 @@ static int msm_compr_pointer(struct snd_compr_stream *cstream, return -EINVAL; } + gapless_transition = prtd->gapless_state.gapless_transition; spin_unlock_irqrestore(&prtd->lock, flags); /* Query timestamp from DSP if some data is with it. This prevents timeouts. */ - if (!first_buffer) { + if (!first_buffer || gapless_transition) { + if (gapless_transition) + pr_info("session time in gapless transition"); + rc = q6asm_get_session_time(prtd->audio_client, ×tamp); if (rc < 0) { pr_err("%s: Get Session Time return value =%lld\n", @@ -1458,7 +1449,11 @@ static int msm_compr_get_codec_caps(struct snd_compr_stream *cstream, case SND_AUDIOCODEC_MP3: codec->num_descriptors = 2; codec->descriptor[0].max_ch = 2; - codec->descriptor[0].sample_rates = SNDRV_PCM_RATE_8000_48000; + memcpy(codec->descriptor[0].sample_rates, + supported_sample_rates, + sizeof(supported_sample_rates)); + codec->descriptor[0].num_sample_rates = + sizeof(supported_sample_rates)/sizeof(unsigned int); codec->descriptor[0].bit_rate[0] = 320; /* 320kbps */ codec->descriptor[0].bit_rate[1] = 128; codec->descriptor[0].num_bitrates = 2; @@ -1469,7 +1464,11 @@ static int msm_compr_get_codec_caps(struct snd_compr_stream *cstream, case SND_AUDIOCODEC_AAC: codec->num_descriptors = 2; codec->descriptor[1].max_ch = 2; - codec->descriptor[1].sample_rates = SNDRV_PCM_RATE_8000_48000; + memcpy(codec->descriptor[1].sample_rates, + supported_sample_rates, + sizeof(supported_sample_rates)); + codec->descriptor[1].num_sample_rates = + sizeof(supported_sample_rates)/sizeof(unsigned int); codec->descriptor[1].bit_rate[0] = 320; /* 320kbps */ codec->descriptor[1].bit_rate[1] = 128; codec->descriptor[1].num_bitrates = 2; @@ -1680,7 +1679,7 @@ static int msm_compr_audio_effects_config_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; - uinfo->count = 128; + uinfo->count = MAX_PP_PARAMS_SZ; uinfo->value.integer.min = 0; uinfo->value.integer.max = 0xFFFFFFFF; return 0; diff --git a/sound/soc/msm/qdsp6v2/msm-dai-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-dai-q6-v2.c index 5c3f9c57641..fd5e9a2a908 100644 --- a/sound/soc/msm/qdsp6v2/msm-dai-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-dai-q6-v2.c @@ -1085,7 +1085,7 @@ static int __devinit msm_auxpcm_dev_probe(struct platform_device *pdev) goto fail_pdata_nomem; } - dev_dbg(&pdev->dev, "%s: dev %p, dai_data %p, auxpcm_pdata %p\n", + dev_dbg(&pdev->dev, "%s: dev %pK, dai_data %pK, auxpcm_pdata %pK\n", __func__, &pdev->dev, dai_data, auxpcm_pdata); rc = of_property_read_u32_array(pdev->dev.of_node, @@ -1421,7 +1421,7 @@ static int msm_dai_q6_mi2s_startup(struct snd_pcm_substream *substream, struct msm_dai_q6_mi2s_dai_data *mi2s_dai_data = dev_get_drvdata(dai->dev); - dev_dbg(dai->dev, "%s: cnst list %p\n", __func__, + dev_dbg(dai->dev, "%s: cnst list %pK\n", __func__, mi2s_dai_data->rate_constraint.list); if (mi2s_dai_data->rate_constraint.list) { diff --git a/sound/soc/msm/qdsp6v2/msm-lsm-client.c b/sound/soc/msm/qdsp6v2/msm-lsm-client.c index 3f570783140..6a0f997fda7 100644 --- a/sound/soc/msm/qdsp6v2/msm-lsm-client.c +++ b/sound/soc/msm/qdsp6v2/msm-lsm-client.c @@ -117,7 +117,7 @@ static int msm_lsm_ioctl(struct snd_pcm_substream *substream, if (copy_from_user(prtd->lsm_client->sound_model.data, snd_model.data, snd_model.data_size)) { - pr_err("%s: copy from user data failed data %p size %d\n", + pr_err("%s: copy from user data failed data %pK size %d\n", __func__, snd_model.data, snd_model.data_size); rc = -EFAULT; break; @@ -170,6 +170,9 @@ static int msm_lsm_ioctl(struct snd_pcm_substream *substream, */ rc = -EFAULT; } else { + if (!access_ok(VERIFY_READ, user, + sizeof(struct snd_lsm_event_status))) + rc = -EFAULT; if (user->payload_size < event_status->payload_size) { pr_debug("%s: provided %dbytes isn't enough, needs %dbytes\n", @@ -259,13 +262,13 @@ static int msm_lsm_open(struct snd_pcm_substream *substream) kfree(prtd); return -ENOMEM; } + prtd->lsm_client->opened = false; ret = q6lsm_open(prtd->lsm_client); if (ret < 0) { pr_err("%s: lsm open failed, %d\n", __func__, ret); - q6lsm_client_free(prtd->lsm_client); - kfree(prtd); return ret; } + prtd->lsm_client->opened = true; pr_debug("%s: Session ID %d\n", __func__, prtd->lsm_client->session); prtd->lsm_client->started = false; @@ -284,7 +287,10 @@ static int msm_lsm_close(struct snd_pcm_substream *substream) pr_debug("%s\n", __func__); - q6lsm_close(prtd->lsm_client); + if (prtd->lsm_client->opened) { + q6lsm_close(prtd->lsm_client); + prtd->lsm_client->opened = false; + } q6lsm_client_free(prtd->lsm_client); spin_lock_irqsave(&prtd->event_lock, flags); diff --git a/sound/soc/msm/qdsp6v2/msm-multi-ch-pcm-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-multi-ch-pcm-q6-v2.c index 39e69343840..e509fbb8bdc 100644 --- a/sound/soc/msm/qdsp6v2/msm-multi-ch-pcm-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-multi-ch-pcm-q6-v2.c @@ -711,7 +711,7 @@ static int msm_pcm_hw_params(struct snd_pcm_substream *substream, } buf = prtd->audio_client->port[dir].buf; - pr_debug("%s:buf = %p\n", __func__, buf); + pr_debug("%s:buf = %pK\n", __func__, buf); dma_buf->dev.type = SNDRV_DMA_TYPE_DEV; dma_buf->dev.dev = substream->pcm->card->dev; dma_buf->private_data = NULL; diff --git a/sound/soc/msm/qdsp6v2/msm-pcm-afe-v2.c b/sound/soc/msm/qdsp6v2/msm-pcm-afe-v2.c index 4c673160d77..60fb5d17cf6 100644 --- a/sound/soc/msm/qdsp6v2/msm-pcm-afe-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-pcm-afe-v2.c @@ -601,7 +601,7 @@ static int msm_afe_hw_params(struct snd_pcm_substream *substream, return -ENOMEM; } - pr_debug("%s:buf = %p\n", __func__, buf); + pr_debug("%s:buf = %pK\n", __func__, buf); dma_buf->dev.type = SNDRV_DMA_TYPE_DEV; dma_buf->dev.dev = substream->pcm->card->dev; dma_buf->private_data = NULL; diff --git a/sound/soc/msm/qdsp6v2/msm-pcm-lpa-v2.c b/sound/soc/msm/qdsp6v2/msm-pcm-lpa-v2.c index 6a52ef0bcd6..9bd2e564459 100644 --- a/sound/soc/msm/qdsp6v2/msm-pcm-lpa-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-pcm-lpa-v2.c @@ -518,7 +518,7 @@ static int msm_pcm_hw_params(struct snd_pcm_substream *substream, if (buf == NULL || buf[0].data == NULL) return -ENOMEM; - pr_debug("%s:buf = %p\n", __func__, buf); + pr_debug("%s:buf = %pK\n", __func__, buf); dma_buf->dev.type = SNDRV_DMA_TYPE_DEV; dma_buf->dev.dev = substream->pcm->card->dev; dma_buf->private_data = NULL; diff --git a/sound/soc/msm/qdsp6v2/msm-pcm-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-pcm-q6-v2.c index 8f98ee6704b..62eaf4ede15 100644 --- a/sound/soc/msm/qdsp6v2/msm-pcm-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-pcm-q6-v2.c @@ -731,7 +731,7 @@ static int msm_pcm_hw_params(struct snd_pcm_substream *substream, if (buf == NULL || buf[0].data == NULL) return -ENOMEM; - pr_debug("%s:buf = %p\n", __func__, buf); + pr_debug("%s:buf = %pK\n", __func__, buf); dma_buf->dev.type = SNDRV_DMA_TYPE_DEV; dma_buf->dev.dev = substream->pcm->card->dev; dma_buf->private_data = NULL; diff --git a/sound/soc/msm/qdsp6v2/q6adm.c b/sound/soc/msm/qdsp6v2/q6adm.c index 2413f47aa5f..68319bb4981 100644 --- a/sound/soc/msm/qdsp6v2/q6adm.c +++ b/sound/soc/msm/qdsp6v2/q6adm.c @@ -18,8 +18,6 @@ #include #include -#include - #include #include #include @@ -33,7 +31,9 @@ #define RESET_COPP_ID 99 #define INVALID_COPP_ID 0xFF -#define ADM_GET_PARAMETER_LENGTH 350 +/* Used for inband payload copy, max size is 4k */ +/* 2 is to account for module & param ID in payload */ +#define ADM_GET_PARAMETER_LENGTH (4096 - APR_HDR_SIZE - 2 * sizeof(uint32_t)) enum { @@ -42,6 +42,7 @@ enum { ADM_RX_AUDVOL_CAL, ADM_TX_AUDVOL_CAL, ADM_CUSTOM_TOP_CAL, + ADM_RTAC, ADM_MAX_CAL_TYPES }; @@ -448,7 +449,7 @@ static int32_t adm_callback(struct apr_client_data *data, void *priv) payload = data->payload; if (data->opcode == RESET_EVENTS) { - pr_debug("adm_callback: Reset event is received: %d %d apr[%p]\n", + pr_debug("adm_callback: Reset event is received: %d %d apr[%pK]\n", data->reset_event, data->reset_proc, this_adm.apr); if (this_adm.apr) { @@ -467,6 +468,10 @@ static int32_t adm_callback(struct apr_client_data *data, void *priv) this_adm.apr = NULL; reset_custom_topology_flags(); this_adm.set_custom_topology = 1; + for (i = 0; i < ADM_MAX_CAL_TYPES; i++) + atomic_set(&this_adm.mem_map_cal_handles[i], + 0); + rtac_clear_mapping(ADM_RTAC_CAL); } pr_debug("Resetting calibration blocks"); for (i = 0; i < MAX_AUDPROC_TYPES; i++) { @@ -580,13 +585,18 @@ static int32_t adm_callback(struct apr_client_data *data, void *priv) if (payload[0] != 0) pr_err("%s: ADM_CMDRSP_GET_PP_PARAMS_V5 returned error = 0x%x\n", __func__, payload[0]); - rtac_make_adm_callback(payload, - data->payload_size); - adm_dolby_get_parameters[0] = payload[3]; - pr_debug("GET_PP PARAM:received parameter length: %x\n", - adm_dolby_get_parameters[0]); - for (i = 0; i < payload[3]; i++) - adm_dolby_get_parameters[1+i] = payload[4+i]; + if (rtac_make_adm_callback(payload, + data->payload_size)) + break; + + if (data->payload_size > (4 * sizeof(uint32_t))) { + adm_dolby_get_parameters[0] = payload[3]; + pr_debug("GET_PP PARAM:received parameter length: %x\n", + adm_dolby_get_parameters[0]); + /* storing param size then params */ + for (i = 0; i < payload[3]; i++) + adm_dolby_get_parameters[1+i] = payload[4+i]; + } atomic_set(&this_adm.copp_stat[index], 1); wake_up(&this_adm.wait[index]); break; @@ -766,11 +776,15 @@ static void send_adm_cal(int port_id, int path, int perf_mode) int result = 0; s32 acdb_path; struct acdb_cal_block aud_cal; - int size = 4096; + int size; pr_debug("%s\n", __func__); /* Maps audio_dev_ctrl path definition to ACDB definition */ acdb_path = path - 1; + if (acdb_path == TX_CAL) + size = 4096 * 4; + else + size = 4096; pr_debug("%s: Sending audproc cal\n", __func__); get_audproc_cal(acdb_path, &aud_cal); @@ -841,6 +855,92 @@ static void send_adm_cal(int port_id, int path, int perf_mode) __func__, port_id, acdb_path); } +int adm_map_rtac_block(struct rtac_cal_block_data *cal_block) +{ + int result = 0; + pr_debug("%s\n", __func__); + + if (cal_block == NULL) { + pr_err("%s: cal_block is NULL!\n", + __func__); + result = -EINVAL; + goto done; + } + + if (cal_block->cal_data.paddr == 0) { + pr_debug("%s: No address to map!\n", + __func__); + result = -EINVAL; + goto done; + } + + if (cal_block->map_data.map_size == 0) { + pr_debug("%s: map size is 0!\n", + __func__); + result = -EINVAL; + goto done; + } + + /* valid port ID needed for callback use primary I2S */ + atomic_set(&this_adm.mem_map_cal_index, ADM_RTAC); + result = adm_memory_map_regions(PRIMARY_I2S_RX, + &cal_block->cal_data.paddr, 0, + &cal_block->map_data.map_size, 1); + if (result < 0) { + pr_err("%s: RTAC mmap did not work! addr = 0x%x, size = %d\n", + __func__, cal_block->cal_data.paddr, + cal_block->map_data.map_size); + goto done; + } + + cal_block->map_data.map_handle = atomic_read( + &this_adm.mem_map_cal_handles[ADM_RTAC]); +done: + return result; +} + +int adm_unmap_rtac_block(uint32_t *mem_map_handle) +{ + int result = 0; + pr_debug("%s\n", __func__); + + if (mem_map_handle == NULL) { + pr_debug("%s: Map handle is NULL, nothing to unmap\n", + __func__); + goto done; + } + + if (*mem_map_handle == 0) { + pr_debug("%s: Map handle is 0, nothing to unmap\n", + __func__); + goto done; + } + + if (*mem_map_handle != atomic_read( + &this_adm.mem_map_cal_handles[ADM_RTAC])) { + pr_err("%s: Map handles do not match! Unmapping RTAC, RTAC map 0x%x, ADM map 0x%x\n", + __func__, *mem_map_handle, atomic_read( + &this_adm.mem_map_cal_handles[ADM_RTAC])); + + /* if mismatch use handle passed in to unmap */ + atomic_set(&this_adm.mem_map_cal_handles[ADM_RTAC], + *mem_map_handle); + } + + /* valid port ID needed for callback use primary I2S */ + atomic_set(&this_adm.mem_map_cal_index, ADM_RTAC); + result = adm_memory_unmap_regions(PRIMARY_I2S_RX); + if (result < 0) { + pr_debug("%s: adm_memory_unmap_regions failed, error %d\n", + __func__, result); + } else { + atomic_set(&this_adm.mem_map_cal_handles[ADM_RTAC], 0); + *mem_map_handle = 0; + } +done: + return result; +} + int adm_unmap_cal_blocks(void) { int i; @@ -861,6 +961,8 @@ int adm_unmap_cal_blocks(void) = 0; } else if (i == ADM_CUSTOM_TOP_CAL) { this_adm.set_custom_topology = 1; + } else { + continue; } atomic_set(&this_adm.mem_map_cal_index, i); diff --git a/sound/soc/msm/qdsp6v2/q6afe.c b/sound/soc/msm/qdsp6v2/q6afe.c index f8042b43426..7eec116e314 100644 --- a/sound/soc/msm/qdsp6v2/q6afe.c +++ b/sound/soc/msm/qdsp6v2/q6afe.c @@ -99,7 +99,7 @@ static int32_t afe_callback(struct apr_client_data *data, void *priv) return -EINVAL; } if (data->opcode == RESET_EVENTS) { - pr_debug("q6afe: reset event = %d %d apr[%p]\n", + pr_debug("q6afe: reset event = %d %d apr[%pK]\n", data->reset_event, data->reset_proc, this_afe.apr); for (i = 0; i < MAX_AFE_CAL_TYPES; i++) { @@ -126,7 +126,7 @@ static int32_t afe_callback(struct apr_client_data *data, void *priv) u8 *payload = data->payload; if ((data->payload_size < sizeof(this_afe.calib_data)) || !payload || (data->token >= AFE_MAX_PORTS)) { - pr_err("%s size %d payload %p token %d\n", + pr_err("%s size %d payload %pK token %d\n", __func__, data->payload_size, payload, data->token); return -EINVAL; } @@ -1922,7 +1922,7 @@ int q6afe_audio_client_buf_alloc_contiguous(unsigned int dir, buf[cnt].used = dir ^ 1; buf[cnt].size = bufsz; buf[cnt].actual_size = bufsz; - pr_debug("%s data[%p]phys[%p][%p]\n", __func__, + pr_debug("%s data[%pK]phys[%pK][%pK]\n", __func__, (void *)buf[cnt].data, (void *)buf[cnt].phys, (void *)&buf[cnt].phys); @@ -2117,14 +2117,16 @@ int q6afe_audio_client_buf_free_contiguous(unsigned int dir, cnt = port->max_buf_cnt - 1; if (port->buf[0].data) { - msm_audio_ion_free(port->buf[0].client, port->buf[0].handle); - pr_debug("%s:data[%p]phys[%p][%p] , client[%p] handle[%p]\n", + pr_debug("%s:data[%pK]phys[%pK][%pK] , client[%pK] handle[%pK]\n", __func__, (void *)port->buf[0].data, (void *)port->buf[0].phys, (void *)&port->buf[0].phys, (void *)port->buf[0].client, (void *)port->buf[0].handle); + msm_audio_ion_free(port->buf[0].client, port->buf[0].handle); + port->buf[0].client = NULL; + port->buf[0].handle = NULL; } while (cnt >= 0) { diff --git a/sound/soc/msm/qdsp6v2/q6asm.c b/sound/soc/msm/qdsp6v2/q6asm.c index a1d0accf400..da73353db7f 100644 --- a/sound/soc/msm/qdsp6v2/q6asm.c +++ b/sound/soc/msm/qdsp6v2/q6asm.c @@ -36,7 +36,6 @@ #include #include -#include #include #include @@ -46,6 +45,7 @@ #define TRUE 0x01 #define FALSE 0x00 +#define FRAME_NUM (8) /* TODO, combine them together */ static DEFINE_MUTEX(session_lock); @@ -478,6 +478,134 @@ void send_asm_custom_topology(struct audio_client *ac) return; } +int q6asm_map_rtac_block(struct rtac_cal_block_data *cal_block) +{ + int result = 0; + struct asm_buffer_node *buf_node = NULL; + struct list_head *ptr, *next; + pr_debug("%s\n", __func__); + + if (cal_block == NULL) { + pr_err("%s: cal_block is NULL!\n", + __func__); + result = -EINVAL; + goto done; + } + + if (cal_block->cal_data.paddr == 0) { + pr_debug("%s: No address to map!\n", + __func__); + result = -EINVAL; + goto done; + } + + if (common_client.mmap_apr == NULL) { + common_client.mmap_apr = q6asm_mmap_apr_reg(); + if (common_client.mmap_apr == NULL) { + pr_err("%s: q6asm_mmap_apr_reg failed\n", + __func__); + result = -EPERM; + goto done; + } + } + + if (cal_block->map_data.map_size == 0) { + pr_debug("%s: map size is 0!\n", + __func__); + result = -EINVAL; + goto done; + } + + /* Use second asm buf to map memory */ + if (common_client.port[OUT].buf == NULL) { + pr_err("%s: common buf is NULL\n", + __func__); + result = -EINVAL; + goto done; + } + + common_client.port[OUT].buf->phys = cal_block->cal_data.paddr; + + result = q6asm_memory_map_regions(&common_client, + OUT, cal_block->map_data.map_size, 1, 1); + if (result < 0) { + pr_err("%s: mmap did not work! addr = 0x%x, size = %d\n", + __func__, cal_block->cal_data.paddr, + cal_block->map_data.map_size); + goto done; + } + + list_for_each_safe(ptr, next, + &common_client.port[OUT].mem_map_handle) { + buf_node = list_entry(ptr, struct asm_buffer_node, + list); + if (buf_node->buf_addr_lsw == cal_block->cal_data.paddr) { + cal_block->map_data.map_handle = buf_node->mmap_hdl; + break; + } + } + + result = q6asm_mmap_apr_dereg(); + if (result < 0) { + pr_err("%s: q6asm_mmap_apr_dereg failed, err %d\n", + __func__, result); + } else { + common_client.mmap_apr = NULL; + } +done: + return result; +} + +int q6asm_unmap_rtac_block(uint32_t *mem_map_handle) +{ + int result = 0; + int result2 = 0; + pr_debug("%s\n", __func__); + + if (mem_map_handle == NULL) { + pr_debug("%s: Map handle is NULL, nothing to unmap\n", + __func__); + goto done; + } + + if (*mem_map_handle == 0) { + pr_debug("%s: Map handle is 0, nothing to unmap\n", + __func__); + goto done; + } + + if (common_client.mmap_apr == NULL) { + common_client.mmap_apr = q6asm_mmap_apr_reg(); + if (common_client.mmap_apr == NULL) { + pr_err("%s: q6asm_mmap_apr_reg failed\n", + __func__); + result = -EPERM; + goto done; + } + } + + + result2 = q6asm_memory_unmap_regions(&common_client, OUT); + if (result2 < 0) { + pr_err("%s: unmap failed, err %d\n", + __func__, result2); + result = result2; + } else { + mem_map_handle = 0; + } + + result2 = q6asm_mmap_apr_dereg(); + if (result2 < 0) { + pr_err("%s: q6asm_mmap_apr_dereg failed, err %d\n", + __func__, result2); + result = result2; + } else { + common_client.mmap_apr = NULL; + } +done: + return result; +} + int q6asm_unmap_cal_blocks(void) { int result = 0; @@ -535,6 +663,8 @@ int q6asm_audio_client_buf_free(unsigned int dir, if (port->buf[cnt].data) { msm_audio_ion_free(port->buf[cnt].client, port->buf[cnt].handle); + port->buf[cnt].client = NULL; + port->buf[cnt].handle = NULL; port->buf[cnt].data = NULL; port->buf[cnt].phys = 0; --(port->max_buf_cnt); @@ -571,14 +701,16 @@ int q6asm_audio_client_buf_free_contiguous(unsigned int dir, } if (port->buf[0].data) { - msm_audio_ion_free(port->buf[0].client, port->buf[0].handle); - pr_debug("%s:data[%p]phys[%p][%p] , client[%p] handle[%p]\n", + pr_debug("%s:data[%pK]phys[%pK][%pK] , client[%pK] handle[%pK]\n", __func__, (void *)port->buf[0].data, (void *)port->buf[0].phys, (void *)&port->buf[0].phys, (void *)port->buf[0].client, (void *)port->buf[0].handle); + msm_audio_ion_free(port->buf[0].client, port->buf[0].handle); + port->buf[0].client = NULL; + port->buf[0].handle = NULL; } while (cnt >= 0) { @@ -786,6 +918,8 @@ int q6asm_audio_client_buf_alloc(unsigned int dir, pr_debug("%s: buffer already allocated\n", __func__); return 0; } + if (bufcnt != FRAME_NUM) + goto fail; mutex_lock(&ac->cmd_lock); buf = kzalloc(((sizeof(struct audio_buffer))*bufcnt), GFP_KERNEL); @@ -816,7 +950,7 @@ int q6asm_audio_client_buf_alloc(unsigned int dir, buf[cnt].used = 1; buf[cnt].size = bufsz; buf[cnt].actual_size = bufsz; - pr_debug("%s data[%p]phys[%p][%p]\n", + pr_debug("%s data[%pK]phys[%pK][%pK]\n", __func__, (void *)buf[cnt].data, (void *)buf[cnt].phys, @@ -876,6 +1010,13 @@ int q6asm_audio_client_buf_alloc_contiguous(unsigned int dir, ac->port[dir].buf = buf; + /* check for integer overflow */ + if ((bufcnt > 0) && ((UINT_MAX / bufcnt) < bufsz)) { + pr_err("%s: integer overflow\n", __func__); + mutex_unlock(&ac->cmd_lock); + goto fail; + } + bytes_to_alloc = bufsz * bufcnt; /* The size to allocate should be multiple of 4K bytes */ @@ -909,7 +1050,7 @@ int q6asm_audio_client_buf_alloc_contiguous(unsigned int dir, buf[cnt].used = dir ^ 1; buf[cnt].size = bufsz; buf[cnt].actual_size = bufsz; - pr_debug("%s data[%p]phys[%p][%p]\n", __func__, + pr_debug("%s data[%pK]phys[%pK][%pK]\n", __func__, (void *)buf[cnt].data, (void *)buf[cnt].phys, (void *)&buf[cnt].phys); @@ -947,7 +1088,7 @@ static int32_t q6asm_srvc_callback(struct apr_client_data *data, void *priv) payload = data->payload; if (data->opcode == RESET_EVENTS) { - pr_debug("%s: Reset event is received: %d %d apr[%p]\n", + pr_debug("%s: Reset event is received: %d %d apr[%pK]\n", __func__, data->reset_event, data->reset_proc, @@ -956,6 +1097,8 @@ static int32_t q6asm_srvc_callback(struct apr_client_data *data, void *priv) this_mmap.apr = NULL; reset_custom_topology_flags(); set_custom_topology = 1; + topology_map_handle = 0; + rtac_clear_mapping(ASM_RTAC_CAL); return 0; } sid = (data->token >> 8) & 0x0F; @@ -1085,15 +1228,13 @@ static int32_t q6asm_callback(struct apr_client_data *data, void *priv) } if (data->opcode == RESET_EVENTS) { - pr_debug("q6asm_callback: Reset event is received: %d %d apr[%p]\n", + pr_debug("q6asm_callback: Reset event is received: %d %d apr[%pK]\n", data->reset_event, data->reset_proc, ac->apr); if (ac->cb) ac->cb(data->opcode, data->token, (uint32_t *)data->payload, ac->priv); apr_reset(ac->apr); ac->apr = NULL; - reset_custom_topology_flags(); - set_custom_topology = 1; return 0; } @@ -1186,7 +1327,7 @@ static int32_t q6asm_callback(struct apr_client_data *data, void *priv) spin_lock_irqsave(&port->dsp_lock, dsp_flags); if (port->buf[data->token].phys != payload[0]) { - pr_err("Buf expected[%p]rxed[%p]\n",\ + pr_err("Buf expected[%pK]rxed[%pK]\n",\ (void *)port->buf[data->token].phys,\ (void *)payload[0]); spin_unlock_irqrestore(&port->dsp_lock, @@ -1243,7 +1384,7 @@ static int32_t q6asm_callback(struct apr_client_data *data, void *priv) port->buf[token].used = 0; if (port->buf[token].phys != payload[READDONE_IDX_BUFADD_LSW]) { - pr_err("Buf expected[%p]rxed[%p]\n",\ + pr_err("Buf expected[%pK]rxed[%pK]\n",\ (void *)port->buf[token].phys,\ (void *)payload[READDONE_IDX_BUFADD_LSW]); spin_unlock_irqrestore(&port->dsp_lock, @@ -1324,7 +1465,7 @@ void *q6asm_is_cpu_buf_avail(int dir, struct audio_client *ac, uint32_t *size, *size = port->buf[idx].actual_size; *index = port->cpu_buf; data = port->buf[idx].data; - pr_debug("%s:session[%d]index[%d] data[%p]size[%d]\n", + pr_debug("%s:session[%d]index[%d] data[%pK]size[%d]\n", __func__, ac->session, port->cpu_buf, @@ -1373,7 +1514,7 @@ void *q6asm_is_cpu_buf_avail_nolock(int dir, struct audio_client *ac, *size = port->buf[idx].actual_size; *index = port->cpu_buf; data = port->buf[idx].data; - pr_debug("%s:session[%d]index[%d] data[%p]size[%d]\n", + pr_debug("%s:session[%d]index[%d] data[%pK]size[%d]\n", __func__, ac->session, port->cpu_buf, data, *size); /* @@ -2978,7 +3119,7 @@ static int q6asm_memory_map_regions(struct audio_client *ac, int dir, mmap_region_cmd; q6asm_add_mmaphdr(ac, &mmap_regions->hdr, cmd_size, TRUE, ((ac->session << 8) | dir)); - pr_debug("mmap_region=0x%p token=0x%x\n", + pr_debug("mmap_region=0x%pK token=0x%x\n", mmap_regions, ((ac->session << 8) | dir)); mmap_regions->hdr.opcode = ASM_CMD_SHARED_MEM_MAP_REGIONS; @@ -3425,7 +3566,7 @@ int q6asm_read(struct audio_client *ac) dsp_buf = port->dsp_buf; ab = &port->buf[dsp_buf]; - pr_debug("%s:session[%d]dsp-buf[%d][%p]cpu_buf[%d][%p]\n", + pr_debug("%s:session[%d]dsp-buf[%d][%pK]cpu_buf[%d][%pK]\n", __func__, ac->session, dsp_buf, @@ -3487,7 +3628,7 @@ int q6asm_read_nolock(struct audio_client *ac) dsp_buf = port->dsp_buf; ab = &port->buf[dsp_buf]; - pr_debug("%s:session[%d]dsp-buf[%d][%p]cpu_buf[%d][%p]\n", + pr_debug("%s:session[%d]dsp-buf[%d][%pK]cpu_buf[%d][%pK]\n", __func__, ac->session, dsp_buf, diff --git a/sound/soc/msm/qdsp6v2/q6core.c b/sound/soc/msm/qdsp6v2/q6core.c index e9352df0177..39a3be6593f 100644 --- a/sound/soc/msm/qdsp6v2/q6core.c +++ b/sound/soc/msm/qdsp6v2/q6core.c @@ -128,7 +128,7 @@ void ocm_core_open(void) if (q6core_lcl.core_handle_q == NULL) q6core_lcl.core_handle_q = apr_register("ADSP", "CORE", aprv2_core_fn_q, 0xFFFFFFFF, NULL); - pr_debug("Open_q %p\n", q6core_lcl.core_handle_q); + pr_debug("Open_q %pK\n", q6core_lcl.core_handle_q); if (q6core_lcl.core_handle_q == NULL) pr_err("%s: Unable to register CORE\n", __func__); } diff --git a/sound/soc/msm/qdsp6v2/q6lsm.c b/sound/soc/msm/qdsp6v2/q6lsm.c index 805b370e670..3ab9ecf5ee9 100644 --- a/sound/soc/msm/qdsp6v2/q6lsm.c +++ b/sound/soc/msm/qdsp6v2/q6lsm.c @@ -252,6 +252,7 @@ void q6lsm_client_free(struct lsm_client *client) q6lsm_mmap_apr_dereg(); mutex_destroy(&client->cmd_lock); kfree(client); + client = NULL; } /* @@ -515,7 +516,7 @@ static int q6lsm_memory_map_regions(struct lsm_client *client, int rc; int cmd_size = 0; - pr_debug("%s: dma_addr_p 0x%x, dma_buf_sz %d, mmap_p 0x%p, session %d\n", + pr_debug("%s: dma_addr_p 0x%x, dma_buf_sz %d, mmap_p 0x%pK, session %d\n", __func__, dma_addr_p, dma_buf_sz, mmap_p, client->session); cmd_size = sizeof(struct avs_cmd_shared_mem_map_regions) + diff --git a/sound/soc/msm/qdsp6v2/q6voice.c b/sound/soc/msm/qdsp6v2/q6voice.c index fe6ef5d0c3b..e0febfc8cf3 100644 --- a/sound/soc/msm/qdsp6v2/q6voice.c +++ b/sound/soc/msm/qdsp6v2/q6voice.c @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -45,6 +44,7 @@ enum { VOC_TOKEN_NONE, VOIP_MEM_MAP_TOKEN, VOC_CAL_MEM_MAP_TOKEN, + VOC_RTAC_MEM_MAP_TOKEN }; static struct common_data common; @@ -88,6 +88,7 @@ static int voice_send_set_pp_enable_cmd(struct voice_data *v, static int is_cal_memory_allocated(void); static int is_voip_memory_allocated(void); static int voice_alloc_cal_mem_map_table(void); +static int voice_alloc_rtac_mem_map_table(void); static int voice_alloc_oob_shared_mem(void); static int voice_free_oob_shared_mem(void); static int voice_alloc_oob_mem_table(void); @@ -1348,6 +1349,59 @@ static int is_cal_memory_allocated(void) return ret; } + +static int free_cal_map_table(void) +{ + int ret = 0; + + if ((common.cal_mem_map_table.client == NULL) || + (common.cal_mem_map_table.handle == NULL)) + goto done; + + ret = msm_audio_ion_free(common.cal_mem_map_table.client, + common.cal_mem_map_table.handle); + if (ret < 0) + pr_err("%s: msm_audio_ion_free failed:\n", __func__); + +done: + common.cal_mem_map_table.client = NULL; + common.cal_mem_map_table.handle = NULL; + return ret; +} + +static int is_rtac_memory_allocated(void) +{ + bool ret; + + if (common.rtac_mem_map_table.client != NULL && + common.rtac_mem_map_table.handle != NULL) + ret = true; + else + ret = false; + + return ret; +} + +static int free_rtac_map_table(void) +{ + int ret = 0; + + if ((common.rtac_mem_map_table.client == NULL) || + (common.rtac_mem_map_table.handle == NULL)) + goto done; + + ret = msm_audio_ion_free(common.rtac_mem_map_table.client, + common.rtac_mem_map_table.handle); + if (ret < 0) + pr_err("%s: msm_audio_ion_free failed:\n", __func__); + +done: + common.rtac_mem_map_table.client = NULL; + common.rtac_mem_map_table.handle = NULL; + return ret; +} + + static int is_voip_memory_allocated(void) { bool ret; @@ -1841,6 +1895,7 @@ static int voice_send_cvs_register_cal_cmd(struct voice_data *v) if (!common.cal_mem_handle) { pr_debug("%s: Cal mem handle is NULL\n", __func__); + ret = -EINVAL; goto done; } @@ -1848,6 +1903,7 @@ static int voice_send_cvs_register_cal_cmd(struct voice_data *v) if (cal_block.cal_size == 0) { pr_err("%s: CVS cal size is 0\n", __func__); + ret = -EINVAL; goto done; } @@ -1868,6 +1924,15 @@ static int voice_send_cvs_register_cal_cmd(struct voice_data *v) /* Get the column info corresponding to CVS cal from ACDB. */ get_voice_col_data(VOCSTRM_CAL, &cal_block); + if (cal_block.cal_size == 0 || + cal_block.cal_size > + sizeof(cvs_reg_cal_cmd.cvs_cal_data.column_info)) { + pr_err("%s: Invalid VOCSTRM_CAL size %d\n", + __func__, cal_block.cal_size); + + ret = -EINVAL; + goto done; + } memcpy(&cvs_reg_cal_cmd.cvs_cal_data.column_info[0], (void *) cal_block.cal_kvaddr, cal_block.cal_size); @@ -2115,6 +2180,7 @@ static int voice_send_cvp_register_cal_cmd(struct voice_data *v) if (!common.cal_mem_handle) { pr_debug("%s: Cal mem handle is NULL\n", __func__); + ret = -EINVAL; goto done; } @@ -2122,6 +2188,7 @@ static int voice_send_cvp_register_cal_cmd(struct voice_data *v) if (cal_block.cal_size == 0) { pr_err("%s: CVP cal size is 0\n", __func__); + ret = -EINVAL; goto done; } @@ -2142,6 +2209,16 @@ static int voice_send_cvp_register_cal_cmd(struct voice_data *v) /* Get the column info corresponding to CVP cal from ACDB. */ get_voice_col_data(VOCPROC_CAL, &cal_block); + if (cal_block.cal_size == 0 || + cal_block.cal_size > + sizeof(cvp_reg_cal_cmd.cvp_cal_data.column_info)) { + pr_err("%s: Invalid VOCPROC_CAL size %d\n", + __func__, cal_block.cal_size); + + ret = -EINVAL; + goto done; + } + memcpy(&cvp_reg_cal_cmd.cvp_cal_data.column_info[0], (void *) cal_block.cal_kvaddr, cal_block.cal_size); @@ -2252,6 +2329,7 @@ static int voice_send_cvp_register_vol_cal_cmd(struct voice_data *v) if (!common.cal_mem_handle) { pr_debug("%s: Cal mem handle is NULL\n", __func__); + ret = -EINVAL; goto done; } @@ -2259,6 +2337,7 @@ static int voice_send_cvp_register_vol_cal_cmd(struct voice_data *v) if (cal_block.cal_size == 0) { pr_err("%s: CVP vol cal size is 0\n", __func__); + ret = -EINVAL; goto done; } @@ -2281,6 +2360,16 @@ static int voice_send_cvp_register_vol_cal_cmd(struct voice_data *v) /* Get the column info corresponding to CVP volume cal from ACDB. */ get_voice_col_data(VOCVOL_CAL, &cal_block); + if (cal_block.cal_size == 0 || + cal_block.cal_size > + sizeof(cvp_reg_vol_cal_cmd.cvp_vol_cal_data.column_info)) { + pr_err("%s: Invalid VOCVOL_CAL size %d\n", + __func__, cal_block.cal_size); + + ret = -EINVAL; + goto done; + } + memcpy(&cvp_reg_vol_cal_cmd.cvp_vol_cal_data.column_info[0], (void *) cal_block.cal_kvaddr, cal_block.cal_size); @@ -2467,6 +2556,7 @@ int voice_unmap_cal_blocks(void) __func__, v->session_id); common.cal_mem_handle = 0; + free_cal_map_table(); } mutex_unlock(&v->lock); } @@ -2573,6 +2663,118 @@ static int voice_map_memory_physical_cmd(struct voice_data *v, return -EINVAL; } +int voc_map_rtac_block(struct rtac_cal_block_data *cal_block) +{ + int result = 0; + struct voice_data *v = NULL; + + pr_debug("%s\n", __func__); + + if (cal_block == NULL) { + pr_err("%s: cal_block is NULL!\n", + __func__); + + result = -EINVAL; + goto done; + } + + if (cal_block->cal_data.paddr == 0) { + pr_debug("%s: No address to map!\n", + __func__); + + result = -EINVAL; + goto done; + } + + if (cal_block->map_data.map_size == 0) { + pr_debug("%s: map size is 0!\n", + __func__); + + result = -EINVAL; + goto done; + } + + mutex_lock(&common.common_lock); + /* use first session */ + v = &common.voice[0]; + mutex_lock(&v->lock); + + if (!is_rtac_memory_allocated()) { + result = voice_alloc_rtac_mem_map_table(); + if (result < 0) { + pr_err("%s: RTAC alloc mem map table did not work! addr = 0x%x, size = %d\n", + __func__, cal_block->cal_data.paddr, + cal_block->map_data.map_size); + + goto done_unlock; + } + } + + result = voice_map_memory_physical_cmd(v, + &common.rtac_mem_map_table, + (dma_addr_t)cal_block->cal_data.paddr, + cal_block->map_data.map_size, + VOC_RTAC_MEM_MAP_TOKEN); + if (result < 0) { + pr_err("%s: RTAC mmap did not work! addr = 0x%x, size = %d\n", + __func__, cal_block->cal_data.paddr, + cal_block->map_data.map_size); + + free_rtac_map_table(); + goto done_unlock; + } + + cal_block->map_data.map_handle = common.rtac_mem_handle; +done_unlock: + mutex_unlock(&v->lock); + mutex_unlock(&common.common_lock); +done: + return result; +} + +int voc_unmap_rtac_block(uint32_t *mem_map_handle) +{ + int result = 0; + struct voice_data *v = NULL; + + pr_debug("%s\n", __func__); + + if (mem_map_handle == NULL) { + pr_debug("%s: Map handle is NULL, nothing to unmap\n", + __func__); + + goto done; + } + + if (*mem_map_handle == 0) { + pr_debug("%s: Map handle is 0, nothing to unmap\n", + __func__); + + goto done; + } + + mutex_lock(&common.common_lock); + /* Use first session */ + /* Only used for apr wait lock */ + v = &common.voice[0]; + mutex_lock(&v->lock); + + result = voice_send_mvm_unmap_memory_physical_cmd( + v, *mem_map_handle); + if (result) { + pr_err("%s: voice_send_mvm_unmap_memory_physical_cmd Failed for session 0x%x!\n", + __func__, v->session_id); + } else { + *mem_map_handle = 0; + common.rtac_mem_handle = 0; + free_rtac_map_table(); + } + mutex_unlock(&v->lock); + mutex_unlock(&common.common_lock); +done: + return result; +} + static int voice_mem_map_cal_block(struct voice_data *v) { int ret = 0; @@ -3203,7 +3405,7 @@ static int voice_send_cvs_packet_exchange_config_cmd(struct voice_data *v) packet_exchange_config_pkt.enc_buf_addr = (uint32_t)enc_buf; packet_exchange_config_pkt.enc_buf_size = 4096; - pr_debug("%s: dec buf: add %p, size %d, enc buf: add %p, size %d\n", + pr_debug("%s: dec buf: add %pK, size %d, enc buf: add %pK, size %d\n", __func__, dec_buf, packet_exchange_config_pkt.dec_buf_size, @@ -4606,6 +4808,8 @@ static int32_t qdsp_mvm_callback(struct apr_client_data *data, void *priv) /* clean up memory handle */ c->cal_mem_handle = 0; + c->rtac_mem_handle = 0; + rtac_clear_mapping(VOICE_RTAC_CAL); /* Sub-system restart is applicable to all sessions. */ for (i = 0; i < MAX_VOC_SESSIONS; i++) { @@ -4693,6 +4897,18 @@ static int32_t qdsp_mvm_callback(struct apr_client_data *data, void *priv) pr_debug("%s: cal mem handle 0x%x\n", __func__, c->cal_mem_handle); + v->mvm_state = CMD_STATUS_SUCCESS; + wake_up(&v->mvm_wait); + } + } else if (data->payload_size && + data->token == VOC_RTAC_MEM_MAP_TOKEN) { + ptr = data->payload; + if (ptr[0]) { + c->rtac_mem_handle = ptr[0]; + + pr_debug("%s: cal mem handle 0x%x\n", + __func__, c->rtac_mem_handle); + v->mvm_state = CMD_STATUS_SUCCESS; wake_up(&v->mvm_wait); } @@ -5130,6 +5346,8 @@ static int voice_free_oob_shared_mem(void) rc = msm_audio_ion_free(v->shmem_info.sh_buf.client, v->shmem_info.sh_buf.handle); + v->shmem_info.sh_buf.client = NULL; + v->shmem_info.sh_buf.handle = NULL; if (rc < 0) { pr_err("%s: Error:%d freeing memory\n", __func__, rc); @@ -5190,12 +5408,12 @@ static int voice_alloc_oob_shared_mem(void) cnt++; } - pr_debug("%s buf[0].data:[%p], buf[0].phys:[%p], &buf[0].phys:[%p],\n", + pr_debug("%s buf[0].data:[%pK], buf[0].phys:[%pK], &buf[0].phys:[%pK],\n", __func__, (void *)v->shmem_info.sh_buf.buf[0].data, (void *)v->shmem_info.sh_buf.buf[0].phys, (void *)&v->shmem_info.sh_buf.buf[0].phys); - pr_debug("%s: buf[1].data:[%p], buf[1].phys[%p], &buf[1].phys[%p]\n", + pr_debug("%s: buf[1].data:[%pK], buf[1].phys[%pK], &buf[1].phys[%pK]\n", __func__, (void *)v->shmem_info.sh_buf.buf[1].data, (void *)v->shmem_info.sh_buf.buf[1].phys, @@ -5237,7 +5455,7 @@ static int voice_alloc_oob_mem_table(void) } v->shmem_info.memtbl.size = sizeof(struct vss_imemory_table_t); - pr_debug("%s data[%p]phys[%p][%p]\n", __func__, + pr_debug("%s data[%pK]phys[%pK][%pK]\n", __func__, (void *)v->shmem_info.memtbl.data, (void *)v->shmem_info.memtbl.phys, (void *)&v->shmem_info.memtbl.phys); @@ -5274,6 +5492,33 @@ static int voice_alloc_cal_mem_map_table(void) return ret; } +static int voice_alloc_rtac_mem_map_table(void) +{ + int ret = 0; + int len; + + ret = msm_audio_ion_alloc("voc_rtac_cal", + &(common.rtac_mem_map_table.client), + &(common.rtac_mem_map_table.handle), + sizeof(struct vss_imemory_table_t), + (ion_phys_addr_t *)&common.rtac_mem_map_table.phys, + (size_t *) &len, + &(common.rtac_mem_map_table.data)); + if (ret < 0) { + pr_err("%s: audio ION alloc failed, rc = %d\n", + __func__, ret); + goto done; + } + + common.rtac_mem_map_table.size = sizeof(struct vss_imemory_table_t); + pr_debug("%s: data 0x%x phys 0x%x\n", __func__, + (unsigned int) common.rtac_mem_map_table.data, + common.rtac_mem_map_table.phys); + +done: + return ret; +} + static int voice_alloc_and_map_cal_mem(struct voice_data *v) { int ret = 0; diff --git a/sound/soc/msm/qdsp6v2/q6voice.h b/sound/soc/msm/qdsp6v2/q6voice.h index 86121ed6c37..6e77f91e0bb 100644 --- a/sound/soc/msm/qdsp6v2/q6voice.h +++ b/sound/soc/msm/qdsp6v2/q6voice.h @@ -13,6 +13,7 @@ #define __QDSP6VOICE_H__ #include +#include #include #include @@ -1328,6 +1329,10 @@ struct common_data { struct mem_map_table cal_mem_map_table; uint32_t cal_mem_handle; + + struct mem_map_table rtac_mem_map_table; + uint32_t rtac_mem_handle; + struct cal_mem cvp_cal; struct cal_mem cvs_cal; @@ -1431,6 +1436,8 @@ int is_voc_initialized(void); int voc_register_vocproc_vol_table(void); int voc_deregister_vocproc_vol_table(void); int voice_unmap_cal_blocks(void); +int voc_map_rtac_block(struct rtac_cal_block_data *cal_block); +int voc_unmap_rtac_block(uint32_t *mem_map_handle); uint32_t voc_get_session_id(char *name); diff --git a/sound/soc/msm/qdsp6v2/rtac.c b/sound/soc/msm/qdsp6v2/rtac.c index 848f470e4b2..d39ae5c7cf9 100644 --- a/sound/soc/msm/qdsp6v2/rtac.c +++ b/sound/soc/msm/qdsp6v2/rtac.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -53,10 +54,26 @@ bool rtac_make_voice_callback(u32 mode, uint32_t *payload, #define RTAC_MAX_ACTIVE_DEVICES 4 #define RTAC_MAX_ACTIVE_VOICE_COMBOS 2 #define RTAC_MAX_ACTIVE_POPP 8 -#define RTAC_BUF_SIZE 4096 +#define RTAC_BUF_SIZE 8192 #define TIMEOUT_MS 1000 +struct rtac_cal_block_data rtac_cal[MAX_RTAC_BLOCKS] = { +/* ADM_RTAC_CAL */ + {{RTAC_BUF_SIZE, 0, 0, 0}, {0, 0, 0} }, +/* ASM_RTAC_CAL */ + {{RTAC_BUF_SIZE, 0, 0, 0}, {0, 0, 0} }, +/* VOICE_RTAC_CAL */ + {{RTAC_BUF_SIZE, 0, 0, 0}, {0, 0, 0} } +}; + +struct rtac_common_data { + atomic_t usage_count; + atomic_t apr_err_code; +}; + +static struct rtac_common_data rtac_common; + /* APR data */ struct rtac_apr_data { void *apr_handle; @@ -83,15 +100,11 @@ struct rtac_adm { struct rtac_adm_data device[RTAC_MAX_ACTIVE_DEVICES]; }; static struct rtac_adm rtac_adm_data; -static u32 rtac_adm_payload_size; -static u32 rtac_adm_user_buf_size; -static u8 *rtac_adm_buffer; +static u32 *rtac_adm_buffer; /* ASM APR */ -static u32 rtac_asm_payload_size; -static u32 rtac_asm_user_buf_size; -static u8 *rtac_asm_buffer; +static u32 *rtac_asm_buffer; /* Voice info & APR */ @@ -110,9 +123,7 @@ struct rtac_voice { }; static struct rtac_voice rtac_voice_data; -static u32 rtac_voice_payload_size; -static u32 rtac_voice_user_buf_size; -static u8 *rtac_voice_buffer; +static u32 *rtac_voice_buffer; static u32 voice_session_id[RTAC_MAX_ACTIVE_VOICE_COMBOS]; @@ -122,16 +133,227 @@ struct mutex rtac_asm_apr_mutex; struct mutex rtac_voice_mutex; struct mutex rtac_voice_apr_mutex; +int rtac_clear_mapping(uint32_t cal_type) +{ + int result = 0; + pr_debug("%s\n", __func__); + + if (cal_type >= MAX_RTAC_BLOCKS) { + pr_debug("%s: invalid cal type %d\n", __func__, cal_type); + result = -EINVAL; + goto done; + } + + rtac_cal[cal_type].map_data.map_handle = 0; +done: + return result; +} + +int rtac_allocate_cal_buffer(uint32_t cal_type) +{ + int result = 0; + int len; + pr_debug("%s\n", __func__); + + if (cal_type >= MAX_RTAC_BLOCKS) { + pr_err("%s: cal_type %d is invalid!\n", + __func__, cal_type); + result = -EINVAL; + goto done; + } + + if (rtac_cal[cal_type].cal_data.paddr != 0) { + pr_err("%s: memory already allocated! cal_type %d, paddr 0x%x\n", + __func__, cal_type, rtac_cal[cal_type].cal_data.paddr); + result = -EPERM; + goto done; + } + + result = msm_audio_ion_alloc("rtac_client", + &rtac_cal[cal_type].map_data.ion_client, + &rtac_cal[cal_type].map_data.ion_handle, + rtac_cal[cal_type].map_data.map_size, + (ion_phys_addr_t *)&rtac_cal[cal_type].cal_data.paddr, + (size_t *)&len, + (void **)&rtac_cal[cal_type].cal_data.kvaddr); + if (result < 0) { + pr_err("%s: ION create client for RTAC failed\n", + __func__); + goto done; + } + + pr_debug("%s: cal_type %d, paddr 0x%x, kvaddr 0x%x, map_size 0x%x\n", + __func__, cal_type, + rtac_cal[cal_type].cal_data.paddr, + rtac_cal[cal_type].cal_data.kvaddr, + rtac_cal[cal_type].map_data.map_size); +done: + return result; +} + +int rtac_free_cal_buffer(uint32_t cal_type) +{ + int result = 0; + pr_debug("%s\n", __func__); + + if (cal_type >= MAX_RTAC_BLOCKS) { + pr_err("%s: cal_type %d is invalid!\n", + __func__, cal_type); + result = -EINVAL; + goto done; + } + + if (rtac_cal[cal_type].map_data.ion_client == NULL) { + pr_debug("%s: cal_type %d not allocated!\n", + __func__, cal_type); + goto done; + } + + result = msm_audio_ion_free(rtac_cal[cal_type].map_data.ion_client, + rtac_cal[cal_type].map_data.ion_handle); + if (result < 0) { + pr_err("%s: ION free for RTAC failed! cal_type %d, paddr 0x%x\n", + __func__, cal_type, rtac_cal[cal_type].cal_data.paddr); + goto done; + } + + rtac_cal[cal_type].map_data.map_handle = 0; + rtac_cal[cal_type].map_data.ion_client = NULL; + rtac_cal[cal_type].map_data.ion_handle = NULL; + rtac_cal[cal_type].cal_data.size = 0; + rtac_cal[cal_type].cal_data.kvaddr = 0; + rtac_cal[cal_type].cal_data.paddr = 0; +done: + return result; +} + +int rtac_map_cal_buffer(uint32_t cal_type) +{ + int result = 0; + pr_debug("%s\n", __func__); + + if (cal_type >= MAX_RTAC_BLOCKS) { + pr_err("%s: cal_type %d is invalid!\n", + __func__, cal_type); + result = -EINVAL; + goto done; + } + + if (rtac_cal[cal_type].map_data.map_handle != 0) { + pr_err("%s: already mapped cal_type %d\n", + __func__, cal_type); + result = -EPERM; + goto done; + } + + if (rtac_cal[cal_type].cal_data.paddr == 0) { + pr_err("%s: physical address is NULL cal_type %d\n", + __func__, cal_type); + result = -EPERM; + goto done; + } + + switch (cal_type) { + case ADM_RTAC_CAL: + result = adm_map_rtac_block(&rtac_cal[cal_type]); + break; + case ASM_RTAC_CAL: + result = q6asm_map_rtac_block(&rtac_cal[cal_type]); + break; + case VOICE_RTAC_CAL: + result = voc_map_rtac_block(&rtac_cal[cal_type]); + break; + } + if (result < 0) { + pr_err("%s: map RTAC failed! cal_type %d\n", + __func__, cal_type); + goto done; + } +done: + return result; +} + +int rtac_unmap_cal_buffer(uint32_t cal_type) +{ + int result = 0; + pr_debug("%s\n", __func__); + + if (cal_type >= MAX_RTAC_BLOCKS) { + pr_err("%s: cal_type %d is invalid!\n", + __func__, cal_type); + result = -EINVAL; + goto done; + } + + if (rtac_cal[cal_type].map_data.map_handle == 0) { + pr_debug("%s: nothing to unmap cal_type %d\n", + __func__, cal_type); + goto done; + } + + switch (cal_type) { + case ADM_RTAC_CAL: + result = adm_unmap_rtac_block( + &rtac_cal[cal_type].map_data.map_handle); + break; + case ASM_RTAC_CAL: + result = q6asm_unmap_rtac_block( + &rtac_cal[cal_type].map_data.map_handle); + break; + case VOICE_RTAC_CAL: + result = voc_unmap_rtac_block( + &rtac_cal[cal_type].map_data.map_handle); + break; + } + if (result < 0) { + pr_err("%s: unmap RTAC failed! cal_type %d\n", + __func__, cal_type); + goto done; + } +done: + return result; +} + static int rtac_open(struct inode *inode, struct file *f) { + int result = 0; pr_debug("%s\n", __func__); - return 0; + + atomic_inc(&rtac_common.usage_count); + return result; } static int rtac_release(struct inode *inode, struct file *f) { + int result = 0; + int result2 = 0; + int i; pr_debug("%s\n", __func__); - return 0; + + atomic_dec(&rtac_common.usage_count); + pr_debug("%s: ref count %d!\n", __func__, + atomic_read(&rtac_common.usage_count)); + + if (atomic_read(&rtac_common.usage_count) > 0) + goto done; + + for (i = 0; i < MAX_RTAC_BLOCKS; i++) { + result2 = rtac_unmap_cal_buffer(i); + if (result2 < 0) { + pr_err("%s: unmap buffer failed! error %d!\n", + __func__, result2); + result = result2; + } + + result2 = rtac_free_cal_buffer(i); + if (result2 < 0) { + pr_err("%s: free buffer failed! error %d!\n", + __func__, result2); + result = result2; + } + } +done: + return result; } /* ADM Info */ @@ -414,52 +636,56 @@ bool rtac_make_adm_callback(uint32_t *payload, u32 payload_size) if (atomic_read(&rtac_adm_apr_data.cmd_state) != 1) return false; - /* Offset data for in-band payload */ - rtac_copy_adm_payload_to_user(payload, payload_size); + pr_debug("%s\n", __func__); + if (payload_size == sizeof(uint32_t)) + atomic_set(&rtac_common.apr_err_code, payload[0]); + else if (payload_size == (2*sizeof(uint32_t))) + atomic_set(&rtac_common.apr_err_code, payload[1]); + atomic_set(&rtac_adm_apr_data.cmd_state, 0); wake_up(&rtac_adm_apr_data.cmd_wait); return true; } -void rtac_copy_adm_payload_to_user(void *payload, u32 payload_size) -{ - pr_debug("%s\n", __func__); - rtac_adm_payload_size = payload_size; - - memcpy(rtac_adm_buffer, &payload_size, sizeof(u32)); - if (payload_size != 0) { - if (payload_size > rtac_adm_user_buf_size) { - pr_err("%s: Buffer set not big enough for returned data, buf size = %d, ret data = %d\n", - __func__, rtac_adm_user_buf_size, payload_size); - rtac_adm_payload_size = 0; - goto done; - } - memcpy(rtac_adm_buffer + sizeof(u32), payload, payload_size); - } -done: - return; -} - u32 send_adm_apr(void *buf, u32 opcode) { s32 result; - u32 count = 0; + u32 user_buf_size = 0; u32 bytes_returned = 0; u32 port_index = 0; u32 copp_id; u32 payload_size; + u32 data_size = 0; struct apr_hdr adm_params; pr_debug("%s\n", __func__); - if (copy_from_user(&count, (void *)buf, sizeof(count))) { - pr_err("%s: Copy to user failed! buf = 0x%x\n", + if (rtac_cal[ADM_RTAC_CAL].map_data.ion_handle == NULL) { + result = rtac_allocate_cal_buffer(ADM_RTAC_CAL); + if (result < 0) { + pr_err("%s: allocate buffer failed!", + __func__); + goto done; + } + } + + if (rtac_cal[ADM_RTAC_CAL].map_data.map_handle == 0) { + result = rtac_map_cal_buffer(ADM_RTAC_CAL); + if (result < 0) { + pr_err("%s: map buffer failed!", + __func__); + goto done; + } + } + + if (copy_from_user(&user_buf_size, (void *)buf, + sizeof(user_buf_size))) { + pr_err("%s: Copy from user failed! buf = 0x%x\n", __func__, (unsigned int)buf); - result = -EFAULT; goto done; } - - if (count <= 0) { - pr_err("%s: Invalid buffer size = %d\n", __func__, count); + if (user_buf_size <= 0) { + pr_err("%s: Invalid buffer size = %d\n", + __func__, user_buf_size); goto done; } @@ -469,13 +695,6 @@ u32 send_adm_apr(void *buf, u32 opcode) goto done; } - - if (payload_size > MAX_PAYLOAD_SIZE) { - pr_err("%s: Invalid payload size = %d\n", - __func__, payload_size); - goto done; - } - if (copy_from_user(&copp_id, buf + 2 * sizeof(u32), sizeof(u32))) { pr_err("%s: Could not copy port id from user buffer\n", __func__); @@ -500,15 +719,42 @@ u32 send_adm_apr(void *buf, u32 opcode) goto err; } - /* Set globals for copy of returned payload */ - rtac_adm_user_buf_size = count; + if (opcode == ADM_CMD_SET_PP_PARAMS_V5) { + /* set payload size to in-band payload */ + /* set data size to actual out of band payload size */ + data_size = payload_size - 4 * sizeof(u32); + if (data_size > rtac_cal[ADM_RTAC_CAL].map_data.map_size) { + pr_err("%s: Invalid data size = %d\n", + __func__, data_size); + goto done; + } + payload_size = 4 * sizeof(u32); + + /* Copy buffer to out-of-band payload */ + if (copy_from_user((void *) + rtac_cal[ADM_RTAC_CAL].cal_data.kvaddr, + buf + 7 * sizeof(u32), data_size)) { + pr_err("%s: Could not copy payload from user buffer\n", + __func__); + goto err; + } + /* set payload size in packet */ + rtac_adm_buffer[8] = data_size; + } else { + if (payload_size > MAX_PAYLOAD_SIZE) { + pr_err("%s: Invalid payload size = %d\n", + __func__, payload_size); + goto done; + } - /* Copy buffer to in-band payload */ - if (copy_from_user(rtac_adm_buffer + sizeof(adm_params), - buf + 3 * sizeof(u32), payload_size)) { - pr_err("%s: Could not copy payload from user buffer\n", - __func__); - goto err; + /* Copy buffer to in-band payload */ + if (copy_from_user(rtac_adm_buffer + + sizeof(adm_params)/sizeof(u32), + buf + 3 * sizeof(u32), payload_size)) { + pr_err("%s: Could not copy payload from user buffer\n", + __func__); + goto err; + } } /* Pack header */ @@ -525,14 +771,20 @@ u32 send_adm_apr(void *buf, u32 opcode) adm_params.token = copp_id; adm_params.opcode = opcode; + /* fill for out-of-band */ + rtac_adm_buffer[5] = rtac_cal[ADM_RTAC_CAL].cal_data.paddr; + rtac_adm_buffer[6] = 0; + rtac_adm_buffer[7] = rtac_cal[ADM_RTAC_CAL].map_data.map_handle; + memcpy(rtac_adm_buffer, &adm_params, sizeof(adm_params)); atomic_set(&rtac_adm_apr_data.cmd_state, 1); - pr_debug("%s: Sending RTAC command size = %d\n", - __func__, adm_params.pkt_size); + pr_debug("%s: Sending RTAC command ioctl 0x%x, paddr 0x%x\n", + __func__, opcode, + rtac_cal[ADM_RTAC_CAL].cal_data.paddr); result = apr_send_pkt(rtac_adm_apr_data.apr_handle, - (uint32_t *)rtac_adm_buffer); + (uint32_t *)rtac_adm_buffer); if (result < 0) { pr_err("%s: Set params failed port = %d, copp = %d\n", __func__, port_index, copp_id); @@ -548,21 +800,34 @@ u32 send_adm_apr(void *buf, u32 opcode) __func__, port_index, copp_id); goto done; } + if (atomic_read(&rtac_common.apr_err_code)) { + pr_err("%s: DSP returned error code = %d, opcode = 0x%x\n", + __func__, atomic_read(&rtac_common.apr_err_code), + opcode); + goto done; + } - if (rtac_adm_payload_size != 0) { - if (copy_to_user(buf, rtac_adm_buffer, - rtac_adm_payload_size + sizeof(u32))) { + if (opcode == ADM_CMD_GET_PP_PARAMS_V5) { + bytes_returned = ((u32 *)rtac_cal[ADM_RTAC_CAL].cal_data. + kvaddr)[2] + 3 * sizeof(u32); + + if (bytes_returned > user_buf_size) { + pr_err("%s: User buf not big enough, size = 0x%x, returned size = 0x%x\n", + __func__, user_buf_size, bytes_returned); + goto done; + } + + if (copy_to_user(buf, (void *) + rtac_cal[ADM_RTAC_CAL].cal_data.kvaddr, + bytes_returned)) { pr_err("%s: Could not copy buffer to user,size = %d\n", - __func__, payload_size); + __func__, bytes_returned); goto done; } + } else { + bytes_returned = data_size; } - /* Return data written for SET & data read for GET */ - if (opcode == ADM_CMD_GET_PP_PARAMS_V5) - bytes_returned = rtac_adm_payload_size; - else - bytes_returned = payload_size; done: return bytes_returned; err: @@ -588,51 +853,54 @@ bool rtac_make_asm_callback(u32 session_id, uint32_t *payload, return false; pr_debug("%s\n", __func__); - /* Offset data for in-band payload */ - rtac_copy_asm_payload_to_user(payload, payload_size); + if (payload_size == sizeof(uint32_t)) + atomic_set(&rtac_common.apr_err_code, payload[0]); + else if (payload_size == (2*sizeof(uint32_t))) + atomic_set(&rtac_common.apr_err_code, payload[1]); + atomic_set(&rtac_asm_apr_data[session_id].cmd_state, 0); wake_up(&rtac_asm_apr_data[session_id].cmd_wait); return true; } -void rtac_copy_asm_payload_to_user(void *payload, u32 payload_size) -{ - pr_debug("%s\n", __func__); - rtac_asm_payload_size = payload_size; - - memcpy(rtac_asm_buffer, &payload_size, sizeof(u32)); - if (payload_size) { - if (payload_size > rtac_asm_user_buf_size) { - pr_err("%s: Buffer set not big enough for returned data, buf size = %d, ret data = %d\n", - __func__, rtac_asm_user_buf_size, payload_size); - rtac_asm_payload_size = 0; - goto done; - } - memcpy(rtac_asm_buffer + sizeof(u32), payload, payload_size); - } -done: - return; -} - u32 send_rtac_asm_apr(void *buf, u32 opcode) { s32 result; - u32 count = 0; + u32 user_buf_size = 0; u32 bytes_returned = 0; u32 session_id = 0; u32 payload_size; + u32 data_size = 0; struct apr_hdr asm_params; pr_debug("%s\n", __func__); - if (copy_from_user(&count, (void *)buf, sizeof(count))) { - pr_err("%s: Copy to user failed! buf = 0x%x\n", + if (rtac_cal[ASM_RTAC_CAL].map_data.ion_handle == NULL) { + result = rtac_allocate_cal_buffer(ASM_RTAC_CAL); + if (result < 0) { + pr_err("%s: allocate buffer failed!", + __func__); + goto done; + } + } + + if (rtac_cal[ASM_RTAC_CAL].map_data.map_handle == 0) { + result = rtac_map_cal_buffer(ASM_RTAC_CAL); + if (result < 0) { + pr_err("%s: map buffer failed!", + __func__); + goto done; + } + } + + if (copy_from_user(&user_buf_size, (void *)buf, + sizeof(user_buf_size))) { + pr_err("%s: Copy from user failed! buf = 0x%x\n", __func__, (unsigned int)buf); - result = -EFAULT; goto done; } - - if (count <= 0) { - pr_err("%s: Invalid buffer size = %d\n", __func__, count); + if (user_buf_size <= 0) { + pr_err("%s: Invalid buffer size = %d\n", + __func__, user_buf_size); goto done; } @@ -642,40 +910,59 @@ u32 send_rtac_asm_apr(void *buf, u32 opcode) goto done; } - if (payload_size > MAX_PAYLOAD_SIZE) { - pr_err("%s: Invalid payload size = %d\n", - __func__, payload_size); - goto done; - } - if (copy_from_user(&session_id, buf + 2 * sizeof(u32), sizeof(u32))) { pr_err("%s: Could not copy session id from user buffer\n", __func__); goto done; } - - if (session_id > (SESSION_MAX + 1)) { + if (session_id >= (SESSION_MAX + 1)) { pr_err("%s: Invalid Session = %d\n", __func__, session_id); goto done; } mutex_lock(&rtac_asm_apr_mutex); - if (session_id < SESSION_MAX+1) { - if (rtac_asm_apr_data[session_id].apr_handle == NULL) { - pr_err("%s: APR not initialized\n", __func__); + if (rtac_asm_apr_data[session_id].apr_handle == NULL) { + pr_err("%s: APR not initialized\n", __func__); + goto err; + } + + if (opcode == ASM_STREAM_CMD_SET_PP_PARAMS_V2) { + /* set payload size to in-band payload */ + /* set data size to actual out of band payload size */ + data_size = payload_size - 4 * sizeof(u32); + if (data_size > rtac_cal[ASM_RTAC_CAL].map_data.map_size) { + pr_err("%s: Invalid data size = %d\n", + __func__, data_size); + goto done; + } + payload_size = 4 * sizeof(u32); + + /* Copy buffer to out-of-band payload */ + if (copy_from_user((void *) + rtac_cal[ASM_RTAC_CAL].cal_data.kvaddr, + buf + 7 * sizeof(u32), data_size)) { + pr_err("%s: Could not copy payload from user buffer\n", + __func__); goto err; } - } + /* set payload size in packet */ + rtac_asm_buffer[8] = data_size; - /* Set globals for copy of returned payload */ - rtac_asm_user_buf_size = count; + } else { + if (payload_size > MAX_PAYLOAD_SIZE) { + pr_err("%s: Invalid payload size = %d\n", + __func__, payload_size); + goto done; + } - /* Copy buffer to in-band payload */ - if (copy_from_user(rtac_asm_buffer + sizeof(asm_params), - buf + 3 * sizeof(u32), payload_size)) { - pr_err("%s: Could not copy payload from user buffer\n", - __func__); - goto err; + /* Copy buffer to in-band payload */ + if (copy_from_user(rtac_asm_buffer + + sizeof(asm_params)/sizeof(u32), + buf + 3 * sizeof(u32), payload_size)) { + pr_err("%s: Could not copy payload from user buffer\n", + __func__); + goto err; + } } /* Pack header */ @@ -692,12 +979,17 @@ u32 send_rtac_asm_apr(void *buf, u32 opcode) asm_params.token = session_id; asm_params.opcode = opcode; + /* fill for out-of-band */ + rtac_asm_buffer[5] = rtac_cal[ASM_RTAC_CAL].cal_data.paddr; + rtac_asm_buffer[6] = 0; + rtac_asm_buffer[7] = rtac_cal[ASM_RTAC_CAL].map_data.map_handle; + memcpy(rtac_asm_buffer, &asm_params, sizeof(asm_params)); - if (session_id < SESSION_MAX+1) - atomic_set(&rtac_asm_apr_data[session_id].cmd_state, 1); + atomic_set(&rtac_asm_apr_data[session_id].cmd_state, 1); - pr_debug("%s: Sending RTAC command size = %d, session_id=%d\n", - __func__, asm_params.pkt_size, session_id); + pr_debug("%s: Sending RTAC command ioctl 0x%x, paddr 0x%x\n", + __func__, opcode, + rtac_cal[ASM_RTAC_CAL].cal_data.paddr); result = apr_send_pkt(rtac_asm_apr_data[session_id].apr_handle, (uint32_t *)rtac_asm_buffer); @@ -717,21 +1009,33 @@ u32 send_rtac_asm_apr(void *buf, u32 opcode) __func__, session_id); goto done; } + if (atomic_read(&rtac_common.apr_err_code)) { + pr_err("%s: DSP returned error code = %d, opcode = 0x%x\n", + __func__, atomic_read(&rtac_common.apr_err_code), + opcode); + goto done; + } - if (rtac_asm_payload_size != 0) { - if (copy_to_user(buf, rtac_asm_buffer, - rtac_asm_payload_size + sizeof(u32))) { + if (opcode == ASM_STREAM_CMD_GET_PP_PARAMS_V2) { + bytes_returned = ((u32 *)rtac_cal[ASM_RTAC_CAL].cal_data. + kvaddr)[2] + 3 * sizeof(u32); + + if (bytes_returned > user_buf_size) { + pr_err("%s: User buf not big enough, size = 0x%x, returned size = 0x%x\n", + __func__, user_buf_size, bytes_returned); + goto done; + } + + if (copy_to_user(buf, (void *) + rtac_cal[ASM_RTAC_CAL].cal_data.kvaddr, + bytes_returned)) { pr_err("%s: Could not copy buffer to user,size = %d\n", - __func__, payload_size); + __func__, bytes_returned); goto done; } + } else { + bytes_returned = data_size; } - - /* Return data written for SET & data read for GET */ - if (opcode == ASM_STREAM_CMD_GET_PP_PARAMS_V2) - bytes_returned = rtac_asm_payload_size; - else - bytes_returned = payload_size; done: return bytes_returned; err: @@ -757,51 +1061,54 @@ bool rtac_make_voice_callback(u32 mode, uint32_t *payload, u32 payload_size) return false; pr_debug("%s\n", __func__); - /* Offset data for in-band payload */ - rtac_copy_voice_payload_to_user(payload, payload_size); + if (payload_size == sizeof(uint32_t)) + atomic_set(&rtac_common.apr_err_code, payload[0]); + else if (payload_size == (2*sizeof(uint32_t))) + atomic_set(&rtac_common.apr_err_code, payload[1]); + atomic_set(&rtac_voice_apr_data[mode].cmd_state, 0); wake_up(&rtac_voice_apr_data[mode].cmd_wait); return true; } -void rtac_copy_voice_payload_to_user(void *payload, u32 payload_size) -{ - pr_debug("%s\n", __func__); - rtac_voice_payload_size = payload_size; - - memcpy(rtac_voice_buffer, &payload_size, sizeof(u32)); - if (payload_size) { - if (payload_size > rtac_voice_user_buf_size) { - pr_err("%s: Buffer set not big enough for returned data, buf size = %d, ret data = %d\n", - __func__, rtac_voice_user_buf_size, payload_size); - rtac_voice_payload_size = 0; - goto done; - } - memcpy(rtac_voice_buffer + sizeof(u32), payload, payload_size); - } -done: - return; -} - u32 send_voice_apr(u32 mode, void *buf, u32 opcode) { s32 result; - u32 count = 0; + u32 user_buf_size = 0; u32 bytes_returned = 0; u32 payload_size; u32 dest_port; + u32 data_size = 0; struct apr_hdr voice_params; pr_debug("%s\n", __func__); - if (copy_from_user(&count, (void *)buf, sizeof(count))) { - pr_err("%s: Copy to user failed! buf = 0x%x\n", + if (rtac_cal[VOICE_RTAC_CAL].map_data.ion_handle == NULL) { + result = rtac_allocate_cal_buffer(VOICE_RTAC_CAL); + if (result < 0) { + pr_err("%s: allocate buffer failed!", + __func__); + goto done; + } + } + + if (rtac_cal[VOICE_RTAC_CAL].map_data.map_handle == 0) { + result = rtac_map_cal_buffer(VOICE_RTAC_CAL); + if (result < 0) { + pr_err("%s: map buffer failed!", + __func__); + goto done; + } + } + + if (copy_from_user(&user_buf_size, (void *)buf, + sizeof(user_buf_size))) { + pr_err("%s: Copy from user failed! buf = 0x%x\n", __func__, (unsigned int)buf); - result = -EFAULT; goto done; } - - if (count <= 0) { - pr_err("%s: Invalid buffer size = %d\n", __func__, count); + if (user_buf_size <= 0) { + pr_err("%s: Invalid buffer size = %d\n", + __func__, user_buf_size); goto done; } @@ -811,12 +1118,6 @@ u32 send_voice_apr(u32 mode, void *buf, u32 opcode) goto done; } - if (payload_size > MAX_PAYLOAD_SIZE) { - pr_err("%s: Invalid payload size = %d\n", - __func__, payload_size); - goto done; - } - if (copy_from_user(&dest_port, buf + 2 * sizeof(u32), sizeof(u32))) { pr_err("%s: Could not copy port id from user buffer\n", __func__); @@ -835,15 +1136,42 @@ u32 send_voice_apr(u32 mode, void *buf, u32 opcode) goto err; } - /* Set globals for copy of returned payload */ - rtac_voice_user_buf_size = count; + if (opcode == VOICE_CMD_SET_PARAM) { + /* set payload size to in-band payload */ + /* set data size to actual out of band payload size */ + data_size = payload_size - 4 * sizeof(u32); + if (data_size > rtac_cal[VOICE_RTAC_CAL].map_data.map_size) { + pr_err("%s: Invalid data size = %d\n", + __func__, data_size); + goto done; + } + payload_size = 4 * sizeof(u32); + + /* Copy buffer to out-of-band payload */ + if (copy_from_user((void *) + rtac_cal[VOICE_RTAC_CAL].cal_data.kvaddr, + buf + 7 * sizeof(u32), data_size)) { + pr_err("%s: Could not copy payload from user buffer\n", + __func__); + goto err; + } + /* set payload size in packet */ + rtac_voice_buffer[8] = data_size; + } else { + if (payload_size > MAX_PAYLOAD_SIZE) { + pr_err("%s: Invalid payload size = %d\n", + __func__, payload_size); + goto done; + } - /* Copy buffer to in-band payload */ - if (copy_from_user(rtac_voice_buffer + sizeof(voice_params), - buf + 3 * sizeof(u32), payload_size)) { - pr_err("%s: Could not copy payload from user buffer\n", - __func__); - goto err; + /* Copy buffer to in-band payload */ + if (copy_from_user(rtac_voice_buffer + + sizeof(voice_params)/sizeof(u32), + buf + 3 * sizeof(u32), payload_size)) { + pr_err("%s: Could not copy payload from user buffer\n", + __func__); + goto err; + } } /* Pack header */ @@ -860,11 +1188,17 @@ u32 send_voice_apr(u32 mode, void *buf, u32 opcode) voice_params.token = 0; voice_params.opcode = opcode; + /* fill for out-of-band */ + rtac_voice_buffer[5] = rtac_cal[VOICE_RTAC_CAL].map_data.map_handle; + rtac_voice_buffer[6] = rtac_cal[VOICE_RTAC_CAL].cal_data.paddr; + rtac_voice_buffer[7] = 0; + memcpy(rtac_voice_buffer, &voice_params, sizeof(voice_params)); atomic_set(&rtac_voice_apr_data[mode].cmd_state, 1); - pr_debug("%s: Sending RTAC command size = %d, opcode = %x\n", - __func__, voice_params.pkt_size, opcode); + pr_debug("%s: Sending RTAC command ioctl 0x%x, paddr 0x%x\n", + __func__, opcode, + rtac_cal[VOICE_RTAC_CAL].cal_data.paddr); result = apr_send_pkt(rtac_voice_apr_data[mode].apr_handle, (uint32_t *)rtac_voice_buffer); @@ -883,21 +1217,33 @@ u32 send_voice_apr(u32 mode, void *buf, u32 opcode) __func__, opcode); goto done; } + if (atomic_read(&rtac_common.apr_err_code)) { + pr_err("%s: DSP returned error code = %d, opcode = 0x%x\n", + __func__, atomic_read(&rtac_common.apr_err_code), + opcode); + goto done; + } - if (rtac_voice_payload_size != 0) { - if (copy_to_user(buf, rtac_voice_buffer, - rtac_voice_payload_size + sizeof(u32))) { + if (opcode == VOICE_CMD_GET_PARAM) { + bytes_returned = ((u32 *)rtac_cal[VOICE_RTAC_CAL].cal_data. + kvaddr)[2] + 3 * sizeof(u32); + + if (bytes_returned > user_buf_size) { + pr_err("%s: User buf not big enough, size = 0x%x, returned size = 0x%x\n", + __func__, user_buf_size, bytes_returned); + goto done; + } + + if (copy_to_user(buf, (void *) + rtac_cal[VOICE_RTAC_CAL].cal_data.kvaddr, + bytes_returned)) { pr_err("%s: Could not copy buffer to user, size = %d\n", - __func__, payload_size); + __func__, bytes_returned); goto done; } + } else { + bytes_returned = data_size; } - - /* Return data written for SET & data read for GET */ - if (opcode == VOICE_CMD_GET_PARAM) - bytes_returned = rtac_voice_payload_size; - else - bytes_returned = payload_size; done: return bytes_returned; err: @@ -991,6 +1337,10 @@ static int __init rtac_init(void) int i = 0; pr_debug("%s\n", __func__); + /* Driver */ + atomic_set(&rtac_common.usage_count, 0); + atomic_set(&rtac_common.apr_err_code, 0); + /* ADM */ memset(&rtac_adm_data, 0, sizeof(rtac_adm_data)); rtac_adm_apr_data.apr_handle = NULL; @@ -999,10 +1349,11 @@ static int __init rtac_init(void) mutex_init(&rtac_adm_mutex); mutex_init(&rtac_adm_apr_mutex); - rtac_adm_buffer = kzalloc(RTAC_BUF_SIZE, GFP_KERNEL); + rtac_adm_buffer = kzalloc( + rtac_cal[ADM_RTAC_CAL].map_data.map_size, GFP_KERNEL); if (rtac_adm_buffer == NULL) { pr_err("%s: Could not allocate payload of size = %d\n", - __func__, RTAC_BUF_SIZE); + __func__, rtac_cal[ADM_RTAC_CAL].map_data.map_size); goto nomem; } @@ -1014,10 +1365,11 @@ static int __init rtac_init(void) } mutex_init(&rtac_asm_apr_mutex); - rtac_asm_buffer = kzalloc(RTAC_BUF_SIZE, GFP_KERNEL); + rtac_asm_buffer = kzalloc( + rtac_cal[ASM_RTAC_CAL].map_data.map_size, GFP_KERNEL); if (rtac_asm_buffer == NULL) { pr_err("%s: Could not allocate payload of size = %d\n", - __func__, RTAC_BUF_SIZE); + __func__, rtac_cal[ASM_RTAC_CAL].map_data.map_size); kzfree(rtac_adm_buffer); goto nomem; } @@ -1032,10 +1384,11 @@ static int __init rtac_init(void) mutex_init(&rtac_voice_mutex); mutex_init(&rtac_voice_apr_mutex); - rtac_voice_buffer = kzalloc(RTAC_BUF_SIZE, GFP_KERNEL); + rtac_voice_buffer = kzalloc( + rtac_cal[VOICE_RTAC_CAL].map_data.map_size, GFP_KERNEL); if (rtac_voice_buffer == NULL) { pr_err("%s: Could not allocate payload of size = %d\n", - __func__, RTAC_BUF_SIZE); + __func__, rtac_cal[VOICE_RTAC_CAL].map_data.map_size); kzfree(rtac_adm_buffer); kzfree(rtac_asm_buffer); goto nomem; diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index ad335b75970..09dfb3e845c 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -2046,13 +2046,17 @@ unsigned int snd_soc_read(struct snd_soc_codec *codec, unsigned int reg) { unsigned int ret; - if (unlikely(!snd_card_is_online_state(codec->card->snd_card))) { - dev_err(codec->dev, "read 0x%02x while offline\n", reg); - return -ENODEV; - } - ret = codec->read(codec, reg); - dev_dbg(codec->dev, "read %x => %x\n", reg, ret); - trace_snd_soc_reg_read(codec, reg, ret); + if (codec->read) { + if (unlikely(!snd_card_is_online_state(codec->card->snd_card))) { + dev_err(codec->dev, "read 0x%02x while offline\n", reg); + return -ENODEV; + } + ret = codec->read(codec, reg); + dev_dbg(codec->dev, "read %x => %x\n", reg, ret); + trace_snd_soc_reg_read(codec, reg, ret); + } + else + ret = -EIO; return ret; } @@ -2061,13 +2065,17 @@ EXPORT_SYMBOL_GPL(snd_soc_read); unsigned int snd_soc_write(struct snd_soc_codec *codec, unsigned int reg, unsigned int val) { - if (unlikely(!snd_card_is_online_state(codec->card->snd_card))) { - dev_err(codec->dev, "write 0x%02x while offline\n", reg); - return -ENODEV; - } - dev_dbg(codec->dev, "write %x = %x\n", reg, val); - trace_snd_soc_reg_write(codec, reg, val); - return codec->write(codec, reg, val); + if (codec->write) { + if (unlikely(!snd_card_is_online_state(codec->card->snd_card))) { + dev_err(codec->dev, "write 0x%02x while offline\n", reg); + return -ENODEV; + } + dev_dbg(codec->dev, "write %x = %x\n", reg, val); + trace_snd_soc_reg_write(codec, reg, val); + return codec->write(codec, reg, val); + } + else + return -EIO; } EXPORT_SYMBOL_GPL(snd_soc_write); diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 27817266867..b7054de9515 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -136,6 +136,7 @@ static int create_fixed_stream_quirk(struct snd_usb_audio *chip, snd_printk(KERN_ERR "cannot memdup\n"); return -ENOMEM; } + INIT_LIST_HEAD(&fp->list); if (fp->nr_rates > MAX_NR_RATES) { kfree(fp); return -EINVAL; @@ -153,24 +154,32 @@ static int create_fixed_stream_quirk(struct snd_usb_audio *chip, stream = (fp->endpoint & USB_DIR_IN) ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK; err = snd_usb_add_audio_stream(chip, stream, fp); - if (err < 0) { - kfree(fp); - kfree(rate_table); - return err; - } + if (err < 0) + goto error; if (fp->iface != get_iface_desc(&iface->altsetting[0])->bInterfaceNumber || fp->altset_idx >= iface->num_altsetting) { + err = -EINVAL; + goto error; + } + alts = &iface->altsetting[fp->altset_idx]; + if (get_iface_desc(alts)->bNumEndpoints < 1) { kfree(fp); kfree(rate_table); return -EINVAL; } - alts = &iface->altsetting[fp->altset_idx]; + fp->datainterval = snd_usb_parse_datainterval(chip, alts); fp->maxpacksize = le16_to_cpu(get_endpoint(alts, 0)->wMaxPacketSize); usb_set_interface(chip->dev, fp->iface, 0); snd_usb_init_pitch(chip, fp->iface, alts, fp); snd_usb_init_sample_rate(chip, fp->iface, alts, fp, fp->rate_max); return 0; + + error: + list_del(&fp->list); /* unlink for avoiding double-free */ + kfree(fp); + kfree(rate_table); + return err; } /* @@ -237,6 +246,7 @@ static int create_uaxx_quirk(struct snd_usb_audio *chip, fp->ep_attr = get_endpoint(alts, 0)->bmAttributes; fp->datainterval = 0; fp->maxpacksize = le16_to_cpu(get_endpoint(alts, 0)->wMaxPacketSize); + INIT_LIST_HEAD(&fp->list); switch (fp->maxpacksize) { case 0x120: @@ -260,6 +270,7 @@ static int create_uaxx_quirk(struct snd_usb_audio *chip, ? SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK; err = snd_usb_add_audio_stream(chip, stream, fp); if (err < 0) { + list_del(&fp->list); /* unlink for avoiding double-free */ kfree(fp); return err; } diff --git a/sound/usb/stream.c b/sound/usb/stream.c index 5ff8010b2d6..9e544a15241 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -77,7 +77,9 @@ static void snd_usb_audio_pcm_free(struct snd_pcm *pcm) /* * add this endpoint to the chip instance. * if a stream with the same endpoint already exists, append to it. - * if not, create a new pcm stream. + * if not, create a new pcm stream. note, fp is added to the substream + * fmt_list and will be freed on the chip instance release. do not free + * fp or do remove it from the substream fmt_list to avoid double-free. */ int snd_usb_add_audio_stream(struct snd_usb_audio *chip, int stream, @@ -395,6 +397,7 @@ int snd_usb_parse_audio_interface(struct snd_usb_audio *chip, int iface_no) * (fp->maxpacksize & 0x7ff); fp->attributes = parse_uac_endpoint_attributes(chip, alts, protocol, iface_no); fp->clock = clock; + INIT_LIST_HEAD(&fp->list); /* some quirks for attributes here */ @@ -438,6 +441,7 @@ int snd_usb_parse_audio_interface(struct snd_usb_audio *chip, int iface_no) snd_printdd(KERN_INFO "%d:%u:%d: add audio endpoint %#x\n", dev->devnum, iface_no, altno, fp->endpoint); err = snd_usb_add_audio_stream(chip, stream, fp); if (err < 0) { + list_del(&fp->list); /* unlink for avoiding double-free */ kfree(fp->rate_table); kfree(fp); return err;