From 4abe756879e780d68300225761caedc3c0f08b72 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 10:14:40 +0100 Subject: [PATCH 01/21] trim returned string when the given offset and length overflows --- ext/mapped_file.c | 5 +++-- test/test_file_window.rb | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ext/mapped_file.c b/ext/mapped_file.c index d835ab5..f639c75 100644 --- a/ext/mapped_file.c +++ b/ext/mapped_file.c @@ -109,7 +109,7 @@ static VALUE sm_mapped_file_close(VALUE vself) } /* - * Document-method: close + * Document-method: read_window_data * call-seq: obj.read_window_data(offset, length) * * Read +length+ bytes starting at +offset+ @@ -140,8 +140,9 @@ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE v buff[i] = sm_map->map[curr++]; } + // If the range overflows, return part that overlaps if ((offset + length) > sm_map->len) - return rb_str_new(buff, ((offset+length) - sm_map->len)-1); + return rb_str_new(buff, sm_map->len - offset); return rb_str_new(buff, length); } diff --git a/test/test_file_window.rb b/test/test_file_window.rb index 77d18a4..c120810 100644 --- a/test/test_file_window.rb +++ b/test/test_file_window.rb @@ -5,7 +5,7 @@ class TestFileWindow < Test::Unit::TestCase def setup @file = Tempfile.new("TestFileWindow.data") - File.open(@file.path, "w"){|f| f.puts(('a'..'z').to_a.join) } + File.open(@file.path, "w"){|f| f.write(('a'..'z').to_a.join) } @fw = SimpleMmap::FileWindow.new(@file.path) end @@ -39,6 +39,15 @@ def test_get_single_byte_at_current_offset_with_index assert_equal 1, @fw.offset end + def test_get_bytes_past_length + assert_equal "z", @fw[25, 10] + end + + def test_nil_on_negative_index + assert_equal nil, @fw[-1] + assert_equal nil, @fw[-1, 2] + end + def test_get_from_x_to_y_with_index_comma_notation assert_equal "cde", @fw[2, 3] assert_equal 5, @fw.pos From c0fe2cdad52576b222b7728681a02feaa00f1e6c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 10:28:13 +0100 Subject: [PATCH 02/21] added method to query mapped file for its length --- ext/mapped_file.c | 19 +++++++++++++++++++ lib/simple_mmap/file_window.rb | 5 +++++ test/test_file_window.rb | 4 ++++ 3 files changed, 28 insertions(+) diff --git a/ext/mapped_file.c b/ext/mapped_file.c index f639c75..e086af9 100644 --- a/ext/mapped_file.c +++ b/ext/mapped_file.c @@ -147,6 +147,24 @@ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE v return rb_str_new(buff, length); } +/* + * Document-method: size + * call-seq: obj.size + * + * Return size of mapped file + */ + +static VALUE sm_mapped_file_size(VALUE vself) +{ + VALUE vsm_map; + simple_mmap_map *sm_map; + + sm_map = ALLOC(simple_mmap_map); + vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); + Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); + return UINT2NUM(sm_map->len); +} + void Init_mapped_file() { mod_simple_mmap = rb_define_module("SimpleMmap"); @@ -155,6 +173,7 @@ void Init_mapped_file() rb_define_private_method(sm_mapped_file, "initialize", sm_mapped_file_initialize, 1); rb_define_method(sm_mapped_file, "close", sm_mapped_file_close, 0); rb_define_method(sm_mapped_file, "read_window_data", sm_mapped_file_read_window_data, 2); + rb_define_method(sm_mapped_file, "size", sm_mapped_file_size, 0); sm_map_data = rb_define_class_under(sm_mapped_file, "MmapData", rb_cObject); } diff --git a/lib/simple_mmap/file_window.rb b/lib/simple_mmap/file_window.rb index b6050f1..9176449 100644 --- a/lib/simple_mmap/file_window.rb +++ b/lib/simple_mmap/file_window.rb @@ -92,5 +92,10 @@ def read(length) @offset += length data end + + # Return size of mapped file + def size + @mmap.size + end end end diff --git a/test/test_file_window.rb b/test/test_file_window.rb index c120810..33cb2e1 100644 --- a/test/test_file_window.rb +++ b/test/test_file_window.rb @@ -68,4 +68,8 @@ def test_read assert_equal "abc", @fw.read(3) assert_equal 3, @fw.offset end + + def test_size + assert_equal 26, @fw.size + end end From a5973055db0bea5605d19e2e7b0bece13cefc8e5 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 11:03:28 +0100 Subject: [PATCH 03/21] use jeweler and prepare for release on gemcutter --- Rakefile | 56 +++++++++++++++++++++++--------------- VERSION | 1 + lib/simple_mmap.rb | 1 - lib/simple_mmap/version.rb | 24 ---------------- 4 files changed, 35 insertions(+), 47 deletions(-) create mode 100644 VERSION delete mode 100644 lib/simple_mmap/version.rb diff --git a/Rakefile b/Rakefile index a2ceecc..df846f3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,38 +1,50 @@ -# encoding: utf-8 - require 'rubygems' -require 'hoe' -require './lib/simple_mmap/version.rb' -require "rake/clean" - -Hoe.new('simple_mmap', SimpleMmap::VERSION) do |p| - p.rubyforge_name = 'simple-mmap' - p.developer('Johan Sørensen', 'johan@johansorensen.com') - p.spec_extras = { - "extensions" => ["Rakefile"] - } +require 'rake' + +begin + require 'jeweler' + Jeweler::Tasks.new do |gem| + gem.name = "simple-mmap" + gem.email = "pcnoordhuis@gmail.com" + gem.homepage = "http://github.com/pietern/simple-mmap" + gem.authors = ["Johan Sørensen", "Pieter Noordhuis"] + + gem.extensions = ["ext/extconf.rb"] + gem.files = FileList['ext/Makefile', 'ext/*.{c,rb}', 'lib/**/*.rb', 'test/**/*.rb'] + gem.require_paths = ["ext", "lib"] + end + Jeweler::GemcutterTasks.new +rescue LoadError + puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler" end DLEXT = Config::CONFIG['DLEXT'] - file 'ext/Makefile' => FileList['ext/{*.c,*.h,*.rb}'] do chdir('ext') { ruby 'extconf.rb' } end -CLEAN.include 'ext/Makefile', 'ext/mkmf.log' file "ext/mapped_file.#{DLEXT}" => FileList['ext/Makefile', 'ext/*.{c,h,rb}'] do |f| sh 'cd ext && make' end -CLEAN.include 'ext/*.{o,bundle,so,dll}' -file "lib/simple_mmap/mapped_file.#{DLEXT}" => "ext/mapped_file.#{DLEXT}" do |f| - cp f.prerequisites, "lib/simple_mmap/", :preserve => true +namespace :ext do + task :clean do + %W[ + ext/Makefile + ext/mapped_file.#{DLEXT} + ].each do |file| + sh "rm -f #{file}" + end + end + + desc 'Build the mapped_file extension' + task :build => [:clean, "ext/mapped_file.#{DLEXT}"] end -#CLEAN.include "lib/simple_mmap/mapped_file.#{DLEXT}" -desc 'Build the mapped_file extension' -task :build => "lib/simple_mmap/mapped_file.#{DLEXT}" - -task :test => [:build] +require 'rake/testtask' +Rake::TestTask.new(:test => "ext:build") do |test| + test.libs << 'ext' + test.verbose = true +end task :default => :test diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..9084fa2 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.0 diff --git a/lib/simple_mmap.rb b/lib/simple_mmap.rb index 972ea91..c79c5ff 100644 --- a/lib/simple_mmap.rb +++ b/lib/simple_mmap.rb @@ -20,6 +20,5 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. $:.unshift File.dirname(__FILE__) -require "simple_mmap/version" require "simple_mmap/mapped_file" require "simple_mmap/file_window" diff --git a/lib/simple_mmap/version.rb b/lib/simple_mmap/version.rb deleted file mode 100644 index 539e453..0000000 --- a/lib/simple_mmap/version.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2009 Johan Sørensen -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# 'Software'), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -module SimpleMmap - VERSION = '1.0.0' -end From 52009f4efc5b0ee3963efa1f1b1c3c3c6f31cf50 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 11:04:56 +0100 Subject: [PATCH 04/21] added summary --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index df846f3..bf05ad7 100644 --- a/Rakefile +++ b/Rakefile @@ -7,6 +7,7 @@ begin gem.name = "simple-mmap" gem.email = "pcnoordhuis@gmail.com" gem.homepage = "http://github.com/pietern/simple-mmap" + gem.summary = "Read-only wrapper for mmap" gem.authors = ["Johan Sørensen", "Pieter Noordhuis"] gem.extensions = ["ext/extconf.rb"] From b0f47c2f038dee34565969caa8d2948ed91dce50 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 11:56:16 +0100 Subject: [PATCH 05/21] use rake-compiler instead of own rakefile hacks for compilation --- .gitignore | 7 +----- Rakefile | 36 +++++++++-------------------- ext/extconf.rb | 5 ---- ext/simple_mmap/extconf.rb | 4 ++++ ext/{ => simple_mmap}/mapped_file.c | 0 5 files changed, 16 insertions(+), 36 deletions(-) delete mode 100644 ext/extconf.rb create mode 100644 ext/simple_mmap/extconf.rb rename ext/{ => simple_mmap}/mapped_file.c (100%) diff --git a/.gitignore b/.gitignore index 6bad8bb..33c6e83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,6 @@ -/ext/*.o -/ext/*.bundle -/ext/*.so -/ext/*.dll -/ext/mkmf.log +/tmp /lib/simple_mmap/*.bundle /lib/simple_mmap/*.so /lib/simple_mmap/*.dll -/ext/Makefile /pkg /doc/* diff --git a/Rakefile b/Rakefile index bf05ad7..9bf8e6a 100644 --- a/Rakefile +++ b/Rakefile @@ -7,43 +7,29 @@ begin gem.name = "simple-mmap" gem.email = "pcnoordhuis@gmail.com" gem.homepage = "http://github.com/pietern/simple-mmap" - gem.summary = "Read-only wrapper for mmap" + gem.summary = "A simplistic interface for reading memory mapped files" gem.authors = ["Johan Sørensen", "Pieter Noordhuis"] - gem.extensions = ["ext/extconf.rb"] - gem.files = FileList['ext/Makefile', 'ext/*.{c,rb}', 'lib/**/*.rb', 'test/**/*.rb'] + gem.extensions = ["ext/**/extconf.rb"] + gem.files = FileList['Rakefile', 'ext/**/*.{c,rb}', 'lib/**/*.rb', 'test/**/*.rb'] gem.require_paths = ["ext", "lib"] + + gem.add_development_dependency "rake-compiler", "0.7.0" end Jeweler::GemcutterTasks.new rescue LoadError puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler" end -DLEXT = Config::CONFIG['DLEXT'] -file 'ext/Makefile' => FileList['ext/{*.c,*.h,*.rb}'] do - chdir('ext') { ruby 'extconf.rb' } -end - -file "ext/mapped_file.#{DLEXT}" => FileList['ext/Makefile', 'ext/*.{c,h,rb}'] do |f| - sh 'cd ext && make' -end - -namespace :ext do - task :clean do - %W[ - ext/Makefile - ext/mapped_file.#{DLEXT} - ].each do |file| - sh "rm -f #{file}" - end - end - - desc 'Build the mapped_file extension' - task :build => [:clean, "ext/mapped_file.#{DLEXT}"] +gem 'rake-compiler', '>= 0.7.0' +require 'rake/extensiontask' +Rake::ExtensionTask.new('simple_mmap') do |ext| + ext.name = "mapped_file" + ext.lib_dir = File.join('lib', 'simple_mmap') end require 'rake/testtask' -Rake::TestTask.new(:test => "ext:build") do |test| +Rake::TestTask.new(:test => :compile) do |test| test.libs << 'ext' test.verbose = true end diff --git a/ext/extconf.rb b/ext/extconf.rb deleted file mode 100644 index f4fe914..0000000 --- a/ext/extconf.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'mkmf' - -dir_config('mapped_file') -have_func('mmap') -create_makefile('mapped_file') diff --git a/ext/simple_mmap/extconf.rb b/ext/simple_mmap/extconf.rb new file mode 100644 index 0000000..c41cbc3 --- /dev/null +++ b/ext/simple_mmap/extconf.rb @@ -0,0 +1,4 @@ +require 'mkmf' + +have_func('mmap') +create_makefile('simple_mmap/mapped_file') diff --git a/ext/mapped_file.c b/ext/simple_mmap/mapped_file.c similarity index 100% rename from ext/mapped_file.c rename to ext/simple_mmap/mapped_file.c From d62786fdffd3496e825694acd9a460d50a3958f9 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 11:57:59 +0100 Subject: [PATCH 06/21] Version bump to 1.1.1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9084fa2..524cb55 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.1.1 From 306a9f3d313e605a546e4d12ac86d2b8d56dcdaa Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 11:58:32 +0100 Subject: [PATCH 07/21] fix build process --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 9bf8e6a..e38c917 100644 --- a/Rakefile +++ b/Rakefile @@ -10,7 +10,7 @@ begin gem.summary = "A simplistic interface for reading memory mapped files" gem.authors = ["Johan Sørensen", "Pieter Noordhuis"] - gem.extensions = ["ext/**/extconf.rb"] + gem.extensions = FileList["ext/**/extconf.rb"] gem.files = FileList['Rakefile', 'ext/**/*.{c,rb}', 'lib/**/*.rb', 'test/**/*.rb'] gem.require_paths = ["ext", "lib"] From f15c95e209e91a003e0b60d4b023a022a4d6afd4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 12:37:48 +0100 Subject: [PATCH 08/21] when length/2 overflowed, uninitialized memory was put in the string buffer --- ext/simple_mmap/mapped_file.c | 5 ++--- test/test_file_window.rb | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index e086af9..baca4a9 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -130,12 +130,11 @@ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE v return Qnil; } - size_t curr; - curr = offset; + size_t curr = offset; size_t i; for(i = 0; i < length; ++i) { //printf("i=%i offset=%i length=%i curr=%i map->len=%i\n", i, offset, length, curr, sm_map->len); - if ((curr + i) > sm_map->len) + if ((offset + i) > sm_map->len) break; buff[i] = sm_map->map[curr++]; } diff --git a/test/test_file_window.rb b/test/test_file_window.rb index 33cb2e1..0ae42e5 100644 --- a/test/test_file_window.rb +++ b/test/test_file_window.rb @@ -43,6 +43,10 @@ def test_get_bytes_past_length assert_equal "z", @fw[25, 10] end + def test_get_all_bytes + assert_equal ('a'..'z').to_a.join, @fw[0, 26] + end + def test_nil_on_negative_index assert_equal nil, @fw[-1] assert_equal nil, @fw[-1, 2] From bc01b190c773c823eb38dbf99d6302e931f2028d Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 12:38:00 +0100 Subject: [PATCH 09/21] Version bump to 1.1.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 524cb55..45a1b3f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.1 +1.1.2 From 3e64c93191b36ca55aea33471041661e6327dc12 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 17:13:09 +0100 Subject: [PATCH 10/21] remove superfluous allocations to stop leaking memory --- ext/simple_mmap/mapped_file.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index baca4a9..12fba92 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -122,7 +122,6 @@ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE v VALUE vsm_map; simple_mmap_map *sm_map; - sm_map = ALLOC(simple_mmap_map); vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); @@ -158,7 +157,6 @@ static VALUE sm_mapped_file_size(VALUE vself) VALUE vsm_map; simple_mmap_map *sm_map; - sm_map = ALLOC(simple_mmap_map); vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); return UINT2NUM(sm_map->len); From 2ea16c8034cc5970b46130f049385a6f7aaa0ea6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Feb 2010 17:13:33 +0100 Subject: [PATCH 11/21] Version bump to 1.1.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 45a1b3f..781dcb0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.2 +1.1.3 From f37a9079dbc534a33fec09c583100e9ca2c73566 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 11 Feb 2010 11:00:07 +0100 Subject: [PATCH 12/21] don't copy the data in the stack, let ruby handle the mmap'ed region --- ext/simple_mmap/mapped_file.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index 12fba92..fb584a9 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -118,31 +118,22 @@ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE v { size_t offset = NUM2INT(voffset); size_t length = NUM2INT(vlength); - char buff[length]; VALUE vsm_map; simple_mmap_map *sm_map; vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); - if (offset > sm_map->len) { + if (offset < 0 || offset > sm_map->len) { return Qnil; } - - size_t curr = offset; - size_t i; - for(i = 0; i < length; ++i) { - //printf("i=%i offset=%i length=%i curr=%i map->len=%i\n", i, offset, length, curr, sm_map->len); - if ((offset + i) > sm_map->len) - break; - buff[i] = sm_map->map[curr++]; - } - + // If the range overflows, return part that overlaps - if ((offset + length) > sm_map->len) - return rb_str_new(buff, sm_map->len - offset); - - return rb_str_new(buff, length); + if ((offset + length) > sm_map->len) { + length = sm_map->len - offset; + } + + return rb_str_new(sm_map->map + offset, length); } /* From cb9823a2b3a7917bf4fd9ee3a260f6a30933ea4f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 11 Feb 2010 17:57:39 +0100 Subject: [PATCH 13/21] Version bump to 1.1.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 781dcb0..65087b4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.3 +1.1.4 From e23f38159d5736120877e05bc1ef6b5b3618d8af Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Tue, 5 Nov 2013 21:29:04 +0900 Subject: [PATCH 14/21] add Gemfile and .travis.yml to try Travis CI --- .travis.yml | 6 ++++++ Gemfile | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 .travis.yml create mode 100644 Gemfile diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f939f55 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: ruby +rvm: + - 2.0.0 + - 1.9.3 + - 1.9.2 + - 1.8.7 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..fb0d0f3 --- /dev/null +++ b/Gemfile @@ -0,0 +1,7 @@ +# A sample Gemfile +source "https://rubygems.org" + +group :test do + gem 'jeweler' + gem 'rake-compiler' +end From cd8a755e698231ba741e3b68b9824ef43673c36e Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Tue, 5 Nov 2013 21:57:13 +0900 Subject: [PATCH 15/21] add magic comment to Rakefile to specify its encoding --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index e38c917..416d347 100644 --- a/Rakefile +++ b/Rakefile @@ -1,3 +1,5 @@ +#encoding: utf-8 + require 'rubygems' require 'rake' From f4fabe82552c5775ea95e949940374e1acb4e5e1 Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Tue, 5 Nov 2013 22:58:14 +0900 Subject: [PATCH 16/21] use NUM2SIZET instead of NUM2INT for offset and length --- ext/simple_mmap/mapped_file.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index fb584a9..52448b3 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -28,6 +28,24 @@ #include #include "ruby.h" +#ifndef NUM2SIZET +# include +# include +# if SIZE_MAX > ULONG_MAX +# define NUM2SIZET NUM2ULL +# define SIZET2NUM ULONG2NUM +# elif SIZE_MAX > UINT_MAX +# define NUM2SIZET NUM2ULONG +# define SIZET2NUM ULONG2NUM +# elif SIZE_MAX > USHRT_MAX +# define NUM2SIZET NUM2UINT +# define SIZET2NUM UINT2NUM +# else +# define NUM2SIZET NUM2USHORT +# define SIZET2NUM USHORT2NUM +# endif +#endif + // a memory mapped file typedef struct { int fd; @@ -116,17 +134,19 @@ static VALUE sm_mapped_file_close(VALUE vself) */ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE vlength) { - size_t offset = NUM2INT(voffset); - size_t length = NUM2INT(vlength); + size_t offset, length; VALUE vsm_map; simple_mmap_map *sm_map; + if (NUM2LL(voffset) < 0) return Qnil; + if (NUM2LL(vlength) < 0) rb_raise(rb_eRangeError, "length out of range: %lld", NUM2LL(vlength)); + offset = NUM2SIZET(voffset); + length = NUM2SIZET(vlength); + vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); - if (offset < 0 || offset > sm_map->len) { - return Qnil; - } + if (offset > sm_map->len) return Qnil; // If the range overflows, return part that overlaps if ((offset + length) > sm_map->len) { From a077d7a13980095ab500813a1aef6c6066d5b281 Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Tue, 5 Nov 2013 23:33:12 +0900 Subject: [PATCH 17/21] add arguments to MappedFile.new and FileWindow.new (offset and length, both are nil by default) --- ext/simple_mmap/mapped_file.c | 34 ++++++++++++++++++++++++++-------- lib/simple_mmap/file_window.rb | 4 ++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index 52448b3..219d472 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -51,6 +51,8 @@ typedef struct { int fd; caddr_t map; size_t len; + caddr_t read_base; + size_t read_len; } simple_mmap_map; static VALUE mod_simple_mmap; @@ -63,7 +65,7 @@ static VALUE sm_map_data; * * mmap() the file at +path+ */ -static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) +static VALUE sm_mapped_file_initialize(int argc, VALUE *argv, VALUE vself) { int fd = -1; size_t length; @@ -72,6 +74,15 @@ static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) VALUE vsm_map; simple_mmap_map *sm_map; + VALUE filename, voffset, vlength; + off_t offset, offset_mod; + + rb_scan_args(argc, argv, "12", &filename, &voffset, &vlength); + if (!NIL_P(voffset) && NUM2LL(voffset) < 0) + rb_raise(rb_eRangeError, "offset out of range: %lld", NUM2LL(voffset)); + if (!NIL_P(vlength) && NUM2LL(vlength) < 0) + rb_raise(rb_eRangeError, "length out of range: %lld", NUM2LL(vlength)); + fd = open(RSTRING_PTR(filename), O_RDONLY); if (fd == -1) { rb_raise(rb_eArgError, "Failed to open file %s", RSTRING_PTR(filename)); @@ -83,10 +94,15 @@ static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) rb_raise(rb_eArgError, "Failed to stat file %s", RSTRING_PTR(filename)); close(fd); } - length = st.st_size; + offset = NIL_P(voffset) ? 0 : NUM2SIZET(voffset); + length = NIL_P(vlength) ? st.st_size : NUM2SIZET(vlength); + if (offset + length > st.st_size) length = st.st_size - offset; + offset_mod = offset % sysconf(_SC_PAGESIZE); + offset = offset - offset_mod; + length = length + offset_mod; // do the mmap - base = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); + base = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, offset); if (base == (caddr_t) -1) { rb_raise(rb_eArgError, "Failed to mmap file %s", RSTRING_PTR(filename)); close(fd); @@ -98,6 +114,8 @@ static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename) sm_map->fd = fd; sm_map->map = base; sm_map->len = length; + sm_map->read_base = base + offset_mod; + sm_map->read_len = length - offset_mod; rb_ivar_set(vself, rb_intern("@mmap_data"), vsm_map); return Qnil; @@ -146,14 +164,14 @@ static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE v vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); - if (offset > sm_map->len) return Qnil; + if (offset > sm_map->read_len) return Qnil; // If the range overflows, return part that overlaps - if ((offset + length) > sm_map->len) { - length = sm_map->len - offset; + if ((offset + length) > sm_map->read_len) { + length = sm_map->read_len - offset; } - return rb_str_new(sm_map->map + offset, length); + return rb_str_new(sm_map->read_base + offset, length); } /* @@ -178,7 +196,7 @@ void Init_mapped_file() mod_simple_mmap = rb_define_module("SimpleMmap"); sm_mapped_file = rb_define_class_under(mod_simple_mmap, "MappedFile", rb_cObject); - rb_define_private_method(sm_mapped_file, "initialize", sm_mapped_file_initialize, 1); + rb_define_private_method(sm_mapped_file, "initialize", sm_mapped_file_initialize, -1); rb_define_method(sm_mapped_file, "close", sm_mapped_file_close, 0); rb_define_method(sm_mapped_file, "read_window_data", sm_mapped_file_read_window_data, 2); rb_define_method(sm_mapped_file, "size", sm_mapped_file_size, 0); diff --git a/lib/simple_mmap/file_window.rb b/lib/simple_mmap/file_window.rb index 9176449..1e90694 100644 --- a/lib/simple_mmap/file_window.rb +++ b/lib/simple_mmap/file_window.rb @@ -25,10 +25,10 @@ module SimpleMmap class FileWindow # Create a mmap'ed window for file at +path+ # You are responsible for closing it when you're done using #close - def initialize(path) + def initialize(path, offset = nil, length = nil) @path = path @offset = 0 - @mmap = SimpleMmap::MappedFile.new(@path) + @mmap = SimpleMmap::MappedFile.new(@path, offset, length) end attr_reader :path, :offset From 8eb82e02177d1da6a8a146271402666a7fc009b3 Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Wed, 6 Nov 2013 10:36:20 +0900 Subject: [PATCH 18/21] introduce bundler instead of jeweler --- Gemfile | 9 +++------ Rakefile | 26 +------------------------- lib/simple_mmap.rb | 1 + lib/simple_mmap/version.rb | 3 +++ simple-mmap.gemspec | 26 ++++++++++++++++++++++++++ 5 files changed, 34 insertions(+), 31 deletions(-) create mode 100644 lib/simple_mmap/version.rb create mode 100644 simple-mmap.gemspec diff --git a/Gemfile b/Gemfile index fb0d0f3..8e2ab11 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,4 @@ -# A sample Gemfile -source "https://rubygems.org" +source 'https://rubygems.org' -group :test do - gem 'jeweler' - gem 'rake-compiler' -end +# Specify your gem's dependencies in simple_mmap.gemspec +gemspec diff --git a/Rakefile b/Rakefile index 416d347..a81bac4 100644 --- a/Rakefile +++ b/Rakefile @@ -1,29 +1,5 @@ -#encoding: utf-8 +require "bundler/gem_tasks" -require 'rubygems' -require 'rake' - -begin - require 'jeweler' - Jeweler::Tasks.new do |gem| - gem.name = "simple-mmap" - gem.email = "pcnoordhuis@gmail.com" - gem.homepage = "http://github.com/pietern/simple-mmap" - gem.summary = "A simplistic interface for reading memory mapped files" - gem.authors = ["Johan Sørensen", "Pieter Noordhuis"] - - gem.extensions = FileList["ext/**/extconf.rb"] - gem.files = FileList['Rakefile', 'ext/**/*.{c,rb}', 'lib/**/*.rb', 'test/**/*.rb'] - gem.require_paths = ["ext", "lib"] - - gem.add_development_dependency "rake-compiler", "0.7.0" - end - Jeweler::GemcutterTasks.new -rescue LoadError - puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler" -end - -gem 'rake-compiler', '>= 0.7.0' require 'rake/extensiontask' Rake::ExtensionTask.new('simple_mmap') do |ext| ext.name = "mapped_file" diff --git a/lib/simple_mmap.rb b/lib/simple_mmap.rb index c79c5ff..972ea91 100644 --- a/lib/simple_mmap.rb +++ b/lib/simple_mmap.rb @@ -20,5 +20,6 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. $:.unshift File.dirname(__FILE__) +require "simple_mmap/version" require "simple_mmap/mapped_file" require "simple_mmap/file_window" diff --git a/lib/simple_mmap/version.rb b/lib/simple_mmap/version.rb new file mode 100644 index 0000000..c12e0d0 --- /dev/null +++ b/lib/simple_mmap/version.rb @@ -0,0 +1,3 @@ +module SimpleMmap + VERSION = "1.1.4" +end diff --git a/simple-mmap.gemspec b/simple-mmap.gemspec new file mode 100644 index 0000000..9d067ea --- /dev/null +++ b/simple-mmap.gemspec @@ -0,0 +1,26 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'simple_mmap/version' + +Gem::Specification.new do |spec| + spec.name = "simple-mmap" + spec.version = SimpleMmap::VERSION + spec.authors = ["Johan Sørensen", "Pieter Noordhuis", "TAKADA Daisuke"] + spec.email = ["pcnoordhuis@gmail.com", "shoulderpower@gmail.com"] +# spec.description = %q{TODO: Write a gem description} + spec.description = %q{A simplistic interface for reading memory mapped files} + spec.summary = %q{A simplistic interface for reading memory mapped files} + spec.homepage = "http://github.com/pietern/simple-mmap" + spec.license = "MIT" + + spec.files = `git ls-files`.split($/) + spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } + spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) + spec.require_paths = ["ext", "lib"] + spec.extensions << 'ext/simple_mmap/extconf.rb' + + spec.add_development_dependency "bundler", "~> 1.3" + spec.add_development_dependency "rake" + spec.add_development_dependency 'rake-compiler', '>= 0.7.0' +end From 2c2ec3218ecf6ba27fe040cbe8851dd8a4a61494 Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Thu, 14 Nov 2013 15:14:29 +0900 Subject: [PATCH 19/21] SimpleMmap::MappedFile#size returns SIZET2NUM(sm_map->read_len) --- ext/simple_mmap/mapped_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index 219d472..7707e1c 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -188,7 +188,7 @@ static VALUE sm_mapped_file_size(VALUE vself) vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data")); Data_Get_Struct(vsm_map, simple_mmap_map, sm_map); - return UINT2NUM(sm_map->len); + return SIZET2NUM(sm_map->read_len); } void Init_mapped_file() From a02abb059e14acae8256b85aa49964fb424892ec Mon Sep 17 00:00:00 2001 From: TAKADA Daisuke Date: Thu, 14 Nov 2013 15:19:26 +0900 Subject: [PATCH 20/21] SimpleMmap::FileWindow#[] --- lib/simple_mmap/file_window.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/simple_mmap/file_window.rb b/lib/simple_mmap/file_window.rb index 1e90694..b6caa4b 100644 --- a/lib/simple_mmap/file_window.rb +++ b/lib/simple_mmap/file_window.rb @@ -70,11 +70,11 @@ def [](*index) offset = index length = 0 when Range - offset = index.begin - length = index.end - index.begin - unless index.exclude_end? - length += 1 - end + offset = index.begin < 0 ? index.begin + @mmap.size : index.begin + return nil if offset < 0 or offset > @mmap.size + length = (index.end < 0 ? index.end + @mmap.size : index.end) - offset + length += 1 unless index.exclude_end? + return '' if length <= 0 end @offset = offset + length From e20c2b0623c3ac953e45596525dcede588028c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20S=C3=B8rensen?= Date: Tue, 4 Oct 2016 20:23:49 +0200 Subject: [PATCH 21/21] Update version and copyright years --- ext/simple_mmap/mapped_file.c | 4 ++-- lib/simple_mmap.rb | 2 +- lib/simple_mmap/version.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/simple_mmap/mapped_file.c b/ext/simple_mmap/mapped_file.c index 7707e1c..c917c44 100644 --- a/ext/simple_mmap/mapped_file.c +++ b/ext/simple_mmap/mapped_file.c @@ -48,9 +48,9 @@ // a memory mapped file typedef struct { - int fd; + int fd; caddr_t map; - size_t len; + size_t len; caddr_t read_base; size_t read_len; } simple_mmap_map; diff --git a/lib/simple_mmap.rb b/lib/simple_mmap.rb index 972ea91..cfbde3e 100644 --- a/lib/simple_mmap.rb +++ b/lib/simple_mmap.rb @@ -1,4 +1,4 @@ -# Copyright (c) 2009 Johan Sørensen +# Copyright (c) 2009-2016 Johan Sørensen # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the diff --git a/lib/simple_mmap/version.rb b/lib/simple_mmap/version.rb index c12e0d0..ec55846 100644 --- a/lib/simple_mmap/version.rb +++ b/lib/simple_mmap/version.rb @@ -1,3 +1,3 @@ module SimpleMmap - VERSION = "1.1.4" + VERSION = "1.2" end