diff --git a/lib/twitter_ebooks/bot.rb b/lib/twitter_ebooks/bot.rb index 33b5445..2506f40 100755 --- a/lib/twitter_ebooks/bot.rb +++ b/lib/twitter_ebooks/bot.rb @@ -43,7 +43,7 @@ module Ebooks config.oauth_token_secret = @oauth_token_secret end - Twitter.configure do |config| + @twitter = Twitter::REST::Client.new do |config| config.consumer_key = @consumer_key config.consumer_secret = @consumer_secret config.oauth_token = @oauth_token @@ -52,7 +52,6 @@ module Ebooks needs_stream = [@on_follow, @on_message, @on_mention, @on_timeline].any? {|e| !e.nil?} - @twitter = Twitter::Client.new @stream = TweetStream::Client.new if needs_stream end @@ -163,7 +162,7 @@ module Ebooks log "Tweeting #{args.inspect}" @twitter.update(*args) end - + # could easily just be *args however the separation keeps it clean. def pictweet(txt, pic, *args) log "Tweeting #{txt.inspect} - #{pic} #{args}" diff --git a/lib/twitter_ebooks/model.rb b/lib/twitter_ebooks/model.rb index 0ed3ad5..6d4af13 100644 --- a/lib/twitter_ebooks/model.rb +++ b/lib/twitter_ebooks/model.rb @@ -79,8 +79,8 @@ module Ebooks @sentences = mass_tokenize(text) @mentions = mass_tokenize(mention_text) - #log "Ranking keywords" - #@keywords = NLP.keywords(@sentences) + log "Ranking keywords" + @keywords = NLP.keywords(@sentences) self end diff --git a/spec/data/0xabad1dea.model b/spec/data/0xabad1dea.model new file mode 100644 index 0000000..90a22ea Binary files /dev/null and b/spec/data/0xabad1dea.model differ diff --git a/spec/memprof.rb b/spec/memprof.rb new file mode 100644 index 0000000..cb552ae --- /dev/null +++ b/spec/memprof.rb @@ -0,0 +1,37 @@ +require 'objspace' + +module MemoryUsage + MemoryReport = Struct.new(:total_memsize) + + def self.full_gc + GC.start(full_mark: true) + end + + def self.report(&block) + rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] + + full_gc + GC.disable + + total_memsize = 0 + + generation = nil + ObjectSpace.trace_object_allocations do + generation = GC.count + block.call + end + + ObjectSpace.each_object do |obj| + next unless generation == ObjectSpace.allocation_generation(obj) + memsize = ObjectSpace.memsize_of(obj) + rvalue_size + # compensate for API bug + memsize = rvalue_size if memsize > 100_000_000_000 + total_memsize += memsize + end + + GC.enable + full_gc + + return MemoryReport.new(total_memsize) + end +end diff --git a/spec/model_spec.rb b/spec/model_spec.rb index 24a1b47..8fdc82d 100644 --- a/spec/model_spec.rb +++ b/spec/model_spec.rb @@ -1,14 +1,14 @@ require 'spec_helper' require 'memory_profiler' +def Process.rss; `ps -o rss= -p #{Process.pid}`.chomp.to_i; end + describe Ebooks::Model do it "does not use a ridiculous amount of memory" do - RubyProf.start - Ebooks::Model.consume(path("data/0xabad1dea.json")) - result = RubyProf.stop + report = MemoryUsage.report do + model = Ebooks::Model.consume(path("data/0xabad1dea.json")) + end - require 'pry'; binding.pry - - expect(report.total_retained).to be < 100000 + expect(report.total_memsize).to be < 1000000000 end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f365da0..64dc11c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,5 @@ require 'twitter_ebooks' +require_relative 'memprof' def path(relpath) File.join(File.dirname(__FILE__), relpath) diff --git a/twitter_ebooks.gemspec b/twitter_ebooks.gemspec index baf8efd..33ae7fe 100644 --- a/twitter_ebooks.gemspec +++ b/twitter_ebooks.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |gem| gem.version = Ebooks::VERSION gem.add_development_dependency 'rspec' - gem.add_development_dependency 'ruby-prof' + gem.add_development_dependency 'memory_profiler' gem.add_development_dependency 'pry-byebug'