More memory profiling

lib/twitter_ebooks/bot.rb

@@ -43,7 +43,7 @@ module Ebooks
         config.oauth_token_secret = @oauth_token_secret
       end
 
-      Twitter.configure do |config|
+      @twitter = Twitter::REST::Client.new do |config|
         config.consumer_key = @consumer_key
         config.consumer_secret = @consumer_secret
         config.oauth_token = @oauth_token
@@ -52,7 +52,6 @@ module Ebooks
 
       needs_stream = [@on_follow, @on_message, @on_mention, @on_timeline].any? {|e| !e.nil?}
 
-      @twitter = Twitter::Client.new
       @stream = TweetStream::Client.new if needs_stream
     end
 
@@ -163,7 +162,7 @@ module Ebooks
       log "Tweeting #{args.inspect}"
       @twitter.update(*args)
     end
-    
+
     # could easily just be *args however the separation keeps it clean.
     def pictweet(txt, pic, *args)
       log "Tweeting #{txt.inspect} - #{pic} #{args}"

lib/twitter_ebooks/model.rb

@@ -79,8 +79,8 @@ module Ebooks
       @sentences = mass_tokenize(text)
       @mentions = mass_tokenize(mention_text)
 
-      #log "Ranking keywords"
-      #@keywords = NLP.keywords(@sentences)
+      log "Ranking keywords"
+      @keywords = NLP.keywords(@sentences)
 
       self
     end

spec/memprof.rb

@@ -0,0 +1,37 @@
+require 'objspace'
+
+module MemoryUsage
+  MemoryReport = Struct.new(:total_memsize)
+
+  def self.full_gc
+    GC.start(full_mark: true)
+  end
+
+  def self.report(&block)
+    rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
+
+    full_gc
+    GC.disable
+
+    total_memsize = 0
+
+    generation = nil
+    ObjectSpace.trace_object_allocations do
+      generation = GC.count
+      block.call
+    end
+
+    ObjectSpace.each_object do |obj|
+      next unless generation == ObjectSpace.allocation_generation(obj)
+      memsize = ObjectSpace.memsize_of(obj) + rvalue_size
+      # compensate for API bug
+      memsize = rvalue_size if memsize > 100_000_000_000
+      total_memsize += memsize
+    end
+
+    GC.enable
+    full_gc
+
+    return MemoryReport.new(total_memsize)
+  end
+end

spec/model_spec.rb

@@ -1,14 +1,14 @@
 require 'spec_helper'
 require 'memory_profiler'
 
+def Process.rss; `ps -o rss= -p #{Process.pid}`.chomp.to_i; end
+
 describe Ebooks::Model do
   it "does not use a ridiculous amount of memory" do
-    RubyProf.start
-    Ebooks::Model.consume(path("data/0xabad1dea.json"))
-    result = RubyProf.stop
+    report = MemoryUsage.report do
+      model = Ebooks::Model.consume(path("data/0xabad1dea.json"))
+    end
 
-    require 'pry'; binding.pry
-
-    expect(report.total_retained).to be < 100000
+    expect(report.total_memsize).to be < 1000000000
   end
 end

spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'twitter_ebooks'
+require_relative 'memprof'
 
 def path(relpath)
   File.join(File.dirname(__FILE__), relpath)

twitter_ebooks.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.version       = Ebooks::VERSION
 
   gem.add_development_dependency 'rspec'
-  gem.add_development_dependency 'ruby-prof'
+  gem.add_development_dependency 'memory_profiler'
   gem.add_development_dependency 'pry-byebug'