Changeset 1610

Show
Ignore:
Timestamp:
06/09/08 17:01:08 (2 months ago)
Author:
mbriggs
Message:

Segment migration of url sources

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • hive/trunk/data_webapp/db/migrate/031_create_job_sources.rb

    r1602 r1610  
    7171 
    7272      # Create url objects and associate to job 
    73       urls = klass.find(:all,:conditions => pair) 
    74       urls.each do |url| 
    75         join_url = join_klass.new(klass_symbol => url,:drone_job => job) 
    76         join_url.save! 
     73      offset = 0 
     74      limit = 100000 
     75      urls = klass.find(:all,:conditions => pair,:limit => limit,:offset => offset,:order => 'id ASC') 
     76      while not urls.size > 0 
     77        urls.each do |url| 
     78          join_url = join_klass.new(klass_symbol => url,:drone_job => job) 
     79          join_url.save! 
     80        end 
     81        offset = urls.last.id + 1 
     82        urls = klass.find(:all,:conditions => pair,:limit => limit,:offset => offset,:order => 'id ASC') 
    7783      end 
    7884    end