Ruby on rails csv解析完成后运行rake任务

我有一个看起来有点像这样的rake任务

require 'open-uri'
require 'csv'
namespace :tm do
  task reload: :environment do
    gzipped = open('csv link')
    csv_text = Zlib::GzipReader.new(gzipped).read
    csv = CSV.parse(csv_text, headers: true)

    csv.each do |row|
      if row[4] == 'logo url'
      else
        tmdate = Date.parse(row[10]).strftime('%Y-%m-%d')
        viatmdate = Date.parse(row[10]).strftime('%d/%m/%Y')
        swtmdate = row[10]
        tmlocation = row[6].split('at ')[1]
        place = row[11].split('|')[1]
        place1 = row[11].split('|')[2]
        place2 = row[11].split('|')[3]
        location = '' + place + ', ' + place1 + ', ' + place2 + ''
        tmtime = row[9]
        text = row[7].gsub('text', '')
        if text.include? '�&#xFFFD'
          eventname = text.gsub('�&#xFFFD', 'e')
        else
          eventname = text.gsub(/[ªÀÈÌÒÙàèìòùÁÉÍÓÚáéíóúÂÊÎÔÛâêîôûÃÑÕãñõÄËÏÖÜŸäëïöüÿ]/, '')
        end

        if text.include? '�&#xFFFD'
          tmname = text.gsub('�&#xFFFD', 'e')
        else
          tmname = text.gsub(/[ªÀÈÌÒÙàèìòùÁÉÍÓÚáéíóúÂÊÎÔÛâêîôûÃÑÕãñõÄËÏÖÜŸäëïöüÿ]/, '')
        end

        if text.include? ' -'
          tmnamesplit = text.split(' -')[0]
        end

        if tmname[/[^0-9]/].present?
          tmnamenn = tmname.gsub(/[^0-9]/i, '')
        end

        text2urldb = text2.where('eventtitle ILIKE ? AND eventdoortime = ? ', "%#{tmname.gsub(/[\-\:\ ]/, '%')}%", tmdate.to_s).first
        text3urldb = text3.where('product_name ILIKE ? AND delivery_time = ? AND valid_from = ?', "%#{tmname}%", tmtime.to_s, tmdate.to_s).first

        text1urldb = text1.where('product_name ILIKE ? AND specifications = ? AND promotional_text = ?', "%#{tmname}%", viatmdate.to_s, "%#{place}%").first

        if tmnamesplit.present?
          if text1urldb.blank?
            text1urldb = text1.where('product_name ILIKE ? AND specifications = ?', "%#{tmnamesplit}%", viatmdate.to_s).first
          end
          if text3urldb.blank?
            text3urldb = text3.where('product_name ILIKE ? AND delivery_time = ? AND valid_from = ?', "%#{tmnamesplit}%", tmtime.to_s, tmdate.to_s).first
          end
        end

        if text1urldb.blank?
          text1urldb = text1.where('product_name ILIKE ? AND specifications = ? AND promotional_text = ?', "%#{tmname}%", viatmdate.to_s, "%#{location}%").first
          if text1urldb.blank?
            text1urldb = text1.where('product_name ILIKE ? AND specifications = ?', "%#{tmname}%", viatmdate.to_s).first
          end
          if text1urldb.blank?
            text1urldb = text1.where('product_name ILIKE ? AND specifications = ? AND promotional_text = ?', "%#{tmname}%", viatmdate.to_s, "%#{tmlocation}%").first
          end
        end

        if text1urldb.present?
          vurl = text1urldb.merchant_deep_link
          txt = vurl
          re1 = '.*?' # Non-greedy match on filler
          re2 = '(?:[a-z][a-z]+)' # Uninteresting: word
          re3 = '.*?' # Non-greedy match on filler
          re4 = '(?:[a-z][a-z]+)' # Uninteresting: word
          re5 = '.*?' # Non-greedy match on filler
          re6 = '(?:[a-z][a-z]+)' # Uninteresting: word
          re7 = '.*?' # Non-greedy match on filler
          re8 = '(?:[a-z][a-z]+)' # Uninteresting: word
          re9 = '.*?' # Non-greedy match on filler
          re10 = '(?:[a-z][a-z]+)'    # Uninteresting: word
          re11 = '.*?'    # Non-greedy match on filler
          re12 = '((?:[a-z][a-z]+))'  # Word 1
          re = (re1 + re2 + re3 + re4 + re5 + re6 + re7 + re8 + re9 + re10 + re11 + re12)
          m = Regexp.new(re, Regexp::IGNORECASE)
          if m.match(txt)
            word1 = m.match(txt)[1]
          end
        end


        gmiurl = text3urldb.merchant_deep_link if text3urldb.present?

        gigurl = text2urldb.eventurl if text2urldb.present?

        api = HTTParty.get(URI.encode('text url' + tmname + '&when_from=' + swtmdate)).parsed_response
        api1 = api['Paging']
        api2 = api1['TotalResultCount']

        if api1.blank?
          newapi = HTTParty.get(URI.encode('texturl' + tmnamenn + '&when_from=' + swtmdate)).parsed_response
          paging = newapi['Paging']
          api2 = paging['TotalResultCount']
          if newapi.blank?
            apisplit = HTTParty.get(URI.encode('texturl' + tmnamesplit + '&when_from=' + swtmdate)).parsed_response
            pagingsplit = apisplit['Paging']
            api2 = pagingsplit['TotalResultCount']
          end
        end
        text1 = vurl
        text3 = gmiurl
        text2 = gigurl

        if api2 == 0
        else
          swurl = api['Events'].first['SwURL']
        end
        event = Event.find_by(time: row[9], date: row[10], eventname: eventname, eventvenuename: location)

        if event.present?
          event.update(event_type: word1, text: row[8], eventimage: row[4], textlink: swurl, text1link: text1, text3url: text3, text2url: text2)
        else
          Event.create(time: row[9], date: row[10], event_type: word1, text: row[8], eventimage: row[4], eventname: eventname, eventvenuename: location, textlink: swurl, text1link: text1, text3url: text3, text2url: text2)
        end
      end
    end
  end
end

现在这需要很长的时间才能运行（大约2-3个小时），所以我想知道拆分它是否会加快一点速度

如果我要把文本12和3从这里拿出来，那么它只是先把它放到数据库中。然后在其完全填充后运行另一个rake任务？希望这能加快速度？所以，如果我有一个名为tm:reload的rake任务，然后在另一个名为BA:reload的文件中有另一个任务，我会怎么做呢

我已经浏览了一些列并为其编制了索引，但这并没有给它带来多大的改进。

我会做以下几点：

• 有一个Rake任务来读取CSV文件并为每一行创建后台作业
• 使用Sidekiq作为后端，这听起来像是可以轻松并行化的东西
• 让此作业通过HTTP执行解析和查找，并在数据库中插入/更新
• 将所有这些REGEXP和模式移动到常量中，这样就不会为每一行实例化它们
• 可以通过创建一个表示CSV行的对象并为所有这些值提供访问器，将代码清理成更小的块/方法

假设：文件中没有重复记录，文件可以按任何顺序处理

我不认为DB是瓶颈。更有可能是您发出的HTTP请求。在你摆弄添加索引之前，我会做一些基准测试，看看时间花在哪里

更新： 类似这样的内容（我没有运行代码）

Rake任务： 作业后端 ActiveJob是Rails（后台）作业API。您可以配置后端，我喜欢Sidekiq，但其他的也可以。详情见:

• （sidekiq需要）

工作执行

---

听起来像是一个把它分成几个部分的计划。为每行执行数据库调用不是很快-最好尝试执行批更新和选择。批更新和选择？你介意告诉我最好的方法吗？嗨，我不知道该怎么做，因为我还是个新手！我怎样才能最好地做到这一点？我已经添加了一些提示让您开始。

...
csv.each do |row|
  ImportEventJob.perform_later row
end

class ImportEventJob < ActiveJob::Base

  def perform(row)
    event_csv = EventCsv.new(row)

    event = Event.find_by(time: event_csv.time, date: event_csv.date, ...)
    if event
      ...
    else
      ...
    end
  end
end

class EventCsv
  RE_WITH_DIACRITICS = /[ªÀÈÌÒÙàèìòùÁÉÍÓÚáéíóúÂÊÎÔÛâêîôûÃÑÕãñõÄËÏÖÜŸäëïöüÿ]/
  def initialize(row)
    @row = row
   end

  def time
    @row[9]
  end
  ...

  def name
    text = @row[7].gsub('text', '')
    if text.include? '�&#xFFFD'
      text.gsub('�&#xFFFD', 'e')
    else
      text.gsub(RE_WITH_DIACRITICS, '')
    end
  end
end