Land #17154, improve qualys import times

lib/msf/core/db_manager/import/qualys.rb

@@ -5,6 +5,9 @@ module Msf::DBManager::Import::Qualys
   include Msf::DBManager::Import::Qualys::Asset
   include Msf::DBManager::Import::Qualys::Scan
 
+  TCP_QID = 82023
+  UDP_QID = 82004
+
   #
   # Qualys report parsing/handling
   #
@@ -47,4 +50,4 @@ def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, t
       )
     end
   end
-end
+end

lib/msf/core/db_manager/import/qualys/asset.rb

@@ -2,11 +2,11 @@ module Msf::DBManager::Import::Qualys::Asset
   # Takes QID numbers and finds the discovered services in
   # a qualys_asset_xml.
   def find_qualys_asset_ports(i,host,wspace,hobj,task_id)
-    return unless (i == 82023 || i == 82004)
-    proto = i == 82023 ? 'tcp' : 'udp'
-    qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"]
-    qid_result = qid.parent.elements["RESULT[@format='table']"] if qid
-    hports = qid_result.first.to_s if qid_result
+    return unless (i == Msf::DBManager::Import::Qualys::TCP_QID || i == Msf::DBManager::Import::Qualys::UDP_QID)
+    proto = i == Msf::DBManager::Import::Qualys::TCP_QID ? 'tcp' : 'udp'
+    qid = host.xpath("VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']").first
+    qid_result = qid.parent.xpath("RESULT[@format='table']") if qid
+    hports = qid_result.first.text if qid_result
     if hports
       hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match|
         if match[2] == nil or match[2].strip == 'unknown'
@@ -21,15 +21,18 @@ def find_qualys_asset_ports(i,host,wspace,hobj,task_id)
 
   def find_qualys_asset_vuln_refs(doc)
     vuln_refs = {}
-    doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln|
-      next unless vuln.elements['QID'] && vuln.elements['QID'].first
-      qid = vuln.elements['QID'].first.to_s
+    doc.xpath("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS").each do |vuln|
+      qid_el = vuln.xpath('QID')
+      next unless qid_el && qid_el.first
+      qid = qid_el.first.text
       vuln_refs[qid] ||= []
-      vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref|
-        vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9,})/.match(ref.elements['ID'].text.to_s)[1])
+      vuln.xpath('CVE_ID_LIST/CVE_ID').each do |ref|
+        id = ref.xpath("ID").first&.text
+        vuln_refs[qid].push(id) if id
       end
-      vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref|
-        vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s)
+      vuln.xpath('BUGTRAQ_ID_LIST/BUGTRAQ_ID').each do |ref|
+        id = ref.xpath("ID").first&.text
+        vuln_refs[qid].push("BID-#{id}") if id
       end
     end
     return vuln_refs
@@ -38,9 +41,9 @@ def find_qualys_asset_vuln_refs(doc)
   # Pull out vulnerabilities that have at least one matching
   # ref -- many "vulns" are not vulns, just audit information.
   def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block)
-    host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi|
-      next unless vi.elements["QID"]
-      vi.elements.each("QID") do |qid|
+    host.xpath("VULN_INFO_LIST/VULN_INFO").each do |vi|
+      next unless vi.xpath("QID").first
+      vi.xpath("QID").each do |qid|
         next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty?
         handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id)
       end
@@ -54,28 +57,32 @@ def import_qualys_asset_xml(args={}, &block)
     data = args[:data]
     wspace = Msf::Util::DBManager.process_opts_workspace(args, framework).name
     bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
-    doc = rexmlify(data)
+    doc = Nokogiri.XML(data)
     vuln_refs = find_qualys_asset_vuln_refs(doc)
 
     # 2nd pass, actually grab the hosts.
-    doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host|
+    doc.xpath("/ASSET_DATA_REPORT/HOST_LIST/HOST").each do |host|
       hobj = nil
-      addr = host.elements["IP"].text if host.elements["IP"]
+      addr_el = host.xpath("IP").first
+      addr = addr_el.text if addr_el
       next unless validate_ips(addr)
       if bl.include? addr
         next
       else
         yield(:address,addr) if block
       end
+      netbios_el = host.xpath("NETBIOS").first
+      dns_el = host.xpath("DNS").first
       hname = ( # Prefer NetBIOS over DNS
-        (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) ||
-         (host.elements["DNS"].text if host.elements["DNS"]) ||
+        (netbios_el.text if netbios_el) ||
+         (dns_el.text if dns_el) ||
          "" )
       hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task])
       report_import_note(wspace,hobj)
 
-      if host.elements["OPERATING_SYSTEM"]
-        hos = host.elements["OPERATING_SYSTEM"].text
+      os_el = host.xpath("OPERATING_SYSTEM").first
+      if os_el
+        hos = os_el.text
         report_note(
           :workspace => wspace,
           :task => args[:task],

lib/msf/core/db_manager/import/qualys/scan.rb

@@ -5,22 +5,23 @@ def import_qualys_scan_xml(args={}, &block)
     bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
 
 
-    doc = rexmlify(data)
-    doc.elements.each('/SCAN/IP') do |host|
+    doc = Nokogiri.XML(data)
+    doc.xpath('/SCAN/IP').each do |host|
       hobj = nil
-      addr  = host.attributes['value']
+      addr  = host.attr('value')
       if bl.include? addr
         next
       else
         yield(:address,addr) if block
       end
-      hname = host.attributes['name'] || ''
+      hname = host.attr('name') || ''
 
       hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task])
       report_import_note(wspace,hobj)
 
-      if host.elements["OS"]
-        hos = host.elements["OS"].text
+      os_el = host.xpath("OS").first
+      if os_el
+        hos = os_el.text
         report_note(
           :workspace => wspace,
           :task => args[:task],
@@ -33,7 +34,7 @@ def import_qualys_scan_xml(args={}, &block)
       end
 
       # Open TCP Services List (Qualys ID 82023)
-      services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"]
+      services_tcp = host.xpath("SERVICES/CAT/SERVICE[@number='#{Msf::DBManager::Import::Qualys::TCP_QID}']/RESULT").first
       if services_tcp
         services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match|
           if match[2] == nil or match[2].strip == 'unknown'
@@ -45,7 +46,7 @@ def import_qualys_scan_xml(args={}, &block)
         end
       end
       # Open UDP Services List (Qualys ID 82004)
-      services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"]
+      services_udp = host.xpath("SERVICES/CAT/SERVICE[@number='#{Msf::DBManager::Import::Qualys::UDP_QID}']/RESULT").first
       if services_udp
         services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match|
           if match[2] == nil or match[2].strip == 'unknown'
@@ -58,22 +59,25 @@ def import_qualys_scan_xml(args={}, &block)
       end
 
       # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities
-      host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat|
-        port = cat.attributes['port']
-        protocol = cat.attributes['protocol']
-        cat.elements.each('VULN | PRACTICE') do |vuln|
+      host.xpath('VULNS/CAT | PRACTICES/CAT').each do |cat|
+        port = cat.attr('port')
+        protocol = cat.attr('protocol')
+        cat.xpath('VULN | PRACTICE').each do |vuln|
           refs = []
-          qid = vuln.attributes['number']
-          severity = vuln.attributes['severity']
-          title = vuln.elements['TITLE'].text.to_s
-          vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref|
-            refs.push(ref.elements['ID'].text.to_s)
+          qid = vuln.attr('number')
+          severity = vuln.attr('severity')
+          title = vuln.xpath('TITLE').first&.text
+          vuln.xpath('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE').each do |ref|
+            id = ref.xpath('ID').first&.text
+            refs.push(id) if id
           end
-          vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref|
-            refs.push('CVE-' + /C..-([0-9\-]{9,})/.match(ref.elements['ID'].text.to_s)[1])
+          vuln.xpath('CVE_ID_LIST/CVE_ID').each do |ref|
+            id = ref.xpath("ID").first&.text
+            refs.push(id) if id
           end
-          vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref|
-            refs.push('BID-' + ref.elements['ID'].text.to_s)
+          vuln.xpath('BUGTRAQ_ID_LIST/BUGTRAQ_ID').each do |ref|
+            id = ref.xpath("ID").first&.text
+            refs.push("BID-#{id}") if id
           end
 
           handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task])