the beginnings of a tech and civic life data parser and importer

.gitignore

@@ -9,6 +9,9 @@
 
 db/schema.rb
 
+# ignore the tech and civic life dataset
+/lib/vip.xml
+
 # Ignore all logfiles and tempfiles.
 /log/*
 /tmp/*

Gemfile

@@ -16,6 +16,9 @@ gem 'sass-rails', '~> 5.0'
 gem 'uglifier', '>= 1.3.0'
 gem 'devise'
 
+#for data import
+gem 'nokogiri'
+
 
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 gem 'jbuilder', '~> 2.5'

Gemfile.lock

@@ -207,6 +207,7 @@ DEPENDENCIES
   jquery-rails
   letter_opener
   listen (>= 3.0.5, < 3.2)
+  nokogiri
   pg (~> 0.18)
   puma (~> 3.7)
   rails (~> 5.1.4)

app/models/techandciviclife.rb

@@ -0,0 +1,5 @@
+module Techandciviclife
+  def self.table_name_prefix
+    'techandciviclife_'
+  end
+end

app/models/techandciviclife/party.rb

@@ -0,0 +1,2 @@
+class Techandciviclife::Party < ApplicationRecord
+end

app/models/techandciviclife/person.rb

@@ -0,0 +1,2 @@
+class Techandciviclife::Person < ApplicationRecord
+end

db/migrate/20180214002402_create_techandciviclife_parties.rb

@@ -0,0 +1,8 @@
+class CreateTechandciviclifeParties < ActiveRecord::Migration[5.1]
+  def change
+    create_table :techandciviclife_parties do |t|
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20180214002416_create_techandciviclife_people.rb

@@ -0,0 +1,8 @@
+class CreateTechandciviclifePeople < ActiveRecord::Migration[5.1]
+  def change
+    create_table :techandciviclife_people do |t|
+
+      t.timestamps
+    end
+  end
+end

lib/techandciviclife_parser.rb

@@ -0,0 +1,52 @@
+module Techandciviclife
+  class Parser
+
+    TOP_LEVEL_NODES = %w(
+    Party
+    )
+=begin
+    Person
+    Candidate
+    BallotMeasureContest
+    ElectoralDistrict
+    Office
+    CandidateSelection
+    CandidateContest
+    Party
+    BallotMeasureSelection
+    Selection
+    RetentionContest
+  )
+=end
+
+    def reader
+      Nokogiri::XML::Reader(File.open(Rails.root + 'lib/vip.xml'))
+    end
+
+    def run
+      t = Time.now
+      unknown_node_names = Set.new
+      reader.each do |node|
+        if node.node_type != Nokogiri::XML::Reader::TYPE_END_ELEMENT
+          if node.name.in?(TOP_LEVEL_NODES)
+            Techandciviclife::Parsers.const_get(node.name).new.parse(node)
+          else
+            unknown_node_names.add(node.name)
+          end
+        end
+      end
+      puts "skipped unknown nodes: #{unknown_node_names.inspect}"
+      puts "finished in: #{Time.now - t} seconds"
+    end
+  end
+
+  module Parsers
+    class Party
+      def parse(node)
+        id = node.attribute('id')
+        name = Nokogiri::XML(node.outer_xml).xpath("//Party/Name/Text[@language='en']").text
+        puts [id, name]
+      end
+    end
+  end
+end

test/fixtures/techandciviclife/parties.yml

@@ -0,0 +1,11 @@
+# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+# This model initially had no columns defined. If you add columns to the
+# model remove the '{}' from the fixture names and add the columns immediately
+# below each fixture, per the syntax in the comments below
+#
+one: {}
+# column: value
+#
+two: {}
+# column: value

test/fixtures/techandciviclife/people.yml

@@ -0,0 +1,11 @@
+# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+# This model initially had no columns defined. If you add columns to the
+# model remove the '{}' from the fixture names and add the columns immediately
+# below each fixture, per the syntax in the comments below
+#
+one: {}
+# column: value
+#
+two: {}
+# column: value

test/models/techandciviclife/party_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class Techandciviclife::PartyTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end

test/models/techandciviclife/person_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class Techandciviclife::PersonTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end