From 6ef32dd0d5285d216065a4c8119acbfa4468e554 Mon Sep 17 00:00:00 2001 From: Doug Cole Date: Mon, 19 Mar 2018 12:16:09 -0700 Subject: [PATCH] finish tech and civic life importing, create models for live site data --- app/models/candidate.rb | 2 + app/models/techandciviclife/candidate.rb | 3 + .../techandciviclife/candidate_contest.rb | 2 + .../techandciviclife/candidate_selection.rb | 2 + .../techandciviclife/electoral_district.rb | 3 + app/models/techandciviclife/office.rb | 2 + app/models/techandciviclife/party.rb | 3 + db/migrate/20180211182437_create_contests.rb | 2 +- ...4002402_create_techandciviclife_parties.rb | 6 +- ...14002416_create_techandciviclife_people.rb | 4 +- ...5727_create_techandciviclife_candidates.rb | 12 ++ ...ate_techandciviclife_candidate_contests.rb | 13 ++ ...e_techandciviclife_candidate_selections.rb | 11 ++ ...te_techandciviclife_electoral_districts.rb | 13 ++ ...6201754_create_techandciviclife_offices.rb | 14 ++ .../20180316222346_create_candidates.rb | 12 ++ .../20180316223038_add_contest_columns.rb | 9 ++ lib/techandciviclife_parser.rb | 125 +++++++++++++++--- test/fixtures/candidates.yml | 11 ++ .../techandciviclife/candidate_contests.yml | 11 ++ .../techandciviclife/candidate_selections.yml | 11 ++ test/fixtures/techandciviclife/candidates.yml | 11 ++ .../techandciviclife/electoral_districts.yml | 11 ++ test/fixtures/techandciviclife/offices.yml | 11 ++ test/models/candidate_test.rb | 7 + .../candidate_contest_test.rb | 7 + .../candidate_selection_test.rb | 7 + .../models/techandciviclife/candidate_test.rb | 7 + .../electoral_district_test.rb | 7 + test/models/techandciviclife/office_test.rb | 7 + 30 files changed, 327 insertions(+), 19 deletions(-) create mode 100644 app/models/candidate.rb create mode 100644 app/models/techandciviclife/candidate.rb create mode 100644 app/models/techandciviclife/candidate_contest.rb create mode 100644 app/models/techandciviclife/candidate_selection.rb create mode 100644 app/models/techandciviclife/electoral_district.rb create mode 100644 app/models/techandciviclife/office.rb create mode 100644 db/migrate/20180316005727_create_techandciviclife_candidates.rb create mode 100644 db/migrate/20180316010024_create_techandciviclife_candidate_contests.rb create mode 100644 db/migrate/20180316201304_create_techandciviclife_candidate_selections.rb create mode 100644 db/migrate/20180316201611_create_techandciviclife_electoral_districts.rb create mode 100644 db/migrate/20180316201754_create_techandciviclife_offices.rb create mode 100644 db/migrate/20180316222346_create_candidates.rb create mode 100644 db/migrate/20180316223038_add_contest_columns.rb create mode 100644 test/fixtures/candidates.yml create mode 100644 test/fixtures/techandciviclife/candidate_contests.yml create mode 100644 test/fixtures/techandciviclife/candidate_selections.yml create mode 100644 test/fixtures/techandciviclife/candidates.yml create mode 100644 test/fixtures/techandciviclife/electoral_districts.yml create mode 100644 test/fixtures/techandciviclife/offices.yml create mode 100644 test/models/candidate_test.rb create mode 100644 test/models/techandciviclife/candidate_contest_test.rb create mode 100644 test/models/techandciviclife/candidate_selection_test.rb create mode 100644 test/models/techandciviclife/candidate_test.rb create mode 100644 test/models/techandciviclife/electoral_district_test.rb create mode 100644 test/models/techandciviclife/office_test.rb diff --git a/app/models/candidate.rb b/app/models/candidate.rb new file mode 100644 index 0000000..02653d7 --- /dev/null +++ b/app/models/candidate.rb @@ -0,0 +1,2 @@ +class Candidate < ApplicationRecord +end diff --git a/app/models/techandciviclife/candidate.rb b/app/models/techandciviclife/candidate.rb new file mode 100644 index 0000000..2c0df85 --- /dev/null +++ b/app/models/techandciviclife/candidate.rb @@ -0,0 +1,3 @@ +class Techandciviclife::Candidate < ApplicationRecord + belongs_to :person, :primary_key => :internal_id +end diff --git a/app/models/techandciviclife/candidate_contest.rb b/app/models/techandciviclife/candidate_contest.rb new file mode 100644 index 0000000..c5a6971 --- /dev/null +++ b/app/models/techandciviclife/candidate_contest.rb @@ -0,0 +1,2 @@ +class Techandciviclife::CandidateContest < ApplicationRecord +end diff --git a/app/models/techandciviclife/candidate_selection.rb b/app/models/techandciviclife/candidate_selection.rb new file mode 100644 index 0000000..0af5824 --- /dev/null +++ b/app/models/techandciviclife/candidate_selection.rb @@ -0,0 +1,2 @@ +class Techandciviclife::CandidateSelection < ApplicationRecord +end diff --git a/app/models/techandciviclife/electoral_district.rb b/app/models/techandciviclife/electoral_district.rb new file mode 100644 index 0000000..7e99d7e --- /dev/null +++ b/app/models/techandciviclife/electoral_district.rb @@ -0,0 +1,3 @@ +class Techandciviclife::ElectoralDistrict < ApplicationRecord + self.inheritance_column = 'unused' +end diff --git a/app/models/techandciviclife/office.rb b/app/models/techandciviclife/office.rb new file mode 100644 index 0000000..9790aef --- /dev/null +++ b/app/models/techandciviclife/office.rb @@ -0,0 +1,2 @@ +class Techandciviclife::Office < ApplicationRecord +end diff --git a/app/models/techandciviclife/party.rb b/app/models/techandciviclife/party.rb index 4ef6c07..55c8863 100644 --- a/app/models/techandciviclife/party.rb +++ b/app/models/techandciviclife/party.rb @@ -1,2 +1,5 @@ +#workaround for https://github.com/rails/rails/issues/21476 +require 'techandciviclife' + class Techandciviclife::Party < ApplicationRecord end diff --git a/db/migrate/20180211182437_create_contests.rb b/db/migrate/20180211182437_create_contests.rb index ca985ed..65f40a9 100644 --- a/db/migrate/20180211182437_create_contests.rb +++ b/db/migrate/20180211182437_create_contests.rb @@ -1,7 +1,7 @@ class CreateContests < ActiveRecord::Migration[5.1] def change create_table :contests do |t| - t.string "name" + t.string :name t.timestamps end end diff --git a/db/migrate/20180214002402_create_techandciviclife_parties.rb b/db/migrate/20180214002402_create_techandciviclife_parties.rb index b41b75e..6b098de 100644 --- a/db/migrate/20180214002402_create_techandciviclife_parties.rb +++ b/db/migrate/20180214002402_create_techandciviclife_parties.rb @@ -1,8 +1,12 @@ class CreateTechandciviclifeParties < ActiveRecord::Migration[5.1] def change create_table :techandciviclife_parties do |t| - + t.text "internal_id" + t.text "name" t.timestamps end + + add_index(:techandciviclife_parties, :internal_id, unique: true) + add_index(:techandciviclife_parties, :name, unique: true) end end diff --git a/db/migrate/20180214002416_create_techandciviclife_people.rb b/db/migrate/20180214002416_create_techandciviclife_people.rb index 230affe..196af55 100644 --- a/db/migrate/20180214002416_create_techandciviclife_people.rb +++ b/db/migrate/20180214002416_create_techandciviclife_people.rb @@ -1,8 +1,10 @@ class CreateTechandciviclifePeople < ActiveRecord::Migration[5.1] def change create_table :techandciviclife_people do |t| - + t.text :internal_id + t.text :name t.timestamps end + add_index(:techandciviclife_people, :internal_id, unique: true) end end diff --git a/db/migrate/20180316005727_create_techandciviclife_candidates.rb b/db/migrate/20180316005727_create_techandciviclife_candidates.rb new file mode 100644 index 0000000..5fa592d --- /dev/null +++ b/db/migrate/20180316005727_create_techandciviclife_candidates.rb @@ -0,0 +1,12 @@ +class CreateTechandciviclifeCandidates < ActiveRecord::Migration[5.1] + def change + create_table :techandciviclife_candidates do |t| + t.text :internal_id, null: false + t.text :person_id, null: false + t.text :party_id + t.text :name + t.timestamps + end + add_index(:techandciviclife_candidates, :internal_id, unique: true) + end +end diff --git a/db/migrate/20180316010024_create_techandciviclife_candidate_contests.rb b/db/migrate/20180316010024_create_techandciviclife_candidate_contests.rb new file mode 100644 index 0000000..6bbbb90 --- /dev/null +++ b/db/migrate/20180316010024_create_techandciviclife_candidate_contests.rb @@ -0,0 +1,13 @@ +class CreateTechandciviclifeCandidateContests < ActiveRecord::Migration[5.1] + def change + create_table :techandciviclife_candidate_contests do |t| + t.text :internal_id, null: false + t.text :name, null: false + t.text :candidate_selections, array: true, null: false, default: [] + t.text :electoral_district_id, null: false + t.text :office_id, null: false + t.timestamps + end + add_index(:techandciviclife_candidate_contests, :internal_id, unique: true) + end +end diff --git a/db/migrate/20180316201304_create_techandciviclife_candidate_selections.rb b/db/migrate/20180316201304_create_techandciviclife_candidate_selections.rb new file mode 100644 index 0000000..300c5c1 --- /dev/null +++ b/db/migrate/20180316201304_create_techandciviclife_candidate_selections.rb @@ -0,0 +1,11 @@ +class CreateTechandciviclifeCandidateSelections < ActiveRecord::Migration[5.1] + def change + create_table :techandciviclife_candidate_selections do |t| + t.text :internal_id, null: false + t.text :candidate_ids, array: true, null: false, default: [] + t.timestamps + end + + add_index(:techandciviclife_candidate_selections, :internal_id, unique: true) + end +end diff --git a/db/migrate/20180316201611_create_techandciviclife_electoral_districts.rb b/db/migrate/20180316201611_create_techandciviclife_electoral_districts.rb new file mode 100644 index 0000000..e4a794a --- /dev/null +++ b/db/migrate/20180316201611_create_techandciviclife_electoral_districts.rb @@ -0,0 +1,13 @@ +class CreateTechandciviclifeElectoralDistricts < ActiveRecord::Migration[5.1] + def change + create_table :techandciviclife_electoral_districts do |t| + + t.text :internal_id, null: false + t.text :name, null: false + t.text :type, null: false + t.timestamps + end + + add_index(:techandciviclife_electoral_districts, :internal_id, unique: true) + end +end diff --git a/db/migrate/20180316201754_create_techandciviclife_offices.rb b/db/migrate/20180316201754_create_techandciviclife_offices.rb new file mode 100644 index 0000000..37b5524 --- /dev/null +++ b/db/migrate/20180316201754_create_techandciviclife_offices.rb @@ -0,0 +1,14 @@ +class CreateTechandciviclifeOffices < ActiveRecord::Migration[5.1] + def change + create_table :techandciviclife_offices do |t| + + t.text :internal_id, null: false + t.text :name, null: false + t.text :description, null: false + t.text :electoral_district_id, null: false + t.timestamps + end + + add_index(:techandciviclife_offices, :internal_id, unique: true) + end +end diff --git a/db/migrate/20180316222346_create_candidates.rb b/db/migrate/20180316222346_create_candidates.rb new file mode 100644 index 0000000..ad8a591 --- /dev/null +++ b/db/migrate/20180316222346_create_candidates.rb @@ -0,0 +1,12 @@ +class CreateCandidates < ActiveRecord::Migration[5.1] + def change + create_table :candidates do |t| + t.text :primary_name + t.text :primary_party + t.text :secondary_name + t.text :secondary_party + t.text :candidate_selection_id + t.timestamps + end + end +end diff --git a/db/migrate/20180316223038_add_contest_columns.rb b/db/migrate/20180316223038_add_contest_columns.rb new file mode 100644 index 0000000..d9df203 --- /dev/null +++ b/db/migrate/20180316223038_add_contest_columns.rb @@ -0,0 +1,9 @@ +class AddContestColumns < ActiveRecord::Migration[5.1] + def change + add_column :contests, :office_name, :text + add_column :contests, :office_description, :text + add_column :contests, :candidate_contest_id, :text + add_column :contests, :electoral_district_name, :text + add_column :contests, :electoral_district_type, :text + end +end diff --git a/lib/techandciviclife_parser.rb b/lib/techandciviclife_parser.rb index 56b8a52..ad1f9e8 100644 --- a/lib/techandciviclife_parser.rb +++ b/lib/techandciviclife_parser.rb @@ -2,20 +2,20 @@ module Techandciviclife class Parser TOP_LEVEL_NODES = %w( - Party + Office + Person + Party + Candidate + CandidateContest + CandidateSelection + ElectoralDistrict ) =begin - Person - Candidate - BallotMeasureContest - ElectoralDistrict + BallotMeasureContest - nope Office - CandidateSelection - CandidateContest - Party - BallotMeasureSelection - Selection - RetentionContest + BallotMeasureSelection - nope + Selection - what? + RetentionContest - nope ) =end @@ -25,11 +25,12 @@ def reader def run t = Time.now + clean! unknown_node_names = Set.new reader.each do |node| if node.node_type != Nokogiri::XML::Reader::TYPE_END_ELEMENT if node.name.in?(TOP_LEVEL_NODES) - Techandciviclife::Parsers.const_get(node.name).new.parse(node) + Techandciviclife::Parsers.const_get(node.name).new(node).parse else unknown_node_names.add(node.name) end @@ -38,14 +39,106 @@ def run puts "skipped unknown nodes: #{unknown_node_names.inspect}" puts "finished in: #{Time.now - t} seconds" end + + def clean! + ::Techandciviclife::Party.delete_all + ::Techandciviclife::Person.delete_all + ::Techandciviclife::Candidate.delete_all + ::Techandciviclife::CandidateContest.delete_all + ::Techandciviclife::CandidateSelection.delete_all + ::Techandciviclife::ElectoralDistrict.delete_all + ::Techandciviclife::Office.delete_all + end end module Parsers - class Party - def parse(node) - id = node.attribute('id') - name = Nokogiri::XML(node.outer_xml).xpath("//Party/Name/Text[@language='en']").text + + class Base + attr_reader :node, :xml, :id + + def initialize(node) + @node = node + @xml = Nokogiri::XML(node.outer_xml) + @id = node.attribute('id') + end + end + + class Party < Base + def parse + name = xml.xpath("//Party/Name/Text[@language='en']").text + ::Techandciviclife::Party.create!(name: name, internal_id: id) + end + end + + #Maybe redundant... can we just use the Candidate name and ignore this? + class Person < Base + def parse + name = xml.xpath("//Person/FullName/Text[@language='en']").text puts [id, name] + ::Techandciviclife::Person.create!(name: name, internal_id: id) + end + end + + class Candidate < Base + def parse + person_id = xml.xpath("//Candidate/PersonId").text + party_id = xml.xpath("//Candidate/PartyId").text + name = xml.xpath("//Candidate/BallotName/Text[@language='en']").text + puts [id, person_id, party_id, name] + ::Techandciviclife::Candidate.create!(name: name, internal_id: id, party_id: party_id, person_id: person_id) + end + end + + class CandidateContest < Base + def parse + name = xml.xpath("//CandidateContest/Name").text + candidate_selections = xml.xpath("//CandidateContest/BallotSelectionIds"). + text.to_s.split(' ') + electoral_district_id = xml.xpath("//CandidateContest/ElectoralDistrictId").text + office_id = xml.xpath("//CandidateContest/OfficeIds").text + ::Techandciviclife::CandidateContest.create!( + name: name, + internal_id: id, + candidate_selections: candidate_selections, + electoral_district_id: electoral_district_id, + office_id: office_id + ) + end + end + + class CandidateSelection < Base + def parse + candidate_ids = xml.xpath("//CandidateSelection/CandidateIds").text.to_s.split(' ') + ::Techandciviclife::CandidateSelection.create!( + candidate_ids: candidate_ids, + internal_id: id + ) + end + end + + class ElectoralDistrict < Base + def parse + name = xml.xpath("//ElectoralDistrict/Name").text + type = xml.xpath("//ElectoralDistrict/Type").text + ::Techandciviclife::ElectoralDistrict.create!( + name: name, + internal_id: id, + type: type + ) + end + end + + class Office < Base + def parse + description = xml.xpath("//Office/Description/Text[@language='en']").text + electoral_district_id = xml.xpath("//Office/ElectoralDistrictId").text + name = xml.xpath("//Office/Name/Text[@language='en']").text + ::Techandciviclife::Office.create!( + description: description, + internal_id: id, + electoral_district_id: electoral_district_id, + name: name + ) end end end diff --git a/test/fixtures/candidates.yml b/test/fixtures/candidates.yml new file mode 100644 index 0000000..80aed36 --- /dev/null +++ b/test/fixtures/candidates.yml @@ -0,0 +1,11 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +# This model initially had no columns defined. If you add columns to the +# model remove the '{}' from the fixture names and add the columns immediately +# below each fixture, per the syntax in the comments below +# +one: {} +# column: value +# +two: {} +# column: value diff --git a/test/fixtures/techandciviclife/candidate_contests.yml b/test/fixtures/techandciviclife/candidate_contests.yml new file mode 100644 index 0000000..80aed36 --- /dev/null +++ b/test/fixtures/techandciviclife/candidate_contests.yml @@ -0,0 +1,11 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +# This model initially had no columns defined. If you add columns to the +# model remove the '{}' from the fixture names and add the columns immediately +# below each fixture, per the syntax in the comments below +# +one: {} +# column: value +# +two: {} +# column: value diff --git a/test/fixtures/techandciviclife/candidate_selections.yml b/test/fixtures/techandciviclife/candidate_selections.yml new file mode 100644 index 0000000..80aed36 --- /dev/null +++ b/test/fixtures/techandciviclife/candidate_selections.yml @@ -0,0 +1,11 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +# This model initially had no columns defined. If you add columns to the +# model remove the '{}' from the fixture names and add the columns immediately +# below each fixture, per the syntax in the comments below +# +one: {} +# column: value +# +two: {} +# column: value diff --git a/test/fixtures/techandciviclife/candidates.yml b/test/fixtures/techandciviclife/candidates.yml new file mode 100644 index 0000000..80aed36 --- /dev/null +++ b/test/fixtures/techandciviclife/candidates.yml @@ -0,0 +1,11 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +# This model initially had no columns defined. If you add columns to the +# model remove the '{}' from the fixture names and add the columns immediately +# below each fixture, per the syntax in the comments below +# +one: {} +# column: value +# +two: {} +# column: value diff --git a/test/fixtures/techandciviclife/electoral_districts.yml b/test/fixtures/techandciviclife/electoral_districts.yml new file mode 100644 index 0000000..80aed36 --- /dev/null +++ b/test/fixtures/techandciviclife/electoral_districts.yml @@ -0,0 +1,11 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +# This model initially had no columns defined. If you add columns to the +# model remove the '{}' from the fixture names and add the columns immediately +# below each fixture, per the syntax in the comments below +# +one: {} +# column: value +# +two: {} +# column: value diff --git a/test/fixtures/techandciviclife/offices.yml b/test/fixtures/techandciviclife/offices.yml new file mode 100644 index 0000000..80aed36 --- /dev/null +++ b/test/fixtures/techandciviclife/offices.yml @@ -0,0 +1,11 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +# This model initially had no columns defined. If you add columns to the +# model remove the '{}' from the fixture names and add the columns immediately +# below each fixture, per the syntax in the comments below +# +one: {} +# column: value +# +two: {} +# column: value diff --git a/test/models/candidate_test.rb b/test/models/candidate_test.rb new file mode 100644 index 0000000..85798b8 --- /dev/null +++ b/test/models/candidate_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class CandidateTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/techandciviclife/candidate_contest_test.rb b/test/models/techandciviclife/candidate_contest_test.rb new file mode 100644 index 0000000..2597e82 --- /dev/null +++ b/test/models/techandciviclife/candidate_contest_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class Techandciviclife::CandidateContestTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/techandciviclife/candidate_selection_test.rb b/test/models/techandciviclife/candidate_selection_test.rb new file mode 100644 index 0000000..20567e9 --- /dev/null +++ b/test/models/techandciviclife/candidate_selection_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class Techandciviclife::CandidateSelectionTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/techandciviclife/candidate_test.rb b/test/models/techandciviclife/candidate_test.rb new file mode 100644 index 0000000..deb2a84 --- /dev/null +++ b/test/models/techandciviclife/candidate_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class Techandciviclife::CandidateTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/techandciviclife/electoral_district_test.rb b/test/models/techandciviclife/electoral_district_test.rb new file mode 100644 index 0000000..1d57f9a --- /dev/null +++ b/test/models/techandciviclife/electoral_district_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class Techandciviclife::ElectoralDistrictTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/techandciviclife/office_test.rb b/test/models/techandciviclife/office_test.rb new file mode 100644 index 0000000..67b13ae --- /dev/null +++ b/test/models/techandciviclife/office_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class Techandciviclife::OfficeTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end