Add Montana Legislative Scraper - Sept. 10th Hackathon #7

smai-f · 2022-09-11T00:15:38Z

WIP

…se audio recording

smai-f · 2022-09-11T05:15:59Z

python/cdp_montana_legislature_backend/scraper.py

+
+                # TODO: Get more metadata from here
+                event_info_text = re.search('EventInfo:(.*),', sliq_html).groups()[0]
+                event_info_json = json.loads(event_info_text)


The timestamps up in here are not the video timestamps but rather the start/end time the actual day of the hearings. We have to find this and match the agenda tree timestamp 😩:

var dataModel = { Manifest:{"id":"40001","children":[{"id":"eventInfo","children":null,"textTags":{"NAME":{"id":"eventInfo","text":"eventInfo"}},"timeTags":{"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}},{"id":"media","children":null,"textTags":{"NAME":{"id":"media","text":"media"}},"timeTags":{"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}},{"id":"agenda","children":null,"textTags":{"NAME":{"id":"agenda","text":"agenda"}},"timeTags":{"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}},{"id":"people","children":null,"textTags":{"NAME":{"id":"people","text":"people"}},"timeTags":{"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}},{"id":"handoutFile","children":null,"textTags":{"NAME":{"id":"handoutFile","text":"handoutFile"}},"timeTags":{"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}}],"textTags":{},"timeTags":{}}, EventInfo:{"id":"40001","children":null,"textTags":{"TITLE":{"id":"40001","text":"House Appropriations"},"DESCRIPTION":{"id":"40001","text":"House Appropriations - Video"},"LOCATION":{"id":"40001","text":"Room 102 "},"MEETINGSTATUS":{"id":"40001","text":"-1"},"SCHEDULEDDURATION":{"id":"40001","text":"14400"},"scheduledDate":{"id":"40001","text":"Tuesday, Jan 5, 2021"},"scheduledTime":{"id":"40001","text":"09:00 - 13:00"},"scheduledDuration":{"id":"40001","text":"4 Hours "},"actualDate":{"id":"40001","text":"Tuesday, Jan 5, 2021"},"actualTime":{"id":"40001","text":"09:02 - 12:04"},"actualDuration":{"id":"40001","text":"3 Hours 2 Minutes"},"eventStatus":{"id":"40001","text":"Adjourned"},"eventStatusClass":{"id":"40001","text":"eventTableEnded"}},"timeTags":{"SCHEDULEDSTART":{"id":null,"timestamp":"2021-01-05T09:00:00.0000000"},"STARTTIME":{"id":null,"timestamp":"2021-01-05T09:02:00.0000000"},"ENDTIME":{"id":null,"timestamp":"2021-01-05T12:04:38.0000000"},"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}}, Media:{"id":"40001","children":[{"id":"150394","children":null,"textTags":{"URL":{"id":"150394","text":"https://sg002-live.sliq.net/00309-vod/_definst_/2021/01/05/House%20Appropriations_2021-01-05-09.02.00_40001_1115.mp4/playlist.m3u8"},"DESCRIPTION":{"id":"150394","text":"Video"},"IsLive":{"id":"150394","text":"False"},"Enabled":{"id":"150394","text":"True"},"GlobalStreamId":{"id":"150394","text":"4"},"NAME":{"id":"150394","text":"Video"},"AudioOnly":{"id":"150394","text":"False"}},"timeTags":{"STARTTIME":{"id":null,"timestamp":"2021-01-05T09:02:00.0000000"}}}],"textTags":{},"timeTags":{"MODIFIED":{"id":null,"timestamp":"2021-06-07T12:23:32.0000000"}}}, AgendaTree:[{"key":"A224723","text":"Call to Order/Roll Call","startTime":"2021-01-05T09:02:00","displayOrder":100000,"lastModified":"2021-06-07T12:23:32.443+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null},{"key":"A224724","text":"HB 3 - Supplemental Appropriations Bill - David Bedey","startTime":"2021-01-05T09:19:00","displayOrder":200000,"lastModified":"2021-06-07T12:23:32.483+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null},{"key":"A224725","text":"HB 4 - Appropriations by Budget Amendment - Dan Bartel","startTime":"2021-01-05T10:37:43","displayOrder":300000,"lastModified":"2021-06-07T12:23:32.523+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null},{"key":"A224726","text":"Recess","startTime":"2021-01-05T11:10:09","displayOrder":400000,"lastModified":"2021-06-07T12:23:32.783+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null},{"key":"A224727","text":"Reconvene","startTime":"2021-01-05T11:46:46","displayOrder":500000,"lastModified":"2021-06-07T12:23:32.82+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null},{"key":"A224728","text":"HB 1 - Feed Bill to fund 67th Legislative session and prepare for 2023 - Llew Jones","startTime":"2021-01-05T11:46:48","displayOrder":600000,"lastModified":"2021-06-07T12:23:32.857+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null},{"key":"A224729","text":"Adjournment","startTime":"2021-01-05T12:04:02","displayOrder":700000,"lastModified":"2021-06-07T12:23:32.893+00:00","parentKey":"-1","externalUrl":"","children":null,"deleted":false,"foreignKey":null}], Speakers:[], ccItems:{}, Handouts: [{"Name":"Agenda","HandoutFileUrl":"http://laws.leg.mt.gov/legprd/LAW0240W$CMTE.ActionQuery?P_SESS=20211&P_COM_NM=%28H%29+Appropriations&P_ACTN_DTM=01%2F05%2F2021&U_ACTN_DTM=01%2F05%2F2021&Z_ACTION2=Find","HandoutFileName":"http://laws.leg.mt.gov/legprd/LAW0240W$CMTE.ActionQuery?P_SESS=20211&P_COM_NM=%28H%29+Appropriations&P_ACTN_DTM=01%2F05%2F2021&U_ACTN_DTM=01%2F05%2F2021&Z_ACTION2=Find","LastModifiedTIme":"2022-09-11T05:08:42.599304+00:00","Id":64672,"Tag":null},{"Name":"Minutes","HandoutFileUrl":"http://sg001-harmony.sliq.net:80/00309/Harmony/en/PowerBrowser/ViewHandoutFile?contentEntityId=40001&handoutId=64674","HandoutFileName":"210105APH_Hm1.pdf","LastModifiedTIme":"2022-09-11T05:08:42.599304+00:00","Id":64674,"Tag":null}] , AspectRatio:0.5625, AspectRatioTxt: '16:9', mediaStartTime : '2021-01-05T09:02:00', mediaEndTime : '2021-01-05T12:04:38', playRangeStart : '2021-01-05T09:02:00', playRangeEnd: '2021-01-05T12:04:38', venueId: '41', silendPeriods:[] };

Also if the agenda is added after the fact, it's very possible at the time of scrape that the subset timestamp wouldn't be available yet. This seems like reason to skip the video processing and try again later, but I think that we'd have to contribute this ability to CDP?

edit: Nope, I don't think we need to contribute to CDP. We can just not use the datetime range arguments at all and scrape everything every run, and only choose to ingest a video if it has the agenda timestamps available. If not, bail early. Then the next time the CRON job runs it will try again for the agenda timestamps. It does mean a bunch of extra scraping will happen every run, but 🤷‍♀️

I created an issue in CDP for this one: CouncilDataProject/cdp-backend#212

ah, at least the agenda tree key minus an "A" is in the Sliq URL:
http://sg001-harmony.sliq.net/00309/Harmony/en/PowerBrowser/PowerBrowserV2/20170221/-1/40001?agendaId=224728

whargrove · 2022-09-11T23:23:39Z

python/cdp_montana_legislature_backend/scraper.py

+    # TODO: Read key bills from config or Capitol Tracker JSON
+    key_bill_names = ["HB 1"]


I created #10 for this TODO

whargrove · 2022-09-11T23:24:11Z

python/cdp_montana_legislature_backend/scraper.py

+    # Start at the big table of all bills from the 2021 session.
+    bills_url_2021 = "http://laws.leg.mt.gov/legprd/LAW0217W$BAIV.return_all_bills?P_SESS=20211"


@smai-f Have you given any thought to how we might want to handle past / future legislative sessions?

I'd say push a commit to update this line once the 2023 site is live. We should only have to update this URL once at the beginning of the 2023 session 😁 Maybe it could live in the config with the key bills

Update: Mark raised big money, we're scraping everything!

whargrove · 2022-09-11T23:26:21Z

python/cdp_montana_legislature_backend/scraper.py

+    bills_table_rows = bills_table.find_all('tr')
+    print(f"Found table with {len(bills_table_rows) - 1} rows.")
+
+    # Of all the bills, narrow down to only the key bills and store off the LAWS bill URL for the next step.


Thank you for adding comments, it really adds to the overall clarity and context for the scraper!

whargrove · 2022-09-11T23:27:02Z

python/cdp_montana_legislature_backend/scraper.py

+
+    # Of all the bills, narrow down to only the key bills and store off the LAWS bill URL for the next step.
+    key_bills_data = []
+    for bill_row in bills_table_rows[1:]:


Consider adding a comment here that we skip the first row because it is actually headings but in a tr and not in a th.

whargrove · 2022-09-11T23:29:00Z

python/cdp_montana_legislature_backend/scraper.py

+    # Go to each LAWS bill URL and find bill actions that have associated recordings.
+    for bill_data in key_bills_data:
+        laws_bill_html = requests.get(bill_data['laws_bill_url']).text
+        bill_rows_with_recordings = re.findall('.*sliq.*', laws_bill_html)


A comment here about why we use regex search on the full html instead of going through BeautifulSoup due to "invalid" HTML returned by the server that can't be parsed by BeautifulSoup.

whargrove · 2022-09-11T23:32:24Z

python/cdp_montana_legislature_backend/scraper.py

+            all_links = bill_cells[-1].find_all('a')
+            sliq_links = [ link for link in all_links if "sliq" in link['href'] ]


BeautifulSoup also supports a parameter to find that you can pass an regex ... so instead of list comprehension here to filter the anchor tags, this becomes...

bill_cells[-1].find_all('a', href=re.compile('sliq'))

oo la la 👍

smai-f · 2022-09-14T01:48:49Z

python/cdp_montana_legislature_backend/scraper.py

+                    parsed_url = urlparse(sliq_link)
+                    # TODO: Handle if this isn't in the query? Does that always mean that the timestamp hasn't been included yet, thus the video
+                    # shouldn't be scraped on this pass? 
+                    agenda_id = 'A' + parse_qs(parsed_url.query)['agendaId'][0]


TODO: Is this always true?

…that have an agendaId aka timestamps

smai-f added 4 commits September 10, 2022 18:14

Hackathon progress

cd4ba7b

Rename and explain things

63bfe27

Prefer video recording over audio recording if it exists, otherwise u…

e593598

…se audio recording

Provide a nicer CDP event title

95055b6

smai-f commented Sep 11, 2022

View reviewed changes

whargrove reviewed Sep 11, 2022

View reviewed changes

smai-f added 2 commits September 13, 2022 19:03

Merge branch 'main' into scraper

ceadb4f

Scrape start and end timestamps, add comments, address lints

634c1b4

smai-f commented Sep 14, 2022

View reviewed changes

smai-f added 9 commits September 13, 2022 19:56

Address more lints

d1859dd

Return event ingestion models

9ebec5d

Ignore whitespace lint

1c54b5a

Minor changes

6f3ac99

Merge branch 'main' into scraper

a7fbc43

Happy path for video start and end time timestamps and offsets working

2472e2f

Skip sliq links without media, cleanup

c26ca64

Fix duplicate timestamp issue and log exceptions

fd94c37

Live life dangerously

9d00703

smai-f force-pushed the scraper branch from a41f107 to 9d00703 Compare November 30, 2022 05:51

smai-f added 7 commits December 5, 2022 19:14

Update default_event_gather_from_days_timedelta to look back for longer

652bca9

Support scraping hearings that only occurred within a date range

8a6e1da

Testing for merge

e90364a

Fixing lints?

4c898ab

Turn off bill page update date check for now, and only scrape videos …

b1407c0

…that have an agendaId aka timestamps

....

7435390

Format

9d32097

smai-f merged commit e500ae9 into main Dec 25, 2022

whargrove deleted the scraper branch January 6, 2023 03:17

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add Montana Legislative Scraper - Sept. 10th Hackathon #7

Add Montana Legislative Scraper - Sept. 10th Hackathon #7

smai-f commented Sep 11, 2022 •

edited

smai-f Sep 11, 2022

smai-f Sep 11, 2022 •

edited

smai-f Sep 11, 2022

whargrove Sep 11, 2022

whargrove Sep 11, 2022

smai-f Sep 12, 2022

smai-f Nov 28, 2022

whargrove Sep 11, 2022

whargrove Sep 11, 2022

whargrove Sep 11, 2022

whargrove Sep 11, 2022

smai-f Sep 12, 2022

smai-f Sep 14, 2022

		# TODO: Read key bills from config or Capitol Tracker JSON
		key_bill_names = ["HB 1"]

		# Start at the big table of all bills from the 2021 session.
		bills_url_2021 = "http://laws.leg.mt.gov/legprd/LAW0217W$BAIV.return_all_bills?P_SESS=20211"

		all_links = bill_cells[-1].find_all('a')
		sliq_links = [ link for link in all_links if "sliq" in link['href'] ]

Add Montana Legislative Scraper - Sept. 10th Hackathon #7

Add Montana Legislative Scraper - Sept. 10th Hackathon #7

Conversation

smai-f commented Sep 11, 2022 • edited

Choose a reason for hiding this comment

smai-f Sep 11, 2022 • edited

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

smai-f commented Sep 11, 2022 •

edited

smai-f Sep 11, 2022 •

edited