Merge pull request #84 from GLAM-Workbench/update

Update Trove Harvester package and add RO-Crate file
GLAM-Workbench · Oct 25, 2023 · 6dbfcee · 6dbfcee
2 parents 6202a61 + f897e23
commit 6dbfcee
Show file tree

Hide file tree

Showing 17 changed files with 671 additions and 64 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@ getting-started/my_nma_dataset.csv
 anzac-day*
 front_pages*
 Untitled*
+scripts/add_nb_metadata.py
diff --git a/.zenodo.json b/.zenodo.json
@@ -5,7 +5,7 @@
   "related_identifiers": [
     {
       "scheme": "url",
-      "identifier": "https://github.com/GLAM-Workbench/trove-newspaper-harvester/tree/v2.0.0",
+      "identifier": "https://github.com/GLAM-Workbench/trove-newspaper-harvester/tree/v2.0.1",
       "relation": "isDerivedFrom",
       "resource_type": "software"
     },
@@ -22,21 +22,21 @@
       "resource_type": "other"
     }
   ],
-  "version": "v2.0.0",
+  "version": "v2.0.1",
   "upload_type": "software",
   "keywords": [
     "Trove",
     "newspapers",
     "Jupyter",
     "GLAM Workbench"
   ],
-  "publication_date": "2023-08-31",
+  "publication_date": "2023-10-25",
   "creators": [
     {
       "orcid": "0000-0001-7956-4498",
       "name": "Sherratt, Tim"
     }
   ],
   "access_right": "open",
-  "description": "<p>Current version: <a href=\"https://github.com/GLAM-Workbench/trove-newspaper-harvester/releases/tag/v2.0.0\">v2.0.0</a></p> <p>The <a href=\"https://pypi.org/project/trove-newspaper-harvester/\">Trove Newspaper &amp; Gazette Harvester Harvester</a> makes it easy to download large quantities of digitised articles from Trove’s newspapers and gazettes. Just give it a search from the Trove web interface, and the harvester will save the metadata of all the articles in a CSV (spreadsheet) file for further analysis. You can also save the full text of every article, as well as copies of the articles as JPG images, and even PDFs. While the web interface will only show you the first 2,000 results matching your search, the Newspaper &amp; Gazette Harvester will get <strong>everything</strong>.</p> <p>The Jupyter notebooks in this repository use the Trove Newspaper and Gazette Harvester to download large quantities of digitised newspaper articles from Trove. There’s also a few examples of how you can analyse and explore the harvested data.</p> <p>The notebooks include:</p> <ul> <li><strong>Using TroveHarvester to get newspaper articles in bulk</strong> — an easy introduction to the TroveHarvester tool</li> <li><strong>Trove Harvester web app</strong> — a simple web interface to the TroveHarvester, the easiest way to harvest data from Trove (runs in Voila)</li> <li><strong>Harvesting articles that mention “Anzac Day” on Anzac Day</strong> – import the Harvester as a Python library to harvest a complex search</li> <li><strong>Display the results of a harvest as a searchable database using Datasette</strong> – load your harvested data into a SQLite database and explore it using Datasette</li> <li><strong>Exploring your TroveHarvester data</strong> — use Pandas to analyse your data and create some visualisations</li> <li><strong>Explore harvested text files</strong> (experimental) — analyse the full text content of harvested articles</li> </ul> <p>See the <a href=\"https://glam-workbench.github.io/trove-harvester/\">GLAM Workbench for more details</a>.</p> <h2 id=\"cite-as\">Cite as</h2> <p>See the GLAM Workbench or <a href=\"https://doi.org/10.5281/zenodo.3545044\">Zenodo</a> for up-to-date citation details.</p> <hr /> <p>This repository is part of the <a href=\"https://glam-workbench.github.io/\">GLAM Workbench</a>.<br /> If you think this project is worthwhile, you might like <a href=\"https://github.com/sponsors/wragge?o=esb\">to sponsor me on GitHub</a>.</p>"
+  "description": "<p>Current version: <a href=\"https://github.com/GLAM-Workbench/trove-newspaper-harvester/releases/tag/v2.0.1\">v2.0.1</a></p> <p>The <a href=\"https://pypi.org/project/trove-newspaper-harvester/\">Trove Newspaper &amp; Gazette Harvester Harvester</a> makes it easy to download large quantities of digitised articles from Trove’s newspapers and gazettes. Just give it a search from the Trove web interface, and the harvester will save the metadata of all the articles in a CSV (spreadsheet) file for further analysis. You can also save the full text of every article, as well as copies of the articles as JPG images, and even PDFs. While the web interface will only show you the first 2,000 results matching your search, the Newspaper &amp; Gazette Harvester will get <strong>everything</strong>.</p> <p>The Jupyter notebooks in this repository use the Trove Newspaper and Gazette Harvester to download large quantities of digitised newspaper articles from Trove. There’s also a few examples of how you can analyse and explore the harvested data.</p> <p>The notebooks include:</p> <ul> <li><strong>Using TroveHarvester to get newspaper articles in bulk</strong> — an easy introduction to the TroveHarvester tool</li> <li><strong>Trove Harvester web app</strong> — a simple web interface to the TroveHarvester, the easiest way to harvest data from Trove (runs in Voila)</li> <li><strong>Harvesting articles that mention “Anzac Day” on Anzac Day</strong> – import the Harvester as a Python library to harvest a complex search</li> <li><strong>Display the results of a harvest as a searchable database using Datasette</strong> – load your harvested data into a SQLite database and explore it using Datasette</li> <li><strong>Exploring your TroveHarvester data</strong> — use Pandas to analyse your data and create some visualisations</li> <li><strong>Explore harvested text files</strong> (experimental) — analyse the full text content of harvested articles</li> </ul> <p>See the <a href=\"https://glam-workbench.github.io/trove-harvester/\">GLAM Workbench for more details</a>.</p> <h2 id=\"cite-as\">Cite as</h2> <p>See the GLAM Workbench or <a href=\"https://doi.org/10.5281/zenodo.3545044\">Zenodo</a> for up-to-date citation details.</p> <hr /> <p>This repository is part of the <a href=\"https://glam-workbench.github.io/\">GLAM Workbench</a>.<br /> If you think this project is worthwhile, you might like <a href=\"https://github.com/sponsors/wragge?o=esb\">to sponsor me on GitHub</a>.</p>"
 }
diff --git a/Explore-harvested-text-files.ipynb b/Explore-harvested-text-files.ipynb
@@ -1399,6 +1399,15 @@
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
   },
+  "rocrate": {
+   "author": [
+    {
+     "name": "Sherratt, Tim",
+     "orcid": "https://orcid.org/0000-0001-7956-4498"
+    }
+   ],
+   "name": "Explore harvested text files"
+  },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {},

diff --git a/Exploring-your-TroveHarvester-data.ipynb b/Exploring-your-TroveHarvester-data.ipynb
@@ -1673,6 +1673,15 @@
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
   },
+  "rocrate": {
+   "author": [
+    {
+     "name": "Sherratt, Tim",
+     "orcid": "https://orcid.org/0000-0001-7956-4498"
+    }
+   ],
+   "name": "Exploring your harvested data"
+  },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {},

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Trove Newspaper and Gazette Harvester
 
-Current version: [v2.0.0](https://github.com/GLAM-Workbench/trove-newspaper-harvester/releases/tag/v2.0.0)
+Current version: [v2.0.1](https://github.com/GLAM-Workbench/trove-newspaper-harvester/releases/tag/v2.0.1)
 
 The [Trove Newspaper & Gazette Harvester Harvester](https://pypi.org/project/trove-newspaper-harvester/) makes it easy to download large quantities of digitised articles from Trove's newspapers and gazettes. Just give it a search from the Trove web interface, and the harvester will save the metadata of all the articles in a CSV (spreadsheet) file for further analysis. You can also save the full text of every article, as well as copies of the articles as JPG images, and even PDFs. While the web interface will only show you the first 2,000 results matching your search, the Newspaper & Gazette Harvester will get **everything**.
 

diff --git a/Using-TroveHarvester-to-get-newspaper-articles-in-bulk.ipynb b/Using-TroveHarvester-to-get-newspaper-articles-in-bulk.ipynb
@@ -423,6 +423,15 @@
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
   },
+  "rocrate": {
+   "author": [
+    {
+     "name": "Sherratt, Tim",
+     "orcid": "https://orcid.org/0000-0001-7956-4498"
+    }
+   ],
+   "name": "Using TroveHarvester to get newspaper and gazette articles in bulk"
+  },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {},

diff --git a/display_harvest_results_using_datasette.ipynb b/display_harvest_results_using_datasette.ipynb
@@ -341,6 +341,15 @@
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
   },
+  "rocrate": {
+   "author": [
+    {
+     "name": "Sherratt, Tim",
+     "orcid": "https://orcid.org/0000-0001-7956-4498"
+    }
+   ],
+   "name": "Display the results of a harvest as a searchable database using Datasette"
+  },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {},

diff --git a/harvest-specific-days.ipynb b/harvest-specific-days.ipynb
@@ -533,6 +533,15 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
+  },
+  "rocrate": {
+   "author": [
+    {
+     "name": "Sherratt, Tim",
+     "orcid": "https://orcid.org/0000-0001-7956-4498"
+    }
+   ],
+   "name": "Harvesting articles that mention \"Anzac Day\" on Anzac Day"
   }
  },
  "nbformat": 4,

diff --git a/newspaper_harvester_app.ipynb b/newspaper_harvester_app.ipynb
@@ -289,6 +289,15 @@
    "pygments_lexer": "ipython3",
    "version": "3.10.12"
   },
+  "rocrate": {
+   "author": [
+    {
+     "name": "Sherratt, Tim",
+     "orcid": "https://orcid.org/0000-0001-7956-4498"
+    }
+   ],
+   "name": "Trove Newspaper & Gazette Harvester"
+  },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {

diff --git a/requirements-dev.in b/requirements-dev.in
@@ -5,4 +5,8 @@ nbqa
 black[jupyter]
 isort
 flake8
-pre-commit
+pre-commit
+rocrate
+giturlparse
+jupyterlab-code-formatter
+gitpython