Disclosure Notes for new Website

[flooie]

Notes:

We've been notified that the AO will no longer be providing financial disclosure deliveries. We now need to download them from the website, which presents some challenges:

Automatic Download Process: Downloading disclosures is cumbersome due to the website's design and the presence of a Google reCAPTCHA.
Limited Download Capability: The website restricts downloads to 100 disclosures at a time.

Current Workaround:

A temporary solution has been found using manual Javascript injection to achieve the following:

Open "show more" buttons until all disclosures are displayed.
Download disclosures in batches of 100 (the website's limit) by repeatedly selecting and deselecting groups of 100 with javascript

Future Considerations:

Automation Potential: I believe this process can be improved with better automation with minimal development effort.
Alternative Approaches: Investigate alternative methods like Selenium for automation, but the current approach might be the most efficient solution.

Next Steps:

Discuss the feasibility and implications of automating the download process.
Explore alternative methods for accessing and downloading disclosures.

Additional Information:

here are the rudimentary scripts I used to accomplish the task of bulk downloading it

function simulateCheckboxClick(checkbox) {
  // Create a click event
  const clickEvent = new MouseEvent('click', {
    bubbles: true,
    cancelable: true,
    view: window
  });

  // Dispatch the click event on the checkbox
  checkbox.dispatchEvent(clickEvent);
}

	
function checkAndDownload(checkboxes) {
  let currentIndex = 0;

  const processNext = () => {
    if (currentIndex < checkboxes.length) {
      const checkbox = checkboxes[currentIndex];
      simulateCheckboxClick(checkbox);
      downloadSelected();

      // Wait one second before unchecking and processing next
      setTimeout(() => {
        checkbox.checked = false; // Uncheck after download
        currentIndex++;
        processNext();
      }, 2000); // Adjust wait time as needed (in milliseconds)
    }
  };

  processNext();
}

function main() {
  const checkboxes = document.querySelectorAll('input[type="checkbox"]');
  if (checkboxes.length > 0) {
    checkAndDownload(checkboxes);
  } else {
    console.error("No checkboxes found"); // Add for error handling
  }
}

// Call the main function
main();

and

function simulateCheckboxClick(checkbox) {
  // Create a click event
  const clickEvent = new MouseEvent('click', {
    bubbles: true,
    cancelable: true,
    view: window
  });

  // Dispatch the click event on the checkbox
  checkbox.dispatchEvent(clickEvent);
}

function downloadSelected() {
  const downloadButton = document.querySelector('#download-some-btn');
  if (downloadButton) {
    downloadButton.click();
    console.log("Clicked download button"); // Add for logging
  } else {
    console.error("Download button not found"); // Add for error handling
  }
}


var checkboxes = document.querySelectorAll('input[type="checkbox"]');
var checking = false
var begun_checking = false
var checked_now = 0
for (let i = 0; i < checkboxes.length; i++) {
    const checkbox = checkboxes[i];
    const isChecked = checkbox.checked; // Check the 'checked' property
    if (begun_checking == true) {
        if (isChecked == false) {
            checked_now += 1
            simulateCheckboxClick(checkbox)

            if (checked_now > 100) {
                break
            }
            continue
        }
    }
    if (checked_now == 100) {
        break
    }
    if (isChecked == true) {
        console.log(checkbox, isChecked);
        begun_checking = true
        simulateCheckboxClick(checkbox)
    }
}