-
Notifications
You must be signed in to change notification settings - Fork 0
/
pup.mjs
71 lines (58 loc) · 1.8 KB
/
pup.mjs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import puppeteer from "puppeteer";
import fs from "fs";
import flightUrls2019 from "./flights-2019.mjs";
(async (fls) => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
page.on("console", log => console[log._type](log._text));
await page.goto("http://xcleague.com/xc/leagues/2019-1.html?vx=2");
let data = {
flights: [],
pages: []
};
data = await page.evaluate(
(data, fls) => {
const exists = (link, urls) => urls.some(url => url == link);
const trs = document.querySelectorAll("#leagueTable tbody tr");
for (const tr of trs) {
let link = tr.querySelector("td:nth-child(9) a").href;
let viewPage = link.match(/leagues/);
if (viewPage && viewPage[0] !== null) {
data.pages.push(link);
} else if (!exists(link, fls)) {
data.flights.push(link);
}
}
return data;
},
data,
fls
);
// console.log(data);
for (const url of data.pages) {
await page.goto(url);
data = await page.evaluate(
(data, fls) => {
const exists = (link, urls) => urls.some(url => url == link);
const links = document.querySelectorAll(
"#leagueTable [class^=flight] a:first-child"
);
for (const link of links) {
if (!exists(link, fls)) data.flights.push(link.href);
}
return data;
},
data,
fls
);
}
console.log('flights', data.flights)
const file = fs.createWriteStream('flights-latest.mjs')
file.on('error', function(err) { console.log('error writing log') })
file.write('const flights = [')
data.flights.forEach(function(flight) { file.write(`"${flight}",\n`) })
file.write('];\n')
file.write('export default flights')
file.end();
browser.close();
})(flightUrls2019);