-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.js
65 lines (51 loc) · 1.69 KB
/
app.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
/**
* Module dependencies.
*/
var express = require('express')
, routes = require('./routes')
, httpAgent = require('http-agent')
, jsdom = require('jsdom').jsdom
, host = 'denver2012.drupal.org'
, urls = ['bofs', 'bofs/2012-03-21', 'bofs/2012-03-22'];
var app = module.exports = express.createServer();
// Configuration
app.configure(function(){
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
app.use(express.bodyParser());
app.use(express.methodOverride());
app.use(app.router);
app.use(express.static(__dirname + '/public'));
});
app.configure('development', function(){
app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});
app.configure('production', function(){
app.use(express.errorHandler());
});
// Routes
app.get('/', routes.index);
app.listen(3000);
console.log("Express server listening on port %d in %s mode", app.address().port, app.settings.env);
// Scraping
function bofScrape() {
var agent = httpAgent.create(host, urls);
console.log('Scraping', url.length, 'pages from', agent.host);
agent.addListener('next', function (err, agent) {
var window = jsdom(agent.body).createWindow()
, $ = require('jquery').create(window);
// Now we can use jQuery to grab some DOM elements.
var start_times = $('.date-display-start')
, end_times = $('.date-display-end')
, bof_titles = $('h3.session-title')
, bof_categories = $('.views-field-field-bof-category')
, bof_rooms = $('views-field-field-session-room');
agent.next();
});
agent.AddListener('stop', function(err, agent) {
if (err) console.log(err);
console.log('Done scraping.');
});
// Start scraping.
agent.start();
}