-
-
Notifications
You must be signed in to change notification settings - Fork 854
/
store.py
147 lines (111 loc) · 5.16 KB
/
store.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import json
import uuid as uuid_builder
import validators
# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?
# Open a github issue if you know something :)
# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change
class ChangeDetectionStore:
def __init__(self):
self.needs_write = False
self.__data = {
'note' : "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
'watching': {},
'settings': {
'headers': {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate', # No support for brolti in python requests yet.
'Accept-Language': 'en-GB,en-US;q=0.9,en;'
},
'requests': {
'timeout': 15, # Default 15 seconds
'minutes_between_check': 3 * 60 # Default 3 hours
}
}
}
# Base definition for all watchers
self.generic_definition = {
'url': None,
'tag': None,
'last_checked': 0,
'last_changed': 0,
'title': None,
'previous_md5': None,
'uuid': str(uuid_builder.uuid4()),
'headers' : {}, # Extra headers to send
'history' : {} # Dict of timestamp and output stripped filename
}
try:
with open('/datastore/url-watches.json') as json_file:
from_disk = json.load(json_file)
# @todo isnt there a way todo this dict.update recursively?
# Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
if 'watching' in from_disk:
self.__data['watching'].update(from_disk['watching'])
if 'settings' in from_disk:
if 'headers' in from_disk['settings']:
self.__data['settings']['headers'].update(from_disk['settings']['headers'])
if 'requests' in from_disk['settings']:
self.__data['settings']['requests'].update(from_disk['settings']['requests'])
# Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
# @todo pretty sure theres a python we todo this with an abstracted(?) object!
i = 0
for uuid, watch in self.data['watching'].items():
_blank = self.generic_definition.copy()
_blank.update(watch)
self.__data['watching'].update({uuid: _blank})
print("Watching:", uuid, _blank['url'])
# First time ran, doesnt exist.
except (FileNotFoundError, json.decoder.JSONDecodeError):
print("Creating JSON store")
self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
self.add_watch(url='https://changedetection.io', tag='Tech news')
# self.entryVariable.get()
def update_watch(self, uuid, val, var):
self.__data['watching'][uuid].update({val: var})
self.needs_write = True
@property
def data(self):
return self.__data
def get_all_tags(self):
tags=[]
for uuid, watch in self.data['watching'].items():
# Support for comma separated list of tags.
for tag in watch['tag'].split(','):
tag = tag.strip()
if not tag in tags:
tags.append(tag)
tags.sort()
return tags
def delete(self, uuid):
# Probably their should be dict...
del(self.__data['watching'][uuid])
self.needs_write = True
def url_exists(self, url):
# Probably their should be dict...
for watch in self.data['watching']:
if watch['url'] == url:
return True
return False
def get_val(self, uuid, val):
# Probably their should be dict...
return self.data['watching'][uuid].get(val)
def add_watch(self, url, tag):
# @todo deal with exception
validators.url(url)
# @todo use a common generic version of this
_blank = self.generic_definition.copy()
_blank.update({
'url': url,
'tag': tag,
'uuid': str(uuid_builder.uuid4())
})
self.data['watching'].update({_blank['uuid']: _blank})
def sync_to_json(self):
print ("Saving index")
with open('/datastore/url-watches.json', 'w') as json_file:
json.dump(self.data, json_file, indent=4)
self.needs_write = False
# body of the constructor