ccppi/job-scrapper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

do not remove stared invalid entries!

Browse Source 
fetchall db querys
This commit is contained in:
ccppi 2024-08-08 08:50:43 +02:00
parent 71209e9b5f
commit 83e0d81ebe
2 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/db.py
View File

@ -125,11 +125,9 @@ def isStillValid(file,skiprows):
rows = cursor.fetchmany(256) rows = cursor.fetchmany(256)
h = httplib2.Http() h = httplib2.Http()
for row in rows: for row in rows:
#with requests.Session() as session:
print("row: ",row[0]) print("row: ",row[0])
try: try:
(resp,content) = h.request(row[0], 'HEAD') (resp,content) = h.request(row[0], 'HEAD')
#page = session.get(row[0])
except IOError as e: except IOError as e:
print("link is no valid URL so remove item") print("link is no valid URL so remove item")
print("error: ",e) print("error: ",e)
@ -137,20 +135,20 @@ def isStillValid(file,skiprows):
except httplib2.error.RelativeURIError: except httplib2.error.RelativeURIError:
isLink = False isLink = False
print("RelativeURIError: Not a valid link") print("RelativeURIError: Not a valid link")
rm_cursor = connection.cursor() #rm_cursor = connection.cursor()
rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ?;""",(row[0],)) #rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ?;""",(row[0],))
finally: finally:
try: try:
resp resp
except NameError: except NameError:
print("Not a valid link") print("Not a valid link")
rm_cursor = connection.cursor() rm_cursor = connection.cursor()
rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ?;""",(row[0],)) rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ? AND star != 1;""",(row[0],))
else: else:
if resp.status >= 400 or isLink == False: if resp.status >= 400 or isLink == False:
print("link is no more valid, remove item") print("link is no more valid, remove item")
rm_cursor = connection.cursor() rm_cursor = connection.cursor()
rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ?;""",(row[0],)) rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ? AND star != 1;""",(row[0],))
print ("Deletion resultet in: ", rm_itm) print ("Deletion resultet in: ", rm_itm)
print("result of commit: ", connection.commit()) print("result of commit: ", connection.commit())
return 0 return 0

3
lib/gui.py
View File

@ -270,6 +270,9 @@ class MainWindow(QMainWindow):
#self.model.setTable("") #self.model.setTable("")
self.model.setQuery(cmd +" ;") self.model.setQuery(cmd +" ;")
#self.model.setTable("jobs") #self.model.setTable("jobs")
while (self.model.canFetchMore()):
print("fetch iterations++")
self.model.fetchMore()
self.proxymodel2 = QSortFilterProxyModel(self) self.proxymodel2 = QSortFilterProxyModel(self)
self.proxymodel2.setSourceModel(self.model) self.proxymodel2.setSourceModel(self.model)