diff --git a/lib/db.py b/lib/db.py index 50252c7..dcf20cf 100644 --- a/lib/db.py +++ b/lib/db.py @@ -3,7 +3,7 @@ import mmh3 import sys #import requests import httplib2 -DEBUG = False +DEBUG = True def log(*s): if DEBUG: @@ -67,13 +67,13 @@ def createnwview(file): createFilterTable(file) def createFilterTable(file): - with sqlite3.connect(file) as connection: + with sqlite3.connect(file,timeout=10) as connection: cmd_create_filter_table = f"""CREATE TABLE IF NOT EXISTS filters(cmd TEXT);""" cursor = connection.cursor() cursor.execute(cmd_create_filter_table) print("db connection:",connection.total_changes) def addFineFilter(file,table,filterstr): - with sqlite3.connect(file) as connection: + with sqlite3.connect(file,timeout=10) as connection: cmd_createFineFilter = f"""INSERT INTO {table}(cmd) VALUES(?);""" cmd_checkIfExists = f"""SELECT * FROM {table} WHERE cmd = ?""" cursor = connection.cursor() @@ -81,7 +81,8 @@ def addFineFilter(file,table,filterstr): cursor.execute(cmd_createFineFilter,(filterstr,)) def writedb(jobs): - with sqlite3.connect("../db/sqlite3.db") as connection: + with sqlite3.connect("../db/sqlite3.db",timeout=10) as connection: + connection.execute("pragma journal_mode=wal") print("db connection", connection.total_changes) cursor = connection.cursor() # cursor.execute("CREATE TABLE jobs (title TEXT, location TEXT, company TEXT,link TEXT,hash INT)") @@ -94,16 +95,19 @@ def writedb(jobs): print("NEW_ENTRY") cursor.execute("INSERT INTO jobs (star,tag,title,company,location,link,pubdate,hash) VALUES (?,?,?,?,?,?,?,?)",(job.starred,job.tag,job.title,job.company,job.location,job.link,job.date,hash1)) -def isStillValid(file): +def isStillValid(file,skiprows): rows = [0,0,0] - with sqlite3.connect(file) as connection: + with sqlite3.connect(file,timeout=10) as connection: cmd_read_chunk = f"""SELECT link from jobs;""" + connection.execute("pragma journal_mode=wal") cursor = connection.cursor() cursor.execute(cmd_read_chunk) + #cursor.fetchmany(skiprows)#drop rows while(len(rows)!=0): isLink = True rows = [] + rows = cursor.fetchmany(256) h = httplib2.Http() for row in rows: @@ -135,3 +139,5 @@ def isStillValid(file): rm_itm = rm_cursor.execute(f"""DELETE from jobs WHERE link = ?;""",(row[0],)) print ("Deletion resultet in: ", rm_itm) print("result of commit: ", connection.commit()) + return 0 + diff --git a/lib/gui.py b/lib/gui.py index a04149f..32d50bc 100644 --- a/lib/gui.py +++ b/lib/gui.py @@ -2,14 +2,31 @@ from PySide6.QtWidgets import QApplication, QWidget, QMainWindow, QTableWidget, from PySide6.QtWebEngineWidgets import QWebEngineView from PySide6.QtCore import QUrl,Qt,QSortFilterProxyModel, qDebug, QSize,QObject,QThread,Signal from PySide6.QtSql import QSqlDatabase, QSqlTableModel, QSqlQueryModel, QSqlQuery - +from PySide6 import QtGui from db import addFineFilter import sysparse import sys +import db as db + +DBFILE = "../db/sqlite3.db" Cantons = ["AG","ZH","BE","SG","SO"] +class TableModel(QAbstractTableModel): + def __init__(self, data): + super(TableModel, self).__init__() + self._data = data + def data(self, index, role): + if role == Qt.BackgroundRole: + return QtGui.QColor('#9C0006') +class ValidationWorker(QObject): + finished = Signal() + + def run(self): + ret = db.isStillValid(DBFILE,0) + if ret == 0: + self.finished.emit() class Worker(QObject): pwprompt = Signal() pw = Signal(str) @@ -56,6 +73,8 @@ class MainWindow(QMainWindow): self.setProxyViewSettings() self.view.clicked.connect(self.cell_clicked) + self.PValidate = QPushButton("links valid") + self.PValidate.clicked.connect(self.runValidation) self.PsyncDB = QPushButton("Perform sync acording to config file") self.PsyncDB.clicked.connect(self.runWorker) @@ -64,6 +83,7 @@ class MainWindow(QMainWindow): self.layout.addWidget(self.b_canton) self.layout.addWidget(self.EditQuery) self.layout.addWidget(self.PsyncDB) + self.layout.addWidget(self.PValidate) self.layout2.addLayout(self.layout) self.layout2.addWidget(self.browser) @@ -90,9 +110,25 @@ class MainWindow(QMainWindow): self.worker.pwprompt.connect(self.showDialog) self.worker.finished.connect(self.thread.quit) self.worker.finished.connect(self.enable_PsyncDB) - - self.thread.start() + def runValidation(self): + self.validationThread = QThread() + self.validationWorker = ValidationWorker() + + self.validationWorker.moveToThread(self.validationThread) + + self.validationThread.started.connect(self.disableValidationButton) + self.validationThread.started.connect(self.validationWorker.run) + + self.validationThread.start() + self.validationWorker.finished.connect(self.validationThread.quit) + self.validationWorker.finished.connect(self.enableValidationButton) + + def enableValidationButton(self): + self.PValidate.setEnabled(True) + def disableValidationButton(self): + self.PValidate.setEnabled(False) + def disable_PsyncDB(self): self.PsyncDB.setText("Sync Running...") self.PsyncDB.setEnabled(False)