job-scrapper/lib/db.py

94 lines
3.7 KiB
Python
Raw Normal View History

2024-06-13 09:14:04 +00:00
import sqlite3
import mmh3
import sys
DEBUG = False
def log(*s):
if DEBUG:
print(s)
def initdb(file):
with sqlite3.connect(file) as connection:
print("db connection", connection.total_changes)
cursor = connection.cursor()
cursor.execute("CREATE TABLE jobs (star TEXT,tag INT ,title TEXT, location TEXT, company TEXT,link TEXT,pubdate TEXT,hash INT)")
sys.exit()
def rmdb(file,table):
with sqlite3.connect(file) as connection:
question = input("Do you really wont to empty the db(press Y)?")
if(question == "Y"):
cursor = connection.cursor()
drop_cmd = f"""DROP TABLE {table}"""
cursor.execute(drop_cmd)
else:
print("abroting removing table")
sys.exit()
def importdb(file,importdb,table):
with sqlite3.connect(file) as connection:
print("db connection",connection.total_changes)
cmd = f"""ATTACH "{importdb}" AS regions"""
cmd2 = f"""CREATE TABLE IF NOT EXISTS {table} AS SELECT * from regions.{table}"""
cmd_view = f"""
CREATE VIEW Canoton_Filter
AS
SELECT * FROM jobs as b
WHERE EXISTS
(SELECT GDENAME FROM {table} as w
where w.GDEKT = 'ZH' AND
b.location LIKE GDENAME);"""
cursor = connection.cursor()
cursor.execute(cmd)
print(cmd,cmd2)
cursor.execute(cmd2)
cursor.execute(cmd_view)
print("db connection",connection.total_changes)
def createnwview(file):
with sqlite3.connect(file) as connection:
2024-06-18 07:58:54 +00:00
cmd_create_nw_table = f"""CREATE VIEW IF NOT EXISTS "Nordwest-SCHWEIZ" AS SELECT * FROM jobs as b
2024-06-13 09:14:04 +00:00
WHERE EXISTS
(SELECT GDENAME FROM Cantons as w
where w.GDEKT = 'ZH' AND
b.location LIKE GDENAME)
OR EXISTS
(SELECT GDENAME FROM Cantons as w
where w.GDEKT = 'AG' AND
b.location LIKE GDENAME)
OR EXISTS
(SELECT GDENAME FROM Cantons as w
where w.GDEKT = 'SO' AND
b.location LIKE GDENAME)"""
cursor = connection.cursor()
cursor.execute(cmd_create_nw_table)
print("db connection",connection.total_changes)
2024-06-18 07:58:54 +00:00
createFilterTable(file)
def createFilterTable(file):
with sqlite3.connect(file) as connection:
cmd_create_filter_table = f"""CREATE TABLE IF NOT EXISTS filters(cmd TEXT);"""
cursor = connection.cursor()
cursor.execute(cmd_create_filter_table)
print("db connection:",connection.total_changes)
def addFineFilter(file,table,filterstr):
with sqlite3.connect(file) as connection:
2024-06-18 10:52:18 +00:00
cmd_createFineFilter = f"""INSERT INTO {table}(cmd) VALUES(?);"""
cmd_checkIfExists = f"""SELECT * FROM {table} WHERE cmd = ?"""
2024-06-18 07:58:54 +00:00
cursor = connection.cursor()
2024-06-18 10:52:18 +00:00
if cursor.execute(cmd_checkIfExists,(filterstr,)).fetchone() == None:
cursor.execute(cmd_createFineFilter,(filterstr,))
2024-06-13 09:14:04 +00:00
def writedb(jobs):
with sqlite3.connect("../db/sqlite3.db") as connection:
print("db connection", connection.total_changes)
cursor = connection.cursor()
# cursor.execute("CREATE TABLE jobs (title TEXT, location TEXT, company TEXT,link TEXT,hash INT)")
for i3,job in enumerate(jobs):
hash1 = mmh3.hash(job.title+job.company+job.location+job.date)
log(hash1);
if(cursor.execute("SELECT * FROM jobs WHERE hash = ?",(hash1,)).fetchone() != None):
log("Hash already exist")
else:
print("NEW_ENTRY")
cursor.execute("INSERT INTO jobs (star,tag,title,company,location,link,pubdate,hash) VALUES (?,?,?,?,?,?,?,?)",(job.starred,job.tag,job.title,job.company,job.location,job.link,job.date,hash1))