job-scrapper/lib/db.py

import sqlite3
import mmh3
import sys
DEBUG = False

def log(*s):
    if DEBUG:
        print(s)
def initdb(file):
    with sqlite3.connect(file) as connection:
        print("db connection", connection.total_changes)
        cursor = connection.cursor()
        cursor.execute("CREATE TABLE jobs (star TEXT,tag INT ,title TEXT, location TEXT, company TEXT,link TEXT,pubdate TEXT,hash INT)")
        sys.exit()
def rmdb(file,table):
    with sqlite3.connect(file) as connection:
        question = input("Do you really wont to empty the db(press Y)?")
        if(question == "Y"):
            cursor = connection.cursor()
            drop_cmd = f"""DROP TABLE {table}"""
            cursor.execute(drop_cmd)
        else:
            print("abroting removing table")
            sys.exit()
def importdb(file,importdb,table):
    with sqlite3.connect(file) as connection:
        print("db connection",connection.total_changes)
        
        cmd = f"""ATTACH "{importdb}" AS regions"""
        cmd2 = f"""CREATE TABLE IF NOT EXISTS {table} AS SELECT * from regions.{table}"""
        cmd_view = f"""
        CREATE VIEW Canoton_Filter
        AS
        SELECT * FROM jobs as b 
        WHERE EXISTS 
        (SELECT GDENAME FROM {table} as w  
        where w.GDEKT = 'ZH' AND 
        b.location LIKE GDENAME);"""
        cursor = connection.cursor()
        cursor.execute(cmd)
        print(cmd,cmd2)
        cursor.execute(cmd2)
        cursor.execute(cmd_view)

        print("db connection",connection.total_changes)

def createnwview(file):
    with sqlite3.connect(file) as connection: 
        cmd_create_nw_table = f"""CREATE VIEW IF NOT EXISTS "Nordwest-SCHWEIZ" AS SELECT * FROM jobs as b 
        WHERE EXISTS 
        (SELECT GDENAME FROM Cantons as w  
        where w.GDEKT = 'ZH' AND 
        b.location LIKE GDENAME)
		OR EXISTS
		(SELECT GDENAME FROM Cantons as w  
        where w.GDEKT = 'AG' AND 
        b.location LIKE GDENAME)
		OR EXISTS	
		(SELECT GDENAME FROM Cantons as w  
        where w.GDEKT = 'SO' AND 
        b.location LIKE GDENAME)"""
        cursor = connection.cursor()
        cursor.execute(cmd_create_nw_table)
        print("db connection",connection.total_changes)
        createFilterTable(file)

def createFilterTable(file):
    with sqlite3.connect(file) as connection:
        cmd_create_filter_table = f"""CREATE TABLE IF NOT EXISTS filters(cmd TEXT);"""
        cursor = connection.cursor()
        cursor.execute(cmd_create_filter_table)
        print("db connection:",connection.total_changes)
def addFineFilter(file,table,filterstr):
    with sqlite3.connect(file) as connection:
        cmd_createFineFilter = f"""INSERT INTO {table}(cmd) VALUES(?);"""
        cmd_checkIfExists = f"""SELECT * FROM {table} WHERE cmd = ?"""
        cursor = connection.cursor()
        if cursor.execute(cmd_checkIfExists,(filterstr,)).fetchone() == None:
            cursor.execute(cmd_createFineFilter,(filterstr,))

def writedb(jobs):
        with sqlite3.connect("../db/sqlite3.db") as connection:
            print("db connection", connection.total_changes)
            cursor = connection.cursor()
        #    cursor.execute("CREATE TABLE jobs (title TEXT, location TEXT, company TEXT,link TEXT,hash INT)")
            for i3,job in enumerate(jobs):
                hash1 = mmh3.hash(job.title+job.company+job.location+job.date)
                log(hash1);
                if(cursor.execute("SELECT * FROM jobs WHERE hash = ?",(hash1,)).fetchone() != None):
                    log("Hash already exist")
                else:
                    print("NEW_ENTRY")
                    cursor.execute("INSERT INTO jobs (star,tag,title,company,location,link,pubdate,hash) VALUES (?,?,?,?,?,?,?,?)",(job.starred,job.tag,job.title,job.company,job.location,job.link,job.date,hash1))