job-scrapper/lib/sysparse.py

import argparse
import config
import sys
from enum import IntEnum
from scrap_jobs import *
from login import *
from time import sleep
from db import *

DEBUG = False
def log(*s):
    if DEBUG:
        print(s)

def choose_scraper(entry,session):
    if not session:
        session = requests.Session()
    domain = extractDomain(entry.scrapurl)
    match domain:
        case 'https://www.jobs.ch':
            runner(entry,session,scrap_jobs,next_url_jobs_ch)
        case 'https://software-job.ch':
            runner(entry,session,scrap_jobagent,next_url_jobagent)
        case 'https://www.jobagent.ch':
            runner(entry,session,scrap_jobagent,next_url_jobagent)
        case 'https://ch.indeed.com':
            runner(entry,session,scrap_indeed_com,next_url_indeed_com)

def parse(**kwargs):
    session=0 
    if len(sys.argv)>1:
        worker=0
        parser = argparse.ArgumentParser()
        parser.add_argument("-c","--config",help = "Specific a config file to use,from where to scrap the jobs")
        parser.add_argument("-t","--test",help = "only for test purposes while developing",action="store_true")
        parser.add_argument("--importregiondb",help = "Import a database used for querring by Regions or Cantons",action="store_true")
        parser.add_argument("--initdb",help = "Initialice a new db from scratch without entrys",action="store_true")
        parser.add_argument("--rmdb",help = "!!reove existing db!!DATALOSS!!",action="store_true")
       # parser.add_argument("--help",help = "print help")
        parser.add_argument("--login",nargs=3,help = "login by specifing login and passwor by a given url",metavar=('USERNAME','PASSWORD','URL'))
        parser.add_argument("--createnwview",help = "Create a VIEW for the Region Nordwest Schweiz",action="store_true")
        parser.add_argument("-VC","--ValidationCheck",help = "Check if links are still valid, if not remove them",action="store_true")
        args = parser.parse_args()

        if args.test:
            addFineFilter("../db/sqlite3.db","filters","testfilterentry")
        if args.importregiondb:
            importdb("../db/sqlite3.db","../db/Cantons.db","Cantons")
        if args.initdb:
            initdb("../db/sqlite3.db")
        if args.rmdb:
            rmdb("../db/sqlite3.db","jobs")
        if args.login:
            user,pw,url = args.login
            session = login(user,pw,url,url)
            choose_scraper(url,session)
        if args.config:
            login_loop(args.config,False,worker)
        if args.createnwview:
            createnwview("../db/sqlite3.db")
        if args.ValidationCheck:
            isStillValid("../db/sqlite3.db")
    if len(kwargs)>0:
        print("no sysargs fiven, running as a module")
        vconfig = kwargs.get('config')
        worker = kwargs.get('worker')
        print("config:",vconfig)
        if vconfig:
            login_loop(vconfig,True,worker)
            worker.finished.emit()
        print("finished sync job")


def login_loop(config_file,gui,worker):
            ret = -1
            ret_login = 0
            session = 0
            while (ret != 0):
                ret = entry2 = config.readConfig(config_file,gui,worker)
                print(entry2)
                if(ret != 0 and ret_login != 1):
                    if(entry2.loginurl != 'NONE'):
                        session = -1
                        while session == -1:
                            session = login(entry2)
                            if session == -1:
                                ret_login = entry2.input_pw(gui,entry2.user,worker)
                    choose_scraper(entry2,session)

def runner(entry,session,scrap_func,next_url_func):
    i=0
    b_url = entry.scrapurl
    while b_url != 0 and i<50:
        sleep(0.3)
        if b_url:
            domain = extractDomain(b_url)
            print(domain)
            if domain == 'https://www.jobagent.ch' or domain == 'https://software-job.ch':
                jobs = scrap_func(b_url,entry,session)
                if jobs:
                    writedb(jobs)
                else:
                    print("nothing found on this page")
                b_url = next_url_func(b_url,session,0)
            elif domain == 'https://www.jobs.ch':
                jobs = scrap_func(b_url,entry,session)
                if jobs:
                    writedb(jobs)
                else:
                    print("nothing found on this page")
                b_url = next_url_func(b_url,session,"https://www.jobs.ch")
            elif domain == 'https://ch.indeed.com':
                jobs = scrap_func(b_url,entry,session)
                if jobs:
                    writedb(jobs)
                else:
                    print("nothing found on this page")
                b_url = next_url_func(b_url,session,domain)



            if b_url != 0:
                print("main:" + b_url)
            if b_url==0:
                print("End of listed items, or did not find any other Nächste Seite Buttons")

        i=i+1
        print(i)