303 lines
9.8 KiB
Python
303 lines
9.8 KiB
Python
import string
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from enum import Enum
|
|
import re
|
|
import shutil
|
|
from dateconverter import *
|
|
from datetime import datetime
|
|
import os
|
|
import sqlite3
|
|
import webbrowser
|
|
from time import sleep
|
|
|
|
DEBUG = True
|
|
number = ['0','1','2','3','4','5','6','7','8','9']
|
|
homePath = os.path.expanduser('~')
|
|
cookiePath = homePath + "/.mozilla/firefox/imibizoh.default/cookies.sqlite"
|
|
tmpPath = "/tmp/cookies.sqlite"
|
|
DBFILE = "../db/sqlite3.db"
|
|
winFirefoxPath = f"""C:\Program Files\Mozilla Firefox\firefox.exe"""
|
|
def log(*s):
|
|
if DEBUG:
|
|
print(s)
|
|
class mode():
|
|
#def set(self,flag,value):
|
|
# self.flag = flag
|
|
# self.value = value
|
|
#def __init__(self,):
|
|
DEFAULT = 0
|
|
LINK = 0
|
|
LOCATION_CLEANUP = 0
|
|
SWAPDATE = 0
|
|
CLEANDATE = 0
|
|
ATTRS = 0
|
|
months = [
|
|
('January','01'),
|
|
('February','02'),
|
|
('March','03'),
|
|
('April','04'),
|
|
('May','05'),
|
|
('June','06'),
|
|
('July','07'),
|
|
('August','08'),
|
|
('September','09'),
|
|
('October','10'),
|
|
('November','11'),
|
|
('December','12')]
|
|
class item():
|
|
def __init__(self,tag,tag_content,index,name=None):
|
|
self.tag = tag
|
|
self.tag_content = tag_content
|
|
self.index = index
|
|
if name is not None:
|
|
self.name = name
|
|
else:
|
|
self.name = "not defined"
|
|
|
|
class job():
|
|
def __init__(self,title,profession,company,location,date,description,link,tag,starred):
|
|
self.title = title
|
|
self.profession = profession
|
|
self.company = company
|
|
self.location = location
|
|
self.date = date
|
|
self.description = description
|
|
self.link = link
|
|
self.tag = tag
|
|
self.starred = starred
|
|
def __str__(self):
|
|
return "%s| %s|%s|%s|%s|%s|%s" % (self.title,self.profession,self.company,self.location,self.date,self.description,self.link)
|
|
|
|
def finder(results,item,**modes):
|
|
GETCHILDREN = modes.get("GETCHILDREN",'')
|
|
ATTRS = modes.get('ATTRS',0)
|
|
LOCATION_CLEANUP = modes.get('LOCATION_CLEANUP',0)
|
|
LINK = modes.get('LINK',0)
|
|
SWAPDATE = modes.get('SWAPDATE',0)
|
|
CLEANDATE = modes.get('CLEANDATE',0)
|
|
BASEURL = modes.get('BASEURL','')
|
|
INDEEDDATE = modes.get('INDEEDDATE',0)
|
|
content = []
|
|
i = item.index
|
|
log("name",item.name)
|
|
log("Item tag: ",item.tag)
|
|
log("Modes:",modes)
|
|
log("tag_content: ",item.tag_content)
|
|
for entry in results:
|
|
if ATTRS==1:
|
|
result = entry.findAll(item.tag,attrs=item.tag_content)
|
|
log(item.tag_content)
|
|
else:
|
|
result = entry.findAll(item.tag,class_=item.tag_content)
|
|
log("found count results:",len(result))
|
|
if len(result)==0 and DEBUG == True:
|
|
log("len result: ",len(result))
|
|
for x in result:
|
|
log("No entry found for: ",item.name,item.tag,item.tag_content," -->", x)
|
|
input()
|
|
if result:
|
|
log("theres a result")
|
|
if i>(len(result)-1):
|
|
log("len:",len(result)-1,"i:",i)
|
|
log("index out of bounds fall back to the %d count",i)
|
|
i=(len(result)-1)
|
|
result2 = result[i]
|
|
if GETCHILDREN != '':
|
|
found = False
|
|
for results in result:
|
|
child = results.find(GETCHILDREN)
|
|
log("[finder] search for '",GETCHILDREN,"' in: ",child)
|
|
if child != None and found == False:
|
|
log("CHILD text strip: ",child.text.strip())
|
|
found = True
|
|
content.append(child.text.strip())
|
|
if found == False:
|
|
log("[finder] No matching Child found: ",child)
|
|
content.append("CHILD_NOT_FOUND: " + GETCHILDREN)
|
|
|
|
elif LOCATION_CLEANUP==1:
|
|
location = CleanLocation(result2.text.strip())
|
|
content.append(location)
|
|
elif LINK==1:
|
|
string = result2.get("href")
|
|
if BASEURL:
|
|
string = BASEURL+string
|
|
content.append(string)
|
|
elif SWAPDATE==1:
|
|
content.append(DateCHToUS(result2.text.strip()))
|
|
elif CLEANDATE==1:
|
|
log("[finder] pre cleandate:",result2.text.strip)
|
|
content.append(jobs_ch_clean_date(result2.text.strip()))
|
|
elif INDEEDDATE==1:
|
|
log("[finder] pre indeeddate:",result2.text.strip)
|
|
content.append(indeedExtractDays(result2.text.strip()))
|
|
else:
|
|
log(result2)
|
|
content.append(result2.text.strip())
|
|
if not result:
|
|
if item.tag_content == "pubdate":
|
|
today = datetime.today().strftime('%Y-%m-%d')
|
|
content.append(today)
|
|
else:
|
|
content.append("NOTFound")
|
|
return content
|
|
|
|
|
|
def CleanLocation(location):
|
|
#p = re.compile('CH-[0-9]{4}')
|
|
location = re.sub('CH-[0-9]{4}|[0-9]{4}| ','',location)
|
|
return location
|
|
|
|
def arrayToClass(titles,companys,locations,dates,links,tag):
|
|
jobs = []
|
|
if(len(titles) == len(companys) == len(locations) == len(dates)):
|
|
log("len:",len(titles))
|
|
for i, title in enumerate(titles):
|
|
jobs.append(job(title,"test_prof",companys[i],locations[i],dates[i],"test_desc",links[i],tag,0))
|
|
log("class job:",jobs[i])
|
|
return jobs
|
|
else:
|
|
log("Something went wrong unequal length of data arrays: ",len(titles),len(companys),len(locations),len(dates))
|
|
return 0
|
|
def jobs_ch_clean_date(date):
|
|
newdate=''
|
|
|
|
for i in range(11,len(date)):#remove string "Published:"
|
|
newdate+=date[i]
|
|
|
|
newdate2 = jobs_ch_switch_month(newdate)
|
|
return newdate2
|
|
|
|
def jobs_ch_switch_month(date):
|
|
newdate=''
|
|
newmonth=''
|
|
day = ''
|
|
year = ''
|
|
|
|
for i in range(3,len(date)-5):
|
|
newmonth += date[i]
|
|
for month in months:
|
|
if(month[0] == newmonth):
|
|
newmonth = month[1]
|
|
|
|
for i in range(0,2):
|
|
day+=date[i]
|
|
for i in range(len(date)-2,len(date)):
|
|
year += date[i]
|
|
newdate = '20'+year+'-'+newmonth+'-'+day
|
|
return newdate
|
|
|
|
def CleanLocation(location):
|
|
#p = re.compile('CH-[0-9]{4}')
|
|
location = re.sub('CH-[0-9]{4}|[0-9]{4}| ','',location)
|
|
return location
|
|
|
|
def extractDomain(url):
|
|
pattern = r'https:\/\/.*\..+?(?=\/)'
|
|
domain = re.match(pattern,url)
|
|
if domain:
|
|
return domain.group()
|
|
else:
|
|
return 0
|
|
|
|
def makeSession(url):
|
|
with requests.Session() as session:
|
|
page = session.get(url)
|
|
return session
|
|
def indeedExtractDays(datestr):
|
|
cleannumstr=''
|
|
cleannumint=-1
|
|
cleandate=''
|
|
foundchar=False
|
|
for a in datestr:
|
|
print(a)
|
|
if a in number and foundchar==False:
|
|
foundchar=True
|
|
cleannumstr+=a
|
|
elif a in number and foundchar == True:
|
|
cleannumstr+=a
|
|
elif a not in number and foundchar == True:
|
|
break
|
|
if cleannumstr != '':
|
|
cleannumint = int(cleannumstr)
|
|
today = int(datetime.utcnow().timestamp())
|
|
cleandate = today - cleannumint * 60 * 60 * 7 * 4
|
|
#print("int:",cleannumint,"today:",today,"cleandate:",datetime.fromtimestamp(cleandate).strftime('%Y-%m-%d'))
|
|
return datetime.fromtimestamp(cleandate).strftime('%Y-%m-%d')
|
|
return "NOTFound"
|
|
def getCookiesFromBrowser(url):
|
|
#workaround for loked database
|
|
tries=0
|
|
cookie = ''
|
|
rows = [0]
|
|
while(cookie == '' and tries < 2):
|
|
tries+=1;
|
|
shutil.copyfile(cookiePath,tmpPath)
|
|
with sqlite3.connect(tmpPath) as connection:
|
|
cmd_read_cookies = f"""SELECT name,value FROM moz_cookies WHERE host like ?;"""
|
|
print(cmd_read_cookies)
|
|
cursor = connection.cursor()
|
|
cursor.execute(cmd_read_cookies,(urlToDomain(url),))
|
|
while len(rows)!=0:
|
|
rows = cursor.fetchmany(25)
|
|
for row in rows:
|
|
print("row:",row)
|
|
cookie = cookie + row[0] + '=' + row[1]
|
|
cookie += ";"
|
|
|
|
print("Cookies:",cookie)
|
|
if cookie == '':
|
|
if os.name == 'posix':
|
|
webbrowser.register("firefox",None,webbrowser.BackgroundBrowser("firefox"))
|
|
webbrowser.get('firefox').open(url)
|
|
elif os.name == 'nt':
|
|
webbrowser.register("firefox",None,webbrowser.BackgroundBrowser(winFirefoxPath))
|
|
webbrowser.get('firefox').open(url)
|
|
sleep(1)
|
|
return cookie
|
|
#access cookies from firefox:
|
|
#copy (because locked): cp .mozilla/firefox/imibizoh.default/cookies.sqlite cookies.sqlite
|
|
#Select value from moz_cookies where host like '%indeed%'
|
|
def urlToDomain(url):
|
|
pos = patternSearch(url,"https://")
|
|
urlCut = dropBeforePos(url,pos)
|
|
posDot = skipAfterChar(urlCut,'.') - 1
|
|
urlCut = dropBeforePos(urlCut,posDot)
|
|
posDot = skipAfterChar(urlCut,'/')
|
|
urlCut = dropAfterPos(urlCut,posDot)
|
|
print("url after cut dot:",urlCut)
|
|
return urlCut
|
|
|
|
def patternSearch(url,pattern):
|
|
x = 0
|
|
for a,i in enumerate(url):
|
|
print("i:",i)
|
|
if i == pattern[x]:
|
|
if x<len(pattern)-1:
|
|
x = x + 1
|
|
elif x==len(pattern)-1:
|
|
print("FULL PATTERN FOUND at pos :",a)
|
|
break
|
|
else:
|
|
x = 0
|
|
return a
|
|
def skipAfterChar(aString,char):
|
|
for a,i in enumerate(aString):
|
|
if i == char:
|
|
break
|
|
return a
|
|
def dropBeforePos(aString,pos):
|
|
aString2=''
|
|
pos+=1
|
|
if pos < len(aString):
|
|
for i in range(pos,len(aString)):
|
|
aString2 += aString[i]
|
|
return aString2
|
|
def dropAfterPos(aString,pos):
|
|
aString2=''
|
|
if pos < len(aString):
|
|
for i in range(0,pos):
|
|
aString2 += aString[i]
|
|
return aString2
|