ccppi/job-scrapper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

helpers.py

Browse Source
This commit is contained in:
ccppi 2024-08-22 11:59:40 +02:00
parent 850538a92f
commit a2b97e9f21
2 changed files with 28 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/helpers.py
View File

@ -8,6 +8,8 @@ from dateconverter import *
from datetime import datetime from datetime import datetime
import os import os
import sqlite3 import sqlite3
import webbrowser
from time import sleep
DEBUG = True DEBUG = True
number = ['0','1','2','3','4','5','6','7','8','9'] number = ['0','1','2','3','4','5','6','7','8','9']
@ -15,6 +17,7 @@ homePath = os.path.expanduser('~')
cookiePath = homePath + "/.mozilla/firefox/imibizoh.default/cookies.sqlite" cookiePath = homePath + "/.mozilla/firefox/imibizoh.default/cookies.sqlite"
tmpPath = "/tmp/cookies.sqlite" tmpPath = "/tmp/cookies.sqlite"
DBFILE = "../db/sqlite3.db" DBFILE = "../db/sqlite3.db"
winFirefoxPath = f"""C:\Program Files\Mozilla Firefox\firefox.exe"""
def log(*s): def log(*s):
if DEBUG: if DEBUG:
print(s) print(s)
@ -225,10 +228,12 @@ def indeedExtractDays(datestr):
return "NOTFound" return "NOTFound"
def getCookiesFromBrowser(url): def getCookiesFromBrowser(url):
#workaround for loked database #workaround for loked database
shutil.copyfile(cookiePath,tmpPath) tries=0
cookie = '' cookie = ''
rows = [0] rows = [0]
while(cookie == '' and tries < 2):
tries+=1;
shutil.copyfile(cookiePath,tmpPath)
with sqlite3.connect(tmpPath) as connection: with sqlite3.connect(tmpPath) as connection:
cmd_read_cookies = f"""SELECT name,value FROM moz_cookies WHERE host like ?;""" cmd_read_cookies = f"""SELECT name,value FROM moz_cookies WHERE host like ?;"""
print(cmd_read_cookies) print(cmd_read_cookies)
@ -242,6 +247,14 @@ def getCookiesFromBrowser(url):
cookie += ";" cookie += ";"
print("Cookies:",cookie) print("Cookies:",cookie)
if cookie == '':
if os.name == 'posix':
webbrowser.register("firefox",None,webbrowser.BackgroundBrowser("firefox"))
webbrowser.get('firefox').open(url)
elif os.name == 'nt':
webbrowser.register("firefox",None,webbrowser.BackgroundBrowser(winFirefoxPath))
webbrowser.get('firefox').open(url)
sleep(1)
return cookie return cookie
#access cookies from firefox: #access cookies from firefox:
#copy (because locked): cp .mozilla/firefox/imibizoh.default/cookies.sqlite cookies.sqlite #copy (because locked): cp .mozilla/firefox/imibizoh.default/cookies.sqlite cookies.sqlite

2
lib/scrap_jobs.py
View File

@ -25,7 +25,7 @@ def scrap_indeed_com(url,entry,session):
log(page) log(page)
solveCaptcha(session,page) solveCaptcha(session,page)
soup = BeautifulSoup(page.content,"html.parser") soup = BeautifulSoup(page.content,"html.parser")
print(soup.prettify()) #print(soup.prettify())
results = soup.find_all("li",class_= 'css-5lfssm eu4oa1w0') #top level list element results = soup.find_all("li",class_= 'css-5lfssm eu4oa1w0') #top level list element