helpers.py

lib/helpers.py

@@ -8,6 +8,8 @@ from dateconverter import *
 from datetime import datetime
 import os
 import sqlite3
+import webbrowser
+from time import sleep
 
 DEBUG = True
 number = ['0','1','2','3','4','5','6','7','8','9']
@@ -15,6 +17,7 @@ homePath = os.path.expanduser('~')
 cookiePath = homePath + "/.mozilla/firefox/imibizoh.default/cookies.sqlite"
 tmpPath = "/tmp/cookies.sqlite"
 DBFILE = "../db/sqlite3.db"
+winFirefoxPath = f"""C:\Program Files\Mozilla Firefox\firefox.exe"""
 def log(*s):
     if DEBUG:
         print(s)
@@ -225,23 +228,33 @@ def indeedExtractDays(datestr):
     return "NOTFound"
 def getCookiesFromBrowser(url):
     #workaround for loked database
-    shutil.copyfile(cookiePath,tmpPath)
+    tries=0
     cookie = ''
     rows = [0]
+    while(cookie == '' and tries < 2):
+        tries+=1;
+        shutil.copyfile(cookiePath,tmpPath)
+        with sqlite3.connect(tmpPath) as connection:
+            cmd_read_cookies = f"""SELECT name,value FROM moz_cookies WHERE host like ?;"""
+            print(cmd_read_cookies)
+            cursor = connection.cursor()
+            cursor.execute(cmd_read_cookies,(urlToDomain(url),))
+            while len(rows)!=0:
+                rows = cursor.fetchmany(25)
+                for row in rows:
+                    print("row:",row)
+                    cookie = cookie + row[0] + '=' + row[1]
+                    cookie += ";"
 
-    with sqlite3.connect(tmpPath) as connection:
+            print("Cookies:",cookie)
-        cmd_read_cookies = f"""SELECT name,value FROM moz_cookies WHERE host like ?;"""
+            if cookie == '':
-        print(cmd_read_cookies)
+                if os.name == 'posix':
-        cursor = connection.cursor()
+                    webbrowser.register("firefox",None,webbrowser.BackgroundBrowser("firefox"))
-        cursor.execute(cmd_read_cookies,(urlToDomain(url),))
+                    webbrowser.get('firefox').open(url)
-        while len(rows)!=0:
+                elif os.name == 'nt':
-            rows = cursor.fetchmany(25)
+                    webbrowser.register("firefox",None,webbrowser.BackgroundBrowser(winFirefoxPath))
-            for row in rows:
+                    webbrowser.get('firefox').open(url)
-                print("row:",row)
+                sleep(1)
-                cookie = cookie + row[0] + '=' + row[1]
-                cookie += ";"
-
-        print("Cookies:",cookie)
     return cookie
 #access cookies from firefox:
 #copy (because locked): cp .mozilla/firefox/imibizoh.default/cookies.sqlite cookies.sqlite

lib/scrap_jobs.py

@@ -25,7 +25,7 @@ def scrap_indeed_com(url,entry,session):
         log(page)
     solveCaptcha(session,page)
     soup = BeautifulSoup(page.content,"html.parser")
-    print(soup.prettify())
+    #print(soup.prettify())
 
 
     results = soup.find_all("li",class_= 'css-5lfssm eu4oa1w0') #top level list element