ccppi/nim-basic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

comparis has a strange dynamicaly loading js bullschit thing therefore the scrapper works but does not get all the data which is seen in the browser

Browse Source
This commit is contained in:
ccppi 2024-07-02 14:00:19 +02:00
parent f629ff784f
commit a2a0001aef
3 changed files with 27 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
main
View File

Binary file not shown.

4
main.nim
View File

@ -10,7 +10,7 @@ var html: string
#var node: XmlNode #var node: XmlNode
echo "URL:" echo "URL:"
var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten"#readLine(stdin) var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
echo "given url is: ",url echo "given url is: ",url
try: try:
@ -20,7 +20,7 @@ try:
var htmlnode: XmlNode var htmlnode: XmlNode
var entry : Entry var entry : Entry
add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-1wvv8q", html_tag : "a",aattrs : "href")) add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
#add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href")) #add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href"))
echo entry echo entry

19
scrap.nim
View File

@ -12,6 +12,7 @@ type
html_context_key* : string html_context_key* : string
contains_string* : string contains_string* : string
aattrs* : string aattrs* : string
attrs_key*: string
content* : seq[string] content* : seq[string]
type type
Entry* = object Entry* = object
@ -40,21 +41,26 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
if c.attrs.hasKey(entry.desc[i].html_context_attrs): if c.attrs.hasKey(entry.desc[i].html_context_attrs):
echo("c.attrs[key]: ",c.attrs[entry.desc[i].html_context_attrs]) echo("c.attrs[key]: ",c.attrs[entry.desc[i].html_context_attrs])
if entry.desc[i].html_context_key != "": if entry.desc[i].html_context_key != "":
if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]): #if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]):
if c.attrs[entry.desc[i].html_context_attrs].contains(entry.desc[i].html_context_key):
context.add(c) context.add(c)
echo("context_key_ MATCH") echo("context_key_ MATCH")
else:
echo("key does NOT match")
else: else:
context.add(c) context.add(c)
else: else:
echo("c.attrs does not have the key") echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
else: else:
echo("no attributes on this tag") echo("no attributes on this tag")
echo("out of for loop") echo("----------CONTEXT RESULT")
echo(context)
#end filter context #end filter context
for a in context: for a in context:
let subContext = a.findAll(entry.desc[i].html_tag) let subContext = a.findAll(entry.desc[i].html_tag)
for b in subContext: for b in subContext:
if entry.desc[i].aattrs != "": if entry.desc[i].aattrs != "":
if b.attrs != nil:
if b.attrs.hasKey(entry.desc[i].aattrs): if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b) echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs]) echo("value: ",b.attrs[entry.desc[i].aattrs])
@ -62,8 +68,13 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
var content_txt :string = b.innerText() var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt) echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string): if content_txt.contains(entry.desc[i].contains_string):
echo("PATTERN MATCH") echo("ATTRS MATCH")
entry.desc[i].content.add(content_txt) entry.desc[i].content.add(content_txt)
elif entry.desc[i].attrs_key != "":
#if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
echo("KEY_MATCH")
entry.desc[i].content.add(b.innerText())
else: else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs]) entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
elif entry.desc[i].contains_string != "": elif entry.desc[i].contains_string != "":