diff --git a/main b/main index feb59cd..59fc770 100755 Binary files a/main and b/main differ diff --git a/main.nim b/main.nim index 8672616..970affc 100644 --- a/main.nim +++ b/main.nim @@ -10,7 +10,7 @@ var html: string #var node: XmlNode echo "URL:" -var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten"#readLine(stdin) +var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin) echo "given url is: ",url try: @@ -20,7 +20,7 @@ try: var htmlnode: XmlNode var entry : Entry - add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-1wvv8q", html_tag : "a",aattrs : "href")) + add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u")) #add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href")) echo entry diff --git a/scrap.nim b/scrap.nim index c38b74f..f3ce1c6 100644 --- a/scrap.nim +++ b/scrap.nim @@ -12,6 +12,7 @@ type html_context_key* : string contains_string* : string aattrs* : string + attrs_key*: string content* : seq[string] type Entry* = object @@ -40,32 +41,42 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry = if c.attrs.hasKey(entry.desc[i].html_context_attrs): echo("c.attrs[key]: ",c.attrs[entry.desc[i].html_context_attrs]) if entry.desc[i].html_context_key != "": - if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]): + #if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]): + if c.attrs[entry.desc[i].html_context_attrs].contains(entry.desc[i].html_context_key): context.add(c) echo("context_key_ MATCH") + else: + echo("key does NOT match") else: context.add(c) else: - echo("c.attrs does not have the key") + echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs) else: echo("no attributes on this tag") - echo("out of for loop") + echo("----------CONTEXT RESULT") + echo(context) #end filter context for a in context: let subContext = a.findAll(entry.desc[i].html_tag) for b in subContext: if entry.desc[i].aattrs != "": - if b.attrs.hasKey(entry.desc[i].aattrs): - echo("found key [",entry.desc[i].aattrs,"] in : ",b) - echo("value: ",b.attrs[entry.desc[i].aattrs]) - if entry.desc[i].contains_string != "": - var content_txt :string = b.innerText() - echo(" and string(innerText): ",content_txt) - if content_txt.contains(entry.desc[i].contains_string): - echo("PATTERN MATCH") - entry.desc[i].content.add(content_txt) - else: - entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs]) + if b.attrs != nil: + if b.attrs.hasKey(entry.desc[i].aattrs): + echo("found key [",entry.desc[i].aattrs,"] in : ",b) + echo("value: ",b.attrs[entry.desc[i].aattrs]) + if entry.desc[i].contains_string != "": + var content_txt :string = b.innerText() + echo(" and string(innerText): ",content_txt) + if content_txt.contains(entry.desc[i].contains_string): + echo("ATTRS MATCH") + entry.desc[i].content.add(content_txt) + elif entry.desc[i].attrs_key != "": + #if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]): + if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key): + echo("KEY_MATCH") + entry.desc[i].content.add(b.innerText()) + else: + entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs]) elif entry.desc[i].contains_string != "": var content_txt = b.innerText() echo(" string(innerText): ",content_txt)