This commit is contained in:
ccppi 2024-07-03 14:03:25 +02:00
parent a2a0001aef
commit 42bd90fb7b
4 changed files with 1981 additions and 41 deletions

BIN
main

Binary file not shown.

View File

@ -4,11 +4,34 @@ import std/xmltree
import std/strtabs
import std/os
import scrap
import std/json
import std/strutils
var client = newHttpClient()
var html: string
#var node: XmlNode
proc do_another_loop( node : JsonNode, level : int, searchString : string = "") =
if node.len() > 0:
for a in node.keys:
if node.kind == JObject:
#for i in 0 .. level:
if searchString == "" or contains($node[a],searchString):
stdout.write ">"
stdout.write a
if node[a].kind == JObject:
do_another_loop(node[a],level+1,searchString)
else:
if searchString == "" or contains($node[a],searchString):
stdout.write " [", node[a].kind, "]\n"
if node[a].kind == JArray:
if searchString == "" or contains($node[a],searchString):
echo node[a]
else:
echo "yyyyyyyy"
else:
echo "\n--------"
echo "URL:"
var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
echo "given url is: ",url
@ -21,19 +44,33 @@ try:
var entry : Entry
add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
#add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href"))
#comparis stores part of the data in a script tag as a json-string, so we get that part:
add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
echo entry
echo("###########")
echo entry.getEntryFromHtml(node)
# echo node
# for a in node.findAll("a"):
# if a.attrs.hasKey "href":
# echo "striping"
# let (dir,filename,ext) = splitFile(a.attrs["href"])
# echo "found a link!",dir & "/" & filename
# else:
# echo "Key has no attribute href"
discard entry.getEntryFromHtml(node)
#echo entry.desc[1].content[0]
let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
let jsonNode = parseJson(jsonString)
#echo "jsonString: ",jsonString
#echo "JSON Node:", jsonNode
#echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
let field = jsonNode.getFields()
do_another_loop(jsonNode,0,"resultitems")
#echo "resultitem", jsonNode["resultItems"]
# for a in jsonNode.keys:
# echo a
# if jsonNode[a].len() > 0:
# for b in jsonNode[a].keys:
# echo ">>>"
# do_another_loop(jsonNode[a][b])
# else:
# echo "---------------"
finally:
client.close()

1896
out.log Normal file

File diff suppressed because one or more lines are too long

View File

@ -14,6 +14,7 @@ type
aattrs* : string
attrs_key*: string
content* : seq[string]
getInline : bool
type
Entry* = object
desc* : seq[Descriptor]
@ -53,36 +54,42 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
else:
echo("no attributes on this tag")
echo("----------CONTEXT RESULT")
echo(context)
#echo("----------CONTEXT RESULT")
#echo(context)
if entry.desc[i].html_tag == "":
echo("No html_tag specified, return whole context as string")
for a in context:
entry.desc[i].content.add(a.innerText)
return entry
else:
#end filter context
for a in context:
let subContext = a.findAll(entry.desc[i].html_tag)
for b in subContext:
if entry.desc[i].aattrs != "":
if b.attrs != nil:
if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs])
if entry.desc[i].contains_string != "":
var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("ATTRS MATCH")
entry.desc[i].content.add(content_txt)
elif entry.desc[i].attrs_key != "":
#if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
echo("KEY_MATCH")
entry.desc[i].content.add(b.innerText())
else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
elif entry.desc[i].contains_string != "":
var content_txt = b.innerText()
echo(" string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("PATTERN MATCH")
entry.desc[i].content.add(content_txt)
for a in context:
let subContext = a.findAll(entry.desc[i].html_tag)
for b in subContext:
if entry.desc[i].aattrs != "":
if b.attrs != nil:
if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs])
if entry.desc[i].contains_string != "":
var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("ATTRS MATCH")
entry.desc[i].content.add(content_txt)
elif entry.desc[i].attrs_key != "":
#if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
echo("KEY_MATCH")
entry.desc[i].content.add(b.innerText())
else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
elif entry.desc[i].contains_string != "":
var content_txt = b.innerText()
echo(" string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("PATTERN MATCH")
entry.desc[i].content.add(content_txt)
return entry
proc test() =