This commit is contained in:
ccppi 2024-07-03 14:03:25 +02:00
parent a2a0001aef
commit 42bd90fb7b
4 changed files with 1981 additions and 41 deletions

BIN
main

Binary file not shown.

View File

@ -4,11 +4,34 @@ import std/xmltree
import std/strtabs import std/strtabs
import std/os import std/os
import scrap import scrap
import std/json
import std/strutils
var client = newHttpClient() var client = newHttpClient()
var html: string var html: string
#var node: XmlNode #var node: XmlNode
proc do_another_loop( node : JsonNode, level : int, searchString : string = "") =
if node.len() > 0:
for a in node.keys:
if node.kind == JObject:
#for i in 0 .. level:
if searchString == "" or contains($node[a],searchString):
stdout.write ">"
stdout.write a
if node[a].kind == JObject:
do_another_loop(node[a],level+1,searchString)
else:
if searchString == "" or contains($node[a],searchString):
stdout.write " [", node[a].kind, "]\n"
if node[a].kind == JArray:
if searchString == "" or contains($node[a],searchString):
echo node[a]
else:
echo "yyyyyyyy"
else:
echo "\n--------"
echo "URL:" echo "URL:"
var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin) var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
echo "given url is: ",url echo "given url is: ",url
@ -21,19 +44,33 @@ try:
var entry : Entry var entry : Entry
add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u")) add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
#add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href")) #comparis stores part of the data in a script tag as a json-string, so we get that part:
add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
echo entry
echo("###########") echo("###########")
echo entry.getEntryFromHtml(node) discard entry.getEntryFromHtml(node)
# echo node
# for a in node.findAll("a"): #echo entry.desc[1].content[0]
# if a.attrs.hasKey "href": let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
# echo "striping" let jsonNode = parseJson(jsonString)
# let (dir,filename,ext) = splitFile(a.attrs["href"]) #echo "jsonString: ",jsonString
# echo "found a link!",dir & "/" & filename #echo "JSON Node:", jsonNode
#echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
let field = jsonNode.getFields()
do_another_loop(jsonNode,0,"resultitems")
#echo "resultitem", jsonNode["resultItems"]
# for a in jsonNode.keys:
# echo a
# if jsonNode[a].len() > 0:
# for b in jsonNode[a].keys:
# echo ">>>"
# do_another_loop(jsonNode[a][b])
# else: # else:
# echo "Key has no attribute href" # echo "---------------"
finally: finally:
client.close() client.close()

1896
out.log Normal file

File diff suppressed because one or more lines are too long

View File

@ -14,6 +14,7 @@ type
aattrs* : string aattrs* : string
attrs_key*: string attrs_key*: string
content* : seq[string] content* : seq[string]
getInline : bool
type type
Entry* = object Entry* = object
desc* : seq[Descriptor] desc* : seq[Descriptor]
@ -53,8 +54,14 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs) echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
else: else:
echo("no attributes on this tag") echo("no attributes on this tag")
echo("----------CONTEXT RESULT") #echo("----------CONTEXT RESULT")
echo(context) #echo(context)
if entry.desc[i].html_tag == "":
echo("No html_tag specified, return whole context as string")
for a in context:
entry.desc[i].content.add(a.innerText)
return entry
else:
#end filter context #end filter context
for a in context: for a in context:
let subContext = a.findAll(entry.desc[i].html_tag) let subContext = a.findAll(entry.desc[i].html_tag)