87 lines
2.7 KiB
Nim
87 lines
2.7 KiB
Nim
import std/httpclient
|
|
import std/htmlparser
|
|
import std/xmltree
|
|
import std/strtabs
|
|
import std/os
|
|
import scrap
|
|
import std/json
|
|
import std/strutils
|
|
var client = newHttpClient()
|
|
var html: string
|
|
#var node: XmlNode
|
|
|
|
proc do_another_loop( node : JsonNode, level : int, searchString : string = "",path :string = "H") =
|
|
if node.len() > 0:
|
|
for a in node.keys:
|
|
if node.kind == JObject:
|
|
#for i in 0 .. level:
|
|
if searchString == "" or a.contains(searchString):
|
|
#for x in 0..level:
|
|
# stdout.write " "
|
|
stdout.write "|", path & "-" & a
|
|
if node[a].kind == JObject:
|
|
stdout.write("\n")
|
|
do_another_loop(node[a],level+1,searchString,path & "-" & a)
|
|
else:
|
|
if searchString == "" or a.contains(searchString):
|
|
stdout.write " [", node[a].kind, "]"
|
|
stdout.write("\n")
|
|
if node[a].kind == JArray:
|
|
if searchString == "" or a.contains(searchString):
|
|
echo "Array Elements: ", node[a].getElems()
|
|
echo "Array Length", node[a].len()
|
|
echo "WWWWWWWWWWWWWWW"
|
|
for b in node[a]:
|
|
echo b
|
|
do_another_loop(b,level,searchString,path & "-" & a)
|
|
|
|
#stdout.write "\n"
|
|
else:
|
|
echo "\n--------"
|
|
|
|
|
|
echo "URL:"
|
|
var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
|
|
echo "given url is: ",url
|
|
|
|
try:
|
|
html = client.getContent(url)
|
|
let node = parseHtml(html)
|
|
|
|
var htmlnode: XmlNode
|
|
var entry : Entry
|
|
|
|
add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
|
|
#comparis stores part of the data in a script tag as a json-string, so we get that part:
|
|
add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
|
|
|
|
|
|
|
|
|
|
echo("###########")
|
|
discard entry.getEntryFromHtml(node)
|
|
|
|
#echo entry.desc[1].content[0]
|
|
let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
|
|
let jsonNode = parseJson(jsonString)
|
|
#echo "jsonString: ",jsonString
|
|
#echo "JSON Node:", jsonNode
|
|
#echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
|
|
let field = jsonNode.getFields()
|
|
do_another_loop(jsonNode,0,searchString = "resultItems")
|
|
|
|
#echo "resultitem", jsonNode["resultItems"]
|
|
# for a in jsonNode.keys:
|
|
# echo a
|
|
# if jsonNode[a].len() > 0:
|
|
# for b in jsonNode[a].keys:
|
|
# echo ">>>"
|
|
# do_another_loop(jsonNode[a][b])
|
|
# else:
|
|
# echo "---------------"
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
|