nim-basic/main.nim
2024-07-05 13:08:22 +02:00

94 lines
3.2 KiB
Nim

import std/httpclient
import std/htmlparser
import std/xmltree
import std/strtabs
import std/os
import scrap
import std/json
import std/strutils
var client = newHttpClient()
var html: string
proc do_another_loop[T]( node : T , level : int, searchString : string = "",path :string = "H")
proc do_loop[T](node: T,level : int, searchString : string = "", path: string = "", key : string) =
if searchString == "" or node.contains(searchString):
stdout.write "|", path & "-" & key
#children of currentObj
if node.kind == JObject:
if searchString == "" or node.contains(searchString):
stdout.write("\n")
echo "childs: "
for y in node.keys:
stdout.write ", " ,y," [", node[y].kind,"] \n"
do_another_loop(node,level+1,searchString,path & "-" & key)
#children of currentObj
proc do_array_loop(node : JsonNode, level : int, searchString : string = "", path: string = "") =
if searchString == "" or node.contains(searchString):
echo "Array Elements: ", node.getElems()
echo "Array Length", node.len()
echo "WWWWWWWWWWWWWWW"
for b in node:
echo "array_node_len: ", b.len()," ,subkind: " ,b.kind, "mainkind: ", node.kind
do_another_loop(b,level+1,searchString,path & "-" & $node)
#if node[a].kind == JString:
#echo "String: ", node[a]
#stdout.write "\n"
proc do_another_loop[T]( node : T , level : int, searchString : string = "",path :string = "H") =
if node.len() > 0:
if node.kind == JObject:
for a in node.keys:
if node[a].kind == JObject:
do_loop(node[a],level,searchString,path, a)
elif node[a].kind == JArray:
do_array_loop(node[a],level,searchString,path)
elif node.kind == JArray:
do_array_loop(node,level,searchString,path)
echo "URL:"
var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
echo "given url is: ",url
try:
html = client.getContent(url)
let node = parseHtml(html)
var htmlnode: XmlNode
var entry : Entry
add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
#comparis stores part of the data in a script tag as a json-string, so we get that part:
add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
echo("###########")
discard entry.getEntryFromHtml(node)
#echo entry.desc[1].content[0]
let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
let jsonNode = parseJson(jsonString)
#echo "jsonString: ",jsonString
#echo "JSON Node:", jsonNode
#echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
let field = jsonNode.getFields()
do_another_loop(jsonNode,0,searchString = "")
#echo "resultitem", jsonNode["resultItems"]
# for a in jsonNode.keys:
# echo a
# if jsonNode[a].len() > 0:
# for b in jsonNode[a].keys:
# echo ">>>"
# do_another_loop(jsonNode[a][b])
# else:
# echo "---------------"
finally:
client.close()