safdoija

main.nim

@@ -4,11 +4,34 @@ import std/xmltree
 import std/strtabs
 import std/os
 import scrap
-
+import std/json
+import std/strutils
 var client = newHttpClient()
 var html: string
 #var node: XmlNode
 
+proc do_another_loop( node : JsonNode, level : int, searchString : string = "") = 
+    if node.len() > 0:
+      for a in node.keys:
+        if node.kind == JObject:
+          #for i in 0 .. level:
+          if searchString == "" or contains($node[a],searchString):
+            stdout.write ">"
+            stdout.write a
+          if node[a].kind == JObject:
+            do_another_loop(node[a],level+1,searchString)
+          else:
+            if searchString == "" or contains($node[a],searchString):
+              stdout.write " [", node[a].kind, "]\n"
+        if node[a].kind == JArray:
+          if searchString == "" or contains($node[a],searchString):
+            echo node[a]
+        else:
+         echo "yyyyyyyy"
+    else:
+      echo "\n--------"
+
+
 echo "URL:"
 var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
 echo "given url is: ",url
@@ -21,19 +44,33 @@ try:
   var entry : Entry
   
   add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
-  #add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href"))
+  #comparis stores part of the data in a script tag as a json-string, so we get that part:
+  add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
+
+  
+
 
-  echo entry
   echo("###########")
-  echo entry.getEntryFromHtml(node)
-#    echo node
-#    for a in node.findAll("a"):
-#      if a.attrs.hasKey "href":
-#        echo "striping"
-#        let (dir,filename,ext) = splitFile(a.attrs["href"])
-#        echo "found a link!",dir & "/" & filename
-#      else:
-#        echo "Key has no attribute href"
+  discard entry.getEntryFromHtml(node)
+
+  #echo entry.desc[1].content[0]
+  let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
+  let jsonNode = parseJson(jsonString)
+  #echo "jsonString: ",jsonString
+  #echo "JSON Node:", jsonNode
+  #echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
+  let field = jsonNode.getFields()
+  do_another_loop(jsonNode,0,"resultitems")
+
+  #echo "resultitem", jsonNode["resultItems"]
+ # for a in jsonNode.keys:
+ #   echo a
+ #   if jsonNode[a].len() > 0:
+ #     for b in jsonNode[a].keys:
+ #       echo ">>>"
+ #       do_another_loop(jsonNode[a][b])
+ #   else:
+ #     echo "---------------"
 finally:
     client.close()
 

scrap.nim

@@ -14,6 +14,7 @@ type
     aattrs* : string
     attrs_key*: string
     content* : seq[string]
+    getInline : bool
 type
   Entry* = object
    desc* : seq[Descriptor]
@@ -53,8 +54,14 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
               echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
           else:
             echo("no attributes on this tag")
-    echo("----------CONTEXT RESULT")
-    echo(context)
+    #echo("----------CONTEXT RESULT")
+    #echo(context)
+    if entry.desc[i].html_tag == "":
+      echo("No html_tag specified, return whole context as string")
+      for a in context:
+        entry.desc[i].content.add(a.innerText)
+      return entry
+    else:
     #end filter context
       for a in context:
         let subContext = a.findAll(entry.desc[i].html_tag)