safdoija

main.nim

@@ -4,11 +4,34 @@ import std/xmltree
 import std/strtabs
 import std/os
 import scrap
-
+import std/json
+import std/strutils
 var client = newHttpClient()
 var html: string
 #var node: XmlNode
 
+proc do_another_loop( node : JsonNode, level : int, searchString : string = "") = 
+    if node.len() > 0:
+      for a in node.keys:
+        if node.kind == JObject:
+          #for i in 0 .. level:
+          if searchString == "" or contains($node[a],searchString):
+            stdout.write ">"
+            stdout.write a
+          if node[a].kind == JObject:
+            do_another_loop(node[a],level+1,searchString)
+          else:
+            if searchString == "" or contains($node[a],searchString):
+              stdout.write " [", node[a].kind, "]\n"
+        if node[a].kind == JArray:
+          if searchString == "" or contains($node[a],searchString):
+            echo node[a]
+        else:
+         echo "yyyyyyyy"
+    else:
+      echo "\n--------"
+
+
 echo "URL:"
 var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
 echo "given url is: ",url
@@ -21,19 +44,33 @@ try:
   var entry : Entry
   
   add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
-  #add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href"))
+  #comparis stores part of the data in a script tag as a json-string, so we get that part:
+  add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
+
+  
+
 
-  echo entry
   echo("###########")
-  echo entry.getEntryFromHtml(node)
+  discard entry.getEntryFromHtml(node)
-#    echo node
+
-#    for a in node.findAll("a"):
+  #echo entry.desc[1].content[0]
-#      if a.attrs.hasKey "href":
+  let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
-#        echo "striping"
+  let jsonNode = parseJson(jsonString)
-#        let (dir,filename,ext) = splitFile(a.attrs["href"])
+  #echo "jsonString: ",jsonString
-#        echo "found a link!",dir & "/" & filename
+  #echo "JSON Node:", jsonNode
-#      else:
+  #echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
-#        echo "Key has no attribute href"
+  let field = jsonNode.getFields()
+  do_another_loop(jsonNode,0,"resultitems")
+
+  #echo "resultitem", jsonNode["resultItems"]
+ # for a in jsonNode.keys:
+ #   echo a
+ #   if jsonNode[a].len() > 0:
+ #     for b in jsonNode[a].keys:
+ #       echo ">>>"
+ #       do_another_loop(jsonNode[a][b])
+ #   else:
+ #     echo "---------------"
 finally:
     client.close()
 

scrap.nim

@@ -14,6 +14,7 @@ type
     aattrs* : string
     attrs_key*: string
     content* : seq[string]
+    getInline : bool
 type
   Entry* = object
    desc* : seq[Descriptor]
@@ -53,36 +54,42 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
               echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
           else:
             echo("no attributes on this tag")
-    echo("----------CONTEXT RESULT")
+    #echo("----------CONTEXT RESULT")
-    echo(context)
+    #echo(context)
+    if entry.desc[i].html_tag == "":
+      echo("No html_tag specified, return whole context as string")
+      for a in context:
+        entry.desc[i].content.add(a.innerText)
+      return entry
+    else:
     #end filter context
-    for a in context:
+      for a in context:
-      let subContext = a.findAll(entry.desc[i].html_tag)
+        let subContext = a.findAll(entry.desc[i].html_tag)
-      for b in subContext:
+        for b in subContext:
-        if entry.desc[i].aattrs != "":
+          if entry.desc[i].aattrs != "":
-          if b.attrs != nil:
+            if b.attrs != nil:
-            if b.attrs.hasKey(entry.desc[i].aattrs):
+              if b.attrs.hasKey(entry.desc[i].aattrs):
-              echo("found key [",entry.desc[i].aattrs,"] in : ",b)
+                echo("found key [",entry.desc[i].aattrs,"] in : ",b)
-              echo("value: ",b.attrs[entry.desc[i].aattrs])
+                echo("value: ",b.attrs[entry.desc[i].aattrs])
-              if entry.desc[i].contains_string != "":
+                if entry.desc[i].contains_string != "":
-                var content_txt :string  = b.innerText()
+                  var content_txt :string  = b.innerText()
-                echo(" and string(innerText): ",content_txt)
+                  echo(" and string(innerText): ",content_txt)
-                if content_txt.contains(entry.desc[i].contains_string):
+                  if content_txt.contains(entry.desc[i].contains_string):
-                  echo("ATTRS MATCH")
+                    echo("ATTRS MATCH")
-                  entry.desc[i].content.add(content_txt)
+                    entry.desc[i].content.add(content_txt)
-              elif entry.desc[i].attrs_key != "":
+                elif entry.desc[i].attrs_key != "":
-                #if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
+                  #if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
-                if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
+                  if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
-                  echo("KEY_MATCH")
+                    echo("KEY_MATCH")
-                  entry.desc[i].content.add(b.innerText())
+                    entry.desc[i].content.add(b.innerText())
-              else:
+                else:
-                entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
+                  entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
-        elif entry.desc[i].contains_string != "":
+          elif entry.desc[i].contains_string != "":
-          var content_txt = b.innerText()
+            var content_txt = b.innerText()
-          echo(" string(innerText): ",content_txt)
+            echo(" string(innerText): ",content_txt)
-          if content_txt.contains(entry.desc[i].contains_string):
+            if content_txt.contains(entry.desc[i].contains_string):
-            echo("PATTERN MATCH")
+              echo("PATTERN MATCH")
-            entry.desc[i].content.add(content_txt)
+              entry.desc[i].content.add(content_txt)
   return entry
 
 proc test() =