ccppi/nim-basic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

safdoija

Browse Source
This commit is contained in:
ccppi 2024-07-03 14:03:25 +02:00
parent a2a0001aef
commit 42bd90fb7b
4 changed files with 1981 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
main
View File

Binary file not shown.

61
main.nim
View File

@ -4,11 +4,34 @@ import std/xmltree
import std/strtabs
import std/os
import scrap
import std/json
import std/strutils
var client = newHttpClient()
var html: string
#var node: XmlNode
proc do_another_loop( node : JsonNode, level : int, searchString : string = "") =
if node.len() > 0:
for a in node.keys:
if node.kind == JObject:
#for i in 0 .. level:
if searchString == "" or contains($node[a],searchString):
stdout.write ">"
stdout.write a
if node[a].kind == JObject:
do_another_loop(node[a],level+1,searchString)
else:
if searchString == "" or contains($node[a],searchString):
stdout.write " [", node[a].kind, "]\n"
if node[a].kind == JArray:
if searchString == "" or contains($node[a],searchString):
echo node[a]
else:
echo "yyyyyyyy"
else:
echo "\n--------"
echo "URL:"
var url: string = "https://www.comparis.ch/immobilien/marktplatz/lenzburg/mieten?page=2"#readLine(stdin)
echo "given url is: ",url
@ -21,19 +44,33 @@ try:
var entry : Entry
add(entry.desc,Descriptor(name : "test", html_context_tag : "div",html_context_attrs : "class",html_context_key : "css-19re50j", html_tag : "p",aattrs : "class",attrs_key : "css-svet6u"))
#add(entry.desc,Descriptor(name:"key_and_string_search",contains_string: "a",html_context_tag : "div", html_tag : "a",attrs : "href"))
#comparis stores part of the data in a script tag as a json-string, so we get that part:
add(entry.desc,Descriptor(name : "script-json", html_context_tag : "script",html_context_attrs : "type",html_context_key : r"application/json", html_tag : "",aattrs : "",attrs_key : ""))
echo entry
echo("###########")
echo entry.getEntryFromHtml(node)
# echo node
# for a in node.findAll("a"):
# if a.attrs.hasKey "href":
# echo "striping"
# let (dir,filename,ext) = splitFile(a.attrs["href"])
# echo "found a link!",dir & "/" & filename
# else:
# echo "Key has no attribute href"
discard entry.getEntryFromHtml(node)
#echo entry.desc[1].content[0]
let jsonString = replaceWord(entry.desc[1].content[0],sub = r"\"",by = r""")
let jsonNode = parseJson(jsonString)
#echo "jsonString: ",jsonString
#echo "JSON Node:", jsonNode
#echo "node fields: ",jsonNode["props"]["pageProps"]["pageTitle"].getStr()
let field = jsonNode.getFields()
do_another_loop(jsonNode,0,"resultitems")
#echo "resultitem", jsonNode["resultItems"]
# for a in jsonNode.keys:
# echo a
# if jsonNode[a].len() > 0:
# for b in jsonNode[a].keys:
# echo ">>>"
# do_another_loop(jsonNode[a][b])
# else:
# echo "---------------"
finally:
client.close()

1896
out.log Normal file
View File

File diff suppressed because one or more lines are too long

65
scrap.nim
View File

@ -14,6 +14,7 @@ type
aattrs* : string
attrs_key*: string
content* : seq[string]
getInline : bool
type
Entry* = object
desc* : seq[Descriptor]
@ -53,36 +54,42 @@ proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
else:
echo("no attributes on this tag")
echo("----------CONTEXT RESULT")
echo(context)
#echo("----------CONTEXT RESULT")
#echo(context)
if entry.desc[i].html_tag == "":
echo("No html_tag specified, return whole context as string")
for a in context:
entry.desc[i].content.add(a.innerText)
return entry
else:
#end filter context
for a in context:
let subContext = a.findAll(entry.desc[i].html_tag)
for b in subContext:
if entry.desc[i].aattrs != "":
if b.attrs != nil:
if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs])
if entry.desc[i].contains_string != "":
var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("ATTRS MATCH")
entry.desc[i].content.add(content_txt)
elif entry.desc[i].attrs_key != "":
#if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
echo("KEY_MATCH")
entry.desc[i].content.add(b.innerText())
else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
elif entry.desc[i].contains_string != "":
var content_txt = b.innerText()
echo(" string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("PATTERN MATCH")
entry.desc[i].content.add(content_txt)
for a in context:
let subContext = a.findAll(entry.desc[i].html_tag)
for b in subContext:
if entry.desc[i].aattrs != "":
if b.attrs != nil:
if b.attrs.hasKey(entry.desc[i].aattrs):
echo("found key [",entry.desc[i].aattrs,"] in : ",b)
echo("value: ",b.attrs[entry.desc[i].aattrs])
if entry.desc[i].contains_string != "":
var content_txt :string = b.innerText()
echo(" and string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("ATTRS MATCH")
entry.desc[i].content.add(content_txt)
elif entry.desc[i].attrs_key != "":
#if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
echo("KEY_MATCH")
entry.desc[i].content.add(b.innerText())
else:
entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
elif entry.desc[i].contains_string != "":
var content_txt = b.innerText()
echo(" string(innerText): ",content_txt)
if content_txt.contains(entry.desc[i].contains_string):
echo("PATTERN MATCH")
entry.desc[i].content.add(content_txt)
return entry
proc test() =