implement tag searching

This commit is contained in:
ccppi 2024-06-28 13:19:47 +02:00
parent fe35b44aa5
commit a706754ab4
4 changed files with 47 additions and 45 deletions

BIN
main

Binary file not shown.

View File

@ -3,6 +3,7 @@ import std/htmlparser
import std/xmltree import std/xmltree
import std/strtabs import std/strtabs
import std/os import std/os
import scrap
var client = newHttpClient() var client = newHttpClient()
var html: string var html: string
@ -13,17 +14,23 @@ var url: string = readLine(stdin)
echo "given url is: ",url echo "given url is: ",url
try: try:
html = client.getContent(url) html = client.getContent(url)
let node = parseHtml(html) let node = parseHtml(html)
echo node
for a in node.findAll("a"): var htmlnode: XmlNode
if a.attrs.hasKey "href": var entry : Entry
echo "striping"
let (dir,filename,ext) = splitFile(a.attrs["href"])
echo "found a link!",dir & "/" & filename
else:
echo "Key has no attribute href"
entry.name = Descriptor(html_context_tag : "div", html_tag : "a",attrs : "href")
entry.getEntryFromHtml(node)
# echo node
# for a in node.findAll("a"):
# if a.attrs.hasKey "href":
# echo "striping"
# let (dir,filename,ext) = splitFile(a.attrs["href"])
# echo "found a link!",dir & "/" & filename
# else:
# echo "Key has no attribute href"
finally: finally:
client.close() client.close()

BIN
scrap

Binary file not shown.

View File

@ -3,51 +3,46 @@ import std/xmltree
import std/strtabs import std/strtabs
type type
Descriptor = object Descriptor* = object
name : string name* : string
html_tag : string html_tag* : string
html_context_tag : string html_context_tag* : string
contains_string : string contains_string* : string
attrs : string attrs* : string
type type
Entry = object Entry* = object
name, tag, description, link, category : Descriptor name*, tag*, description*, link*, category* : Descriptor
proc setDescriptor(desc : var Descriptor, descToChange : string, value : string) = proc getEntryFromHtml*(entry : Entry, node : XmlNode) =
case descToChange:
of "name":
desc.name = value
of "html_tag":
desc.html_tag = value
of "html_context_tag":
desc.html_context_tag = value
of "contains_string":
desc.contains_string = value
of "attrs":
desc.attrs = value
#proc newEntry() : Entry =
# newDescriptor()
proc getEntryFromHtml(entry : Entry, node : XmlNode) =
echo(entry.description) echo(entry.description)
var childrens = node.findAll(entry.name.html_tag)
let context = node.findAll(entry.name.html_context_tag)
for a in context:
let subContext = a.findAll(entry.name.html_tag)
for b in subContext:
echo(b)
if entry.name.attrs != "":
if b.attrs.hasKey(entry.name.attrs):
echo("found key")
if entry.name.contains_string != "":
echo(" and string")
#check if b contains contains_string
#return function
if entry.name.contains_string != "":
echo("found string")
#check if b contains contains_string
#return function
proc test() = proc test() =
var htmlnode : XmlNode var htmlnode : XmlNode
var str_html : string var str_html : string
var entry: Entry var entry: Entry
# entry = Entry(name : "testname",description : "testdescription",link : "testlink",category : "testcategory") entry.name = Descriptor(html_tag : "p", contains_string : "test_container")
entry.name.setDescriptor("name","testname")
entry.name.setDescriptor("html_tag","p")
entry.link.setDescriptor("name","inserat_link")
entry.link.setDescriptor("html_tag","a")
entry.link.setDescriptor("attrs","href")
echo entry echo entry
htmlnode = parseHtml(str_html) htmlnode = parseHtml(str_html)
#entry.getEntryFromHtml() entry.getEntryFromHtml(htmlnode)
test() #test()