nim-basic/scrap.nim

import std/htmlparser
import std/xmltree
import std/strtabs
import std/strutils

type 
  Descriptor* = object
    name* : string
    html_tag* : string
    html_context_tag* : string
    html_context_attrs* : string
    html_context_key* : string
    contains_string* : string
    aattrs* : string
    attrs_key*: string
    content* : seq[string]
    getInline : bool
type
  Entry* = object
   desc* : seq[Descriptor]

proc getEntryFromHtml*(entry :var Entry, node : XmlNode) : Entry =
  var 
    context : seq[XmlNode]
    tmpContext : seq[XmlNode]
  for i,desc in entry.desc:
    context = @[] #clear sequences
    tmpContext = @[]
    echo("descriptor ",i," content: ",entry.desc[i])
    #filter context
    if entry.desc[i].html_context_attrs == "":
      context = node.findAll(entry.desc[i].html_context_tag)
    else:
      tmpContext = node.findAll(entry.desc[i].html_context_tag)
      echo(tmpContext.len)
      #filter by attributes
      for c in tmpContext:
        echo c.len
        #echo("c: ",c.attrs, " i3: ", i3, " i: ", i, " len tmpContext: ", "entry: ",entry.desc[i])
        if entry.desc[i].html_context_attrs != "":
          if c.attrs != nil:
            if c.attrs.hasKey(entry.desc[i].html_context_attrs):
              echo("c.attrs[key]: ",c.attrs[entry.desc[i].html_context_attrs])
              if entry.desc[i].html_context_key != "":
                #if entry.desc[i].html_context_key.contains(c.attrs[entry.desc[i].html_context_attrs]):
                if c.attrs[entry.desc[i].html_context_attrs].contains(entry.desc[i].html_context_key):
                  context.add(c)
                  echo("context_key_ MATCH")
                else:
                  echo("key does NOT match")
              else:
                context.add(c)
            else:
              echo("c.attrs does not have the key: ", entry.desc[i].html_context_attrs)
          else:
            echo("no attributes on this tag")
    #echo("----------CONTEXT RESULT")
    #echo(context)
    if entry.desc[i].html_tag == "":
      echo("No html_tag specified, return whole context as string")
      for a in context:
        entry.desc[i].content.add(a.innerText)
      return entry
    else:
    #end filter context
      for a in context:
        let subContext = a.findAll(entry.desc[i].html_tag)
        for b in subContext:
          if entry.desc[i].aattrs != "":
            if b.attrs != nil:
              if b.attrs.hasKey(entry.desc[i].aattrs):
                echo("found key [",entry.desc[i].aattrs,"] in : ",b)
                echo("value: ",b.attrs[entry.desc[i].aattrs])
                if entry.desc[i].contains_string != "":
                  var content_txt :string  = b.innerText()
                  echo(" and string(innerText): ",content_txt)
                  if content_txt.contains(entry.desc[i].contains_string):
                    echo("ATTRS MATCH")
                    entry.desc[i].content.add(content_txt)
                elif entry.desc[i].attrs_key != "":
                  #if entry.desc[i].attrs_key.contains(b.attrs[entry.desc[i].aattrs]):
                  if b.attrs[entry.desc[i].aattrs].contains(entry.desc[i].attrs_key):
                    echo("KEY_MATCH")
                    entry.desc[i].content.add(b.innerText())
                else:
                  entry.desc[i].content.add(b.attrs[entry.desc[i].aattrs])
          elif entry.desc[i].contains_string != "":
            var content_txt = b.innerText()
            echo(" string(innerText): ",content_txt)
            if content_txt.contains(entry.desc[i].contains_string):
              echo("PATTERN MATCH")
              entry.desc[i].content.add(content_txt)
  return entry

proc test() =
  var htmlnode : XmlNode
  var str_html : string
  var entry: Entry
  entry.desc[0] = Descriptor(html_tag : "p", contains_string : "test_container")
  echo entry

  htmlnode = parseHtml(str_html)
  discard entry.getEntryFromHtml(htmlnode)

#test()