Commit 2f69c13d authored by Michael Raskin's avatar Michael Raskin

Syncing from Monotone at 20200218-024739. Last commit message was Play more with extraction

parent ddcde14e
......@@ -78,8 +78,9 @@ as readable content in case this element is picked as a part of main content.
text
"Alphabetic" "Sentence_Terminal" "Punctuation")
(let* ((commas (- punctuation sentences))
(length-points (min 3 (truncate letters 90))))
(if (< letters 25) 0 (+ commas length-points 1)))))
(length-points (min 3 (truncate letters 90)))
(sentence-points (min 20 (truncate sentences 5))))
(if (< letters 25) 0 (+ commas length-points sentence-points 1)))))
(defmethod html-content-score-summarize ((protocol html-text-score-protocol)
(scores list))
......
......@@ -195,7 +195,18 @@
(defmethod html-element-to-text-dispatch
((protocol show-ui-texts-mixin) (type (eql :element)) tag element)
(let* ((attributes '("name" "id" "alt" "title"))
(let* ((attributes '( "alt" "title"))
(names (loop for a in attributes
for v := (html5-parser:element-attribute element a)
when v collect v))
(text (call-next-method)))
(if names (format nil "~{ ⚓[ ~a ]~}~a" names text) text)))
(defclass show-id-texts-mixin (html-textifier-protocol) ())
(defmethod html-element-to-text-dispatch
((protocol show-id-texts-mixin) (type (eql :element)) tag element)
(let* ((attributes '("name" "id"))
(names (loop for a in attributes
for v := (html5-parser:element-attribute element a)
when v collect v))
......@@ -253,11 +264,15 @@
(html5-parser:node-first-child
element))))))
(defclass html-textifier-protocol-inspector
(defclass html-textifier-protocol-formatting-inspector
(recur-on-noscript-mixin
show-formatting-mixin show-ui-texts-mixin show-urls-mixin
html-textifier-protocol) ())
(defclass html-textifier-protocol-inspector
(show-id-texts-mixin
html-textifier-protocol-formatting-inspector) ())
(defun html-inner-text (element)
(html-element-to-text
(make-instance 'html-textifier-protocol)
......
......@@ -18,6 +18,7 @@
#:real-url
;; Converting HTML to text
#:html-textifier-protocol
#:html-textifier-protocol-formatting-inspector
#:html-textifier-protocol-inspector
#:html-element-to-text
#:html-inner-text
......
......@@ -33,38 +33,42 @@
(or
(ignore-errors
(unless skip-extract-main-content
(html-extract-main-content
(parsed-content f)
(or html-content-score-protocol
(make-instance 'html-classname-score-protocol))
(or
html-textifier-protocol
(make-instance 'html-textifier-protocol-inspector)))))
(cl-ppcre:regex-replace-all
" *\\n *\\n"
(html-extract-main-content
(parsed-content f)
(or html-content-score-protocol
(make-instance 'html-classname-score-protocol))
(or
html-textifier-protocol
(make-instance 'html-textifier-protocol-formatting-inspector)))
(coerce (list #\Newline #\Newline) 'string))))
"")
(when (and (not skip-forms) (parsed-content f))
(loop for form in (css-selectors:query
"form" (parsed-content f))
collect (form-parameters form :fetcher f)))
(unless skip-cookies
:cookie-jar
`(make-instance
'drakma:cookie-jar
:cookies
(list
,@(loop for c in (drakma:cookie-jar-cookies (cookie-jar f))
collect
`(make-instance
'drakma:cookie
:name
,(drakma:cookie-name c)
:value
,(drakma:cookie-value c)
:domain
,(drakma:cookie-domain c)
:path
,(drakma:cookie-path c)
:expires
,(drakma:cookie-expires c))))))
(list
:cookie-jar
`(make-instance
'drakma:cookie-jar
:cookies
(list
,@(loop for c in (drakma:cookie-jar-cookies (cookie-jar f))
collect
`(make-instance
'drakma:cookie
:name
,(drakma:cookie-name c)
:value
,(drakma:cookie-value c)
:domain
,(drakma:cookie-domain c)
:path
,(drakma:cookie-path c)
:expires
,(drakma:cookie-expires c)))))))
(if (parsed-content f)
(html-element-to-text
(or html-textifier-protocol
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment