Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Michael Raskin
thoughtful-theridion
Commits
2f69c13d
Commit
2f69c13d
authored
Feb 18, 2020
by
Michael Raskin
Browse files
Syncing from Monotone at 20200218-024739. Last commit message was Play more with extraction
parent
ddcde14e
Changes
4
Hide whitespace changes
Inline
Side-by-side
html-main-text.lisp
View file @
2f69c13d
...
...
@@ -78,8 +78,9 @@ as readable content in case this element is picked as a part of main content.
text
"Alphabetic"
"Sentence_Terminal"
"Punctuation"
)
(
let*
((
commas
(
-
punctuation
sentences
))
(
length-points
(
min
3
(
truncate
letters
90
))))
(
if
(
<
letters
25
)
0
(
+
commas
length-points
1
)))))
(
length-points
(
min
3
(
truncate
letters
90
)))
(
sentence-points
(
min
20
(
truncate
sentences
5
))))
(
if
(
<
letters
25
)
0
(
+
commas
length-points
sentence-points
1
)))))
(
defmethod
html-content-score-summarize
((
protocol
html-text-score-protocol
)
(
scores
list
))
...
...
html-textify.lisp
View file @
2f69c13d
...
...
@@ -195,7 +195,18 @@
(
defmethod
html-element-to-text-dispatch
((
protocol
show-ui-texts-mixin
)
(
type
(
eql
:element
))
tag
element
)
(
let*
((
attributes
'
(
"name"
"id"
"alt"
"title"
))
(
let*
((
attributes
'
(
"alt"
"title"
))
(
names
(
loop
for
a
in
attributes
for
v
:=
(
html5-parser:element-attribute
element
a
)
when
v
collect
v
))
(
text
(
call-next-method
)))
(
if
names
(
format
nil
"~{ ⚓[ ~a ]~}~a"
names
text
)
text
)))
(
defclass
show-id-texts-mixin
(
html-textifier-protocol
)
())
(
defmethod
html-element-to-text-dispatch
((
protocol
show-id-texts-mixin
)
(
type
(
eql
:element
))
tag
element
)
(
let*
((
attributes
'
(
"name"
"id"
))
(
names
(
loop
for
a
in
attributes
for
v
:=
(
html5-parser:element-attribute
element
a
)
when
v
collect
v
))
...
...
@@ -253,11 +264,15 @@
(
html5-parser:node-first-child
element
))))))
(
defclass
html-textifier-protocol-inspector
(
defclass
html-textifier-protocol-
formatting-
inspector
(
recur-on-noscript-mixin
show-formatting-mixin
show-ui-texts-mixin
show-urls-mixin
html-textifier-protocol
)
())
(
defclass
html-textifier-protocol-inspector
(
show-id-texts-mixin
html-textifier-protocol-formatting-inspector
)
())
(
defun
html-inner-text
(
element
)
(
html-element-to-text
(
make-instance
'html-textifier-protocol
)
...
...
package.lisp
View file @
2f69c13d
...
...
@@ -18,6 +18,7 @@
#:real-url
;; Converting HTML to text
#:html-textifier-protocol
#:html-textifier-protocol-formatting-inspector
#:html-textifier-protocol-inspector
#:html-element-to-text
#:html-inner-text
...
...
util.lisp
View file @
2f69c13d
...
...
@@ -33,38 +33,42 @@
(
or
(
ignore-errors
(
unless
skip-extract-main-content
(
html-extract-main-content
(
parsed-content
f
)
(
or
html-content-score-protocol
(
make-instance
'html-classname-score-protocol
))
(
or
html-textifier-protocol
(
make-instance
'html-textifier-protocol-inspector
)))))
(
cl-ppcre:regex-replace-all
" *\\n *\\n"
(
html-extract-main-content
(
parsed-content
f
)
(
or
html-content-score-protocol
(
make-instance
'html-classname-score-protocol
))
(
or
html-textifier-protocol
(
make-instance
'html-textifier-protocol-formatting-inspector
)))
(
coerce
(
list
#\Newline
#\Newline
)
'string
))))
""
)
(
when
(
and
(
not
skip-forms
)
(
parsed-content
f
))
(
loop
for
form
in
(
css-selectors:query
"form"
(
parsed-content
f
))
collect
(
form-parameters
form
:fetcher
f
)))
(
unless
skip-cookies
:cookie-jar
`
(
make-instance
'drakma:cookie-jar
:cookies
(
list
,@
(
loop
for
c
in
(
drakma:cookie-jar-cookies
(
cookie-jar
f
))
collect
`
(
make-instance
'drakma:cookie
:name
,
(
drakma:cookie-name
c
)
:value
,
(
drakma:cookie-value
c
)
:domain
,
(
drakma:cookie-domain
c
)
:path
,
(
drakma:cookie-path
c
)
:expires
,
(
drakma:cookie-expires
c
))))))
(
list
:cookie-jar
`
(
make-instance
'drakma:cookie-jar
:cookies
(
list
,@
(
loop
for
c
in
(
drakma:cookie-jar-cookies
(
cookie-jar
f
))
collect
`
(
make-instance
'drakma:cookie
:name
,
(
drakma:cookie-name
c
)
:value
,
(
drakma:cookie-value
c
)
:domain
,
(
drakma:cookie-domain
c
)
:path
,
(
drakma:cookie-path
c
)
:expires
,
(
drakma:cookie-expires
c
)))))))
(
if
(
parsed-content
f
)
(
html-element-to-text
(
or
html-textifier-protocol
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment