Rules
6 fields: name, price, old price, brand, image url, stock
Language
extract.pyOne file. Give it links or a sitemap, and it writes one line per page: every field with how sure it is.
Show the code
# extract.py: pulls name, price, old price, brand, image url, stock out of a web page.
# Made on scrape.fairhall.nl.
#
# Use it:
# python3 extract.py https://shop.example/p/1
# one page (a link or a saved .html file): its fields as JSON
# python3 extract.py page-1.html page-2.html
# several pages: one JSON line per page
# from extract import extract; extract(html) in your own code: a dict
#
# Many pages, with the downloader:
# python3 extract.py --download links.txt > results.jsonl
# links.txt has one link per line; results.jsonl gets one JSON line per page
# python3 extract.py --download https://shop.example/sitemap.xml --match /p/ > results.jsonl
# every link in the sitemap with /p/ in it (a sitemap index is followed)
# It fetches one page at a time, one a second (--wait 2 for slower), and keeps every page in pages/
# (--save other-folder): run it again and only new links are fetched. A page that fails is reported and skipped.
#
# Needs Python 3.8+ and the regex module (pip install regex): it reads the rules exactly like PHP does.
#
# Every field comes back with a value and how sure it is:
# sure the main rule found it and no backup disagrees
# fallback only a backup found it: this page looks different from the examples
# conflict the rules found different values: check this page by hand
# none nothing found
# A page that is not "sure" everywhere is the best next example to send in.
#
# Cuts: a field first cuts the page to the part between two landmarks (an empty end: the start tag's own closing
# tag), so a look-alike value elsewhere on the page (a related product's price) can't answer. No cut on a page:
# the whole page, as a fallback.
import json
import os
import sys
import time
import urllib.request
from html import unescape
import regex
# The recipe: per field the main rule, then its backups, tried in that order.
RECIPE = {
"name": {
"main": r'\s*›\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))',
"backups": [
r'<h1(?![\w:-])[^>]*>\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))',
r'<img(?![\w:-])[^>]*?-800\.jpg"\s+alt="\s*(?<value>[^<>"\s][^<>"]{0,399}?)\s*"\s+w',
r'<meta\s+property="og:title"\s+content="\s*(?<value>[^<>"\s][^<>"]{0,399}?)\s*">',
r'''<meta(?![\w:-])(?=[^>]*\sname\s*=\s*["']?viewport["'\s>/])(?=[^>]*\scontent\s*=\s*["']?width=device-width, initial-scale=1["'\s>/])[^>]*>\s*<title(?![\w:-])[^>]*>\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*\|\s+T''',
r'''<meta(?![\w:-])(?=[^>]*\sname\s*=\s*["']?viewport["'\s>/])(?=[^>]*\scontent\s*=\s*["']?width=device-width, initial-scale=1["'\s>/])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*\|\s+T''',
r'<h1(?![\w:-])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))',
r'''<meta(?![\w:-])(?=[^>]*\sname\s*=\s*["']?viewport["'\s>/])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*\|\s+T''',
r'<title(?![\w:-])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*\|\s+T',
],
"cuts": [
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?menu["'\s>/])[^>]*>''', r'<main(?![\w:-])[^>]*>'),
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?top["'\s>/])[^>]*>''', r'''<img(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?pic["'\s>/])[^>]*>'''),
(r'<\/header\s*>', r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?brand["'\s>/])[^>]*>'''),
],
},
"price": {
"main": r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?price["'\s>/])[^>]*>\s*€ \s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))''',
"backups": [
r'''<script(?![\w:-])(?=[^>]*\stype\s*=\s*["']?application\/ld\+json["'\s>/])[^>]*>\s*[^<]*?"price"\s*:\s*"\s*(?<value>\p{N}+(?:(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{N}+)+(?![\p{L}\p{N}]))''',
r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?price["'\s>/])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))''',
r'[^<]*?"price"\s*:\s*"\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>\p{N}+(?:(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{N}+)+(?![\p{L}\p{N}]))',
],
"cuts": [
(r'<main(?![\w:-])[^>]*>', ""),
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?brand["'\s>/])[^>]*>''', r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?stock["'\s>/])[^>]*>'''),
(r'''<img(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?pic["'\s>/])[^>]*>''', r'''<ul(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?ship["'\s>/])[^>]*>'''),
],
},
"old price": {
"main": r'<s(?![\w:-])[^>]*>\s*€ \s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))',
"backups": [
r'<s(?![\w:-])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))',
],
"cuts": [
(r'<main(?![\w:-])[^>]*>', ""),
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?brand["'\s>/])[^>]*>''', r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?price["'\s>/])[^>]*>'''),
(r'''<img(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?pic["'\s>/])[^>]*>''', r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?stock["'\s>/])[^>]*>'''),
],
},
"brand": {
"main": r'by\s*<a(?![\w:-])[^>]*>\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))',
"backups": [
r'''<script(?![\w:-])(?=[^>]*\stype\s*=\s*["']?application\/ld\+json["'\s>/])[^>]*>\s*[^<]*?"name"\s*:\s*"\s*(?<value>[^<"\s][^<"]{0,399}?)\s*"\},''',
r'[^<]*?"name"\s*:\s*"\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<"\s][^<"]{0,399}?)\s*"\},',
],
"cuts": [
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?brand["'\s>/])[^>]*>''', ""),
(r'<main(?![\w:-])[^>]*>', r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?price["'\s>/])[^>]*>'''),
(r'''<img(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?pic["'\s>/])[^>]*>''', r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?stock["'\s>/])[^>]*>'''),
],
},
"image url": {
"main": r'<img(?![\w:-])[^>]*?-200\.jpg"\s+data-src="\s*(?<value>[^<>"\s][^<>"]{0,399}?)\s*"\s+a',
"backups": [
r'''<script(?![\w:-])(?=[^>]*\stype\s*=\s*["']?application\/ld\+json["'\s>/])[^>]*>\s*[^<]*?"image"\s*:\s*"\s*(?<value>\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{N}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?![\p{L}\p{N}]))''',
r'[^<]*?"image"\s*:\s*"\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{N}+(?:&#?[a-z0-9]{2,8};|[^\p{L}\p{N}\s<>])\p{L}+(?![\p{L}\p{N}]))',
],
"cuts": [
(r'<main(?![\w:-])[^>]*>', ""),
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?menu["'\s>/])[^>]*>''', r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?brand["'\s>/])[^>]*>'''),
(r'<\/nav\s*>', r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?price["'\s>/])[^>]*>'''),
],
},
"stock": {
"main": r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?stock["'\s>/])[^>]*>\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))''',
"backups": [
r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?stock["'\s>/])[^>]*>\s*(?:\s*(?:<[^>]*>|[^<]++)){0,8}?\s*(?<value>[^<\s][^<]{0,399}?(?:(?:<!--[^>]*-->|<\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b[^>]*>)[^<]{0,399}?){0,20}?)\s*(?=<(?!!--|\/?(?:b|i|em|strong|u|mark|small|sup|sub|abbr|wbr)\b))''',
],
"cuts": [
(r'<main(?![\w:-])[^>]*>', ""),
(r'''<span(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?price["'\s>/])[^>]*>''', r'''<ul(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?ship["'\s>/])[^>]*>'''),
(r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?brand["'\s>/])[^>]*>''', r'''<div(?![\w:-])(?=[^>]*\sclass\s*=\s*["']?(?:[^"'>]*\s)?seller["'\s>/])[^>]*>'''),
],
},
}
USER_AGENT = "Mozilla/5.0 (compatible; extract-script)"
FLAGS = regex.IGNORECASE | regex.DOTALL
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}
def extract(html):
"""Every field of the recipe on this page: {field: {"value": ..., "confidence": ...}}; a list's value is a list, a
records field's a list of records (dicts)."""
return {name: _run_field(rules, html) for name, rules in RECIPE.items()}
def _run_field(rules, html):
if rules.get("records"):
return _records(rules, html)
return (_list if rules.get("list") else _cut_field)(rules, html)
def read(src):
"""A page as text, from a link or a saved file."""
if src.startswith(("http://", "https://")):
request = urllib.request.Request(src, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(request, timeout=20) as response:
return response.read().decode("utf-8", "replace")
with open(src, encoding="utf-8", errors="replace") as f:
return f.read()
def _field(rules, html):
values = [_run(rule, html) for rule in rules]
found = [v for v in values if v is not None]
if not found:
return {"value": None, "confidence": "none"}
if len({_compare_key(v) for v in found}) > 1:
confidence = "conflict"
elif values[0] is None:
confidence = "fallback"
else:
confidence = "sure"
return {"value": found[0], "confidence": confidence}
def _run(rule, html):
m = regex.search(rule, html, FLAGS)
value = _value(m) if m else None
return None if value is None else _clean(value)
def _value(m):
groups = m.groupdict()
return groups["value"] if "value" in groups else (m.group(1) if m.re.groups else m.group(0))
def _cut(html, start, end, once):
"""The part of the page between the first match of start and the first match of end after it, or None. An empty
end is the start tag's own closing tag. With once, a start that matches more than once is no cut."""
if once and sum(1 for _ in regex.finditer(start, html, FLAGS)) != 1:
return None
m = regex.search(start, html, FLAGS)
if not m:
return None
if end == "":
to = _close_of(html, m.group(0), m.end())
return None if to is None else html[m.end():to]
e = regex.compile(end, FLAGS).search(html, m.end())
return None if e is None else html[m.end():e.start()]
def _close_of(html, start_tag, pos):
"""Where the element of this start tag closes: its own closing tag, counting the tags of its name in between."""
t = regex.match(r"<([a-z][\w:-]*)", start_tag, regex.IGNORECASE)
if not t or start_tag.endswith("/>") or t.group(1).lower() in VOID:
return None
depth = 1
for m in regex.compile(r"<(/?)" + regex.escape(t.group(1)) + r"(?![\w:-])[^>]*>", regex.IGNORECASE).finditer(html, pos):
if m.group(1):
depth -= 1
if depth == 0:
return m.start()
elif not m.group(0).endswith("/>"):
depth += 1
return None
def _cut_field(rules, html):
"""One value: in the first cut found (a later cut: fallback), else the whole page (a fallback when there are cuts)."""
chain = [rules["main"], *rules["backups"]]
cuts = rules.get("cuts", [])
for i, (start, end) in enumerate(cuts):
block = _cut(html, start, end, True)
result = None if block is None else _field(chain, block)
if result and result["value"] is not None:
if i > 0 and result["confidence"] == "sure":
result["confidence"] = "fallback"
return result
result = _field(chain, html)
if cuts and result["confidence"] == "sure":
result["confidence"] = "fallback"
return result
def _list(rules, html):
"""Every item inside the cut; backup cuts must agree. Sure: the main cut and the main rule answered."""
chain = [rules["main"], *rules["backups"]]
answers = []
for i, (start, end) in enumerate(rules.get("cuts", [])):
block = _cut(html, start, end, False)
if block is None:
continue
for j, rule in enumerate(chain):
matches = list(regex.finditer(rule, block, FLAGS))
if matches:
answers.append((i, j, _keep(rules, block, matches)))
break
if not answers:
return {"value": [], "confidence": "none"}
i, j, items = answers[0]
if len({tuple(_compare_key(v) for v in a[2]) for a in answers}) > 1:
confidence = "conflict"
elif i or j:
confidence = "fallback"
else:
confidence = "sure"
return {"value": items, "confidence": confidence}
def _keep(rules, block, matches):
"""The items, without those whose stretch of HTML (before or after them) matches the exclude rule."""
exclude = rules.get("exclude")
items = []
for k, m in enumerate(matches):
if exclude:
if rules.get("exclude_side") == "before":
stretch = block[matches[k - 1].end() if k else 0:m.start()]
else:
stretch = block[m.start():matches[k + 1].start() if k + 1 < len(matches) else len(block)]
if regex.search(exclude, stretch, FLAGS):
continue
value = _value(m)
if value is not None:
items.append(_clean(value))
return items
RANK = {"sure": 0, "fallback": 1, "conflict": 2}
def _records(rules, html):
"""One record per card inside the cut; backup cuts must agree. Sure: the main cut answered, every field in every
card was sure, and no card lacks a required field."""
answers = []
for i, (start, end) in enumerate(rules.get("cuts", [])):
block = _cut(html, start, end, False)
if block is None:
continue
worst, found = _cards(rules, block)
if found:
answers.append((i, worst, found))
if not answers:
return {"value": [], "confidence": "none"}
i, worst, found = answers[0]
def key(records):
return json.dumps([sorted((k, [_compare_key(v) for v in (r[k] if isinstance(r[k], list) else [r[k]])]) for k in r) for r in records])
if len({key(a[2]) for a in answers}) > 1 or worst == "conflict":
confidence = "conflict"
elif i or worst == "fallback":
confidence = "fallback"
else:
confidence = "sure"
return {"value": found, "confidence": confidence}
def _cards(rules, block):
"""The cards in a block: from each match of the card rule to its own closing tag (or the next card), without those
whose stretch matches the exclude rule; each card's fields, and the worst confidence among them."""
starts = list(regex.finditer(rules["main"], block, FLAGS))
exclude = rules.get("exclude")
worst = "sure"
found = []
for k, m in enumerate(starts):
nxt = starts[k + 1].start() if k + 1 < len(starts) else len(block)
if exclude:
if rules.get("exclude_side") == "before":
stretch = block[starts[k - 1].end() if k else 0:m.start()]
else:
stretch = block[m.start():nxt]
if regex.search(exclude, stretch, FLAGS):
continue
close = _close_of(block, m.group(0), m.end())
card = block[m.start():nxt if close is None or close > nxt else block.index(">", close) + 1]
card_worst = "sure"
record = {}
for name, sub in rules["fields"].items():
r = _run_field(sub, card)
if r["value"] is None or r["value"] == []:
continue
record[name] = r["value"]
if RANK[r["confidence"]] > RANK[card_worst]:
card_worst = r["confidence"]
if not record:
continue
if card_worst == "sure" and any(f not in record for f in rules.get("required", [])):
card_worst = "fallback"
if RANK[card_worst] > RANK[worst]:
worst = card_worst
found.append(record)
return worst, found
def _clean(raw):
"""The text a person sees: no tags, no entities, one space between words."""
text = unescape(regex.sub(r"<[^>]*>", " ", raw))
return regex.sub(r"[\s\u00a0\u2009\u202f]+", " ", text).strip()
def _compare_key(value):
return regex.sub(r"[^\p{L}\p{N}]+", "", value).lower()
# ---- the downloader: many pages from a links file or a sitemap, fetched one at a time, kept in a folder ----
_last_fetch = 0.0
def _polite_fetch(link, wait):
"""A link's text, at most one fetch per wait seconds: kind to the site, and less likely to get blocked."""
global _last_fetch
pause = _last_fetch + wait - time.time()
if pause > 0:
time.sleep(pause)
_last_fetch = time.time()
return read(link)
def _links(src, wait):
"""The page links in a source: a sitemap (.xml), a links file (one per line) or one link."""
is_link = src.startswith(("http://", "https://"))
if regex.search(r"\.xml(\?|$)", src, regex.IGNORECASE):
return _sitemap(_polite_fetch(src, wait) if is_link else read(src), wait)
if is_link:
return [src]
return [line.strip() for line in read(src).splitlines() if line.strip().startswith(("http://", "https://"))]
def _sitemap(xml, wait):
"""The links in a sitemap; a sitemap index is followed into its sitemaps."""
locs = [unescape(loc) for loc in regex.findall(r"<loc>\s*([^<\s]+)\s*</loc>", xml, regex.IGNORECASE)]
if not regex.search(r"<sitemapindex", xml, regex.IGNORECASE):
return locs
return [link for loc in locs for link in _sitemap(_polite_fetch(loc, wait), wait)]
def _saved(link, folder, wait):
"""A page from the folder, or fetched and put there: a second run only fetches new links."""
name = regex.sub(r"[^A-Za-z0-9._-]+", "_", regex.sub(r"^https?://", "", link, flags=regex.IGNORECASE))[:150]
path = os.path.join(folder, name + ".html")
if os.path.exists(path):
with open(path, encoding="utf-8", errors="replace") as f:
return f.read()
html = _polite_fetch(link, wait)
with open(path, "w", encoding="utf-8") as f:
f.write(html)
return html
def download(args):
"""--download <links.txt | sitemap.xml | link> [more ...] [--match text] [--wait seconds] [--save folder]"""
opts = {"--match": "", "--wait": "1", "--save": "pages"}
sources = []
i = 0
while i < len(args):
if args[i] in opts and i + 1 < len(args):
opts[args[i]] = args[i + 1]
i += 2
else:
sources.append(args[i])
i += 1
wait = float(opts["--wait"])
found = [link for src in sources for link in _links(src, wait)]
todo = [link for link in dict.fromkeys(found) if opts["--match"] in link]
os.makedirs(opts["--save"], exist_ok=True)
print(f"{len(todo)} pages, kept in {opts['--save']}/", file=sys.stderr)
for link in todo:
try:
print(json.dumps({"page": link, "fields": extract(_saved(link, opts["--save"], wait))}, ensure_ascii=False), flush=True)
except Exception as e:
print(f"{link}: {e}", file=sys.stderr)
def main(sources):
if not sources:
sys.exit("Usage: python3 extract.py <link or file> [more ...]")
if sources[0] == "--download":
download(sources[1:])
return
if len(sources) == 1:
print(json.dumps(extract(read(sources[0])), ensure_ascii=False, indent=2))
return
for src in sources:
print(json.dumps({"page": src, "fields": extract(read(src))}, ensure_ascii=False))
if __name__ == "__main__":
main(sys.argv[1:])
How you use it
$ pip install regex
$ python3 extract.py --download https://shop.example/sitemap.xml > results.jsonl
5 pages, kept in pages/What comes out, in results.jsonl: one line per page
{"page": "https://shop.example/p/mug", "fields": {"name": {"value": "Left-handed coffee mug", "confidence": "sure"}, "price": {"value": "8,99", "confidence": "sure"}, "old price": {"value": null, "confidence": "none"}, "brand": {"value": "Mugs & More", "confidence": "sure"}, "image url": {"value": "https://shop.example/img/mug-800.jpg", "confidence": "sure"}, "stock": {"value": "In stock", "confidence": "sure"}}}
{"page": "https://shop.example/p/lamp", "fields": {"name": {"value": "Table lamp that hums", "confidence": "sure"}, "price": {"value": "15,99", "confidence": "sure"}, "old price": {"value": "19,99", "confidence": "sure"}, "brand": {"value": "Lumen", "confidence": "sure"}, "image url": {"value": "https://shop.example/img/lamp-800.jpg", "confidence": "sure"}, "stock": {"value": "In stock", "confidence": "sure"}}}
{"page": "https://shop.example/p/notebook", "fields": {"name": {"value": "Notebook with no pages", "confidence": "sure"}, "price": {"value": "5,25", "confidence": "sure"}, "old price": {"value": "6,50", "confidence": "sure"}, "brand": {"value": "Paper Co.", "confidence": "sure"}, "image url": {"value": "https://shop.example/img/notebook-800.jpg", "confidence": "sure"}, "stock": {"value": "Sold out", "confidence": "sure"}}}
…- sure the main rule found it and the backups agree
- fallback only a backup found it: this page looks different
- conflict the rules disagree: check this page by hand
- none not on this page
A page that isn't sure everywhere is the best next example to send me.
No rules yet? Send me a few of your pages, free.