From 4d3c399ee985385e888ba068d973e4653d9f50b9 Mon Sep 17 00:00:00 2001 From: Kyle Anthony Williams Date: Sun, 27 Dec 2020 23:46:11 -0500 Subject: [feat] add bandcamp engine --- searx/engines/bandcamp.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++ searx/settings.yml | 5 ++++ 2 files changed, 77 insertions(+) create mode 100644 searx/engines/bandcamp.py diff --git a/searx/engines/bandcamp.py b/searx/engines/bandcamp.py new file mode 100644 index 000000000..b1b5214fe --- /dev/null +++ b/searx/engines/bandcamp.py @@ -0,0 +1,72 @@ +""" +Bandcamp (Music) + +@website https://bandcamp.com/ +@provide-api no +@results HTML +@parse url, title, content, publishedDate, embedded, thumbnail +""" + +from urllib.parse import urlencode, urlparse, parse_qs +from dateutil.parser import parse as dateparse +from lxml import html +from searx.utils import extract_text + +categories = ['music'] +paging = True + +base_url = "https://bandcamp.com/" +search_string = search_string = 'search?{query}&page={page}' +embedded_url = '''''' + + +def request(query, params): + '''pre-request callback + params: + method : POST/GET + headers : {} + data : {} # if method == POST + url : '' + category: 'search category' + pageno : 1 # number of the requested page + ''' + + search_path = search_string.format( + query=urlencode({'q': query}), + page=params['pageno']) + + params['url'] = base_url + search_path + + return params + + +def response(resp): + '''post-response callback + resp: requests response object + ''' + results = [] + tree = html.fromstring(resp.text) + search_results = tree.xpath('//li[contains(@class, "searchresult")]') + for result in search_results: + link = result.xpath('//div[@class="itemurl"]/a')[0] + result_id = parse_qs(urlparse(link.get('href')).query)["search_item_id"][0] + title = result.xpath('//div[@class="heading"]/a/text()')[0] + date = dateparse(result.xpath('//div[@class="released"]/text()')[0].replace("released ", "")) + content = result.xpath('//div[@class="subhead"]/text()')[0] + thumbnail = result.xpath('//div[@class="art"]/img/@src')[0] + new_result = { + "url": extract_text(link), + "title": title, + "content": content, + "publishedDate": date, + "thumbnail": thumbnail, + } + if "album" in result.classes: + new_result["embedded"] = embedded_url.format(type='album', result_id=result_id) + elif "track" in result.classes: + new_result["embedded"] = embedded_url.format(type='track', result_id=result_id) + results.append(new_result) + return results diff --git a/searx/settings.yml b/searx/settings.yml index c289cde5c..058867092 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -197,6 +197,11 @@ engines: # engine : base # shortcut : bs + - name: bandcamp + engine: bandcamp + shortcut: bc + categories: music + - name : wikipedia engine : wikipedia shortcut : wp -- cgit v1.2.3 From 062d589f865cf736620f4ff5d6a8476dfe980ba7 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Dec 2020 15:49:41 +0100 Subject: [fix] xpath expressions to grap all items from bandcamp's response I also found some items missing a thumbnail and I used text_extract for content and title, to remove unneeded whitespaces. BTW: added bandcamp's favicon Signed-off-by: Markus Heiser --- searx/engines/bandcamp.py | 15 ++++++++------- searx/static/themes/oscar/img/icons/bandcamp.png | Bin 0 -> 919 bytes 2 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 searx/static/themes/oscar/img/icons/bandcamp.png diff --git a/searx/engines/bandcamp.py b/searx/engines/bandcamp.py index b1b5214fe..dafb3ee16 100644 --- a/searx/engines/bandcamp.py +++ b/searx/engines/bandcamp.py @@ -51,19 +51,20 @@ def response(resp): tree = html.fromstring(resp.text) search_results = tree.xpath('//li[contains(@class, "searchresult")]') for result in search_results: - link = result.xpath('//div[@class="itemurl"]/a')[0] + link = result.xpath('.//div[@class="itemurl"]/a')[0] result_id = parse_qs(urlparse(link.get('href')).query)["search_item_id"][0] - title = result.xpath('//div[@class="heading"]/a/text()')[0] + title = result.xpath('.//div[@class="heading"]/a/text()') date = dateparse(result.xpath('//div[@class="released"]/text()')[0].replace("released ", "")) - content = result.xpath('//div[@class="subhead"]/text()')[0] - thumbnail = result.xpath('//div[@class="art"]/img/@src')[0] + content = result.xpath('.//div[@class="subhead"]/text()') new_result = { "url": extract_text(link), - "title": title, - "content": content, + "title": extract_text(title), + "content": extract_text(content), "publishedDate": date, - "thumbnail": thumbnail, } + thumbnail = result.xpath('.//div[@class="art"]/img/@src') + if thumbnail: + new_result['thumbnail'] = thumbnail[0] if "album" in result.classes: new_result["embedded"] = embedded_url.format(type='album', result_id=result_id) elif "track" in result.classes: diff --git a/searx/static/themes/oscar/img/icons/bandcamp.png b/searx/static/themes/oscar/img/icons/bandcamp.png new file mode 100644 index 000000000..2de405afe Binary files /dev/null and b/searx/static/themes/oscar/img/icons/bandcamp.png differ -- cgit v1.2.3 From f637bfc63530c36ff7d92284807f5fff5ff58bf9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 29 Dec 2020 15:58:18 +0100 Subject: [mod] oscar's "default" template should make use of result.thumbnail Some engine do have set result.img_src, other return a result.thumbnail. If result.img_src is unset and a result.thumbnail is given, show it to the UI. Signed-off-by: Markus Heiser --- searx/templates/oscar/result_templates/default.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searx/templates/oscar/result_templates/default.html b/searx/templates/oscar/result_templates/default.html index d743f928e..53cfee5cb 100644 --- a/searx/templates/oscar/result_templates/default.html +++ b/searx/templates/oscar/result_templates/default.html @@ -13,10 +13,10 @@ {%- endif -%} -{%- if result.img_src -%} +{%- if result.img_src or result.thumbnail -%}
{{- "" -}}
{{- "" -}} - + {%- if result.content %}

{{ result.content|safe }}

{% endif -%}
{{- "" -}}
-- cgit v1.2.3