From 3e3672e0790266fc7f2482fdd854d7789a915d4d Mon Sep 17 00:00:00 2001 From: jibe-b Date: Sat, 23 Sep 2017 14:16:06 +0200 Subject: [add] arxiv engine --- searx/engines/arxiv.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 searx/engines/arxiv.py (limited to 'searx/engines') diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py new file mode 100644 index 000000000..cbeac0089 --- /dev/null +++ b/searx/engines/arxiv.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +""" + ArXiV (Scientific preprints) + @website https://axiv.org + @provide-api yes (export.arxiv.org/api/query) + @using-api yes + @results XML-RSS + @stable yes + @parse url, title, publishedDate, content + More info on api: https://arxiv.org/help/api/user-manual +""" + +from lxml import html +from datetime import datetime +from searx.url_utils import urlencode + + +categories = ['science'] + +base_url = 'http://export.arxiv.org/api/query?search_query=all:'\ + + '{query}&start={offset}&max_results={number_of_results}' + +# engine dependent config +number_of_results = 10 + + +def request(query, params): + # basic search + offset = (params['pageno'] - 1) * number_of_results + + string_args = dict(query=query, + offset=offset, + number_of_results=number_of_results) + + params['url'] = base_url.format(**string_args) + + return params + + +def response(resp): + results = [] + + search_results = html.fromstring(resp.text.encode('utf-8')).xpath('//entry') + + for entry in search_results: + title = entry.xpath('.//title')[0].text + + url = entry.xpath('.//id')[0].text + + content = entry.xpath('.//summary')[0].text + + # If a doi is available, add it to the snipppet + try: + doi = entry.xpath('.//link[@title="doi"]')[0].text + content = 'DOI: ' + doi + ' Abstract: ' + content + except: + pass + + if len(content) > 300: + content = content[0:300] + "..." + # TODO: center snippet on query term + + publishedDate = datetime.strptime(entry.xpath('.//published')[0].text, '%Y-%m-%dT%H:%M:%SZ') + + res_dict = {'url': url, + 'title': title, + 'publishedDate': publishedDate, + 'content': content} + + results.append(res_dict) + + return results -- cgit v1.2.3 From 5278fa666c193e5ccb30e7b5f8dddf1b053f97ca Mon Sep 17 00:00:00 2001 From: jibe-b Date: Wed, 27 Sep 2017 16:01:31 +0200 Subject: [enh] use format to concatenate strings --- searx/engines/arxiv.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py index cbeac0089..4b6440cbc 100644 --- a/searx/engines/arxiv.py +++ b/searx/engines/arxiv.py @@ -48,14 +48,16 @@ def response(resp): url = entry.xpath('.//id')[0].text - content = entry.xpath('.//summary')[0].text + content_string = '{doi_content}{abstract_content}' + + abstract = entry.xpath('.//summary')[0].text # If a doi is available, add it to the snipppet try: - doi = entry.xpath('.//link[@title="doi"]')[0].text - content = 'DOI: ' + doi + ' Abstract: ' + content + doi_content = entry.xpath('.//link[@title="doi"]')[0].text + content = content_string.format(doi_content=doi_content, abstract_content=abstract_content) except: - pass + content = content_string.format(abstract_content=abstract_content) if len(content) > 300: content = content[0:300] + "..." -- cgit v1.2.3 From e391b2d970a19cdc39dd550929e91ace4aee8832 Mon Sep 17 00:00:00 2001 From: jibe-b Date: Wed, 27 Sep 2017 16:05:28 +0200 Subject: [fix] remove .encode for python3 compatibility --- searx/engines/arxiv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/engines') diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py index 4b6440cbc..826b77690 100644 --- a/searx/engines/arxiv.py +++ b/searx/engines/arxiv.py @@ -41,7 +41,7 @@ def request(query, params): def response(resp): results = [] - search_results = html.fromstring(resp.text.encode('utf-8')).xpath('//entry') + search_results = html.fromstring(resp.text).xpath('//entry') for entry in search_results: title = entry.xpath('.//title')[0].text -- cgit v1.2.3 From 9c2b7a82f0c515fd1df88ed80349eda7f49e0825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 1 Nov 2017 12:28:18 +0100 Subject: minor fixes of arxiv Closes #1050 --- searx/engines/arxiv.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py index 826b77690..5ef84f0c1 100644 --- a/searx/engines/arxiv.py +++ b/searx/engines/arxiv.py @@ -2,7 +2,7 @@ """ ArXiV (Scientific preprints) - @website https://axiv.org + @website https://arxiv.org @provide-api yes (export.arxiv.org/api/query) @using-api yes @results XML-RSS @@ -41,7 +41,8 @@ def request(query, params): def response(resp): results = [] - search_results = html.fromstring(resp.text).xpath('//entry') + dom = html.fromstring(resp.content) + search_results = dom.xpath('//entry') for entry in search_results: title = entry.xpath('.//title')[0].text @@ -49,15 +50,15 @@ def response(resp): url = entry.xpath('.//id')[0].text content_string = '{doi_content}{abstract_content}' - + abstract = entry.xpath('.//summary')[0].text # If a doi is available, add it to the snipppet try: doi_content = entry.xpath('.//link[@title="doi"]')[0].text - content = content_string.format(doi_content=doi_content, abstract_content=abstract_content) + content = content_string.format(doi_content=doi_content, abstract_content=abstract) except: - content = content_string.format(abstract_content=abstract_content) + content = content_string.format(doi_content="", abstract_content=abstract) if len(content) > 300: content = content[0:300] + "..." -- cgit v1.2.3 From df0d915806b6e4488099130cd1d7fb1775fe475c Mon Sep 17 00:00:00 2001 From: jibe-b Date: Fri, 22 Sep 2017 22:09:33 +0200 Subject: [add] pubmed engine --- searx/engines/pubmed.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 searx/engines/pubmed.py (limited to 'searx/engines') diff --git a/searx/engines/pubmed.py b/searx/engines/pubmed.py new file mode 100644 index 000000000..abb59d2ed --- /dev/null +++ b/searx/engines/pubmed.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +""" + PubMed (Scholar publications) + @website https://www.ncbi.nlm.nih.gov/pubmed/ + @provide-api yes (https://www.ncbi.nlm.nih.gov/home/develop/api/) + @using-api yes + @results XML + @stable yes + @parse url, title, publishedDate, content + More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/ +""" + +from lxml import etree +from datetime import datetime +from searx.url_utils import urlencode, urlopen + + +categories = ['science'] + +base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi'\ + + '?db=pubmed&{query}&retstart={offset}&retmax={hits}' + +# engine dependent config +number_of_results = 10 +pubmed_url = 'https://www.ncbi.nlm.nih.gov/pubmed/' + + +def request(query, params): + # basic search + offset = (params['pageno'] - 1) * number_of_results + + string_args = dict(query=urlencode({'term': query}), + offset=offset, + hits=number_of_results) + + params['url'] = base_url.format(**string_args) + + return params + + +def response(resp): + results = [] + + # First retrieve notice of each result + pubmed_retrieve_api_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?'\ + + 'db=pubmed&retmode=xml&id={pmids_string}' + + # handle Python2 vs Python3 management of bytes and strings + try: + pmids_results = etree.XML(resp.text.encode('utf-8')) + except AttributeError: + pmids_results = etree.XML(resp.text) + + pmids = pmids_results.xpath('//eSearchResult/IdList/Id') + pmids_string = '' + + for item in pmids: + pmids_string += item.text + ',' + + retrieve_notice_args = dict(pmids_string=pmids_string) + + retrieve_url_encoded = pubmed_retrieve_api_url.format(**retrieve_notice_args) + + search_results_xml = urlopen(retrieve_url_encoded).read() + search_results = etree.XML(search_results_xml).xpath('//PubmedArticleSet/PubmedArticle/MedlineCitation') + + for entry in search_results: + title = entry.xpath('.//Article/ArticleTitle')[0].text + + pmid = entry.xpath('.//PMID')[0].text + url = pubmed_url + pmid + + try: + content = entry.xpath('.//Abstract/AbstractText')[0].text + except: + content = 'No abstract is available for this publication.' + + # If a doi is available, add it to the snipppet + try: + doi = entry.xpath('.//ELocationID[@EIdType="doi"]')[0].text + content = 'DOI: ' + doi + ' Abstract: ' + content + except: + pass + + if len(content) > 300: + content = content[0:300] + "..." + # TODO: center snippet on query term + + publishedDate = datetime.strptime(entry.xpath('.//DateCreated/Year')[0].text + + '-' + entry.xpath('.//DateCreated/Month')[0].text + + '-' + entry.xpath('.//DateCreated/Day')[0].text, '%Y-%m-%d') + + res_dict = {'url': url, + 'title': title, + 'publishedDate': publishedDate, + 'content': content} + + results.append(res_dict) + + return results -- cgit v1.2.3 From d20bba6dc74ded16556acf2a404d01ec47455ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 1 Nov 2017 14:20:47 +0100 Subject: minor fixes of pubmed engine Closes #1045 --- searx/engines/pubmed.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/pubmed.py b/searx/engines/pubmed.py index abb59d2ed..6451f1467 100644 --- a/searx/engines/pubmed.py +++ b/searx/engines/pubmed.py @@ -11,9 +11,11 @@ More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/ """ +from flask_babel import gettext from lxml import etree from datetime import datetime -from searx.url_utils import urlencode, urlopen +from searx.url_utils import urlencode +from searx.poolrequests import get categories = ['science'] @@ -46,12 +48,7 @@ def response(resp): pubmed_retrieve_api_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?'\ + 'db=pubmed&retmode=xml&id={pmids_string}' - # handle Python2 vs Python3 management of bytes and strings - try: - pmids_results = etree.XML(resp.text.encode('utf-8')) - except AttributeError: - pmids_results = etree.XML(resp.text) - + pmids_results = etree.XML(resp.content) pmids = pmids_results.xpath('//eSearchResult/IdList/Id') pmids_string = '' @@ -62,7 +59,7 @@ def response(resp): retrieve_url_encoded = pubmed_retrieve_api_url.format(**retrieve_notice_args) - search_results_xml = urlopen(retrieve_url_encoded).read() + search_results_xml = get(retrieve_url_encoded).content search_results = etree.XML(search_results_xml).xpath('//PubmedArticleSet/PubmedArticle/MedlineCitation') for entry in search_results: @@ -74,12 +71,12 @@ def response(resp): try: content = entry.xpath('.//Abstract/AbstractText')[0].text except: - content = 'No abstract is available for this publication.' + content = gettext('No abstract is available for this publication.') # If a doi is available, add it to the snipppet try: doi = entry.xpath('.//ELocationID[@EIdType="doi"]')[0].text - content = 'DOI: ' + doi + ' Abstract: ' + content + content = 'DOI: {doi} Abstract: {content}'.format(doi=doi, content=content) except: pass -- cgit v1.2.3 From 5954a8e16a64a369072a7487f62b6396a451ae5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=83=C2=A9mi=20V=C3=83=C2=A1nyi?= Date: Wed, 1 Nov 2017 16:50:27 +0100 Subject: minor fix of BASE engine --- searx/engines/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/engines') diff --git a/searx/engines/base.py b/searx/engines/base.py index ff006a3bc..be0b7d247 100755 --- a/searx/engines/base.py +++ b/searx/engines/base.py @@ -73,7 +73,7 @@ def request(query, params): def response(resp): results = [] - search_results = etree.XML(resp.text) + search_results = etree.XML(resp.content) for entry in search_results.xpath('./result/doc'): content = "No description available" -- cgit v1.2.3