Merge pull request #375 from dalf/searxng_extra

SearXNG: searx_extra

1 files changed, 31 insertions, 0 deletions

diff --git a/searxng_extra/update/update_ahmia_blacklist.py b/searxng_extra/update/update_ahmia_blacklist.py
new file mode 100755
index 000000000..f7695deae
--- /dev/null
+++ b/searxng_extra/update/update_ahmia_blacklist.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# This script saves Ahmia's blacklist for onion sites.
+# More info in https://ahmia.fi/blacklist/
+
+# set path
+from os.path import join
+
+import requests
+from searx import searx_dir
+
+URL = 'https://ahmia.fi/blacklist/banned/'
+
+
+def fetch_ahmia_blacklist():
+    resp = requests.get(URL, timeout=3.0)
+    if resp.status_code != 200:
+        raise Exception("Error fetching Ahmia blacklist, HTTP code " + resp.status_code)
+    else:
+        blacklist = resp.text.split()
+        return blacklist
+
+
+def get_ahmia_blacklist_filename():
+    return join(join(searx_dir, "data"), "ahmia_blacklist.txt")
+
+
+blacklist = fetch_ahmia_blacklist()
+with open(get_ahmia_blacklist_filename(), "w") as f:
+    f.write('\n'.join(blacklist))