Merge pull request #565 from MarcAbonce/onions

New category: Onions

1 files changed, 33 insertions, 0 deletions

diff --git a/utils/fetch_ahmia_blacklist.py b/utils/fetch_ahmia_blacklist.py
new file mode 100755
index 000000000..3e393edbe
--- /dev/null
+++ b/utils/fetch_ahmia_blacklist.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+# This script saves Ahmia's blacklist for onion sites.
+# More info in https://ahmia.fi/blacklist/
+
+# set path
+from sys import path
+from os.path import realpath, dirname, join
+path.append(realpath(dirname(realpath(__file__)) + '/../'))
+
+#
+import requests
+from searx import searx_dir
+
+URL = 'https://ahmia.fi/blacklist/banned/'
+
+
+def fetch_ahmia_blacklist():
+    resp = requests.get(URL, timeout=3.0)
+    if resp.status_code != 200:
+        raise Exception("Error fetching Ahmia blacklist, HTTP code " + resp.status_code)
+    else:
+        blacklist = resp.text.split()
+        return blacklist
+
+
+def get_ahmia_blacklist_filename():
+    return join(join(searx_dir, "data"), "ahmia_blacklist.txt")
+
+
+blacklist = fetch_ahmia_blacklist()
+with open(get_ahmia_blacklist_filename(), "w") as f:
+    f.write('\n'.join(blacklist))