euscan: respect robots.txt

Signed-off-by: Corentin Chary <corentincj@iksaif.net>

pym/euscan/handlers/generic.py

@@ -57,6 +57,9 @@ def scan_directory_recursive(cpv, url, steps):
     except IOError:
         return []
 
+    if not fp:
+        return []
+
     data = fp.read()
 
     results = []

pym/euscan/handlers/php.py

@@ -38,6 +38,9 @@ def scan(cpv, url):
     except IOError:
         return []
 
+    if not fp:
+        return []
+
     data = fp.read()
 
     dom = xml.dom.minidom.parseString(data)

pym/euscan/handlers/rubygem.py

@@ -33,6 +33,9 @@ def scan(cpv, url):
     except IOError:
         return []
 
+    if not fp:
+        return []
+
     data = fp.read()
     versions = json.loads(data)