dns/py-dns-crawler: Add py-dns-crawler 1.4.9

Despite the name, the crawler gets info for more services than just DNS:
- DNS:
  - all A/AAAA records (for the 2nd level domain and www.subdomain), annotated
    with GeoIP
  - TXT records (with SPF and DMARC parsed for easier filtering)
  - TLSA (for the 2nd level domain and www.subdomain)
  - MX
  - DNSSEC validation
  - nameservers:
    - each server IP annotated with GeoIP
    - HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs)
  - users can add custom additional RRs in the config file
- E-mail (for every server from MX):
  - SMTP server banners (optional, ports are configurable)
  - TLSA records
- Web:
  - HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP
    from A/AAAA records
  - certificate info for HTTPS (optionally with an entire cert chain)
  - webpage content (optional)
  - everything of the above is saved for each step in the redirect history --
    the crawler follows redirects until it gets a non-redirecting status or hits
    a configurable limit

WWW: https://gitlab.nic.cz/adam/dns-crawler

5 files changed, 109 insertions, 0 deletions

diff --git a/dns/Makefile b/dns/Makefile
index 10be006cbbee..71c200092077 100644
--- a/dns/Makefile
+++ b/dns/Makefile
@@ -176,6 +176,7 @@
     SUBDIR += py-adns
     SUBDIR += py-aiodns
     SUBDIR += py-cloudflare
+    SUBDIR += py-dns-crawler
     SUBDIR += py-dns-lexicon
     SUBDIR += py-dnspython
     SUBDIR += py-dnspython2
diff --git a/dns/py-dns-crawler/Makefile b/dns/py-dns-crawler/Makefile
new file mode 100644
index 000000000000..b0b1ece176ec
--- /dev/null
+++ b/dns/py-dns-crawler/Makefile
@@ -0,0 +1,39 @@
+# Created by: Po-Chuan Hsieh <sunpoet@FreeBSD.org>
+
+PORTNAME=	dns-crawler
+PORTVERSION=	1.4.9
+CATEGORIES=	dns python
+MASTER_SITES=	CHEESESHOP
+PKGNAMEPREFIX=	${PYTHON_PKGNAMEPREFIX}
+
+MAINTAINER=	sunpoet@FreeBSD.org
+COMMENT=	Crawler for getting info about DNS domains and services
+
+LICENSE=	GPLv3
+LICENSE_FILE=	${WRKSRC}/LICENSE
+
+BUILD_DEPENDS=	${PYTHON_PKGNAMEPREFIX}setuptools_scm>=0:devel/py-setuptools_scm@${PY_FLAVOR}
+RUN_DEPENDS=	${PYTHON_PKGNAMEPREFIX}asn1crypto>=1.3.0:devel/py-asn1crypto@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}cert-human>=1.0.7:security/py-cert-human@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}cryptography>=2.8:security/py-cryptography@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}dnspython>=0:dns/py-dnspython@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}ecdsa>=0.15:security/py-ecdsa@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}forcediphttpsadapter>=1.0.1:www/py-forcediphttpsadapter@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}GeoIP2>=3.0.0:net/py-GeoIP2@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}hstspreload>=0:www/py-hstspreload@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}idna>=2.9:dns/py-idna@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}pyaml>=19.12.0:textproc/py-pyaml@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}pycryptodome>=3.9.7:security/py-pycryptodome@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}pyicu>=2.4.3:devel/py-pyicu@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}openssl>=19.1.0:security/py-openssl@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}redis>=3.4.1:databases/py-redis@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}requests>=2.23.0:www/py-requests@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}requests-toolbelt>=0.9.1:www/py-requests-toolbelt@${PY_FLAVOR} \
+		${PYTHON_PKGNAMEPREFIX}rq>=1.2.2:devel/py-rq@${PY_FLAVOR}
+
+USES=		python:3.6+
+USE_PYTHON=	autoplist concurrent distutils
+
+NO_ARCH=	yes
+
+.include <bsd.port.mk>
diff --git a/dns/py-dns-crawler/distinfo b/dns/py-dns-crawler/distinfo
new file mode 100644
index 000000000000..bd3eda879206
--- /dev/null
+++ b/dns/py-dns-crawler/distinfo
@@ -0,0 +1,3 @@
+TIMESTAMP = 1622213966
+SHA256 (dns-crawler-1.4.9.tar.gz) = f59340c4ef1769a04a4f9ebc07f27114fae289b91d287433a42c326958afb71d
+SIZE (dns-crawler-1.4.9.tar.gz) = 66705
diff --git a/dns/py-dns-crawler/files/patch-setup.py b/dns/py-dns-crawler/files/patch-setup.py
new file mode 100644
index 000000000000..dbc26bca9f5c
--- /dev/null
+++ b/dns/py-dns-crawler/files/patch-setup.py
@@ -0,0 +1,41 @@
+--- setup.py.orig	2020-07-10 06:16:32 UTC
++++ setup.py
+@@ -39,23 +39,23 @@ setup(
+         ]
+     },
+     install_requires=[
+-        "asn1crypto==1.3.0",
+-        "cert_human==1.0.7",
+-        "cryptography==2.8",
++        "asn1crypto>=1.3.0",
++        "cert_human>=1.0.7",
++        "cryptography>=2.8",
+         "dnspython",
+-        "ecdsa==0.15",
+-        "forcediphttpsadapter==1.0.1",
+-        "geoip2==3.0.0",
++        "ecdsa>=0.15",
++        "forcediphttpsadapter>=1.0.1",
++        "geoip2>=3.0.0",
+         "hstspreload",
+-        "idna==2.9",
+-        "pyaml==19.12.0",
+-        "PyICU==2.4.3",
+-        "pycryptodome==3.9.7",
+-        "pyopenssl==19.1.0",
+-        "redis==3.4.1",
+-        "requests_toolbelt==0.9.1",
+-        "requests==2.23.0",
+-        "rq==1.2.2",
++        "idna>=2.9",
++        "pyaml>=19.12.0",
++        "PyICU>=2.4.3",
++        "pycryptodome>=3.9.7",
++        "pyopenssl>=19.1.0",
++        "redis>=3.4.1",
++        "requests_toolbelt>=0.9.1",
++        "requests>=2.23.0",
++        "rq>=1.2.2",
+     ],
+     keywords=["crawler", "dns", "http", "https"],
+     classifiers=[
diff --git a/dns/py-dns-crawler/pkg-descr b/dns/py-dns-crawler/pkg-descr
new file mode 100644
index 000000000000..b309f887ca79
--- /dev/null
+++ b/dns/py-dns-crawler/pkg-descr
@@ -0,0 +1,25 @@
+Despite the name, the crawler gets info for more services than just DNS:
+- DNS:
+  - all A/AAAA records (for the 2nd level domain and www.subdomain), annotated
+    with GeoIP
+  - TXT records (with SPF and DMARC parsed for easier filtering)
+  - TLSA (for the 2nd level domain and www.subdomain)
+  - MX
+  - DNSSEC validation
+  - nameservers:
+    - each server IP annotated with GeoIP
+    - HOSTNAME.BIND, VERSION.BIND, AUTHORS.BIND and fortune (also for all IPs)
+  - users can add custom additional RRs in the config file
+- E-mail (for every server from MX):
+  - SMTP server banners (optional, ports are configurable)
+  - TLSA records
+- Web:
+  - HTTP status & headers (inc. parsed cookies) for ports 80 & 443 on each IP
+    from A/AAAA records
+  - certificate info for HTTPS (optionally with an entire cert chain)
+  - webpage content (optional)
+  - everything of the above is saved for each step in the redirect history --
+    the crawler follows redirects until it gets a non-redirecting status or hits
+    a configurable limit
+
+WWW: https://gitlab.nic.cz/adam/dns-crawler