python3: Use hash-checking mode when installing host pip packages

In hash-checking mode[1], pip will verify downloaded package archives
(source tarballs in our case) against known SHA256 hashes before
installing the packages.

As a consequence, this requires the use of requirements files[2] and
pinning packages to known versions.

The syntax for package Makefiles has changed slightly;
HOST_PYTHON3_PACKAGE_BUILD_DEPENDS no longer accepts requirement
specifiers like "foo>=1.0", only requirements file names (which are the
same as package names in the most common case).

This also updates affected packages, in particular:

* python-zipp: "setuptools_scm[toml]" has been split into
  "setuptools-scm toml" to reuse the requirements file for
  setuptools-scm (the extra depends installed by "setuptools_scm[toml]"
  is toml).

* python-pycparser: This previously used ply 3.10, whereas the
  requirements file will now install 3.11.

[1]: https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode
[2]: https://pip.pypa.io/en/stable/user_guide/#requirements-files

Signed-off-by: Jeffery To <jeffery.to@gmail.com>

diff --git a/lang/python/README.md b/lang/python/README.md
index ac3bf51c76bd70d2b2aafaab27be46a6dc79532a..f5b07c01cf132053bc93a5001777bc4e365f994a 100644 (file)
--- a/lang/python/README.md
+++ b/lang/python/README.md
@@ -330,13 +330,45 @@ endef
 
 These can be installed via pip and ideally they should only be installed like this, because it's a bit simpler than running them through the OpenWrt build system.
 
-Which is why [for example] if you need python cffi on the host build, it's easier to just add it via:
+#### Requirements files
+
+All host-side Python packages are installed with pip using [requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files), with [hash-checking mode](https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode) enabled. These requirements files are stored in the [host-pip-requirements](./host-pip-requirements) directory.
+
+Each requirements file is named after the Python package it installs and contains the package's pinned version and `--hash` option. The `--hash` option value is the SHA256 hash of the package's source tarball; this value can be found on [pypi.org](https://pypi.org/).
+
+For example, the requirements file for setuptools-scm ([setuptools-scm.txt](./host-pip-requirements/setuptools-scm.txt)) contains:
+
+```
+setuptools-scm==4.1.2 --hash=sha256:a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8
+```
+
+If the Python package to be installed depends on other Python packages, those dependencies, with their pinned versions and `--hash` options, also need to be specified in the requirements file. For instance, [cffi.txt](./host-pip-requirements/cffi.txt) includes information for pycparser because pycparser is a dependency of cffi and will be installed with cffi.
+
+There are two types of requirements files in [host-pip-requirements](./host-pip-requirements):
+
+* Installs the latest version of a Python package.
+
+  A requirements file of this type is named with the package name only (for example, [setuptools-scm.txt](./host-pip-requirements/setuptools-scm.txt)) and is used when there is no strict version requirement.
+
+  These files will be updated as newer versions of the Python packages are available.
+
+* Installs a specific version of a Python package.
+
+  A requirements file of this type is named with the package name and version number (for example, [Django-1.11.txt](./host-pip-requirements/Django-1.11.txt)) and is used when a specific (usually older) version is required.
+
+  Installing the latest versions of packages is preferred over specific versions whenever possible.
+
+#### Installing host-side Python packages
+
+Set `HOST_PYTHON3_PACKAGE_BUILD_DEPENDS` to the names of one or more requirements files in [host-pip-requirements](./host-pip-requirements), without the directory path or ".txt" extension.
+
+For example:
+
 ```
 PKG_BUILD_PARALLEL:=0
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi==$(PKG_VERSION)"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm
 ```
-[cffi is one of those packages that needs a host-side package installed].
 
-This works reasonably well in the current OpenWrt build system, as binaries get built for this package and get installed in the staging-dir `$(STAGING_DIR)/usr/lib/pythonX.Y/site-packages`.
+The Python package will be installed in `$(STAGING_DIR_HOSTPKG)/lib/pythonX.Y/site-packages`.
 
-`PKG_BUILD_PARALLEL:=0` is necessary because installing packages with multiple concurrent pip processes can lead to [errors or unexpected results](https://github.com/pypa/pip/issues/2361).
+Parallel builds need to be disabled because installing packages with multiple concurrent pip processes can lead to [errors or unexpected results](https://github.com/pypa/pip/issues/2361).

diff --git a/lang/python/bcrypt/Makefile b/lang/python/bcrypt/Makefile
index 3a8bdf7d920592d2c4bdef1bf47387a81598ad74..f480112152e579d8c82270795395099f86b1a2ea 100644 (file)
--- a/lang/python/bcrypt/Makefile
+++ b/lang/python/bcrypt/Makefile
@@ -17,7 +17,7 @@ PKG_LICENSE_FILES:=LICENSE
 
 PKG_BUILD_DEPENDS:=libffi/host
 PKG_BUILD_PARALLEL:=0
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.1"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi  # cffi>=1.1
 
 include ../pypi.mk
 include $(INCLUDE_DIR)/package.mk

diff --git a/lang/python/host-pip-requirements/Cython.txt b/lang/python/host-pip-requirements/Cython.txt
new file mode 100644 (file)
index 0000000..fb7a7f4
--- /dev/null
+++ b/lang/python/host-pip-requirements/Cython.txt
@@ -0,0 +1 @@
+Cython==0.29.21 --hash=sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad

diff --git a/lang/python/host-pip-requirements/Django-1.11.txt b/lang/python/host-pip-requirements/Django-1.11.txt
new file mode 100644 (file)
index 0000000..48e981e
--- /dev/null
+++ b/lang/python/host-pip-requirements/Django-1.11.txt
@@ -0,0 +1,2 @@
+Django==1.11.29 --hash=sha256:4200aefb6678019a0acf0005cd14cfce3a5e6b9b90d06145fcdd2e474ad4329c
+pytz==2020.1 --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048

diff --git a/lang/python/host-pip-requirements/cffi.txt b/lang/python/host-pip-requirements/cffi.txt
new file mode 100644 (file)
index 0000000..869ce38
--- /dev/null
+++ b/lang/python/host-pip-requirements/cffi.txt
@@ -0,0 +1,2 @@
+cffi==1.14.2 --hash=sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b
+pycparser==2.20 --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0

diff --git a/lang/python/host-pip-requirements/ply.txt b/lang/python/host-pip-requirements/ply.txt
new file mode 100644 (file)
index 0000000..807e19e
--- /dev/null
+++ b/lang/python/host-pip-requirements/ply.txt
@@ -0,0 +1 @@
+ply==3.11 --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3

diff --git a/lang/python/host-pip-requirements/setuptools-scm.txt b/lang/python/host-pip-requirements/setuptools-scm.txt
new file mode 100644 (file)
index 0000000..35f77a2
--- /dev/null
+++ b/lang/python/host-pip-requirements/setuptools-scm.txt
@@ -0,0 +1 @@
+setuptools-scm==4.1.2 --hash=sha256:a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8

diff --git a/lang/python/host-pip-requirements/toml.txt b/lang/python/host-pip-requirements/toml.txt
new file mode 100644 (file)
index 0000000..abff675
--- /dev/null
+++ b/lang/python/host-pip-requirements/toml.txt
@@ -0,0 +1 @@
+toml==0.10.1 --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f

diff --git a/lang/python/numpy/Makefile b/lang/python/numpy/Makefile
index d073754e5e88ac8eed9c6ea7878f0e16591deb98..7cf6f72c04c7ccc9eb87db44eeebb48b983cc3a3 100644 (file)
--- a/lang/python/numpy/Makefile
+++ b/lang/python/numpy/Makefile
@@ -22,7 +22,7 @@ PKG_CPE_ID:=cpe:/a:numpy:numpy
 # yes, zip... sigh
 PYPI_SOURCE_EXT:=zip
 PKG_BUILD_PARALLEL:=0
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="Cython==0.29.21"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=Cython  # Cython>=0.29.21
 
 include ../pypi.mk
 include $(INCLUDE_DIR)/package.mk

diff --git a/lang/python/python-cryptography/Makefile b/lang/python/python-cryptography/Makefile
index 5693d57343fc47e17a4f3516046339b9bf21be5a..ba3335b651a0d3cd6f8a40e6d3b875b1c5b61988 100644 (file)
--- a/lang/python/python-cryptography/Makefile
+++ b/lang/python/python-cryptography/Makefile
@@ -21,7 +21,7 @@ PKG_MAINTAINER:=Jeffery To <jeffery.to@gmail.com>, Alexandru Ardelean <ardeleana
 PKG_BUILD_DEPENDS:=libffi/host
 PKG_BUILD_PARALLEL:=0
 
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.8,!=1.11.3"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi  # cffi>=1.8,!=1.11.3
 
 include ../pypi.mk
 include $(INCLUDE_DIR)/package.mk

diff --git a/lang/python/python-jsonschema/Makefile b/lang/python/python-jsonschema/Makefile
index efcb76fc6a34586ad0304cadbafcab523cc71ac8..be38dd1247ac50029912ff22f76de8163a5bb668 100644 (file)
--- a/lang/python/python-jsonschema/Makefile
+++ b/lang/python/python-jsonschema/Makefile
@@ -12,7 +12,7 @@ PKG_LICENSE:=MIT
 PKG_LICENSE_FILES:=COPYING
 
 PKG_BUILD_PARALLEL:=0
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools_scm
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm
 
 include ../pypi.mk
 include $(INCLUDE_DIR)/package.mk

diff --git a/lang/python/python-pycparser/Makefile b/lang/python/python-pycparser/Makefile
index ce19a5ba7aa07366fc13818aedf3fd0fbdfbf9f3..99cb104ace0fcff2e3f2780f2350da63e4af2214 100644 (file)
--- a/lang/python/python-pycparser/Makefile
+++ b/lang/python/python-pycparser/Makefile
@@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=python-pycparser
 PKG_VERSION:=2.20
-PKG_RELEASE:=3
+PKG_RELEASE:=4
 
 PYPI_NAME:=pycparser
 PKG_HASH:=2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0
@@ -19,7 +19,7 @@ PKG_LICENSE_FILES:=LICENSE
 PKG_MAINTAINER:=Jeffery To <jeffery.to@gmail.com>
 
 PKG_BUILD_PARALLEL:=0
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="ply==3.10"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=ply  # ply==3.10
 
 include ../pypi.mk
 include $(INCLUDE_DIR)/package.mk

diff --git a/lang/python/python-pynacl/Makefile b/lang/python/python-pynacl/Makefile
index 3d3d02a2e351e4c11ce91d514a11be3d2d031244..a879a99546a66df8926fa859933631e7a16680ec 100644 (file)
--- a/lang/python/python-pynacl/Makefile
+++ b/lang/python/python-pynacl/Makefile
@@ -14,7 +14,7 @@ PKG_LICENSE_FILES:=LICENSE
 PKG_BUILD_DEPENDS:=libffi/host
 PKG_BUILD_PARALLEL:=0
 
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.4.1"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi  # cffi>=1.4.1
 
 PYTHON3_PKG_SETUP_VARS:= SODIUM_INSTALL=system
 

diff --git a/lang/python/python-zipp/Makefile b/lang/python/python-zipp/Makefile
index fede7f14fb7be612fd521bc37024aad7bd53ca4a..0044a0cf824fdbf30f757708eea2376a9b6d5d66 100644 (file)
--- a/lang/python/python-zipp/Makefile
+++ b/lang/python/python-zipp/Makefile
@@ -12,7 +12,7 @@ PKG_LICENSE:=MIT
 PKG_LICENSE_FILES:=LICENSE
 
 PKG_BUILD_PARALLEL:=0
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="setuptools_scm[toml] >= 3.4.1"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm toml  # setuptools_scm[toml] >= 3.4.1
 
 include ../pypi.mk
 include $(INCLUDE_DIR)/package.mk

diff --git a/lang/python/python3-host.mk b/lang/python/python3-host.mk
index f5016ed570b8e77479eea40cef2452bfacc42c38..9dd5e13c438ebdf89cfe980b72470b3af0fcbf80 100644 (file)
--- a/lang/python/python3-host.mk
+++ b/lang/python/python3-host.mk
@@ -55,10 +55,11 @@ HOST_PYTHON3_PIP_CACHE_DIR:=$(DL_DIR)/pip-cache
 define HostPython3/PipInstall
        $(HOST_PYTHON3_VARS) \
        $(HOST_PYTHON3_PIP) \
-               --disable-pip-version-check \
                --cache-dir "$(HOST_PYTHON3_PIP_CACHE_DIR)" \
+               --disable-pip-version-check \
                install \
                --no-binary :all: \
+               --require-hashes \
                $(1)
   ifdef CONFIG_PYTHON3_HOST_PIP_CACHE_WORLD_READABLE
        $(FIND) $(HOST_PYTHON3_PIP_CACHE_DIR) -not -type d -exec chmod go+r  '{}' \;

diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk
index 36f4ab07e25b339c742b75640326a30a9540e921..5673d15bb003285294a8f5f2595fc25e5b08a245 100644 (file)
--- a/lang/python/python3-package.mk
+++ b/lang/python/python3-package.mk
@@ -194,13 +194,19 @@ PYTHON3_PKG_SETUP_GLOBAL_ARGS ?=
 PYTHON3_PKG_SETUP_ARGS ?= --single-version-externally-managed
 PYTHON3_PKG_SETUP_VARS ?=
 
+PYTHON3_PKG_HOST_PIP_INSTALL_ARGS = \
+       $(foreach req,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS), \
+               --requirement \
+               $(if $(findstring /,$(req)),$(req),$(python3_mk_path)host-pip-requirements/$(req).txt) \
+       )
+
 define Py3Build/FindStdlibDepends
        $(SHELL) $(python3_mk_path)python3-find-stdlib-depends.sh -n "$(PKG_NAME)" "$(PKG_BUILD_DIR)"
 endef
 
 define Py3Build/Compile/Default
-       $(if $(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS),
-               $(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS))
+       $(if $(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS), \
+               $(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS)) \
        )
        $(call Python3/ModSetup, \
                $(PYTHON3_PKG_SETUP_DIR), \

diff --git a/net/seafile-seahub/Makefile b/net/seafile-seahub/Makefile
index 28396aaa6014b90a1d0296f7599ab66dc2baef6e..a6705b37cda52d3e68cf45b16d1d5787aa692dbd 100644 (file)
--- a/net/seafile-seahub/Makefile
+++ b/net/seafile-seahub/Makefile
@@ -21,7 +21,7 @@ PKG_LICENSE_FILES:=LICENSE.txt
 
 PKG_BUILD_DIR:=$(BUILD_DIR)/seahub-$(PKG_VERSION)-server
 
-HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="Django~=1.11"
+HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=Django-1.11
 
 PKG_BUILD_PARALLEL:=0
 PYTHON3_PKG_BUILD:=0
@@ -76,7 +76,7 @@ MAKE_VARS += \
        DJANGO_ADMIN_PY="$(STAGING_DIR_HOSTPKG)/bin/django-admin"
 
 define Build/Compile
-       $(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS))
+       $(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS))
        $(call Build/Compile/Default,locale)
 endef