lib: use separate packages for pycriu and crit

Newer versions of pip use an isolated virtual environment when building
Python projects. However, when the source code of CRIT is copied into
the isolated environment, the symlink for `../lib/py` (pycriu) becomes
invalid. As a workaround, we used the `--no-build-isolation` option for
`pip install`. However, this functionality has issues in some versions
of PIP [1, 2]. To fix this problem, this patch adds separate packages
for pycriu and crit, and each package is installed independently.

[1] https://github.com/pypa/pip/pull/8221
[2] https://github.com/pypa/pip/issues/8165#issuecomment-625401463

Signed-off-by: Radostin Stoyanov <rstoyanov@fedoraproject.org>

Makefile

@@ -164,7 +164,7 @@ HOSTCFLAGS		+= $(WARNINGS) $(DEFINES) -iquote include/
 export AFLAGS CFLAGS USERCLFAGS HOSTCFLAGS
 
 # Default target
-all: criu lib
+all: criu lib crit
 .PHONY: all
 
 #
@@ -288,9 +288,9 @@ clean mrproper:
 	$(Q) $(MAKE) $(build)=criu $@
 	$(Q) $(MAKE) $(build)=soccr $@
 	$(Q) $(MAKE) $(build)=lib $@
+	$(Q) $(MAKE) $(build)=crit $@
 	$(Q) $(MAKE) $(build)=compel $@
 	$(Q) $(MAKE) $(build)=compel/plugins $@
-	$(Q) $(MAKE) $(build)=lib $@
 .PHONY: clean mrproper
 
 clean-amdgpu_plugin:
@@ -337,6 +337,10 @@ amdgpu_plugin: criu
 	$(Q) $(MAKE) -C plugins/amdgpu all
 .PHONY: amdgpu_plugin
 
+crit: lib
+	$(Q) $(MAKE) -C crit
+.PHONY: crit
+
 #
 # Generating tar requires tag matched CRIU_VERSION.
 # If not found then simply use GIT's describe with
@@ -402,6 +406,7 @@ help:
 	@echo '    Targets:'
 	@echo '      all             - Build all [*] targets'
 	@echo '    * criu            - Build criu'
+	@echo '    * crit            - Build crit'
 	@echo '      zdtm            - Build zdtm test-suite'
 	@echo '      docs            - Build documentation'
 	@echo '      install         - Install CRIU (see INSTALL.md)'
@@ -425,11 +430,12 @@ lint:
 	flake8 --config=scripts/flake8.cfg test/zdtm.py
 	flake8 --config=scripts/flake8.cfg test/inhfd/*.py
 	flake8 --config=scripts/flake8.cfg test/others/rpc/config_file.py
-	flake8 --config=scripts/flake8.cfg lib/py/images/pb2dict.py
+	flake8 --config=scripts/flake8.cfg lib/pycriu/images/pb2dict.py
-	flake8 --config=scripts/flake8.cfg lib/py/images/images.py
+	flake8 --config=scripts/flake8.cfg lib/pycriu/images/images.py
 	flake8 --config=scripts/flake8.cfg scripts/criu-ns
 	flake8 --config=scripts/flake8.cfg test/others/criu-ns/run.py
-	flake8 --config=scripts/flake8.cfg crit/setup.py
+	flake8 --config=scripts/flake8.cfg crit/*.py
+	flake8 --config=scripts/flake8.cfg crit/crit/*.py
 	flake8 --config=scripts/flake8.cfg scripts/uninstall_module.py
 	flake8 --config=scripts/flake8.cfg coredump/ coredump/coredump
 	flake8 --config=scripts/flake8.cfg scripts/github-indent-warnings.py

Makefile.install

@@ -37,6 +37,10 @@ install-lib: lib
 	$(Q) $(MAKE) $(build)=lib install
 .PHONY: install-lib
 
+install-crit: lib
+	$(Q) $(MAKE) $(build)=crit install
+.PHONY: install-crit
+
 install-criu: criu
 	$(Q) $(MAKE) $(build)=criu install
 .PHONY: install-criu
@@ -50,12 +54,13 @@ install-compel: $(compel-install-targets)
 	$(Q) $(MAKE) $(build)=compel/plugins install
 .PHONY: install-compel
 
-install: install-man install-lib install-criu install-compel install-amdgpu_plugin ;
+install: install-man install-lib install-crit install-criu install-compel install-amdgpu_plugin ;
 .PHONY: install
 
 uninstall:
 	$(Q) $(MAKE) -C Documentation $@
 	$(Q) $(MAKE) $(build)=lib $@
+	$(Q) $(MAKE) $(build)=crit $@
 	$(Q) $(MAKE) $(build)=criu $@
 	$(Q) $(MAKE) $(build)=compel $@
 	$(Q) $(MAKE) $(build)=compel/plugins $@

coredump/pycriu

@@ -1 +1 @@
-../lib/py/
+../lib/pycriu

crit/.gitignore

@@ -1,2 +1,4 @@
 crit.egg-info/
 build/
+dist/
+version.py

crit/Makefile

@@ -0,0 +1,40 @@
+PYTHON_EXTERNALLY_MANAGED := $(shell $(PYTHON) -c 'import os, sysconfig; print(int(os.path.isfile(os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED"))))')
+PIP_BREAK_SYSTEM_PACKAGES := 0
+
+VERSION_FILE := $(if $(obj),$(addprefix $(obj)/,crit/version.py),crit/version.py)
+
+all-y	+= ${VERSION_FILE}
+cleanup-y	+= ${VERSION_FILE}
+
+${VERSION_FILE}:
+	$(Q) echo "__version__ = '${CRIU_VERSION}'" > $@
+
+install: ${VERSION_FILE}
+ifeq ($(PYTHON_EXTERNALLY_MANAGED),1)
+ifeq ($(PIP_BREAK_SYSTEM_PACKAGES),0)
+	$(E) "  SKIP INSTALL crit: Externally managed python environment (See PEP 668 for more information)"
+	$(E) "  Consider using PIP_BREAK_SYSTEM_PACKAGES=1 make install"
+else
+	$(E) "  INSTALL " crit
+	$(Q) $(PYTHON) -m pip install --upgrade --ignore-installed --prefix=$(DESTDIR)$(PREFIX) ./crit
+endif
+else
+	$(E) "  INSTALL " crit
+	$(Q) $(PYTHON) -m pip install --upgrade --ignore-installed --prefix=$(DESTDIR)$(PREFIX) ./crit
+endif
+.PHONY: install
+
+uninstall:
+ifeq ($(PYTHON_EXTERNALLY_MANAGED),1)
+ifeq ($(PIP_BREAK_SYSTEM_PACKAGES),0)
+	$(E) " SKIP UNINSTALL crit: Externally managed python environment (See PEP 668 for more information)"
+	$(E) " Consider using PIP_BREAK_SYSTEM_PACKAGES=1 make uninstall"
+else
+	$(E) " UNINSTALL" crit
+	$(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) crit
+endif
+else
+	$(E) " UNINSTALL" crit
+	$(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) crit
+endif
+.PHONY: uninstall

crit/crit/__init__.py

@@ -0,0 +1 @@
+from .version import __version__

crit/crit/__main__.py

@@ -5,6 +5,7 @@ import json
 import os
 
 import pycriu
+from . import __version__
 
 
 def inf(opts):
@@ -41,8 +42,8 @@ def decode(opts):
     try:
         img = pycriu.images.load(inf(opts), opts['pretty'], opts['nopl'])
     except pycriu.images.MagicException as exc:
-        print("Unknown magic %#x.\n"\
+        print("Unknown magic %#x.\n"
-          "Maybe you are feeding me an image with "\
+              "Maybe you are feeding me an image with "
               "raw data(i.e. pages.img)?" % exc.magic, file=sys.stderr)
         sys.exit(1)
 
@@ -59,8 +60,8 @@ def encode(opts):
     try:
         img = json.load(inf(opts))
     except UnicodeDecodeError:
-        print("Cannot read JSON.\n"\
+        print("Cannot read JSON.\n"
-          "Maybe you are feeding me an image with protobuf data? "\
+              "Maybe you are feeding me an image with protobuf data? "
               "Encode expects JSON input.", file=sys.stderr)
         sys.exit(1)
     pycriu.images.dump(img, outf(opts, False))
@@ -131,7 +132,7 @@ def ftype_find_in_files(opts, ft, fid):
     if files_img is None:
         try:
             files_img = pycriu.images.load(dinf(opts, "files.img"))['entries']
-        except:
+        except Exception:
             files_img = []
 
     if len(files_img) == 0:
@@ -364,7 +365,7 @@ def main():
     desc = 'CRiu Image Tool'
     parser = argparse.ArgumentParser(
         description=desc, formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument('--version', action='version', version=pycriu.__version__)
+    parser.add_argument('--version', action='version', version=__version__)
 
     subparsers = parser.add_subparsers(
         help='Use crit CMD --help for command-specific help')
@@ -374,8 +375,7 @@ def main():
         'decode', help='convert criu image from binary type to json')
     decode_parser.add_argument(
         '--pretty',
-        help=
+        help='Multiline with indents and some numerical fields in field-specific format',
-        'Multiline with indents and some numerical fields in field-specific format',
         action='store_true')
     decode_parser.add_argument(
         '-i',

crit/pycriu

@@ -1 +0,0 @@
-../lib/py/

crit/pyproject.toml

@@ -1,3 +1,22 @@
 [build-system]
-# Minimum requirements for the build system to execute.
+requires = ["setuptools"]
-requires = ["setuptools", "wheel"]  # PEP 508 specifications.
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "crit"
+description = "CRiu Image Tool"
+authors = [
+    {name = "CRIU team", email = "criu@openvz.org"},
+]
+license = {text = "GPLv2"}
+dynamic = ["version"]
+requires-python = ">=3.6"
+
+[project.scripts]
+crit = "crit.__main__:main"
+
+[tool.setuptools]
+packages = ["crit"]
+
+[tool.setuptools.dynamic]
+version = {attr = "crit.__version__"}

crit/requirements.txt

@@ -1,7 +0,0 @@
-# We need pip version 20.1 or newer to correctly build with 'pycriu' symlink.
-# - Building of local directories with pip 20.1 or newer is done in place,
-#   instead of a temporary location containing a copy of the directory tree.
-#   (https://github.com/pypa/pip/issues/7555)
-pip>=20.1
-setuptools>=42.0.0
-wheel

crit/setup.cfg

@@ -0,0 +1,20 @@
+# Configuring setuptools using pyproject.toml files was introduced in setuptools 61.0.0
+# https://setuptools.pypa.io/en/latest/history.html#v61-0-0
+# For older versions of setuptools, we need to use the setup.cfg file
+# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html#declarative-config
+
+[metadata]
+name = crit
+description = CRiu Image Tool
+author = CRIU team
+author_email = criu@openvz.org
+license = GPLv2
+version = attr: crit.__version__
+
+[options]
+packages = crit
+python_requires = >=3.6
+
+[options.entry_points]
+console_scripts =
+    crit = crit.__main__:main

crit/setup.py

@@ -1,15 +1,6 @@
-from setuptools import setup, find_packages
+#!/usr/bin/env python3
-import pycriu
+import setuptools
 
-setup(
+
-    name='crit',
+if __name__ == '__main__':
-    version=pycriu.__version__,
+    setuptools.setup()
-    description='CRiu Image Tool',
-    author='CRIU team',
-    author_email='criu@openvz.org',
-    license='GPLv2',
-    url='https://github.com/checkpoint-restore/criu',
-    packages=find_packages('.'),
-    scripts=['crit'],
-    install_requires=[],
-)

lib/.gitignore

@@ -0,0 +1 @@
+pycriu.egg-info/

lib/Makefile

@@ -28,17 +28,17 @@ lib-a: lib/c/$(CRIU_A)
 
 #
 # Python bindings.
-lib/py/Makefile: ;
+lib/pycriu/Makefile: ;
-lib/py/%: .FORCE
+lib/pycriu/%: .FORCE
 	$(call msg-gen, $@)
-	$(Q) $(MAKE) $(build)=lib/py $@
+	$(Q) $(MAKE) $(build)=lib/pycriu $@
 lib-py:
-	$(Q) $(MAKE) $(build)=lib/py all
+	$(Q) $(MAKE) $(build)=lib/pycriu all
 .PHONY: lib-py
 
 clean-lib:
 	$(Q) $(MAKE) $(build)=lib/c clean
-	$(Q) $(MAKE) $(build)=lib/py clean
+	$(Q) $(MAKE) $(build)=lib/pycriu clean
 .PHONY: clean-lib
 clean: clean-lib
 cleanup-y	+= lib/c/$(CRIU_SO) lib/c/$(CRIU_A) lib/c/criu.pc
@@ -59,17 +59,15 @@ install: lib-c lib-a lib-py lib/c/criu.pc.in
 	$(Q) install -m 644 lib/c/criu.pc $(DESTDIR)$(LIBDIR)/pkgconfig
 ifeq ($(PYTHON_EXTERNALLY_MANAGED),1)
 ifeq ($(PIP_BREAK_SYSTEM_PACKAGES),0)
-	$(E) "  SKIP INSTALL crit: Externally managed python environment (See PEP 668 for more information)"
+	$(E) "  SKIP INSTALL pycriu: Externally managed python environment (See PEP 668 for more information)"
 	$(E) "  Consider using PIP_BREAK_SYSTEM_PACKAGES=1 make install"
 else
-	$(E) "  INSTALL " crit
+	$(E) "  INSTALL " pycriu
-	$(Q) $(PYTHON) -m pip install -r ./crit/requirements.txt
+	$(Q) $(PYTHON) -m pip install --upgrade --ignore-installed --prefix=$(DESTDIR)$(PREFIX) ./lib
-	$(Q) $(PYTHON) -m pip install --no-build-isolation --upgrade --ignore-installed --prefix=$(DESTDIR)$(PREFIX) ./crit
 endif
 else
-	$(E) "  INSTALL " crit
+	$(E) "  INSTALL " pycriu
-	$(Q) $(PYTHON) -m pip install -r ./crit/requirements.txt
+	$(Q) $(PYTHON) -m pip install --upgrade --ignore-installed --prefix=$(DESTDIR)$(PREFIX) ./lib
-	$(Q) $(PYTHON) -m pip install --no-build-isolation --upgrade --ignore-installed --prefix=$(DESTDIR)$(PREFIX) ./crit
 endif
 .PHONY: install
 
@@ -84,14 +82,14 @@ uninstall:
 	$(Q) $(RM) $(addprefix $(DESTDIR)$(LIBDIR)/pkgconfig/,criu.pc)
 ifeq ($(PYTHON_EXTERNALLY_MANAGED),1)
 ifeq ($(PIP_BREAK_SYSTEM_PACKAGES),0)
-	$(E) " SKIP UNINSTALL crit: Externally managed python environment (See PEP 668 for more information)"
+	$(E) " SKIP UNINSTALL pycriu: Externally managed python environment (See PEP 668 for more information)"
 	$(E) " Consider using PIP_BREAK_SYSTEM_PACKAGES=1 make uninstall"
 else
-	$(E) " UNINSTALL" crit
+	$(E) " UNINSTALL" pycriu
-	$(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) crit
+	$(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) pycriu
 endif
 else
-	$(E) " UNINSTALL" crit
+	$(E) " UNINSTALL" pycriu
-	$(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) crit
+	$(Q) $(PYTHON) ./scripts/uninstall_module.py --prefix=$(DESTDIR)$(PREFIX) pycriu
 endif
 .PHONY: uninstall

lib/pycriu/.gitignore

@@ -1,3 +1,4 @@
+__pycache__
 *_pb2.py
 *.pyc
 version.py

lib/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools", "protobuf<4.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pycriu"
+description = "Python bindings for CRIU"
+authors = [
+    {name = "CRIU team", email = "criu@openvz.org"},
+]
+license = {text = "GPLv2"}
+dynamic = ["version"]
+requires-python = ">=3.6"
+
+[tool.setuptools]
+packages = ["pycriu", "pycriu.images"]
+
+[tool.setuptools.dynamic]
+version = {attr = "pycriu.__version__"}

lib/setup.cfg

@@ -0,0 +1,16 @@
+# Configuring setuptools using pyproject.toml files was introduced in setuptools 61.0.0
+# https://setuptools.pypa.io/en/latest/history.html#v61-0-0
+# For older versions of setuptools, we need to use the setup.cfg file
+# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html#declarative-config
+
+[metadata]
+name = pycriu
+description = Python bindings for CRIU
+author = CRIU team
+author_email = criu@openvz.org
+license = GPLv2
+version = attr: pycriu.__version__
+
+[options]
+packages = find:
+python_requires = >=3.6

lib/setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
+import setuptools
 
-from pycriu import cli
 
 if __name__ == '__main__':
-        cli.main()
+    setuptools.setup()

test/others/env.sh

@@ -1,8 +1,13 @@
 #!/bin/sh
 
-CRIU=$(readlink -f `dirname ${BASH_SOURCE[0]}`/../../criu/criu)
+BASE_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/../../")"
+
+CRIU="${BASE_DIR}/criu/criu"
 criu=$CRIU
-CRIT=$(readlink -f `dirname ${BASH_SOURCE[0]}`/../../crit/crit)
+
+export PYTHONPATH="${BASE_DIR}/lib:${BASE_DIR}/crit:${PYTHONPATH-}"
+CRIT="python3 -m crit"
 crit=$CRIT
-CRIU_COREDUMP=$(readlink -f `dirname ${BASH_SOURCE[0]}`/../../coredump/coredump)
+
+CRIU_COREDUMP="${BASE_DIR}/coredump/coredump"
 criu_coredump=$CRIU_COREDUMP

test/pycriu

@@ -1 +1 @@
-../lib/py/
+../lib/pycriu

test/zdtm/static/socket-tcp-fin-wait1.hook

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import sys
 
-sys.path.append("../crit")
+sys.path.append("../lib")
 
 import pycriu
 import os, os.path