Get the new bypy based freezing process working with linux builds

bypy/init_env.py

@@ -7,12 +7,13 @@ import re
 import shutil
 import subprocess
 import sys
+import tempfile
 from contextlib import suppress
 
 from bypy.constants import (
     LIBDIR, PREFIX, PYTHON, SRC as KITTY_DIR, ismacos, worker_env
 )
-from bypy.utils import run_shell
+from bypy.utils import run_shell, walk
 
 
 def read_src_file(name):
@@ -43,8 +44,34 @@ def run(*args, **extra_env):
     return subprocess.call(list(args), env=env, cwd=cwd)
 
 
+def build_frozen_launcher(extra_include_dirs):
+    inc_dirs = [f'--extra-include-dirs={x}' for x in extra_include_dirs]
+    cmd = [PYTHON, 'setup.py', '--prefix', build_frozen_launcher.prefix] + inc_dirs + ['build-frozen-launcher']
+    if run(*cmd, cwd=build_frozen_launcher.writeable_src_dir) != 0:
+        print('Building of frozen kitty launcher failed', file=sys.stderr)
+        os.chdir(KITTY_DIR)
+        run_shell()
+        raise SystemExit('Building of kitty launcher failed')
+    return build_frozen_launcher.writeable_src_dir
+
+
+def check_build(kitty_exe):
+    with tempfile.TemporaryDirectory() as tdir:
+        env = {
+            'KITTY_CONFIG_DIRECTORY': os.path.join(tdir, 'conf'),
+            'KITTY_CACHE_DIRECTORY': os.path.join(tdir, 'cache')
+        }
+        [os.mkdir(x) for x in env.values()]
+        if subprocess.call([kitty_exe, '+runpy', 'from kitty.check_build import main; main()'], env=env) != 0:
+            print('Checking of kitty build failed', file=sys.stderr)
+            os.chdir(os.path.dirname(kitty_exe))
+            run_shell()
+            raise SystemExit('Checking of kitty build failed')
+
+
 def build_c_extensions(ext_dir, args):
     writeable_src_dir = os.path.join(ext_dir, 'src')
+    build_frozen_launcher.writeable_src_dir = writeable_src_dir
     shutil.copytree(
         KITTY_DIR, writeable_src_dir, symlinks=True,
         ignore=shutil.ignore_patterns('b', 'build', 'dist', '*_commands.json', '*.o'))
@@ -58,11 +85,19 @@ def build_c_extensions(ext_dir, args):
         run_shell()
         raise SystemExit('Building of kitty launcher failed')
 
+    for x in walk(writeable_src_dir):
+        if x.rpartition('.') in ('o', 'so', 'dylib', 'pyd'):
+            os.unlink(x)
     cmd = [PYTHON, 'setup.py']
+    if run(*cmd, cwd=writeable_src_dir) != 0:
+        print('Building of kitty failed', file=sys.stderr)
+        os.chdir(KITTY_DIR)
+        run_shell()
+        raise SystemExit('Building of kitty package failed')
     bundle = 'macos-freeze' if ismacos else 'linux-freeze'
     cmd.append(bundle)
     dest = kitty_constants['appname'] + ('.app' if ismacos else '')
-    dest = os.path.join(ext_dir, dest)
+    dest = build_frozen_launcher.prefix = os.path.join(ext_dir, dest)
     cmd += ['--prefix', dest]
     if run(*cmd, cwd=writeable_src_dir) != 0:
         print('Building of kitty package failed', file=sys.stderr)

bypy/linux/__main__.py

@@ -14,12 +14,14 @@ import time
 from bypy.constants import (
     OUTPUT_DIR, PREFIX, is64bit, python_major_minor_version
 )
-from bypy.utils import get_dll_path, py_compile, walk
+from bypy.freeze import (
+    extract_extension_modules, freeze_python, path_to_freeze_dir
+)
+from bypy.utils import get_dll_path, mkdtemp, py_compile, walk
 
 j = os.path.join
-self_dir = os.path.dirname(os.path.abspath(__file__))
 arch = 'x86_64' if is64bit else 'i686'
-
+self_dir = os.path.dirname(os.path.abspath(__file__))
 py_ver = '.'.join(map(str, python_major_minor_version()))
 iv = globals()['init_env']
 kitty_constants = iv['kitty_constants']
@@ -27,12 +29,8 @@ kitty_constants = iv['kitty_constants']
 
 def binary_includes():
     return tuple(map(get_dll_path, (
-            'expat', 'sqlite3', 'ffi', 'z', 'lzma', 'png16', 'lcms2',
-
-            # dont include freetype because fontconfig is closely coupled to it
-            # and also distros often patch freetype
-            # 'iconv.so.2', 'pcre.so.1', 'graphite2.so.3', 'glib-2.0.so.0', 'freetype.so.6',
-
+            'expat', 'sqlite3', 'ffi', 'z', 'lzma', 'png16', 'lcms2', 'crypt',
+            'iconv', 'pcre', 'graphite2', 'glib-2.0', 'freetype',
             'harfbuzz', 'xkbcommon', 'xkbcommon-x11',
             'ncursesw', 'readline',
         ))) + (
@@ -49,6 +47,7 @@ class Env:
         self.py_dir = j(self.lib_dir, 'python' + py_ver)
         os.makedirs(self.py_dir)
         self.bin_dir = j(self.base, 'bin')
+        self.obj_dir = mkdtemp('launchers-')
 
 
 def ignore_in_lib(base, items, ignored_dirs=None):
@@ -106,10 +105,30 @@ def copy_python(env):
             shutil.copy2(y, env.py_dir)
 
     srcdir = j(srcdir, 'site-packages')
-    dest = j(env.py_dir, 'site-packages')
-    import_site_packages(srcdir, dest)
+    site_packages_dir = j(env.py_dir, 'site-packages')
+    import_site_packages(srcdir, site_packages_dir)
+    pdir = os.path.join(env.lib_dir, 'kitty-extensions')
+    os.makedirs(pdir, exist_ok=True)
+    kitty_dir = os.path.join(env.base, 'lib', 'kitty')
+    for x in ('kitty', 'kittens'):
+        dest = os.path.join(env.py_dir, x)
+        os.rename(os.path.join(kitty_dir, x), dest)
+        if x == 'kitty':
+            shutil.rmtree(os.path.join(dest, 'launcher'))
+    os.rename(os.path.join(kitty_dir, '__main__.py'), os.path.join(env.py_dir, 'kitty_main.py'))
+    shutil.rmtree(os.path.join(kitty_dir, '__pycache__'))
+    print('Extracting extension modules from', env.py_dir, 'to', pdir)
+    ext_map = extract_extension_modules(env.py_dir, pdir)
+    shutil.copy(os.path.join(os.path.dirname(self_dir), 'site.py'), os.path.join(env.py_dir, 'site.py'))
+    for q in walk(os.path.join(env.py_dir, 'kitty')):
+        if os.path.splitext(q)[1] not in ('.py', '.glsl'):
+            os.unlink(q)
     py_compile(env.py_dir)
-    py_compile(os.path.join(env.base, 'lib', 'kitty'))
+    freeze_python(env.py_dir, pdir, env.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM')
+
+
+def build_launcher(env):
+    iv['build_frozen_launcher']([path_to_freeze_dir(), env.obj_dir])
 
 
 def is_elf(path):
@@ -204,10 +223,12 @@ def main():
     env = Env(os.path.join(ext_dir, kitty_constants['appname']))
     copy_libs(env)
     copy_python(env)
+    build_launcher(env)
     files = find_binaries(env)
     fix_permissions(files)
     if not args.dont_strip:
         strip_binaries(files)
+    iv['check_build'](os.path.join(env.base, 'bin', 'kitty'))
     create_tarfile(env, args.compression_level)
 
 

bypy/site.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+import builtins
+import sys
+
+import _sitebuiltins
+
+
+def set_quit():
+    eof = 'Ctrl-D (i.e. EOF)'
+    builtins.quit = _sitebuiltins.Quitter('quit', eof)
+    builtins.exit = _sitebuiltins.Quitter('exit', eof)
+
+
+def set_helper():
+    builtins.help = _sitebuiltins._Helper()
+
+
+def main():
+    sys.argv[0] = sys.calibre_basename
+    set_helper()
+    set_quit()
+    mod = __import__(sys.calibre_module, fromlist=[1])
+    func = getattr(mod, sys.calibre_function)
+    return func()
+
+
+if __name__ == '__main__':
+    main()

bypy/sources.json

@@ -110,7 +110,6 @@
 
     {
         "name": "xz",
-        "os": "macos,linux",
         "unix": {
             "filename": "xz-5.2.5.tar.gz",
             "hash": "sha256:f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10",
@@ -118,6 +117,15 @@
         }
     },
 
+    {
+        "name": "xcrypt",
+        "unix": {
+            "filename": "xcrypt-4.4.17.tar.gz",
+            "hash": "sha256:7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07",
+            "urls": ["https://github.com/besser82/libxcrypt/archive/v4.4.17.tar.gz"]
+        }
+    },
+
     {
         "name": "python",
         "unix": {