Add appimage build script and github action

This commit is contained in:
Rebecca Breu 2024-04-09 22:04:31 +02:00
parent 82787d08aa
commit 1119004fe1
7 changed files with 612 additions and 0 deletions

26
.github/workflows/build_appimage.yml vendored Normal file
View file

@ -0,0 +1,26 @@
name: build_appimage
on: workflow_dispatch
jobs:
build_appimage:
name: build_appimage
runs-on: 'ubuntu-20.04'
strategy:
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Build appimage
run: |
bash tools/build_appimage --version=${{ github.ref_name }} --jsonfile=tools/linux_libs.json
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
path: BeeRef*.appimage
retention-days: 5

2
.gitignore vendored
View file

@ -28,6 +28,8 @@ share/python-wheels/
.installed.cfg
*.egg
MANIFEST
*.appimage
squashfs-root/
# PyInstaller
# Usually these files are written by a python script from a template

View file

@ -103,6 +103,7 @@ def main():
logger.info(f'Starting {constants.APPNAME} version {constants.VERSION}')
logger.debug('System: %s', ' '.join(platform.uname()))
logger.debug('Python: %s', platform.python_version())
logger.debug('LD_LIBRARY_PATH: %s', os.environ.get('LD_LIBRARY_PATH'))
settings = BeeSettings()
logger.info(f'Using settings: {settings.fileName()}')
logger.info(f'Logging to: {logfile_name()}')

View file

@ -1,5 +1,9 @@
[flake8]
exclude = squashfs-root
[coverage:run]
source = beeref
[tool:pytest]
norecursedirs = squashfs-root
addopts = --cov-report html --cov-config=setup.cfg

170
tools/build_appimage.py Executable file
View file

@ -0,0 +1,170 @@
#!/usr/bin/env python3
# Build the BeeRef appimage. Run from the git root directory.
# On github actions:
# ./tools/build_appimage --version=${{ github.ref_name }}\
# --jsonfile=tools/linux_libs.json
# Locally:
# ./tools/build_appimage --version=0.3.3-dev --jsonfile=tools/linux_libs.json
# --skip-apt
import argparse
import json
import logging
import os
import shutil
import subprocess
from urllib.request import urlretrieve
parser = argparse.ArgumentParser(
description=('Create an appimage for BeeRef. '
'Run from the git root directory.'))
parser.add_argument(
'-v', '--version',
required=True,
help='BeeRef version number/tag for output file')
parser.add_argument(
'-j', '--jsonfile',
required=True,
help='Json with lib files and packages as generated by find_linux_libs')
parser.add_argument(
'--redownload',
default=False,
action='store_true',
help='Re-use downloaded files if present')
parser.add_argument(
'--skip-apt',
default=False,
action='store_true',
help='Skip apt install step')
parser.add_argument(
'-l', '--loglevel',
default='INFO',
choices=list(logging._nameToLevel.keys()),
help='log level for console output')
args = parser.parse_args()
BEEVERSION = args.version.removeprefix('v')
APPIMAGE = 'python3.11.8-cp311-cp311-manylinux_2_28_x86_64.AppImage'
PYVER = '3.11'
logger = logging.getLogger(__name__)
logging.basicConfig(level=getattr(logging, args.loglevel))
def run_command(*args, capture_output=False):
logger.info(f'Running command: {args}')
result = subprocess.run(args, capture_output=capture_output)
assert result.returncode == 0, f'Failed with exit code {result.returncode}'
def download_file(url, filename):
if not args.redownload and os.path.exists(filename):
logger.info(f'Found file: {filename}')
else:
logger.info(f'Downloading: {url}')
logger.info(f'Saving as: {filename}')
urlretrieve(url, filename=filename)
os.chmod(filename, 0o755)
url = ('https://github.com/niess/python-appimage/releases/download/'
f'python{PYVER}/{APPIMAGE}')
download_file(url, filename='python.appimage')
try:
shutil.rmtree('squashfs-root')
except FileNotFoundError:
pass
run_command('./python.appimage', '--appimage-extract',
capture_output=True)
run_command('squashfs-root/usr/bin/pip',
'install',
'.',
f'--target=squashfs-root/opt/python{PYVER}/lib/python{PYVER}/')
logger.info(f'Reading from: {args.jsonfile}')
with open(args.jsonfile, 'r') as f:
data = json.loads(f.read())
libs = data['libs']
packages = data['packages']
excludes = data['excludes']
paths = set()
if not args.skip_apt:
run_command('sudo', 'apt', 'install', *packages)
logger.info('Copying .so files to appimage...')
existing_files = []
for root, subdirs, files in os.walk('squashfs-root'):
existing_files.extend(files)
for lib in libs:
if os.path.basename(lib) in existing_files:
logger.debug(f'Skipping {lib} (already in appimage)')
continue
if os.path.basename(lib) in excludes:
logger.debug(f'Skipping {lib} (excluded)')
continue
paths.add(os.path.dirname(lib))
if os.path.exists(lib):
filename = lib
else:
filename, _ = os.path.splitext(lib)
dest = f'squashfs-root{filename}'
os.makedirs(os.path.dirname(dest), exist_ok=True)
logger.debug(f'Copying {filename} to {dest}')
shutil.copyfile(filename, f'squashfs-root{filename}')
logger.info('Writing run script...')
# Adapted from usr/bin/python3.x in the python appimage
os.remove('squashfs-root/AppRun')
# ^ This is only a symlink to usr/bin/python3.x
paths = [
'/usr/lib', # The libs that come with the python appimage ar in /usr/lib
] + list(paths)
ld_paths = ['${APPDIR}' + p for p in paths] + ['${LD_LIBRARY_PATH}']
ld_paths = ':'.join(ld_paths)
logger.debug(f'LD_LIBRARY_PATH: {ld_paths}')
content = """#! /bin/bash
# If running from an extracted image, then export ARGV0 and APPDIR
if [ -z "${APPIMAGE}" ]; then
export ARGV0="$0"
self=$(readlink -f -- "$0") # Protect spaces (issue 55)
here="${self%/*}"
tmp="${here%/*}"
export APPDIR="${tmp%/*}"
fi
# Resolve the calling command (preserving symbolic links).
export APPIMAGE_COMMAND=$(command -v -- "$ARGV0")
# Export SSL certificate
export SSL_CERT_FILE="${APPDIR}/opt/_internal/certs.pem"
"""
content += f'export LD_LIBRARY_PATH="{ld_paths}"\n'
content += f'"$APPDIR/opt/python{PYVER}/bin/python{PYVER}" -I -m beeref "$@"\n'
with open('squashfs-root/AppRun', 'w') as f:
f.write(content)
os.chmod('squashfs-root/AppRun', 0o755)
url = ('https://github.com/AppImage/AppImageKit/releases/download/'
'continuous/appimagetool-x86_64.AppImage')
download_file(url, filename='appimagetool.appimage')
run_command('./appimagetool.appimage',
'squashfs-root',
f'BeeRef-{BEEVERSION}.appimage',
'--no-appstream')

171
tools/find_linux_libs.py Executable file
View file

@ -0,0 +1,171 @@
#!/usr/bin/env python3
# Create JSON with Linux libs needed for BeeRef appimage
import argparse
import json
import logging
import os
import pathlib
import re
import subprocess
import sys
from urllib import request
parser = argparse.ArgumentParser(
description=('Create JSON with Linux libs needed for BeeRef appimage'))
parser.add_argument(
'pid',
nargs=1,
default=None,
help='PID of running BeeRef process')
parser.add_argument(
'-l', '--loglevel',
default='INFO',
choices=list(logging._nameToLevel.keys()),
help='log level for console output')
parser.add_argument(
'--jsonfile',
default='linux_libs.json',
help='JSON input/output file')
parser.add_argument(
'--check-appimage',
default=False,
action='store_true',
help='Check a running appimage process for missing libraries')
args = parser.parse_args()
def strip_minor_versions(path):
# foo2.so.2.1.1 -> foo2.so.2
return re.sub('(.so.[0-9]*)[.0-9]*$', r'\1', path)
def what_links_to(path):
links = set()
dirname = os.path.dirname(path)
for filename in os.listdir(dirname):
filename = os.path.join(dirname, filename)
if (os.path.islink(filename)
and str(pathlib.Path(filename).resolve()) == path):
links.add(filename)
return sorted(links, key=len)
def is_lib(path):
return ('.so' in path
and os.path.expanduser('~') not in path
and 'python3' not in path
and 'mesa-diverted' not in path)
def iter_lsofoutput(output):
for line in output.splitlines():
line = line.split()
if line[3] == 'mem':
path = line[-1]
if is_lib(path):
yield path
PID = args.pid[0]
logger = logging.getLogger(__name__)
logging.basicConfig(level=getattr(logging, args.loglevel))
result = subprocess.run(('lsof', '-p', PID), capture_output=True)
assert result.returncode == 0, result.stderr
output = result.stdout.decode('utf-8')
if args.check_appimage:
logger.info('Checking appimage...')
errors = False
for lib in iter_lsofoutput(output):
if 'mount_BeeRef' not in lib:
print(f'Not in appimage: {lib}')
errors = True
if not errors:
print('No missing libs found.')
sys.exit()
libs = []
if os.path.exists(args.jsonfile):
logger.info(f'Reading from: {args.jsonfile}')
with open(args.jsonfile, 'r') as f:
data = json.loads(f.read())
known_libs = data['libs']
packages = set(data['packages'])
else:
logger.info(f'No file {args.jsonfile}; starting from scratch')
known_libs = []
packages = set()
for lib in iter_lsofoutput(output):
links = what_links_to(lib)
if len(links) == 1:
lib = links[0]
else:
logger.warning(f'Double check: {lib} {links}')
lib = links[0]
if lib in known_libs:
logger.debug(f'Found known lib: {lib}')
else:
logger.debug(f'Found unknown lib: {lib}')
libs.append(lib)
for lib in libs:
result = subprocess.run(('apt-file', 'search', lib), capture_output=True)
if result.returncode != 0:
logger.warning(f'Fix manually: {lib}')
continue
output = result.stdout.decode('utf-8')
pkgs = set()
for line in output.splitlines():
pkg = line.split(': ')[0]
if not (pkg.endswith('-dev') or pkg.endswith('-dbg')):
pkgs.add(pkg)
if len(pkgs) == 1:
pkg = pkgs.pop()
logger.debug(f'Found package: {pkg}')
packages.add(pkg)
else:
logger.warning(f'Fix manually: {lib}')
# Find the libs we shouldn't include in the appimage
with request.urlopen(
'https://raw.githubusercontent.com/AppImageCommunity/pkg2appimage/'
'master/excludelist') as f:
response = f.read().decode()
exclude_masterlist = set()
for line in response.splitlines():
if not line or line.startswith('#'):
continue
line = line.split()[0]
line = strip_minor_versions(line)
exclude_masterlist.add(line)
excludes = []
for ex in exclude_masterlist:
for lib in (libs + known_libs):
if lib.endswith(ex):
excludes.append(ex)
continue
logger.info(f'Writing to: {args.jsonfile}')
with open(args.jsonfile, 'w') as f:
data = {'libs': sorted(libs + known_libs),
'packages': sorted(packages),
'excludes': sorted(excludes)}
f.write(json.dumps(data, indent=4))

238
tools/linux_libs.json Normal file
View file

@ -0,0 +1,238 @@
{
"libs": [
"/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2",
"/lib/x86_64-linux-gnu/libbz2.so.1",
"/lib/x86_64-linux-gnu/libc.so.6",
"/lib/x86_64-linux-gnu/libcom_err.so.2",
"/lib/x86_64-linux-gnu/libdbus-1.so.3",
"/lib/x86_64-linux-gnu/libdl.so.2",
"/lib/x86_64-linux-gnu/libexpat.so.1",
"/lib/x86_64-linux-gnu/libgcc_s.so.1",
"/lib/x86_64-linux-gnu/libgpg-error.so.0",
"/lib/x86_64-linux-gnu/libkeyutils.so.1",
"/lib/x86_64-linux-gnu/liblzma.so.5",
"/lib/x86_64-linux-gnu/libm.so.6",
"/lib/x86_64-linux-gnu/libpcre.so.3",
"/lib/x86_64-linux-gnu/libpthread.so.0",
"/lib/x86_64-linux-gnu/libresolv.so.2",
"/lib/x86_64-linux-gnu/librt.so.1",
"/lib/x86_64-linux-gnu/libselinux.so.1",
"/lib/x86_64-linux-gnu/libutil.so.1",
"/lib/x86_64-linux-gnu/libz.so.1",
"/usr/lib/x86_64-linux-gnu/gio/modules/libgvfsdbus.so",
"/usr/lib/x86_64-linux-gnu/gtk-3.0/modules/libcanberra-gtk-module.so",
"/usr/lib/x86_64-linux-gnu/gvfs/libgvfscommon.so",
"/usr/lib/x86_64-linux-gnu/libGLX.so.0",
"/usr/lib/x86_64-linux-gnu/libGLdispatch.so.0",
"/usr/lib/x86_64-linux-gnu/libX11-xcb.so.1",
"/usr/lib/x86_64-linux-gnu/libX11.so.6",
"/usr/lib/x86_64-linux-gnu/libXau.so.6",
"/usr/lib/x86_64-linux-gnu/libXcomposite.so.1",
"/usr/lib/x86_64-linux-gnu/libXcursor.so.1",
"/usr/lib/x86_64-linux-gnu/libXdamage.so.1",
"/usr/lib/x86_64-linux-gnu/libXdmcp.so.6",
"/usr/lib/x86_64-linux-gnu/libXext.so.6",
"/usr/lib/x86_64-linux-gnu/libXfixes.so.3",
"/usr/lib/x86_64-linux-gnu/libXi.so.6",
"/usr/lib/x86_64-linux-gnu/libXinerama.so.1",
"/usr/lib/x86_64-linux-gnu/libXrandr.so.2",
"/usr/lib/x86_64-linux-gnu/libXrender.so.1",
"/usr/lib/x86_64-linux-gnu/libatk-1.0.so.0",
"/usr/lib/x86_64-linux-gnu/libatk-bridge-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libatspi.so.0",
"/usr/lib/x86_64-linux-gnu/libblkid.so.1",
"/usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1",
"/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1",
"/usr/lib/x86_64-linux-gnu/libbsd.so.0",
"/usr/lib/x86_64-linux-gnu/libcairo-gobject.so.2",
"/usr/lib/x86_64-linux-gnu/libcairo.so.2",
"/usr/lib/x86_64-linux-gnu/libcanberra-gtk3.so.0",
"/usr/lib/x86_64-linux-gnu/libcanberra.so.0",
"/usr/lib/x86_64-linux-gnu/libcrypto.so",
"/usr/lib/x86_64-linux-gnu/libdatrie.so.1",
"/usr/lib/x86_64-linux-gnu/libepoxy.so.0",
"/usr/lib/x86_64-linux-gnu/libffi.so.7",
"/usr/lib/x86_64-linux-gnu/libfontconfig.so.1",
"/usr/lib/x86_64-linux-gnu/libfreetype.so.6",
"/usr/lib/x86_64-linux-gnu/libfribidi.so.0",
"/usr/lib/x86_64-linux-gnu/libgcrypt.so.20",
"/usr/lib/x86_64-linux-gnu/libgdk-3.so.0",
"/usr/lib/x86_64-linux-gnu/libgdk_pixbuf-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libgio-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libglib-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libgmodule-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libgraphite2.so.3",
"/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2",
"/usr/lib/x86_64-linux-gnu/libgthread-2.0.so.0",
"/usr/lib/x86_64-linux-gnu/libgtk-3.so.0",
"/usr/lib/x86_64-linux-gnu/libharfbuzz.so.0",
"/usr/lib/x86_64-linux-gnu/libk5crypto.so.3",
"/usr/lib/x86_64-linux-gnu/libkrb5.so.3",
"/usr/lib/x86_64-linux-gnu/libkrb5support.so.0",
"/usr/lib/x86_64-linux-gnu/libltdl.so.7",
"/usr/lib/x86_64-linux-gnu/liblz4.so.1",
"/usr/lib/x86_64-linux-gnu/libmd.so.0",
"/usr/lib/x86_64-linux-gnu/libmount.so.1",
"/usr/lib/x86_64-linux-gnu/libogg.so.0",
"/usr/lib/x86_64-linux-gnu/libpango-1.0.so.0",
"/usr/lib/x86_64-linux-gnu/libpangocairo-1.0.so.0",
"/usr/lib/x86_64-linux-gnu/libpangoft2-1.0.so.0",
"/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0",
"/usr/lib/x86_64-linux-gnu/libpixman-1.so.0",
"/usr/lib/x86_64-linux-gnu/libpng16.so.16",
"/usr/lib/x86_64-linux-gnu/libsqlite3.so",
"/usr/lib/x86_64-linux-gnu/libssl.so",
"/usr/lib/x86_64-linux-gnu/libstdc++.so.6",
"/usr/lib/x86_64-linux-gnu/libsystemd.so.0",
"/usr/lib/x86_64-linux-gnu/libtdb.so.1",
"/usr/lib/x86_64-linux-gnu/libthai.so.0",
"/usr/lib/x86_64-linux-gnu/libuuid.so.1",
"/usr/lib/x86_64-linux-gnu/libvorbis.so.0",
"/usr/lib/x86_64-linux-gnu/libvorbisfile.so.3",
"/usr/lib/x86_64-linux-gnu/libwayland-client.so.0",
"/usr/lib/x86_64-linux-gnu/libwayland-cursor.so.0",
"/usr/lib/x86_64-linux-gnu/libwayland-egl.so.1",
"/usr/lib/x86_64-linux-gnu/libxcb-cursor.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-icccm.so.4",
"/usr/lib/x86_64-linux-gnu/libxcb-image.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-keysyms.so.1",
"/usr/lib/x86_64-linux-gnu/libxcb-randr.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-render-util.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-render.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-shape.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-shm.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-sync.so.1",
"/usr/lib/x86_64-linux-gnu/libxcb-util.so.1",
"/usr/lib/x86_64-linux-gnu/libxcb-xfixes.so.0",
"/usr/lib/x86_64-linux-gnu/libxcb-xkb.so.1",
"/usr/lib/x86_64-linux-gnu/libxcb.so.1",
"/usr/lib/x86_64-linux-gnu/libxkbcommon-x11.so.0",
"/usr/lib/x86_64-linux-gnu/libxkbcommon.so.0",
"/usr/lib/x86_64-linux-gnu/libzstd.so.1"
],
"packages": [
"gvfs",
"gvfs-libs",
"libatk-bridge2.0-0",
"libatk1.0-0",
"libatspi2.0-0",
"libblkid1",
"libbrotli1",
"libbsd0",
"libbz2-1.0",
"libc6",
"libcairo-gobject2",
"libcairo2",
"libcanberra-gtk3-0",
"libcanberra-gtk3-module",
"libcanberra0",
"libcom-err2",
"libdatrie1",
"libdbus-1-3",
"libepoxy0",
"libexpat1",
"libffi7",
"libfontconfig1",
"libfreetype6",
"libfribidi0",
"libgcc-s1",
"libgcrypt20",
"libgdk-pixbuf-2.0-0",
"libglib2.0-0",
"libglvnd0",
"libglx0",
"libgpg-error0",
"libgraphite2-3",
"libgssapi-krb5-2",
"libgtk-3-0",
"libharfbuzz0b",
"libk5crypto3",
"libkeyutils1",
"libkrb5-3",
"libkrb5support0",
"libltdl7",
"liblz4-1",
"liblzma5",
"libmd0",
"libmount1",
"libogg0",
"libpango-1.0-0",
"libpangocairo-1.0-0",
"libpangoft2-1.0-0",
"libpcre2-8-0",
"libpcre3",
"libpixman-1-0",
"libpng16-16",
"libselinux1",
"libsqlite3-0",
"libssl1.1",
"libstdc++6",
"libsystemd0",
"libtdb1",
"libthai0",
"libuuid1",
"libvorbis0a",
"libvorbisfile3",
"libwayland-client0",
"libwayland-cursor0",
"libwayland-egl1",
"libx11-6",
"libx11-xcb1",
"libxau6",
"libxcb-cursor0",
"libxcb-icccm4",
"libxcb-image0",
"libxcb-keysyms1",
"libxcb-randr0",
"libxcb-render-util0",
"libxcb-render0",
"libxcb-shape0",
"libxcb-shm0",
"libxcb-sync1",
"libxcb-util1",
"libxcb-xfixes0",
"libxcb-xkb1",
"libxcb1",
"libxcomposite1",
"libxcursor1",
"libxdamage1",
"libxdmcp6",
"libxext6",
"libxfixes3",
"libxi6",
"libxinerama1",
"libxkbcommon-x11-0",
"libxkbcommon0",
"libxrandr2",
"libxrender1",
"libzstd1",
"zlib1g"
],
"excludes": [
"ld-linux-x86-64.so.2",
"libGLX.so.0",
"libGLdispatch.so.0",
"libX11-xcb.so.1",
"libX11.so.6",
"libc.so.6",
"libcom_err.so.2",
"libdl.so.2",
"libexpat.so.1",
"libfontconfig.so.1",
"libfreetype.so.6",
"libfribidi.so.0",
"libgcc_s.so.1",
"libgpg-error.so.0",
"libharfbuzz.so.0",
"libm.so.6",
"libpthread.so.0",
"libresolv.so.2",
"librt.so.1",
"libstdc++.so.6",
"libthai.so.0",
"libutil.so.1",
"libxcb.so.1",
"libz.so.1"
]
}