From 090b3b6f1fd5783bce15dc88b8a1e433b04a3dda Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 3 Jul 2021 10:57:06 +0200 Subject: [PATCH 01/16] Added support for aab --- README.md | 7 +- doc/source/quickstart.rst | 18 ++ pythonforandroid/archs.py | 2 +- pythonforandroid/bootstrap.py | 2 +- .../bootstraps/common/build/build.py | 26 ++- .../java/org/kivy/android/PythonUtil.java | 71 +++++- .../java/org/renpy/android/AssetExtract.java | 10 +- .../common/build/templates/gradle.properties | 2 - .../build/templates/gradle.tmpl.properties | 5 + pythonforandroid/bootstraps/sdl2/__init__.py | 26 +-- .../java/org/kivy/android/PythonActivity.java | 3 +- .../bootstraps/service_only/__init__.py | 2 +- .../java/org/kivy/android/PythonActivity.java | 3 +- .../bootstraps/webview/__init__.py | 2 +- .../java/org/kivy/android/PythonActivity.java | 3 +- pythonforandroid/build.py | 216 +++++++++--------- pythonforandroid/distribution.py | 26 +-- pythonforandroid/recipe.py | 4 +- pythonforandroid/recipes/icu/__init__.py | 2 +- pythonforandroid/recipes/kivy3/__init__.py | 2 +- pythonforandroid/recipes/libiconv/__init__.py | 2 +- .../recipes/libsecp256k1/__init__.py | 2 +- pythonforandroid/recipes/libzbar/__init__.py | 2 +- pythonforandroid/recipes/pandas/__init__.py | 2 +- .../recipes/protobuf_cpp/__init__.py | 2 +- pythonforandroid/recipes/psycopg2/__init__.py | 2 +- pythonforandroid/recipes/pycrypto/__init__.py | 2 +- pythonforandroid/recipes/pyicu/__init__.py | 2 +- pythonforandroid/recipes/python3/__init__.py | 6 +- pythonforandroid/recipes/pyzbar/__init__.py | 2 +- pythonforandroid/recipes/scipy/__init__.py | 2 +- pythonforandroid/recipes/zbar/__init__.py | 2 +- .../recipes/zbarlight/__init__.py | 2 +- pythonforandroid/toolchain.py | 33 ++- tests/recipes/test_pandas.py | 2 +- tests/recipes/test_python3.py | 2 +- tests/test_bootstrap.py | 8 +- 37 files changed, 302 insertions(+), 205 deletions(-) delete mode 100644 pythonforandroid/bootstraps/common/build/templates/gradle.properties create mode 100644 pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties diff --git a/README.md b/README.md index 519e2762fb..8c0dee6913 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ python-for-android python-for-android is a packaging tool for Python apps on Android. You can create your own Python distribution including the modules and -dependencies you want, and bundle it in an APK along with your own code. +dependencies you want, and bundle it in an APK or AAB along with your own code. Features include: @@ -19,6 +19,7 @@ Features include: sqlalchemy. - Multiple architecture targets, for APKs optimised on any given device. +- AAB: Android App Bundle support. For documentation and support, see: @@ -30,7 +31,7 @@ For documentation and support, see: Follow the [quickstart instructions]() -to install and begin creating APKs. +to install and begin creating APKs and AABs. **Quick instructions**: install python-for-android with: @@ -52,6 +53,8 @@ With everything installed, build an APK with SDL2 with e.g.: p4a apk --requirements=kivy --private /home/username/devel/planewave_frozen/ --package=net.inclem.planewavessdl2 --name="planewavessdl2" --version=0.5 --bootstrap=sdl2 +**If you need to deploy your app on Google Play, Android App Bundle (aab) is required since 1 August 2021:** + **For full instructions and parameter options,** see [the documentation](https://python-for-android.readthedocs.io/en/latest/quickstart/#usage). diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 90c15cd461..01a57f012d 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -213,6 +213,24 @@ You can also replace flask with another web framework. Replace ``--port=5000`` with the port on which your app will serve a website. The default for Flask is 5000. +Exporting the Android App Bundle (aab) for distributing it on Google Play +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Starting from August 2021 for new apps and from November 2021 for updates to existings apps, +Google Play Console will require the Android App Bundle instead of the long lived apk. + +python-for-android handles by itself the needed work to accomplish the new requirements: + + p4a aab --private $HOME/code/myapp --package=org.example.myapp --name="My App" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy --arch=arm64-v8a --arch=armeabi-v7a --release + +This `p4a aab ...` command builds a distribution with `python3`, +`kivy`, and everything else you specified in the requirements. +It will be packaged using a SDL2 bootstrap, and produce +an `.aab` file that contains binaries for both `armeabi-v7a` and `arm64-v8a` ABIs. + +The Android App Bundle, is supposed to be used for distributing your app. +If you need to test it locally, on your device, you can use `bundletool ` + Other options ~~~~~~~~~~~~~ diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index aa661fecce..bc7e0f2cbe 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -133,7 +133,7 @@ def get_env(self, with_flags_in_cc=True): ctx=self.ctx, command_prefix=self.command_prefix, python_includes=join( - self.ctx.get_python_install_dir(), + self.ctx.get_python_install_dir(self.arch), 'include/python{}'.format(self.ctx.python_recipe.version[0:3]), ), ) diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index a9e7f4d911..0a5225e526 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -374,7 +374,7 @@ def strip_libraries(self, arch): if len(tokens) > 1: strip = strip.bake(tokens[1:]) - libs_dir = join(self.dist_dir, '_python_bundle', + libs_dir = join(self.dist_dir, f'_python_bundle__{arch.arch}', '_python_bundle', 'modules') filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'), '-iname', '*.so', _env=env).stdout.decode('utf-8') diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index f7c2314c1e..3af3cc7567 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -267,7 +267,7 @@ def make_package(args): # Package up the private data (public not supported). use_setup_py = get_dist_info_for("use_setup_py", error_if_missing=False) is True - tar_dirs = [env_vars_tarpath] + private_tar_dirs = [env_vars_tarpath] _temp_dirs_to_clean = [] try: if args.private: @@ -277,7 +277,7 @@ def make_package(args): ): print('No setup.py/pyproject.toml used, copying ' 'full private data into .apk.') - tar_dirs.append(args.private) + private_tar_dirs.append(args.private) else: print("Copying main.py's ONLY, since other app data is " "expected in site-packages.") @@ -309,10 +309,7 @@ def make_package(args): ) # Append directory with all main.py's to result apk paths: - tar_dirs.append(main_py_only_dir) - for python_bundle_dir in ('private', '_python_bundle'): - if exists(python_bundle_dir): - tar_dirs.append(python_bundle_dir) + private_tar_dirs.append(main_py_only_dir) if get_bootstrap_name() == "webview": for asset in listdir('webview_includes'): shutil.copy(join('webview_includes', asset), join(assets_dir, asset)) @@ -326,8 +323,13 @@ def make_package(args): shutil.copytree(realpath(asset_src), join(assets_dir, asset_dest)) if args.private or args.launcher: + for arch in get_dist_info_for("archs"): + libs_dir = f"libs/{arch}" + make_tar( + join(libs_dir, 'libpybundle.so'), [f'_python_bundle__{arch}'], args.ignore_path, + optimize_python=args.optimize_python) make_tar( - join(assets_dir, 'private.mp3'), tar_dirs, args.ignore_path, + join(assets_dir, 'private.tar'), private_tar_dirs, args.ignore_path, optimize_python=args.optimize_python) finally: for directory in _temp_dirs_to_clean: @@ -358,9 +360,6 @@ def make_package(args): print("WARNING: Received an --icon_fg or an --icon_bg argument, but not both. " "Ignoring.") - if args.enable_androidx: - shutil.copy('templates/gradle.properties', 'gradle.properties') - if get_bootstrap_name() != "service_only": lottie_splashscreen = join(res_dir, 'raw/splashscreen.json') if args.presplash_lottie: @@ -410,7 +409,6 @@ def make_package(args): version_code = 0 if not args.numeric_version: # Set version code in format (arch-minsdk-app_version) - arch = get_dist_info_for("archs")[0] arch_dict = {"x86_64": "9", "arm64-v8a": "8", "armeabi-v7a": "7", "x86": "6"} arch_code = arch_dict.get(arch, '1') min_sdk = args.min_sdk_version @@ -549,6 +547,12 @@ def make_package(args): is_library=(get_bootstrap_name() == 'service_library'), ) + # gradle properties + render( + 'gradle.tmpl.properties', + 'gradle.properties', + args=args) + # ant build templates render( 'build.tmpl.xml', diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index d67570e18a..38738d3065 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -127,7 +127,7 @@ protected static void recursiveDelete(File f) { f.delete(); } - public static void unpackData( + public static void unpackAsset( Context ctx, final String resource, File target, @@ -170,7 +170,74 @@ public static void unpackData( target.mkdirs(); AssetExtract ae = new AssetExtract(ctx); - if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) { + if (!ae.extractTar(resource + ".tar", target.getAbsolutePath(), "private")) { + String msg = "Could not extract " + resource + " data."; + if (ctx instanceof Activity) { + toastError((Activity)ctx, msg); + } else { + Log.v(TAG, msg); + } + } + + try { + // Write .nomedia. + new File(target, ".nomedia").createNewFile(); + + // Write version file. + FileOutputStream os = new FileOutputStream(diskVersionFn); + os.write(dataVersion.getBytes()); + os.close(); + } catch (Exception e) { + Log.w("python", e); + } + } + } + + public static void unpackPyBundle( + Context ctx, + final String resource, + File target, + boolean cleanup_on_version_update) { + + Log.v(TAG, "Unpacking " + resource + " " + target.getName()); + + // The version of data in memory and on disk. + String dataVersion = "p4aisawesome"; // FIXME: Assets method is not usable for fake .so files bundled as a library. + String diskVersion = null; + + Log.v(TAG, "Data version is " + dataVersion); + + // If no version, no unpacking is necessary. + if (dataVersion == null) { + return; + } + + // Check the current disk version, if any. + String filesDir = target.getAbsolutePath(); + String diskVersionFn = filesDir + "/" + resource + ".version"; + + // FIXME: Keeping that for later. Now it is surely failing. + try { + byte buf[] = new byte[64]; + InputStream is = new FileInputStream(diskVersionFn); + int len = is.read(buf); + diskVersion = new String(buf, 0, len); + is.close(); + } catch (Exception e) { + diskVersion = ""; + } + + // If the disk data is out of date, extract it and write the version file. + if (! dataVersion.equals(diskVersion)) { + Log.v(TAG, "Extracting " + resource + " assets."); + + if (cleanup_on_version_update) { + recursiveDelete(target); + } + target.mkdirs(); + + AssetExtract ae = new AssetExtract(ctx); + if (!ae.extractTar(resource + ".so", target.getAbsolutePath(), "pybundle")) { String msg = "Could not extract " + resource + " data."; if (ctx instanceof Activity) { toastError((Activity)ctx, msg); diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java index e7383258f1..0a5dda6567 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java @@ -13,6 +13,7 @@ import java.io.FileOutputStream; import java.io.FileNotFoundException; import java.io.File; +import java.io.FileInputStream; import java.util.zip.GZIPInputStream; @@ -28,7 +29,7 @@ public AssetExtract(Context context) { mAssetManager = context.getAssets(); } - public boolean extractTar(String asset, String target) { + public boolean extractTar(String asset, String target, String method) { byte buf[] = new byte[1024 * 1024]; @@ -36,7 +37,12 @@ public boolean extractTar(String asset, String target) { TarInputStream tis = null; try { - assetStream = mAssetManager.open(asset, AssetManager.ACCESS_STREAMING); + if(method == "private"){ + assetStream = mAssetManager.open(asset, AssetManager.ACCESS_STREAMING); + } else if (method == "pybundle") { + assetStream = new FileInputStream(asset); + } + tis = new TarInputStream(new BufferedInputStream(new GZIPInputStream(new BufferedInputStream(assetStream, 8192)), 8192)); } catch (IOException e) { Log.e("python", "opening up extract tar", e); diff --git a/pythonforandroid/bootstraps/common/build/templates/gradle.properties b/pythonforandroid/bootstraps/common/build/templates/gradle.properties deleted file mode 100644 index 646c51b977..0000000000 --- a/pythonforandroid/bootstraps/common/build/templates/gradle.properties +++ /dev/null @@ -1,2 +0,0 @@ -android.useAndroidX=true -android.enableJetifier=true diff --git a/pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties b/pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties new file mode 100644 index 0000000000..334714c1f7 --- /dev/null +++ b/pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties @@ -0,0 +1,5 @@ +{% if args.enable_androidx %} +android.useAndroidX=true +android.enableJetifier=true +{% endif %} +android.bundle.enableUncompressedNativeLibs=false \ No newline at end of file diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index 5f7c9cee9a..662d43c0ef 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -15,12 +15,7 @@ class SDL2GradleBootstrap(Bootstrap): def assemble_distribution(self): info_main("# Creating Android project ({})".format(self.name)) - arch = self.ctx.archs[0] - - if len(self.ctx.archs) > 1: - raise ValueError("SDL2/gradle support only one arch") - - info("Copying SDL2/gradle build for {}".format(arch)) + info("Copying SDL2/gradle build") shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) @@ -33,23 +28,24 @@ def assemble_distribution(self): with current_directory(self.dist_dir): info("Copying Python distribution") - python_bundle_dir = join('_python_bundle', '_python_bundle') - - self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) - ensure_dir(python_bundle_dir) - site_packages_dir = self.ctx.python_recipe.create_python_bundle( - join(self.dist_dir, python_bundle_dir), arch) + for arch in self.ctx.archs: + python_bundle_dir = join(f'_python_bundle__{arch.arch}', '_python_bundle') + ensure_dir(python_bundle_dir) + + self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) + site_packages_dir = self.ctx.python_recipe.create_python_bundle( + join(self.dist_dir, python_bundle_dir), arch) + if not self.ctx.with_debug_symbols: + self.strip_libraries(arch) + self.fry_eggs(site_packages_dir) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') - if not self.ctx.with_debug_symbols: - self.strip_libraries(arch) - self.fry_eggs(site_packages_dir) super().assemble_distribution() diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index a0d3ee0a2b..f1fd943a34 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -104,7 +104,8 @@ private class UnpackFilesTask extends AsyncTask { protected String doInBackground(String... params) { File app_root_file = new File(params[0]); Log.v(TAG, "Ready to unpack"); - PythonUtil.unpackData(mActivity, "private", app_root_file, true); + PythonUtil.unpackAsset(mActivity, "private", app_root_file, true); + PythonUtil.unpackPyBundle(mActivity, getApplicationInfo().nativeLibraryDir + "/" + "libpybundle", app_root_file, false); return null; } diff --git a/pythonforandroid/bootstraps/service_only/__init__.py b/pythonforandroid/bootstraps/service_only/__init__.py index 41016d73d0..f00baddf8a 100644 --- a/pythonforandroid/bootstraps/service_only/__init__.py +++ b/pythonforandroid/bootstraps/service_only/__init__.py @@ -37,7 +37,7 @@ def assemble_distribution(self): self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) - python_bundle_dir = join('_python_bundle', '_python_bundle') + python_bundle_dir = join(f'_python_bundle__{arch.arch}', '_python_bundle') ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) diff --git a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java index 8abff40bb6..919c42b0ea 100644 --- a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java @@ -74,7 +74,8 @@ protected void onCreate(Bundle savedInstanceState) { Log.v(TAG, "Ready to unpack"); File app_root_file = new File(getAppRoot()); - PythonUtil.unpackData(mActivity, "private", app_root_file, true); + PythonUtil.unpackAsset(mActivity, "private", app_root_file, true); + PythonUtil.unpackPyBundle(mActivity, getApplicationInfo().nativeLibraryDir + "/" + "libpybundle", app_root_file, false); Log.v(TAG, "About to do super onCreate"); super.onCreate(savedInstanceState); diff --git a/pythonforandroid/bootstraps/webview/__init__.py b/pythonforandroid/bootstraps/webview/__init__.py index 24b4cf4bce..c7a0117b98 100644 --- a/pythonforandroid/bootstraps/webview/__init__.py +++ b/pythonforandroid/bootstraps/webview/__init__.py @@ -34,7 +34,7 @@ def assemble_distribution(self): self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) - python_bundle_dir = join('_python_bundle', '_python_bundle') + python_bundle_dir = join(f'_python_bundle__{arch.arch}', '_python_bundle') ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) diff --git a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java index 4dcddec1f0..b8499849da 100644 --- a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java @@ -106,7 +106,8 @@ private class UnpackFilesTask extends AsyncTask { protected String doInBackground(String... params) { File app_root_file = new File(params[0]); Log.v(TAG, "Ready to unpack"); - PythonUtil.unpackData(mActivity, "private", app_root_file, true); + PythonUtil.unpackAsset(mActivity, "private", app_root_file, true); + PythonUtil.unpackPyBundle(mActivity, getApplicationInfo().nativeLibraryDir + "/" + "libpybundle", app_root_file, false); return null; } diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 6c1975a8fd..d728f9d997 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -162,8 +162,8 @@ def python_installs_dir(self): ensure_dir(directory) return directory - def get_python_install_dir(self): - return join(self.python_installs_dir, self.bootstrap.distribution.name) + def get_python_install_dir(self, arch): + return join(self.python_installs_dir, self.bootstrap.distribution.name, arch) def setup_dirs(self, storage_dir): '''Calculates all the storage and build dirs, and makes sure @@ -492,11 +492,11 @@ def prepare_bootstrap(self, bootstrap): def prepare_dist(self): self.bootstrap.prepare_dist_dir() - def get_site_packages_dir(self, arch=None): + def get_site_packages_dir(self, arch): '''Returns the location of site-packages in the python-install build dir. ''' - return self.get_python_install_dir() + return self.get_python_install_dir(arch.arch) def get_libs_dir(self, arch): '''The libs dir for a given arch.''' @@ -613,7 +613,7 @@ def project_has_setup_py(project_dir): )) -def run_setuppy_install(ctx, project_dir, env=None): +def run_setuppy_install(ctx, project_dir, env=None, arch=None): env = env or {} with current_directory(project_dir): @@ -651,7 +651,7 @@ def run_setuppy_install(ctx, project_dir, env=None): # Reference: # https://github.com/pypa/pip/issues/6223 ctx_site_packages_dir = os.path.normpath( - os.path.abspath(ctx.get_site_packages_dir()) + os.path.abspath(ctx.get_site_packages_dir(arch)) ) venv_site_packages_dir = os.path.normpath(os.path.join( ctx.build_dir, "venv", "lib", [ @@ -690,7 +690,7 @@ def run_setuppy_install(ctx, project_dir, env=None): ctx.build_dir, "venv", "bin", "pip" ).replace("'", "'\"'\"'") + "' " + "install -c ._tmp_p4a_recipe_constraints.txt -v ." - ).format(ctx.get_site_packages_dir(). + ).format(ctx.get_site_packages_dir(arch). replace("'", "'\"'\"'")), _env=copy.copy(env)) @@ -737,110 +737,118 @@ def run_pymodules_install(ctx, modules, project_dir=None, """ info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***') - modules = list(filter(ctx.not_has_package, modules)) - - # We change current working directory later, so this has to be an absolute - # path or `None` in case that we didn't supply the `project_dir` via kwargs - project_dir = abspath(project_dir) if project_dir else None - - # Bail out if no python deps and no setup.py to process: - if not modules and ( - ignore_setup_py or - project_dir is None or - not project_has_setup_py(project_dir) - ): - info('No Python modules and no setup.py to process, skipping') - return - # Output messages about what we're going to do: - if modules: - info('The requirements ({}) don\'t have recipes, attempting to ' - 'install them with pip'.format(', '.join(modules))) - info('If this fails, it may mean that the module has compiled ' - 'components and needs a recipe.') - if project_dir is not None and \ - project_has_setup_py(project_dir) and not ignore_setup_py: - info('Will process project install, if it fails then the ' - 'project may not be compatible for Android install.') - - # Use our hostpython to create the virtualenv - host_python = sh.Command(ctx.hostpython) - with current_directory(join(ctx.build_dir)): - shprint(host_python, '-m', 'venv', 'venv') - - # Prepare base environment and upgrade pip: - base_env = dict(copy.copy(os.environ)) - base_env["PYTHONPATH"] = ctx.get_site_packages_dir() - info('Upgrade pip to latest version') - shprint(sh.bash, '-c', ( - "source venv/bin/activate && pip install -U pip" - ), _env=copy.copy(base_env)) - - # Install Cython in case modules need it to build: - info('Install Cython in case one of the modules needs it to build') - shprint(sh.bash, '-c', ( - "venv/bin/pip install Cython" - ), _env=copy.copy(base_env)) - - # Get environment variables for build (with CC/compiler set): - standard_recipe = CythonRecipe() - standard_recipe.ctx = ctx - # (note: following line enables explicit -lpython... linker options) - standard_recipe.call_hostpython_via_targetpython = False - recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) - env = copy.copy(base_env) - env.update(recipe_env) - - # Make sure our build package dir is available, and the virtualenv - # site packages come FIRST (so the proper pip version is used): - env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir() - env["PYTHONPATH"] = os.path.abspath(join( - ctx.build_dir, "venv", "lib", - "python" + ctx.python_recipe.major_minor_version_string, - "site-packages")) + ":" + env["PYTHONPATH"] - - # Install the manually specified requirements first: - if not modules: - info('There are no Python modules to install, skipping') - else: - info('Creating a requirements.txt file for the Python modules') - with open('requirements.txt', 'w') as fileh: - for module in modules: - key = 'VERSION_' + module - if key in environ: - line = '{}=={}\n'.format(module, environ[key]) - else: - line = '{}\n'.format(module) - fileh.write(line) + for arch in ctx.archs: + modules = [m for m in modules if ctx.not_has_package(m, arch)] + + # We change current working directory later, so this has to be an absolute + # path or `None` in case that we didn't supply the `project_dir` via kwargs + project_dir = abspath(project_dir) if project_dir else None + + # Bail out if no python deps and no setup.py to process: + if not modules and ( + ignore_setup_py or + project_dir is None or + not project_has_setup_py(project_dir) + ): + info('No Python modules and no setup.py to process, skipping') + return + + # Output messages about what we're going to do: + if modules: + info( + "The requirements ({}) don\'t have recipes, attempting to " + "install them with pip".format(', '.join(modules)) + ) + info( + "If this fails, it may mean that the module has compiled " + "components and needs a recipe." + ) + if project_dir is not None and \ + project_has_setup_py(project_dir) and not ignore_setup_py: + info( + "Will process project install, if it fails then the " + "project may not be compatible for Android install." + ) - info('Installing Python modules with pip') - info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. ' - 'A reason for this is often modules compiling ' - 'native code that is unaware of Android cross-compilation ' - 'and does not work without additional ' - 'changes / workarounds.') + # Use our hostpython to create the virtualenv + host_python = sh.Command(ctx.hostpython) + with current_directory(join(ctx.build_dir)): + shprint(host_python, '-m', 'venv', 'venv') + # Prepare base environment and upgrade pip: + base_env = dict(copy.copy(os.environ)) + base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch) + info('Upgrade pip to latest version') shprint(sh.bash, '-c', ( - "venv/bin/pip " + - "install -v --target '{0}' --no-deps -r requirements.txt" - ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")), - _env=copy.copy(env)) + "source venv/bin/activate && pip install -U pip" + ), _env=copy.copy(base_env)) - # Afterwards, run setup.py if present: - if project_dir is not None and ( - project_has_setup_py(project_dir) and not ignore_setup_py - ): - run_setuppy_install(ctx, project_dir, env) - elif not ignore_setup_py: - info("No setup.py found in project directory: " + - str(project_dir) + # Install Cython in case modules need it to build: + info('Install Cython in case one of the modules needs it to build') + shprint(sh.bash, '-c', ( + "venv/bin/pip install Cython" + ), _env=copy.copy(base_env)) + + # Get environment variables for build (with CC/compiler set): + standard_recipe = CythonRecipe() + standard_recipe.ctx = ctx + # (note: following line enables explicit -lpython... linker options) + standard_recipe.call_hostpython_via_targetpython = False + recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) + env = copy.copy(base_env) + env.update(recipe_env) + + # Make sure our build package dir is available, and the virtualenv + # site packages come FIRST (so the proper pip version is used): + env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch) + env["PYTHONPATH"] = os.path.abspath(join( + ctx.build_dir, "venv", "lib", + "python" + ctx.python_recipe.major_minor_version_string, + "site-packages")) + ":" + env["PYTHONPATH"] + + # Install the manually specified requirements first: + if not modules: + info('There are no Python modules to install, skipping') + else: + info('Creating a requirements.txt file for the Python modules') + with open('requirements.txt', 'w') as fileh: + for module in modules: + key = 'VERSION_' + module + if key in environ: + line = '{}=={}\n'.format(module, environ[key]) + else: + line = '{}\n'.format(module) + fileh.write(line) + + info('Installing Python modules with pip') + info( + "IF THIS FAILS, THE MODULES MAY NEED A RECIPE. " + "A reason for this is often modules compiling " + "native code that is unaware of Android cross-compilation " + "and does not work without additional " + "changes / workarounds." ) - # Strip object files after potential Cython or native code builds: - if not ctx.with_debug_symbols: - standard_recipe.strip_object_files( - ctx.archs[0], env, build_dir=ctx.build_dir - ) + shprint(sh.bash, '-c', ( + "venv/bin/pip " + + "install -v --target '{0}' --no-deps -r requirements.txt" + ).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")), + _env=copy.copy(env)) + + # Afterwards, run setup.py if present: + if project_dir is not None and ( + project_has_setup_py(project_dir) and not ignore_setup_py + ): + run_setuppy_install(ctx, project_dir, env, arch.arch) + elif not ignore_setup_py: + info("No setup.py found in project directory: " + str(project_dir)) + + # Strip object files after potential Cython or native code builds: + if not ctx.with_debug_symbols: + standard_recipe.strip_object_files( + arch, env, build_dir=ctx.build_dir + ) def biglink(ctx, arch): diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index 8607766eba..334f0c67ed 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -176,11 +176,7 @@ def get_distribution( dist.name = name dist.dist_dir = join( ctx.dist_dir, - generate_dist_folder_name( - name, - [arch_name] if arch_name is not None else None, - ) - ) + name) dist.recipes = recipes dist.ndk_api = ctx.ndk_api dist.archs = [arch_name] @@ -265,23 +261,3 @@ def pretty_log_dists(dists, log_func=info): for line in infos: log_func('\t' + line) - - -def generate_dist_folder_name(base_dist_name, arch_names=None): - """Generate the distribution folder name to use, based on a - combination of the input arguments. - - Parameters - ---------- - base_dist_name : str - The core distribution identifier string - arch_names : list of str - The architecture compile targets - """ - if arch_names is None: - arch_names = ["no_arch_specified"] - - return '{}__{}'.format( - base_dist_name, - '_'.join(arch_names) - ) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 87a9ae9ac4..1e58e7f3e1 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -949,7 +949,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): def should_build(self, arch): name = self.folder_name - if self.ctx.has_package(name): + if self.ctx.has_package(name, arch): info('Python package already exists in site-packages') return False info('{} apparently isn\'t already in site-packages'.format(name)) @@ -976,7 +976,7 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): hpenv = env.copy() with current_directory(self.get_build_dir(arch.arch)): shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--install-lib=.', _env=hpenv, *self.setup_extra_args) diff --git a/pythonforandroid/recipes/icu/__init__.py b/pythonforandroid/recipes/icu/__init__.py index 56e2d49798..f6c43100e0 100644 --- a/pythonforandroid/recipes/icu/__init__.py +++ b/pythonforandroid/recipes/icu/__init__.py @@ -114,7 +114,7 @@ def install_libraries(self, arch): src_include = join( self.get_build_dir(arch.arch), "icu_build", "include") dst_include = join( - self.ctx.get_python_install_dir(), "include", "icu") + self.ctx.get_python_install_dir(arch.arch), "include", "icu") ensure_dir(dst_include) shprint(sh.cp, "-r", join(src_include, "layout"), dst_include) shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include) diff --git a/pythonforandroid/recipes/kivy3/__init__.py b/pythonforandroid/recipes/kivy3/__init__.py index 43b55a4e5d..6f27f62cc9 100644 --- a/pythonforandroid/recipes/kivy3/__init__.py +++ b/pythonforandroid/recipes/kivy3/__init__.py @@ -15,7 +15,7 @@ class Kivy3Recipe(PythonRecipe): def build_arch(self, arch): super().build_arch(arch) suffix = '/kivy3/default.glsl' - shutil.copyfile(self.get_build_dir(arch.arch) + suffix, self.ctx.get_python_install_dir() + suffix) + shutil.copyfile(self.get_build_dir(arch.arch) + suffix, self.ctx.get_python_install_dir(arch.arch) + suffix) recipe = Kivy3Recipe() diff --git a/pythonforandroid/recipes/libiconv/__init__.py b/pythonforandroid/recipes/libiconv/__init__.py index 530497a2ed..111e422d2f 100644 --- a/pythonforandroid/recipes/libiconv/__init__.py +++ b/pythonforandroid/recipes/libiconv/__init__.py @@ -21,7 +21,7 @@ def build_arch(self, arch): shprint( sh.Command('./configure'), '--host=' + arch.command_prefix, - '--prefix=' + self.ctx.get_python_install_dir(), + '--prefix=' + self.ctx.get_python_install_dir(arch.arch), _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) diff --git a/pythonforandroid/recipes/libsecp256k1/__init__.py b/pythonforandroid/recipes/libsecp256k1/__init__.py index caa5a6fc37..b2e10cb8f2 100644 --- a/pythonforandroid/recipes/libsecp256k1/__init__.py +++ b/pythonforandroid/recipes/libsecp256k1/__init__.py @@ -20,7 +20,7 @@ def build_arch(self, arch): shprint( sh.Command('./configure'), '--host=' + arch.toolchain_prefix, - '--prefix=' + self.ctx.get_python_install_dir(), + '--prefix=' + self.ctx.get_python_install_dir(arch.arch), '--enable-shared', '--enable-module-recovery', '--enable-experimental', diff --git a/pythonforandroid/recipes/libzbar/__init__.py b/pythonforandroid/recipes/libzbar/__init__.py index 7a9c15650d..a4b5292abd 100644 --- a/pythonforandroid/recipes/libzbar/__init__.py +++ b/pythonforandroid/recipes/libzbar/__init__.py @@ -34,7 +34,7 @@ def build_arch(self, arch): sh.Command('./configure'), '--host=' + arch.command_prefix, '--target=' + arch.toolchain_prefix, - '--prefix=' + self.ctx.get_python_install_dir(), + '--prefix=' + self.ctx.get_python_install_dir(arch.arch), # Python bindings are compiled in a separated recipe '--with-python=no', '--with-gtk=no', diff --git a/pythonforandroid/recipes/pandas/__init__.py b/pythonforandroid/recipes/pandas/__init__.py index 40da2fb73c..a217ab635a 100644 --- a/pythonforandroid/recipes/pandas/__init__.py +++ b/pythonforandroid/recipes/pandas/__init__.py @@ -20,7 +20,7 @@ def get_recipe_env(self, arch): # we need the includes from our installed numpy at site packages # because we need some includes generated at numpy's compile time env['NUMPY_INCLUDES'] = join( - self.ctx.get_python_install_dir(), "numpy/core/include", + self.ctx.get_python_install_dir(arch.arch), "numpy/core/include", ) # this flag below is to fix a runtime error: diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index 9bde198376..5c43e33828 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -115,7 +115,7 @@ def install_python_package(self, arch): hpenv = env.copy() shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--install-lib=.', _env=hpenv, *self.setup_extra_args) diff --git a/pythonforandroid/recipes/psycopg2/__init__.py b/pythonforandroid/recipes/psycopg2/__init__.py index e35694e6ce..1d946e7d42 100644 --- a/pythonforandroid/recipes/psycopg2/__init__.py +++ b/pythonforandroid/recipes/psycopg2/__init__.py @@ -43,7 +43,7 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): shprint(hostpython, 'setup.py', 'build_ext', '--static-libpq', _env=env) shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--install-lib=.', _env=env) diff --git a/pythonforandroid/recipes/pycrypto/__init__.py b/pythonforandroid/recipes/pycrypto/__init__.py index ae2a375df4..f142d3776d 100644 --- a/pythonforandroid/recipes/pycrypto/__init__.py +++ b/pythonforandroid/recipes/pycrypto/__init__.py @@ -36,7 +36,7 @@ def build_compiled_components(self, arch): with current_directory(self.get_build_dir(arch.arch)): configure = sh.Command('./configure') shprint(configure, '--host=arm-eabi', - '--prefix={}'.format(self.ctx.get_python_install_dir()), + '--prefix={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--enable-shared', _env=env) super().build_compiled_components(arch) diff --git a/pythonforandroid/recipes/pyicu/__init__.py b/pythonforandroid/recipes/pyicu/__init__.py index 17b7c619b3..d1e3749fb7 100644 --- a/pythonforandroid/recipes/pyicu/__init__.py +++ b/pythonforandroid/recipes/pyicu/__init__.py @@ -13,7 +13,7 @@ def get_recipe_env(self, arch): env = super().get_recipe_env(arch) icu_include = join( - self.ctx.get_python_install_dir(), "include", "icu") + self.ctx.get_python_install_dir(arch.arch), "include", "icu") icu_recipe = self.get_recipe('icu', self.ctx) icu_link_libs = icu_recipe.built_libraries.keys() diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index cb03709a30..04f71761b7 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -370,7 +370,7 @@ def create_python_bundle(self, dirn, arch): # Compile to *.pyc/*.pyo the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib')) # Compile to *.pyc/*.pyo the other python packages (site-packages) - self.compile_python_files(self.ctx.get_python_install_dir()) + self.compile_python_files(self.ctx.get_python_install_dir(arch.arch)) # Bundle compiled python modules to a folder modules_dir = join(dirn, 'modules') @@ -399,9 +399,9 @@ def create_python_bundle(self, dirn, arch): # copy the site-packages into place ensure_dir(join(dirn, 'site-packages')) - ensure_dir(self.ctx.get_python_install_dir()) + ensure_dir(self.ctx.get_python_install_dir(arch.arch)) # TODO: Improve the API around walking and copying the files - with current_directory(self.ctx.get_python_install_dir()): + with current_directory(self.ctx.get_python_install_dir(arch.arch)): filens = list(walk_valid_filens( '.', self.site_packages_dir_blacklist, self.site_packages_filen_blacklist)) diff --git a/pythonforandroid/recipes/pyzbar/__init__.py b/pythonforandroid/recipes/pyzbar/__init__.py index 32aaf89185..cf78a558cd 100644 --- a/pythonforandroid/recipes/pyzbar/__init__.py +++ b/pythonforandroid/recipes/pyzbar/__init__.py @@ -16,7 +16,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) libzbar = self.get_recipe('libzbar', self.ctx) libzbar_dir = libzbar.get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir(arch.arch) env['CFLAGS'] += ' -I' + join(libzbar_dir, 'include') env['LDFLAGS'] += ' -L' + join(libzbar_dir, 'zbar', '.libs') env['LIBS'] = env.get('LIBS', '') + ' -landroid -lzbar' diff --git a/pythonforandroid/recipes/scipy/__init__.py b/pythonforandroid/recipes/scipy/__init__.py index de22a79c54..6d9a2cdda4 100644 --- a/pythonforandroid/recipes/scipy/__init__.py +++ b/pythonforandroid/recipes/scipy/__init__.py @@ -33,7 +33,7 @@ def get_recipe_env(self, arch): sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}" sysroot_include = f'{self.ctx.ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include' libgfortran = f'{self.ctx.ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}/{prefix}/{LIB}' - numpylib = self.ctx.get_python_install_dir() + '/numpy/core/lib' + numpylib = self.ctx.get_python_install_dir(arch.arch) + '/numpy/core/lib' LDSHARED_opts = env['LDSHARED'].split('clang')[1] env['LAPACK'] = f'{lapack_dir}/lib' diff --git a/pythonforandroid/recipes/zbar/__init__.py b/pythonforandroid/recipes/zbar/__init__.py index 1656895795..c24971e21d 100644 --- a/pythonforandroid/recipes/zbar/__init__.py +++ b/pythonforandroid/recipes/zbar/__init__.py @@ -23,7 +23,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) libzbar = self.get_recipe('libzbar', self.ctx) libzbar_dir = libzbar.get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir(arch.arch) env['CFLAGS'] += ' -I' + join(libzbar_dir, 'include') env['LDFLAGS'] += ' -L' + join(libzbar_dir, 'zbar', '.libs') env['LIBS'] = env.get('LIBS', '') + ' -landroid -lzbar' diff --git a/pythonforandroid/recipes/zbarlight/__init__.py b/pythonforandroid/recipes/zbarlight/__init__.py index 0f1d718835..36365cd03d 100644 --- a/pythonforandroid/recipes/zbarlight/__init__.py +++ b/pythonforandroid/recipes/zbarlight/__init__.py @@ -16,7 +16,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) libzbar = self.get_recipe('libzbar', self.ctx) libzbar_dir = libzbar.get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir(arch.arch) env['CFLAGS'] += ' -I' + join(libzbar_dir, 'include') env['LDFLAGS'] += ' -L' + join(libzbar_dir, 'zbar', '.libs') env['LIBS'] = env.get('LIBS', '') + ' -landroid -lzbar' diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index aa242a4170..3f106b400d 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -99,8 +99,6 @@ def check_python_dependencies(): toolchain_dir = dirname(__file__) sys.path.insert(0, join(toolchain_dir, "tools", "external")) -APK_SUFFIX = '.apk' - def add_boolean_option(parser, names, no_names=None, default=True, dest=None, description=None): @@ -313,8 +311,8 @@ def __init__(self): '(default: {})'.format(default_storage_dir))) generic_parser.add_argument( - '--arch', help='The arch to build for.', - default='armeabi-v7a') + '--arch', help='The archs to build for.', + action='append', default=[]) # Options for specifying the Distribution generic_parser.add_argument( @@ -563,6 +561,11 @@ def add_parser(subparsers, *args, **kwargs): 'apk', help='Build an APK', parents=[parser_packaging]) + add_parser( + subparsers, + 'aab', help='Build an AAB', + parents=[parser_packaging]) + add_parser( subparsers, 'create', help='Compile a set of requirements into a dist', @@ -712,7 +715,7 @@ def add_parser(subparsers, *args, **kwargs): self.ctx.symlink_bootstrap_files = args.symlink_bootstrap_files self.ctx.java_build_tool = args.java_build_tool - self._archs = split_argument_list(args.arch) + self._archs = args.arch self.ctx.local_recipes = args.local_recipes self.ctx.copy_libs = args.copy_libs @@ -1028,7 +1031,7 @@ def _build_package(self, args, package_type): """ Creates an android package using gradle :param args: parser args - :param package_type: one of 'apk', 'aar' + :param package_type: one of 'apk', 'aar', 'aab' :return (gradle output, build_args) """ ctx = self.ctx @@ -1078,7 +1081,10 @@ def _build_package(self, args, package_type): if args.build_mode == "debug": gradle_task = "assembleDebug" elif args.build_mode == "release": - gradle_task = "assembleRelease" + if package_type == "apk": + gradle_task = "assembleRelease" + elif package_type == "aab": + gradle_task = "bundleRelease" else: raise BuildInterruptingException( "Unknown build mode {} for apk()".format(args.build_mode)) @@ -1092,7 +1098,7 @@ def _finish_package(self, args, output, build_args, package_type, output_dir): :param args: the parser args :param output: RunningCommand output :param build_args: build args as returned by build.parse_args - :param package_type: one of 'apk', 'aar' + :param package_type: one of 'apk', 'aar', 'aab' :param output_dir: where to put the package file """ @@ -1129,11 +1135,12 @@ def _finish_package(self, args, output, build_args, package_type, output_dir): raise BuildInterruptingException('Couldn\'t find the built APK') info_main('# Found android package file: {}'.format(package_file)) + package_extension = f".{package_type}" if package_add_version: info('# Add version number to android package') - package_name = basename(package_file)[:-len(APK_SUFFIX)] + package_name = basename(package_file)[:-len(package_extension)] package_file_dest = "{}-{}-{}".format( - package_name, build_args.version, APK_SUFFIX) + package_name, build_args.version, package_extension) info('# Android package renamed to {}'.format(package_file_dest)) shprint(sh.cp, package_file, package_file_dest) else: @@ -1151,6 +1158,12 @@ def aar(self, args): output_dir = join(self._dist.dist_dir, "build", "outputs", 'aar') self._finish_package(args, output, build_args, 'aar', output_dir) + @require_prebuilt_dist + def aab(self, args): + output, build_args = self._build_package(args, package_type='aab') + output_dir = join(self._dist.dist_dir, "build", "outputs", 'bundle', args.build_mode) + self._finish_package(args, output, build_args, 'aab', output_dir) + @require_prebuilt_dist def create(self, args): """Create a distribution directory if it doesn't already exist, run diff --git a/tests/recipes/test_pandas.py b/tests/recipes/test_pandas.py index 23cefb43c9..3ac34d1d3b 100644 --- a/tests/recipes/test_pandas.py +++ b/tests/recipes/test_pandas.py @@ -38,7 +38,7 @@ def test_get_recipe_env( self.ctx.recipe_build_order ) numpy_includes = join( - self.ctx.get_python_install_dir(), "numpy/core/include", + self.ctx.get_python_install_dir(self.arch.arch), "numpy/core/include", ) env = self.recipe.get_recipe_env(self.arch) self.assertIn(numpy_includes, env["NUMPY_INCLUDES"]) diff --git a/tests/recipes/test_python3.py b/tests/recipes/test_python3.py index 66698c9162..481c4b3e24 100644 --- a/tests/recipes/test_python3.py +++ b/tests/recipes/test_python3.py @@ -194,7 +194,7 @@ def test_create_python_bundle( expected_sp_paths = [ modules_build_dir, join(recipe_build_dir, 'Lib'), - self.ctx.get_python_install_dir(), + self.ctx.get_python_install_dir(self.arch.arch), ] for n, (sp_call, kw) in enumerate(mock_subprocess.call_args_list): self.assertEqual(sp_call[0][-1], expected_sp_paths[n]) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 8fcedb53a7..39e016ad9c 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -8,7 +8,7 @@ from pythonforandroid.bootstrap import ( _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, ) -from pythonforandroid.distribution import Distribution, generate_dist_folder_name +from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe from pythonforandroid.archs import ArchARMv7_a from pythonforandroid.build import Context @@ -85,7 +85,7 @@ def test_attributes(self): # test dist_dir success self.setUp_distribution_with_bootstrap(bs) - expected_folder_name = generate_dist_folder_name('test_prj', [self.TEST_ARCH]) + expected_folder_name = 'test_prj' self.assertTrue( bs.dist_dir.endswith(f"dists/{expected_folder_name}")) @@ -438,8 +438,8 @@ def test_assemble_distribution( mock_strip_libraries.assert_called() expected__python_bundle = os.path.join( self.ctx.dist_dir, - f"{self.ctx.bootstrap.distribution.name}__{self.TEST_ARCH}", - "_python_bundle", + self.ctx.bootstrap.distribution.name, + f"_python_bundle__{self.TEST_ARCH}", "_python_bundle", ) self.assertIn( From 17ac5e88bc67318c9a1aa38430679b24186f788d Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 31 Jul 2021 17:03:21 +0200 Subject: [PATCH 02/16] Move to build:gradle:3.5.4 (adds support for API 30), fix some tests --- .../bootstraps/common/build/templates/build.tmpl.gradle | 2 +- tests/test_build.py | 2 ++ tests/test_toolchain.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle index b59ee6df41..1f3fa72cc6 100644 --- a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle +++ b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle @@ -5,7 +5,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.5.2' + classpath 'com.android.tools.build:gradle:3.5.4' } } diff --git a/tests/test_build.py b/tests/test_build.py index 27e3f40a24..073377df34 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -4,6 +4,7 @@ import jinja2 from pythonforandroid.build import run_pymodules_install +from pythonforandroid.archs import ArchARMv7_a, ArchAarch_64 class TestBuildBasic(unittest.TestCase): @@ -14,6 +15,7 @@ def test_run_pymodules_install_optional_project_dir(self): `project_dir` optional parameter is None, refs #1898 """ ctx = mock.Mock() + ctx.archs = [ArchARMv7_a(ctx), ArchAarch_64(ctx)] modules = [] project_dir = None with mock.patch('pythonforandroid.build.info') as m_info: diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index d7c73c6319..50f3a35aae 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -62,6 +62,8 @@ def test_create(self): '--dist-name=test_toolchain', '--activity-class-name=abc.myapp.android.CustomPythonActivity', '--service-class-name=xyz.myapp.android.CustomPythonService', + '--arch=arm64-v8a', + '--arch=armeabi-v7a' ] with patch_sys_argv(argv), mock.patch( 'pythonforandroid.build.get_available_apis' @@ -116,7 +118,7 @@ def test_create_no_sdk_dir(self): """ The `--sdk-dir` is mandatory to `create` a distribution. """ - argv = ['toolchain.py', 'create'] + argv = ['toolchain.py', 'create', '--arch=arm64-v8a', '--arch=armeabi-v7a'] with patch_sys_argv(argv), pytest.raises( BuildInterruptingException ) as ex_info: From 71d11748a267f73ba81b420d7a66e40ffd95d7f2 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 31 Jul 2021 18:24:34 +0200 Subject: [PATCH 03/16] Github actions test apps (apk + aab) --- .github/workflows/push.yml | 40 ++++++++++++++++++++++++++++++-------- Makefile | 11 ++++++++--- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bc1f71dbdf..3691185e71 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -41,14 +41,10 @@ jobs: pip install tox>=2.0 make test - build: + build_apk: name: Unit test apk needs: [flake8] runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - build-arch: ['arm64-v8a', 'armeabi-v7a', 'x86_64', 'x86'] steps: - name: Checkout python-for-android uses: actions/checkout@v2 @@ -64,15 +60,43 @@ jobs: - name: Pull docker image run: | make docker/pull - - name: Build apk Python 3 ${{ matrix.build-arch }} + - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) run: | mkdir -p apks - make docker/run/make/with-artifact/testapps-with-numpy/${{ matrix.build-arch }} + make docker/run/make/with-artifact/testapps-with-numpy - uses: actions/upload-artifact@v1 with: - name: bdist_test_app_unittests__${{ matrix.build-arch }}-debug-1.1.apk + name: bdist_unit_tests_app-debug-1.1-.apk path: apks + build_aab: + name: Unit test aab + needs: [flake8] + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@v2 + # helps with GitHub runner getting out of space + - name: Free disk space + run: | + df -h + sudo swapoff -a + sudo rm -f /swapfile + sudo apt -y clean + docker rmi $(docker image ls -aq) + df -h + - name: Pull docker image + run: | + make docker/pull + - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) + run: | + mkdir -p aabs + make docker/run/make/with-artifact/testapps-with-numpy-aab + - uses: actions/upload-artifact@v1 + with: + name: bdist_unit_tests_app-release-1.1-.aab + path: aabs + rebuild_updated_recipes: name: Test updated recipes needs: [flake8] diff --git a/Makefile b/Makefile index be6c1b7c89..c4e0da5bfb 100644 --- a/Makefile +++ b/Makefile @@ -34,12 +34,17 @@ rebuild_updated_recipes: virtualenv ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py -testapps-with-numpy/%: virtualenv - $(eval $@_APP_ARCH := $(shell basename $*)) +testapps-with-numpy: virtualenv . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ - --arch=$($@_APP_ARCH) + --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 + +testapps-with-numpy-aab: virtualenv + . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ + python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ + --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release testapps/%: virtualenv $(eval $@_APP_ARCH := $(shell basename $*)) From 69a6449d880616b721cafc491aab060be565a609 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 31 Jul 2021 19:08:11 +0200 Subject: [PATCH 04/16] Add missing bdistaab --- pythonforandroid/bdistapk.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index d4b2c7953a..884095950f 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -143,6 +143,14 @@ class BdistAAR(Bdist): package_type = 'aar' +class BdistAAB(Bdist): + """ + distutil command handler for 'aab' + """ + description = 'Create an AAB with python-for-android' + package_type = 'aab' + + def _set_user_options(): # This seems like a silly way to do things, but not sure if there's a # better way to pass arbitrary options onwards to p4a From 30f30415abb8f0136df1fa6d0276407137cc239d Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sun, 1 Aug 2021 17:47:36 +0200 Subject: [PATCH 05/16] Fix automated tests --- .github/workflows/push.yml | 4 ++-- Makefile | 14 +++++++++----- pythonforandroid/bdistapk.py | 1 + setup.py | 1 + testapps/on_device_unit_tests/setup.py | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3691185e71..40850ffbc0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -63,7 +63,7 @@ jobs: - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) run: | mkdir -p apks - make docker/run/make/with-artifact/testapps-with-numpy + make docker/run/make/with-artifact/apk/testapps-with-numpy - uses: actions/upload-artifact@v1 with: name: bdist_unit_tests_app-debug-1.1-.apk @@ -91,7 +91,7 @@ jobs: - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) run: | mkdir -p aabs - make docker/run/make/with-artifact/testapps-with-numpy-aab + make docker/run/make/with-artifact/aab/testapps-with-numpy-aab - uses: actions/upload-artifact@v1 with: name: bdist_unit_tests_app-release-1.1-.aab diff --git a/Makefile b/Makefile index c4e0da5bfb..80fc8fcf1f 100644 --- a/Makefile +++ b/Makefile @@ -74,14 +74,18 @@ docker/run/test: docker/build docker/run/command: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" -docker/run/make/%: docker/build - docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* +docker/run/make/with-artifact/apk/%: docker/build + docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1-.apk ./apks + docker rm -fv p4a-latest -docker/run/make/with-artifact/%: docker/build - $(eval $@_APP_ARCH := $(shell basename $*)) +docker/run/make/with-artifact/aab/%: docker/build docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app__$($@_APP_ARCH)-debug-1.1-.apk ./apks + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1-.aab ./aabs docker rm -fv p4a-latest +docker/run/make/%: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* + docker/run/shell: docker/build docker run --rm --env-file=.env -it $(DOCKER_IMAGE) diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 884095950f..70858da466 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -164,6 +164,7 @@ def _set_user_options(): user_options.append((arg[2:], None, None)) BdistAPK.user_options = user_options + BdistAAB.user_options = user_options _set_user_options() diff --git a/setup.py b/setup.py index ef9ee0cfb2..18bd3e4e45 100644 --- a/setup.py +++ b/setup.py @@ -103,6 +103,7 @@ def recursively_include(results, directory, patterns): 'distutils.commands': [ 'apk = pythonforandroid.bdistapk:BdistAPK', 'aar = pythonforandroid.bdistapk:BdistAAR', + 'aab = pythonforandroid.bdistapk:BdistAAB', ], }, classifiers=[ diff --git a/testapps/on_device_unit_tests/setup.py b/testapps/on_device_unit_tests/setup.py index be33963e89..fb9570af3f 100644 --- a/testapps/on_device_unit_tests/setup.py +++ b/testapps/on_device_unit_tests/setup.py @@ -38,6 +38,20 @@ # define a basic test app, which can be override passing the proper args to cli options = { 'apk': + { + 'requirements': + 'sqlite3,libffi,openssl,pyjnius,kivy,python3,requests,urllib3,' + 'chardet,idna', + 'android-api': 27, + 'ndk-api': 21, + 'dist-name': 'bdist_unit_tests_app', + 'arch': 'armeabi-v7a', + 'bootstrap' : 'sdl2', + 'permissions': ['INTERNET', 'VIBRATE'], + 'orientation': 'sensor', + 'service': 'P4a_test_service:app_service.py', + }, + 'aab': { 'requirements': 'sqlite3,libffi,openssl,pyjnius,kivy,python3,requests,urllib3,' From e06db1bce2398988bdb1b441372f9cd054096c4b Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sun, 8 Aug 2021 14:17:28 +0200 Subject: [PATCH 06/16] ndk lib folder (or ndk platform) now is ABI specific --- pythonforandroid/archs.py | 25 +++- pythonforandroid/build.py | 136 +++++++++--------- pythonforandroid/recipe.py | 2 +- pythonforandroid/recipes/Pillow/__init__.py | 6 +- .../recipes/audiostream/__init__.py | 2 +- pythonforandroid/recipes/cffi/__init__.py | 4 +- pythonforandroid/recipes/evdev/__init__.py | 2 +- pythonforandroid/recipes/freetype/__init__.py | 4 +- pythonforandroid/recipes/libogg/__init__.py | 2 +- pythonforandroid/recipes/librt/__init__.py | 9 +- .../recipes/libvorbis/__init__.py | 2 +- pythonforandroid/recipes/lxml/__init__.py | 4 +- pythonforandroid/recipes/openal/__init__.py | 2 +- pythonforandroid/recipes/openssl/__init__.py | 3 +- pythonforandroid/recipes/pygame/__init__.py | 6 +- pythonforandroid/recipes/python3/__init__.py | 4 +- pythonforandroid/recommendations.py | 1 + tests/recipes/recipe_ctx.py | 6 +- tests/recipes/test_icu.py | 1 - tests/test_toolchain.py | 4 - 20 files changed, 117 insertions(+), 108 deletions(-) diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index bc7e0f2cbe..95d94b5f1b 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -1,9 +1,10 @@ from distutils.spawn import find_executable from os import environ -from os.path import join, split +from os.path import join, split, exists from multiprocessing import cpu_count from glob import glob +from pythonforandroid.logger import warning from pythonforandroid.recipe import Recipe from pythonforandroid.util import BuildInterruptingException, build_platform @@ -30,7 +31,7 @@ class Arch: common_cppflags = [ '-DANDROID', '-D__ANDROID_API__={ctx.ndk_api}', - '-I{ctx.ndk_dir}/sysroot/usr/include/{command_prefix}', + '-I{ctx.ndk_sysroot}/usr/include/{command_prefix}', '-I{python_includes}', ] @@ -57,6 +58,24 @@ def __init__(self, ctx): def __str__(self): return self.arch + @property + def ndk_lib_dir(self): + return join(self.ctx.ndk_sysroot, 'usr', 'lib', self.command_prefix, str(self.ctx.ndk_api)) + + @property + def ndk_platform(self): + warning("ndk_platform is deprecated and should be avoided in new recipes") + ndk_platform = join( + self.ctx.ndk_dir, + 'platforms', + 'android-{}'.format(self.ctx.ndk_api), + self.platform_dir) + if not exists(ndk_platform): + BuildInterruptingException( + "The requested platform folder doesn't exist. If you're building on ndk >= r22, and seeing this error, one of the required recipe is using a removed feature." + ) + return ndk_platform + @property def include_dirs(self): return [ @@ -213,7 +232,7 @@ def get_env(self, with_flags_in_cc=True): # Android's arch/toolchain env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) - env['TOOLCHAIN_PREFIX'] = self.ctx.toolchain_prefix + env['TOOLCHAIN_PREFIX'] = self.toolchain_prefix env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version # Custom linker options diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index d728f9d997..f0c2ec476b 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -24,20 +24,18 @@ from pythonforandroid.recommendations import ( check_ndk_version, check_target_api, check_ndk_api, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) +from pythonforandroid.util import build_platform -def get_ndk_platform_dir(ndk_dir, ndk_api, arch): - ndk_platform_dir_exists = True - platform_dir = arch.platform_dir - ndk_platform = join( - ndk_dir, - 'platforms', - 'android-{}'.format(ndk_api), - platform_dir) - if not exists(ndk_platform): - warning("ndk_platform doesn't exist: {}".format(ndk_platform)) - ndk_platform_dir_exists = False - return ndk_platform, ndk_platform_dir_exists +def get_ndk_standalone(ndk_dir): + return join(ndk_dir, 'toolchains', 'llvm', 'prebuilt', build_platform) + + +def get_ndk_sysroot(ndk_dir): + sysroot = join(get_ndk_standalone(ndk_dir), 'sysroot') + if not exists(sysroot): + warning("sysroot doesn't exist: {}".format(sysroot)) + return sysroot def get_toolchain_versions(ndk_dir, arch): @@ -114,7 +112,9 @@ class Context: ccache = None # whether to use ccache - ndk_platform = None # the ndk platform directory + ndk_standalone = None + ndk_sysroot = None + ndk_include_dir = None # usr/include bootstrap = None bootstrap_build_dir = None @@ -296,7 +296,9 @@ def prepare_build_environment(self, android_api = int(android_api) self.android_api = android_api - check_target_api(android_api, self.archs[0].arch) + for arch in self.archs: + # Maybe We could remove this one in a near future (ARMv5 is definitely old) + check_target_api(android_api, arch) apis = get_available_apis(self.sdk_dir) info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) @@ -375,60 +377,61 @@ def prepare_build_environment(self, ' a python 3 target (which is the default)' ' then THINGS WILL BREAK.') - # This would need to be changed if supporting multiarch APKs - arch = self.archs[0] - toolchain_prefix = arch.toolchain_prefix - self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir( - self.ndk_dir, self.ndk_api, arch) - ok = ok and ndk_platform_dir_exists - py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' - toolchain_versions, toolchain_path_exists = get_toolchain_versions( - self.ndk_dir, arch) - ok = ok and toolchain_path_exists - toolchain_versions.sort() - - toolchain_versions_gcc = [] - for toolchain_version in toolchain_versions: - if toolchain_version[0].isdigit(): - # GCC toolchains begin with a number - toolchain_versions_gcc.append(toolchain_version) - - if toolchain_versions: - info('Found the following toolchain versions: {}'.format( - toolchain_versions)) - info('Picking the latest gcc toolchain, here {}'.format( - toolchain_versions_gcc[-1])) - toolchain_version = toolchain_versions_gcc[-1] - else: - warning('Could not find any toolchain for {}!'.format( - toolchain_prefix)) - ok = False - - self.toolchain_prefix = toolchain_prefix - self.toolchain_version = toolchain_version - # Modify the path so that sh finds modules appropriately - environ['PATH'] = ( - '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' - 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' - '{toolchain_prefix}-{toolchain_version}/prebuilt/' - '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' - 'tools:{path}').format( - sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, - toolchain_prefix=toolchain_prefix, - toolchain_version=toolchain_version, - py_platform=py_platform, path=environ.get('PATH')) - - for executable in ("pkg-config", "autoconf", "automake", "libtoolize", - "tar", "bzip2", "unzip", "make", "gcc", "g++"): - if not sh.which(executable): - warning(f"Missing executable: {executable} is not installed") - - if not ok: - raise BuildInterruptingException( - 'python-for-android cannot continue due to the missing executables above') + + self.ndk_standalone = get_ndk_standalone(self.ndk_dir) + self.ndk_sysroot = get_ndk_sysroot(self.ndk_dir) + ok = ok and exists(self.ndk_sysroot) + self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include') + + for arch in self.archs: + + toolchain_versions, toolchain_path_exists = get_toolchain_versions( + self.ndk_dir, arch) + ok = ok and toolchain_path_exists + toolchain_versions.sort() + + toolchain_versions_gcc = [] + for toolchain_version in toolchain_versions: + if toolchain_version[0].isdigit(): + # GCC toolchains begin with a number + toolchain_versions_gcc.append(toolchain_version) + + if toolchain_versions: + info('Found the following toolchain versions: {}'.format( + toolchain_versions)) + info('Picking the latest gcc toolchain, here {}'.format( + toolchain_versions_gcc[-1])) + toolchain_version = toolchain_versions_gcc[-1] + else: + warning('Could not find any toolchain for {}!'.format( + arch.toolchain_prefix)) + ok = False + + # Modify the path so that sh finds modules appropriately + environ['PATH'] = ( + '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' + 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' + '{toolchain_prefix}-{toolchain_version}/prebuilt/' + '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' + 'tools:{path}').format( + sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, + toolchain_prefix=arch.toolchain_prefix, + toolchain_version=toolchain_version, + py_platform=py_platform, path=environ.get('PATH')) + + for executable in ("pkg-config", "autoconf", "automake", "libtoolize", + "tar", "bzip2", "unzip", "make", "gcc", "g++"): + if not sh.which(executable): + warning(f"Missing executable: {executable} is not installed") + + if not ok: + raise BuildInterruptingException( + 'python-for-android cannot continue due to the missing executables above') + + self.toolchain_version = toolchain_version # We assume that the toolchain version is the same for all the archs def __init__(self): self.include_dirs = [] @@ -441,7 +444,6 @@ def __init__(self): self._ndk_api = None self.ndk = None - self.toolchain_prefix = None self.toolchain_version = None self.local_recipes = None @@ -887,7 +889,7 @@ def biglink(ctx, arch): # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? - with current_directory(join(ctx.ndk_platform, 'usr', 'lib')): + with current_directory(arch.ndk_lib_dir): do_biglink( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 1e58e7f3e1..b28a947367 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1142,7 +1142,7 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): env['LDSHARED'] = env['CC'] + ' -shared' # shprint(sh.whereis, env['LDSHARED'], _env=env) env['LIBLINK'] = 'NOTNONE' - env['NDKPLATFORM'] = self.ctx.ndk_platform + env['NDKPLATFORM'] = self.ctx.ndk_sysroot # FIXME? if self.ctx.copy_libs: env['COPYLIBS'] = '1' diff --git a/pythonforandroid/recipes/Pillow/__init__.py b/pythonforandroid/recipes/Pillow/__init__.py index 08abba0f24..a2da43c278 100644 --- a/pythonforandroid/recipes/Pillow/__init__.py +++ b/pythonforandroid/recipes/Pillow/__init__.py @@ -35,9 +35,9 @@ class PillowRecipe(CompiledComponentsPythonRecipe): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) - env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr') - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') - ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + env['ANDROID_ROOT'] = join(arch.ndk_platform, 'usr') + ndk_lib_dir = arch.ndk_lib_dir + ndk_include_dir = self.ctx.ndk_include_dir png = self.get_recipe('png', self.ctx) png_lib_dir = join(png.get_build_dir(arch.arch), '.libs') diff --git a/pythonforandroid/recipes/audiostream/__init__.py b/pythonforandroid/recipes/audiostream/__init__.py index 69e5ab4c49..93f007a5f6 100644 --- a/pythonforandroid/recipes/audiostream/__init__.py +++ b/pythonforandroid/recipes/audiostream/__init__.py @@ -25,7 +25,7 @@ def get_recipe_env(self, arch): jni_path=join(self.ctx.bootstrap.build_dir, 'jni'), sdl_include=sdl_include, sdl_mixer_include=sdl_mixer_include) - env['NDKPLATFORM'] = self.ctx.ndk_platform + env['NDKPLATFORM'] = arch.ndk_platform env['LIBLINK'] = 'NOTNONE' # Hacky fix. Needed by audiostream setup.py return env diff --git a/pythonforandroid/recipes/cffi/__init__.py b/pythonforandroid/recipes/cffi/__init__.py index dbe805eced..06966e0138 100644 --- a/pythonforandroid/recipes/cffi/__init__.py +++ b/pythonforandroid/recipes/cffi/__init__.py @@ -35,9 +35,7 @@ def get_recipe_env(self, arch=None): self.ctx.get_libs_dir(arch.arch)) env['LDFLAGS'] += ' -L{}'.format(os.path.join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)) # required for libc and libdl - ndk_dir = self.ctx.ndk_platform - ndk_lib_dir = os.path.join(ndk_dir, 'usr', 'lib') - env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) + env['LDFLAGS'] += ' -L{}'.format(arch.ndk_lib_dir) env['PYTHONPATH'] = ':'.join([ self.ctx.get_site_packages_dir(), env['BUILDLIB_PATH'], diff --git a/pythonforandroid/recipes/evdev/__init__.py b/pythonforandroid/recipes/evdev/__init__.py index 2d53f9b0b8..1973612fa3 100644 --- a/pythonforandroid/recipes/evdev/__init__.py +++ b/pythonforandroid/recipes/evdev/__init__.py @@ -18,7 +18,7 @@ class EvdevRecipe(CompiledComponentsPythonRecipe): def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) - env['NDKPLATFORM'] = self.ctx.ndk_platform + env['NDKPLATFORM'] = arch.ndk_platform return env diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index 00dbeec6a1..130f6708c8 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -47,8 +47,8 @@ def get_recipe_env(self, arch=None, with_harfbuzz=False): ) # android's zlib support - zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') - zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + zlib_lib_path = arch.ndk_lib_dir + zlib_includes = self.ctx.ndk_include_dir def add_flag_if_not_added(flag, env_key): if flag not in env[env_key]: diff --git a/pythonforandroid/recipes/libogg/__init__.py b/pythonforandroid/recipes/libogg/__init__.py index a96eca9c34..51320f429d 100644 --- a/pythonforandroid/recipes/libogg/__init__.py +++ b/pythonforandroid/recipes/libogg/__init__.py @@ -12,7 +12,7 @@ def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) flags = [ - '--with-sysroot=' + self.ctx.ndk_platform, + '--with-sysroot=' + arch.ndk_platform, '--host=' + arch.toolchain_prefix, ] configure = sh.Command('./configure') diff --git a/pythonforandroid/recipes/librt/__init__.py b/pythonforandroid/recipes/librt/__init__.py index 9eb56b3b18..fcd7d5048c 100644 --- a/pythonforandroid/recipes/librt/__init__.py +++ b/pythonforandroid/recipes/librt/__init__.py @@ -18,11 +18,8 @@ class LibRt(Recipe): libc, so we create a symbolic link which we will remove when our build finishes''' - @property - def libc_path(self): - return join(self.ctx.ndk_platform, 'usr', 'lib', 'libc') - def build_arch(self, arch): + libc_path = join(arch.ndk_platform, 'usr', 'lib', 'libc') # Create a temporary folder to add to link path with a fake librt.so: fake_librt_temp_folder = join( self.get_build_dir(arch.arch), @@ -35,13 +32,13 @@ def build_arch(self, arch): if exists(join(fake_librt_temp_folder, "librt.so")): remove(join(fake_librt_temp_folder, "librt.so")) shprint(sh.ln, '-sf', - self.libc_path + '.so', + libc_path + '.so', join(fake_librt_temp_folder, "librt.so"), ) if exists(join(fake_librt_temp_folder, "librt.a")): remove(join(fake_librt_temp_folder, "librt.a")) shprint(sh.ln, '-sf', - self.libc_path + '.a', + libc_path + '.a', join(fake_librt_temp_folder, "librt.a"), ) diff --git a/pythonforandroid/recipes/libvorbis/__init__.py b/pythonforandroid/recipes/libvorbis/__init__.py index 5f1e3254c1..4599d319a8 100644 --- a/pythonforandroid/recipes/libvorbis/__init__.py +++ b/pythonforandroid/recipes/libvorbis/__init__.py @@ -21,7 +21,7 @@ def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) flags = [ - '--with-sysroot=' + self.ctx.ndk_platform, + '--with-sysroot=' + arch.ndk_platform, '--host=' + arch.toolchain_prefix, ] configure = sh.Command('./configure') diff --git a/pythonforandroid/recipes/lxml/__init__.py b/pythonforandroid/recipes/lxml/__init__.py index 75e7fcd9f7..13e95bf28e 100644 --- a/pythonforandroid/recipes/lxml/__init__.py +++ b/pythonforandroid/recipes/lxml/__init__.py @@ -51,8 +51,8 @@ def get_recipe_env(self, arch): env['LIBS'] += ' -lxml2' # android's ndk flags - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') - ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + ndk_lib_dir = arch.ndk_lib_dir + ndk_include_dir = self.ndk_include_dir cflags += ' -I' + ndk_include_dir env['LDFLAGS'] += ' -L' + ndk_lib_dir env['LIBS'] += ' -lz -lm -lc' diff --git a/pythonforandroid/recipes/openal/__init__.py b/pythonforandroid/recipes/openal/__init__.py index cfb62f6148..1fc72159c7 100644 --- a/pythonforandroid/recipes/openal/__init__.py +++ b/pythonforandroid/recipes/openal/__init__.py @@ -22,7 +22,7 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) cmake_args = [ '-DCMAKE_TOOLCHAIN_FILE={}'.format('XCompile-Android.txt'), - '-DHOST={}'.format(self.ctx.toolchain_prefix) + '-DHOST={}'.format(arch.toolchain_prefix) ] shprint( sh.cmake, '.', diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index da4fdb6e61..20a93ac4ab 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -96,7 +96,8 @@ def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) env['OPENSSL_VERSION'] = self.version env['MAKE'] = 'make' # This removes the '-j5', which isn't safe - env['ANDROID_NDK'] = self.ctx.ndk_dir + env['CC'] = 'clang' + env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir return env def select_build_arch(self, arch): diff --git a/pythonforandroid/recipes/pygame/__init__.py b/pythonforandroid/recipes/pygame/__init__.py index 3088b6e8c0..8ec416617d 100644 --- a/pythonforandroid/recipes/pygame/__init__.py +++ b/pythonforandroid/recipes/pygame/__init__.py @@ -28,9 +28,7 @@ def prebuild_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): setup_template = open(join("buildconfig", "Setup.Android.SDL2.in")).read() env = self.get_recipe_env(arch) - env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr') - - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') + env['ANDROID_ROOT'] = join(arch.ndk_platform, 'usr') png = self.get_recipe('png', self.ctx) png_lib_dir = join(png.get_build_dir(arch.arch), '.libs') @@ -43,7 +41,7 @@ def prebuild_arch(self, arch): sdl_includes=( " -I" + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL', 'include') + " -L" + join(self.ctx.bootstrap.build_dir, "libs", str(arch)) + - " -L" + png_lib_dir + " -L" + jpeg_lib_dir + " -L" + ndk_lib_dir), + " -L" + png_lib_dir + " -L" + jpeg_lib_dir + " -L" + arch.ndk_lib_dir), sdl_ttf_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_ttf'), sdl_image_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_image'), sdl_mixer_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_mixer'), diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 04f71761b7..7d5c488feb 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -265,8 +265,8 @@ def add_flags(include_flags, link_dirs, link_libs): # the build of zlib module, here we search for android's zlib version # and sets the right flags, so python can be build with android's zlib info("Activating flags for android's zlib") - zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') - zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + zlib_lib_path = arch.ndk_lib_dir + zlib_includes = self.ctx.ndk_include_dir zlib_h = join(zlib_includes, 'zlib.h') try: with open(zlib_h) as fileh: diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 00caad1484..5550861282 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -153,6 +153,7 @@ def check_target_api(api, arch): recommendation """ + # FIXME: Should We remove support for armeabi (ARMv5)? if api >= ARMEABI_MAX_TARGET_API and arch == 'armeabi': raise BuildInterruptingException( UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE.format( diff --git a/tests/recipes/recipe_ctx.py b/tests/recipes/recipe_ctx.py index a162e8f0dc..4a2bf3aca2 100644 --- a/tests/recipes/recipe_ctx.py +++ b/tests/recipes/recipe_ctx.py @@ -44,10 +44,8 @@ def setUp(self): self.ctx.recipe_build_order = self.recipe_build_order self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) self.arch = ArchAarch_64(self.ctx) - self.ctx.ndk_platform = ( - f"{self.ctx._ndk_dir}/platforms/" - f"android-{self.ctx.ndk_api}/{self.arch.platform_dir}" - ) + self.ctx.ndk_sysroot = f'{self.ctx._ndk_dir}/sysroot' + self.ctx.ndk_include_dir = f'{self.ctx.ndk_sysroot}/usr/include' self.recipe = Recipe.get_recipe(self.recipe_name, self.ctx) def tearDown(self): diff --git a/tests/recipes/test_icu.py b/tests/recipes/test_icu.py index 39c29047f7..de062d7231 100644 --- a/tests/recipes/test_icu.py +++ b/tests/recipes/test_icu.py @@ -53,7 +53,6 @@ def test_build_arch( mock_archs_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") ] - self.ctx.toolchain_prefix = self.arch.toolchain_prefix self.ctx.toolchain_version = "4.9" self.recipe.build_arch(self.arch) diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index 50f3a35aae..25343100f1 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -70,8 +70,6 @@ def test_create(self): ) as m_get_available_apis, mock.patch( 'pythonforandroid.build.get_toolchain_versions' ) as m_get_toolchain_versions, mock.patch( - 'pythonforandroid.build.get_ndk_platform_dir' - ) as m_get_ndk_platform_dir, mock.patch( 'pythonforandroid.toolchain.build_recipes' ) as m_build_recipes, mock.patch( 'pythonforandroid.bootstraps.service_only.' @@ -79,8 +77,6 @@ def test_create(self): ) as m_run_distribute: m_get_available_apis.return_value = [27] m_get_toolchain_versions.return_value = (['4.9'], True) - m_get_ndk_platform_dir.return_value = ( - '/tmp/android-ndk/platforms/android-21/arch-arm', True) tchain = ToolchainCL() assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity' assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService' From 77579c3c72176bf28b0eb228d6d40d33363edbf5 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Mon, 23 Aug 2021 14:29:02 +0200 Subject: [PATCH 07/16] Fixes dist lookup + some tests --- pythonforandroid/build.py | 8 +++++--- pythonforandroid/distribution.py | 12 ++++++------ pythonforandroid/toolchain.py | 2 +- tests/recipes/recipe_ctx.py | 2 +- tests/test_archs.py | 2 +- tests/test_bootstrap.py | 2 +- tests/test_distribution.py | 8 ++++---- tests/test_toolchain.py | 17 +++++++++++++---- 8 files changed, 32 insertions(+), 21 deletions(-) diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index f0c2ec476b..fa72fd9fc2 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -33,9 +33,11 @@ def get_ndk_standalone(ndk_dir): def get_ndk_sysroot(ndk_dir): sysroot = join(get_ndk_standalone(ndk_dir), 'sysroot') + sysroot_exists = True if not exists(sysroot): warning("sysroot doesn't exist: {}".format(sysroot)) - return sysroot + sysroot_exists = False + return sysroot, sysroot_exists def get_toolchain_versions(ndk_dir, arch): @@ -382,8 +384,8 @@ def prepare_build_environment(self, py_platform = 'linux' self.ndk_standalone = get_ndk_standalone(self.ndk_dir) - self.ndk_sysroot = get_ndk_sysroot(self.ndk_dir) - ok = ok and exists(self.ndk_sysroot) + self.ndk_sysroot, ndk_sysroot_exists = get_ndk_sysroot(self.ndk_dir) + ok = ok and ndk_sysroot_exists self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include') for arch in self.archs: diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index 334f0c67ed..ff97f92bfe 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -46,7 +46,7 @@ def get_distribution( cls, ctx, *, - arch_name, # required keyword argument: there is no sensible default + archs, # required keyword argument: there is no sensible default name=None, recipes=[], ndk_api=None, @@ -70,8 +70,8 @@ def get_distribution( ndk_api : int The NDK API to compile against, included in the dist because it cannot be changed later during APK packaging. - arch_name : str - The target architecture name to compile against, included in the dist because + archs : list + The target architectures list to compile against, included in the dist because it cannot be changed later during APK packaging. recipes : list The recipes that the distribution must contain. @@ -99,7 +99,7 @@ def get_distribution( if name is not None and name: possible_dists = [ d for d in possible_dists if - (d.name == name) and (arch_name in d.archs)] + (d.name == name) and all(arch_name in d.archs for arch_name in archs)] if possible_dists: # There should only be one folder with a given dist name *and* arch. @@ -136,7 +136,7 @@ def get_distribution( continue if ndk_api is not None and dist.ndk_api != ndk_api: continue - if arch_name is not None and arch_name not in dist.archs: + if not all(arch_name in dist.archs for arch_name in archs): continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and @@ -179,7 +179,7 @@ def get_distribution( name) dist.recipes = recipes dist.ndk_api = ctx.ndk_api - dist.archs = [arch_name] + dist.archs = archs return dist diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 3f106b400d..3529e9eed2 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -161,7 +161,7 @@ def dist_from_args(ctx, args): ctx, name=args.dist_name, recipes=split_argument_list(args.requirements), - arch_name=args.arch, + archs=args.arch, ndk_api=args.ndk_api, force_build=args.force_build, require_perfect_match=args.require_perfect_match, diff --git a/tests/recipes/recipe_ctx.py b/tests/recipes/recipe_ctx.py index 4a2bf3aca2..056ca88454 100644 --- a/tests/recipes/recipe_ctx.py +++ b/tests/recipes/recipe_ctx.py @@ -39,7 +39,7 @@ def setUp(self): self.ctx.setup_dirs(os.getcwd()) self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="sdl2", recipes=self.recipes, arch_name=self.TEST_ARCH, + self.ctx, name="sdl2", recipes=self.recipes, archs=[self.TEST_ARCH], ) self.ctx.recipe_build_order = self.recipe_build_order self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) diff --git a/tests/test_archs.py b/tests/test_archs.py index 3b6b86e218..bfd4bebfcc 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -64,7 +64,7 @@ def setUp(self): self.ctx, name="sdl2", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) # Here we define the expected compiler, which, as per ndk >= r19, diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 39e016ad9c..64e11c52ae 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -50,7 +50,7 @@ def setUp_distribution_with_bootstrap(self, bs): self.ctx.bootstrap.distribution = Distribution.get_distribution( self.ctx, name="test_prj", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) def tearDown(self): diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 3342b5f143..423d572252 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -53,7 +53,7 @@ def setUp_distribution_with_bootstrap(self, bs, **kwargs): self.ctx, name=kwargs.pop("name", "test_prj"), recipes=kwargs.pop("recipes", ["python3", "kivy"]), - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], **kwargs ) @@ -111,7 +111,7 @@ def test_get_distribution_no_name(self, mock_exists): returns the proper result which should `unnamed_dist_1`.""" mock_exists.return_value = False self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) - dist = Distribution.get_distribution(self.ctx, arch_name=self.TEST_ARCH) + dist = Distribution.get_distribution(self.ctx, archs=[self.TEST_ARCH]) self.assertEqual(dist.name, "unnamed_dist_1") @mock.patch("pythonforandroid.util.chdir") @@ -213,7 +213,7 @@ def test_get_distributions_error_ndk_api_mismatch( self.ctx, name="test_prj", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) mock_get_dists.return_value = [expected_dist] mock_glob.return_value = ["sdl2-python3"] @@ -264,7 +264,7 @@ def test_get_distributions_possible_dists(self, mock_get_dists): self.ctx, name="test_prj", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) mock_get_dists.return_value = [expected_dist] self.setUp_distribution_with_bootstrap( diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index 25343100f1..c6747885ca 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -1,10 +1,12 @@ import io import sys +from os.path import join import pytest from unittest import mock from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import ToolchainCL from pythonforandroid.util import BuildInterruptingException +from pythonforandroid.build import get_ndk_standalone def patch_sys_argv(argv): @@ -70,6 +72,8 @@ def test_create(self): ) as m_get_available_apis, mock.patch( 'pythonforandroid.build.get_toolchain_versions' ) as m_get_toolchain_versions, mock.patch( + 'pythonforandroid.build.get_ndk_sysroot' + ) as m_get_ndk_sysroot, mock.patch( 'pythonforandroid.toolchain.build_recipes' ) as m_build_recipes, mock.patch( 'pythonforandroid.bootstraps.service_only.' @@ -77,6 +81,10 @@ def test_create(self): ) as m_run_distribute: m_get_available_apis.return_value = [27] m_get_toolchain_versions.return_value = (['4.9'], True) + m_get_ndk_sysroot.return_value = ( + join(get_ndk_standalone("/tmp/android-ndk"), "sysroot"), + True, + ) tchain = ToolchainCL() assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity' assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService' @@ -84,10 +92,11 @@ def test_create(self): [mock.call('/tmp/android-sdk')], # linux case [mock.call('/private/tmp/android-sdk')] # macos case ] - assert m_get_toolchain_versions.call_args_list in [ - [mock.call('/tmp/android-ndk', mock.ANY)], # linux case - [mock.call('/private/tmp/android-ndk', mock.ANY)], # macos case - ] + for callargs in m_get_toolchain_versions.call_args_list: + assert callargs in [ + mock.call("/tmp/android-ndk", mock.ANY), # linux case + mock.call("/private/tmp/android-ndk", mock.ANY), # macos case + ] build_order = [ 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', From aa129bb14d72c901ad49117277867350640155c1 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Wed, 25 Aug 2021 09:30:10 +0200 Subject: [PATCH 08/16] Added .aab and .apks to blacklist --- pythonforandroid/bootstraps/sdl2/build/blacklist.txt | 2 ++ pythonforandroid/bootstraps/service_only/build/blacklist.txt | 2 ++ pythonforandroid/bootstraps/webview/build/blacklist.txt | 2 ++ 3 files changed, 6 insertions(+) diff --git a/pythonforandroid/bootstraps/sdl2/build/blacklist.txt b/pythonforandroid/bootstraps/sdl2/build/blacklist.txt index 96f8af8133..d5e230c89a 100644 --- a/pythonforandroid/bootstraps/sdl2/build/blacklist.txt +++ b/pythonforandroid/bootstraps/sdl2/build/blacklist.txt @@ -1,5 +1,7 @@ # prevent user to include invalid extensions *.apk +*.aab +*.apks *.pxd # eggs diff --git a/pythonforandroid/bootstraps/service_only/build/blacklist.txt b/pythonforandroid/bootstraps/service_only/build/blacklist.txt index f5d05d44d5..53cc634b7d 100644 --- a/pythonforandroid/bootstraps/service_only/build/blacklist.txt +++ b/pythonforandroid/bootstraps/service_only/build/blacklist.txt @@ -1,5 +1,7 @@ # prevent user to include invalid extensions *.apk +*.aab +*.apks *.pxd # eggs diff --git a/pythonforandroid/bootstraps/webview/build/blacklist.txt b/pythonforandroid/bootstraps/webview/build/blacklist.txt index f5d05d44d5..53cc634b7d 100644 --- a/pythonforandroid/bootstraps/webview/build/blacklist.txt +++ b/pythonforandroid/bootstraps/webview/build/blacklist.txt @@ -1,5 +1,7 @@ # prevent user to include invalid extensions *.apk +*.aab +*.apks *.pxd # eggs From d049f9e4d789451533ec417bb3f483f0623e8f34 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Wed, 25 Aug 2021 09:55:25 +0200 Subject: [PATCH 09/16] Interrupt build and alert the user if tried to build an aab in debug mode --- pythonforandroid/toolchain.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 3529e9eed2..2636eaca5b 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -1079,6 +1079,11 @@ def _build_package(self, args, package_type): _tail=20, _critical=True, _env=env ) if args.build_mode == "debug": + if package_type == "aab": + raise BuildInterruptingException( + "aab is meant only for distribution and is not available in debug mode. " + "Instead, you can use apk while building for debugging purposes." + ) gradle_task = "assembleDebug" elif args.build_mode == "release": if package_type == "apk": From 049dfdf0d389990de841010f2509c2b20c329ac4 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 28 Aug 2021 17:36:23 +0200 Subject: [PATCH 10/16] Updates troubleshooting instructions to reflect current structure. --- doc/source/troubleshooting.rst | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/doc/source/troubleshooting.rst b/doc/source/troubleshooting.rst index 884d24d384..4d04c9954e 100644 --- a/doc/source/troubleshooting.rst +++ b/doc/source/troubleshooting.rst @@ -85,31 +85,35 @@ At the top level, this will always contain the same set of files:: AndroidManifest.xml classes.dex META-INF res assets lib YourApk.apk resources.arsc -The Python distribution is in the assets folder:: +The user app data (code, images, fonts ..) is packaged into a single tarball contained in the assets folder:: $ cd assets $ ls - private.mp3 + private.tar -``private.mp3`` is actually a tarball containing all your packaged -data, and the Python distribution. Extract it:: +``private.tar`` is a tarball containing all your packaged +data. Extract it:: - $ tar xf private.mp3 + $ tar xf private.tar -This will reveal all the Python-related files:: +This will reveal all the user app data (the files shown below are from the touchtracer demo):: $ ls - android_runnable.pyo include interpreter_subprocess main.kv pipinterface.kv settings.pyo - assets __init__.pyo interpreterwrapper.pyo main.pyo pipinterface.pyo utils.pyo - editor.kv interpreter.kv _python_bundle menu.kv private.mp3 widgets.pyo - editor.pyo interpreter.pyo libpymodules.so menu.pyo settings.kv + README.txt android.txt icon.png main.pyc p4a_env_vars.txt particle.png + private.tar touchtracer.kv -Most of these files have been included by the user (in this case, they -come from one of my own apps), the rest relate to the python -distribution. +Due to how We're required to ship ABI-specific things in Android App Bundle, +the Python installation is packaged separately, as (most of it) is ABI-specific. + +For example, the Python installation for ``arm64-v8a`` is available in ``lib/arm64-v8a/libpybundle.so`` + +``libpybundle.so`` is a tarball (but named like a library for packaging requirements), that contains our ``_python_bundle``:: + + $ tar xf libpybundle.so + $ cd _python_bundle + $ ls + modules site-packages stdlib.zip -The python installation, along with all side-packages, is mostly contained -inside the `_python_bundle` folder. Common errors From fe0a27ada4ea5de2b6d90a124b8ed65afc6082cb Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 28 Aug 2021 19:52:02 +0200 Subject: [PATCH 11/16] Exclude gdbserver and gdb.setup from release builds --- .../common/build/templates/build.tmpl.gradle | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle index 1f3fa72cc6..4d9e287b4b 100644 --- a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle +++ b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle @@ -39,11 +39,16 @@ android { manifestPlaceholders = {{ args.manifest_placeholders}} } - {% if debug_build -%} + packagingOptions { - doNotStrip '**/*.so' + {% if debug_build -%} + doNotStrip '**/*.so' + {% else %} + exclude 'lib/**/gdbserver' + exclude 'lib/**/gdb.setup' + {%- endif %} } - {%- endif %} + {% if args.sign -%} signingConfigs { From cad31be21346cac6893dca06b155a482dc42b2aa Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Mon, 30 Aug 2021 12:39:29 +0200 Subject: [PATCH 12/16] Add a paragraph in history + fixes --arch docs --- README.md | 3 +++ doc/source/commands.rst | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c0dee6913..cec5c6e75a 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ api level below 21, you should use an older version of python-for-android On March of 2020 we dropped support for creating apps that use Python 2. The latest python-for-android release that supported building Python 2 was version 2019.10.6. +On August of 2021, We added support for Android App Bundle (aab). As a collateral, +now We support multi-arch apk. + ## Contributors This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)]. diff --git a/doc/source/commands.rst b/doc/source/commands.rst index 952c31d6b2..5a0884aa5e 100644 --- a/doc/source/commands.rst +++ b/doc/source/commands.rst @@ -71,8 +71,9 @@ supply those that you need. Whether the distribution must be compiled from scratch. ``--arch`` - The architecture to build for. Currently only one architecture can be - targeted at a time, and a given distribution can only include one architecture. + The architecture to build for. You can specify multiple architectures to build for + at the same time. As an example ``p4a ... --arch arm64-v8a --arch armeabi-v7a ...`` + will build a distribution for both ``arm64-v8a`` and ``armeabi-v7a``. ``--bootstrap BOOTSTRAP`` The Java bootstrap to use for your application. You mostly don't From 3d54a3512a7f1120c2fa495507777139a96b979c Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 11 Sep 2021 14:52:58 +0200 Subject: [PATCH 13/16] Fix versioning --- pythonforandroid/bootstraps/common/build/build.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 3af3cc7567..dcb6d2ac3b 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -408,14 +408,17 @@ def make_package(args): version_code = 0 if not args.numeric_version: - # Set version code in format (arch-minsdk-app_version) - arch_dict = {"x86_64": "9", "arm64-v8a": "8", "armeabi-v7a": "7", "x86": "6"} - arch_code = arch_dict.get(arch, '1') + """ + Set version code in format (10 + minsdk + app_version) + Historically versioning was (arch + minsdk + app_version), + with arch expressed with a single digit from 6 to 9. + Since the multi-arch support, has been changed to 10. + """ min_sdk = args.min_sdk_version for i in args.version.split('.'): version_code *= 100 version_code += int(i) - args.numeric_version = "{}{}{}".format(arch_code, min_sdk, version_code) + args.numeric_version = "{}{}{}".format("10", min_sdk, version_code) if args.intent_filters: with open(args.intent_filters) as fd: From 40816f7e61a33044ff61c853c5c995c732d7d088 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 11 Sep 2021 15:10:39 +0200 Subject: [PATCH 14/16] Minor fixes to docs --- README.md | 2 +- doc/source/quickstart.rst | 2 +- pythonforandroid/bdistapk.py | 12 +++--------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cec5c6e75a..9f69f1bc5c 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ api level below 21, you should use an older version of python-for-android On March of 2020 we dropped support for creating apps that use Python 2. The latest python-for-android release that supported building Python 2 was version 2019.10.6. -On August of 2021, We added support for Android App Bundle (aab). As a collateral, +On August of 2021, we added support for Android App Bundle (aab). As a collateral, now We support multi-arch apk. ## Contributors diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 01a57f012d..5aa910812a 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -214,7 +214,7 @@ Replace ``--port=5000`` with the port on which your app will serve a website. The default for Flask is 5000. Exporting the Android App Bundle (aab) for distributing it on Google Play -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Starting from August 2021 for new apps and from November 2021 for updates to existings apps, Google Play Console will require the Android App Bundle instead of the long lived apk. diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 70858da466..bfc5279736 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -128,25 +128,19 @@ def prepare_build_dir(self): class BdistAPK(Bdist): - """ - distutil command handler for 'apk' - """ + """distutil command handler for 'apk'.""" description = 'Create an APK with python-for-android' package_type = 'apk' class BdistAAR(Bdist): - """ - distutil command handler for 'aar' - """ + """distutil command handler for 'aar'.""" description = 'Create an AAR with python-for-android' package_type = 'aar' class BdistAAB(Bdist): - """ - distutil command handler for 'aab' - """ + """distutil command handler for 'aab'.""" description = 'Create an AAB with python-for-android' package_type = 'aab' From 3f3420dbc061cc8eba598312ce67eefa483f6976 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 11 Sep 2021 18:54:23 +0200 Subject: [PATCH 15/16] Some code cleanup --- pythonforandroid/build.py | 326 ++++++++++++++++++++------------------ tests/test_build.py | 6 +- 2 files changed, 172 insertions(+), 160 deletions(-) diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index fa72fd9fc2..b76deb21a8 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -56,6 +56,62 @@ def get_toolchain_versions(ndk_dir, arch): return toolchain_versions, toolchain_path_exists +def select_and_check_toolchain_version(sdk_dir, ndk_dir, arch, ndk_sysroot_exists, py_platform): + toolchain_versions, toolchain_path_exists = get_toolchain_versions(ndk_dir, arch) + ok = ndk_sysroot_exists and toolchain_path_exists + toolchain_versions.sort() + + toolchain_versions_gcc = [] + for toolchain_version in toolchain_versions: + if toolchain_version[0].isdigit(): + # GCC toolchains begin with a number + toolchain_versions_gcc.append(toolchain_version) + + if toolchain_versions: + info('Found the following toolchain versions: {}'.format( + toolchain_versions)) + info('Picking the latest gcc toolchain, here {}'.format( + toolchain_versions_gcc[-1])) + toolchain_version = toolchain_versions_gcc[-1] + else: + warning('Could not find any toolchain for {}!'.format( + arch.toolchain_prefix)) + ok = False + + # Modify the path so that sh finds modules appropriately + environ['PATH'] = ( + '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' + 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' + '{toolchain_prefix}-{toolchain_version}/prebuilt/' + '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' + 'tools:{path}').format( + sdk_dir=sdk_dir, ndk_dir=ndk_dir, + toolchain_prefix=arch.toolchain_prefix, + toolchain_version=toolchain_version, + py_platform=py_platform, path=environ.get('PATH')) + + for executable in ( + "pkg-config", + "autoconf", + "automake", + "libtoolize", + "tar", + "bzip2", + "unzip", + "make", + "gcc", + "g++", + ): + if not sh.which(executable): + warning(f"Missing executable: {executable} is not installed") + + if not ok: + raise BuildInterruptingException( + 'python-for-android cannot continue due to the missing executables above') + + return toolchain_version + + def get_targets(sdk_dir): if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) @@ -253,8 +309,6 @@ def prepare_build_environment(self, if self._build_env_prepared: return - ok = True - # Work out where the Android SDK is sdk_dir = None if user_sdk_dir: @@ -385,55 +439,13 @@ def prepare_build_environment(self, self.ndk_standalone = get_ndk_standalone(self.ndk_dir) self.ndk_sysroot, ndk_sysroot_exists = get_ndk_sysroot(self.ndk_dir) - ok = ok and ndk_sysroot_exists self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include') for arch in self.archs: - - toolchain_versions, toolchain_path_exists = get_toolchain_versions( - self.ndk_dir, arch) - ok = ok and toolchain_path_exists - toolchain_versions.sort() - - toolchain_versions_gcc = [] - for toolchain_version in toolchain_versions: - if toolchain_version[0].isdigit(): - # GCC toolchains begin with a number - toolchain_versions_gcc.append(toolchain_version) - - if toolchain_versions: - info('Found the following toolchain versions: {}'.format( - toolchain_versions)) - info('Picking the latest gcc toolchain, here {}'.format( - toolchain_versions_gcc[-1])) - toolchain_version = toolchain_versions_gcc[-1] - else: - warning('Could not find any toolchain for {}!'.format( - arch.toolchain_prefix)) - ok = False - - # Modify the path so that sh finds modules appropriately - environ['PATH'] = ( - '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' - 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' - '{toolchain_prefix}-{toolchain_version}/prebuilt/' - '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' - 'tools:{path}').format( - sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, - toolchain_prefix=arch.toolchain_prefix, - toolchain_version=toolchain_version, - py_platform=py_platform, path=environ.get('PATH')) - - for executable in ("pkg-config", "autoconf", "automake", "libtoolize", - "tar", "bzip2", "unzip", "make", "gcc", "g++"): - if not sh.which(executable): - warning(f"Missing executable: {executable} is not installed") - - if not ok: - raise BuildInterruptingException( - 'python-for-android cannot continue due to the missing executables above') - - self.toolchain_version = toolchain_version # We assume that the toolchain version is the same for all the archs + # We assume that the toolchain version is the same for all the archs. + self.toolchain_version = select_and_check_toolchain_version( + self.sdk_dir, self.ndk_dir, arch, ndk_sysroot_exists, py_platform + ) def __init__(self): self.include_dirs = [] @@ -604,10 +616,11 @@ def build_recipes(build_order, python_modules, ctx, project_dir, recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') - run_pymodules_install( - ctx, python_modules, project_dir, - ignore_setup_py=ignore_project_setup_py - ) + for arch in ctx.archs: + run_pymodules_install( + ctx, arch, python_modules, project_dir, + ignore_setup_py=ignore_project_setup_py + ) def project_has_setup_py(project_dir): @@ -728,7 +741,7 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None): os.remove("._tmp_p4a_recipe_constraints.txt") -def run_pymodules_install(ctx, modules, project_dir=None, +def run_pymodules_install(ctx, arch, modules, project_dir=None, ignore_setup_py=False): """ This function will take care of all non-recipe things, by: @@ -740,119 +753,118 @@ def run_pymodules_install(ctx, modules, project_dir=None, """ - info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***') + info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch)) - for arch in ctx.archs: - modules = [m for m in modules if ctx.not_has_package(m, arch)] + modules = [m for m in modules if ctx.not_has_package(m, arch)] - # We change current working directory later, so this has to be an absolute - # path or `None` in case that we didn't supply the `project_dir` via kwargs - project_dir = abspath(project_dir) if project_dir else None + # We change current working directory later, so this has to be an absolute + # path or `None` in case that we didn't supply the `project_dir` via kwargs + project_dir = abspath(project_dir) if project_dir else None - # Bail out if no python deps and no setup.py to process: - if not modules and ( - ignore_setup_py or - project_dir is None or - not project_has_setup_py(project_dir) - ): - info('No Python modules and no setup.py to process, skipping') - return + # Bail out if no python deps and no setup.py to process: + if not modules and ( + ignore_setup_py or + project_dir is None or + not project_has_setup_py(project_dir) + ): + info('No Python modules and no setup.py to process, skipping') + return - # Output messages about what we're going to do: - if modules: - info( - "The requirements ({}) don\'t have recipes, attempting to " - "install them with pip".format(', '.join(modules)) - ) - info( - "If this fails, it may mean that the module has compiled " - "components and needs a recipe." - ) - if project_dir is not None and \ - project_has_setup_py(project_dir) and not ignore_setup_py: + # Output messages about what we're going to do: + if modules: + info( + "The requirements ({}) don\'t have recipes, attempting to " + "install them with pip".format(', '.join(modules)) + ) + info( + "If this fails, it may mean that the module has compiled " + "components and needs a recipe." + ) + if project_dir is not None and \ + project_has_setup_py(project_dir) and not ignore_setup_py: + info( + "Will process project install, if it fails then the " + "project may not be compatible for Android install." + ) + + # Use our hostpython to create the virtualenv + host_python = sh.Command(ctx.hostpython) + with current_directory(join(ctx.build_dir)): + shprint(host_python, '-m', 'venv', 'venv') + + # Prepare base environment and upgrade pip: + base_env = dict(copy.copy(os.environ)) + base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch) + info('Upgrade pip to latest version') + shprint(sh.bash, '-c', ( + "source venv/bin/activate && pip install -U pip" + ), _env=copy.copy(base_env)) + + # Install Cython in case modules need it to build: + info('Install Cython in case one of the modules needs it to build') + shprint(sh.bash, '-c', ( + "venv/bin/pip install Cython" + ), _env=copy.copy(base_env)) + + # Get environment variables for build (with CC/compiler set): + standard_recipe = CythonRecipe() + standard_recipe.ctx = ctx + # (note: following line enables explicit -lpython... linker options) + standard_recipe.call_hostpython_via_targetpython = False + recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) + env = copy.copy(base_env) + env.update(recipe_env) + + # Make sure our build package dir is available, and the virtualenv + # site packages come FIRST (so the proper pip version is used): + env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch) + env["PYTHONPATH"] = os.path.abspath(join( + ctx.build_dir, "venv", "lib", + "python" + ctx.python_recipe.major_minor_version_string, + "site-packages")) + ":" + env["PYTHONPATH"] + + # Install the manually specified requirements first: + if not modules: + info('There are no Python modules to install, skipping') + else: + info('Creating a requirements.txt file for the Python modules') + with open('requirements.txt', 'w') as fileh: + for module in modules: + key = 'VERSION_' + module + if key in environ: + line = '{}=={}\n'.format(module, environ[key]) + else: + line = '{}\n'.format(module) + fileh.write(line) + + info('Installing Python modules with pip') info( - "Will process project install, if it fails then the " - "project may not be compatible for Android install." + "IF THIS FAILS, THE MODULES MAY NEED A RECIPE. " + "A reason for this is often modules compiling " + "native code that is unaware of Android cross-compilation " + "and does not work without additional " + "changes / workarounds." ) - # Use our hostpython to create the virtualenv - host_python = sh.Command(ctx.hostpython) - with current_directory(join(ctx.build_dir)): - shprint(host_python, '-m', 'venv', 'venv') - - # Prepare base environment and upgrade pip: - base_env = dict(copy.copy(os.environ)) - base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch) - info('Upgrade pip to latest version') shprint(sh.bash, '-c', ( - "source venv/bin/activate && pip install -U pip" - ), _env=copy.copy(base_env)) + "venv/bin/pip " + + "install -v --target '{0}' --no-deps -r requirements.txt" + ).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")), + _env=copy.copy(env)) - # Install Cython in case modules need it to build: - info('Install Cython in case one of the modules needs it to build') - shprint(sh.bash, '-c', ( - "venv/bin/pip install Cython" - ), _env=copy.copy(base_env)) - - # Get environment variables for build (with CC/compiler set): - standard_recipe = CythonRecipe() - standard_recipe.ctx = ctx - # (note: following line enables explicit -lpython... linker options) - standard_recipe.call_hostpython_via_targetpython = False - recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) - env = copy.copy(base_env) - env.update(recipe_env) - - # Make sure our build package dir is available, and the virtualenv - # site packages come FIRST (so the proper pip version is used): - env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch) - env["PYTHONPATH"] = os.path.abspath(join( - ctx.build_dir, "venv", "lib", - "python" + ctx.python_recipe.major_minor_version_string, - "site-packages")) + ":" + env["PYTHONPATH"] - - # Install the manually specified requirements first: - if not modules: - info('There are no Python modules to install, skipping') - else: - info('Creating a requirements.txt file for the Python modules') - with open('requirements.txt', 'w') as fileh: - for module in modules: - key = 'VERSION_' + module - if key in environ: - line = '{}=={}\n'.format(module, environ[key]) - else: - line = '{}\n'.format(module) - fileh.write(line) - - info('Installing Python modules with pip') - info( - "IF THIS FAILS, THE MODULES MAY NEED A RECIPE. " - "A reason for this is often modules compiling " - "native code that is unaware of Android cross-compilation " - "and does not work without additional " - "changes / workarounds." - ) - - shprint(sh.bash, '-c', ( - "venv/bin/pip " + - "install -v --target '{0}' --no-deps -r requirements.txt" - ).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")), - _env=copy.copy(env)) - - # Afterwards, run setup.py if present: - if project_dir is not None and ( - project_has_setup_py(project_dir) and not ignore_setup_py - ): - run_setuppy_install(ctx, project_dir, env, arch.arch) - elif not ignore_setup_py: - info("No setup.py found in project directory: " + str(project_dir)) - - # Strip object files after potential Cython or native code builds: - if not ctx.with_debug_symbols: - standard_recipe.strip_object_files( - arch, env, build_dir=ctx.build_dir - ) + # Afterwards, run setup.py if present: + if project_dir is not None and ( + project_has_setup_py(project_dir) and not ignore_setup_py + ): + run_setuppy_install(ctx, project_dir, env, arch.arch) + elif not ignore_setup_py: + info("No setup.py found in project directory: " + str(project_dir)) + + # Strip object files after potential Cython or native code builds: + if not ctx.with_debug_symbols: + standard_recipe.strip_object_files( + arch, env, build_dir=ctx.build_dir + ) def biglink(ctx, arch): diff --git a/tests/test_build.py b/tests/test_build.py index 073377df34..1ffa46fc52 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -19,7 +19,7 @@ def test_run_pymodules_install_optional_project_dir(self): modules = [] project_dir = None with mock.patch('pythonforandroid.build.info') as m_info: - assert run_pymodules_install(ctx, modules, project_dir) is None + assert run_pymodules_install(ctx, ctx.archs[0], modules, project_dir) is None assert m_info.call_args_list[-1] == mock.call( 'No Python modules and no setup.py to process, skipping') @@ -44,13 +44,13 @@ def test_strip_if_with_debug_symbols(self): # Make sure it is NOT called when `with_debug_symbols` is true: ctx.with_debug_symbols = True - assert run_pymodules_install(ctx, modules, project_dir) is None + assert run_pymodules_install(ctx, ctx.archs[0], modules, project_dir) is None assert m_CythonRecipe().strip_object_files.called is False # Make sure strip object files IS called when # `with_debug_symbols` is fasle: ctx.with_debug_symbols = False - assert run_pymodules_install(ctx, modules, project_dir) is None + assert run_pymodules_install(ctx, ctx.archs[0], modules, project_dir) is None assert m_CythonRecipe().strip_object_files.called is True From 9d96664930792ffd98cb362139bbe77d554935e6 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 11 Sep 2021 19:46:53 +0200 Subject: [PATCH 16/16] Removes unusued versioning logic in unpackPyBundle and add a FIXME --- .../java/org/kivy/android/PythonUtil.java | 66 ++++--------------- 1 file changed, 14 insertions(+), 52 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index 38738d3065..7b3e45f739 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -201,61 +201,23 @@ public static void unpackPyBundle( Log.v(TAG, "Unpacking " + resource + " " + target.getName()); - // The version of data in memory and on disk. - String dataVersion = "p4aisawesome"; // FIXME: Assets method is not usable for fake .so files bundled as a library. - String diskVersion = null; - - Log.v(TAG, "Data version is " + dataVersion); - - // If no version, no unpacking is necessary. - if (dataVersion == null) { - return; - } - - // Check the current disk version, if any. - String filesDir = target.getAbsolutePath(); - String diskVersionFn = filesDir + "/" + resource + ".version"; - - // FIXME: Keeping that for later. Now it is surely failing. - try { - byte buf[] = new byte[64]; - InputStream is = new FileInputStream(diskVersionFn); - int len = is.read(buf); - diskVersion = new String(buf, 0, len); - is.close(); - } catch (Exception e) { - diskVersion = ""; - } + // FIXME: Implement a versioning logic to speed-up the startup process (maybe hash-based?). // If the disk data is out of date, extract it and write the version file. - if (! dataVersion.equals(diskVersion)) { - Log.v(TAG, "Extracting " + resource + " assets."); + Log.v(TAG, "Extracting " + resource + " assets."); - if (cleanup_on_version_update) { - recursiveDelete(target); - } - target.mkdirs(); - - AssetExtract ae = new AssetExtract(ctx); - if (!ae.extractTar(resource + ".so", target.getAbsolutePath(), "pybundle")) { - String msg = "Could not extract " + resource + " data."; - if (ctx instanceof Activity) { - toastError((Activity)ctx, msg); - } else { - Log.v(TAG, msg); - } - } - - try { - // Write .nomedia. - new File(target, ".nomedia").createNewFile(); - - // Write version file. - FileOutputStream os = new FileOutputStream(diskVersionFn); - os.write(dataVersion.getBytes()); - os.close(); - } catch (Exception e) { - Log.w("python", e); + if (cleanup_on_version_update) { + recursiveDelete(target); + } + target.mkdirs(); + + AssetExtract ae = new AssetExtract(ctx); + if (!ae.extractTar(resource + ".so", target.getAbsolutePath(), "pybundle")) { + String msg = "Could not extract " + resource + " data."; + if (ctx instanceof Activity) { + toastError((Activity)ctx, msg); + } else { + Log.v(TAG, msg); } } }