https://conda-forge.org/assets/js/b2bfd945.3e78536e.js

1"use strict";(globalThis.webpackChunkcf_infra_docs=globalThis.webpackChunkcf_infra_docs||[]).push([[8855],{56263(e,n,i){i.r(n),i.d(n,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>r,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"maintainer/understanding_conda_forge/cross-compilation","title":"Cross-compilation","description":"Cross-compilation refers to building binaries for a different platform than the one on which the","source":"@site/docs/maintainer/understanding_conda_forge/cross-compilation.md","sourceDirName":"maintainer/understanding_conda_forge","slug":"/maintainer/understanding_conda_forge/cross-compilation","permalink":"/docs/maintainer/understanding_conda_forge/cross-compilation","draft":false,"unlisted":false,"editUrl":"https://github.com/conda-forge/conda-forge.github.io/tree/refs/heads/main/docs/maintainer/understanding_conda_forge/cross-compilation.md","tags":[],"version":"current","lastUpdatedAt":1790351612000,"frontMatter":{"title":"Cross-compilation"},"sidebar":"diataxis","previous":{"title":"Compilation concepts","permalink":"/docs/maintainer/understanding_conda_forge/compilation-concepts"},"next":{"title":"The directory structure","permalink":"/docs/maintainer/understanding_conda_forge/directory-structure"}}');var t=i(74848),o=i(28453);const r={title:"Cross-compilation"},a=void 0,l={},c=[{value:"Platform triplets",id:"platform-triplets",level:2},{value:"Platform types",id:"platform-types",level:2},{value:"Cross-compilation environment",id:"cross-compilation-environment",level:2},{value:"Limitations of cross-compilation",id:"limitations-of-cross-compilation",level:2},{value:"Emulator use",id:"emulator-use",level:2},{value:"Toolchain setup",id:"toolchain-setup",level:2},{value:"Cross-compiled Python packages",id:"cross-compiled-python-packages",level:2}];function d(e){const n={a:"a",code:"code",em:"em",h2:"h2",li:"li",ol:"ol",p:"p",pre:"pre",ul:"ul",...(0,o.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.p,{children:"Cross-compilation refers to building binaries for a different platform than the one on which the\ncompiler is being run. In the most common case, a cross-compiler is used to build for a different\narchitecture, for example for AArch64 from an x86-64 system. However, cross-compilers can also be\nused to target different operating systems, for example to build Windows executables on a Linux\nsystem."}),"\n",(0,t.jsx)(n.h2,{id:"platform-triplets",children:"Platform triplets"}),"\n",(0,t.jsx)(n.p,{children:"Individual platforms are usually identified using triplets. These are strings of the form:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"{machine}-{vendor}-{operating system}\n"})}),"\n",(0,t.jsx)(n.p,{children:"A few examples are:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"aarch64-conda-linux-gnu\narm64-apple-darwin20.0.0\npowerpc64le-conda-linux-gnu\nx86_64-w64-mingw32\n"})}),"\n",(0,t.jsxs)(n.p,{children:["The machine field corresponds to the architecture such as ",(0,t.jsx)(n.code,{children:"x86_64"}),". Note that different operating\nsystems may be using different identifiers for the same architecture, e.g. ",(0,t.jsx)(n.code,{children:"aarch64"})," on Linux\ncorresponds to ",(0,t.jsx)(n.code,{children:"arm64"})," on macOS. Sometimes it also includes architecture version (",(0,t.jsx)(n.code,{children:"armv7a"}),")\nor endianness (",(0,t.jsx)(n.code,{children:"ppc64le"}),")."]}),"\n",(0,t.jsxs)(n.p,{children:["The vendor field can contain an arbitrary string. Conda-forge uses ",(0,t.jsx)(n.code,{children:"conda"})," on Linux and ",(0,t.jsx)(n.code,{children:"apple"})," on\nmacOS. Again, sometimes it conveys additional information, for example ",(0,t.jsx)(n.code,{children:"w64"})," is used to indicate\n",(0,t.jsx)(n.code,{children:"win-64"})," target for MinGW."]}),"\n",(0,t.jsxs)(n.p,{children:["The operating system field indicates the operating system, optionally followed by its version\n(such as ",(0,t.jsx)(n.code,{children:"linux-gnu"}),", ",(0,t.jsx)(n.code,{children:"linux-musl"})," or ",(0,t.jsx)(n.code,{children:"darwin20.0.0"}),"). Sometimes it may include additional ABI\nindication, such as ",(0,t.jsx)(n.code,{children:"linux-gnueabi"}),", or ",(0,t.jsx)(n.code,{children:"linux-gnueabihf"})," for ARM processors with hardware\nfloating-point unit."]}),"\n",(0,t.jsx)(n.h2,{id:"platform-types",children:"Platform types"}),"\n",(0,t.jsx)(n.p,{children:"When discussing toolchains, three types of platform are referenced:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"The build platform, referring to the platform on which the cross-toolchain is being run."}),"\n",(0,t.jsx)(n.li,{children:"The host platform, referring to the platform for which the binaries are being produced."}),"\n",(0,t.jsxs)(n.li,{children:["The target platform, which is only relevant if the binaries being produced will ",(0,t.jsx)(n.em,{children:"themselves"})," target\nsome specific architecture. This is generally only needed when building a cross-toolchain, in which\ncase it refers to the platform for which the resulting toolchain will produce binaries."]}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["Toolchain executables are often prefixed by the host platform triplet, e.g.\n",(0,t.jsx)(n.code,{children:"x86_64-conda-linux-gnu-gcc"})," indicates a compiler producing binaries for ",(0,t.jsx)(n.code,{children:"x86_64-conda-linux-gnu"}),"\nhost platform."]}),"\n",(0,t.jsxs)(n.p,{children:["The platform designations are relative to the particular build process. For example, when building\na cross-compiler on ",(0,t.jsx)(n.code,{children:"x86_64-conda-linux-gnu"})," platform that will run on the same platform, but\nproduce co

1de for ",(0,t.jsx)(n.code,{children:"aarch64-conda-linux-gnu"}),", build and host platforms will be\n",(0,t.jsx)(n.code,{children:"x86_64-conda-linux-gnu"})," and target platform will be ",(0,t.jsx)(n.code,{children:"aarch64-conda-linux-gnu"}),". However, when\nusing that compiler afterwards, ",(0,t.jsx)(n.code,{children:"x86_64-conda-linux-gnu"})," will be the build platform, whereas\n",(0,t.jsx)(n.code,{children:"aarch64-conda-linux-gnu"})," will be the host platform."]}),"\n",(0,t.jsx)(n.h2,{id:"cross-compilation-environment",children:"Cross-compilation environment"}),"\n",(0,t.jsx)(n.p,{children:"A cross-compilation environment requires a number of components:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsxs)(n.li,{children:["A cross-toolchain, i.e. compiler, linker and other tools that run on the build platform and are\ncapable of producing binaries for the host platform. In some cases (such as GCC) a dedicated\ncross-compiler is built for every target, whereas in others, such as Clang, a single compiler is\ncapable of producing binaries for multiple targets. The prefix containing these executables\nis de

1signated by the ",(0,t.jsx)(n.code,{children:"${BUILD_PREFIX}"})," variable, and the tools themselves are frequently prefixed\nby the host triplet."]}),"\n",(0,t.jsxs)(n.li,{children:["Other binaries that are run throughout the build process, such as the build system executables,\nshell tools, etc. These are also installed into the ",(0,t.jsx)(n.code,{children:"${BUILD_PREFIX}"}),"."]}),"\n",(0,t.jsxs)(n.li,{children:["The sysroot built for the host platform, including the system libraries or system library stubs,\nand system header files. While conda-forge packages use the underlying operating system libraries\nat runtime, packages are built against prepackaged fixed versions of these libraries for\nbackwards compatibility; this is also naturally a necessity when system libraries are provided\nfor the different platform. The sysroot is installed into a subdirectory of ",(0,t.jsx)(n.code,{children:"${BUILD_PREFIX}"}),",\nand are used by the compiler automatically."]}),"\n",(0,t.jsxs)(n.li,{children:["The files specific to host platform that are needed during the build process, such as libraries,\nheader files, pkg-config files, etc. These files are installed into the location designated\nby the ",(0,t.jsx)(n.code,{children:"${PREFIX}"})," variable, which is also used as the install location for the built package\nfiles."]}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["The distinction between the two prefixes is essential while cross-compiling. The build process can\nonly run programs built for the build platform, and therefore all the tools needed at build time\nneed to be available in ",(0,t.jsx)(n.code,{children:"${BUILD_PREFIX}"}),", and consequently they are specified as ",(0,t.jsx)(n.code,{children:"build"}),"\ndependencies. However, the headers, libraries, pkg-config files and other files describing the\ncharacteristics of the host platform need to be used from ",(0,t.jsx)(n.code,{children:"${PREFIX}"}),", and specified as ",(0,t.jsx)(n.code,{children:"host"}),"\ndependencies."]}),"\n",(0,t.jsxs)(n.p,{children:["In conda-forge, the build setup and compiler activation scripts attempt to configure the build\nprocess for correct locations, but in some cases (e.g. when dealing with custom ",(0,t.jsx)(n.code,{children:"*-config"})," tools),\nadditional effort may be required from feedstock maintainers to point the build system at the\ncorrect locations."]}),"\n",(0,t.jsx)(n.h2,{id:"limitations-of-cross-compilation",children:"Limitations of cross-compilation"}),"\n",(0,t.jsx)(n.p,{children:"While performing cross-builds without an emulator, the build process cannot run executables built by\nthe cross-compiler, since they are built for a different platform. This has various implications,\nsuch as:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"Programs are unable to run their test suites. Testing is often skipped when cross-compiling, or\nemulators are used to run tests."}),"\n",(0,t.jsxs)(n.li,{children:["Build-time utility programs have to be built for the build platform rather than the host platform.\nSome build systems handle this automatically by using the build platform compiler in addition to\nthe host compiler. In other cases, it may be necessary to build these utilities manually. This\nalso implies that their dependencies need to be present in the ",(0,t.jsx)(n.code,{children:"build"})," requirements section."]}),"\n",(0,t.jsx)(n.li,{children:"Some platform checks cannot be performed. The correct values for the relevant platform\ncharacteristics need then to be provided directly to the build system."}),"\n"]}),"\n",(0,t.jsx)(n.h2,{id:"emulator-use",children:"Emulator use"}),"\n",(0,t.jsxs)(n.p,{children:["Many of the aforementioned limitations can be overcome by using an emulator. Conda-forge supports\ntwo kinds of emulation: full-system emulation and userspace emulation. The former is covered in\n",(0,t.jsx)(n.a,{href:"/docs/maintainer/knowledge_base/#emulated-builds",children:"Emulated builds"}),", but it is generally discouraged\nbecause it is substantially slower (roughly 5x-6x longer runtime) than cross-compiling. On the other\nhand, the latter can be successfully comb

1ined with cross-compilation to limit its overhead."]}),"\n",(0,t.jsxs)(n.p,{children:["The conda-forge CI setups provide userspace emulation support on Linux, using ",(0,t.jsx)(n.code,{children:"qemu-user"}),". The path\nto the emulator is then provided in the ",(0,t.jsx)(n.code,{children:"CROSSCOMPILING_EMULATOR"})," variable. However, usually there\nis no need to use it directly, as ",(0,t.jsx)(n.code,{children:"binfmt_misc"})," permits executing non-native executables directly."]}),"\n",(0,t.jsx)(n.p,{children:"This could eliminate the need to build executables for the build platform, as the executables built\nfor the host platform can be run in CI. This is useful if the executable queries some information\nabout the system it runs on. However, if the executable does not query system information, or if the\nbuild system and the host system are sufficiently similar for the queries run, it's advisable to\nbuild the executable for the build platform, and for two reasons:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsxs)(n.li,{children:["Userspace emulation is not available on all platforms. For example, emulation cannot be used\nwhile cross-compiling from ",(0,t.jsx)(n.code,{children:"osx-64"})," to ",(0,t.jsx)(n.code,{children:"osx-arm64"}),"."]}),"\n",(0,t.jsx)(n.li,{children:"Enabling userspace emulation requires superuser privileges, which may make it harder for users to\nbuild the package locally, in particular to build the build process."}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["Parts of the recipe such as tests can require an emulator, but they must be guarded accordingly, and\nrun only for native builds or when ",(0,t.jsx)(n.code,{children:"CROSSCOMPILING_EMULATOR"})," is not empty."]}),"\n",(0,t.jsx)(n.h2,{id:"toolchain-setup",children:"Toolchain setup"}),"\n",(0,t.jsxs)(n.p,{children:["In conda-forge packages, the toolchain packages are always listed in the ",(0,t.jsx)(n.code,{children:"build"})," requirements\nsection. They are generally declared using the ",(0,t.jsx)(n.code,{children:"compiler"})," and ",(0,t.jsx)(n.code,{children:"stdlib"})," macros that map them through\nconda-forge-pinning and then expand into packages appropriate for the host platform. For example,\na ",(0,t.jsx)(n.code,{children:'compiler("c")'})," entry will be ",(0,t.jsx)(n.a,{href:"https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/38b2437526e47309199fc18ca05596e20ceaf05c/recipe/conda_build_config.yaml#L1-L4",children:"mapped to one of the appropriate compiler\nnames"}),"\nsuch as ",(0,t.jsx)(n.code,{children:"gcc"}),", then expanded into a platform-suffixed package such as ",(0,t.jsx)(n.code,{children:"gcc_linux-aarch64"}),"."]}),"\n",(0,t.jsxs)(n.p,{children:["When built for a native environment, this package pulls the native toolchain and sysroot for the\nplatform in question. However, when built for a cross-compilation environment, it pulls both the\nnative and cross-toolchain, effectively making it possible both to compile the host binaries, and\nany binaries needed for the build process itself. So ",(0,t.jsx)(n.code,{children:"gcc_linux-aarch64"})," on ",(0,t.jsx)(n.code,{children:"linux-64"})," pulls in both\n",(0,t.jsx)(n.code,{children:"*_linux-aarch64"})," packages and ",(0,t.jsx)(n.code,{children:"*_linux-64"})," packages."]}),"\n",(0,t.jsxs)(n.p,{children:["In addition to that, the activation scripts installed by these packages ensure that the environment\nis set correctly for cross-compiling. This includes setting generic variables for use of\ncross-compiler (such as ",(0,t.jsx)(n.code,{children:"CC"}),") and for the native compiler (",(0,t.jsx)(n.code,{children:"CC_FOR_BUILD"}),"), as well as best-effort\nsetup needed for different build systems such as CMake and Meson."]}),"\n",(0,t.jsx)(n.h2,{id:"cross-compiled-python-packages",children:"Cross-compiled Python packages"}),"\n",(0,t.jsxs)(n.p,{children:["Conda-forge employs a series of workarounds to make cross-compilation work for Python packages.\nThe problem is discussed in greater detail in ",(0,t.jsx)(n.a,{href:"https://peps.python.org/pep-0720/",children:"PEP720"})," and\n",(0,t.jsx)(n.a,{href:"https://github.com/conda-forge/conda-forge.github.io/issues/1841",children:"conda-forge.github.io#1841"}),"."]}),"\n",(0,t.jsx)(n.p,{children:'In the terms of the PEP720, the conda-forge setup implements the "faking the target environment"\napproach. More specifically, this will result in the following changes before the builds scripts\nrun:'}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:["A modified ",(0,t.jsx)(n.code,{children:"crosse

1nv"})," installation in ",(0,t.jsx)(n.code,{children:"$BUILD_PREFIX/venv"}),", mimicking the architecture of\n",(0,t.jsx)(n.code,{children:"$PREFIX"}),"."]}),"\n",(0,t.jsxs)(n.li,{children:["Forwarder binaries in ",(0,t.jsx)(n.code,{children:"$BUILD_PREFIX/bin"})," that point to the ",(0,t.jsx)(n.code,{children:"crossenv"})," installation."]}),"\n",(0,t.jsxs)(n.li,{children:["Symlinks that expose the ",(0,t.jsx)(n.code,{children:"$BUILD_PREFIX"})," site-packages in the ",(0,t.jsx)(n.code,{children:"crossenv"})," installation, which\nis also included in ",(0,t.jsx)(n.code,{children:"$PYTHONPATH"}),"."]}),"\n",(0,t.jsxs)(n.li,{children:["A copy of all ",(0,t.jsx)(n.code,{children:"$PREFIX"})," site-packages to ",(0,t.jsx)(n.code,{children:"$BUILD_PREFIX"})," (except the compiled libraries)."]}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["All in all, this results in a setup where ",(0,t.jsx)(n.code,{children:"conda-build"})," can run a ",(0,t.jsx)(n.code,{children:"$BUILD_PREFIX"}),"-architecture\n",(0,t.jsx)(n.code,{children:"python"})," interpreter that can see the packages in ",(0,t.jsx)(n.code,{children:"$PREFIX"})," (with the compiled bits provided by\ntheir corresponding counterparts in ",(0,t.jsx)(n.code,{children:"$BUILD_PREFIX"}),") and sufficiently mimic that target\narchitecture."]})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(d,{...e})}):d(e)}},28453(e,n,i){i.d(n,{R:()=>r,x:()=>a});var s=i(96540);const t={},o=s.createContext(t);function r(e){const n=s.useContext(o);return s.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:r(e.components),s.createElement(o.Provider,{value:n},e.children)}}}]);