https://curvineio.github.io/assets/js/793a26a6.7df220d3.js

1"use strict";(self.webpackChunkcurvine_doc=self.webpackChunkcurvine_doc||[]).push([[3474],{28453:(e,n,i)=>{i.d(n,{R:()=>d,x:()=>l});var r=i(96540);const o={},s=r.createContext(o);function d(e){const n=r.useContext(s);return r.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function l(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:d(e.components),r.createElement(s.Provider,{value:n},e.children)}},91018:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>c,contentTitle:()=>l,default:()=>h,frontMatter:()=>d,metadata:()=>r,toc:()=>t});const r=JSON.parse('{"id":"Deploy/Deploy-Curvine-Cluster/Preparation/compile","title":"Download and Compile Curvine","description":"This chapter introduces how to download and compile Curvine.","source":"@site/docs/2-Deploy/2-Deploy-Curvine-Cluster/1-Preparation/02-compile.md","sourceDirName":"2-Deploy/2-Deploy-Curvine-Cluster/1-Preparation","slug":"/Deploy/Deploy-Curvine-Cluster/Preparation/compile","permalink":"/docs/Deploy/Deploy-Curvine-Cluster/Preparation/compile","draft":false,"unlisted":false,"editUrl":"https://github.com/curvineio/curvine-doc/edit/main/docs/2-Deploy/2-Deploy-Curvine-Cluster/1-Preparation/02-compile.md","tags":[],"version":"current","sidebarPosition":1,"frontMatter":{"sidebar_position":1},"sidebar":"tutorialSidebar","previous":{"title":"Build Instructions","permalink":"/docs/Deploy/build-instructions"},"next":{"title":"Deployment Preparation","permalink":"/docs/Deploy/Deploy-Curvine-Cluster/Preparation/deployment-setup"}}');var o=i(74848),s=i(28453);const d={sidebar_position:1},l="Download and Compile Curvine",c={},t=[{value:"Supported Linux Distributions",id:"supported-linux-distributions",level:2},{value:"Local Compilation",id:"local-compilation",level:2},{value:"Docker Compilation",id:"docker-compilation",level:2},{value:"1. Using Curvine-provided Compilation Images",id:"1-using-curvine-provided-compilation-images",level:3},{value:"2. Advanced: Build Your Own Compilation Image",id:"2-advanced-build-your-own-compilation-image",level:3},{value:"UFS Module Support",id:"ufs-module-support",level:2}];function a(e){const n={a:"a",admonition:"admonition",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(n.header,{children:(0,o.jsx)(n.h1,{id:"download-and-compile-curvine",children:"Download and Compile Curvine"})}),"\n",(0,o.jsx)(n.p,{children:"This chapter introduces how to download and compile Curvine."}),"\n",(0,o.jsx)(n.h2,{id:"supported-linux-distributions",children:"Supported Linux Distributions"}),"\n",(0,o.jsxs)(n.table,{children:[(0,o.jsx)(n.thead,{children:(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.th,{children:"OS Distribution"}),(0,o.jsx)(n.th,{children:"Kernel Requirement"}),(0,o.jsx)(n.th,{children:"Tested Version"}),(0,o.jsx)(n.th,{children:"Dependencies"})]})}),(0,o.jsxs)(n.tbody,{children:[(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:(0,o.jsx)(n.strong,{children:"CentOS 7"})}),(0,o.jsx)(n.td,{children:"\u22653.10.0"}),(0,o.jsx)(n.td,{children:"7.6"}),(0,o.jsx)(n.td,{children:"fuse2-2.9.2"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:(0,o.jsx)(n.strong,{children:"CentOS 8"})}),(0,o.jsx)(n.td,{children:"\u22654.18.0"}),(0,o.jsx)(n.td,{children:"8.5"}),(0,o.jsx)(n.td,{children:"fuse3-3.9.1"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:(0,o.jsx)(n.strong,{children:"Rocky Linux 9"})}),(0,o.jsx)(n.td,{children:"\u22655.14.0"}),(0,o.jsx)(n.td,{children:"9.5"}),(0,o.jsx)(n.td,{children:"fuse3-3.10.2"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:(0,o.jsx)(n.strong,{children:"RHEL 9"})}),(0,o.jsx)(n.td,{children:"\u22655.14.0"}),(0,o.jsx)(n.td,{children:"9.5"}),(0,o.jsx)(n.td,{children:"fuse3-3.10.2"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:(0,o.jsx)(n.strong,{children:"Ubuntu 22"})}),(0,o.jsx)(n.td,{children:"\u22655.15.0"}),(0,o.jsx)(n.td,{children:"22.4"}),(0,o.jsx)(n.td,{children:"fuse3-3.10.5"})]})]})]}),"\n",(0,o.jsxs)(n.p,{children:["Download the source code (clone into an empty directory, or use a new folder name in place of ",(0,o.jsx)(n.code,{children:"./"}),"):"]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"git clone https://github.com/CurvineIO/curvine.git ./\n"})}),"\n",(0,o.jsx)(n.h2,{id:"local-compilation",children:"Local Compilation"}),"\n",(0,o.jsx)(n.admonition,{type:"warning",children:(0,o.jsxs)(n.p,{children:["Please ensure that the prerequisite dependencies are installed and configured. See the ",(0,o.jsx)(n.a,{href:"/docs/Deploy/Deploy-Curvine-Cluster/Preparation/prerequisites",children:"Environment Initialization Tutorial"})," or the ",(0,o.jsx)(n.a,{href:"https://github.com/CurvineIO/curvine/blob/main/curvine-docker/compile/Dockerfile_rocky9",children:"Dockerfile for a reference environment"}),"."]})}),"\n",(0,o.jsx)(n.p,{children:"Optional: check that the build environment is ready:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"make check-env\n"})}),"\n",(0,o.jsxs)(n.p,{children:["Then build. The compiled output is under ",(0,o.jsx)(n.code,{children:"build/dist"})," (contains ",(0,o.jsx)(n.code,{children:"bin/"}),", ",(0,o.jsx)(n.code,{children:"conf/"}),", ",(0,o.jsx)(n.code,{children:"lib/"}),", etc.\u2014the same layout used for deployment):"]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"make all\n"})}),"\n",(0,o.jsx)(n.p,{children:"You can also call the build script directly (e.g. to see all options):"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"sh build/build.sh          # build all (release)\nsh build/build.sh -h       # show help and package/ufs options\nsh build/build.sh -p core  # build only server, client, cli\n"})}),"\n",(0,o.jsxs)(n.p,{children:["To create a distribution archive for deployment, see ",(0,o.jsx)(n.a,{href:"/docs/Deploy/Deploy-Curvine-Cluster/Preparation/deployment-setup#create-installation-package",children:"Deployment Preparation"})," (",(0,o.jsx)(n.code,{children:"make dist"}),")."]}),"\n",(0,o.jsxs)(n.admonition,{type:"note",children:[(0,o.jsxs)(n.p,{children:["More make targets: run ",(0,o.jsx)(n.code,{children:"make"})," or ",(0,o.jsx)(n.code,{children:"make help"}),". Summary:"]}),(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Environment:"})," ",(0,o.jsx)(n.code,{children:"make check-env"})," \u2014 Check build dependencies (also run automatically by ",(0,o.jsx)(n.code,{children:"make build"}),")."]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Building:"})," ",(0,o.jsx)(n.code,{children:"make build"})," / ",(0,o.jsx)(n.code,{children:"make all"})," \u2014 Build; output in ",(0,o.jsx)(n.code,{children:"build/dist"}),". ",(0,o.jsx)(n.code,{children:"make dist"})," \u2014 Build and create tar.gz in project root. ",(0,o.jsx)(n.code,{children:"make dist-only"})," \u2014 Package existing ",(0,o.jsx)(n.code,{children:"build/dist"})," only."]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Partial builds:"})," ",(0,o.jsx)(n.code,{children:"make build ARGS='-p core'"})," (server+client+cli), ",(0,o.jsx)(n.code,{children:"make build ARGS='-p core -p fuse'"}),", ",(0,o.jsx)(n.code,{children:"make build ARGS='-p object'"})," (S3 gateway), ",(0,o.jsx)(n.code,{children:"make build ARGS='-d'"})," (debug). Use ",(0,o.jsx)(n.code,{children:"make build-hdfs"})," for HDFS support (native + WebHDFS); ",(0,o.jsx)(n.code,{children:"make build ARGS='--skip-java-sdk'"})," to skip Java SDK."]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Docker:"})," ",(0,o.jsx)(n.code,{children:"make docker-build"})," \u2014 Build ",(0,o.jsx)(n.strong,{children:"runtime"})," image from source. ",(0,o.jsx)(n.code,{children:"make docker-build-compile"})," \u2014 Build compilation image (interactive). ",(0,o.jsx)(n.code,{children:"make docker-compile"})," \u2014 Compile in container, output to local ",(0,o.jsx)(n.code,{children:"build/dist"}),"."]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"CSI:"})," ",(0,o.jsx)(n.code,{children:"make curvine-csi"})," \u2014 Build curvine-csi Docker image."]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Example:"})," ",(0,o.jsx)(n.code,{children:"RELEASE_VERSION=v1.0.0 make dist"})," \u2014 Build and package with a version tag."]}),"\n"]})]}),"\n",(0,o.jsx)(n.h2,{id:"docker-compilation",children:"Docker Compilation"}),"\n",(0,o.jsx)(n.admonition,{type:"tip",children:(0,o.jsxs)(n.p,{children:["If your system environment is macOS or Windows, or your Linux version is not in the ",(0,o.jsx)(n.a,{href:"#supported-linux-distributions",children:"supported list"}

1),", we recommend using Docker compilation. This allows you to operate safely in an isolated environment without affecting your system environment."]})}),"\n",(0,o.jsx)(n.h3,{id:"1-using-curvine-provided-compilation-images",children:"1. Using Curvine-provided Compilation Images"}),"\n",(0,o.jsxs)(n.p,{children:["Curvine provides compilation images on Docker Hub: ",(0,o.jsx)(n.code,{children:"curvine/curvine-compile:latest"})," (and optionally ",(0,o.jsx)(n.code,{children:"curvine/curvine-compile:build-cached"})," with cached dependencies)."]}),"\n",(0,o.jsx)(n.admonition,{type:"tip",children:(0,o.jsxs)(n.p,{children:["Use the compile image as a sandbox: run compilation (and optionally execution) inside the container, with the repo mounted at ",(0,o.jsx)(n.code,{children:"/workspace"}),"."]})}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsxs)(n.strong,{children:["Option A \u2014 Compile in container, output to local ",(0,o.jsx)(n.code,{children:"build/dist"}),":"]})}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"make docker-compile\n"})}),"\n",(0,o.jsxs)(n.p,{children:["(Uses image ",(0,o.jsx)(n.code,{children:"curvine/curvine-compile:build-cached"}),"; result appears in ",(0,o.jsx)(n.code,{children:"build/dist"}),".)"]}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.strong,{children:"Option B \u2014 Persistent development container:"})}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"docker run -itd --name curvine-compile \\\n  -u root --privileged=true \\\n  -v $(pwd):/workspace -w /workspace \\\n  --network host \\\n  curvine/curvine-compile:latest /bin/bash\n\ndocker exec -it curvine-compile /bin/bash\n# inside container: make all\n"})}),"\n",(0,o.jsx)(n.h3,{id:"2-advanced-build-your-own-compilation-image",children:"2. Advanced: Build Your Own Compilation Image"}),"\n",(0,o.jsx)(n.admonition,{type:"tip",children:(0,o.jsxs)(n.p,{children:["If you cannot use the official images (e.g. network restrictions), build the compilation image locally from the Dockerfiles in ",(0,o.jsx)(n.code,{children:"curvine-docker/compile"}),"."]})}),"\n",(0,o.jsxs)(n.p,{children:["Available Dockerfiles under ",(0,o.jsx)(n.code,{children:"curvine-docker/compile/"}),":"]}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Dockerfile_rocky9"})," \u2014 Rocky Linux 9 (recommended baseline)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Dockerfile_rocky9_cached"})," \u2014 Rocky 9 with dependency cache (faster rebuilds)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Dockerfile_ubuntu22"})," \u2014 Ubuntu 22.04"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Dockerfile_ubuntu24"})," \u2014 Ubuntu 24.04"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Dockerfile_amzn2"})," \u2014 Amazon Linux 2"]}),"\n"]}),"\n",(0,o.jsxs)(n.p,{children:["From the ",(0,o.jsx)(n.strong,{children:"project root"}),":"]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:"# Build the compilation image (choose one Dockerfile)\ndocker build -f curvine-docker/compile/Dockerfile_rocky9 -t curvine-compile:rocky9 curvine-docker/compile\n\n# Run container and compile\ndocker run -itd --name curvine-compile \\\n  -u root --privileged=true \\\n  -v $(pwd):/workspace -w /workspace \\\n  --network host \\\n  curvine-compile:rocky9 /bin/bash\n\ndocker exec -it curvine-compile /bin/bash\n# inside: make all\n"})}),"\n",(0,o.jsxs)(n.p,{children:["Compiled output is in ",(0,o.jsx)(n.code,{children:"build/dist"})," inside the container (and on the host if you mounted the repo)."]}),"\n",(0,o.jsxs)(n.admonition,{type:"warning",children:[(0,o.jsx)(n.p,{children:"If there are significant differences between your compilation image's OS version and the host machine's OS version, or they are not from the same distribution, the Docker-compiled artifacts may not run directly on the host machine due to libc or ABI incompatibilities."}),(0,o.jsx)(n.p,{children:"Therefore, for Docker-compiled artifacts, we strongly recommend running them on the same OS version or within Docker containers!"})]}),"\n",(0,o.jsx)(n.h2,{id:"ufs-module-support",children:"UFS Module Support"}),"\n",(0,o.jsxs)(n.p,{children:["Curvine supports multiple storage backends including OSS, HDFS, S3, MinIO, and more. By default, these are integrated via OpenDAL. Curvine also provides optimized implementations for OSS and HDFS. You can compile with specific modules using ",(0,o.jsx)(n.code,{children:"make all --ufs oss-hdfs"}),"."]}),"\n",(0,o.jsx)(n.p,{children:"Supported parameters:"}),"\n",(0,o.jsxs)(n.table,{children:[(0,o.jsx)(n.thead,{children:(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.th,{children:"Parameter"}),(0,o.jsx)(n.th,{children:"Description"})]})}),(0,o.jsxs)(n.tbody,{children:[(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:"oss-hdfs"}),(0,o.jsx)(n.td,{children:"Use JindoSDK for Alibaba Cloud"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:"opendal-oss"}),(0,o.jsx)(n.td,{children:"Use OpenDAL for OSS object storage"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:"opendal-hdfs"}),(0,o.jsx)(n.td,{children:"Run in native mode"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:"opendal-s3"}),(0,o.jsx)(n.td,{children:"Use OpenDAL for S3 object storage"})]}),(0,o.jsxs)(n.tr,{children:[(0,o.jsx)(n.td,{children:"opendal-azblob"}),(0,o.jsx)(n.td,{children:"Use OpenDAL for Azure Blob"})]})]})]}),"\n",(0,o.jsxs)(n.admonition,{type:"warning",children:[(0,o.jsxs)(n.p,{children:["When compiling the ",(0,o.jsx)(n.code,{children:"oss-hdfs"})," module, you need to set JindoSDK environment variables:"]}),(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-bash",children:'export JINDOSDK_HOME=/opt/jindosdk-6.10.3\

1nexport LD_LIBRARY_PATH="${JINDOSDK_HOME}/lib/native:${LD_LIBRARY_PATH}"\nmake all --ufs oss-hdfs\n'})})]}),"\n",(0,o.jsxs)(n.p,{children:["To compile multiple UFS modules simultaneously, specify ",(0,o.jsx)(n.code,{children:"--ufs"})," multiple times, e.g. ",(0,o.jsx)(n.code,{children:"make all --ufs oss-hdfs --ufs opendal-oss"})]}),"\n",(0,o.jsxs)(n.p,{children:["Note that both OSS accelerated buckets and standard object storage buckets use the ",(0,o.jsx)(n.code,{children:"oss://test-bucket"})," format. When mounting OSS storage, you need to specify the appropriate module. For detailed mount usage, see the CLI mount section."]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(a,{...e})}):a(e)}}}]);