diff --git a/docs/.vitepress/config/sidebar/apiSidebar.zh.json b/docs/.vitepress/config/sidebar/apiSidebar.zh.json index d7af646..b6ce80e 100644 --- a/docs/.vitepress/config/sidebar/apiSidebar.zh.json +++ b/docs/.vitepress/config/sidebar/apiSidebar.zh.json @@ -85,4 +85,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/docs/zh/api/cli/cli.md b/docs/zh/api/cli/cli.md index 0e327fb..855fd66 100644 --- a/docs/zh/api/cli/cli.md +++ b/docs/zh/api/cli/cli.md @@ -3,6 +3,10 @@ This is a reference for the CLI commands of envd. +::: tip +The documentation is auto-generated from [envd app](https://github.com/tensorchord/envd/blob/main/pkg/app/app.go), please do not edit it manually. +::: + # NAME envd - Development environment for data science and AI/ML teams @@ -12,9 +16,9 @@ envd - Development environment for data science and AI/ML teams envd ``` -[--buildkitd-container-name]=[value] -[--buildkitd-image]=[value] +[--analytics-enabled] [--debug] +[--help|-h] ``` **Usage**: @@ -25,109 +29,338 @@ envd [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...] # GLOBAL OPTIONS -**--buildkitd-container-name**="": buildkitd container to use for buildkitd (default: envd_buildkitd) - -**--buildkitd-image**="": docker image to use for buildkitd (default: docker.io/moby/buildkit:v0.10.3) +**--analytics-enabled**: enable analytics **--debug**: enable debug output in logs +**--help, -h**: show help + # COMMANDS ## bootstrap -Bootstraps envd installation including shell autocompletion and buildkit image download +Bootstrap the envd installation **--buildkit, -b**: Download the image and bootstrap buildkit -**--dockerhub-mirror, -m**="": Dockerhub mirror to use +**--dockerhub-mirror, -m**="": DockerHub mirror to use + +**--registry, -r**="": Specify the registry to pull the image from (default: "docker.io") + +**--registry-ca-keypair, --ca**="": Specify the ca/key/cert file path for the private registry (format: 'ca=/etc/config/ca.pem,key=/etc/config/key.pem,cert=/etc/config/cert.pem') + +**--registry-config**="": Path to a JSON file containing registry configuration. Cannot be used with 'registry' or 'registry-ca-keypair' + +**--ssh-keypair, -k**="": Manually specify ssh key pair as `publicKey,privateKey`. envd will generate a keypair at /home/runner/.config/envd/id_rsa_envd.pub and /home/runner/.config/envd/id_rsa_envd if not specified + +**--use-http**: Use HTTP instead of HTTPS for the registry + +**--with-autocomplete**: Add envd auto-completions + +## run, c + +Run the envd environment from the existing image + +**--cpus**="": Request CPU resources (number of cores), such as 0.5, 1, 2 + +**--detach**: Detach from the container + +**--gpu-set**="": GPU devices used in this environment, such as `all`, `'"device=1,3"'`, `count=2`(all to pass all GPUs). This will override the `--gpus` + +**--gpus**="": Request GPU resources (number of gpus), such as 1, 2 (default: 0) + +**--host**="": Assign the host address for the environment SSH access server listening (default: "127.0.0.1") + +**--image**="": image name (default: PROJECT:dev) + +**--memory**="": Request Memory, such as 512M, 2G + +**--name**="": environment name + +**--path, -p**="": Working directory path to be used as project root (default: ".") + +**--shm-size**="": Configure the shared memory size (megabyte) (default: 2048) + +**--sync**: Sync the local directory with the remote container (only supported in envd-server runner currently) + +**--timeout**="": Timeout of container creation (default: 30m0s) + +**--volume, -v**="": Mount host directory into container + +## completion + +Install shell completion scripts for envd + +**--no-install**: Only output the completion script and don't install it + +**--shell, -s**="": Shell type to install completion + +## context + +Manage envd contexts -**--with-autocomplete**: Add envd autocompletions +### create + +Create envd context + +**--builder**="": Builder to use (docker-container, kube-pod, tcp, unix, moby-worker, nerdctl-container) (default: "docker-container") + +**--builder-address**="": Builder address (default: "envd_buildkitd") + +**--name**="": Name of the context + +**--runner**="": Runner to use(docker, envd-server) (default: "docker") + +**--runner-address**="": Runner address + +**--use**: Use the context + +### ls + +List envd contexts + +**--format, -f**="": Format of output, could be "json" or "table" (default: "table") + +### rm + +Remove envd context + +**--name**="": Name of the context + +### use + +Use the specified envd context + +**--name**="": Name of the context ## build, b -build envd environment +Build the envd environment + +**--export-cache, --ec**="": Export the cache (e.g. `type=registry,ref=`). The default `moby-worker` builder doesn't support this unless the docker-ce has enabled the `containerd` image store. You can run `envd context create --name docker --builder docker-container --use` to use this feature. + +**--force**: Force rebuild the image + +**--from, -f**="": Function to execute, format `file:func` (default: "build.envd:build") -**--file, -f**="": Name of the build.envd (default: build.envd) +**--import-cache, --ic**="": Import the cache (e.g. `type=registry,ref=`) -**--output, -o**="": Output destination (format: type=tar,dest=path) +**--output, -o**="": Output destination (e.g. `type=tar,dest=path,push=true`) -**--path, -p**="": Path to the directory containing the build.envd (default: .) +**--path, -p**="": Path to the directory containing the build.envd (default: ".") -**--public-key, --pubk**="": Path to the public key (default: /home/runner/.config/envd/id_rsa_envd.pub) +**--platform**="": Specify the target platforms for the build output (for example, "windows/amd64" or "linux/amd64,darwin/arm64"). +Build images with same tags could cause image overwriting, platform suffixes will be added to differentiate the images. (default: linux/amd64) **--tag, -t**="": Name and optionally a tag in the 'name:tag' format (default: PROJECT:dev) -## destroy, d +**--use-proxy, --proxy**: Use HTTPS_PROXY/HTTP_PROXY/NO_PROXY in the build process -destroys the envd environment +## destroy, down, d -**--name, -n**="": Name of the environment +Destroy the envd environment + +**--name, -n**="": Name of the environment or container ID + +**--path, -p**="": Path to the directory containing the build.envd (default: current directory) + +## envs, env, e -**--path, -p**="": Path to the directory containing the build.envd +Manage envd environments -## get, g +### describe, d -Get images, or environments +Show details about environments, including dependencies and port binding -### envs, env, e +**--env, -e**="": Specify the envd environment to use (default: "envd-docs") + +**--format, -f**="": Format of output, could be "json" or "table" (default: "table") + +### list, ls, l List envd environments -#### deps, dep, d +**--format, -f**="": Format of output, could be "json" or "table" (default: "table") -List all dependencies +## images, image -**--env, -e**="": Specify the envd environment to use +Manage envd images -**--private-key, -k**="": Path to the private key (default: /home/runner/.config/envd/id_rsa_envd) +### describe, d -### images, image, i +Show details about image, including dependencies + +**--format, -f**="": Format of output, could be "json" or "table" (default: "table") + +**--image, -i**="": Specify the image to use + +### list, ls, l List envd images -#### deps, dep, d +**--format, -f**="": Format of output, could be "json" or "table" (default: "table") -List all dependencies in the image +### prune -**--image, -i**="": Specify the image to use +Remove unused images + +### remove, r, rm + +Remove an envd image + +**--image, -i**="": Specify the image name to be removed + +**--tag, -t**="": Remove the image with a specific tag (default: dev) + +## init, i + +Automatically generate the build.envd + +**--force, -f**: overwrite the build.envd if existed + +**--lang, -l**="": language usage. Support Python, R, Julia + +**--path, -p**="": Path to the directory containing the build.envd (default: ".") + +## login + +Login to the envd server defined in the current context + +**--password, -p**="": password + +**--username, -u**="": the login name in envd server ## pause, p -pause the envd environment +Pause the envd environment + +**--env, -e**="": Environment name + +## prune + +Clean up the build cache -**--env, -e**="": environment name +**--all**: Include internal caches (oh-my-zsh, vscode extensions and other envd caches) + +**--keep-duration**="": Keep data newer than this limit (default: 0s) + +**--keep-storage**="": Keep data below this limit (in MB) (default: 0) + +**--verbose**: Verbose output + +## exec + +Spawns a command installed into the environment. + +**--command, -c**="": Command defined in build.envd to execute + +**--from, -f**="": Function to execute, format `file:func` (default: "build.envd:build") + +**--name, -n**="": Name of the environment + +**--path, -p**="": Path to the directory containing the build.envd (default: ".") + +**--raw, -r**="": Raw command to execute ## resume, r -resume the envd environment +Resume the envd environment -**--env, -e**="": environment name +**--env, -e**="": Environment name ## up, u -build and run the envd environment +Build and run the envd environment + +**--cap**="": Add capabilities to the environment (e.g. `SYS_PTRACE`) + +**--cpu-set**="": Limit the specific CPUs or cores the environment can use, such as `0-3`, `1,3` + +**--cpus**="": Request CPU resources (number of cores), such as 0.5, 1, 2 + +**--detach**: Detach from the container -**--detach**: detach from the container +**--export-cache, --ec**="": Export the cache (e.g. `type=registry,ref=`). The default `moby-worker` builder doesn't support this unless the docker-ce has enabled the `containerd` image store. You can run `envd context create --name docker --builder docker-container --use` to use this feature. -**--file, -f**="": Name of the build.envd (default: build.envd) +**--force**: Force rebuild and run the container although the previous container is running -**--path, -p**="": Path to the directory containing the build.envd (default: .) +**--from, -f**="": Function to execute, format `file:func` (default: "build.envd:build") -**--private-key, -k**="": Path to the private key (default: /home/runner/.config/envd/id_rsa_envd) +**--gpu-set**="": GPU devices used in this environment, such as `all`, `'"device=1,3"'`, `count=2`(all to pass all GPUs). This will override the `--gpus` -**--public-key, --pubk**="": Path to the public key (default: /home/runner/.config/envd/id_rsa_envd.pub) +**--gpus**="": Number of GPUs used in this environment, this will override the `config.gpu()` (default: 0) + +**--host**="": Assign the host address for the environment SSH access server listening (default: "127.0.0.1") + +**--import-cache, --ic**="": Import the cache (e.g. `type=registry,ref=`) + +**--memory**="": Request Memory, such as 512Mb, 2Gb + +**--name**="": environment name + +**--no-gpu**: Launch the CPU container even if it's a GPU image + +**--path, -p**="": Path to the directory containing the build.envd (default: ".") + +**--platform**="": Specify the target platform for the build output, (for example, windows/amd64, linux/amd64, or darwin/arm64) (default: linux/amd64) + +**--shm-size**="": Configure the shared memory size (megabyte) (default: 2048) **--tag, -t**="": Name and optionally a tag in the 'name:tag' format (default: PROJECT:dev) **--timeout**="": Timeout of container creation (default: 30s) +**--use-proxy, --proxy**: Use HTTPS_PROXY/HTTP_PROXY/NO_PROXY in the build process + **--volume, -v**="": Mount host directory into container +## debug + +Debug commands + +### llb, b + +dump buildkit LLB in human-readable format. + +**--dot**: Output dot format + +**--from, -f**="": Function to execute, format `file:func` (default: "build.envd:build") + +**--path, -p**="": Path to the directory containing the build.envd (default: ".") + ## version, v Print envd version information +**--detail, -d**: Print details about the envd environment + +**--format, -f**="": Format of output, could be "json" or "table" (default: "table") + **--short, -s**: Only print the version number -The documentation is auto-generated from [api package](https://github.com/tensorchord/envd/tree/main/envd/api), please do not edit it manually. +## top + +Show statistics about the containers managed by the environment. + +## reference + +Print envd reference documentation + +**--help, -h**: show help + +**--output**="": Output file, if not specified, print to stderr + +### help, h + +Shows a list of commands or help for one command + +## new, n + +Create a new `build.envd` file from pre-defined templates + +**--force, -f**: Overwrite the build.envd if existed + +**--path, -p**="": Path to the directory of the build.envd (default: ".") + +**--template, -t**="": Template name to use (`envd bootstrap` will add some default templates to '$HOME/.config/envd/templates') diff --git a/docs/zh/api/starlark/lib/lang.md b/docs/zh/api/starlark/lib/lang.md index 16c9268..bd6bab4 100644 --- a/docs/zh/api/starlark/lib/lang.md +++ b/docs/zh/api/starlark/lib/lang.md @@ -12,3 +12,11 @@ Install Golang. - `version` _Optional[str]_ - golang version +## rust + +```python +def rust() +``` + +Install Rust. + diff --git a/docs/zh/api/starlark/v0/config.md b/docs/zh/api/starlark/v0/config.md index 3e51085..68a25aa 100644 --- a/docs/zh/api/starlark/v0/config.md +++ b/docs/zh/api/starlark/v0/config.md @@ -8,6 +8,12 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: +::: warning +v0 syntax is no longer supported from `envd>=v1.0`. Check the [upgrade guide](https://envd.tensorchord.ai/guide/v1.html). + +If you want to use v0 syntax, try `pip install 'envd<1'`. +::: + ## apt\_source ```python @@ -18,17 +24,17 @@ Configure apt sources Example usage: -``` +```python apt_source(source=''' - deb https://mirror.sjtu.edu.cn/ubuntu focal main restricted - deb https://mirror.sjtu.edu.cn/ubuntu focal-updates main restricted - deb https://mirror.sjtu.edu.cn/ubuntu focal universe - deb https://mirror.sjtu.edu.cn/ubuntu focal-updates universe - deb https://mirror.sjtu.edu.cn/ubuntu focal multiverse - deb https://mirror.sjtu.edu.cn/ubuntu focal-updates multiverse - deb https://mirror.sjtu.edu.cn/ubuntu focal-backports main restricted universe multiverse - deb http://archive.canonical.com/ubuntu focal partner - deb https://mirror.sjtu.edu.cn/ubuntu focal-security main restricted universe multiverse + deb https://mirror.sjtu.edu.cn/ubuntu jammy main restricted + deb https://mirror.sjtu.edu.cn/ubuntu jammy-updates main restricted + deb https://mirror.sjtu.edu.cn/ubuntu jammy universe + deb https://mirror.sjtu.edu.cn/ubuntu jammy-updates universe + deb https://mirror.sjtu.edu.cn/ubuntu jammy multiverse + deb https://mirror.sjtu.edu.cn/ubuntu jammy-updates multiverse + deb https://mirror.sjtu.edu.cn/ubuntu jammy-backports main restricted universe multiverse + deb http://archive.canonical.com/ubuntu jammy partner + deb https://mirror.sjtu.edu.cn/ubuntu jammy-security main restricted universe multiverse ''') ``` @@ -52,7 +58,7 @@ Configure jupyter notebook configuration ## pip\_index ```python -def pip_index(url: str, extra_url: str) +def pip_index(url: str, extra_url: str = "", trust: bool = False) ``` Configure pypi index mirror @@ -62,6 +68,7 @@ Configure pypi index mirror - `url` _str_ - PyPI index URL (i.e. https://mirror.sjtu.edu.cn/pypi/web/simple) - `extra_url` _str_ - PyPI extra index URL. `url` and `extra_url` will be treated equally, see https://github.com/pypa/pip/issues/8606 +- `trust` _bool_ - trust the provided index ## conda\_channel @@ -73,7 +80,7 @@ Configure conda channel mirror Example usage: -``` +```python config.conda_channel(channel=''' channels: - defaults @@ -101,7 +108,7 @@ Configure entrypoint for custom base image Example usage: -``` +```python config.entrypoint(["date", "-u"]) ``` @@ -119,7 +126,7 @@ Configure the number of GPUs required Example usage: -``` +```python config.gpu(count=2) ``` @@ -127,6 +134,24 @@ config.gpu(count=2) - `count` _int_ - number of GPUs +## shm\_size + +```python +def shm_size(size: int) +``` + +Configure the shared memory size (megabyte) of docker containers + +Example usage: + +```python +config.shm_size(size=1024) +``` + +**Arguments**: + +- `size` _int_ - the shared memory size (megabyte) of docker containers + ## cran\_mirror ```python @@ -158,7 +183,7 @@ Since Julia 1.5, https://pkg.julialang.org is the default pkg server. def rstudio_server() ``` -Enable the RStudio Server (only work for `base(os="ubuntu20.04", language="r")`) +Enable the RStudio Server (only work for `base(os="ubuntu22.04", language="r")`) ## repo diff --git a/docs/zh/api/starlark/v0/global.md b/docs/zh/api/starlark/v0/global.md index 22dfbc1..82309a3 100644 --- a/docs/zh/api/starlark/v0/global.md +++ b/docs/zh/api/starlark/v0/global.md @@ -8,6 +8,12 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: +::: warning +v0 syntax is no longer supported from `envd>=v1.0`. Check the [upgrade guide](https://envd.tensorchord.ai/guide/v1.html). + +If you want to use v0 syntax, try `pip install 'envd<1'`. +::: + ## base ```python @@ -18,9 +24,9 @@ Set base image **Arguments**: -- `os` _str_ - The operating system (i.e. `ubuntu20.04`) -- `language` _str_ - The programing language dependency (i.e. `python3.8`) -- `image` _Optional[str]_ - Custom image (i.e. `python:3.9-slim`) +- `os` _str_ - The operating system (i.e. `ubuntu22.04`) +- `language` _str_ - The programming language dependency (i.e. `python3.8`) +- `image` _Optional[str]_ - Custom image (i.e. `python:3.11-slim`) ## shell @@ -37,21 +43,21 @@ Interactive shell ## run ```python -def run(commands: str, mount_host: bool = False) +def run(commands: List[str], mount_host: bool = False) ``` Execute command **Arguments**: -- `commands` _str_ - command to run during the building process +- `commands` _List[str]_ - command to run during the building process - `mount_host` _bool_ - mount the host directory. Default is False. Enabling this will disable the build cache for this operation. **Example**: -``` +```python run(commands=["conda install -y -c conda-forge exa"]) ``` @@ -72,7 +78,7 @@ Setup git config - `editor` _optional, str_ - Editor for git operations Example usage: -``` +```python git_config(name="My Name", email="my@email.com", editor="vim") ``` @@ -85,18 +91,18 @@ def include(git: str) Import from another git repo This will pull the git repo and execute all the `envd` files. The return value will be a module -contains all the variables/functions defined (expect those has `_` prefix). +contains all the variables/functions defined (except the ones with `_` prefix). **Arguments**: - `git` _str_ - git URL Example usage: -``` +```python envd = include("https://github.com/tensorchord/envdlib") def build(): - base(os="ubuntu20.04", language="python") + base(os="ubuntu22.04", language="python") envd.tensorboard(8000) ``` diff --git a/docs/zh/api/starlark/v0/install.md b/docs/zh/api/starlark/v0/install.md index b55f5f5..57cd6b5 100644 --- a/docs/zh/api/starlark/v0/install.md +++ b/docs/zh/api/starlark/v0/install.md @@ -8,17 +8,23 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: +::: warning +v0 syntax is no longer supported from `envd>=v1.0`. Check the [upgrade guide](https://envd.tensorchord.ai/guide/v1.html). + +If you want to use v0 syntax, try `pip install 'envd<1'`. +::: + ## apt\_packages ```python def apt_packages(name: List[str]) ``` -Install package by system-level package manager (apt on Ubuntu) +Install package using the system package manager (apt on Ubuntu) **Arguments**: -- `name` _str_ - apt package name list +- `name` _List[str]_ - apt package name list ## python\_packages diff --git a/docs/zh/api/starlark/v0/io.md b/docs/zh/api/starlark/v0/io.md index 460e847..10141a0 100644 --- a/docs/zh/api/starlark/v0/io.md +++ b/docs/zh/api/starlark/v0/io.md @@ -8,6 +8,12 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: +::: warning +v0 syntax is no longer supported from `envd>=v1.0`. Check the [upgrade guide](https://envd.tensorchord.ai/guide/v1.html). + +If you want to use v0 syntax, try `pip install 'envd<1'`. +::: + ## copy ```python diff --git a/docs/zh/api/starlark/v0/runtime.md b/docs/zh/api/starlark/v0/runtime.md index 5430cf2..1605d9c 100644 --- a/docs/zh/api/starlark/v0/runtime.md +++ b/docs/zh/api/starlark/v0/runtime.md @@ -8,6 +8,12 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: +::: warning +v0 syntax is no longer supported from `envd>=v1.0`. Check the [upgrade guide](https://envd.tensorchord.ai/guide/v1.html). + +If you want to use v0 syntax, try `pip install 'envd<1'`. +::: + ## command ```python @@ -21,7 +27,7 @@ Execute commands during runtime - `commands` _Dict[str, str]_ - map name to command, similar to Makefile Example usage: -``` +```python runtime.command(commands={ "train": "python train.py --epoch 20 --notify me@tensorchord.ai", "run": "python server.py --batch 1 --host 0.0.0.0 --port 8000", @@ -33,7 +39,7 @@ runtime.command(commands={ ## expose ```python -def expose(envd_port: str, host_port: Optional[str], service: Optional[str], +def expose(envd_port: int, host_port: Optional[int], service: Optional[str], listen_addr: Optional[str]) ``` @@ -42,8 +48,8 @@ Proposal: https://github.com/tensorchord/envd/pull/780 **Arguments**: -- `envd_port` _str_ - port in `envd` container -- `host_port` _Optional[str]_ - port in the host, if not provided or +- `envd_port` _int_ - port in `envd` container +- `host_port` _Optional[int]_ - port in the host, if not provided or `host_port=0`, `envd` will randomly choose a free port - `service` _Optional[str]_ - service name - `listen_addr` _Optional[str]_ - address to listen on @@ -57,14 +63,17 @@ def daemon(commands: List[List[str]]) Run daemon processes in the container Proposal: https://github.com/tensorchord/envd/pull/769 -It's better to redirect the logs to local files for debug purposes. +It's better to redirect the logs to local files for debugging purposes. + +You can find the generated horust config files under `/etc/horust/services` +and log files under `/var/log/horust` in the container. **Arguments**: - `commands` _List[List[str]]_ - run multiple commands in the background Example usage: -``` +```python runtime.daemon(commands=[ ["jupyter-lab", "--port", "8080"], ["python3", "serving.py", ">>serving.log", "2>&1"], @@ -85,7 +94,7 @@ Add runtime environments - `extra_path` _List[str]_ - additional PATH Example usage: -``` +```python runtime.environ(env={"ENVD_MODE": "DEV"}, extra_path=["/usr/bin/go/bin"]) ``` @@ -110,6 +119,9 @@ def init(commands: List[str]) Commands to be executed when start the container +You can find the generated horust config files under `/etc/horust/services` +and log files under `/var/log/horust` in the container. + **Arguments**: - `commands` _List[str]_ - list of commands diff --git a/docs/zh/api/starlark/v1/config.md b/docs/zh/api/starlark/v1/config.md index 4ba4e20..12000d6 100644 --- a/docs/zh/api/starlark/v1/config.md +++ b/docs/zh/api/starlark/v1/config.md @@ -8,12 +8,6 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: -::: warning -Enable v1 by adding `# syntax=v1` to the 1st line of your envd file. - -v1 is experimental and may change in the future. Make sure to freeze the envd version for online CI/CD. -::: - ## apt\_source ```python @@ -24,17 +18,17 @@ Configure apt sources Example usage: -``` +```python apt_source(source=''' - deb https://mirror.sjtu.edu.cn/ubuntu focal main restricted - deb https://mirror.sjtu.edu.cn/ubuntu focal-updates main restricted - deb https://mirror.sjtu.edu.cn/ubuntu focal universe - deb https://mirror.sjtu.edu.cn/ubuntu focal-updates universe - deb https://mirror.sjtu.edu.cn/ubuntu focal multiverse - deb https://mirror.sjtu.edu.cn/ubuntu focal-updates multiverse - deb https://mirror.sjtu.edu.cn/ubuntu focal-backports main restricted universe multiverse - deb http://archive.canonical.com/ubuntu focal partner - deb https://mirror.sjtu.edu.cn/ubuntu focal-security main restricted universe multiverse + deb https://mirror.sjtu.edu.cn/ubuntu jammy main restricted + deb https://mirror.sjtu.edu.cn/ubuntu jammy-updates main restricted + deb https://mirror.sjtu.edu.cn/ubuntu jammy universe + deb https://mirror.sjtu.edu.cn/ubuntu jammy-updates universe + deb https://mirror.sjtu.edu.cn/ubuntu jammy multiverse + deb https://mirror.sjtu.edu.cn/ubuntu jammy-updates multiverse + deb https://mirror.sjtu.edu.cn/ubuntu jammy-backports main restricted universe multiverse + deb http://archive.canonical.com/ubuntu jammy partner + deb https://mirror.sjtu.edu.cn/ubuntu jammy-security main restricted universe multiverse ''') ``` @@ -58,7 +52,7 @@ Configure jupyter notebook configuration ## pip\_index ```python -def pip_index(url: str, extra_url: str) +def pip_index(url: str, extra_url: str = "", trust: bool = False) ``` Configure pypi index mirror @@ -68,6 +62,7 @@ Configure pypi index mirror - `url` _str_ - PyPI index URL (i.e. https://mirror.sjtu.edu.cn/pypi/web/simple) - `extra_url` _str_ - PyPI extra index URL. `url` and `extra_url` will be treated equally, see https://github.com/pypa/pip/issues/8606 +- `trust` _bool_ - trust the provided index ## conda\_channel @@ -79,7 +74,7 @@ Configure conda channel mirror Example usage: -``` +```python config.conda_channel(channel=''' channels: - defaults @@ -107,7 +102,7 @@ Configure entrypoint for custom base image Example usage: -``` +```python config.entrypoint(["date", "-u"]) ``` @@ -125,7 +120,7 @@ Configure the number of GPUs required Example usage: -``` +```python config.gpu(count=2) ``` @@ -133,6 +128,24 @@ config.gpu(count=2) - `count` _int_ - number of GPUs +## shm\_size + +```python +def shm_size(size: int) +``` + +Configure the shared memory size (megabyte) of docker containers + +Example usage: + +```python +config.shm_size(size=1024) +``` + +**Arguments**: + +- `size` _int_ - the shared memory size (megabyte) of docker containers + ## cran\_mirror ```python @@ -164,7 +177,7 @@ Since Julia 1.5, https://pkg.julialang.org is the default pkg server. def rstudio_server() ``` -Enable the RStudio Server (only work for `base(os="ubuntu20.04", language="r")`) +Enable the RStudio Server (only work for `base(os="ubuntu22.04", language="r")`) ## repo @@ -179,3 +192,18 @@ Setup repo related information. Will save to the image labels. - `url` _str_ - repo URL - `description` _str_ - repo description +## owner + +```python +def owner(uid: int, gid: int) +``` + +Configure uid:gid as the environment owner. +This can also be achieved by using flag `envd --owner uid:gid build` or environment +variable `ENVD_BUILD_OWNER=uid:gid envd build` + +**Arguments**: + +- `uid` _int_ - UID +- `gid` _int_ - GID + diff --git a/docs/zh/api/starlark/v1/global.md b/docs/zh/api/starlark/v1/global.md index dbf74d3..285eb35 100644 --- a/docs/zh/api/starlark/v1/global.md +++ b/docs/zh/api/starlark/v1/global.md @@ -9,56 +9,50 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: -::: warning -Enable v1 by adding `# syntax=v1` to the 1st line of your envd file. - -v1 is experimental and may change in the future. Make sure to freeze the envd version for online CI/CD. -::: - ## base ```python -def base(image: str = "ubuntu:20.04", dev: bool = False) +def base(image: str = "ubuntu:22.04", dev: bool = False) ``` Set up the base env. **Arguments**: -- `image` _str_ - docker image, can be any Debian-based images +- `image` _str_ - docker image, can be any Debian-based image - `dev` _bool_ - enabling the dev env will add lots of development related libraries like envd-sshd, vim, git, shell prompt, vscode extensions, etc. ## shell ```python -def shell(name: str = "base") +def shell(name: str = "bash") ``` Interactive shell **Arguments**: -- `name` _str_ - shell name (i.e. `zsh`, `bash`) +- `name` _str_ - shell name (i.e. `zsh`, `bash`, `fish`) ## run ```python -def run(commands: str, mount_host: bool = False) +def run(commands: List[str], mount_host: bool = False) ``` Execute command **Arguments**: -- `commands` _str_ - command to run during the building process +- `commands` _List[str]_ - command to run during the building process - `mount_host` _bool_ - mount the host directory. Default is False. Enabling this will disable the build cache for this operation. **Example**: -``` +```python run(commands=["conda install -y -c conda-forge exa"]) ``` @@ -70,7 +64,7 @@ def git_config(name: Optional[str] = None, editor: Optional[str] = None) ``` -Setup git config +Setup git config. **Arguments**: @@ -79,7 +73,7 @@ Setup git config - `editor` _str_ - Editor for git operations Example usage: -``` +```python git_config(name="My Name", email="my@email.com", editor="vim") ``` @@ -92,18 +86,18 @@ def include(git: str) Import from another git repo This will pull the git repo and execute all the `envd` files. The return value will be a module -contains all the variables/functions defined (expect those has `_` prefix). +contains all the variables/functions defined (except the ones with `_` prefix). **Arguments**: - `git` _str_ - git URL Example usage: -``` +```python envd = include("https://github.com/tensorchord/envdlib") def build(): - base(os="ubuntu20.04", language="python") + base(os="ubuntu22.04", language="python") envd.tensorboard(host_port=8000) ``` diff --git a/docs/zh/api/starlark/v1/install.md b/docs/zh/api/starlark/v1/install.md index bb4c802..8906a96 100644 --- a/docs/zh/api/starlark/v1/install.md +++ b/docs/zh/api/starlark/v1/install.md @@ -8,16 +8,10 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: -::: warning -Enable v1 by adding `# syntax=v1` to the 1st line of your envd file. - -v1 is experimental and may change in the future. Make sure to freeze the envd version for online CI/CD. -::: - ## python ```python -def python(version: str = "3.9") +def python(version: str = "3.11") ``` Install python. @@ -41,6 +35,45 @@ Install MiniConda or MicroMamba. - `use_mamba` _bool_ - use mamba instead of conda +## pixi + +```python +def pixi(use_pixi_mirror: bool = False, pypi_index: Optional[str] = None) +``` + +Install Pixi (https://github.com/prefix-dev/pixi). + +`pixi` is an alternative to `conda` that is written in Rust and provides faster +dependency resolution and installation. It also simplify the project management. + +This doesn't support installing Python packages through `install.python_packages` +because that part should be managed by `pixi`. You can run `pixi shell` in the +`envd` environment to sync all the dependencies. + +**Arguments**: + +- `use_pixi_mirror` _bool_ - use pixi mirror +- `pypi_index` _Optional[str]_ - customize pypi index url + +## uv + +```python +def uv(python_version: str = "3.11") +``` + +Install UV (an extremely fast Python package and project manager). + +`uv` is much faster than `conda`. Choose this one instead of `conda` if you don't +need any machine learning packages. + +This doesn't support installing Python packages through `install.python_packages` +because that part should be managed by `uv`. You can run `uv sync` in the `envd` +environment to install all the dependencies. + +**Arguments**: + +- `python_version` _str_ - install this Python version through UV + ## r\_lang ```python @@ -49,8 +82,6 @@ def r_lang() Install R Lang. -Not implemented yet. Please use v0 if you need R. - ## julia ```python @@ -59,93 +90,139 @@ def julia() Install Julia. -Not implemented yet. Please use v0 if you need Julia. +## rust + +```python +def rust(version: Optional[str] = None) +``` + +Install Rust programming language. + +**Arguments**: + +- `version` _Optional[str]_ - Rust version, such as '1.72.0'. + If not specified, the latest stable version will be installed. + +## go + +```python +def go(version: Optional[str] = None) +``` + +Install Go programming language. + +**Arguments**: + +- `version` _Optional[str]_ - Go version, such as '1.25.3'. + +## nodejs + +```python +def nodejs(version: Optional[str] = None) +``` + +Install NodeJS programming language. + +**Arguments**: + +- `version` _Optional[str]_ - NodeJS version, such as '25.1.0'. + +## codex + +```python +def codex(version: Optional[str] = None) +``` + +Install Codex agent. + +**Arguments**: + +- `version` _Optional[str]_ - Codex GitHub release tag, such as 'rust-v0.98.0'. + If None is provided, envd will attempt to use the latest tag. + If the latest tag cannot be resolved (due to network or rate limit), + a built-in default version will be used. ## apt\_packages ```python -def apt_packages(name: List[str] = []) +def apt_packages(name: Sequence[str] = ()) ``` -Install package by system-level package manager (apt on Ubuntu). +Install package using the system package manager (apt on Ubuntu). **Arguments**: -- `name` _str_ - apt package name list +- `name` _Sequence[str]_ - apt package name list ## python\_packages ```python -def python_packages(name: List[str] = [], +def python_packages(name: Sequence[str] = (), requirements: str = "", - local_wheels: List[str] = []) + local_wheels: Sequence[str] = ()) ``` Install python package by pip. **Arguments**: -- `name` _List[str]_ - package name list +- `name` _Sequence[str]_ - package name list - `requirements` _str_ - requirements file path -- `local_wheels` _List[str]_ - local wheels +- `local_wheels` _Sequence[str]_ - local wheels (wheel files should be placed under the current directory) ## conda\_packages ```python -def conda_packages(name: List[str] = [], - channel: List[str] = [], - env_file: str = "") +def conda_packages( + name: Sequence[str] = (), + channel: Sequence[str] = (), env_file: str = "") ``` Install python package by Conda **Arguments**: -- `name` _List[str]_ - List of package names with optional version assignment, +- `name` _Sequence[str]_ - List of package names with optional version assignment, such as ['pytorch', 'tensorflow==1.13.0'] -- `channel` _List[str]_ - additional channels +- `channel` _Sequence[str]_ - additional channels - `env_file` _str_ - conda env file path ## r\_packages ```python -def r_packages(name: List[str]) +def r_packages(name: Sequence[str]) ``` Install R packages by R package manager. -Not implemented yet. Please use v0 if you need R. - **Arguments**: -- `name` _List[str]_ - package name list +- `name` _Sequence[str]_ - package name list ## julia\_packages ```python -def julia_packages(name: List[str]) +def julia_packages(name: Sequence[str]) ``` Install Julia packages. -Not implemented yet. Please use v0 if you need Julia. - **Arguments**: - name (List(str)): List of Julia packages +- `name` _Sequence[str]_ - List of Julia packages ## vscode\_extensions ```python -def vscode_extensions(name: List[str]) +def vscode_extensions(name: Sequence[str]) ``` Install VS Code extensions **Arguments**: -- `name` _List[str]_ - extension names, such as ['ms-python.python'] +- `name` _Sequence[str]_ - extension names, such as ['ms-python.python'] ## cuda @@ -153,10 +230,19 @@ Install VS Code extensions def cuda(version: str, cudnn: Optional[str] = "8") ``` -Install CUDA dependency +Replace the base image with a `nvidia/cuda` image. + +This will replace the default base image to an `nvidia/cuda` image. You can +also use a CUDA base image directly like +`base(image="nvidia/cuda:12.2.0-devel-ubuntu22.04", dev=True)`. **Arguments**: - `version` _str_ - CUDA version, such as '11.6.2' - `cudnn` _optional, str_ - CUDNN version, such as '8' + + Example usage: +```python +install.cuda(version="11.6.2", cudnn="8") +``` diff --git a/docs/zh/api/starlark/v1/io.md b/docs/zh/api/starlark/v1/io.md index 88f97f4..ac8c990 100644 --- a/docs/zh/api/starlark/v1/io.md +++ b/docs/zh/api/starlark/v1/io.md @@ -8,24 +8,29 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: -::: warning -Enable v1 by adding `# syntax=v1` to the 1st line of your envd file. - -v1 is experimental and may change in the future. Make sure to freeze the envd version for online CI/CD. -::: - ## copy ```python -def copy(host_path: str, envd_path: str) +def copy(source: str, target: str, image: Optional[str]) ``` Copy from host path to container path (build time) **Arguments**: -- `host_path` _str_ - source path in the host machine -- `envd_path` _str_ - destination path in the envd container +- `source` _str_ - source path in the host machine or in the ``image`` +- `target` _str_ - destination path in the envd container +- `image(Optional[str])` - image name, if not specified, will use the host + + +**Examples**: + +```python +# copy from host to container +io.copy(source='main.py', target='/home/envd/') +# copy from image to container +io.copy(source='/bin/micromamba', target='/usr/local/bin/micromamba', image='mambaorg/micromamba:1.0.0') +``` ## http diff --git a/docs/zh/api/starlark/v1/runtime.md b/docs/zh/api/starlark/v1/runtime.md index 4f2873a..6d6585b 100644 --- a/docs/zh/api/starlark/v1/runtime.md +++ b/docs/zh/api/starlark/v1/runtime.md @@ -8,12 +8,6 @@ in [tensorchord/envd](https://github.com/tensorchord/envd/tree/main/envd/api) re Please update the python file there instead of directly editing file inside envd-docs repo. ::: -::: warning -Enable v1 by adding `# syntax=v1` to the 1st line of your envd file. - -v1 is experimental and may change in the future. Make sure to freeze the envd version for online CI/CD. -::: - ## command ```python @@ -27,7 +21,7 @@ Execute commands during runtime - `commands` _Dict[str, str]_ - map name to command, similar to Makefile Example usage: -``` +```python runtime.command(commands={ "train": "python train.py --epoch 20 --notify me@tensorchord.ai", "run": "python server.py --batch 1 --host 0.0.0.0 --port 8000", @@ -39,7 +33,7 @@ runtime.command(commands={ ## expose ```python -def expose(envd_port: str, host_port: Optional[str], service: Optional[str], +def expose(envd_port: int, host_port: Optional[int], service: Optional[str], listen_addr: Optional[str]) ``` @@ -48,8 +42,8 @@ Proposal: https://github.com/tensorchord/envd/pull/780 **Arguments**: -- `envd_port` _str_ - port in `envd` container -- `host_port` _Optional[str]_ - port in the host, if not provided or +- `envd_port` _int_ - port in `envd` container +- `host_port` _Optional[int]_ - port in the host, if not provided or `host_port=0`, `envd` will randomly choose a free port - `service` _Optional[str]_ - service name - `listen_addr` _Optional[str]_ - address to listen on @@ -63,14 +57,17 @@ def daemon(commands: List[List[str]]) Run daemon processes in the container Proposal: https://github.com/tensorchord/envd/pull/769 -It's better to redirect the logs to local files for debug purposes. +It's better to redirect the logs to local files for debugging purposes. + +You can find the generated horust config files under `/etc/horust/services` +and log files under `/var/log/horust` in the container. **Arguments**: - `commands` _List[List[str]]_ - run multiple commands in the background Example usage: -``` +```python runtime.daemon(commands=[ ["jupyter-lab", "--port", "8080"], ["python3", "serving.py", ">>serving.log", "2>&1"], @@ -91,7 +88,7 @@ Add runtime environments - `extra_path` _List[str]_ - additional PATH Example usage: -``` +```python runtime.environ(env={"ENVD_MODE": "DEV"}, extra_path=["/usr/bin/go/bin"]) ``` @@ -116,6 +113,9 @@ def init(commands: List[str]) Commands to be executed when start the container +You can find the generated horust config files under `/etc/horust/services` +and log files under `/var/log/horust` in the container. + **Arguments**: - `commands` _List[str]_ - list of commands