diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/fetch-reference.sh | 200 |
1 files changed, 200 insertions, 0 deletions
diff --git a/bin/fetch-reference.sh b/bin/fetch-reference.sh new file mode 100755 index 0000000..2837d46 --- /dev/null +++ b/bin/fetch-reference.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# +# fetch-reference.sh +# +# Clones every repository and downloads every external HTML/text document +# referenced by HTML_RENDERING_ENGINES.md, RESEARCH.md, and X11_COMPOSITING_WMS.md +# into ./reference/repos and ./reference/docs respectively. +# +# - All git clones use --depth 1 --single-branch (shallow, single branch only). +# - All operations are strictly sequential: one git clone at a time, one curl +# at a time. No backgrounding, no parallelism. +# - Safe to re-run: existing repos are refreshed with a shallow fetch so they +# stay at depth 1; existing docs are re-downloaded. + +set -u + +# Resolve repo root (script lives in <repo>/bin/) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." &>/dev/null && pwd)" + +REPOS_DIR="${REPO_ROOT}/reference/repos" +DOCS_DIR="${REPO_ROOT}/reference/docs" + +mkdir -p "${REPOS_DIR}" "${DOCS_DIR}" + +# --------------------------------------------------------------------------- +# Repositories to clone +# --------------------------------------------------------------------------- +REPOS=( + # HTML/CSS rendering engines + "https://github.com/chromiumembedded/cef.git" + "https://github.com/WebKit/WebKit.git" + "https://github.com/VehicularEpic/webkit2-osr.git" + "https://github.com/WebPlatformForEmbedded/WPEWebKit.git" + "https://github.com/WebPlatformForEmbedded/libwpe.git" + "https://github.com/servo/servo.git" + "https://github.com/paulrouget/servo-embedding-example.git" + "https://github.com/mikke89/RmlUi.git" + "https://github.com/NimbusFox/raylib-RmlUi.git" + "https://github.com/litehtml/litehtml.git" + "https://github.com/lexbor/lexbor.git" + "https://github.com/lexborisov/Modest.git" + "https://github.com/webview/webview.git" + + # Reference compositor / WM implementations from RESEARCH.md + "https://github.com/obiwac/x-compositing-wm.git" + "https://gitlab.freedesktop.org/xorg/app/xcompmgr.git" + "https://github.com/yshui/picom.git" + "https://gitlab.com/compiz/compiz-core.git" + "https://github.com/mackstann/tinywm.git" + "https://github.com/jichu4n/basic_wm.git" + "https://github.com/jaelpark/chamferwm.git" + "https://github.com/deurzen/wzrd.git" + + # Desktop-environment compositing WMs + "https://github.com/KDE/kwin.git" + "https://gitlab.gnome.org/GNOME/mutter.git" + "https://gitlab.xfce.org/xfce/xfwm4.git" + "https://github.com/linuxmint/muffin.git" + "https://github.com/mate-desktop/marco.git" + "https://github.com/elementary/gala.git" + "https://github.com/Enlightenment/enlightenment.git" + "https://github.com/JeffHoogland/moksha.git" + + # Standalone compositing WMs + "https://github.com/compiz-reloaded/compiz.git" + "https://github.com/noodlylight/fusilli.git" + "https://github.com/ammen99/fire.git" + "https://github.com/gschwind/page.git" + "https://github.com/michaelshiel/eink-wm.git" + + # Picom forks & siblings + "https://github.com/pijulius/picom.git" + "https://github.com/jonaburg/picom.git" + "https://github.com/ibhagwan/picom.git" + "https://github.com/chjj/compton.git" + "https://github.com/comick/finalcm.git" +) + +# --------------------------------------------------------------------------- +# Documents / web pages to curl +# --------------------------------------------------------------------------- +DOCS=( + # Project websites referenced by HTML_RENDERING_ENGINES.md + "https://webkitgtk.org/" + "https://wpewebkit.org/" + "https://servo.org/" + "https://mikke89.github.io/RmlUiDoc/" + "http://www.litehtml.com/" + "https://lexbor.com/" + + # Specifications & articles referenced by RESEARCH.md + "https://www.x.org/releases/X11R7.5/doc/compositeproto/compositeproto.txt" + "https://www.x.org/releases/current/doc/man/man3/Xcomposite.3.xhtml" + "https://www.x.org/archive/X11R7.5/doc/damageproto/damageproto.txt" + "https://registry.khronos.org/OpenGL/extensions/EXT/GLX_EXT_texture_from_pixmap.txt" + "https://specifications.freedesktop.org/wm-spec/1.5/index.html" + "https://jichu4n.com/posts/how-x-window-managers-work-and-how-to-write-one-part-i/" + "https://jichu4n.com/posts/how-x-window-managers-work-and-how-to-write-one-part-ii/" + "https://wingolog.org/archives/2008/07/26/so-you-want-to-build-a-compositor" + "https://magcius.github.io/xplain/article/composite.html" + "https://deepwiki.com/yshui/picom" + "https://download.nvidia.com/XFree86/Linux-x86_64/396.51/README/xcompositeextension.html" + "https://registry.khronos.org/OpenGL/extensions/SGI/GLX_SGI_swap_control.txt" + "https://www.x.org/releases/X11R7.7/doc/fixesproto/fixesproto.txt" +) + +# --------------------------------------------------------------------------- +# Clone repos — STRICTLY SEQUENTIAL, one at a time, all shallow (depth 1). +# --------------------------------------------------------------------------- +echo "==> Cloning ${#REPOS[@]} repositories into ${REPOS_DIR} (sequential, depth 1)" +repo_ok=0 +repo_fail=0 +for url in "${REPOS[@]}"; do + # Derive a directory name: "<owner>__<repo>" to avoid collisions + # (e.g. yshui/picom vs pijulius/picom). + stripped="${url%.git}" + repo_name="${stripped##*/}" + owner_path="${stripped%/*}" + owner="${owner_path##*/}" + dest_name="${owner}__${repo_name}" + dest="${REPOS_DIR}/${dest_name}" + + if [[ -d "${dest}/.git" ]]; then + # Re-run path: refresh via a shallow fetch so the clone stays at depth 1. + echo "--> [${dest_name}] already present, shallow-refreshing" + if git -C "${dest}" fetch --depth 1 origin \ + && git -C "${dest}" reset --hard FETCH_HEAD; then + repo_ok=$((repo_ok + 1)) + else + echo " !! refresh failed for ${url}" + repo_fail=$((repo_fail + 1)) + fi + else + echo "--> [${dest_name}] cloning ${url}" + if git clone --depth 1 --single-branch "${url}" "${dest}"; then + repo_ok=$((repo_ok + 1)) + else + echo " !! clone failed for ${url}" + repo_fail=$((repo_fail + 1)) + fi + fi + # Wait a moment so the previous clone has fully released resources before + # the next one starts. (Belt-and-suspenders for "one at a time".) + wait +done + +# --------------------------------------------------------------------------- +# Download docs — STRICTLY SEQUENTIAL, one curl at a time. +# --------------------------------------------------------------------------- +echo +echo "==> Downloading ${#DOCS[@]} documents into ${DOCS_DIR} (sequential)" +doc_ok=0 +doc_fail=0 +for url in "${DOCS[@]}"; do + # Build a filesystem-safe filename from the URL. + # Strip scheme, replace path separators, and ensure a sensible extension. + no_scheme="${url#*://}" + safe="${no_scheme//\//_}" + safe="${safe//\?/_}" + safe="${safe//&/_}" + safe="${safe//=/_}" + safe="${safe//:/_}" + # Trim trailing underscores (from URLs ending with "/"). + safe="${safe%_}" + # If there's no file extension, append .html so browsers/editors know. + base="${safe##*/}" + if [[ "${base}" != *.* ]]; then + safe="${safe}.html" + fi + dest="${DOCS_DIR}/${safe}" + + echo "--> [${safe}] ${url}" + if curl --fail --location --silent --show-error \ + --user-agent "Mozilla/5.0 (fetch-reference.sh)" \ + --output "${dest}" \ + "${url}"; then + doc_ok=$((doc_ok + 1)) + else + echo " !! download failed for ${url}" + rm -f "${dest}" + doc_fail=$((doc_fail + 1)) + fi + # Belt-and-suspenders: make sure the previous curl has fully exited before + # the next one is launched. + wait +done + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo +echo "==> Done." +echo " repos: ${repo_ok} ok, ${repo_fail} failed (in ${REPOS_DIR})" +echo " docs: ${doc_ok} ok, ${doc_fail} failed (in ${DOCS_DIR})" + +# Non-zero exit if anything failed, so callers/CI can detect it. +if (( repo_fail > 0 || doc_fail > 0 )); then + exit 1 +fi |