From 991837d4b2512788e14caf4f5d4d414f2fcbe035 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Mon, 13 Apr 2026 10:33:45 +0300 Subject: [PATCH 01/16] chore: switch provisioning SSH key to local-db-op --- .github/workflows/deploy.yaml | 22 ++++++++++++++++++++++ .github/workflows/provision-host.yaml | 22 +++++++++++++++++----- infra/variables.tf | 2 +- keys.nix | 12 ++++-------- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index f060ba1..84184f0 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -3,6 +3,16 @@ name: Deploy on: workflow_dispatch: inputs: + target_ref: + description: Git ref to deploy from + required: true + default: main + type: string + allow_non_main: + description: Allow deployment from a non-main ref + required: true + default: false + type: boolean settings_url: description: Settings YAML URL to deploy required: true @@ -24,16 +34,27 @@ env: SPACES_REGION: ${{ secrets.SPACES_REGION }} SPACES_BUCKET: ${{ secrets.SPACES_BUCKET }} SPACES_ENDPOINT: ${{ secrets.SPACES_ENDPOINT }} + TARGET_REF: ${{ inputs.target_ref }} jobs: deploy: runs-on: ubuntu-latest steps: + - name: Validate target ref + shell: bash + run: | + set -euo pipefail + if [ "$TARGET_REF" != "main" ] && [ "${{ inputs.allow_non_main }}" != "true" ]; then + echo "Set allow_non_main=true to deploy from a non-main ref." >&2 + exit 1 + fi + - name: Free disk space uses: jlumbroso/free-disk-space@v1.3.1 - uses: actions/checkout@v4 with: + ref: ${{ env.TARGET_REF }} submodules: recursive fetch-depth: 0 @@ -124,6 +145,7 @@ jobs: { echo "## Local DB Remote Deployment" echo + echo "- Target ref: $TARGET_REF" echo "- Host IP: $HOST_IP" echo "- Settings URL: ${{ inputs.settings_url }}" echo "- Timer: local-db-sync.timer" diff --git a/.github/workflows/provision-host.yaml b/.github/workflows/provision-host.yaml index 12afb45..fe5c073 100644 --- a/.github/workflows/provision-host.yaml +++ b/.github/workflows/provision-host.yaml @@ -3,6 +3,16 @@ name: Provision Host on: workflow_dispatch: inputs: + target_ref: + description: Git ref to provision from + required: true + default: main + type: string + allow_non_main: + description: Allow provisioning from a non-main ref + required: true + default: false + type: boolean bootstrap_nixos: description: Bootstrap NixOS onto the provisioned droplet required: true @@ -22,13 +32,14 @@ jobs: runs-on: ubuntu-latest env: TF_VAR_do_token: ${{ secrets.DO_TOKEN }} + TARGET_REF: ${{ inputs.target_ref }} steps: - - name: Require main branch + - name: Validate target ref shell: bash run: | set -euo pipefail - if [ "${GITHUB_REF_NAME}" != "main" ]; then - echo "This workflow must be run from the main branch." >&2 + if [ "$TARGET_REF" != "main" ] && [ "${{ inputs.allow_non_main }}" != "true" ]; then + echo "Set allow_non_main=true to provision from a non-main ref." >&2 exit 1 fi @@ -37,7 +48,7 @@ jobs: - uses: actions/checkout@v4 with: - ref: main + ref: ${{ env.TARGET_REF }} submodules: recursive fetch-depth: 0 @@ -76,7 +87,7 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add infra/terraform.tfstate.age infra/terraform.tfvars.age git commit -m "chore: update local-db remote terraform state" - git push origin HEAD:main + git push origin HEAD:"$TARGET_REF" - name: Resolve host IP id: resolve_ip @@ -100,6 +111,7 @@ jobs: { echo "## Host Provisioning" echo + echo "- Target ref: $TARGET_REF" echo "- Host IP: ${{ steps.resolve_ip.outputs.host_ip }}" echo "- Bootstrap NixOS: ${{ inputs.bootstrap_nixos }}" echo "- Terraform apply ran successfully" diff --git a/infra/variables.tf b/infra/variables.tf index 0559c6c..c662e86 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -7,7 +7,7 @@ variable "do_token" { variable "ssh_key_name" { description = "Name of the SSH key in DigitalOcean to add to the droplet" type = string - default = "st0x-op" + default = "github_do" } variable "region" { diff --git a/keys.nix b/keys.nix index 07833ab..bba7014 100644 --- a/keys.nix +++ b/keys.nix @@ -1,17 +1,13 @@ rec { keys = { - st0x-op = - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPZ56nOYbGDd0ZfbqxeY7AbvaQGQrHnlC80ccpRGpCoj"; - ci = - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTd2zKSwHgWegi290EiK5nYp1Wp4+x2fDYqFxbd0WLN"; + github_do = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIMYFMrh1LLoYCZhir9LA8FpbwKOpWrZ3gpZ5VvFT5Bu github_do"; arda = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAyTREGZCOzMsl7N9dp1saN/t7DCs7YesusVUKApMJ78"; - sid = - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPl3/6RlR6Rvz0ZRyZukzFtt4zUYNz5OVuTsajJl7V3n"; }; roles = with keys; { - infra = [ st0x-op ci sid ]; - ssh = [ st0x-op ci arda sid ]; + infra = [ github_do arda ]; + ssh = [ github_do arda ]; }; } From 3f38d0021174c18dbab60f6ddd7e2b373fdc3ecd Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Tue, 14 Apr 2026 08:28:14 +0300 Subject: [PATCH 02/16] Fix shellcheck for raindex COMMIT_SHA export --- .../create-empty-local-db-manifest.yaml | 24 +++++++++++++++--- .github/workflows/deploy.yaml | 21 ++++++++++++--- config/runtime.env.age | Bin 0 -> 622 bytes config/runtime.env.example | 4 +++ config/secrets.nix | 3 +++ flake.nix | 9 +++++-- infra/terraform.tfvars.age | Bin 0 -> 486 bytes infra/terraform.tfvars.example | 5 ++++ infra/variables.tf | 2 +- lib/raindex | 2 +- 10 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 config/runtime.env.age create mode 100644 config/runtime.env.example create mode 100644 config/secrets.nix create mode 100644 infra/terraform.tfvars.age diff --git a/.github/workflows/create-empty-local-db-manifest.yaml b/.github/workflows/create-empty-local-db-manifest.yaml index 8bb6026..6cbc27f 100644 --- a/.github/workflows/create-empty-local-db-manifest.yaml +++ b/.github/workflows/create-empty-local-db-manifest.yaml @@ -3,9 +3,6 @@ name: Create Empty Local DB Manifest env: SPACES_ACCESS_KEY: ${{ secrets.SPACES_ACCESS_KEY }} SPACES_SECRET_KEY: ${{ secrets.SPACES_SECRET_KEY }} - SPACES_REGION: ${{ secrets.SPACES_REGION }} - SPACES_BUCKET: ${{ secrets.SPACES_BUCKET }} - SPACES_ENDPOINT: ${{ secrets.SPACES_ENDPOINT }} on: workflow_dispatch: @@ -30,12 +27,31 @@ jobs: keep-env-derivations = true keep-outputs = true + - name: Load encrypted runtime config + shell: bash + run: | + set -euo pipefail + mkdir -p ~/.ssh + printf '%s\n' "${{ secrets.SSH_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + nix shell nixpkgs#rage -c sh -c 'rage -d -i ~/.ssh/id_ed25519 config/runtime.env.age > /tmp/local-db-runtime.env' + set -a + # shellcheck disable=SC1091 + source /tmp/local-db-runtime.env + set +a + { + echo "DUMP_BASE_URL=$DUMP_BASE_URL" + echo "SPACES_REGION=$SPACES_REGION" + echo "SPACES_BUCKET=$SPACES_BUCKET" + echo "SPACES_ENDPOINT=$SPACES_ENDPOINT" + } >> "$GITHUB_ENV" + - name: Create and upload empty manifest id: create_manifest shell: bash run: | set -euo pipefail - manifest_url="$(nix run .#local-db-create-empty-manifest)" + manifest_url="$(nix run .#local-db-create-empty-manifest | tail -n 1)" echo "manifest_url=$manifest_url" >>"$GITHUB_OUTPUT" - name: Summarize manifest URL diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 84184f0..ff603e4 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -28,12 +28,9 @@ permissions: env: HYPER_RPC_API_TOKEN: ${{ secrets.HYPER_RPC_API_TOKEN }} - DUMP_BASE_URL: ${{ secrets.DUMP_BASE_URL }} + PUBLIC_WALLETCONNECT_PROJECT_ID: "00000000000000000000000000000000" SPACES_ACCESS_KEY: ${{ secrets.SPACES_ACCESS_KEY }} SPACES_SECRET_KEY: ${{ secrets.SPACES_SECRET_KEY }} - SPACES_REGION: ${{ secrets.SPACES_REGION }} - SPACES_BUCKET: ${{ secrets.SPACES_BUCKET }} - SPACES_ENDPOINT: ${{ secrets.SPACES_ENDPOINT }} TARGET_REF: ${{ inputs.target_ref }} jobs: @@ -75,6 +72,22 @@ jobs: printf '%s\n' "${{ secrets.SSH_KEY }}" > ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 + - name: Load encrypted runtime config + shell: bash + run: | + set -euo pipefail + nix shell nixpkgs#rage -c sh -c 'rage -d -i ~/.ssh/id_ed25519 config/runtime.env.age > /tmp/local-db-runtime.env' + set -a + # shellcheck disable=SC1091 + source /tmp/local-db-runtime.env + set +a + { + echo "DUMP_BASE_URL=$DUMP_BASE_URL" + echo "SPACES_REGION=$SPACES_REGION" + echo "SPACES_BUCKET=$SPACES_BUCKET" + echo "SPACES_ENDPOINT=$SPACES_ENDPOINT" + } >> "$GITHUB_ENV" + - name: Prepare raindex submodule run: ./prep.sh diff --git a/config/runtime.env.age b/config/runtime.env.age new file mode 100644 index 0000000000000000000000000000000000000000..dd29cb94c3c8225cc6ccff52bbd4019bf70aa83a GIT binary patch literal 622 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCSnHqUm;E?00VG%H9; ziAqV!DYJA;b@xp!@i*1?uZRqd3NlLeao6_^@(%LLOV7#9a^>mYxw&DWfs>_IR#|SYnX{i^u!l#WXNg&bdr3N%uCA_vi&LdhMqakN zNkO)Dcxh&+p?7GSV}_YeRd%jRqFH68vA=6!h?%Rtg;z1x@}kO&{nt4zXx^$f7C73w zC1UrDJ^qX5mFI2F-^u-*8*^VjJloN^-uA^Z00yb- ArvLx| literal 0 HcmV?d00001 diff --git a/config/runtime.env.example b/config/runtime.env.example new file mode 100644 index 0000000..f7dd0af --- /dev/null +++ b/config/runtime.env.example @@ -0,0 +1,4 @@ +DUMP_BASE_URL= +SPACES_REGION= +SPACES_BUCKET= +SPACES_ENDPOINT= diff --git a/config/secrets.nix b/config/secrets.nix new file mode 100644 index 0000000..f9a43a4 --- /dev/null +++ b/config/secrets.nix @@ -0,0 +1,3 @@ +{ + "runtime.env.age".publicKeys = (import ../keys.nix).roles.infra; +} diff --git a/flake.nix b/flake.nix index 1d61907..4e0b350 100644 --- a/flake.nix +++ b/flake.nix @@ -126,6 +126,11 @@ export LIBRARY_PATH="${pkgs.gmp}/lib''${LIBRARY_PATH:+:$LIBRARY_PATH}" export PKG_CONFIG_PATH="${pkgs.gmp.dev}/lib/pkgconfig''${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" export RUSTFLAGS="-L native=${pkgs.gmp}/lib ''${RUSTFLAGS:-}" + export OPENSSL_DIR="${pkgs.openssl.dev}" + export OPENSSL_LIB_DIR="${pkgs.openssl.out}/lib" + export OPENSSL_INCLUDE_DIR="${pkgs.openssl.dev}/include" + COMMIT_SHA="$(git -C "$raindex_root" rev-parse HEAD)" + export COMMIT_SHA target_triple="$(rustc -vV | sed -n 's/^host: //p')" @@ -138,11 +143,11 @@ ;; esac - binary_source="$raindex_root/target/$target_triple/release/rain_orderbook_cli" + binary_source="$raindex_root/target/$target_triple/release/raindex_cli" binary_output="$repo_root/rain-orderbook-cli" echo "Building local raindex CLI artifact..." - cargo build --release --manifest-path "$raindex_manifest" -p rain_orderbook_cli --target "$target_triple" + cargo build --release --manifest-path "$raindex_manifest" -p raindex_cli --target "$target_triple" cp "$binary_source" "$binary_output" chmod 755 "$binary_output" diff --git a/infra/terraform.tfvars.age b/infra/terraform.tfvars.age new file mode 100644 index 0000000000000000000000000000000000000000..04440ca7db44542e107cc6daffedf08e79f713a2 GIT binary patch literal 486 zcmZ9_%WKnc003aU28WD!aBN^iHf(K7S5U+m!z&CUq+zG6th*B>Mw!*|MO$h|Qi$Vx+ z$0li-sG%0)bN&ns>OMb_q(GJCy_}dusk+012GQvIu?jn_J;!Wm)!Hn<1_lBIdXUDs z%4&cPyE{FyWMPoy3Nf{GvmvFLW>REK0D^J<-?FO4r7Rj!^^s;NJ%JGm7v}zG9%(vk zskxYn+kx5386ht+GAijVk_@+ly?zq5>54IKiUJIoMwqAyRrodqBD+P7^IA3sa=H3% z+m~+S9_}X25IQJGRpkLH$iw!AJsF@tz8TLH8!7Xcsg_5o#~8tIw=!qFSglWFFB;X$ zx&R0}kjId3N?L~rMJ$9q-}De1$I-4sPP!P?j;Fj$^tq|A>oN{d*eum8nRAPL%27<# z+3&eN^7{6jvy10XjvodaD^Ka8C%?t}SKpLRSB%XU*RC8tJ9$1ZlV6|2m(c!ft{;l= zkJa^C2g}Rv9?NT;BjVEA>uXEM`_`i@`hH>Yb?eXRYVhr{@F8hb&b|8blc^jmmhNpX I4nOLD0eUW~=>Px# literal 0 HcmV?d00001 diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 841ba90..0ee92e4 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -4,6 +4,11 @@ # `do_token` is intentionally omitted from this example so CI can supply it # through `TF_VAR_do_token` without being overridden by a checked-in file. # +# Optional overrides (current deployment defaults shown): +# ssh_key_name = "github_do" +# region = "lon1" +# droplet_size = "s-2vcpu-4gb" +# # Narrow this to trusted admin IPs once you have them. Leaving the default keeps # SSH reachable from anywhere while the rest of the droplet stays closed. # ssh_allowed_cidrs = ["203.0.113.10/32"] diff --git a/infra/variables.tf b/infra/variables.tf index c662e86..7fcfe93 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -19,7 +19,7 @@ variable "region" { variable "droplet_size" { description = "Droplet size slug" type = string - default = "s-1vcpu-1gb" + default = "s-2vcpu-4gb" } variable "ssh_allowed_cidrs" { diff --git a/lib/raindex b/lib/raindex index e686b4d..5daff74 160000 --- a/lib/raindex +++ b/lib/raindex @@ -1 +1 @@ -Subproject commit e686b4d353bfa271b1eef99c2569426eaa075b9b +Subproject commit 5daff745070a06c809839bf1cd2ecfc734b3e438 From ea23fd7d668abd3b7987e139d657ea4d18d606b4 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Wed, 15 Apr 2026 11:21:56 +0300 Subject: [PATCH 03/16] Restore provisioned terraform state --- infra/terraform.tfstate.age | Bin 0 -> 7342 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 infra/terraform.tfstate.age diff --git a/infra/terraform.tfstate.age b/infra/terraform.tfstate.age new file mode 100644 index 0000000000000000000000000000000000000000..89726fcc55dd181f7e82f4aac52fcd60aacfc22b GIT binary patch literal 7342 zcmV;f98u$8XJsvAZewzJaCB*JZZ2fF?Yj+?* zS9Lg4FKBg5T1IPXMOrg&Pf1Qmba^o=PHJ;YNHkMXIB_^bHC0$QXH^PoP)sm4PG~WD zcrbQZNiS1!V^er~ZZB_BOhGbXLq%dad21_83N1b$AxvOkMp`Xr za%Ew2Wgtu_RVpB6WII+NAZI;4d?0!>T2w)DP6{@3FLgnBZgEslQ%yKcd2n<_V^dFb zRc~WcMG7q~Eg)uhL18_MK)opFaOtNAl0y~%ELA(%5k_W!ozQugN(cE^jM&?KlAUOf7{R| zbJB;Q)9a{ViI4qR?>;O_D1VCO*tSqQUmo;h3+Bas!6EWWOJH`2ol%vv8u8&*!R*Kc zlq(MXlJex?*XgX>`$6304iC4Hh;;AD>u0;9{{Vbe(gj7Lv_0mcjfQ-{sonlcTjRq2*a!vq5jhc!(8AiNG28+^uq*J$c>BvuIM4epwLH;f#w)q z3WGSWh=dnIRm}ypI$&?;o&_{a`X2i+B%xA_*fO?6j0UYr<#HW7I(KtwFnE$)0|@de*)n#wQJSys_&yvRa&KylbhEM8+sbs9y4g7)fy*KrYzhYFyFJ=MzzMayXboaE1JWi97Q~+ z9iG%5(I_g!==+F@^nqPip!Ino_Z~dmYED#vZcn}N9qeylsqo+}r4=w2jY9d=`RA^1 z#jTe}Ba*8f@U8^c85&mb-dD#3kq7fB?8mt0%Z#F#KZUHp=Um0zDl@ zI~Dx3%h__jx8@vQOubyg*?hi=Fn(vA{&ly*+O$2N-+_F^gUca#IUlY$33ulH4J7Z4q5$|R+$K)6}X#zj&3<`BQR}6L-aVDtl?_DP-S~sXPPXrTMlA zil6ykpcF?09y!jr9MJ`>ZG36*U`OZUwUs-#ZmEQSah@6WKGo~~vASrtYW#3byKI{z z0DnVZcrmQZjmO9Ovi@Px8RS6a{OLIJj6ZqQH|XBx6^tdzShf*o-os9DiDmZlC6;%! zivTJi#$XBtgrq*kX#7-LbST);CY=8EDMS+;ee5O$D1XGuu= zDR}37_=NGN$9^$k2FGz1D`zH~8A(vA$LvAb`r5#3-?1n$mCR01=LOmzNp^eF)yExZ z8taxuZ705${w!@;ss+$HL$|_srfUSXoct@!Ih_l;juMc0F?bEqRPxILuz(pvej#}ICH?RjN zRGsbEb=zD&6{Ltfqb@gB*?q*Ku(jIjt(nP)-pI2K9~eA}dyoH2tMDhS9jA@^xLv$+ z)FyK*d0JPO!mWV@=?PPIQ^>bw@34alziH{xka|Oe2{#e-dDc!@s=q1g&d;!*4ACjaEoM^(04yj+m^pGdHYT!>cZkQQkZ*{$4A;#wM_2idMx^IWCly z#(v{D@gC(0r3$>CL}7i{CFH8^3%YF;|1F-10s&V9xgb)Q3>96>0($U&^&>cDeT7@k zJ>WwR?!?~Yn&qTv>JI0tS-0lm8vtdfAI}u=7rF&Rg!qp>xZ^$=%FU**1;03VVRfbL z*dbZEFj^5|-;#eUh6mb8ji{!rh1Y?a$=jwwp`Ml3xXosilZ zW!T`0)=NAn(})VOCAieEeAHRi%r^^&4BY<#)eC{Xgy*n7;va8Yy9d&9kX`C|#F!25 zo3o}aWX^sv%U|mMl%`CH^RDy>POHVt$UDW~WQr}$iZ}2iJceb1e!eq&_=W^a_cy29 z%oA$DAmt<%rqS<_fJU8uS|sTq%&7pT;|~E1$_2V8@r|vKGC{}FYGwz`pb`r4xB9c! zgayM$NQQ}!@hhbws;KMeXi&2#pez?*HmdWHyYy zYzBdp1Mpgg#k0N>?bE=-W1c6($Pc3)TUc zOROU9S^jGV#I7mya^xr+4+7aUEylmOxw3uWwFKVUgLDp7Grw)9Hbkk3Ox~bLc?{}L zbaJns5uz96FMoLVAc}v?IFM3YVtLwVqk9uN$5QaofNMd_D%E3(Li)g(9;?Al5gadC zk&&k5{{EzuU}v2zW22J6OZs+0mpj0|?yHN^8pgcQ(sYNWow1l{?phBvmR7Tn zyTAleD!-EhD^nJX$W(_F=?DsC%Cu^_whvX;^vB1#>ZO&%uYa~XW{mRiJ4s18^8EA& znA`9@p&O_rMhPBOR)e%K;My+y#8<+>2Lmw$_)O9x{z>EZbuKHxY34x&R#($P4PcV1 z1DxjH^I7zNw^}OrQh9h}N1;aklq~4Z>YcB+M`gCAX!(I1mp)UY-$tJoMW`#v+bqSpI^UOA&LV;2DNu7P~7|r*Yg4i+uI~ z#2*D_m_oD+mI)%vd^JH=;&R1WGCE`sz0y(VONgOtvq-&k6D#14W2G$~X5(o!soA#O zuGsBVZkhuH2a7EZYjD$s|B9kQ-&gfFs6hG$`lNvM-kP76wMPne6V0_{T!CTDTsJ#w zY6^MF(PXl`h5cYi0{tL|&ihSUzD^c{%p8s+EH1DdZoC{tAKK5So6Bg+rGWf$!M^sC z0`|U8Vc_ROs^ee^i!y3J`GOvPXXrew;#g3+CVfv=m4IC^P?t92pKM;6@7cwx!jn(c1s~QCLV@`$;ds>GUew@qpC7;Q8^e4%s zNzZFpX$YWxT}Jw|Dzo3HTM}t06O~F1VA}A3=*8zy4H||}!X1&Et>4XECZ)v-yF1NH z#=urF!+_P81X0~u25QM61XfFq2Gu8}>Bv#*KQekK>~-sZrRe`%)DY~o4U746^n-lz zPgA%LQ@4^HaG~WC>2W^uBjqgV1=JK!VEw%pM*E4@l<`b`VgrXMk5*2c=nc(skZ9cD zofqPJ^30MJ7XexgiOg(JQct4I&(B&coeq;>t-cgE}G9@5DTMWi0w56r> z))+AiRoXjENpG-|N%g%#07%}t&BOukQEwow38D>8)>hiZ1SNQZ0B=%;#%dBlmASz! zNR48^^}J&a60y73QIh96otDSv!S>zMh#3>aFt7MOYHNsQ#tP0iK!`R?uZEv5Q2`yi z;!!eZ<{ef8O(GuxX_xygP@>MoT9xJ2K9vEL5`QeVu~} z;_NNKE{jwnX%_eBDEuxQU3l0Ek_1!U&6WxznEixXWw03Fj12iD3S0YpdSqOt`YvRp zta8wr)T5##DFt8u-F>ZhouR7Q+cD0aXJEmzp_hy?9&#^UH;KLoLYV^3nagj zD1y3oyjscFq#7J@G3#gxRg~Z|n=2UzM5qKIo540huCs+tp+p_ewxhz+y|n(c$+hsD`HUzP=m_zQq|Cg!kxN|DKy6)91k3DfeCean-7 zCE`*vwYA}H(izX>0cOMI{#_-*`>BTDzM)GXEJxzzdo}H{X|E!H$DFroiDx|?bk|?3 zk(#;)#SHUr4+;%NiMBEs>{y#=a;r!~sL{cakn~=@;1L}?$f_TFD<4GC+!WC%{r;-- zb@I@n83wu_vLYEEM`*$DB(Qpi{?GW@>uVPIbqr;c%yf__Zqr^2Ezx90^<6X5S!eQGQpBdg%;A7>Ri3RceG7*d2E zkz$$A*1MuP5pAF%8>qB{FUM)DKn5jO-t?s&l3~eE;p=XhN}3p%8zLi1Q%9?&IxT$7 z_R7Xyv=GIclZI4M4i=}ic6E>B9}H|D968{b26#5J^>I#XJUNs%(HPE{(xD z&;}2>qa}=&X=9)eWTSDI+!m&y*h)Q!ikj zLl~Cvf%{!A84YgQHrr&abCKteVRc zZx&|!Y_qfbyD^Yr?ID;RGB_g6uybtb&n|x*Smg7!_=M*JU~r5QUaTOYb?nt9RapN4 zrKKY#{iSA%)DdGR=-*PWTAX4SZ?gxt!Ac+Ux|ErfVVU2fY(@z!flk!PXdu*d0ja)bt%&sB(&F{NDsWy?UMWY*}>B96E|4I+$#2?IF%ZC)V zFQY04q%uXNwt=6)-krm(ExxK9x=%9W(|KAYJqXB!J5bx>bCz$#w^%6K>g>b2z> zs#oExJ*=2b?pA-Cr9I3xs!EstV;>@k8JjP>pN63=H2c0QuP-#dB z4Jw1KzaXpE4~vm{oKgB#cMAm^UmAk&VlV*OzV^FaSC9Cz@3~+)w3$VSbrFdULi7xv z-eQ`Fqd%ry>1a)$Rb^NVlgp4$QL%)A2FL6TSiyF9K71m03-%C5p>m!yFeENN5~T~x z5~v}>c{6^uia{vI(l+`4ybuyMDrjKyY~ne<#K$S4r1DE0{hVMl1V%mF^A6J{pq#!! zv!c#*PNVBjyo0d>W?^_*l!dO5m5T$rrqkfHI*K#}w{Gjwff$OhXT*v$gOJfDjTP@U|1#xyI|3SnzvOtA zqAjBQi)LKQ$w0jRk0HMsOU%}e@qkWPH&`JDidISgPvhN!jMHMWBj0P{Wc zjBH?b!|4#LDecb$??j3VM#f1DWMN2f%5Y^1T9knC(L>zyzh1o$jm6zNKs=T685%Jc z$ml>SM6>A(;!F0uT`itx*9ktFi09zl*TSzl3*Sr7T!O?FCh(ftUUE&F9m^<~qC9cbH-(A4UjPHzR&Rg2avm7@Jk2?N%~5j_g6Zy9vf1(79cTrf z&IB5AEXS>&IEz=26W}oR2Ie0xZS&&+?4aMEhs+ihf8g}6Q;pFp+CX#bE|_3sE!SP7 zn-W&_n((CO`$dk4$gxp|nQN#+v8Dz$LzW+l;yl=%g;d)VV-pL}lGDHvrVWiHJ!T`o zgN@x&AR_dM17oh;(pI|k-0q05!M^xxhUFvg)hY8I2en>G&K(CqL0O<%N2V8Zw17kPgfpFdsy=4y zNLahK*>a1K3UJJ7O_gt1yZQc>(l#(Oja2&CR%=0T?8Wlr=6T{SK)ToAa7@Mvg65iw zcnUs2NE(3W42$r_YF08{R4U0|T_(=W{pBu*RB;Hks-rUM78P=?{(dkd&p(~cz zhP|qLcQV#4zRZ2tnutC;PfQGvp&JF6>eFO%v{*CSPn}2Wu#xl}c=}5iRDHYk@gM$bwI$;&xOqBjwL-7>dag-sO73&PIk5j zTMtg)YhV=9g4?pK30I1?mpV)AQ_%bb#^*L`|cHFs~loO z%hcy56cNTET^vOX-N3dYlmf<;rd=q^@BmOoOW5V#ld4)-I9{L&BKwqq1F>0C!%bV} z-(q9)^5M50mv#3XjtxxwsYoX~dE|1aImqYan?k9VEriA&uYI>dmdAh|NZl+ETH9e4 zTcbDagwYcG4*ztri07I3aGe82xs%Rz>)Rr08saidz1ezJ&x=%eh*h) z&Q;XVzw%P`(8^qG?Bp*91*;&>j@S*Xr7g!=cXa_b?^)WBV3Ko{mY9|Lux@f;N$%Wj zq(e!}PJKIJ&9|_oap+=z52X>pF?aG-1_2{=R_G#*G|Au&_Ym-3dJsIFqLoKga4XEl z%+4N0!VJBK24_b!`};LtRc@ugsa2Mbx0TUdJ!6>~m5%+cN5RJM(hcA&v<}iZ@a($I znVYVCx6TYk93zAzPVQ^utve!JtDCpOD>%~9sN(8kIob@BwD z?XKO7WLqH175P2#nqIJU)-19Y5X*2-IpObz;_QDR%!dzxV^>J>;7MgB-I+T?V;4TL zDTr0)OnkI0wGw23T5p{UlpeM+U6|4;e+X0a3wcDN6J&EmBw~i8GFV~rd6n~LI1rTs zKh9yz3D9@8!^@?D^W~NxL&NzxG8+23d-WrU`WaR5;BuSfc>p8B_kjgw&Rc)*LAfY-Wsd z>HAe6qmwc#v19MyG|D*uqzYcnBK2#|Pj#a+xFxVl9Q| Date: Wed, 15 Apr 2026 18:56:54 +0300 Subject: [PATCH 04/16] feat: deploy over tailscale --- .github/workflows/deploy.yaml | 14 ++++++++++++-- deploy.nix | 8 ++++++-- os.nix | 7 +++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index ff603e4..da6f436 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -31,6 +31,7 @@ env: PUBLIC_WALLETCONNECT_PROJECT_ID: "00000000000000000000000000000000" SPACES_ACCESS_KEY: ${{ secrets.SPACES_ACCESS_KEY }} SPACES_SECRET_KEY: ${{ secrets.SPACES_SECRET_KEY }} + TAILSCALE_HOSTNAME: local-db-remote TARGET_REF: ${{ inputs.target_ref }} jobs: @@ -88,17 +89,24 @@ jobs: echo "SPACES_ENDPOINT=$SPACES_ENDPOINT" } >> "$GITHUB_ENV" + - name: Connect runner to Tailscale + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + - name: Prepare raindex submodule run: ./prep.sh - name: Build local CLI artifact run: nix run .#build-raindex-cli - - name: Resolve host IP + - name: Resolve host over Tailscale shell: bash run: | set -euo pipefail - host_ip="$(nix run .#resolve-ip)" + host_ip="$(tailscale ip -4 "$TAILSCALE_HOSTNAME")" echo "::add-mask::$host_ip" echo "HOST_IP=$host_ip" >> "$GITHUB_ENV" @@ -109,6 +117,8 @@ jobs: ssh-keyscan -H "$HOST_IP" >> ~/.ssh/known_hosts - name: Deploy NixOS configuration + env: + DEPLOY_HOST: ${{ env.HOST_IP }} run: nix run .#deploy-nixos - name: Upload runtime assets and restart service diff --git a/deploy.nix b/deploy.nix index 6a2eb9a..ccfec00 100644 --- a/deploy.nix +++ b/deploy.nix @@ -22,8 +22,12 @@ in { infraPkgs.buildInputs ++ [ deploy-rs.packages.${localSystem}.deploy-rs ]; deployPreamble = '' - ${infraPkgs.resolveIp} - export DEPLOY_HOST="$host_ip" + if [ -n "''${DEPLOY_HOST:-}" ]; then + host_ip="$DEPLOY_HOST" + else + ${infraPkgs.resolveIp} + export DEPLOY_HOST="$host_ip" + fi export NIX_SSHOPTS="-i $identity" ssh_flag="--ssh-opts=-i $identity" ''; diff --git a/os.nix b/os.nix index bcc1054..9543bc2 100644 --- a/os.nix +++ b/os.nix @@ -50,6 +50,11 @@ in { mode = "aggressive"; }; }; + + tailscale = { + enable = true; + openFirewall = true; + }; }; users.users.root.openssh.authorizedKeys.keys = roles.ssh; @@ -57,6 +62,7 @@ in { networking.firewall = { enable = true; allowedTCPPorts = [ 22 ]; + trustedInterfaces = [ "tailscale0" ]; }; nix = { @@ -120,6 +126,7 @@ in { git htop jq + tailscale zellij ]; From 9dcb0cc76d08609bdb332e4e0175e5efa83a10fc Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Wed, 15 Apr 2026 19:57:53 +0300 Subject: [PATCH 05/16] fix: parse ssh identity for tailscale deploys --- deploy.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy.nix b/deploy.nix index ccfec00..f450f16 100644 --- a/deploy.nix +++ b/deploy.nix @@ -22,6 +22,7 @@ in { infraPkgs.buildInputs ++ [ deploy-rs.packages.${localSystem}.deploy-rs ]; deployPreamble = '' + ${infraPkgs.parseIdentity} if [ -n "''${DEPLOY_HOST:-}" ]; then host_ip="$DEPLOY_HOST" else From fa386b8f28595e0b8fc8dc12c1a056f60fa9a3bb Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Thu, 16 Apr 2026 09:47:04 +0300 Subject: [PATCH 06/16] fix: remove cloud-init from steady-state host config --- os.nix | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/os.nix b/os.nix index 9543bc2..fb0ffdf 100644 --- a/os.nix +++ b/os.nix @@ -18,16 +18,6 @@ in { networking.useDHCP = lib.mkForce false; services = { - cloud-init = { - enable = true; - network.enable = true; - settings = { - datasource_list = [ "ConfigDrive" "Digitalocean" ]; - datasource.ConfigDrive = { }; - datasource.Digitalocean = { }; - }; - }; - openssh = { enable = true; settings = { From 4e46e5ea50de56e33d396aedf167c60974aa7d5f Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Thu, 16 Apr 2026 12:10:57 +0300 Subject: [PATCH 07/16] fix: enable nix-ld for deployed cli --- os.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/os.nix b/os.nix index fb0ffdf..b64ddeb 100644 --- a/os.nix +++ b/os.nix @@ -68,6 +68,16 @@ in { }; }; + programs.nix-ld = { + enable = true; + libraries = with pkgs; [ + openssl + sqlite + stdenv.cc.cc + zlib + ]; + }; + systemd.tmpfiles.rules = [ "d /etc/local-db-remote 0750 root root -" "d /var/lib/local-db-remote 0755 root root -" From bb712b188426a8cb002f6771ea9b7abec2981486 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Thu, 16 Apr 2026 15:06:30 +0300 Subject: [PATCH 08/16] fix: publish local db to manifest target --- flake.nix | 57 ++++++++++++++++------ nixos/local-db-remote-run.sh | 36 +++++++++----- scripts/local-db-publish-target.sh | 78 ++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 25 deletions(-) create mode 100644 scripts/local-db-publish-target.sh diff --git a/flake.nix b/flake.nix index 4e0b350..1537491 100644 --- a/flake.nix +++ b/flake.nix @@ -52,6 +52,9 @@ baseDevShells = rainix.devShells.${system}; rainixPkgs = rainix.packages.${system}; + localDbPublishTargetHelpers = + pkgs.writeText "local-db-publish-target.sh" + (builtins.readFile ./scripts/local-db-publish-target.sh); addBuildInputs = shell: extraInputs: shell.overrideAttrs (old: { @@ -104,6 +107,7 @@ exit 1 fi } + ''; buildRaindexCliCommand = pkgs.writeShellApplication { @@ -165,6 +169,9 @@ ]; text = '' set -euo pipefail + export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" + # shellcheck disable=SC1090 + source "$LOCAL_DB_PUBLISH_TARGET_HELPERS" ${raindexSetup} ${envSetup} ${shellHelpers} @@ -173,7 +180,6 @@ require_var SETTINGS_YAML_URL require_var HYPER_RPC_API_TOKEN - require_var DUMP_BASE_URL cli_bin="$repo_root/rain-orderbook-cli" if [ ! -x "$cli_bin" ]; then @@ -186,12 +192,13 @@ echo "🌐 Fetching settings YAML from $SETTINGS_YAML_URL" settings_yaml="$(curl -fsSL "$SETTINGS_YAML_URL")" + resolve_publish_target echo "🚀 Running local-db sync via $cli_bin" "$cli_bin" local-db sync \ --settings-yaml "$settings_yaml" \ --api-token "$HYPER_RPC_API_TOKEN" \ - --release-base-url "$DUMP_BASE_URL" \ + --release-base-url "$manifest_dir_url" \ --out-root "$out_root" \ --debug-status ''; @@ -206,12 +213,16 @@ ]; text = '' set -euo pipefail + export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" + # shellcheck disable=SC1090 + source "$LOCAL_DB_PUBLISH_TARGET_HELPERS" ${repoRootSetup} ${envSetup} ${shellHelpers} load_env_file + require_var SETTINGS_YAML_URL require_var SPACES_ACCESS_KEY require_var SPACES_SECRET_KEY require_var SPACES_REGION @@ -229,28 +240,43 @@ exit 1 fi - echo "🚀 Uploading dump files and manifest from $local_dir to Spaces bucket: $SPACES_BUCKET" - echo " Using endpoint: $SPACES_ENDPOINT" - echo + echo "🌐 Fetching settings YAML from $SETTINGS_YAML_URL" + settings_yaml="$(curl -fsSL "$SETTINGS_YAML_URL")" + resolve_publish_target + manifest_object_key="$(object_key_from_url "$manifest_url")" + publish_prefix_key="''${manifest_object_key%/*}" - if [ -f "$local_dir/manifest.yaml" ]; then - echo "📄 Uploading manifest.yaml..." - aws s3 cp "$local_dir/manifest.yaml" "s3://$SPACES_BUCKET/manifest.yaml" \ - --endpoint-url "$SPACES_ENDPOINT" \ - --acl public-read \ - --content-type "text/yaml" + if [ "$publish_prefix_key" = "$manifest_object_key" ]; then + publish_prefix_key="" fi - echo "🗂️ Uploading SQL dump files (flattened to bucket root)..." + echo "🚀 Uploading dump files and manifest from $local_dir to Spaces bucket: $SPACES_BUCKET" + echo " Manifest URL: $manifest_url" + echo + + echo "🗂️ Uploading SQL dump files..." find "$local_dir" -type f -iname "*-0x*.sql.gz" | while IFS= read -r file; do filename="$(basename "$file")" + if [ -n "$publish_prefix_key" ]; then + object_key="$publish_prefix_key/$filename" + else + object_key="$filename" + fi echo "→ Uploading: $filename" - aws s3 cp "$file" "s3://$SPACES_BUCKET/$filename" \ + aws s3 cp "$file" "s3://$SPACES_BUCKET/$object_key" \ --endpoint-url "$SPACES_ENDPOINT" \ --acl public-read \ --content-type "application/gzip" done + if [ -f "$local_dir/manifest.yaml" ]; then + echo "📄 Uploading manifest.yaml to $manifest_url..." + aws s3 cp "$local_dir/manifest.yaml" "s3://$SPACES_BUCKET/$manifest_object_key" \ + --endpoint-url "$SPACES_ENDPOINT" \ + --acl public-read \ + --content-type "text/yaml" + fi + echo echo "✅ Upload complete!" ''; @@ -343,7 +369,10 @@ curl findutils ]; - text = builtins.readFile ./nixos/local-db-remote-run.sh; + text = '' + export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" + ${builtins.readFile ./nixos/local-db-remote-run.sh} + ''; }; infraPkgs = import ./infra { diff --git a/nixos/local-db-remote-run.sh b/nixos/local-db-remote-run.sh index 1b62381..4c30003 100644 --- a/nixos/local-db-remote-run.sh +++ b/nixos/local-db-remote-run.sh @@ -20,7 +20,6 @@ trap cleanup EXIT require_var SETTINGS_YAML_URL require_var HYPER_RPC_API_TOKEN -require_var DUMP_BASE_URL require_var SPACES_ACCESS_KEY require_var SPACES_SECRET_KEY require_var SPACES_REGION @@ -36,16 +35,26 @@ export AWS_ACCESS_KEY_ID="$SPACES_ACCESS_KEY" export AWS_SECRET_ACCESS_KEY="$SPACES_SECRET_KEY" export AWS_DEFAULT_REGION="$SPACES_REGION" +# shellcheck disable=SC1090 +source "${LOCAL_DB_PUBLISH_TARGET_HELPERS:?missing LOCAL_DB_PUBLISH_TARGET_HELPERS}" + mkdir -p "$state_root" echo "Fetching settings YAML from $SETTINGS_YAML_URL" settings_yaml="$(curl -fsSL "$SETTINGS_YAML_URL")" +resolve_publish_target +manifest_object_key="$(object_key_from_url "$manifest_url")" +publish_prefix_key="${manifest_object_key%/*}" + +if [ "$publish_prefix_key" = "$manifest_object_key" ]; then + publish_prefix_key="" +fi echo "Running local-db sync via $cli_bin" "$cli_bin" local-db sync \ --settings-yaml "$settings_yaml" \ --api-token "$HYPER_RPC_API_TOKEN" \ - --release-base-url "$DUMP_BASE_URL" \ + --release-base-url "$manifest_dir_url" \ --out-root "$out_root" \ --debug-status @@ -54,22 +63,27 @@ if [ ! -d "$out_root" ]; then exit 1 fi -if [ -f "$out_root/manifest.yaml" ]; then - echo "Uploading manifest.yaml" - aws s3 cp "$out_root/manifest.yaml" "s3://$SPACES_BUCKET/manifest.yaml" \ - --endpoint-url "$SPACES_ENDPOINT" \ - --acl public-read \ - --content-type "text/yaml" -fi - echo "Uploading SQL dump files" find "$out_root" -type f -iname "*-0x*.sql.gz" | while IFS= read -r file; do filename="$(basename "$file")" + if [ -n "$publish_prefix_key" ]; then + object_key="$publish_prefix_key/$filename" + else + object_key="$filename" + fi echo "Uploading $filename" - aws s3 cp "$file" "s3://$SPACES_BUCKET/$filename" \ + aws s3 cp "$file" "s3://$SPACES_BUCKET/$object_key" \ --endpoint-url "$SPACES_ENDPOINT" \ --acl public-read \ --content-type "application/gzip" done +if [ -f "$out_root/manifest.yaml" ]; then + echo "Uploading manifest.yaml to $manifest_url" + aws s3 cp "$out_root/manifest.yaml" "s3://$SPACES_BUCKET/$manifest_object_key" \ + --endpoint-url "$SPACES_ENDPOINT" \ + --acl public-read \ + --content-type "text/yaml" +fi + echo "Local DB remote sync completed" diff --git a/scripts/local-db-publish-target.sh b/scripts/local-db-publish-target.sh new file mode 100644 index 0000000..0901bab --- /dev/null +++ b/scripts/local-db-publish-target.sh @@ -0,0 +1,78 @@ +extract_manifest_urls() { + printf '%s\n' "$settings_yaml" | perl -ne ' + if (/^local-db-remotes:\s*$/) { + $in_remotes = 1; + next; + } + + if ($in_remotes) { + if (/^\S/) { + exit; + } + + if (/^\s+[^:#][^:]*:\s*(.+?)\s*$/) { + my $value = $1; + $value =~ s/\s+#.*$//; + $value =~ s/^["\x27]//; + $value =~ s/["\x27]$//; + print "$value\n" if length $value; + } + } + ' +} + +resolve_publish_target() { + mapfile -t manifest_urls < <(extract_manifest_urls | sort -u) + + if [ "${#manifest_urls[@]}" -ne 1 ]; then + echo "Expected exactly one unique local-db remote manifest URL, found ${#manifest_urls[@]}" >&2 + printf ' %s\n' "${manifest_urls[@]}" >&2 + exit 1 + fi + + manifest_url="${manifest_urls[0]}" + manifest_dir_url="${manifest_url%/*}" + + if [ "$manifest_dir_url" = "$manifest_url" ]; then + echo "Manifest URL does not contain a publish directory: $manifest_url" >&2 + exit 1 + fi +} + +url_host() { + printf '%s\n' "$1" | sed -E 's#^https?://([^/]+).*$#\1#' +} + +url_path() { + printf '%s\n' "$1" | sed -E 's#^https?://[^/]+(/.*)?$#\1#' +} + +object_key_from_url() { + local url="$1" + local host path endpoint_host + + host="$(url_host "$url")" + path="$(url_path "$url")" + endpoint_host="$(url_host "$SPACES_ENDPOINT")" + + case "$host" in + "$SPACES_BUCKET.$endpoint_host") + printf '%s\n' "${path#/}" + ;; + "$endpoint_host") + case "$path" in + "/$SPACES_BUCKET/"*) + printf '%s\n' "${path#/$SPACES_BUCKET/}" + ;; + *) + echo "Manifest URL path is not inside bucket $SPACES_BUCKET: $url" >&2 + exit 1 + ;; + esac + ;; + *) + echo "Manifest URL host does not match bucket endpoint: $url" >&2 + exit 1 + ;; + esac +} From 2768a2d3a617c9db2bea27271ccff0728dbae3b2 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Thu, 16 Apr 2026 16:02:05 +0300 Subject: [PATCH 09/16] fix: satisfy shellcheck for publish target vars --- nixos/local-db-remote-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/local-db-remote-run.sh b/nixos/local-db-remote-run.sh index 4c30003..e841598 100644 --- a/nixos/local-db-remote-run.sh +++ b/nixos/local-db-remote-run.sh @@ -37,6 +37,8 @@ export AWS_DEFAULT_REGION="$SPACES_REGION" # shellcheck disable=SC1090 source "${LOCAL_DB_PUBLISH_TARGET_HELPERS:?missing LOCAL_DB_PUBLISH_TARGET_HELPERS}" +manifest_url="" +manifest_dir_url="" mkdir -p "$state_root" From 118da9925aa6e50ea3f1838ae4899b0ddb87f582 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Thu, 16 Apr 2026 17:05:53 +0300 Subject: [PATCH 10/16] fix: remove perl from publish target helper --- scripts/local-db-publish-target.sh | 42 ++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/scripts/local-db-publish-target.sh b/scripts/local-db-publish-target.sh index 0901bab..212b9bb 100644 --- a/scripts/local-db-publish-target.sh +++ b/scripts/local-db-publish-target.sh @@ -1,28 +1,42 @@ extract_manifest_urls() { - printf '%s\n' "$settings_yaml" | perl -ne ' - if (/^local-db-remotes:\s*$/) { - $in_remotes = 1; - next; + printf '%s\n' "$settings_yaml" | awk ' + /^local-db-remotes:[[:space:]]*$/ { + in_remotes = 1 + next } - if ($in_remotes) { - if (/^\S/) { - exit; + in_remotes { + if ($0 ~ /^[^[:space:]]/) { + exit } - if (/^\s+[^:#][^:]*:\s*(.+?)\s*$/) { - my $value = $1; - $value =~ s/\s+#.*$//; - $value =~ s/^["\x27]//; - $value =~ s/["\x27]$//; - print "$value\n" if length $value; + if ($0 ~ /^[[:space:]]+[^:#][^:]*:[[:space:]]*/) { + value = $0 + sub(/^[[:space:]]+[^:#][^:]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*$/, "", value) + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + + if (value ~ /^".*"$/ || value ~ /^'\''.*'\''$/) { + value = substr(value, 2, length(value) - 2) + } + + if (length(value) > 0) { + print value + } } } ' } resolve_publish_target() { - mapfile -t manifest_urls < <(extract_manifest_urls | sort -u) + manifest_urls=() + + while IFS= read -r manifest_url_candidate; do + if [ -n "$manifest_url_candidate" ]; then + manifest_urls+=("$manifest_url_candidate") + fi + done < <(extract_manifest_urls | sort -u) if [ "${#manifest_urls[@]}" -ne 1 ]; then echo "Expected exactly one unique local-db remote manifest URL, found ${#manifest_urls[@]}" >&2 From 1c8e961ad66369294d5e689ba0524c656b63c106 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Thu, 16 Apr 2026 18:17:45 +0300 Subject: [PATCH 11/16] fix: add gawk to local db remote runner --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 1537491..be0467c 100644 --- a/flake.nix +++ b/flake.nix @@ -368,6 +368,7 @@ coreutils curl findutils + gawk ]; text = '' export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" From 6adc5c38c5ad0ff48eb408014c2a990f580e9197 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Mon, 20 Apr 2026 11:44:42 +0300 Subject: [PATCH 12/16] fix: pin newer tailscale package --- flake.lock | 17 +++++++++++++++++ flake.nix | 10 ++++++++-- os.nix | 5 +++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index f7f4e4f..accdccc 100644 --- a/flake.lock +++ b/flake.lock @@ -412,6 +412,22 @@ "type": "github" } }, + "nixpkgs-tailscale": { + "locked": { + "lastModified": 1776329215, + "narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b86751bc4085f48661017fa226dee99fab6c651b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1761672384, @@ -544,6 +560,7 @@ "rainix", "nixpkgs" ], + "nixpkgs-tailscale": "nixpkgs-tailscale", "ragenix": "ragenix", "rainix": "rainix" } diff --git a/flake.nix b/flake.nix index be0467c..1af8c33 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,7 @@ rainix.url = "github:rainlanguage/rainix"; flake-utils.url = "github:numtide/flake-utils"; nixpkgs.follows = "rainix/nixpkgs"; + nixpkgs-tailscale.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; ragenix.url = "github:yaxitech/ragenix"; deploy-rs.url = "github:serokell/deploy-rs"; @@ -21,6 +22,7 @@ flake-utils, rainix, nixpkgs, + nixpkgs-tailscale, ragenix, deploy-rs, disko, @@ -31,9 +33,13 @@ deploySystem = "x86_64-linux"; in { nixosConfigurations.local-db-remote = - rainix.inputs.nixpkgs.lib.nixosSystem { + let + tailscalePkgs = import nixpkgs-tailscale { + system = deploySystem; + }; + in rainix.inputs.nixpkgs.lib.nixosSystem { system = deploySystem; - specialArgs = { inherit self; }; + specialArgs = { inherit self tailscalePkgs; }; modules = [ disko.nixosModules.disko ./os.nix diff --git a/os.nix b/os.nix index b64ddeb..a426775 100644 --- a/os.nix +++ b/os.nix @@ -1,4 +1,4 @@ -{ lib, modulesPath, pkgs, self, ... }: +{ lib, modulesPath, pkgs, self, tailscalePkgs, ... }: let inherit (import ./keys.nix) roles; @@ -44,6 +44,7 @@ in { tailscale = { enable = true; openFirewall = true; + package = tailscalePkgs.tailscale; }; }; @@ -126,7 +127,7 @@ in { git htop jq - tailscale + tailscalePkgs.tailscale zellij ]; From be9f00d52101b4b6f8180e49ee1e05914083fbeb Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Mon, 20 Apr 2026 13:05:15 +0300 Subject: [PATCH 13/16] chore: debug tailscale deploy connectivity --- .github/workflows/deploy.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index da6f436..b458103 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -96,6 +96,15 @@ jobs: oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:ci + - name: Debug Tailscale connectivity + shell: bash + run: | + set -euo pipefail + tailscale status --json | jq '{CurrentTailnet, Self}' + tailscale status + tailscale ip -4 "$TAILSCALE_HOSTNAME" + tailscale ping --c 3 "$TAILSCALE_HOSTNAME" + - name: Prepare raindex submodule run: ./prep.sh From f2314dba72684c58f68685255287d221f5ffa777 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Mon, 20 Apr 2026 13:13:32 +0300 Subject: [PATCH 14/16] chore: debug tailscale ssh connectivity --- .github/workflows/deploy.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index b458103..0596729 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -102,8 +102,11 @@ jobs: set -euo pipefail tailscale status --json | jq '{CurrentTailnet, Self}' tailscale status - tailscale ip -4 "$TAILSCALE_HOSTNAME" - tailscale ping --c 3 "$TAILSCALE_HOSTNAME" + host_ip="$(tailscale ip -4 "$TAILSCALE_HOSTNAME")" + echo "Resolved $TAILSCALE_HOSTNAME to $host_ip" + tailscale ping --c 3 "$TAILSCALE_HOSTNAME" || true + timeout 10 bash -c "cat < /dev/null > /dev/tcp/$host_ip/22" + ssh-keyscan -T 10 -H "$host_ip" >/dev/null - name: Prepare raindex submodule run: ./prep.sh From 1ee5ba3a3553a99f1b973bc0a10a6c36e348a8c6 Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Mon, 20 Apr 2026 14:41:48 +0300 Subject: [PATCH 15/16] chore: use raindex manifest URL command --- flake.nix | 173 +++++++++++++++++------------ lib/raindex | 2 +- nixos/local-db-remote-run.sh | 79 ++++++++++++- scripts/local-db-publish-target.sh | 92 --------------- 4 files changed, 182 insertions(+), 164 deletions(-) delete mode 100644 scripts/local-db-publish-target.sh diff --git a/flake.nix b/flake.nix index 1af8c33..d151cc4 100644 --- a/flake.nix +++ b/flake.nix @@ -17,37 +17,21 @@ nixos-anywhere.inputs.nixpkgs.follows = "rainix/nixpkgs"; }; - outputs = { - self, - flake-utils, - rainix, - nixpkgs, - nixpkgs-tailscale, - ragenix, - deploy-rs, - disko, - nixos-anywhere, - ... - }: - let - deploySystem = "x86_64-linux"; + outputs = { self, flake-utils, rainix, nixpkgs, nixpkgs-tailscale, ragenix + , deploy-rs, disko, nixos-anywhere, ... }: + let deploySystem = "x86_64-linux"; in { nixosConfigurations.local-db-remote = - let - tailscalePkgs = import nixpkgs-tailscale { - system = deploySystem; - }; + let tailscalePkgs = import nixpkgs-tailscale { system = deploySystem; }; in rainix.inputs.nixpkgs.lib.nixosSystem { system = deploySystem; specialArgs = { inherit self tailscalePkgs; }; - modules = [ - disko.nixosModules.disko - ./os.nix - ]; + modules = [ disko.nixosModules.disko ./os.nix ]; }; deploy = (import ./deploy.nix { inherit deploy-rs self; }).config; - checks.${deploySystem} = deploy-rs.lib.${deploySystem}.deployChecks self.deploy; + checks.${deploySystem} = + deploy-rs.lib.${deploySystem}.deployChecks self.deploy; } // flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { @@ -58,14 +42,10 @@ baseDevShells = rainix.devShells.${system}; rainixPkgs = rainix.packages.${system}; - localDbPublishTargetHelpers = - pkgs.writeText "local-db-publish-target.sh" - (builtins.readFile ./scripts/local-db-publish-target.sh); addBuildInputs = shell: extraInputs: - shell.overrideAttrs (old: { - buildInputs = (old.buildInputs or [ ]) ++ extraInputs; - }); + shell.overrideAttrs + (old: { buildInputs = (old.buildInputs or [ ]) ++ extraInputs; }); repoRootSetup = '' repo_root="''${LOCAL_DB_REPO_ROOT:-$(pwd -P)}" @@ -114,6 +94,81 @@ fi } + resolve_manifest_publish_target() { + local urls first_url + + urls="$("$cli_bin" local-db manifest-urls --settings-yaml "$settings_yaml")" + first_url="" + + while IFS= read -r url; do + if [ -n "$url" ]; then + first_url="$url" + break + fi + done <<<"$urls" + + if [ -z "$first_url" ]; then + echo "❌ Expected at least one local-db remote manifest URL" + exit 1 + fi + + manifest_url="$first_url" + manifest_dir_url="''${manifest_url%/*}" + + if [ "$manifest_dir_url" = "$manifest_url" ]; then + echo "❌ Manifest URL does not contain a publish directory: $manifest_url" + exit 1 + fi + } + + url_host() { + local without_scheme="$1" + without_scheme="''${without_scheme#http://}" + without_scheme="''${without_scheme#https://}" + printf '%s\n' "''${without_scheme%%/*}" + } + + url_path() { + local without_scheme="$1" + without_scheme="''${without_scheme#http://}" + without_scheme="''${without_scheme#https://}" + + case "$without_scheme" in + */*) printf '/%s\n' "''${without_scheme#*/}" ;; + *) printf '\n' ;; + esac + } + + object_key_from_url() { + local url="$1" + local host path endpoint_host + + host="$(url_host "$url")" + path="$(url_path "$url")" + endpoint_host="$(url_host "$SPACES_ENDPOINT")" + + case "$host" in + "$SPACES_BUCKET.$endpoint_host") + printf '%s\n' "''${path#/}" + ;; + "$endpoint_host") + case "$path" in + "/$SPACES_BUCKET/"*) + printf '%s\n' "''${path#/"$SPACES_BUCKET"/}" + ;; + *) + echo "Manifest URL path is not inside bucket $SPACES_BUCKET: $url" >&2 + exit 1 + ;; + esac + ;; + *) + echo "Manifest URL host does not match bucket endpoint: $url" >&2 + exit 1 + ;; + esac + } + ''; buildRaindexCliCommand = pkgs.writeShellApplication { @@ -169,15 +224,9 @@ localDbSyncCommand = pkgs.writeShellApplication { name = "local-db-sync"; - runtimeInputs = with pkgs; [ - coreutils - curl - ]; + runtimeInputs = with pkgs; [ coreutils curl ]; text = '' set -euo pipefail - export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" - # shellcheck disable=SC1090 - source "$LOCAL_DB_PUBLISH_TARGET_HELPERS" ${raindexSetup} ${envSetup} ${shellHelpers} @@ -198,7 +247,7 @@ echo "🌐 Fetching settings YAML from $SETTINGS_YAML_URL" settings_yaml="$(curl -fsSL "$SETTINGS_YAML_URL")" - resolve_publish_target + resolve_manifest_publish_target echo "🚀 Running local-db sync via $cli_bin" "$cli_bin" local-db sync \ @@ -212,16 +261,9 @@ localDbUploadCommand = pkgs.writeShellApplication { name = "local-db-upload"; - runtimeInputs = with pkgs; [ - awscli2 - coreutils - findutils - ]; + runtimeInputs = with pkgs; [ awscli2 coreutils curl findutils ]; text = '' set -euo pipefail - export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" - # shellcheck disable=SC1090 - source "$LOCAL_DB_PUBLISH_TARGET_HELPERS" ${repoRootSetup} ${envSetup} ${shellHelpers} @@ -240,6 +282,13 @@ export AWS_DEFAULT_REGION="$SPACES_REGION" local_dir="$repo_root/local-db" + cli_bin="$repo_root/rain-orderbook-cli" + + if [ ! -x "$cli_bin" ]; then + echo "❌ Local CLI artifact missing at $cli_bin" + echo " Run: nix run .#build-raindex-cli" + exit 1 + fi if [ ! -d "$local_dir" ]; then echo "❌ Local DB directory not found at $local_dir" @@ -248,7 +297,7 @@ echo "🌐 Fetching settings YAML from $SETTINGS_YAML_URL" settings_yaml="$(curl -fsSL "$SETTINGS_YAML_URL")" - resolve_publish_target + resolve_manifest_publish_target manifest_object_key="$(object_key_from_url "$manifest_url")" publish_prefix_key="''${manifest_object_key%/*}" @@ -290,11 +339,7 @@ localDbCreateEmptyManifestCommand = pkgs.writeShellApplication { name = "local-db-create-empty-manifest"; - runtimeInputs = with pkgs; [ - awscli2 - coreutils - gnused - ]; + runtimeInputs = with pkgs; [ awscli2 coreutils gnused ]; text = '' set -euo pipefail ${raindexSetup} @@ -369,32 +414,24 @@ localDbRemoteRunner = pkgs.writeShellApplication { name = "local-db-remote-run"; - runtimeInputs = with pkgs; [ - awscli2 - coreutils - curl - findutils - gawk - ]; + runtimeInputs = with pkgs; [ awscli2 coreutils curl findutils ]; text = '' - export LOCAL_DB_PUBLISH_TARGET_HELPERS="${localDbPublishTargetHelpers}" ${builtins.readFile ./nixos/local-db-remote-run.sh} ''; }; - infraPkgs = import ./infra { - inherit pkgs ragenix rainix system; - }; + infraPkgs = import ./infra { inherit pkgs ragenix rainix system; }; - deployPkgs = (import ./deploy.nix { inherit deploy-rs self; }).wrappers { - inherit pkgs infraPkgs; - localSystem = system; - }; + deployPkgs = + (import ./deploy.nix { inherit deploy-rs self; }).wrappers { + inherit pkgs infraPkgs; + localSystem = system; + }; bootstrapNixos = rainix.mkTask.${system} { name = "bootstrap-nixos"; - additionalBuildInputs = - infraPkgs.buildInputs ++ [ nixos-anywhere.packages.${system}.default ]; + additionalBuildInputs = infraPkgs.buildInputs + ++ [ nixos-anywhere.packages.${system}.default ]; body = '' ${infraPkgs.resolveIp} ssh_opts="-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i $identity" diff --git a/lib/raindex b/lib/raindex index 5daff74..9b33a3c 160000 --- a/lib/raindex +++ b/lib/raindex @@ -1 +1 @@ -Subproject commit 5daff745070a06c809839bf1cd2ecfc734b3e438 +Subproject commit 9b33a3c4eca44726b4602f1a2d4046982584ef93 diff --git a/nixos/local-db-remote-run.sh b/nixos/local-db-remote-run.sh index e841598..ee4f990 100644 --- a/nixos/local-db-remote-run.sh +++ b/nixos/local-db-remote-run.sh @@ -8,6 +8,81 @@ require_var() { fi } +resolve_manifest_publish_target() { + local urls first_url + + urls="$("$cli_bin" local-db manifest-urls --settings-yaml "$settings_yaml")" + first_url="" + + while IFS= read -r url; do + if [ -n "$url" ]; then + first_url="$url" + break + fi + done <<<"$urls" + + if [ -z "$first_url" ]; then + echo "Expected at least one local-db remote manifest URL" >&2 + exit 1 + fi + + manifest_url="$first_url" + manifest_dir_url="${manifest_url%/*}" + + if [ "$manifest_dir_url" = "$manifest_url" ]; then + echo "Manifest URL does not contain a publish directory: $manifest_url" >&2 + exit 1 + fi +} + +url_host() { + local without_scheme="$1" + without_scheme="${without_scheme#http://}" + without_scheme="${without_scheme#https://}" + printf '%s\n' "${without_scheme%%/*}" +} + +url_path() { + local without_scheme="$1" + without_scheme="${without_scheme#http://}" + without_scheme="${without_scheme#https://}" + + case "$without_scheme" in + */*) printf '/%s\n' "${without_scheme#*/}" ;; + *) printf '\n' ;; + esac +} + +object_key_from_url() { + local url="$1" + local host path endpoint_host + + host="$(url_host "$url")" + path="$(url_path "$url")" + endpoint_host="$(url_host "$SPACES_ENDPOINT")" + + case "$host" in + "$SPACES_BUCKET.$endpoint_host") + printf '%s\n' "${path#/}" + ;; + "$endpoint_host") + case "$path" in + "/$SPACES_BUCKET/"*) + printf '%s\n' "${path#/"$SPACES_BUCKET"/}" + ;; + *) + echo "Manifest URL path is not inside bucket $SPACES_BUCKET: $url" >&2 + exit 1 + ;; + esac + ;; + *) + echo "Manifest URL host does not match bucket endpoint: $url" >&2 + exit 1 + ;; + esac +} + cli_bin="/var/lib/local-db-remote/bin/rain-orderbook-cli" state_root="/var/lib/local-db-remote/work" out_root="$state_root/local-db" @@ -35,8 +110,6 @@ export AWS_ACCESS_KEY_ID="$SPACES_ACCESS_KEY" export AWS_SECRET_ACCESS_KEY="$SPACES_SECRET_KEY" export AWS_DEFAULT_REGION="$SPACES_REGION" -# shellcheck disable=SC1090 -source "${LOCAL_DB_PUBLISH_TARGET_HELPERS:?missing LOCAL_DB_PUBLISH_TARGET_HELPERS}" manifest_url="" manifest_dir_url="" @@ -44,7 +117,7 @@ mkdir -p "$state_root" echo "Fetching settings YAML from $SETTINGS_YAML_URL" settings_yaml="$(curl -fsSL "$SETTINGS_YAML_URL")" -resolve_publish_target +resolve_manifest_publish_target manifest_object_key="$(object_key_from_url "$manifest_url")" publish_prefix_key="${manifest_object_key%/*}" diff --git a/scripts/local-db-publish-target.sh b/scripts/local-db-publish-target.sh deleted file mode 100644 index 212b9bb..0000000 --- a/scripts/local-db-publish-target.sh +++ /dev/null @@ -1,92 +0,0 @@ -extract_manifest_urls() { - printf '%s\n' "$settings_yaml" | awk ' - /^local-db-remotes:[[:space:]]*$/ { - in_remotes = 1 - next - } - - in_remotes { - if ($0 ~ /^[^[:space:]]/) { - exit - } - - if ($0 ~ /^[[:space:]]+[^:#][^:]*:[[:space:]]*/) { - value = $0 - sub(/^[[:space:]]+[^:#][^:]*:[[:space:]]*/, "", value) - sub(/[[:space:]]+#.*$/, "", value) - sub(/^[[:space:]]+/, "", value) - sub(/[[:space:]]+$/, "", value) - - if (value ~ /^".*"$/ || value ~ /^'\''.*'\''$/) { - value = substr(value, 2, length(value) - 2) - } - - if (length(value) > 0) { - print value - } - } - } - ' -} - -resolve_publish_target() { - manifest_urls=() - - while IFS= read -r manifest_url_candidate; do - if [ -n "$manifest_url_candidate" ]; then - manifest_urls+=("$manifest_url_candidate") - fi - done < <(extract_manifest_urls | sort -u) - - if [ "${#manifest_urls[@]}" -ne 1 ]; then - echo "Expected exactly one unique local-db remote manifest URL, found ${#manifest_urls[@]}" >&2 - printf ' %s\n' "${manifest_urls[@]}" >&2 - exit 1 - fi - - manifest_url="${manifest_urls[0]}" - manifest_dir_url="${manifest_url%/*}" - - if [ "$manifest_dir_url" = "$manifest_url" ]; then - echo "Manifest URL does not contain a publish directory: $manifest_url" >&2 - exit 1 - fi -} - -url_host() { - printf '%s\n' "$1" | sed -E 's#^https?://([^/]+).*$#\1#' -} - -url_path() { - printf '%s\n' "$1" | sed -E 's#^https?://[^/]+(/.*)?$#\1#' -} - -object_key_from_url() { - local url="$1" - local host path endpoint_host - - host="$(url_host "$url")" - path="$(url_path "$url")" - endpoint_host="$(url_host "$SPACES_ENDPOINT")" - - case "$host" in - "$SPACES_BUCKET.$endpoint_host") - printf '%s\n' "${path#/}" - ;; - "$endpoint_host") - case "$path" in - "/$SPACES_BUCKET/"*) - printf '%s\n' "${path#/$SPACES_BUCKET/}" - ;; - *) - echo "Manifest URL path is not inside bucket $SPACES_BUCKET: $url" >&2 - exit 1 - ;; - esac - ;; - *) - echo "Manifest URL host does not match bucket endpoint: $url" >&2 - exit 1 - ;; - esac -} From b1aca2492f472d209fdbdfea3c88de3f634d518c Mon Sep 17 00:00:00 2001 From: Arda Nakisci Date: Mon, 20 Apr 2026 16:04:57 +0300 Subject: [PATCH 16/16] chore: remove deploy tailnet debug step --- .github/workflows/deploy.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0596729..da6f436 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -96,18 +96,6 @@ jobs: oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:ci - - name: Debug Tailscale connectivity - shell: bash - run: | - set -euo pipefail - tailscale status --json | jq '{CurrentTailnet, Self}' - tailscale status - host_ip="$(tailscale ip -4 "$TAILSCALE_HOSTNAME")" - echo "Resolved $TAILSCALE_HOSTNAME to $host_ip" - tailscale ping --c 3 "$TAILSCALE_HOSTNAME" || true - timeout 10 bash -c "cat < /dev/null > /dev/tcp/$host_ip/22" - ssh-keyscan -T 10 -H "$host_ip" >/dev/null - - name: Prepare raindex submodule run: ./prep.sh