From a6445cc8ff2f0a56ef2150adb091b52182705888 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 13:53:36 -0500 Subject: [PATCH 01/26] fix: update npm version for unprivileged container use --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c0a3a2df..c1ddc52c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,9 +39,13 @@ ARG LEGO_VERSION=v4.28.1 RUN curl -fsSL "https://github.com/go-acme/lego/releases/download/${LEGO_VERSION}/lego_${LEGO_VERSION}_linux_amd64.tar.gz" \ | tar -xz -C /usr/local/bin lego +# We install the nodesource repo for newer versions of NPM fixing compatibility +# with unprivileged containers. This sets 24.x up which is the LTS at this time +RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - + # Install requisites: git for updating the software, make and npm for installing # and management. -RUN apt update && apt -y install git make npm +RUN apt update && apt -y install git make nodejs # Install the software. We include the .git directory so that the software can # update itself without replacing the entire container. @@ -81,4 +85,4 @@ EXPOSE 686 # Configure systemd to run properly in a container. This isn't nessary for LXC # in Proxmox, but is useful for testing with Docker directly. STOPSIGNAL SIGRTMIN+3 -ENTRYPOINT [ "/sbin/init" ] \ No newline at end of file +ENTRYPOINT [ "/sbin/init" ] From e37c3a8cbecbde9f0c3e771ce0a8f7c0e981432a Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 14:04:41 -0500 Subject: [PATCH 02/26] fix: build docker on all pushes --- .github/workflows/docker-build-push.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 9db13405..46bb5625 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -2,11 +2,6 @@ name: Build and Push Docker Image on: push: - branches: - - main - pull_request: - branches: - - main env: REGISTRY: ghcr.io From cedcce5c688e83ba1b9c606a57347d561e625281 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 14:17:22 -0500 Subject: [PATCH 03/26] fix: disable nginx-debug service --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c1ddc52c..1c3f9dec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apt update && apt -y install curl gnupg2 ca-certificates lsb-release debian- && cat /etc/apt/preferences.d/99nginx \ && apt update \ && apt install -y nginx ssl-cert \ - && systemctl enable nginx + && echo 'disable nginx-debug.service' >/etc/systemd/system-preset/00-nginx.preset # Install DNSMasq and configure it to only get it's config from our pull-config RUN apt update && apt -y install dnsmasq && systemctl enable dnsmasq From 1a396e8c9f92bce5f228dd6003a26f1c07495510 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 16:09:52 -0500 Subject: [PATCH 04/26] fix: prefer username/password for root@pam privileges Fixes #135 --- create-a-container/models/node.js | 36 +++++++++++++++++++++++ create-a-container/routers/containers.js | 23 ++------------- create-a-container/routers/nodes.js | 25 +++++++--------- create-a-container/views/nodes/form.ejs | 10 +++---- create-a-container/views/nodes/import.ejs | 8 +++-- 5 files changed, 60 insertions(+), 42 deletions(-) diff --git a/create-a-container/models/node.js b/create-a-container/models/node.js index 7181d9d9..3e6052a1 100644 --- a/create-a-container/models/node.js +++ b/create-a-container/models/node.js @@ -2,6 +2,9 @@ const { Model } = require('sequelize'); +const https = require('https'); +const ProxmoxApi = require('../utils/proxmox-api'); + module.exports = (sequelize, DataTypes) => { class Node extends Model { /** @@ -19,6 +22,39 @@ module.exports = (sequelize, DataTypes) => { as: 'site' }); } + + /** + * Create an authenticated ProxmoxApi client for this node. + * Detects whether stored credentials are username/password or API token + * based on presence of '!' in tokenId (Proxmox convention). + * @returns {Promise} Authenticated API client + * @throws {Error} If credentials are missing or authentication fails + */ + async api() { + if (!this.tokenId || !this.secret) { + throw new Error(`Node ${this.name}: Missing credentials (tokenId and secret required)`); + } + + const httpsAgent = new https.Agent({ + rejectUnauthorized: this.tlsVerify !== false + }); + + const isApiToken = this.tokenId.includes('!'); + + if (isApiToken) { + // API token authentication - pass directly to constructor + return new ProxmoxApi(this.apiUrl, this.tokenId, this.secret, { httpsAgent }); + } + + // Username/password authentication - authenticate and return client + const client = new ProxmoxApi(this.apiUrl, null, null, { httpsAgent }); + try { + await client.authenticate(this.tokenId, this.secret); + return client; + } catch (error) { + throw new Error(`Node ${this.name}: Authentication failed - ${error.message}`); + } + } } Node.init({ name: { diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index a03006c5..a6c04f5b 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -32,11 +32,7 @@ router.get('/new', requireAuth, async (req, res) => { // TODO: use datamodel backed templates instead of querying Proxmox here for (const node of nodes) { - const client = new ProxmoxApi(node.apiUrl, node.tokenId, node.secret, { - httpsAgent: new https.Agent({ - rejectUnauthorized: node.tlsVerify !== false - }) - }); + const client = await node.api(); const lxcTemplates = await client.getLxcTemplates(node.name); @@ -212,11 +208,7 @@ router.post('/', async (req, res) => { const { hostname, template, services } = req.body; const [ nodeName, templateVmid ] = template.split(','); const node = await Node.findOne({ where: { name: nodeName, siteId } }); - const client = new ProxmoxApi(node.apiUrl, node.tokenId, node.secret, { - httpsAgent: new https.Agent({ - rejectUnauthorized: node.tlsVerify !== false - }) - }); + const client = await node.api(); const vmid = await client.nextId(); const upid = await client.cloneLxc(node.name, parseInt(templateVmid, 10), vmid, { hostname, @@ -561,16 +553,7 @@ router.delete('/:id', requireAuth, async (req, res) => { // Delete from Proxmox try { - const api = new ProxmoxApi( - node.apiUrl, - node.tokenId, - node.secret, - { - httpsAgent: new https.Agent({ - rejectUnauthorized: node.tlsVerify !== false, - }) - } - ); + const api = await node.api(); await api.deleteContainer(node.name, container.containerId, true, true); } catch (error) { diff --git a/create-a-container/routers/nodes.js b/create-a-container/routers/nodes.js index 710e10fe..e2fd139b 100644 --- a/create-a-container/routers/nodes.js +++ b/create-a-container/routers/nodes.js @@ -149,23 +149,20 @@ router.post('/import', async (req, res) => { const { apiUrl, username, password, tlsVerify } = req.body; const httpsAgent = new https.Agent({ rejectUnauthorized: tlsVerify !== 'false' }); - let tokenId = username.includes('!') ? username : null; - let secret = tokenId ? password : null; + const tokenId = username; + const secret = password; - // create an api token if a username/password was provided + // Create temporary node instance to use api() method for authentication try { - if (!tokenId) { - const client = new ProxmoxApi(apiUrl, null, null, { httpsAgent }); - await client.authenticate(username, password); - const ticketData = await client.createApiToken(username, `import-${Date.now()}`); - tokenId = ticketData['full-tokenid']; - secret = ticketData['value']; - - // set privileges for the created token - await client.updateAcl('/', 'Administrator', null, true, tokenId, null); - } + const tempNode = Node.build({ + name: 'temp', + apiUrl, + tokenId, + secret, + tlsVerify: tlsVerify !== 'false' + }); - const client = new ProxmoxApi(apiUrl, tokenId, secret, { httpsAgent }); + const client = await tempNode.api(); const nodes = await client.nodes(); // Fetch network information for each node to get IP address diff --git a/create-a-container/views/nodes/form.ejs b/create-a-container/views/nodes/form.ejs index 79cad2be..7f70782c 100644 --- a/create-a-container/views/nodes/form.ejs +++ b/create-a-container/views/nodes/form.ejs @@ -59,21 +59,21 @@
- + -
API token identifier (optional)
+
Proxmox username (e.g., root@pam)
- +
- <%= isEdit ? 'Leave blank to keep existing secret. Enter new secret to update.' : 'API token secret (optional)' %> + <%= isEdit ? 'Leave blank to keep existing password. Enter new password to update.' : 'Proxmox password' %>
diff --git a/create-a-container/views/nodes/import.ejs b/create-a-container/views/nodes/import.ejs index d8b550a0..451c7d4a 100644 --- a/create-a-container/views/nodes/import.ejs +++ b/create-a-container/views/nodes/import.ejs @@ -28,19 +28,20 @@
- + +
Proxmox username (e.g., root@pam)
- + +
Proxmox password
From c1d417ba7ac5bae165c55de45b757e8b9a8588d1 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 16:36:16 -0500 Subject: [PATCH 05/26] fix: disable the required auth for search setting for compatibility --- create-a-container/routers/sites.js | 1 + 1 file changed, 1 insertion(+) diff --git a/create-a-container/routers/sites.js b/create-a-container/routers/sites.js index 803b5da4..4c67fd64 100644 --- a/create-a-container/routers/sites.js +++ b/create-a-container/routers/sites.js @@ -115,6 +115,7 @@ router.get('/:siteId/ldap.conf', requireLocalhost, async (req, res) => { // define the environment object const env = { DIRECTORY_BACKEND: 'sql', + REQUIRE_AUTH_FOR_SEARCH: false, }; // Configure AUTH_BACKENDS and NOTIFICATION_URL based on push notification settings From e4acb7c9522cfa241cd373a82410a614d9c20f8d Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 16:47:24 -0500 Subject: [PATCH 06/26] fix: catch database deletion errors --- create-a-container/routers/containers.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index a6c04f5b..eed6a8c3 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -551,20 +551,19 @@ router.delete('/:id', requireAuth, async (req, res) => { return res.redirect(`/sites/${siteId}/containers`); } - // Delete from Proxmox try { + // Delete from Proxmox const api = await node.api(); - await api.deleteContainer(node.name, container.containerId, true, true); + + // Delete from database (cascade deletes associated services) + await container.destroy(); } catch (error) { console.error(error); req.flash('error', `Failed to delete container from Proxmox: ${error.message}`); return res.redirect(`/sites/${siteId}/containers`); } - // Delete from database (cascade deletes associated services) - await container.destroy(); - req.flash('success', `Container ${container.hostname} deleted successfully`); return res.redirect(`/sites/${siteId}/containers`); }); From 9b44d18dd1caf141e0ca9b4a84607fa3eb87a480 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 16:15:06 -0500 Subject: [PATCH 07/26] feat: enable features for docker --- create-a-container/routers/containers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index eed6a8c3..8dd7de71 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -225,7 +225,7 @@ router.post('/', async (req, res) => { // Configure the cloned container await client.updateLxcConfig(node.name, vmid, { cores: 4, - features: 'nesting=1', + features: 'nesting=1,keyctl=1,fuse=1', memory: 4096, net0: 'name=eth0,ip=dhcp,bridge=vmbr0', searchdomain: site.internalDomain, @@ -260,7 +260,7 @@ router.post('/', async (req, res) => { } } console.error('DNS lookup failed after maximum retries'); - return null + return null; })(); const container = await Container.create({ From 2dc6b407b5fb004f6cec1e20990b0842a59391e4 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Mon, 26 Jan 2026 17:04:44 -0500 Subject: [PATCH 08/26] fix: add cascade delete back to services table --- ...60126120000-fix-services-cascade-delete.js | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 create-a-container/migrations/20260126120000-fix-services-cascade-delete.js diff --git a/create-a-container/migrations/20260126120000-fix-services-cascade-delete.js b/create-a-container/migrations/20260126120000-fix-services-cascade-delete.js new file mode 100644 index 00000000..6e998210 --- /dev/null +++ b/create-a-container/migrations/20260126120000-fix-services-cascade-delete.js @@ -0,0 +1,33 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // Fix: Restore ON DELETE CASCADE to Services.containerId foreign key + // This was lost during the 20251202180408-refactor-services-to-sti migration + // when columns were removed + + await queryInterface.changeColumn('Services', 'containerId', { + type: Sequelize.INTEGER, + allowNull: false, + references: { + model: 'Containers', + key: 'id' + }, + onDelete: 'CASCADE' + }); + }, + + async down(queryInterface, Sequelize) { + // This migration fixes a bug, so down migration would recreate the bug + // We'll leave the constraint as-is + await queryInterface.changeColumn('Services', 'containerId', { + type: Sequelize.INTEGER, + allowNull: false, + references: { + model: 'Containers', + key: 'id' + } + }); + } +}; From 771b5c7a3975dca65f445dea8f53e02efbdb0dba Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 08:25:11 -0500 Subject: [PATCH 09/26] refactor: cleanup --- create-a-container/{ => bin}/test-api-key.sh | 0 .../create-container-wrapper.sh | 223 ------------------ create-a-container/example.env | 10 +- 3 files changed, 1 insertion(+), 232 deletions(-) rename create-a-container/{ => bin}/test-api-key.sh (100%) delete mode 100644 create-a-container/create-container-wrapper.sh diff --git a/create-a-container/test-api-key.sh b/create-a-container/bin/test-api-key.sh similarity index 100% rename from create-a-container/test-api-key.sh rename to create-a-container/bin/test-api-key.sh diff --git a/create-a-container/create-container-wrapper.sh b/create-a-container/create-container-wrapper.sh deleted file mode 100644 index 1817f58d..00000000 --- a/create-a-container/create-container-wrapper.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/bin/bash -# Wrapper for non-interactive container creation -# Reads all inputs from environment variables and validates them -# Exits with error messages if invalid/missing - -set -euo pipefail - -GH_ACTION="${GH_ACTION:-}" - -RESET="\033[0m" -BOLD="\033[1m" -MAGENTA='\033[35m' - -outputError() { - echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" - echo -e "$1" - echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - exit 1 -} - -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -echo -e "${BOLD}${MAGENTA}📦 MIE Container Creation Script (Wrapper)${RESET}" -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - -# Required variables, fail if not set or empty -: "${PROXMOX_USERNAME:?Environment variable PROXMOX_USERNAME is required}" -: "${PROXMOX_PASSWORD:?Environment variable PROXMOX_PASSWORD is required}" -: "${CONTAINER_NAME:?Environment variable CONTAINER_NAME is required}" -: "${LINUX_DISTRIBUTION:?Environment variable LINUX_DISTRIBUTION is required}" -: "${HTTP_PORT:?Environment variable HTTP_PORT is required}" -: "${DEPLOY_ON_START:=n}" # default to "n" if not set - -# Convert container name and linux distribution to lowercase -CONTAINER_NAME="${CONTAINER_NAME,,}" -LINUX_DISTRIBUTION="${LINUX_DISTRIBUTION,,}" -DEPLOY_ON_START="${DEPLOY_ON_START,,}" - -# Optional: AI_CONTAINER (default to "N" if not set) -AI_CONTAINER="${AI_CONTAINER:-N}" -AI_CONTAINER="${AI_CONTAINER^^}" # normalize - -# Validate allowed values -if [[ "$AI_CONTAINER" != "N" && "$AI_CONTAINER" != "PHOENIX" && "$AI_CONTAINER" != "FORTWAYNE" ]]; then - outputError "AI_CONTAINER must be one of: N, PHOENIX, FORTWAYNE." -fi - - -# Validate Proxmox credentials using your Node.js authenticateUser -USER_AUTHENTICATED=$(node /root/bin/js/runner.js authenticateUser "$PROXMOX_USERNAME" "$PROXMOX_PASSWORD") -if [ "$USER_AUTHENTICATED" != "true" ]; then - outputError "Invalid Proxmox Credentials." -fi - -echo "🎉 Proxmox user '$PROXMOX_USERNAME' authenticated." - -# Validate container name: alphanumeric + dash only -if ! [[ "$CONTAINER_NAME" =~ ^[a-z0-9-]+$ ]]; then - outputError "Invalid container name: Only lowercase letters, numbers, and dashes are allowed." -fi - -# Check if hostname already exists remotely -HOST_NAME_EXISTS=$(ssh root@10.15.20.69 "node /etc/nginx/checkHostnameRunner.js checkHostnameExists ${CONTAINER_NAME}") -if [ "$HOST_NAME_EXISTS" == "true" ]; then - outputError "Container hostname '$CONTAINER_NAME' already exists." -fi -echo "✅ Container name '$CONTAINER_NAME' is available." - -# Validate Linux distribution choice -if [[ "$LINUX_DISTRIBUTION" != "debian" && "$LINUX_DISTRIBUTION" != "rocky" ]]; then - outputError "Linux distribution must be 'debian' or 'rocky'." -fi - -# Validate HTTP_PORT: integer between 80 and 60000 -if ! [[ "$HTTP_PORT" =~ ^[0-9]+$ ]] || [ "$HTTP_PORT" -lt 80 ] || [ "$HTTP_PORT" -gt 60000 ]; then - outputError "HTTP_PORT must be a number between 80 and 60000." -fi - -echo "✅ HTTP port set to $HTTP_PORT." - -# Public key optional -if [ -n "${PUBLIC_KEY-}" ]; then - # Validate public key format (simple check) - if echo "$PUBLIC_KEY" | ssh-keygen -l -f - &>/dev/null; then - AUTHORIZED_KEYS="/root/.ssh/authorized_keys" - echo "$PUBLIC_KEY" > "$AUTHORIZED_KEYS" - systemctl restart ssh - echo "$PUBLIC_KEY" > "/root/bin/ssh/temp_pubs/key_$(shuf -i 100000-999999 -n1).pub" - sudo /root/bin/ssh/publicKeyAppendJumpHost.sh "$PUBLIC_KEY" - echo "🔐 Public key added." - else - outputError "Invalid PUBLIC_KEY format." - fi -else - echo "ℹ️ No public key provided." -fi - -# Protocol list handling (optional) -PROTOCOL_BASE_FILE="protocol_list_$(shuf -i 100000-999999 -n 1).txt" -PROTOCOL_FILE="/root/bin/protocols/$PROTOCOL_BASE_FILE" -touch "$PROTOCOL_FILE" - -# --- Logic for named protocols from a list (existing) --- -if [[ "${USE_OTHER_PROTOCOLS-}" == "y" || "${USE_OTHER_PROTOCOLS-}" == "Y" ]]; then - if [ -z "${OTHER_PROTOCOLS_LIST-}" ]; then - outputError "USE_OTHER_PROTOCOLS is yes but OTHER_PROTOCOLS_LIST is empty." - fi - IFS=',' read -ra PROTOCOLS <<< "$OTHER_PROTOCOLS_LIST" - for PROTOCOL_NAME in "${PROTOCOLS[@]}"; do - PROTOCOL_NAME=$(echo "$PROTOCOL_NAME" | tr '[:lower:]' '[:upper:]') - FOUND=0 - while read -r line; do - PROTOCOL_ABBRV=$(echo "$line" | awk '{print $1}') - if [[ "$PROTOCOL_ABBRV" == "$PROTOCOL_NAME" ]]; then - echo "$line" >> "$PROTOCOL_FILE" - echo " ^|^e Protocol $PROTOCOL_NAME added." - FOUND=1 - break - fi - done < "/root/bin/protocols/master_protocol_list.txt" - if [ "$FOUND" -eq 0 ]; then - echo " ^}^l Protocol $PROTOCOL_NAME not found, skipping." - fi - done -fi - -# --- START: Added logic for single custom port --- -# Check if the OTHER_PORT variable is set and not empty -if [ -n "${OTHER_PORT-}" ]; then - # Validate that it's an integer - if [[ "$OTHER_PORT" =~ ^[0-9]+$ ]]; then - echo "TCP $OTHER_PORT" >> "$PROTOCOL_FILE" - echo "UDP $OTHER_PORT" >> "$PROTOCOL_FILE" - echo " ^|^e Custom port $OTHER_PORT (TCP/UDP) added." - else - echo " ^}^l Invalid custom port specified: $OTHER_PORT. Must be an integer. Skipping." - fi -fi - -# Deploy on start must be y or n -if [[ "$DEPLOY_ON_START" != "y" && "$DEPLOY_ON_START" != "n" ]]; then - outputError "DEPLOY_ON_START must be 'y' or 'n'." -fi - -if [ "$DEPLOY_ON_START" == "y" ]; then - source /root/bin/deploy-application.sh -fi - -# Send files to hypervisor (public keys, protocols, env vars, services) -send_file_to_hypervisor() { - local LOCAL_FILE="$1" - local REMOTE_FOLDER="$2" - if [ "$REMOTE_FOLDER" != "container-env-vars" ]; then - if [ -s "$LOCAL_FILE" ]; then - sftp root@10.15.0.4 < /dev/null -put $LOCAL_FILE /var/lib/vz/snippets/$REMOTE_FOLDER/ -EOF - fi - else - if [ -d "$LOCAL_FILE" ]; then - sftp root@10.15.0.4 < /dev/null -put -r $LOCAL_FILE /var/lib/vz/snippets/$REMOTE_FOLDER/ -EOF - fi - fi -} - -# Example paths, set or export these in environment if used -send_file_to_hypervisor "/root/bin/ssh/temp_pubs/key_*.pub" "container-public-keys" -send_file_to_hypervisor "$PROTOCOL_FILE" "container-port-maps" -send_file_to_hypervisor "${ENV_FOLDER_PATH:-}" "container-env-vars" -send_file_to_hypervisor "${TEMP_SERVICES_FILE_PATH:-}" "container-services" - -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" -echo -e "${BOLD}${MAGENTA}🚀 Starting Container Creation...${RESET}" -echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - -# Safely get the basename of the temporary public key file. -KEY_BASENAME="" -# The 'find' command is safer than 'ls' for script usage. -KEY_FILE=$(find /root/bin/ssh/temp_pubs -type f -name "*.pub" | head -n1) - -if [[ -n "$KEY_FILE" ]]; then - KEY_BASENAME=$(basename "$KEY_FILE") -fi - -# Run your create-container.sh remotely over SSH with corrected quoting and simplified variable -ssh -t root@10.15.0.4 "bash -c \"/var/lib/vz/snippets/create-container-new.sh \ - '$CONTAINER_NAME' \ - '$GH_ACTION' \ - '$HTTP_PORT' \ - '$PROXMOX_USERNAME' \ - '$KEY_BASENAME' \ - '$PROTOCOL_BASE_FILE' \ - '$DEPLOY_ON_START' \ - '${PROJECT_REPOSITORY:-}' \ - '${PROJECT_BRANCH:-}' \ - '${PROJECT_ROOT:-}' \ - '${INSTALL_COMMAND:-}' \ - '${BUILD_COMMAND:-}' \ - '${START_COMMAND:-}' \ - '${RUNTIME_LANGUAGE:-}' \ - '${ENV_FOLDER:-}' \ - '${SERVICES_FILE:-}' \ - '$LINUX_DISTRIBUTION' \ - '${MULTI_COMPONENT:-}' \ - '${ROOT_START_COMMAND:-}' \ - '${SELF_HOSTED_RUNNER:-}' \ - '${VERSIONS_DICT:-}' \ - '$AI_CONTAINER' # Corrected: Pass the variable's value in the correct position -\"" - -# Clean up temp files -rm -f "$PROTOCOL_FILE" -rm -f /root/bin/ssh/temp_pubs/key_*.pub -rm -f "${TEMP_SERVICES_FILE_PATH:-}" -rm -rf "${ENV_FOLDER_PATH:-}" - -# Unset sensitive variables -unset CONFIRM_PASSWORD -unset PUBLIC_KEY - -echo "✅ Container creation wrapper script finished successfully." \ No newline at end of file diff --git a/create-a-container/example.env b/create-a-container/example.env index 45aa572f..c403cc30 100644 --- a/create-a-container/example.env +++ b/create-a-container/example.env @@ -1,6 +1,3 @@ -# secret used to compute express-session hash, generate randomly -SESSION_SECRET= - # dialect used by sequelize (mysql, sqlite, postgres) # you also need to configure the relevant variables for the selected dialect DATABASE_DIALECT= @@ -20,9 +17,4 @@ POSTGRES_HOST= POSTGRES_PORT= POSTGRES_USER= POSTGRES_PASSWORD= -POSTGRES_DATABASE= - -# Only used for bin/json-to-sql.js (for now) -PROXMOX_URL= -PROXMOX_USER= -PROXMOX_PASSWORD= \ No newline at end of file +POSTGRES_DATABASE= \ No newline at end of file From 13ce272a5764af43b9fb7faa4352e782433a4b37 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 08:39:13 -0500 Subject: [PATCH 10/26] refactor: prefer postgres in development over sqlite --- .github/workflows/docker-build-push.yml | 6 -- Dockerfile | 8 +- create-a-container/compose.yml | 15 ++++ .../docs/developers/core-technologies.md | 14 +++ .../docs/developers/development-workflow.md | 88 ++++++++++++++++++- 5 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 create-a-container/compose.yml diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 46bb5625..66087122 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -29,10 +29,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract branch name - id: branch - run: echo "name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - - name: Extract metadata id: meta uses: docker/metadata-action@v5 @@ -49,7 +45,5 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-args: | - OPENSOURCE_SERVER_BRANCH=${{ steps.branch.outputs.name }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 1c3f9dec..32e6c6b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,12 +49,8 @@ RUN apt update && apt -y install git make nodejs # Install the software. We include the .git directory so that the software can # update itself without replacing the entire container. -ARG OPENSOURCE_SERVER_BRANCH=main -RUN git clone \ - --branch=${OPENSOURCE_SERVER_BRANCH} \ - https://github.com/mieweb/opensource-server.git \ - /opt/opensource-server \ - && cd /opt/opensource-server \ +COPY . /opt/opensource-server +RUN cd /opt/opensource-server \ && make install # Install the ldap-gateway package diff --git a/create-a-container/compose.yml b/create-a-container/compose.yml new file mode 100644 index 00000000..d3a6a344 --- /dev/null +++ b/create-a-container/compose.yml @@ -0,0 +1,15 @@ +--- +services: + postgres: + image: postgres:18 + volumes: + - postgres-data:/var/lib/postgresql + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Missing password} + POSTGRES_USER: ${POSTGRES_USER:?Missing user} + POSTGRES_DB: ${POSTGRES_DATABASE:?Missing database} + ports: + - 5432:5432 + +volumes: + postgres-data: \ No newline at end of file diff --git a/mie-opensource-landing/docs/developers/core-technologies.md b/mie-opensource-landing/docs/developers/core-technologies.md index 6746b45a..d9503a6c 100644 --- a/mie-opensource-landing/docs/developers/core-technologies.md +++ b/mie-opensource-landing/docs/developers/core-technologies.md @@ -127,6 +127,20 @@ The MIE Opensource Proxmox Cluster is built on several key open-source technolog - **Official Documentation**: [docs.npmjs.com](https://docs.npmjs.com/) - **CLI Commands**: [npm CLI](https://docs.npmjs.com/cli/v10/commands) +### Docker & Docker Compose + +**Docker** provides containerization for development and deployment, while **Docker Compose** orchestrates multi-container applications. + +- **Docker Documentation**: [docs.docker.com](https://docs.docker.com/) +- **Docker Compose Documentation**: [docs.docker.com/compose](https://docs.docker.com/compose/) +- **Dockerfile Reference**: [Dockerfile Reference](https://docs.docker.com/reference/dockerfile/) + +**Used For:** +- Building container images for deployment +- Local development environment (PostgreSQL via compose.yml) +- CI/CD pipeline image building +- Testing in isolated environments + ## Related Resources - [System Architecture](system-architecture): Understand how these technologies work together diff --git a/mie-opensource-landing/docs/developers/development-workflow.md b/mie-opensource-landing/docs/developers/development-workflow.md index 4ce050a4..cd5c6528 100644 --- a/mie-opensource-landing/docs/developers/development-workflow.md +++ b/mie-opensource-landing/docs/developers/development-workflow.md @@ -17,6 +17,42 @@ To contribute to the cluster management software: ## Local Development Setup +### Option 1: Using Docker Compose (Recommended) + +The simplest way to develop is using the included Docker Compose setup which provides a PostgreSQL database: + +```bash +# Clone the repository +git clone https://github.com/mieweb/opensource-server +cd opensource-server/create-a-container + +# Configure environment +cp example.env .env +# Edit .env with your Proxmox settings and database configuration: +# DATABASE_DIALECT=postgres +# POSTGRES_HOST=localhost +# POSTGRES_PORT=5432 +# POSTGRES_USER=your_user +# POSTGRES_PASSWORD=your_password +# POSTGRES_DATABASE=your_db + +# Start PostgreSQL +docker compose up -d + +# Install dependencies +npm install + +# Run database migrations +npm run db:migrate + +# Start the development server +npm run dev +``` + +### Option 2: Manual Setup + +For development without Docker: + ```bash # Clone the repository git clone https://github.com/mieweb/opensource-server @@ -28,9 +64,11 @@ npm install # Configure environment cp example.env .env # Edit .env with your Proxmox and database settings +# For SQLite (default): no additional database setup required +# For PostgreSQL/MySQL: ensure database server is running # Run database migrations -npx sequelize-cli db:migrate +npm run db:migrate # Start the development server npm run dev @@ -99,6 +137,45 @@ Before submitting changes: ## Debugging +### Local Docker Image Build + +You can build and test the Docker image locally before deploying: + +```bash +# Build the Docker image from the repository root +docker build -t opensource-server:dev . + +# Run the container (requires systemd support) +docker run -d --privileged \ + --name opensource-test \ + -p 80:80 -p 443:443 -p 53:53/udp \ + opensource-server:dev + +# View container logs +docker logs -f opensource-test + +# Access a shell in the container +docker exec -it opensource-test bash + +# Stop and remove the test container +docker stop opensource-test && docker rm opensource-test +``` + +**Note:** The Dockerfile copies your local repository code (including uncommitted changes), making it ideal for testing changes before pushing to GitHub. + +### CI/CD Pipeline + +The project uses GitHub Actions to automatically build and push Docker images: + +- **Trigger**: On every push to any branch +- **Registry**: GitHub Container Registry (ghcr.io) +- **Tags**: + - Branch name (e.g., `sprint`, `main`) + - `latest` tag for main branch only +- **Build optimization**: Uses GitHub Actions cache for faster builds + +The workflow file is located at `.github/workflows/docker-build-push.yml`. + ### API Server ```bash @@ -125,8 +202,15 @@ DB_LOGGING=true **Database Connection Errors:** - Verify database credentials in `.env` -- Ensure database service is running +- Ensure database service is running (for Docker: `docker compose ps`) - Check network connectivity +- For PostgreSQL via Docker Compose: ensure ports are not in use + +**Docker Compose Issues:** +- Check container status: `docker compose ps` +- View logs: `docker compose logs postgres` +- Restart services: `docker compose restart` +- Clean start: `docker compose down -v && docker compose up -d` **Proxmox API Errors:** - Verify API credentials are correct From 02c6c1b66e3af297052074f223af2af6c2d2bb1c Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 12:08:21 -0500 Subject: [PATCH 11/26] fix: postgres fixes in migrations --- ...93722-convert-container-node-to-node-id.js | 18 +-- ...20251202180408-refactor-services-to-sti.js | 127 +++++++++++++----- .../20251202201123-add-dns-services.js | 50 +++++-- 3 files changed, 145 insertions(+), 50 deletions(-) diff --git a/create-a-container/migrations/20251104193722-convert-container-node-to-node-id.js b/create-a-container/migrations/20251104193722-convert-container-node-to-node-id.js index 078aba06..17963119 100644 --- a/create-a-container/migrations/20251104193722-convert-container-node-to-node-id.js +++ b/create-a-container/migrations/20251104193722-convert-container-node-to-node-id.js @@ -60,17 +60,13 @@ module.exports = { allowNull: true }); - // Populate node from nodeId - const [nodes, _] = await queryInterface.sequelize.query( - 'SELECT id, name FROM Nodes' - ); - for (const { id, name } of nodes) { - await queryInterface.bulkUpdate('Containers', { - node: name - }, { - nodeId: id - }); - } + // Populate node from nodeId using a LEFT JOIN to handle case where Nodes table might not exist or is empty + await queryInterface.sequelize.query(` + UPDATE "Containers" c + SET node = n.name + FROM "Nodes" n + WHERE c."nodeId" = n.id + `); // Make node NOT NULL await queryInterface.changeColumn('Containers', 'node', { diff --git a/create-a-container/migrations/20251202180408-refactor-services-to-sti.js b/create-a-container/migrations/20251202180408-refactor-services-to-sti.js index 0bd4d5a7..ebdada03 100644 --- a/create-a-container/migrations/20251202180408-refactor-services-to-sti.js +++ b/create-a-container/migrations/20251202180408-refactor-services-to-sti.js @@ -126,15 +126,48 @@ module.exports = { await queryInterface.removeColumn('Services', 'externalDomainId'); // rename tcp and udp service types to transport - await queryInterface.changeColumn('Services', 'type', { - type: Sequelize.ENUM('http', 'transport', 'tcp', 'udp'), - allowNull: false - }); - await queryInterface.bulkUpdate('Services', { type: 'transport' }, { [Sequelize.Op.or]: [ { type: 'tcp' }, { type: 'udp' } ] }); - await queryInterface.changeColumn('Services', 'type', { - type: Sequelize.ENUM('http', 'transport'), - allowNull: false - }); + // For PostgreSQL, we need to handle ENUM modification differently + const dialect = queryInterface.sequelize.getDialect(); + + if (dialect === 'postgres') { + // Rename the existing enum to a backup name + await queryInterface.sequelize.query('ALTER TYPE "enum_Services_type" RENAME TO "enum_Services_type_old"'); + + // Create new enum with transport added + await queryInterface.sequelize.query("CREATE TYPE \"enum_Services_type\" AS ENUM ('http', 'transport', 'tcp', 'udp')"); + + // Update the column to use the new enum + await queryInterface.sequelize.query('ALTER TABLE "Services" ALTER COLUMN "type" TYPE "enum_Services_type" USING "type"::text::"enum_Services_type"'); + + // Update tcp and udp to transport + await queryInterface.bulkUpdate('Services', { type: 'transport' }, { [Sequelize.Op.or]: [ { type: 'tcp' }, { type: 'udp' } ] }); + + // Drop old enum + await queryInterface.sequelize.query('DROP TYPE "enum_Services_type_old"'); + + // Rename enum again to update it to final values + await queryInterface.sequelize.query('ALTER TYPE "enum_Services_type" RENAME TO "enum_Services_type_old"'); + + // Create final enum with only http and transport + await queryInterface.sequelize.query("CREATE TYPE \"enum_Services_type\" AS ENUM ('http', 'transport')"); + + // Update the column to use the final enum + await queryInterface.sequelize.query('ALTER TABLE "Services" ALTER COLUMN "type" TYPE "enum_Services_type" USING "type"::text::"enum_Services_type"'); + + // Drop old enum + await queryInterface.sequelize.query('DROP TYPE "enum_Services_type_old"'); + } else { + // SQLite and other databases + await queryInterface.changeColumn('Services', 'type', { + type: Sequelize.ENUM('http', 'transport', 'tcp', 'udp'), + allowNull: false + }); + await queryInterface.bulkUpdate('Services', { type: 'transport' }, { [Sequelize.Op.or]: [ { type: 'tcp' }, { type: 'udp' } ] }); + await queryInterface.changeColumn('Services', 'type', { + type: Sequelize.ENUM('http', 'transport'), + allowNull: false + }); + } // insert migrated data into new tables AFTER schema changes because of how sqlite3 handles cascades if (httpServices.length > 0) @@ -144,21 +177,7 @@ module.exports = { }, async down (queryInterface, Sequelize) { - // Recreate old indexes on Services table - await queryInterface.addIndex('Services', ['externalHostname', 'externalDomainId'], { - unique: true, - name: 'services_http_unique_hostname_domain', - where: { - type: 'http' - } - }); - - await queryInterface.addIndex('Services', ['type', 'externalPort'], { - unique: true, - name: 'services_layer4_unique_port' - }); - - // Add columns back to Services table + // Add columns back to Services table first await queryInterface.addColumn('Services', 'externalHostname', { type: Sequelize.STRING(255), allowNull: true @@ -184,10 +203,27 @@ module.exports = { }); // Change type enum back to include tcp and udp - await queryInterface.changeColumn('Services', 'type', { - type: Sequelize.ENUM('http', 'transport', 'tcp', 'udp'), - allowNull: false - }); + const dialect = queryInterface.sequelize.getDialect(); + + if (dialect === 'postgres') { + // Rename the existing enum + await queryInterface.sequelize.query('ALTER TYPE "enum_Services_type" RENAME TO "enum_Services_type_old"'); + + // Create new enum with tcp, udp, and transport + await queryInterface.sequelize.query("CREATE TYPE \"enum_Services_type\" AS ENUM ('http', 'transport', 'tcp', 'udp')"); + + // Update the column to use the new enum + await queryInterface.sequelize.query('ALTER TABLE "Services" ALTER COLUMN "type" TYPE "enum_Services_type" USING "type"::text::"enum_Services_type"'); + + // Drop old enum + await queryInterface.sequelize.query('DROP TYPE "enum_Services_type_old"'); + } else { + // SQLite and other databases + await queryInterface.changeColumn('Services', 'type', { + type: Sequelize.ENUM('http', 'transport', 'tcp', 'udp'), + allowNull: false + }); + } // Migrate data back from child tables const servicesTable = queryInterface.quoteIdentifier('Services'); @@ -218,9 +254,38 @@ module.exports = { } // Remove transport from enum, leaving only http, tcp, udp - await queryInterface.changeColumn('Services', 'type', { - type: Sequelize.ENUM('http', 'tcp', 'udp'), - allowNull: false + if (dialect === 'postgres') { + // Rename the existing enum + await queryInterface.sequelize.query('ALTER TYPE "enum_Services_type" RENAME TO "enum_Services_type_old"'); + + // Create new enum with only http, tcp, udp + await queryInterface.sequelize.query("CREATE TYPE \"enum_Services_type\" AS ENUM ('http', 'tcp', 'udp')"); + + // Update the column to use the new enum + await queryInterface.sequelize.query('ALTER TABLE "Services" ALTER COLUMN "type" TYPE "enum_Services_type" USING "type"::text::"enum_Services_type"'); + + // Drop old enum + await queryInterface.sequelize.query('DROP TYPE "enum_Services_type_old"'); + } else { + // SQLite and other databases + await queryInterface.changeColumn('Services', 'type', { + type: Sequelize.ENUM('http', 'tcp', 'udp'), + allowNull: false + }); + } + + // Recreate old indexes on Services table + await queryInterface.addIndex('Services', ['externalHostname', 'externalDomainId'], { + unique: true, + name: 'services_http_unique_hostname_domain', + where: { + type: 'http' + } + }); + + await queryInterface.addIndex('Services', ['type', 'externalPort'], { + unique: true, + name: 'services_layer4_unique_port' }); // Drop child tables diff --git a/create-a-container/migrations/20251202201123-add-dns-services.js b/create-a-container/migrations/20251202201123-add-dns-services.js index 4f1964d3..8ca9da61 100644 --- a/create-a-container/migrations/20251202201123-add-dns-services.js +++ b/create-a-container/migrations/20251202201123-add-dns-services.js @@ -4,10 +4,27 @@ module.exports = { async up (queryInterface, Sequelize) { // Add 'dns' to Service type enum - await queryInterface.changeColumn('Services', 'type', { - type: Sequelize.ENUM('http', 'transport', 'dns'), - allowNull: false - }); + const dialect = queryInterface.sequelize.getDialect(); + + if (dialect === 'postgres') { + // Rename the existing enum + await queryInterface.sequelize.query('ALTER TYPE "enum_Services_type" RENAME TO "enum_Services_type_old"'); + + // Create new enum with dns added + await queryInterface.sequelize.query("CREATE TYPE \"enum_Services_type\" AS ENUM ('http', 'transport', 'dns')"); + + // Update the column to use the new enum + await queryInterface.sequelize.query('ALTER TABLE "Services" ALTER COLUMN "type" TYPE "enum_Services_type" USING "type"::text::"enum_Services_type"'); + + // Drop old enum + await queryInterface.sequelize.query('DROP TYPE "enum_Services_type_old"'); + } else { + // SQLite and other databases + await queryInterface.changeColumn('Services', 'type', { + type: Sequelize.ENUM('http', 'transport', 'dns'), + allowNull: false + }); + } // Create DnsServices table await queryInterface.createTable('DnsServices', { @@ -51,9 +68,26 @@ module.exports = { await queryInterface.dropTable('DnsServices'); // Remove 'dns' from Service type enum - await queryInterface.changeColumn('Services', 'type', { - type: Sequelize.ENUM('http', 'transport'), - allowNull: false - }); + const dialect = queryInterface.sequelize.getDialect(); + + if (dialect === 'postgres') { + // Rename the existing enum + await queryInterface.sequelize.query('ALTER TYPE "enum_Services_type" RENAME TO "enum_Services_type_old"'); + + // Create new enum without dns + await queryInterface.sequelize.query("CREATE TYPE \"enum_Services_type\" AS ENUM ('http', 'transport')"); + + // Update the column to use the new enum + await queryInterface.sequelize.query('ALTER TABLE "Services" ALTER COLUMN "type" TYPE "enum_Services_type" USING "type"::text::"enum_Services_type"'); + + // Drop old enum + await queryInterface.sequelize.query('DROP TYPE "enum_Services_type_old"'); + } else { + // SQLite and other databases + await queryInterface.changeColumn('Services', 'type', { + type: Sequelize.ENUM('http', 'transport'), + allowNull: false + }); + } } }; From e85e233f4b8072bf1a86d787a6a4d16648ea6eaa Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 12:08:38 -0500 Subject: [PATCH 12/26] feat: install postgres into docker appliance --- .dockerignore | 5 +++- Dockerfile | 19 ++++++++------ Makefile | 12 +++++---- .../systemd/container-creator-init.service | 25 +++++++++++++++++++ .../systemd/container-creator.service | 3 ++- 5 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 create-a-container/systemd/container-creator-init.service diff --git a/.dockerignore b/.dockerignore index 6c8e6fb5..2c97c009 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,5 @@ /create-a-container/data/database.sqlite -/create-a-container/certs/* \ No newline at end of file +/create-a-container/certs/* +/create-a-container/.envcompose.override.yml +/mie-opensource-landing/build +*/node_modules diff --git a/Dockerfile b/Dockerfile index 32e6c6b3..7920f89b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,19 +39,18 @@ ARG LEGO_VERSION=v4.28.1 RUN curl -fsSL "https://github.com/go-acme/lego/releases/download/${LEGO_VERSION}/lego_${LEGO_VERSION}_linux_amd64.tar.gz" \ | tar -xz -C /usr/local/bin lego +# Install Postgres 18 from the PGDG repository +RUN apt update && apt -y install postgresql-common \ + && /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y \ + && apt -y install postgresql-18 + # We install the nodesource repo for newer versions of NPM fixing compatibility # with unprivileged containers. This sets 24.x up which is the LTS at this time RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - # Install requisites: git for updating the software, make and npm for installing # and management. -RUN apt update && apt -y install git make nodejs - -# Install the software. We include the .git directory so that the software can -# update itself without replacing the entire container. -COPY . /opt/opensource-server -RUN cd /opt/opensource-server \ - && make install +RUN apt update && apt -y install git make nodejs sudo # Install the ldap-gateway package ARG LDAP_GATEWAY_BRANCH=main @@ -67,6 +66,12 @@ RUN git clone \ && cp /opt/ldap-gateway/nfpm/systemd/ldap-gateway.service /etc/systemd/system/ldap-gateway.service \ && systemctl enable ldap-gateway +# Install the software. We include the .git directory so that the software can +# update itself without replacing the entire container. +COPY . /opt/opensource-server +RUN cd /opt/opensource-server \ + && make install + # Tag the exposed ports for services handled by the container # NGINX (http, https, quic) EXPOSE 80 diff --git a/Makefile b/Makefile index 376c0564..be8b61e2 100644 --- a/Makefile +++ b/Makefile @@ -12,13 +12,15 @@ help: install: install-create-container install-pull-config install-docs +SYSTEMD_DIR := create-a-container/systemd +SERVICES := $(wildcard $(SYSTEMD_DIR)/*.service) install-create-container: - cd create-a-container && npm install --production - cd create-a-container && npm run db:migrate - install -m644 -oroot -groot create-a-container/systemd/container-creator.service /etc/systemd/system/container-creator.service + cd create-a-container && npm install --omit=dev + install -m 644 -o root -g root $(SERVICES) /etc/systemd/system/ systemctl daemon-reload || true - systemctl enable container-creator.service - systemctl start container-creator.service || true + @for service in $(notdir $(SERVICES)); do \ + systemctl enable $$service; \ + done install-pull-config: cd pull-config && bash install.sh diff --git a/create-a-container/systemd/container-creator-init.service b/create-a-container/systemd/container-creator-init.service new file mode 100644 index 00000000..10d5112a --- /dev/null +++ b/create-a-container/systemd/container-creator-init.service @@ -0,0 +1,25 @@ +[Unit] +Description=Initialize PostgreSQL for Container Creator +After=postgresql.service +ConditionPathExists=!/opt/opensource-server/create-a-container/.env + +[Service] +Type=oneshot +WorkingDirectory=/opt/opensource-server/create-a-container +ExecStart=/bin/bash -c '\ + for i in {1..30}; do pg_isready -h localhost && break || sleep 1; done; \ + POSTGRES_PASSWORD=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 24); \ + POSTGRES_USER="cluster_manager"; \ + POSTGRES_DATABASE="cluster_manager"; \ + POSTGRES_HOST="localhost"; \ + sudo -u postgres psql -c "CREATE USER $${POSTGRES_USER} WITH PASSWORD '"'"'$${POSTGRES_PASSWORD}'"'"';"; \ + sudo -u postgres psql -c "CREATE DATABASE $${POSTGRES_DATABASE} OWNER $${POSTGRES_USER};"; \ + echo "DATABASE_DIALECT=postgres" > .env; \ + echo "POSTGRES_HOST=$${POSTGRES_HOST}" >> .env; \ + echo "POSTGRES_DATABASE=$${POSTGRES_DATABASE}" >> .env; \ + echo "POSTGRES_USER=$${POSTGRES_USER}" >> .env; \ + echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> .env; \ + npm run db:migrate;' + +[Install] +WantedBy=multi-user.target diff --git a/create-a-container/systemd/container-creator.service b/create-a-container/systemd/container-creator.service index cd734568..27ab76ba 100644 --- a/create-a-container/systemd/container-creator.service +++ b/create-a-container/systemd/container-creator.service @@ -1,6 +1,7 @@ [Unit] Description=Container Creator Node.js App -After=network.target +After=container-creator-init.service +Wants=container-creator-init.service [Service] Type=simple From 5c9b7faa1b907b2f1845ef4518a3958d22536a96 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 13:08:05 -0500 Subject: [PATCH 13/26] fix: dumb --- create-a-container/routers/sites.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create-a-container/routers/sites.js b/create-a-container/routers/sites.js index 4c67fd64..a32b0666 100644 --- a/create-a-container/routers/sites.js +++ b/create-a-container/routers/sites.js @@ -150,9 +150,9 @@ router.get('/:siteId/ldap.conf', requireLocalhost, async (req, res) => { // config/config.js and construct the SQL URL const config = require('../config/config')[process.env.NODE_ENV || 'development']; const sqlUrlBuilder = new URL(`${config.dialect}://`); + sqlUrlBuilder.hostname = config.host || ''; sqlUrlBuilder.username = config.username || ''; sqlUrlBuilder.password = config.password || ''; - sqlUrlBuilder.hostname = config.host || ''; sqlUrlBuilder.port = config.port || ''; sqlUrlBuilder.pathname = config.database || path.resolve(config.storage); env.SQL_URI = sqlUrlBuilder.toString(); From be065667206e889d0169bac73fe7861f10c30b50 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 13:53:15 -0500 Subject: [PATCH 14/26] feat: async container creation --- create-a-container/README.md | 28 +- create-a-container/bin/create-container.js | 298 ++++++++++++++++++ create-a-container/bin/json-to-sql.js | 11 +- .../20260127180000-add-container-status.js | 15 + ...127180001-add-container-creation-job-id.js | 20 ++ .../20260127180002-add-container-template.js | 14 + ...60127180003-remove-container-os-release.js | 14 + ...-make-container-network-fields-nullable.js | 29 ++ create-a-container/models/container.js | 21 +- .../models/transport-service.js | 13 +- create-a-container/routers/containers.js | 295 ++++++++--------- create-a-container/routers/sites.js | 6 +- create-a-container/utils/proxmox-api.js | 14 + create-a-container/views/containers/index.ejs | 35 +- .../docs/developers/database-schema.md | 23 +- 15 files changed, 646 insertions(+), 190 deletions(-) create mode 100755 create-a-container/bin/create-container.js create mode 100644 create-a-container/migrations/20260127180000-add-container-status.js create mode 100644 create-a-container/migrations/20260127180001-add-container-creation-job-id.js create mode 100644 create-a-container/migrations/20260127180002-add-container-template.js create mode 100644 create-a-container/migrations/20260127180003-remove-container-os-release.js create mode 100644 create-a-container/migrations/20260127180004-make-container-network-fields-nullable.js diff --git a/create-a-container/README.md b/create-a-container/README.md index a89352e1..927f8797 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -22,11 +22,13 @@ erDiagram int id PK string hostname UK "FQDN hostname" string username "Owner username" - string osRelease "OS distribution" + string status "pending,creating,running,failed" + string template "Template name" + int creationJobId FK "References Job" int nodeId FK "References Node" int containerId UK "Proxmox VMID" - string macAddress UK "MAC address" - string ipv4Address UK "IPv4 address" + string macAddress UK "MAC address (nullable)" + string ipv4Address UK "IPv4 address (nullable)" string aiContainer "Node type flag" datetime createdAt datetime updatedAt @@ -206,12 +208,13 @@ List all containers for authenticated user Display container creation form #### `POST /containers` -Create or register a container -- **Query Parameter**: `init` (boolean) - If true, requires auth and spawns container creation -- **Body (init=true)**: `{ hostname, osRelease, httpPort, aiContainer }` -- **Body (init=false)**: Container registration data (for scripts) -- **Returns (init=true)**: Redirect to status page -- **Returns (init=false)**: `{ containerId, message }` +Create a container asynchronously via a background job +- **Body**: `{ hostname, template, services }` where: + - `hostname`: Container hostname + - `template`: Template selection in format "nodeName,vmid" + - `services`: Object of service definitions +- **Returns**: Redirect to containers list with flash message +- **Process**: Creates pending container, services, and job in a single transaction. The job-runner executes the actual Proxmox operations. #### `DELETE /containers/:id` (Auth Required) Delete a container from both Proxmox and the database @@ -430,8 +433,11 @@ Test email configuration (development/testing) id INT PRIMARY KEY AUTO_INCREMENT hostname VARCHAR(255) UNIQUE NOT NULL username VARCHAR(255) NOT NULL -osRelease VARCHAR(255) -containerId INT UNSIGNED UNIQUE +status VARCHAR(20) NOT NULL DEFAULT 'pending' +template VARCHAR(255) +creationJobId INT FOREIGN KEY REFERENCES Jobs(id) +nodeId INT FOREIGN KEY REFERENCES Nodes(id) +containerId INT UNSIGNED NOT NULL macAddress VARCHAR(17) UNIQUE ipv4Address VARCHAR(45) UNIQUE aiContainer VARCHAR(50) DEFAULT 'N' diff --git a/create-a-container/bin/create-container.js b/create-a-container/bin/create-container.js new file mode 100755 index 00000000..9c158526 --- /dev/null +++ b/create-a-container/bin/create-container.js @@ -0,0 +1,298 @@ +#!/usr/bin/env node +/** + * create-container.js + * + * Background job script that performs the actual Proxmox container creation. + * This script is executed by the job-runner after a pending container record + * has been created in the database. + * + * Usage: node bin/create-container.js --container-id= + * + * The script will: + * 1. Load the container record from the database + * 2. Clone the template in Proxmox + * 3. Configure the container (cores, memory, network) + * 4. Start the container + * 5. Query MAC address from Proxmox config + * 6. Query IP address from Proxmox interfaces API + * 7. Update the container record with MAC, IP, and status='running' + * + * All output is logged to STDOUT for capture by the job-runner. + * Exit code 0 = success, non-zero = failure. + */ + +const path = require('path'); + +// Load models from parent directory +const db = require(path.join(__dirname, '..', 'models')); +const { Container, Node, Site } = db; + +/** + * Parse command line arguments + * @returns {object} Parsed arguments + */ +function parseArgs() { + const args = {}; + for (const arg of process.argv.slice(2)) { + const match = arg.match(/^--([^=]+)=(.+)$/); + if (match) { + args[match[1]] = match[2]; + } + } + return args; +} + +/** + * Wait for a Proxmox task to complete + * @param {ProxmoxApi} client - The Proxmox API client + * @param {string} nodeName - The node name + * @param {string} upid - The task UPID + * @param {number} pollInterval - Polling interval in ms (default 2000) + * @param {number} timeout - Timeout in ms (default 300000 = 5 minutes) + * @returns {Promise} The final task status + */ +async function waitForTask(client, nodeName, upid, pollInterval = 2000, timeout = 300000) { + const startTime = Date.now(); + while (true) { + const status = await client.taskStatus(nodeName, upid); + console.log(`Task ${upid}: status=${status.status}, exitstatus=${status.exitstatus || 'N/A'}`); + + if (status.status === 'stopped') { + if (status.exitstatus && status.exitstatus !== 'OK') { + throw new Error(`Task failed with status: ${status.exitstatus}`); + } + return status; + } + + if (Date.now() - startTime > timeout) { + throw new Error(`Task ${upid} timed out after ${timeout}ms`); + } + + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } +} + +/** + * Query IP address from Proxmox interfaces API with retries + * @param {ProxmoxApi} client - The Proxmox API client + * @param {string} nodeName - The node name + * @param {number} vmid - The container VMID + * @param {number} maxRetries - Maximum number of retries + * @param {number} retryDelay - Delay between retries in ms + * @returns {Promise} The IPv4 address or null if not found + */ +async function getIpFromInterfaces(client, nodeName, vmid, maxRetries = 10, retryDelay = 3000) { + console.log(`Querying IP address from Proxmox interfaces API...`); + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const interfaces = await client.lxcInterfaces(nodeName, vmid); + + // Find eth0 interface and get its IPv4 address + const eth0 = interfaces.find(iface => iface.name === 'eth0'); + if (eth0 && eth0['ip-addresses']) { + const ipv4 = eth0['ip-addresses'].find(addr => addr['ip-address-type'] === 'inet'); + if (ipv4 && ipv4['ip-address']) { + console.log(`IP address found (attempt ${attempt}): ${ipv4['ip-address']}`); + return ipv4['ip-address']; + } + } + + // Also check the 'inet' field as fallback + if (eth0 && eth0.inet) { + const ip = eth0.inet.split('/')[0]; + console.log(`IP address found from inet field (attempt ${attempt}): ${ip}`); + return ip; + } + + console.log(`IP address not yet available (attempt ${attempt}/${maxRetries})`); + if (attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, retryDelay)); + } + } catch (err) { + console.log(`Interfaces query attempt ${attempt}/${maxRetries} failed: ${err.message}`); + if (attempt < maxRetries) { + await new Promise(resolve => setTimeout(resolve, retryDelay)); + } + } + } + + console.error(`Failed to get IP address after ${maxRetries} attempts`); + return null; +} + +/** + * Main function + */ +async function main() { + const args = parseArgs(); + + if (!args['container-id']) { + console.error('Usage: node create-container.js --container-id='); + process.exit(1); + } + + const containerId = parseInt(args['container-id'], 10); + console.log(`Starting container creation for container ID: ${containerId}`); + + // Load the container record with its node and site + const container = await Container.findByPk(containerId, { + include: [{ + model: Node, + as: 'node', + include: [{ + model: Site, + as: 'site' + }] + }] + }); + + if (!container) { + console.error(`Container with ID ${containerId} not found`); + process.exit(1); + } + + if (container.status !== 'pending') { + console.error(`Container is not in pending status (current: ${container.status})`); + process.exit(1); + } + + const node = container.node; + const site = node.site; + + if (!node) { + console.error('Container has no associated node'); + process.exit(1); + } + + if (!site) { + console.error('Node has no associated site'); + process.exit(1); + } + + console.log(`Container: ${container.hostname}`); + console.log(`Node: ${node.name}`); + console.log(`Site: ${site.name} (${site.internalDomain})`); + console.log(`Template: ${container.template}`); + console.log(`Target VMID: ${container.containerId}`); + + try { + // Update status to 'creating' + await container.update({ status: 'creating' }); + console.log('Status updated to: creating'); + + // Get the Proxmox API client + const client = await node.api(); + console.log('Proxmox API client initialized'); + + // Find the template VMID by matching the template name + console.log(`Looking for template: ${container.template}`); + const templates = await client.getLxcTemplates(node.name); + const templateContainer = templates.find(t => t.name === container.template); + + if (!templateContainer) { + throw new Error(`Template "${container.template}" not found on node ${node.name}`); + } + + const templateVmid = templateContainer.vmid; + console.log(`Found template VMID: ${templateVmid}`); + + // Clone the template + console.log(`Cloning template ${templateVmid} to VMID ${container.containerId}...`); + const cloneUpid = await client.cloneLxc(node.name, templateVmid, container.containerId, { + hostname: container.hostname, + description: `Cloned from template ${container.template}`, + full: 1 + }); + console.log(`Clone task started: ${cloneUpid}`); + + // Wait for clone to complete + await waitForTask(client, node.name, cloneUpid); + console.log('Clone completed successfully'); + + // Configure the container + console.log('Configuring container...'); + await client.updateLxcConfig(node.name, container.containerId, { + cores: 4, + features: 'nesting=1,keyctl=1,fuse=1', + memory: 4096, + net0: 'name=eth0,ip=dhcp,bridge=vmbr0', + searchdomain: site.internalDomain, + swap: 0, + onboot: 1, + tags: container.username + }); + console.log('Container configured'); + + // Start the container + console.log('Starting container...'); + const startUpid = await client.startLxc(node.name, container.containerId); + console.log(`Start task started: ${startUpid}`); + + // Wait for start to complete + await waitForTask(client, node.name, startUpid); + console.log('Container started successfully'); + + // Get MAC address from config + console.log('Querying container configuration...'); + const config = await client.lxcConfig(node.name, container.containerId); + const net0 = config['net0']; + const macMatch = net0.match(/hwaddr=([0-9A-Fa-f:]+)/); + + if (!macMatch) { + throw new Error('Could not extract MAC address from container configuration'); + } + + const macAddress = macMatch[1]; + console.log(`MAC address: ${macAddress}`); + + // Get IP address from Proxmox interfaces API + const ipv4Address = await getIpFromInterfaces(client, node.name, container.containerId); + + if (!ipv4Address) { + throw new Error('Could not get IP address from Proxmox interfaces API'); + } + + console.log(`IP address: ${ipv4Address}`); + + // Update the container record + console.log('Updating container record...'); + await container.update({ + macAddress, + ipv4Address, + status: 'running' + }); + + console.log('Container creation completed successfully!'); + console.log(` Hostname: ${container.hostname}`); + console.log(` VMID: ${container.containerId}`); + console.log(` MAC: ${macAddress}`); + console.log(` IP: ${ipv4Address}`); + console.log(` Status: running`); + + process.exit(0); + } catch (err) { + console.error('Container creation failed:', err.message); + + // Log axios error details if available + if (err.response?.data) { + console.error('API Error Details:', JSON.stringify(err.response.data, null, 2)); + } + + // Update status to failed + try { + await container.update({ status: 'failed' }); + console.log('Status updated to: failed'); + } catch (updateErr) { + console.error('Failed to update container status:', updateErr.message); + } + + process.exit(1); + } +} + +// Run the main function +main().catch(err => { + console.error('Unhandled error:', err); + process.exit(1); +}); diff --git a/create-a-container/bin/json-to-sql.js b/create-a-container/bin/json-to-sql.js index 6f9958e4..5dba9845 100644 --- a/create-a-container/bin/json-to-sql.js +++ b/create-a-container/bin/json-to-sql.js @@ -177,7 +177,7 @@ async function run() { console.log(`Container: hostname=${hostname}`); console.log(` ipv4Address=${obj.ip}`); console.log(` username=${obj.user}`); - console.log(` osRelease=${obj.os_release}`); + console.log(` template=${obj.template || 'N/A'}`); console.log(` containerId=${obj.ctid}`); console.log(` macAddress=${obj.mac}`); if (obj.ports) { @@ -196,13 +196,12 @@ async function run() { for (const [hostname, obj] of Object.entries(data)) { // If fields missing and Proxmox creds provided, try to fill them - if ((obj.user === undefined || obj.os_release === undefined || obj.ctid === undefined || obj.mac === undefined || obj.ip === undefined) && (PROXMOX_URL && PROXMOX_USER && PROXMOX_PASSWORD)) { + if ((obj.user === undefined || obj.ctid === undefined || obj.mac === undefined || obj.ip === undefined) && (PROXMOX_URL && PROXMOX_USER && PROXMOX_PASSWORD)) { const pmx = await lookupProxmoxByHostname(hostname); if (pmx.ctid && obj.ctid === undefined) obj.ctid = pmx.ctid; if (pmx.mac && obj.mac === undefined) obj.mac = pmx.mac; if (pmx.ip && obj.ip === undefined) obj.ip = pmx.ip; if (pmx.user && obj.user === undefined) obj.user = pmx.user; - if (pmx.os_release && obj.os_release === undefined) obj.os_release = pmx.os_release; } // Upsert Container by hostname @@ -215,7 +214,8 @@ async function run() { hostname, ipv4Address: obj.ip, username: obj.user || '', - osRelease: obj.os_release, + status: 'running', + template: obj.template || null, containerId: obj.ctid, macAddress: obj.mac }); @@ -224,7 +224,8 @@ async function run() { await container.update({ ipv4Address: obj.ip, username: obj.user || '', - osRelease: obj.os_release, + status: container.status || 'running', + template: obj.template || container.template, containerId: obj.ctid, macAddress: obj.mac }); diff --git a/create-a-container/migrations/20260127180000-add-container-status.js b/create-a-container/migrations/20260127180000-add-container-status.js new file mode 100644 index 00000000..172cfb66 --- /dev/null +++ b/create-a-container/migrations/20260127180000-add-container-status.js @@ -0,0 +1,15 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Containers', 'status', { + type: Sequelize.STRING(20), + allowNull: false, + defaultValue: 'running' + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn('Containers', 'status'); + } +}; diff --git a/create-a-container/migrations/20260127180001-add-container-creation-job-id.js b/create-a-container/migrations/20260127180001-add-container-creation-job-id.js new file mode 100644 index 00000000..be86d716 --- /dev/null +++ b/create-a-container/migrations/20260127180001-add-container-creation-job-id.js @@ -0,0 +1,20 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Containers', 'creationJobId', { + type: Sequelize.INTEGER, + allowNull: true, + references: { + model: 'Jobs', + key: 'id' + }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL' + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn('Containers', 'creationJobId'); + } +}; diff --git a/create-a-container/migrations/20260127180002-add-container-template.js b/create-a-container/migrations/20260127180002-add-container-template.js new file mode 100644 index 00000000..2e291cb6 --- /dev/null +++ b/create-a-container/migrations/20260127180002-add-container-template.js @@ -0,0 +1,14 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Containers', 'template', { + type: Sequelize.STRING(255), + allowNull: true + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn('Containers', 'template'); + } +}; diff --git a/create-a-container/migrations/20260127180003-remove-container-os-release.js b/create-a-container/migrations/20260127180003-remove-container-os-release.js new file mode 100644 index 00000000..2059b736 --- /dev/null +++ b/create-a-container/migrations/20260127180003-remove-container-os-release.js @@ -0,0 +1,14 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.removeColumn('Containers', 'osRelease'); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.addColumn('Containers', 'osRelease', { + type: Sequelize.STRING(255), + allowNull: true + }); + } +}; diff --git a/create-a-container/migrations/20260127180004-make-container-network-fields-nullable.js b/create-a-container/migrations/20260127180004-make-container-network-fields-nullable.js new file mode 100644 index 00000000..f4b0c823 --- /dev/null +++ b/create-a-container/migrations/20260127180004-make-container-network-fields-nullable.js @@ -0,0 +1,29 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.changeColumn('Containers', 'macAddress', { + type: Sequelize.STRING(17), + allowNull: true, + unique: true + }); + await queryInterface.changeColumn('Containers', 'ipv4Address', { + type: Sequelize.STRING(45), + allowNull: true, + unique: true + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.changeColumn('Containers', 'macAddress', { + type: Sequelize.STRING(17), + allowNull: false, + unique: true + }); + await queryInterface.changeColumn('Containers', 'ipv4Address', { + type: Sequelize.STRING(45), + allowNull: false, + unique: true + }); + } +}; diff --git a/create-a-container/models/container.js b/create-a-container/models/container.js index 02d94231..e8d5a5e1 100644 --- a/create-a-container/models/container.js +++ b/create-a-container/models/container.js @@ -14,6 +14,8 @@ module.exports = (sequelize, DataTypes) => { Container.hasMany(models.Service, { foreignKey: 'containerId', as: 'services' }); // a container belongs to a node Container.belongsTo(models.Node, { foreignKey: 'nodeId', as: 'node' }); + // a container may have a creation job + Container.belongsTo(models.Job, { foreignKey: 'creationJobId', as: 'creationJob' }); } } Container.init({ @@ -26,10 +28,23 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.STRING(255), allowNull: false }, - osRelease: { + status: { + type: DataTypes.STRING(20), + allowNull: false, + defaultValue: 'pending' + }, + template: { type: DataTypes.STRING(255), allowNull: true }, + creationJobId: { + type: DataTypes.INTEGER, + allowNull: true, + references: { + model: 'Jobs', + key: 'id' + } + }, nodeId: { type: DataTypes.INTEGER, allowNull: false, @@ -44,12 +59,12 @@ module.exports = (sequelize, DataTypes) => { }, macAddress: { type: DataTypes.STRING(17), - allowNull: false, + allowNull: true, unique: true }, ipv4Address: { type: DataTypes.STRING(45), - allowNull: false, + allowNull: true, unique: true }, aiContainer: { diff --git a/create-a-container/models/transport-service.js b/create-a-container/models/transport-service.js index 42b103be..10397805 100644 --- a/create-a-container/models/transport-service.js +++ b/create-a-container/models/transport-service.js @@ -8,8 +8,8 @@ module.exports = (sequelize, DataTypes) => { } // Find the next available external port for the given protocol in the specified range - static async nextAvailablePortInRange(protocol, minPort, maxPort) { - const usedServices = await TransportService.findAll({ + static async nextAvailablePortInRange(protocol, minPort, maxPort, transaction = null) { + const queryOptions = { where: { protocol: protocol, externalPort: { @@ -18,7 +18,14 @@ module.exports = (sequelize, DataTypes) => { }, attributes: ['externalPort'], order: [['externalPort', 'ASC']] - }); + }; + + if (transaction) { + queryOptions.transaction = transaction; + queryOptions.lock = sequelize.Sequelize.Transaction.LOCK.UPDATE; + } + + const usedServices = await TransportService.findAll(queryOptions); const usedPorts = new Set(usedServices.map(s => s.externalPort)); diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index 8dd7de71..3c7497a2 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -2,7 +2,7 @@ const express = require('express'); const router = express.Router({ mergeParams: true }); // Enable access to :siteId param const https = require('https'); const dns = require('dns').promises; -const { Container, Service, HTTPService, TransportService, DnsService, Node, Site, ExternalDomain, Sequelize, sequelize } = require('../models'); +const { Container, Service, HTTPService, TransportService, DnsService, Node, Site, ExternalDomain, Job, Sequelize, sequelize } = require('../models'); const { requireAuth } = require('../middlewares'); const ProxmoxApi = require('../utils/proxmox-api'); const serviceMap = require('../data/services.json'); @@ -109,7 +109,9 @@ router.get('/', requireAuth, async (req, res) => { id: c.id, hostname: c.hostname, ipv4Address: c.ipv4Address, - osRelease: c.osRelease, + status: c.status, + template: c.template, + creationJobId: c.creationJobId, sshPort, httpPort, nodeName: c.node ? c.node.name : '-' @@ -192,7 +194,7 @@ router.get('/:id/edit', requireAuth, async (req, res) => { }); }); -// POST /sites/:siteId/containers - Create a new container +// POST /sites/:siteId/containers - Create a new container (async via job) router.post('/', async (req, res) => { const siteId = parseInt(req.params.siteId, 10); @@ -203,170 +205,153 @@ router.post('/', async (req, res) => { return res.redirect('/sites'); } - // TODO: build the container async in a Job - try { - const { hostname, template, services } = req.body; - const [ nodeName, templateVmid ] = template.split(','); - const node = await Node.findOne({ where: { name: nodeName, siteId } }); - const client = await node.api(); - const vmid = await client.nextId(); - const upid = await client.cloneLxc(node.name, parseInt(templateVmid, 10), vmid, { - hostname, - description: `Cloned from template ${templateVmid}`, - full: 1 - }); - - // wait for the task to complete - while (true) { - const status = await client.taskStatus(node.name, upid); - if (status.status === 'stopped') break; - } - - // Configure the cloned container - await client.updateLxcConfig(node.name, vmid, { - cores: 4, - features: 'nesting=1,keyctl=1,fuse=1', - memory: 4096, - net0: 'name=eth0,ip=dhcp,bridge=vmbr0', - searchdomain: site.internalDomain, - swap: 0, - onboot: 1, - tags: req.session.user, - }); - - // Start the container - const startUpid = await client.startLxc(node.name, vmid); + const t = await sequelize.transaction(); - // wait for the start task to complete - while (true) { - const status = await client.taskStatus(node.name, startUpid); - if (status.status === 'stopped') break; - } - - // record container information - const config = await client.lxcConfig(node.name, vmid); - const macAddress = config['net0'].match(/hwaddr=([0-9A-Fa-f:]+)/)[1]; - const ipv4Address = await (async () => { - const maxRetries = 10; - const retryDelay = 3000; - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - const domainName = `${hostname}.${site.internalDomain}`; - const lookup = await dns.lookup(domainName); - return lookup.address; - } catch (err) { - console.error('DNS lookup failed:', err); - await new Promise(resolve => setTimeout(resolve, retryDelay)); - } + try { + const { hostname, template, services } = req.body; + const [ nodeName, templateVmid ] = template.split(','); + const node = await Node.findOne({ where: { name: nodeName, siteId } }); + + if (!node) { + throw new Error(`Node "${nodeName}" not found`); } - console.error('DNS lookup failed after maximum retries'); - return null; - })(); - - const container = await Container.create({ - hostname, - username: req.session.user, - nodeId: node.id, - containerId: vmid, - macAddress, - ipv4Address - }); - - // Create services if provided - if (services && typeof services === 'object') { - for (const key in services) { - const service = services[key]; - const { type, internalPort, externalHostname, externalDomainId, dnsName } = service; - - // Validate required fields - if (!type || !internalPort) continue; - - // Determine the service type (http, transport, or dns) - let serviceType; - let protocol = null; - - if (type === 'http') { - serviceType = 'http'; - } else if (type === 'srv') { - serviceType = 'dns'; - } else { - // tcp or udp - serviceType = 'transport'; - protocol = type; - } - - const serviceData = { - containerId: container.id, - type: serviceType, - internalPort: parseInt(internalPort, 10) - }; - - // Create the base service - const createdService = await Service.create(serviceData); - - if (serviceType === 'http') { - // Validate that both hostname and domain are set - if (!externalHostname || !externalDomainId || externalDomainId === '') { - req.flash('error', 'HTTP services must have both an external hostname and external domain'); - return res.redirect(`/sites/${siteId}/containers/new`); - } + + // Get the template name from Proxmox + const client = await node.api(); + const templates = await client.getLxcTemplates(node.name); + const templateContainer = templates.find(t => t.vmid === parseInt(templateVmid, 10)); + + if (!templateContainer) { + throw new Error(`Template with VMID ${templateVmid} not found on node ${nodeName}`); + } + + const templateName = templateContainer.name; + + // Allocate VMID immediately + const vmid = await client.nextId(); + + // Create the container record in pending status + const container = await Container.create({ + hostname, + username: req.session.user, + status: 'pending', + template: templateName, + nodeId: node.id, + containerId: vmid, + macAddress: null, + ipv4Address: null + }, { transaction: t }); + + // Create services if provided (validate within transaction) + if (services && typeof services === 'object') { + for (const key in services) { + const service = services[key]; + const { type, internalPort, externalHostname, externalDomainId, dnsName } = service; - // Create HTTPService entry - await HTTPService.create({ - serviceId: createdService.id, - externalHostname, - externalDomainId: parseInt(externalDomainId, 10) - }); - } else if (serviceType === 'dns') { - // Validate DNS name is set - if (!dnsName) { - req.flash('error', 'DNS services must have a DNS name'); - return res.redirect(`/sites/${siteId}/containers/new`); - } + // Validate required fields + if (!type || !internalPort) continue; - // Create DnsService entry - await DnsService.create({ - serviceId: createdService.id, - recordType: 'SRV', - dnsName - }); - } else { - // For TCP/UDP services, auto-assign external port - const minPort = 2000; - const maxPort = 65565; - const externalPort = await TransportService.nextAvailablePortInRange(protocol, minPort, maxPort); + // Determine the service type (http, transport, or dns) + let serviceType; + let protocol = null; - // Create TransportService entry - await TransportService.create({ - serviceId: createdService.id, - protocol: protocol, - externalPort - }); + if (type === 'http') { + serviceType = 'http'; + } else if (type === 'srv') { + serviceType = 'dns'; + } else { + // tcp or udp + serviceType = 'transport'; + protocol = type; + } + + const serviceData = { + containerId: container.id, + type: serviceType, + internalPort: parseInt(internalPort, 10) + }; + + // Create the base service + const createdService = await Service.create(serviceData, { transaction: t }); + + if (serviceType === 'http') { + // Validate that both hostname and domain are set + if (!externalHostname || !externalDomainId || externalDomainId === '') { + throw new Error('HTTP services must have both an external hostname and external domain'); + } + + // Create HTTPService entry + await HTTPService.create({ + serviceId: createdService.id, + externalHostname, + externalDomainId: parseInt(externalDomainId, 10) + }, { transaction: t }); + } else if (serviceType === 'dns') { + // Validate DNS name is set + if (!dnsName) { + throw new Error('DNS services must have a DNS name'); + } + + // Create DnsService entry + await DnsService.create({ + serviceId: createdService.id, + recordType: 'SRV', + dnsName + }, { transaction: t }); + } else { + // For TCP/UDP services, auto-assign external port + const minPort = 2000; + const maxPort = 65565; + const externalPort = await TransportService.nextAvailablePortInRange(protocol, minPort, maxPort, t); + + // Create TransportService entry + await TransportService.create({ + serviceId: createdService.id, + protocol: protocol, + externalPort + }, { transaction: t }); + } } } - } - return res.redirect(`/sites/${siteId}/containers`); -} catch (err) { - console.error('Error creating container:', err); - - // Handle axios errors with detailed messages - let errorMessage = 'Failed to create container: '; - if (err.response?.data) { - if (err.response.data.errors) { - errorMessage += JSON.stringify(err.response.data.errors); - } else if (err.response.data.message) { - errorMessage += err.response.data.message; + // Create the job to perform the actual container creation + const job = await Job.create({ + command: `node bin/create-container.js --container-id=${container.id}`, + createdBy: req.session.user, + status: 'pending' + }, { transaction: t }); + + // Link the container to the job + await container.update({ creationJobId: job.id }, { transaction: t }); + + // Commit the transaction + await t.commit(); + + req.flash('success', `Container "${hostname}" is being created. Check back shortly for status updates.`); + return res.redirect(`/sites/${siteId}/containers`); + } catch (err) { + // Rollback the transaction + await t.rollback(); + + console.error('Error creating container:', err); + + // Handle axios errors with detailed messages + let errorMessage = 'Failed to create container: '; + if (err.response?.data) { + if (err.response.data.errors) { + errorMessage += JSON.stringify(err.response.data.errors); + } else if (err.response.data.message) { + errorMessage += err.response.data.message; + } else { + errorMessage += err.message; + } } else { errorMessage += err.message; } - } else { - errorMessage += err.message; + + req.flash('error', errorMessage); + return res.redirect(`/sites/${siteId}/containers/new`); } - - req.flash('error', errorMessage); - return res.redirect(`/sites/${siteId}/containers/new`); -} }); // PUT /sites/:siteId/containers/:id - Update container services diff --git a/create-a-container/routers/sites.js b/create-a-container/routers/sites.js index a32b0666..17fc23a0 100644 --- a/create-a-container/routers/sites.js +++ b/create-a-container/routers/sites.js @@ -19,6 +19,8 @@ router.get('/:siteId/dnsmasq.conf', requireLocalhost, async (req, res) => { include: [{ model: Container, as: 'containers', + where: { status: 'running' }, + required: false, attributes: ['macAddress', 'ipv4Address', 'hostname'], include: [{ model: Service, @@ -44,7 +46,7 @@ router.get('/:siteId/dnsmasq.conf', requireLocalhost, async (req, res) => { router.get('/:siteId/nginx.conf', requireLocalhost, async (req, res) => { const siteId = parseInt(req.params.siteId, 10); - // fetch services for the specific site + // fetch services for the specific site (only from running containers) const site = await Site.findByPk(siteId, { include: [{ model: Node, @@ -52,6 +54,8 @@ router.get('/:siteId/nginx.conf', requireLocalhost, async (req, res) => { include: [{ model: Container, as: 'containers', + where: { status: 'running' }, + required: false, include: [{ model: Service, as: 'services', diff --git a/create-a-container/utils/proxmox-api.js b/create-a-container/utils/proxmox-api.js index 3e2d56bf..8d132f31 100644 --- a/create-a-container/utils/proxmox-api.js +++ b/create-a-container/utils/proxmox-api.js @@ -319,6 +319,20 @@ class ProxmoxApi { ); return response.data.data; } + + /** + * Get LXC container network interfaces + * @param {string} node - The node name + * @param {number} vmid - The container VMID + * @returns {Promise} - Array of network interfaces + */ + async lxcInterfaces(node, vmid) { + const response = await axios.get( + `${this.baseUrl}/api2/json/nodes/${node}/lxc/${vmid}/interfaces`, + this.options + ); + return response.data.data; + } } module.exports = ProxmoxApi; diff --git a/create-a-container/views/containers/index.ejs b/create-a-container/views/containers/index.ejs index 4c0e26bb..5cff1407 100644 --- a/create-a-container/views/containers/index.ejs +++ b/create-a-container/views/containers/index.ejs @@ -20,8 +20,9 @@ Hostname + Status IPv4 - OS Release + Template Node SSH Port HTTP Port @@ -33,13 +34,39 @@ <% rows.forEach(r => { %> <%= r.hostname %> + + <% if (r.status === 'running') { %> + Running + <% } else if (r.status === 'pending') { %> + + + Pending + + <% } else if (r.status === 'creating') { %> + + + Creating + + <% } else if (r.status === 'failed') { %> + Failed + <% } else { %> + <%= r.status || 'Unknown' %> + <% } %> + <% if (r.creationJobId && (r.status === 'pending' || r.status === 'creating' || r.status === 'failed')) { %> + + Details + + <% } %> + <%= r.ipv4Address || '-' %> - <%= r.osRelease || '-' %> + <%= r.template || '-' %> <%= r.nodeName %> <%= r.sshPort || '-' %> <%= r.httpPort || '-' %> - Edit + <% if (r.status === 'running') { %> + Edit + <% } %>
@@ -49,7 +76,7 @@ <% }) %> <% } else { %> - + No containers found. Click "New Container" to create your first one. diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index cf2a5e75..7a8d5d0e 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -14,6 +14,7 @@ erDiagram Sites ||--o{ ExternalDomains : has Nodes ||--o{ Containers : hosts Containers ||--o{ Services : exposes + Containers }o--o| Jobs : "created by" Services ||--|| HTTPServices : "type: http" Services ||--|| TransportServices : "type: transport" Services ||--|| DnsServices : "type: dns" @@ -47,13 +48,15 @@ erDiagram Containers { int id PK string hostname UK - string name - string description + string username + string status "pending,creating,running,failed" + string template + int creationJobId FK int nodeId FK int containerId string macAddress UK string ipv4Address UK - string status + string aiContainer } Services { @@ -184,16 +187,20 @@ The **Node** model represents a Proxmox VE server within a site. The **Container** model represents an LXC container running on a Proxmox node. **Key Fields:** -- `hostname`: Container hostname -- `name`: Display name +- `hostname`: Container hostname (unique) +- `username`: Owner of the container (who created it) +- `status`: Container creation state ('pending', 'creating', 'running', 'failed') +- `template`: Name of the Proxmox template used to create this container +- `creationJobId`: Foreign key to the Job that created this container (nullable) - `containerId`: Proxmox container ID (CTID) -- `macAddress`: Unique MAC address -- `ipv4Address`: Assigned IP address -- `status`: Container state (e.g., 'running', 'stopped') +- `macAddress`: Unique MAC address (nullable for pending containers) +- `ipv4Address`: Assigned IP address (nullable for pending containers) +- `aiContainer`: AI container flag (default: 'N') **Relationships:** - Belongs to Node - Has many Services +- Belongs to Job (optional, via creationJobId) **Constraints:** - Unique composite index on `(nodeId, containerId)` From af4be9e45a46a41807b506886f32e7e4d99e74bd Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 14:17:27 -0500 Subject: [PATCH 15/26] feat: job streaming --- create-a-container/routers/jobs.js | 148 ++++++++++++++-- create-a-container/views/jobs/show.ejs | 228 +++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 12 deletions(-) create mode 100644 create-a-container/views/jobs/show.ejs diff --git a/create-a-container/routers/jobs.js b/create-a-container/routers/jobs.js index 46fa03c8..874790a7 100644 --- a/create-a-container/routers/jobs.js +++ b/create-a-container/routers/jobs.js @@ -1,11 +1,20 @@ const express = require('express'); const router = express.Router(); -const { Job, JobStatus, sequelize } = require('../models'); +const { Job, JobStatus, Container, Node, sequelize } = require('../models'); const { requireAuth, requireAdmin } = require('../middlewares'); // All job endpoints require authentication router.use(requireAuth); +/** + * Helper to check if user can access a job + */ +async function canAccessJob(job, req) { + const username = req.session && req.session.user; + const isAdmin = req.session && req.session.isAdmin; + return isAdmin || job.createdBy === username; +} + // POST /jobs - enqueue a new job (admins only) router.post('/', requireAdmin, async (req, res) => { try { @@ -28,26 +37,145 @@ router.post('/', requireAdmin, async (req, res) => { } }); -// GET /jobs/:id - job metadata +// GET /jobs/:id - job metadata (HTML or JSON based on Accept header) router.get('/:id', async (req, res) => { try { const id = parseInt(req.params.id, 10); const job = await Job.findByPk(id); - if (!job) return res.status(404).json({ error: 'Job not found' }); + if (!job) { + if (req.accepts('html')) { + req.flash('error', 'Job not found'); + return res.redirect('/'); + } + return res.status(404).json({ error: 'Job not found' }); + } + // Authorization: only owner or admin can view - const username = req.session && req.session.user; - const isAdmin = req.session && req.session.isAdmin; - if (!isAdmin && job.createdBy !== username) { + if (!await canAccessJob(job, req)) { + if (req.accepts('html')) { + req.flash('error', 'Job not found'); + return res.redirect('/'); + } return res.status(404).json({ error: 'Job not found' }); } - return res.json({ id: job.id, command: job.command, status: job.status, createdAt: job.createdAt, updatedAt: job.updatedAt, createdBy: job.createdBy }); + // If client accepts HTML, render the job view + if (req.accepts('html')) { + // Get initial output for completed jobs or first batch for running jobs + const initialOutput = await JobStatus.findAll({ + where: { jobId: id }, + order: [['id', 'ASC']], + limit: 1000 + }); + + // Find the container associated with this job (if any) + const container = await Container.findOne({ + where: { creationJobId: id }, + include: [{ model: Node, as: 'node' }] + }); + + return res.render('jobs/show', { + job, + initialOutput, + container, + req + }); + } + + // JSON response for API clients + return res.json({ + id: job.id, + command: job.command, + status: job.status, + createdAt: job.createdAt, + updatedAt: job.updatedAt, + createdBy: job.createdBy + }); } catch (err) { console.error('Failed to fetch job:', err); + if (req.accepts('html')) { + req.flash('error', 'Failed to load job'); + return res.redirect('/'); + } return res.status(500).json({ error: 'Failed to fetch job' }); } }); +// GET /jobs/:id/stream - SSE endpoint for streaming job output +router.get('/:id/stream', async (req, res) => { + const id = parseInt(req.params.id, 10); + + try { + const job = await Job.findByPk(id); + if (!job || !await canAccessJob(job, req)) { + return res.status(404).json({ error: 'Job not found' }); + } + + // Set up SSE headers + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); // Disable nginx buffering + res.flushHeaders(); + + // Track last sent ID for incremental updates + let lastId = req.query.lastId ? parseInt(req.query.lastId, 10) : 0; + let isRunning = true; + + // Send keepalive every 15 seconds + const keepaliveInterval = setInterval(() => { + if (isRunning) { + res.write(':keepalive\n\n'); + } + }, 15000); + + // Poll for new output every 2 seconds + const pollInterval = setInterval(async () => { + try { + // Fetch new log entries + const newLogs = await JobStatus.findAll({ + where: { + jobId: id, + id: { [sequelize.Sequelize.Op.gt]: lastId } + }, + order: [['id', 'ASC']], + limit: 100 + }); + + // Send each new log entry + for (const log of newLogs) { + res.write(`event: log\ndata: ${JSON.stringify({ id: log.id, output: log.output, timestamp: log.createdAt })}\n\n`); + lastId = log.id; + } + + // Check if job is still running + const currentJob = await Job.findByPk(id); + if (!currentJob || !['pending', 'running'].includes(currentJob.status)) { + // Send final status and close + res.write(`event: status\ndata: ${JSON.stringify({ status: currentJob ? currentJob.status : 'unknown' })}\n\n`); + cleanup(); + res.end(); + } + } catch (err) { + console.error('SSE poll error:', err); + } + }, 2000); + + function cleanup() { + isRunning = false; + clearInterval(keepaliveInterval); + clearInterval(pollInterval); + } + + // Clean up on client disconnect + req.on('close', cleanup); + + } catch (err) { + console.error('SSE setup error:', err); + res.status(500).json({ error: 'Failed to start stream' }); + } +}); + // GET /jobs/:id/status - fetch job status rows router.get('/:id/status', async (req, res) => { try { @@ -62,11 +190,7 @@ router.get('/:id/status', async (req, res) => { // Ensure only owner or admin can fetch statuses const job = await Job.findByPk(id); - if (!job) return res.status(404).json({ error: 'Job not found' }); - const username = req.session && req.session.user; - const isAdmin = req.session && req.session.isAdmin; - if (!isAdmin && job.createdBy !== username) { - // Hide existence to prevent information leakage + if (!job || !await canAccessJob(job, req)) { return res.status(404).json({ error: 'Job not found' }); } diff --git a/create-a-container/views/jobs/show.ejs b/create-a-container/views/jobs/show.ejs new file mode 100644 index 00000000..b4e19984 --- /dev/null +++ b/create-a-container/views/jobs/show.ejs @@ -0,0 +1,228 @@ +<%- include('../layouts/header', { + title: `Job #${job.id} - MIE`, + breadcrumbs: [ + { label: 'Jobs', url: '/jobs' }, + { label: `Job #${job.id}`, url: `/jobs/${job.id}` } + ], + colWidth: 'col-12 col-lg-10', + req +}) %> + +
+
+

+ Job #<%= job.id %> + + <% if (job.status === 'pending' || job.status === 'running') { %> + + <% } %> + <%= job.status.charAt(0).toUpperCase() + job.status.slice(1) %> + +

+ <% if (container) { %> + + Back to Containers + + <% } %> +
+ +
+
+
+

Command:

+ <%= job.command %> +
+
+

Created:

+ <%= new Date(job.createdAt).toLocaleString() %> +
+
+

Created By:

+ <%= job.createdBy || 'System' %> +
+
+ + <% if (container) { %> + + <% } %> + +
+
Output
+
+ + +
+
+ +
+ <% if (initialOutput && initialOutput.length > 0) { %> + <% initialOutput.forEach(line => { %><%= line.output %><% }) %> + <% } %> +
+ +
+ <% if (job.status === 'pending' || job.status === 'running') { %> + Connecting to live stream... + <% } else { %> + Job completed + <% } %> +
+
+ + +
+ + + + + +<%- include('../layouts/footer') %> From bf4f530b072d0d5d05161580c378de0416c05802 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 14:29:06 -0500 Subject: [PATCH 16/26] feat: delete sanity checks --- create-a-container/bin/create-container.js | 24 +++++++---- ...260127191000-make-container-id-nullable.js | 17 ++++++++ create-a-container/models/container.js | 2 +- create-a-container/routers/containers.js | 43 +++++++++++++++---- 4 files changed, 68 insertions(+), 18 deletions(-) create mode 100644 create-a-container/migrations/20260127191000-make-container-id-nullable.js diff --git a/create-a-container/bin/create-container.js b/create-a-container/bin/create-container.js index 9c158526..6e1d07ed 100755 --- a/create-a-container/bin/create-container.js +++ b/create-a-container/bin/create-container.js @@ -174,7 +174,6 @@ async function main() { console.log(`Node: ${node.name}`); console.log(`Site: ${site.name} (${site.internalDomain})`); console.log(`Template: ${container.template}`); - console.log(`Target VMID: ${container.containerId}`); try { // Update status to 'creating' @@ -197,9 +196,14 @@ async function main() { const templateVmid = templateContainer.vmid; console.log(`Found template VMID: ${templateVmid}`); + // Allocate VMID right before cloning to minimize race condition window + console.log('Allocating VMID from Proxmox...'); + const vmid = await client.nextId(); + console.log(`Allocated VMID: ${vmid}`); + // Clone the template - console.log(`Cloning template ${templateVmid} to VMID ${container.containerId}...`); - const cloneUpid = await client.cloneLxc(node.name, templateVmid, container.containerId, { + console.log(`Cloning template ${templateVmid} to VMID ${vmid}...`); + const cloneUpid = await client.cloneLxc(node.name, templateVmid, vmid, { hostname: container.hostname, description: `Cloned from template ${container.template}`, full: 1 @@ -210,9 +214,13 @@ async function main() { await waitForTask(client, node.name, cloneUpid); console.log('Clone completed successfully'); + // Store the VMID now that clone succeeded + await container.update({ containerId: vmid }); + console.log(`Container VMID ${vmid} stored in database`); + // Configure the container console.log('Configuring container...'); - await client.updateLxcConfig(node.name, container.containerId, { + await client.updateLxcConfig(node.name, vmid, { cores: 4, features: 'nesting=1,keyctl=1,fuse=1', memory: 4096, @@ -226,7 +234,7 @@ async function main() { // Start the container console.log('Starting container...'); - const startUpid = await client.startLxc(node.name, container.containerId); + const startUpid = await client.startLxc(node.name, vmid); console.log(`Start task started: ${startUpid}`); // Wait for start to complete @@ -235,7 +243,7 @@ async function main() { // Get MAC address from config console.log('Querying container configuration...'); - const config = await client.lxcConfig(node.name, container.containerId); + const config = await client.lxcConfig(node.name, vmid); const net0 = config['net0']; const macMatch = net0.match(/hwaddr=([0-9A-Fa-f:]+)/); @@ -247,7 +255,7 @@ async function main() { console.log(`MAC address: ${macAddress}`); // Get IP address from Proxmox interfaces API - const ipv4Address = await getIpFromInterfaces(client, node.name, container.containerId); + const ipv4Address = await getIpFromInterfaces(client, node.name, vmid); if (!ipv4Address) { throw new Error('Could not get IP address from Proxmox interfaces API'); @@ -265,7 +273,7 @@ async function main() { console.log('Container creation completed successfully!'); console.log(` Hostname: ${container.hostname}`); - console.log(` VMID: ${container.containerId}`); + console.log(` VMID: ${vmid}`); console.log(` MAC: ${macAddress}`); console.log(` IP: ${ipv4Address}`); console.log(` Status: running`); diff --git a/create-a-container/migrations/20260127191000-make-container-id-nullable.js b/create-a-container/migrations/20260127191000-make-container-id-nullable.js new file mode 100644 index 00000000..3479ea4c --- /dev/null +++ b/create-a-container/migrations/20260127191000-make-container-id-nullable.js @@ -0,0 +1,17 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.changeColumn('Containers', 'containerId', { + type: Sequelize.INTEGER.UNSIGNED, + allowNull: true + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.changeColumn('Containers', 'containerId', { + type: Sequelize.INTEGER.UNSIGNED, + allowNull: false + }); + } +}; diff --git a/create-a-container/models/container.js b/create-a-container/models/container.js index e8d5a5e1..eaf309d6 100644 --- a/create-a-container/models/container.js +++ b/create-a-container/models/container.js @@ -55,7 +55,7 @@ module.exports = (sequelize, DataTypes) => { }, containerId: { type: DataTypes.INTEGER.UNSIGNED, - allowNull: false + allowNull: true }, macAddress: { type: DataTypes.STRING(17), diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index 3c7497a2..0326029b 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -227,17 +227,14 @@ router.post('/', async (req, res) => { const templateName = templateContainer.name; - // Allocate VMID immediately - const vmid = await client.nextId(); - - // Create the container record in pending status + // Create the container record in pending status (VMID allocated by job) const container = await Container.create({ hostname, username: req.session.user, status: 'pending', template: templateName, nodeId: node.id, - containerId: vmid, + containerId: null, macAddress: null, ipv4Address: null }, { transaction: t }); @@ -537,15 +534,43 @@ router.delete('/:id', requireAuth, async (req, res) => { } try { - // Delete from Proxmox - const api = await node.api(); - await api.deleteContainer(node.name, container.containerId, true, true); + // Only attempt Proxmox deletion if containerId exists + if (container.containerId) { + const api = await node.api(); + + // Sanity check: verify the container in Proxmox matches our database record + try { + const proxmoxConfig = await api.lxcConfig(node.name, container.containerId); + const proxmoxHostname = proxmoxConfig.hostname; + + if (proxmoxHostname && proxmoxHostname !== container.hostname) { + console.error(`Hostname mismatch: DB has "${container.hostname}", Proxmox has "${proxmoxHostname}" for VMID ${container.containerId}`); + req.flash('error', `Safety check failed: Proxmox container hostname "${proxmoxHostname}" does not match database hostname "${container.hostname}". Manual intervention required.`); + return res.redirect(`/sites/${siteId}/containers`); + } + + // Delete from Proxmox + await api.deleteContainer(node.name, container.containerId, true, true); + console.log(`Deleted container ${container.containerId} from Proxmox node ${node.name}`); + } catch (proxmoxError) { + // If container doesn't exist in Proxmox (404 or similar), continue with DB deletion + if (proxmoxError.response?.status === 500 && proxmoxError.response?.data?.errors?.vmid) { + console.log(`Container ${container.containerId} not found in Proxmox, proceeding with DB deletion`); + } else if (proxmoxError.response?.status === 404) { + console.log(`Container ${container.containerId} not found in Proxmox, proceeding with DB deletion`); + } else { + throw proxmoxError; + } + } + } else { + console.log(`Container ${container.hostname} has no containerId, skipping Proxmox deletion`); + } // Delete from database (cascade deletes associated services) await container.destroy(); } catch (error) { console.error(error); - req.flash('error', `Failed to delete container from Proxmox: ${error.message}`); + req.flash('error', `Failed to delete container: ${error.message}`); return res.redirect(`/sites/${siteId}/containers`); } From a3a3bfd9f915a575e7cd1eaf616408f74c5339c9 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 15:16:57 -0500 Subject: [PATCH 17/26] feat: docker image templates --- create-a-container/README.md | 8 +- create-a-container/bin/create-container.js | 195 ++++++++++++++---- .../20260127200000-add-node-image-storage.js | 16 ++ create-a-container/models/node.js | 5 + create-a-container/routers/containers.js | 112 ++++++++-- create-a-container/utils/proxmox-api.js | 20 ++ create-a-container/views/containers/form.ejs | 33 ++- .../docs/developers/database-schema.md | 4 +- 8 files changed, 330 insertions(+), 63 deletions(-) create mode 100644 create-a-container/migrations/20260127200000-add-node-image-storage.js diff --git a/create-a-container/README.md b/create-a-container/README.md index 927f8797..8bdb74b8 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -58,6 +58,7 @@ erDiagram - **User Authentication** - Proxmox VE authentication integration - **Container Management** - Create, list, and track LXC containers +- **Docker/OCI Support** - Pull and deploy containers from Docker Hub, GHCR, or any OCI registry - **Service Registry** - Track HTTP/TCP/UDP services running on containers - **Dynamic Nginx Config** - Generate nginx reverse proxy configurations on-demand - **Real-time Progress** - SSE (Server-Sent Events) for container creation progress @@ -209,12 +210,13 @@ Display container creation form #### `POST /containers` Create a container asynchronously via a background job -- **Body**: `{ hostname, template, services }` where: +- **Body**: `{ hostname, template, customTemplate, services }` where: - `hostname`: Container hostname - - `template`: Template selection in format "nodeName,vmid" + - `template`: Template selection in format "nodeName,vmid" OR "custom" for Docker images + - `customTemplate`: Docker image reference when template="custom" (e.g., `nginx`, `nginx:alpine`, `myorg/myapp:v1`, `ghcr.io/org/image:tag`) - `services`: Object of service definitions - **Returns**: Redirect to containers list with flash message -- **Process**: Creates pending container, services, and job in a single transaction. The job-runner executes the actual Proxmox operations. +- **Process**: Creates pending container, services, and job in a single transaction. Docker image references are normalized to full format (`host/org/image:tag`). The job-runner executes the actual Proxmox operations. #### `DELETE /containers/:id` (Auth Required) Delete a container from both Proxmox and the database diff --git a/create-a-container/bin/create-container.js b/create-a-container/bin/create-container.js index 6e1d07ed..22fea10f 100755 --- a/create-a-container/bin/create-container.js +++ b/create-a-container/bin/create-container.js @@ -10,13 +10,16 @@ * * The script will: * 1. Load the container record from the database - * 2. Clone the template in Proxmox + * 2. Either clone a Proxmox template OR pull a Docker image and create from it * 3. Configure the container (cores, memory, network) * 4. Start the container * 5. Query MAC address from Proxmox config * 6. Query IP address from Proxmox interfaces API * 7. Update the container record with MAC, IP, and status='running' * + * Docker images are detected by the presence of '/' in the template field. + * Format: host/org/image:tag (e.g., docker.io/library/nginx:latest) + * * All output is logged to STDOUT for capture by the job-runner. * Exit code 0 = success, non-zero = failure. */ @@ -42,6 +45,62 @@ function parseArgs() { return args; } +/** + * Check if a template is a Docker image reference (contains '/') + * @param {string} template - The template string + * @returns {boolean} True if Docker image, false if Proxmox template + */ +function isDockerImage(template) { + return template.includes('/'); +} + +/** + * Parse a normalized Docker image reference into components + * Format: host/org/image:tag + * @param {string} ref - The normalized Docker reference + * @returns {object} Parsed components: { registry, namespace, image, tag } + */ +function parseDockerRef(ref) { + // Split off tag + const [imagePart, tag] = ref.split(':'); + const parts = imagePart.split('/'); + + // Format is always host/org/image after normalization + const registry = parts[0]; + const image = parts[parts.length - 1]; + const namespace = parts.slice(1, -1).join('/'); + + return { registry, namespace, image, tag }; +} + +/** + * Generate a filename for a pulled Docker image + * Replaces special chars with underscores + * Note: Proxmox automatically appends .tar, so we don't include it here + * @param {object} parsed - Parsed Docker ref components + * @returns {string} Sanitized filename (e.g., "docker.io_library_nginx_latest") + */ +function generateImageFilename(parsed) { + const { registry, namespace, image, tag } = parsed; + const sanitized = `${registry}_${namespace}_${image}_${tag}`.replace(/[/:]/g, '_'); + return sanitized; +} + +/** + * Parse command line arguments + * @returns {object} Parsed arguments + */ +function parseArgs() { + const args = {}; + for (const arg of process.argv.slice(2)) { + const match = arg.match(/^--([^=]+)=(.+)$/); + if (match) { + args[match[1]] = match[2]; + } + } + return args; +} + /** * Wait for a Proxmox task to complete * @param {ProxmoxApi} client - The Proxmox API client @@ -175,6 +234,9 @@ async function main() { console.log(`Site: ${site.name} (${site.internalDomain})`); console.log(`Template: ${container.template}`); + const isDocker = isDockerImage(container.template); + console.log(`Template type: ${isDocker ? 'Docker image' : 'Proxmox template'}`); + try { // Update status to 'creating' await container.update({ status: 'creating' }); @@ -184,54 +246,105 @@ async function main() { const client = await node.api(); console.log('Proxmox API client initialized'); - // Find the template VMID by matching the template name - console.log(`Looking for template: ${container.template}`); - const templates = await client.getLxcTemplates(node.name); - const templateContainer = templates.find(t => t.name === container.template); - - if (!templateContainer) { - throw new Error(`Template "${container.template}" not found on node ${node.name}`); - } - - const templateVmid = templateContainer.vmid; - console.log(`Found template VMID: ${templateVmid}`); - - // Allocate VMID right before cloning to minimize race condition window + // Allocate VMID right before creating to minimize race condition window console.log('Allocating VMID from Proxmox...'); const vmid = await client.nextId(); console.log(`Allocated VMID: ${vmid}`); - // Clone the template - console.log(`Cloning template ${templateVmid} to VMID ${vmid}...`); - const cloneUpid = await client.cloneLxc(node.name, templateVmid, vmid, { - hostname: container.hostname, - description: `Cloned from template ${container.template}`, - full: 1 - }); - console.log(`Clone task started: ${cloneUpid}`); - - // Wait for clone to complete - await waitForTask(client, node.name, cloneUpid); - console.log('Clone completed successfully'); + if (isDocker) { + // Docker image: pull from OCI registry, then create container + const parsed = parseDockerRef(container.template); + console.log(`Docker image: ${parsed.registry}/${parsed.namespace}/${parsed.image}:${parsed.tag}`); + + const filename = generateImageFilename(parsed); + console.log(`Target filename: ${filename}`); + + const storage = node.imageStorage || 'local'; + console.log(`Using storage: ${storage}`); + + // Pull the image from OCI registry using full image reference + const imageRef = container.template; + console.log(`Pulling image ${imageRef}...`); + const pullUpid = await client.pullOciImage(node.name, storage, { + reference: imageRef, + filename + }); + console.log(`Pull task started: ${pullUpid}`); + + // Wait for pull to complete + await waitForTask(client, node.name, pullUpid); + console.log('Image pulled successfully'); + + // Create container from the pulled image (Proxmox adds .tar to the filename) + console.log(`Creating container from ${filename}.tar...`); + const ostemplate = `${storage}:vztmpl/${filename}.tar`; + const createUpid = await client.createLxc(node.name, { + vmid, + hostname: container.hostname, + ostemplate, + description: `Created from Docker image ${container.template}`, + cores: 4, + features: 'nesting=1,keyctl=1,fuse=1', + memory: 4096, + net0: 'name=eth0,ip=dhcp,bridge=vmbr0,host-managed=1', + searchdomain: site.internalDomain, + swap: 0, + onboot: 1, + tags: container.username, + unprivileged: 1, + storage: 'local-lvm' + }); + console.log(`Create task started: ${createUpid}`); + + // Wait for create to complete + await waitForTask(client, node.name, createUpid); + console.log('Container created successfully'); + + } else { + // Proxmox template: clone existing container + console.log(`Looking for template: ${container.template}`); + const templates = await client.getLxcTemplates(node.name); + const templateContainer = templates.find(t => t.name === container.template); + + if (!templateContainer) { + throw new Error(`Template "${container.template}" not found on node ${node.name}`); + } + + const templateVmid = templateContainer.vmid; + console.log(`Found template VMID: ${templateVmid}`); + + // Clone the template + console.log(`Cloning template ${templateVmid} to VMID ${vmid}...`); + const cloneUpid = await client.cloneLxc(node.name, templateVmid, vmid, { + hostname: container.hostname, + description: `Cloned from template ${container.template}`, + full: 1 + }); + console.log(`Clone task started: ${cloneUpid}`); + + // Wait for clone to complete + await waitForTask(client, node.name, cloneUpid); + console.log('Clone completed successfully'); + + // Configure the container (Docker containers are configured at creation time) + console.log('Configuring container...'); + await client.updateLxcConfig(node.name, vmid, { + cores: 4, + features: 'nesting=1,keyctl=1,fuse=1', + memory: 4096, + net0: 'name=eth0,ip=dhcp,bridge=vmbr0', + searchdomain: site.internalDomain, + swap: 0, + onboot: 1, + tags: container.username + }); + console.log('Container configured'); + } - // Store the VMID now that clone succeeded + // Store the VMID now that creation succeeded await container.update({ containerId: vmid }); console.log(`Container VMID ${vmid} stored in database`); - // Configure the container - console.log('Configuring container...'); - await client.updateLxcConfig(node.name, vmid, { - cores: 4, - features: 'nesting=1,keyctl=1,fuse=1', - memory: 4096, - net0: 'name=eth0,ip=dhcp,bridge=vmbr0', - searchdomain: site.internalDomain, - swap: 0, - onboot: 1, - tags: container.username - }); - console.log('Container configured'); - // Start the container console.log('Starting container...'); const startUpid = await client.startLxc(node.name, vmid); diff --git a/create-a-container/migrations/20260127200000-add-node-image-storage.js b/create-a-container/migrations/20260127200000-add-node-image-storage.js new file mode 100644 index 00000000..27feaaec --- /dev/null +++ b/create-a-container/migrations/20260127200000-add-node-image-storage.js @@ -0,0 +1,16 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Nodes', 'imageStorage', { + type: Sequelize.STRING(255), + allowNull: false, + defaultValue: 'local' + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn('Nodes', 'imageStorage'); + } +}; diff --git a/create-a-container/models/node.js b/create-a-container/models/node.js index 3e6052a1..2d812ca4 100644 --- a/create-a-container/models/node.js +++ b/create-a-container/models/node.js @@ -81,6 +81,11 @@ module.exports = (sequelize, DataTypes) => { tlsVerify: { type: DataTypes.BOOLEAN, allowNull: true + }, + imageStorage: { + type: DataTypes.STRING(255), + allowNull: false, + defaultValue: 'local' } }, { sequelize, diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index 0326029b..d2488171 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -7,6 +7,60 @@ const { requireAuth } = require('../middlewares'); const ProxmoxApi = require('../utils/proxmox-api'); const serviceMap = require('../data/services.json'); +/** + * Normalize a Docker image reference to full format: host/org/image:tag + * Examples: + * nginx → docker.io/library/nginx:latest + * nginx:alpine → docker.io/library/nginx:alpine + * myorg/myapp → docker.io/myorg/myapp:latest + * myorg/myapp:v1 → docker.io/myorg/myapp:v1 + * ghcr.io/org/app:v1 → ghcr.io/org/app:v1 + */ +function normalizeDockerRef(ref) { + // Split off tag first + let tag = 'latest'; + let imagePart = ref; + + const lastColon = ref.lastIndexOf(':'); + if (lastColon !== -1) { + const potentialTag = ref.substring(lastColon + 1); + // Make sure this isn't a port number in a registry URL (e.g., registry:5000/image) + if (!potentialTag.includes('/')) { + tag = potentialTag; + imagePart = ref.substring(0, lastColon); + } + } + + const parts = imagePart.split('/'); + + let host = 'docker.io'; + let org = 'library'; + let image; + + if (parts.length === 1) { + // Just image name: nginx + image = parts[0]; + } else if (parts.length === 2) { + // Could be org/image or host/image + // If first part contains a dot or colon, it's a registry host + if (parts[0].includes('.') || parts[0].includes(':')) { + host = parts[0]; + image = parts[1]; + } else { + // org/image + org = parts[0]; + image = parts[1]; + } + } else { + // host/org/image or host/path/to/image + host = parts[0]; + image = parts[parts.length - 1]; + org = parts.slice(1, -1).join('/'); + } + + return `${host}/${org}/${image}:${tag}`; +} + // GET /sites/:siteId/containers/new - Display form for creating a new container router.get('/new', requireAuth, async (req, res) => { // verify site exists @@ -208,25 +262,53 @@ router.post('/', async (req, res) => { const t = await sequelize.transaction(); try { - const { hostname, template, services } = req.body; - const [ nodeName, templateVmid ] = template.split(','); - const node = await Node.findOne({ where: { name: nodeName, siteId } }); + const { hostname, template, customTemplate, services } = req.body; - if (!node) { - throw new Error(`Node "${nodeName}" not found`); - } - - // Get the template name from Proxmox - const client = await node.api(); - const templates = await client.getLxcTemplates(node.name); - const templateContainer = templates.find(t => t.vmid === parseInt(templateVmid, 10)); + let nodeName, templateName, node; - if (!templateContainer) { - throw new Error(`Template with VMID ${templateVmid} not found on node ${nodeName}`); + if (template === 'custom' || !template) { + // Custom Docker image - parse and normalize the reference + if (!customTemplate || customTemplate.trim() === '') { + throw new Error('Custom template image is required'); + } + + templateName = normalizeDockerRef(customTemplate.trim()); + + // For custom templates, pick the first available node in the site + node = await Node.findOne({ + where: { + siteId, + apiUrl: { [Sequelize.Op.ne]: null }, + tokenId: { [Sequelize.Op.ne]: null }, + secret: { [Sequelize.Op.ne]: null } + } + }); + + if (!node) { + throw new Error('No nodes with API access available in this site'); + } + } else { + // Standard Proxmox template + const [ nodeNamePart, templateVmid ] = template.split(','); + nodeName = nodeNamePart; + node = await Node.findOne({ where: { name: nodeName, siteId } }); + + if (!node) { + throw new Error(`Node "${nodeName}" not found`); + } + + // Get the template name from Proxmox + const client = await node.api(); + const templates = await client.getLxcTemplates(node.name); + const templateContainer = templates.find(t => t.vmid === parseInt(templateVmid, 10)); + + if (!templateContainer) { + throw new Error(`Template with VMID ${templateVmid} not found on node ${nodeName}`); + } + + templateName = templateContainer.name; } - const templateName = templateContainer.name; - // Create the container record in pending status (VMID allocated by job) const container = await Container.create({ hostname, diff --git a/create-a-container/utils/proxmox-api.js b/create-a-container/utils/proxmox-api.js index 8d132f31..df134ca9 100644 --- a/create-a-container/utils/proxmox-api.js +++ b/create-a-container/utils/proxmox-api.js @@ -333,6 +333,26 @@ class ProxmoxApi { ); return response.data.data; } + + /** + * Pull an OCI/Docker image from a registry to Proxmox storage + * @param {string} node - The node name + * @param {string} storage - The storage name (e.g., 'local') + * @param {Object} options - Pull options + * @param {string} options.reference - Full image reference (e.g., 'docker.io/library/nginx:latest') + * @param {string} [options.filename] - Target filename (e.g., 'nginx_latest.tar') + * @param {string} [options.username] - Registry username for private images + * @param {string} [options.password] - Registry password for private images + * @returns {Promise} - UPID of the pull task + */ + async pullOciImage(node, storage, options) { + const response = await axios.post( + `${this.baseUrl}/api2/json/nodes/${node}/storage/${storage}/oci-registry-pull`, + options, + this.options + ); + return response.data.data; + } } module.exports = ProxmoxApi; diff --git a/create-a-container/views/containers/form.ejs b/create-a-container/views/containers/form.ejs index cad0d069..3232422a 100644 --- a/create-a-container/views/containers/form.ejs +++ b/create-a-container/views/containers/form.ejs @@ -36,7 +36,7 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; <% if (isEdit) { %> <% } else { %> - <% if (typeof templates !== 'undefined' && templates && templates.length > 0) { %> <% templates.forEach(template => { %> @@ -44,10 +44,17 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; <%= template.name %> (<%= template.node %>) <% }) %> - <% } else { %> - <% } %> + + <% } %> @@ -114,6 +121,26 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; const externalDomains = <%- JSON.stringify((externalDomains || []).map(d => ({ id: d.id, name: d.name }))) %>; let serviceCounter = 0; + // Custom template toggle logic + const templateSelect = document.getElementById('templateSelect'); + const customTemplateContainer = document.getElementById('customTemplateContainer'); + const customTemplateInput = document.getElementById('customTemplate'); + + if (templateSelect) { + templateSelect.addEventListener('change', function() { + if (this.value === 'custom') { + customTemplateContainer.style.display = 'block'; + customTemplateInput.required = true; + this.removeAttribute('name'); // Don't submit the select value + } else { + customTemplateContainer.style.display = 'none'; + customTemplateInput.required = false; + customTemplateInput.value = ''; + this.setAttribute('name', 'template'); // Submit select value + } + }); + } + function addServiceRow(type = 'http', internalPort = '', externalHostname = '', externalDomainId = '', serviceId = null, externalPort = null, dnsName = '') { const row = document.createElement('tr'); row.id = `service-row-${serviceCounter}`; diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index 7a8d5d0e..a0a9fb49 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -42,6 +42,7 @@ erDiagram string apiTokenIdOrUsername string apiTokenSecretOrPassword boolean disableTlsVerification + string imageStorage "default: local" int siteId FK } @@ -174,6 +175,7 @@ The **Node** model represents a Proxmox VE server within a site. - `apiTokenIdOrUsername`: Authentication credential (username or token ID) - `apiTokenSecretOrPassword`: Authentication secret - `disableTlsVerification`: Skip TLS certificate validation +- `imageStorage`: Proxmox storage name for pulled Docker/OCI images (default: 'local') **Relationships:** - Belongs to Site @@ -190,7 +192,7 @@ The **Container** model represents an LXC container running on a Proxmox node. - `hostname`: Container hostname (unique) - `username`: Owner of the container (who created it) - `status`: Container creation state ('pending', 'creating', 'running', 'failed') -- `template`: Name of the Proxmox template used to create this container +- `template`: Name of the Proxmox template or Docker image reference (e.g., `docker.io/library/nginx:latest`) - `creationJobId`: Foreign key to the Job that created this container (nullable) - `containerId`: Proxmox container ID (CTID) - `macAddress`: Unique MAC address (nullable for pending containers) From 5a4d5f3e9b1ea7d7c686ded630fe66b52f55ef38 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 16:07:49 -0500 Subject: [PATCH 18/26] feat: add nodes storage option --- create-a-container/routers/nodes.js | 84 ++++++++++++++++++------- create-a-container/views/nodes/form.ejs | 34 ++++++++++ 2 files changed, 94 insertions(+), 24 deletions(-) diff --git a/create-a-container/routers/nodes.js b/create-a-container/routers/nodes.js index e2fd139b..921949ec 100644 --- a/create-a-container/routers/nodes.js +++ b/create-a-container/routers/nodes.js @@ -117,7 +117,7 @@ router.post('/', async (req, res) => { return res.redirect('/sites'); } - const { name, ipv4Address, apiUrl, tokenId, secret, tlsVerify } = req.body; + const { name, ipv4Address, apiUrl, tokenId, secret, tlsVerify, imageStorage } = req.body; await Node.create({ name, @@ -126,6 +126,7 @@ router.post('/', async (req, res) => { tokenId: tokenId || null, secret: secret || null, tlsVerify: tlsVerify === '' || tlsVerify === null ? null : tlsVerify === 'true', + imageStorage: imageStorage || 'local', siteId }); @@ -165,37 +166,43 @@ router.post('/import', async (req, res) => { const client = await tempNode.api(); const nodes = await client.nodes(); - // Fetch network information for each node to get IP address + // Fetch network information and storage for each node const nodesWithIp = await Promise.all(nodes.map(async (n) => { + let ipv4Address = null; + let imageStorage = 'local'; + try { const networkInterfaces = await client.nodeNetwork(n.node); // Find the primary network interface (usually vmbr0 or the one with type 'bridge' and active) const primaryInterface = networkInterfaces.find(iface => iface.iface === 'vmbr0' || (iface.type === 'bridge' && iface.active) ); - const ipv4Address = primaryInterface?.address || null; - - return { - name: n.node, - ipv4Address, - apiUrl, - tokenId, - secret, - tlsVerify: tlsVerify === '' || tlsVerify === null ? null : tlsVerify === 'true', - siteId - }; + ipv4Address = primaryInterface?.address || null; } catch (err) { console.error(`Failed to fetch network info for node ${n.node}:`, err.message); - return { - name: n.node, - ipv4Address: null, - apiUrl, - tokenId, - secret, - tlsVerify: tlsVerify === '' || tlsVerify === null ? null : tlsVerify === 'true', - siteId - }; } + + // Find largest storage supporting CT templates (vztmpl) + try { + const storages = await client.datastores(n.node, 'vztmpl', true); + if (storages.length > 0) { + const largest = storages.reduce((max, s) => (s.total > max.total ? s : max), storages[0]); + imageStorage = largest.storage; + } + } catch (err) { + console.error(`Failed to fetch storages for node ${n.node}:`, err.message); + } + + return { + name: n.node, + ipv4Address, + apiUrl, + tokenId, + secret, + tlsVerify: tlsVerify === '' || tlsVerify === null ? null : tlsVerify === 'true', + imageStorage, + siteId + }; })); const importedNodes = await Node.bulkCreate(nodesWithIp); @@ -242,14 +249,15 @@ router.put('/:id', async (req, res) => { return res.redirect(`/sites/${siteId}/nodes`); } - const { name, ipv4Address, apiUrl, tokenId, secret, tlsVerify } = req.body; + const { name, ipv4Address, apiUrl, tokenId, secret, tlsVerify, imageStorage } = req.body; const updateData = { name, ipv4Address: ipv4Address || null, apiUrl: apiUrl || null, tokenId: tokenId || null, - tlsVerify: tlsVerify === '' || tlsVerify === null ? null : tlsVerify === 'true' + tlsVerify: tlsVerify === '' || tlsVerify === null ? null : tlsVerify === 'true', + imageStorage: imageStorage || 'local' }; // Only update secret if a new value was provided @@ -268,6 +276,34 @@ router.put('/:id', async (req, res) => { } }); +// GET /sites/:siteId/nodes/:id/storages - Get storages supporting CT templates +router.get('/:id/storages', async (req, res) => { + const siteId = parseInt(req.params.siteId, 10); + const nodeId = parseInt(req.params.id, 10); + + try { + const node = await Node.findOne({ + where: { id: nodeId, siteId } + }); + + if (!node || !node.apiUrl || !node.tokenId || !node.secret) { + return res.json([]); + } + + const client = await node.api(); + const storages = await client.datastores(node.name, 'vztmpl', true); + + return res.json(storages.map(s => ({ + name: s.storage, + total: s.total, + available: s.avail + }))); + } catch (err) { + console.error('Error fetching storages:', err.message); + return res.json([]); + } +}); + // DELETE /sites/:siteId/nodes/:id - Delete a node router.delete('/:id', async (req, res) => { const siteId = parseInt(req.params.siteId, 10); diff --git a/create-a-container/views/nodes/form.ejs b/create-a-container/views/nodes/form.ejs index 7f70782c..23849bba 100644 --- a/create-a-container/views/nodes/form.ejs +++ b/create-a-container/views/nodes/form.ejs @@ -98,6 +98,22 @@
Whether to verify TLS certificates when connecting to this node
+
+ + + +
Storage for CT Template images used when building containers
+
+
Cancel
<%- include('../layouts/footer') %> + +<% if (isEdit) { %> + +<% } %> From a87af7211decc1abc6ad2fad8f24008e8676a442 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 16:44:48 -0500 Subject: [PATCH 19/26] feat: add env and entrypoint options for OCI containers --- create-a-container/bin/create-container.js | 78 ++------ .../bin/reconfigure-container.js | 143 ++++++++++++++ ...0127210000-add-container-env-entrypoint.js | 23 +++ create-a-container/models/container.js | 45 +++++ create-a-container/routers/containers.js | 68 ++++++- create-a-container/utils/cli.js | 22 +++ create-a-container/utils/proxmox-api.js | 44 +++++ create-a-container/views/containers/form.ejs | 187 ++++++++++++++---- create-a-container/views/containers/index.ejs | 5 + .../users/creating-containers/web-gui.mdx | 35 ++++ 10 files changed, 544 insertions(+), 106 deletions(-) create mode 100644 create-a-container/bin/reconfigure-container.js create mode 100644 create-a-container/migrations/20260127210000-add-container-env-entrypoint.js create mode 100644 create-a-container/utils/cli.js diff --git a/create-a-container/bin/create-container.js b/create-a-container/bin/create-container.js index 22fea10f..cbc8fb8a 100755 --- a/create-a-container/bin/create-container.js +++ b/create-a-container/bin/create-container.js @@ -30,20 +30,8 @@ const path = require('path'); const db = require(path.join(__dirname, '..', 'models')); const { Container, Node, Site } = db; -/** - * Parse command line arguments - * @returns {object} Parsed arguments - */ -function parseArgs() { - const args = {}; - for (const arg of process.argv.slice(2)) { - const match = arg.match(/^--([^=]+)=(.+)$/); - if (match) { - args[match[1]] = match[2]; - } - } - return args; -} +// Load utilities +const { parseArgs } = require(path.join(__dirname, '..', 'utils', 'cli')); /** * Check if a template is a Docker image reference (contains '/') @@ -86,51 +74,6 @@ function generateImageFilename(parsed) { return sanitized; } -/** - * Parse command line arguments - * @returns {object} Parsed arguments - */ -function parseArgs() { - const args = {}; - for (const arg of process.argv.slice(2)) { - const match = arg.match(/^--([^=]+)=(.+)$/); - if (match) { - args[match[1]] = match[2]; - } - } - return args; -} - -/** - * Wait for a Proxmox task to complete - * @param {ProxmoxApi} client - The Proxmox API client - * @param {string} nodeName - The node name - * @param {string} upid - The task UPID - * @param {number} pollInterval - Polling interval in ms (default 2000) - * @param {number} timeout - Timeout in ms (default 300000 = 5 minutes) - * @returns {Promise} The final task status - */ -async function waitForTask(client, nodeName, upid, pollInterval = 2000, timeout = 300000) { - const startTime = Date.now(); - while (true) { - const status = await client.taskStatus(nodeName, upid); - console.log(`Task ${upid}: status=${status.status}, exitstatus=${status.exitstatus || 'N/A'}`); - - if (status.status === 'stopped') { - if (status.exitstatus && status.exitstatus !== 'OK') { - throw new Error(`Task failed with status: ${status.exitstatus}`); - } - return status; - } - - if (Date.now() - startTime > timeout) { - throw new Error(`Task ${upid} timed out after ${timeout}ms`); - } - - await new Promise(resolve => setTimeout(resolve, pollInterval)); - } -} - /** * Query IP address from Proxmox interfaces API with retries * @param {ProxmoxApi} client - The Proxmox API client @@ -272,7 +215,7 @@ async function main() { console.log(`Pull task started: ${pullUpid}`); // Wait for pull to complete - await waitForTask(client, node.name, pullUpid); + await client.waitForTask(node.name, pullUpid); console.log('Image pulled successfully'); // Create container from the pulled image (Proxmox adds .tar to the filename) @@ -297,7 +240,7 @@ async function main() { console.log(`Create task started: ${createUpid}`); // Wait for create to complete - await waitForTask(client, node.name, createUpid); + await client.waitForTask(node.name, createUpid); console.log('Container created successfully'); } else { @@ -323,7 +266,7 @@ async function main() { console.log(`Clone task started: ${cloneUpid}`); // Wait for clone to complete - await waitForTask(client, node.name, cloneUpid); + await client.waitForTask(node.name, cloneUpid); console.log('Clone completed successfully'); // Configure the container (Docker containers are configured at creation time) @@ -341,6 +284,15 @@ async function main() { console.log('Container configured'); } + // Apply environment variables and entrypoint if set + const envConfig = container.buildLxcEnvConfig(); + if (Object.keys(envConfig).length > 0) { + console.log('Applying environment variables and entrypoint...'); + console.log('Config:', JSON.stringify(envConfig, null, 2)); + await client.updateLxcConfig(node.name, vmid, envConfig); + console.log('Environment/entrypoint configuration applied'); + } + // Store the VMID now that creation succeeded await container.update({ containerId: vmid }); console.log(`Container VMID ${vmid} stored in database`); @@ -351,7 +303,7 @@ async function main() { console.log(`Start task started: ${startUpid}`); // Wait for start to complete - await waitForTask(client, node.name, startUpid); + await client.waitForTask(node.name, startUpid); console.log('Container started successfully'); // Get MAC address from config diff --git a/create-a-container/bin/reconfigure-container.js b/create-a-container/bin/reconfigure-container.js new file mode 100644 index 00000000..237ff23a --- /dev/null +++ b/create-a-container/bin/reconfigure-container.js @@ -0,0 +1,143 @@ +#!/usr/bin/env node +/** + * reconfigure-container.js + * + * Background job script that applies configuration changes and restarts a container. + * This script is executed by the job-runner when environment variables or entrypoint + * are changed on an existing container. + * + * Usage: node bin/reconfigure-container.js --container-id= + * + * The script will: + * 1. Load the container record from the database + * 2. Apply env and entrypoint config via Proxmox API + * 3. Stop the container + * 4. Start the container + * 5. Update the container status to 'running' + * + * All output is logged to STDOUT for capture by the job-runner. + * Exit code 0 = success, non-zero = failure. + */ + +const path = require('path'); + +// Load models from parent directory +const db = require(path.join(__dirname, '..', 'models')); +const { Container, Node, Site } = db; + +// Load utilities +const { parseArgs } = require(path.join(__dirname, '..', 'utils', 'cli')); + +/** + * Main function + */ +async function main() { + const args = parseArgs(); + + if (!args['container-id']) { + console.error('Usage: node reconfigure-container.js --container-id='); + process.exit(1); + } + + const containerId = parseInt(args['container-id'], 10); + console.log(`Starting container reconfiguration for container ID: ${containerId}`); + + // Load the container record with its node and site + const container = await Container.findByPk(containerId, { + include: [{ + model: Node, + as: 'node', + include: [{ + model: Site, + as: 'site' + }] + }] + }); + + if (!container) { + console.error(`Container with ID ${containerId} not found`); + process.exit(1); + } + + if (!container.containerId) { + console.error('Container has no Proxmox VMID - cannot reconfigure'); + process.exit(1); + } + + const node = container.node; + + if (!node) { + console.error('Container has no associated node'); + process.exit(1); + } + + console.log(`Container: ${container.hostname}`); + console.log(`Node: ${node.name}`); + console.log(`VMID: ${container.containerId}`); + + try { + // Get the Proxmox API client + const client = await node.api(); + console.log('Proxmox API client initialized'); + + // Build config from environment variables and entrypoint + const lxcConfig = container.buildLxcEnvConfig(); + + if (Object.keys(lxcConfig).length > 0) { + console.log('Applying LXC configuration...'); + console.log('Config:', JSON.stringify(lxcConfig, null, 2)); + await client.updateLxcConfig(node.name, container.containerId, lxcConfig); + console.log('Configuration applied'); + } else { + console.log('No configuration changes to apply'); + } + + // Stop the container + console.log('Stopping container...'); + const stopUpid = await client.stopLxc(node.name, container.containerId); + console.log(`Stop task started: ${stopUpid}`); + + // Wait for stop to complete (shorter timeout for stop/start) + await client.waitForTask(node.name, stopUpid, 2000, 60000); + console.log('Container stopped'); + + // Start the container + console.log('Starting container...'); + const startUpid = await client.startLxc(node.name, container.containerId); + console.log(`Start task started: ${startUpid}`); + + // Wait for start to complete + await client.waitForTask(node.name, startUpid, 2000, 60000); + console.log('Container started'); + + // Update status to running + await container.update({ status: 'running' }); + console.log('Status updated to: running'); + + console.log('Container reconfiguration completed successfully!'); + process.exit(0); + } catch (err) { + console.error('Container reconfiguration failed:', err.message); + + // Log axios error details if available + if (err.response?.data) { + console.error('API Error Details:', JSON.stringify(err.response.data, null, 2)); + } + + // Update status to failed + try { + await container.update({ status: 'failed' }); + console.log('Status updated to: failed'); + } catch (updateErr) { + console.error('Failed to update container status:', updateErr.message); + } + + process.exit(1); + } +} + +// Run the main function +main().catch(err => { + console.error('Unhandled error:', err); + process.exit(1); +}); diff --git a/create-a-container/migrations/20260127210000-add-container-env-entrypoint.js b/create-a-container/migrations/20260127210000-add-container-env-entrypoint.js new file mode 100644 index 00000000..26161f73 --- /dev/null +++ b/create-a-container/migrations/20260127210000-add-container-env-entrypoint.js @@ -0,0 +1,23 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Containers', 'environmentVars', { + type: Sequelize.TEXT, + allowNull: true, + defaultValue: null + }); + + await queryInterface.addColumn('Containers', 'entrypoint', { + type: Sequelize.STRING(2000), + allowNull: true, + defaultValue: null + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn('Containers', 'entrypoint'); + await queryInterface.removeColumn('Containers', 'environmentVars'); + } +}; diff --git a/create-a-container/models/container.js b/create-a-container/models/container.js index eaf309d6..88c621f5 100644 --- a/create-a-container/models/container.js +++ b/create-a-container/models/container.js @@ -17,6 +17,41 @@ module.exports = (sequelize, DataTypes) => { // a container may have a creation job Container.belongsTo(models.Job, { foreignKey: 'creationJobId', as: 'creationJob' }); } + + /** + * Build LXC config object for environment variables and entrypoint + * Returns config suitable for Proxmox API updateLxcConfig + * @returns {object} Config object with 'env' and 'entrypoint' properties + */ + buildLxcEnvConfig() { + const config = {}; + + // Parse environment variables from JSON and format as NUL-separated list + // Format: KEY1=value1\0KEY2=value2\0KEY3=value3 + if (this.environmentVars) { + try { + const envObj = JSON.parse(this.environmentVars); + const envPairs = []; + for (const [key, value] of Object.entries(envObj)) { + if (key && value !== undefined) { + envPairs.push(`${key}=${value}`); + } + } + if (envPairs.length > 0) { + config['env'] = envPairs.join('\0'); + } + } catch (err) { + console.error('Failed to parse environment variables JSON:', err.message); + } + } + + // Set entrypoint command + if (this.entrypoint && this.entrypoint.trim()) { + config['entrypoint'] = this.entrypoint.trim(); + } + + return config; + } } Container.init({ hostname: { @@ -71,6 +106,16 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.STRING(50), allowNull: false, defaultValue: 'N' + }, + environmentVars: { + type: DataTypes.TEXT, + allowNull: true, + defaultValue: null + }, + entrypoint: { + type: DataTypes.STRING(2000), + allowNull: true, + defaultValue: null } }, { sequelize, diff --git a/create-a-container/routers/containers.js b/create-a-container/routers/containers.js index d2488171..167d35a2 100644 --- a/create-a-container/routers/containers.js +++ b/create-a-container/routers/containers.js @@ -262,7 +262,21 @@ router.post('/', async (req, res) => { const t = await sequelize.transaction(); try { - const { hostname, template, customTemplate, services } = req.body; + const { hostname, template, customTemplate, services, environmentVars, entrypoint } = req.body; + + // Convert environment variables array to JSON object + let envVarsJson = null; + if (environmentVars && Array.isArray(environmentVars)) { + const envObj = {}; + for (const env of environmentVars) { + if (env.key && env.key.trim()) { + envObj[env.key.trim()] = env.value || ''; + } + } + if (Object.keys(envObj).length > 0) { + envVarsJson = JSON.stringify(envObj); + } + } let nodeName, templateName, node; @@ -318,7 +332,9 @@ router.post('/', async (req, res) => { nodeId: node.id, containerId: null, macAddress: null, - ipv4Address: null + ipv4Address: null, + environmentVars: envVarsJson, + entrypoint: entrypoint && entrypoint.trim() ? entrypoint.trim() : null }, { transaction: t }); // Create services if provided (validate within transaction) @@ -465,10 +481,50 @@ router.put('/:id', requireAuth, async (req, res) => { return res.redirect(`/sites/${siteId}/containers`); } - const { services } = req.body; + const { services, environmentVars, entrypoint } = req.body; + + // Convert environment variables array to JSON object + let envVarsJson = null; + if (environmentVars && Array.isArray(environmentVars)) { + const envObj = {}; + for (const env of environmentVars) { + if (env.key && env.key.trim()) { + envObj[env.key.trim()] = env.value || ''; + } + } + if (Object.keys(envObj).length > 0) { + envVarsJson = JSON.stringify(envObj); + } + } + + const newEntrypoint = entrypoint && entrypoint.trim() ? entrypoint.trim() : null; + + // Check if env vars or entrypoint changed + const envChanged = container.environmentVars !== envVarsJson; + const entrypointChanged = container.entrypoint !== newEntrypoint; + const needsRestart = envChanged || entrypointChanged; // Wrap all database operations in a transaction + let restartJob = null; await sequelize.transaction(async (t) => { + // Update environment variables and entrypoint + if (envChanged || entrypointChanged) { + await container.update({ + environmentVars: envVarsJson, + entrypoint: newEntrypoint, + status: needsRestart && container.containerId ? 'restarting' : container.status + }, { transaction: t }); + } + + // Create restart job if needed and container has a VMID + if (needsRestart && container.containerId) { + restartJob = await Job.create({ + command: `node bin/reconfigure-container.js --container-id=${container.id}`, + createdBy: req.session.user, + status: 'pending' + }, { transaction: t }); + } + // Process services in two phases: delete first, then create new if (services && typeof services === 'object') { // Phase 1: Delete marked services @@ -559,7 +615,11 @@ router.put('/:id', requireAuth, async (req, res) => { } }); - req.flash('success', 'Container services updated successfully'); + if (restartJob) { + req.flash('success', 'Container configuration updated. Restarting container...'); + } else { + req.flash('success', 'Container services updated successfully'); + } return res.redirect(`/sites/${siteId}/containers`); } catch (err) { console.error('Error updating container:', err); diff --git a/create-a-container/utils/cli.js b/create-a-container/utils/cli.js new file mode 100644 index 00000000..6a74a61d --- /dev/null +++ b/create-a-container/utils/cli.js @@ -0,0 +1,22 @@ +/** + * CLI utility functions for job scripts + */ + +/** + * Parse command line arguments in --key=value format + * @returns {object} Parsed arguments as key-value pairs + */ +function parseArgs() { + const args = {}; + for (const arg of process.argv.slice(2)) { + const match = arg.match(/^--([^=]+)=(.+)$/); + if (match) { + args[match[1]] = match[2]; + } + } + return args; +} + +module.exports = { + parseArgs +}; diff --git a/create-a-container/utils/proxmox-api.js b/create-a-container/utils/proxmox-api.js index df134ca9..33563a07 100644 --- a/create-a-container/utils/proxmox-api.js +++ b/create-a-container/utils/proxmox-api.js @@ -234,6 +234,35 @@ class ProxmoxApi { return response.data.data; } + /** + * Wait for a Proxmox task to complete + * @param {string} node - The node name + * @param {string} upid - The task UPID + * @param {number} pollInterval - Polling interval in ms (default 2000) + * @param {number} timeout - Timeout in ms (default 300000 = 5 minutes) + * @returns {Promise} The final task status + */ + async waitForTask(node, upid, pollInterval = 2000, timeout = 300000) { + const startTime = Date.now(); + while (true) { + const status = await this.taskStatus(node, upid); + console.log(`Task ${upid}: status=${status.status}, exitstatus=${status.exitstatus || 'N/A'}`); + + if (status.status === 'stopped') { + if (status.exitstatus && status.exitstatus !== 'OK') { + throw new Error(`Task failed with status: ${status.exitstatus}`); + } + return status; + } + + if (Date.now() - startTime > timeout) { + throw new Error(`Task ${upid} timed out after ${timeout}ms`); + } + + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } + } + /** * Delete a container * @param {string} nodeName @@ -320,6 +349,21 @@ class ProxmoxApi { return response.data.data; } + /** + * Stop an LXC container + * @param {string} node - The node name + * @param {number} vmid - The container VMID + * @returns {Promise} - The task UPID + */ + async stopLxc(node, vmid) { + const response = await axios.post( + `${this.baseUrl}/api2/json/nodes/${node}/lxc/${vmid}/status/stop`, + {}, + this.options + ); + return response.data.data; + } + /** * Get LXC container network interfaces * @param {string} node - The node name diff --git a/create-a-container/views/containers/form.ejs b/create-a-container/views/containers/form.ejs index 3232422a..561fb432 100644 --- a/create-a-container/views/containers/form.ejs +++ b/create-a-container/views/containers/form.ejs @@ -49,7 +49,7 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; -
- - -
-
- - - - - - - - - - - - -
TypeInternal Port - External - - - - Action
-
+
+ + Services + +
+ +
+
+ + + + + + + + + + + + +
TypeInternal Port + External + + + + Action
+
+
+ +
+ + Environment Variables + +
+ +
+
+ + + + + + + + + + + +
KeyValueAction
+
+
- +
+ + Entrypoint Command + +
+ +
The command to run when the container starts (overrides the default entrypoint)
+
+
+ +
@@ -119,7 +163,9 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; } : null }))) : '[]' %>; const externalDomains = <%- JSON.stringify((externalDomains || []).map(d => ({ id: d.id, name: d.name }))) %>; + const existingEnvVars = <%- isEdit && container && container.environmentVars ? container.environmentVars : '{}' %>; let serviceCounter = 0; + let envVarCounter = 0; // Custom template toggle logic const templateSelect = document.getElementById('templateSelect'); @@ -388,9 +434,6 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; addServiceRow(type, service.internalPort, externalHostname, externalDomainId, service.id, externalPort, dnsName); }); - } else { - // Add default HTTP service on port 80 with hostname from form for new containers - addServiceRow('http', '80', hostnameField.value || '', ''); } // Update externalHostname when hostname field changes (only for new containers) @@ -404,6 +447,72 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; }); }); } + + // Environment Variables functionality + function addEnvVarRow(key = '', value = '') { + const row = document.createElement('tr'); + row.id = `env-row-${envVarCounter}`; + + // Key cell + const keyCell = document.createElement('td'); + keyCell.style.cssText = 'border: 1px solid #ddd; padding: 8px;'; + + const keyInput = document.createElement('input'); + keyInput.type = 'text'; + keyInput.name = `environmentVars[${envVarCounter}][key]`; + keyInput.value = key; + keyInput.placeholder = 'KEY_NAME'; + keyInput.pattern = '[A-Za-z_][A-Za-z0-9_]*'; + keyInput.style.cssText = 'width: 100%; padding: 4px;'; + + keyCell.appendChild(keyInput); + + // Value cell + const valueCell = document.createElement('td'); + valueCell.style.cssText = 'border: 1px solid #ddd; padding: 8px;'; + + const valueInput = document.createElement('input'); + valueInput.type = 'text'; + valueInput.name = `environmentVars[${envVarCounter}][value]`; + valueInput.value = value; + valueInput.placeholder = 'value'; + valueInput.style.cssText = 'width: 100%; padding: 4px;'; + + valueCell.appendChild(valueInput); + + // Action cell + const actionCell = document.createElement('td'); + actionCell.style.cssText = 'border: 1px solid #ddd; padding: 8px; text-align: center;'; + + const removeBtn = document.createElement('button'); + removeBtn.type = 'button'; + removeBtn.textContent = 'Delete'; + removeBtn.style.cssText = 'padding: 4px 8px; cursor: pointer; background-color: #dc3545; color: white; border: none; border-radius: 4px;'; + const currentIndex = envVarCounter; + removeBtn.onclick = () => { + document.getElementById(`env-row-${currentIndex}`).remove(); + }; + + actionCell.appendChild(removeBtn); + + row.appendChild(keyCell); + row.appendChild(valueCell); + row.appendChild(actionCell); + + document.getElementById('envVarsTableBody').appendChild(row); + envVarCounter++; + } + + document.getElementById('addEnvBtn').addEventListener('click', () => { + addEnvVarRow(); + }); + + // Initialize existing environment variables + if (existingEnvVars && typeof existingEnvVars === 'object') { + for (const [key, value] of Object.entries(existingEnvVars)) { + addEnvVarRow(key, value); + } + } <%- include('../layouts/footer') %> diff --git a/create-a-container/views/containers/index.ejs b/create-a-container/views/containers/index.ejs index 5cff1407..1de5ef31 100644 --- a/create-a-container/views/containers/index.ejs +++ b/create-a-container/views/containers/index.ejs @@ -49,6 +49,11 @@ <% } else if (r.status === 'failed') { %> Failed + <% } else if (r.status === 'restarting') { %> + + + Restarting + <% } else { %> <%= r.status || 'Unknown' %> <% } %> diff --git a/mie-opensource-landing/docs/users/creating-containers/web-gui.mdx b/mie-opensource-landing/docs/users/creating-containers/web-gui.mdx index 3b2ba607..9aa733e1 100644 --- a/mie-opensource-landing/docs/users/creating-containers/web-gui.mdx +++ b/mie-opensource-landing/docs/users/creating-containers/web-gui.mdx @@ -160,4 +160,39 @@ As of writing, you are able to reboot, start, and shutdown your container as you --- +## Docker Container Configuration + +When creating containers from Docker images, you can configure additional settings that are specific to containerized applications. + +### Environment Variables + +Environment variables allow you to pass configuration to your containerized application at runtime. These are commonly used for: +- Database connection strings +- API keys and secrets +- Feature flags +- Application modes (development, production, etc.) + +To add environment variables, expand the **Environment Variables** section and click **Add Variable** to create key-value pairs. + +:::warning System Containers +Environment variables are intended for **Docker-based containers** only. System containers (created from Proxmox templates like Debian or Rocky Linux) typically expect `init` as PID 1 and may not use environment variables in the same way. If you're using a standard Linux template, you generally don't need to set environment variables here. +::: + +### Entrypoint Command + +The entrypoint command overrides the default startup command for a Docker container. This is useful when you need to: +- Run a specific script or binary +- Pass custom arguments to your application +- Chain multiple startup commands + +:::warning System Containers +The entrypoint command is intended for **Docker-based containers** only. System containers expect `init` as PID 1 and should not have their entrypoint overridden. Changing the entrypoint on a system container may prevent it from starting correctly. +::: + +### Restarting After Configuration Changes + +When you modify environment variables or the entrypoint command on an existing container, the system will automatically restart the container to apply the changes. You can monitor the restart progress from the container list page. + +--- + **Need Help?**: For questions about container configuration or troubleshooting, contact the MIE team. From 16761c6dcd26845a658698ce1ea86a24e7670401 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Tue, 27 Jan 2026 17:07:42 -0500 Subject: [PATCH 20/26] fix: prevent colisions by appending image hash --- create-a-container/bin/create-container.js | 206 +++++++++++++++++-- create-a-container/views/containers/form.ejs | 2 +- 2 files changed, 186 insertions(+), 22 deletions(-) diff --git a/create-a-container/bin/create-container.js b/create-a-container/bin/create-container.js index cbc8fb8a..001df2c3 100755 --- a/create-a-container/bin/create-container.js +++ b/create-a-container/bin/create-container.js @@ -25,6 +25,7 @@ */ const path = require('path'); +const https = require('https'); // Load models from parent directory const db = require(path.join(__dirname, '..', 'models')); @@ -33,6 +34,90 @@ const { Container, Node, Site } = db; // Load utilities const { parseArgs } = require(path.join(__dirname, '..', 'utils', 'cli')); +/** + * Fetch JSON from a URL with optional headers + * @param {string} url - The URL to fetch + * @param {object} headers - Optional headers + * @returns {Promise} Parsed JSON response + */ +function fetchJson(url, headers = {}) { + return new Promise((resolve, reject) => { + const req = https.get(url, { headers }, (res) => { + let data = ''; + res.on('data', chunk => data += chunk); + res.on('end', () => { + if (res.statusCode >= 400) { + reject(new Error(`HTTP ${res.statusCode}: ${data}`)); + } else { + try { + resolve(JSON.parse(data)); + } catch (e) { + reject(new Error(`Failed to parse JSON: ${e.message}`)); + } + } + }); + }); + req.on('error', reject); + }); +} + +/** + * Get the digest (sha256 hash) of a Docker/OCI image from the registry + * Handles both single-arch and multi-arch (manifest list) images + * @param {string} registry - Registry hostname (e.g., 'docker.io') + * @param {string} repo - Repository (e.g., 'library/nginx') + * @param {string} tag - Tag (e.g., 'latest') + * @returns {Promise} Short digest (first 12 chars of sha256) + */ +async function getImageDigest(registry, repo, tag) { + let headers = {}; + + // Docker Hub requires auth token + if (registry === 'docker.io' || registry === 'registry-1.docker.io') { + const tokenUrl = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull`; + const tokenData = await fetchJson(tokenUrl); + headers['Authorization'] = `Bearer ${tokenData.token}`; + } + + const registryHost = registry === 'docker.io' ? 'registry-1.docker.io' : registry; + + // Fetch manifest - accept both single manifest and manifest list + headers['Accept'] = [ + 'application/vnd.docker.distribution.manifest.v2+json', + 'application/vnd.oci.image.manifest.v1+json', + 'application/vnd.docker.distribution.manifest.list.v2+json', + 'application/vnd.oci.image.index.v1+json' + ].join(', '); + + const manifestUrl = `https://${registryHost}/v2/${repo}/manifests/${tag}`; + let manifest = await fetchJson(manifestUrl, headers); + + // Handle manifest list (multi-arch) - select amd64/linux + if (manifest.manifests && Array.isArray(manifest.manifests)) { + const amd64Manifest = manifest.manifests.find(m => + m.platform?.architecture === 'amd64' && m.platform?.os === 'linux' + ); + if (!amd64Manifest) { + throw new Error('No amd64/linux manifest found in manifest list'); + } + + // Fetch the actual manifest for amd64 + headers['Accept'] = 'application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json'; + const archManifestUrl = `https://${registryHost}/v2/${repo}/manifests/${amd64Manifest.digest}`; + manifest = await fetchJson(archManifestUrl, headers); + } + + // Get config digest from manifest + const configDigest = manifest.config?.digest; + if (!configDigest) { + throw new Error('No config digest in manifest'); + } + + // Return short hash (sha256:abc123... -> abc123...) + const hash = configDigest.replace('sha256:', ''); + return hash.substring(0, 12); +} + /** * Check if a template is a Docker image reference (contains '/') * @param {string} template - The template string @@ -63,14 +148,15 @@ function parseDockerRef(ref) { /** * Generate a filename for a pulled Docker image - * Replaces special chars with underscores + * Replaces special chars with underscores, includes digest for cache busting * Note: Proxmox automatically appends .tar, so we don't include it here * @param {object} parsed - Parsed Docker ref components - * @returns {string} Sanitized filename (e.g., "docker.io_library_nginx_latest") + * @param {string} digest - Short digest hash + * @returns {string} Sanitized filename (e.g., "docker.io_library_nginx_latest_abc123def456") */ -function generateImageFilename(parsed) { +function generateImageFilename(parsed, digest) { const { registry, namespace, image, tag } = parsed; - const sanitized = `${registry}_${namespace}_${image}_${tag}`.replace(/[/:]/g, '_'); + const sanitized = `${registry}_${namespace}_${image}_${tag}_${digest}`.replace(/[/:]/g, '_'); return sanitized; } @@ -199,24 +285,39 @@ async function main() { const parsed = parseDockerRef(container.template); console.log(`Docker image: ${parsed.registry}/${parsed.namespace}/${parsed.image}:${parsed.tag}`); - const filename = generateImageFilename(parsed); - console.log(`Target filename: ${filename}`); - const storage = node.imageStorage || 'local'; console.log(`Using storage: ${storage}`); - // Pull the image from OCI registry using full image reference - const imageRef = container.template; - console.log(`Pulling image ${imageRef}...`); - const pullUpid = await client.pullOciImage(node.name, storage, { - reference: imageRef, - filename - }); - console.log(`Pull task started: ${pullUpid}`); + // Get image digest from registry to create unique filename + const repo = parsed.namespace ? `${parsed.namespace}/${parsed.image}` : parsed.image; + console.log(`Fetching digest for ${parsed.registry}/${repo}:${parsed.tag}...`); + const digest = await getImageDigest(parsed.registry, repo, parsed.tag); + console.log(`Image digest: ${digest}`); + + const filename = generateImageFilename(parsed, digest); + console.log(`Target filename: ${filename}`); + + // Check if image already exists in storage + const existingContents = await client.storageContents(node.name, storage, 'vztmpl'); + const expectedVolid = `${storage}:vztmpl/${filename}.tar`; + const imageExists = existingContents.some(item => item.volid === expectedVolid); - // Wait for pull to complete - await client.waitForTask(node.name, pullUpid); - console.log('Image pulled successfully'); + if (imageExists) { + console.log(`Image already exists in storage: ${expectedVolid}`); + } else { + // Pull the image from OCI registry + const imageRef = container.template; + console.log(`Pulling image ${imageRef}...`); + const pullUpid = await client.pullOciImage(node.name, storage, { + reference: imageRef, + filename + }); + console.log(`Pull task started: ${pullUpid}`); + + // Wait for pull to complete + await client.waitForTask(node.name, pullUpid); + console.log('Image pulled successfully'); + } // Create container from the pulled image (Proxmox adds .tar to the filename) console.log(`Creating container from ${filename}.tar...`); @@ -284,11 +385,47 @@ async function main() { console.log('Container configured'); } - // Apply environment variables and entrypoint if set - const envConfig = container.buildLxcEnvConfig(); + // Apply environment variables and entrypoint + // First read defaults from the image, then merge with user-specified values + const defaultConfig = await client.lxcConfig(node.name, vmid); + const defaultEntrypoint = defaultConfig['entrypoint'] || null; + const defaultEnvStr = defaultConfig['env'] || null; + + // Parse default env vars + let mergedEnvVars = {}; + if (defaultEnvStr) { + const pairs = defaultEnvStr.split('\0'); + for (const pair of pairs) { + const eqIndex = pair.indexOf('='); + if (eqIndex > 0) { + mergedEnvVars[pair.substring(0, eqIndex)] = pair.substring(eqIndex + 1); + } + } + } + + // Merge user-specified env vars (user values override defaults) + const userEnvVars = container.environmentVars ? JSON.parse(container.environmentVars) : {}; + mergedEnvVars = { ...mergedEnvVars, ...userEnvVars }; + + // Use user entrypoint if specified, otherwise keep default + const finalEntrypoint = container.entrypoint || defaultEntrypoint; + + // Build config to apply + const envConfig = {}; + if (finalEntrypoint) { + envConfig.entrypoint = finalEntrypoint; + } + if (Object.keys(mergedEnvVars).length > 0) { + envConfig.env = Object.entries(mergedEnvVars) + .map(([key, value]) => `${key}=${value}`) + .join('\0'); + } + if (Object.keys(envConfig).length > 0) { console.log('Applying environment variables and entrypoint...'); - console.log('Config:', JSON.stringify(envConfig, null, 2)); + if (defaultEntrypoint) console.log(`Default entrypoint: ${defaultEntrypoint}`); + if (defaultEnvStr) console.log(`Default env vars: ${Object.keys(mergedEnvVars).length - Object.keys(userEnvVars).length} from image`); + if (Object.keys(userEnvVars).length > 0) console.log(`User env vars: ${Object.keys(userEnvVars).length} overrides`); await client.updateLxcConfig(node.name, vmid, envConfig); console.log('Environment/entrypoint configuration applied'); } @@ -319,6 +456,31 @@ async function main() { const macAddress = macMatch[1]; console.log(`MAC address: ${macAddress}`); + // Read back entrypoint and environment variables from config + const actualEntrypoint = config['entrypoint'] || null; + const actualEnv = config['env'] || null; + + // Parse NUL-separated env string back to JSON object + let environmentVars = {}; + if (actualEnv) { + const pairs = actualEnv.split('\0'); + for (const pair of pairs) { + const eqIndex = pair.indexOf('='); + if (eqIndex > 0) { + const key = pair.substring(0, eqIndex); + const value = pair.substring(eqIndex + 1); + environmentVars[key] = value; + } + } + } + + if (actualEntrypoint) { + console.log(`Entrypoint: ${actualEntrypoint}`); + } + if (Object.keys(environmentVars).length > 0) { + console.log(`Environment variables: ${Object.keys(environmentVars).length} vars`); + } + // Get IP address from Proxmox interfaces API const ipv4Address = await getIpFromInterfaces(client, node.name, vmid); @@ -333,6 +495,8 @@ async function main() { await container.update({ macAddress, ipv4Address, + entrypoint: actualEntrypoint, + environmentVars: JSON.stringify(environmentVars), status: 'running' }); diff --git a/create-a-container/views/containers/form.ejs b/create-a-container/views/containers/form.ejs index 561fb432..a3644d42 100644 --- a/create-a-container/views/containers/form.ejs +++ b/create-a-container/views/containers/form.ejs @@ -34,7 +34,7 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New';
<% if (isEdit) { %> - + <% } else { %> + + + <% } %>
From 67eecd77b13f0340cad94acd6fce3a8e762100e1 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 28 Jan 2026 16:15:41 -0500 Subject: [PATCH 25/26] fix: fail recreate job when container fails to come up cleanly --- .../bin/reconfigure-container.js | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/create-a-container/bin/reconfigure-container.js b/create-a-container/bin/reconfigure-container.js index e29cd77c..b08026fe 100644 --- a/create-a-container/bin/reconfigure-container.js +++ b/create-a-container/bin/reconfigure-container.js @@ -119,20 +119,29 @@ async function main() { console.log('Container started'); // Get MAC address from config (in case it wasn't captured during failed create) - const macAddress = await client.getLxcMacAddress(node.name, container.containerId) || container.macAddress; + const macAddress = await client.getLxcMacAddress(node.name, container.containerId); + + if (!macAddress) { + throw new Error('Could not get MAC address from container configuration'); + } // Get IP address from Proxmox interfaces API - const ipv4Address = await client.getLxcIpAddress(node.name, container.containerId) || container.ipv4Address; + const ipv4Address = await client.getLxcIpAddress(node.name, container.containerId); + + if (!ipv4Address) { + throw new Error('Could not get IP address from Proxmox interfaces API'); + } // Update container record with MAC/IP and running status - const updateData = { status: 'running' }; - if (macAddress) updateData.macAddress = macAddress; - if (ipv4Address) updateData.ipv4Address = ipv4Address; + await container.update({ + status: 'running', + macAddress, + ipv4Address + }); - await container.update(updateData); console.log('Status updated to: running'); - if (macAddress) console.log(` MAC: ${macAddress}`); - if (ipv4Address) console.log(` IP: ${ipv4Address}`); + console.log(` MAC: ${macAddress}`); + console.log(` IP: ${ipv4Address}`); console.log('Container reconfiguration completed successfully!'); process.exit(0); From a44e659b1ee995b711c73666b300e6a01b2cbad4 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Wed, 28 Jan 2026 16:27:16 -0500 Subject: [PATCH 26/26] fix: improve service delete UX --- create-a-container/views/containers/form.ejs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/create-a-container/views/containers/form.ejs b/create-a-container/views/containers/form.ejs index a3644d42..e1d57637 100644 --- a/create-a-container/views/containers/form.ejs +++ b/create-a-container/views/containers/form.ejs @@ -376,6 +376,17 @@ const breadcrumbLabel = isEdit ? 'Edit' : 'New'; if (!row || !deletedInput || !removeBtn) return; + // Check if this is an existing service (has an ID hidden field) + const serviceIdInput = row.querySelector('input[name*="[id]"]'); + const isExistingService = serviceIdInput && serviceIdInput.value; + + // For new services (no ID), remove immediately from DOM + if (!isExistingService) { + row.remove(); + return; + } + + // For existing services, toggle between deleted and undo states const isDeleted = deletedInput.value === 'true'; if (isDeleted) {