From 6c577d4df6b25ce886207286568069f331e9e84e Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Tue, 23 Dec 2025 12:49:20 -0700 Subject: [PATCH 1/4] Update repositories in README and add helper scripts --- README.md | 44 ++--- scripts/nexus_mirror_discovery.sh | 286 ++++++++++++++++++++++++++++++ scripts/test_nexus_proxies.sh | 30 ++++ 3 files changed, 339 insertions(+), 21 deletions(-) create mode 100755 scripts/nexus_mirror_discovery.sh create mode 100644 scripts/test_nexus_proxies.sh diff --git a/README.md b/README.md index f1e8f884..266ead21 100644 --- a/README.md +++ b/README.md @@ -218,32 +218,34 @@ the Deployer will either install the required repository or download the package | Component | Location | Protocol | Notes | | :-------- | :------- | :------- | :---- | -| Ansible Control Node | | TCP | | -| Ansible Control Node | | TCP | | -| Itential Gateway | | TCP | | -| Itential Gateway | | TCP | | -| Itential Gateway | | TCP | | -| Itential Platform | | TCP | | -| Itential Platform | | TCP | Core npm package access | -| Itential Platform | | TCP | GitHub-hosted dependencies | -| Itential Platform | | TCP | GitHub tarballs | -| MongoDB | | TCP | | -| MongoDB | | TCP | | -| MongoDB | | TCP | | -| MongoDB | | TCP | | -| Redis | | TCP | When installing Redis from the Remi repository | -| Redis | | TCP | When installing Redis from the Remi repository | -| Redis | | TCP | When installing Redis from source | -| Redis | | TCP | | -| Vault | | TCP | | +| Ansible Control Node | | TCP | Itential and community Ansible collections | +| Ansible Control Node | | TCP | Python modules | +| Itential Gateway | | TCP | Itential packages | +| Itential Gateway | | TCP | Community Ansible collections | +| Itential Gateway | | TCP | Python modules | +| Itential Platform | | TCP | Itential packages | +| Itential Platform | | TCP | Core NPM packages | +| Itential Platform | | TCP | Python modules | +| Itential Platform | | TCP | Platform Opensource Adapter packages | +| MongoDB | | TCP | MongoDB YUM RPMs | +| MongoDB | | TCP | MongoDB YUM RPMs | +| MongoDB | | TCP | MongoDB YUM repository GPG Key | +| MongoDB | | TCP | MongoDB YUM repository GPG Key | +| MongoDB | | TCP | Python modules | +| Redis | | TCP | Redis YUM RPMs
When installing Redis from the Remi repository | +| Redis | | TCP | EPEL YUM RPMsWhen installing Redis from the Remi repository | +| Redis | | TCP | Redis source packages
When installing Redis from source | +| Redis | | TCP | Redis source packages
When installing Redis from source | +| Vault | | TCP | Vault YUM RPMs | If internal YUM repositories are used, refer to the [Using Internal YUM Repositories](#using-internal-yum-repositories) section. > [! WARNING] -> The Itential Deployer nor the maintainers of the project can not know if any of the above URLs -> will result in a redirect. If a customer is using a proxy or other such method to restrict access -> this list may not represent the final URLs that are used. +> The Itential Deployer and the maintainers of the project can not know if any of the above URLs +> will result in a redirect. Reporitories may rely on CDNs or mirrors. If a customer is using a +> proxy or other such method to restrict access, this list may not represent the final URLs that are +> required. ### Ports and Networking diff --git a/scripts/nexus_mirror_discovery.sh b/scripts/nexus_mirror_discovery.sh new file mode 100755 index 00000000..0e4097c3 --- /dev/null +++ b/scripts/nexus_mirror_discovery.sh @@ -0,0 +1,286 @@ +#!/bin/bash +# nexus_mirror_discovery.sh +# Discovers mirrors, redirects, and CDN endpoints for upstream repositories + +set -eo pipefail + +# Color output for terminal +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Output files +MARKDOWN_OUTPUT="mirror_discovery_$(date +%Y%m%d_%H%M%S).md" +CSV_OUTPUT="mirror_discovery_$(date +%Y%m%d_%H%M%S).csv" + +# URL definitions - format: "name|url" +URLS=( + "PyPI|https://pypi.org/simple/" + "Ansible_Galaxy|https://galaxy.ansible.com/api/" + "Itential_Registry|https://registry.aws.itential.com" + "NPM_Registry|https://registry.npmjs.org/" + "GitHub|https://github.com" + "GitHub_Codeload|https://codeload.github.com" + "MongoDB_Repo|https://repo.mongodb.org/yum/redhat/8/mongodb-org/7.0/x86_64/" + "MongoDB_WWW|https://www.mongodb.org" + "MongoDB_GPG_org|https://pgp.mongodb.org/server-7.0.asc" + "MongoDB_GPG_com|https://pgp.mongodb.com/server-7.0.asc" + "Remi_Repo|http://rpms.remirepo.net/enterprise/8/remi/x86_64/" + "Fedora_EPEL|https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/" + "HashiCorp_Vault|https://rpm.releases.hashicorp.com/RHEL/8/x86_64/stable/" +) + +# Function to extract domain from URL +extract_domain() { + echo "$1" | awk -F[/:] '{print $4}' +} + +# Function to check DNS resolution +check_dns() { + local domain="$1" + local result + + # Get A records + result=$(dig +short "$domain" A 2>/dev/null | head -5 | tr '\n' ',' | sed 's/,$//') + if [ -z "$result" ]; then + echo "No A records" + return + fi + + # Check for CNAME + local cname=$(dig +short "$domain" CNAME 2>/dev/null | tr '\n' ',' | sed 's/,$//') + if [ -n "$cname" ]; then + echo "CNAME: $cname - IPs: $result" + else + echo "Direct: $result" + fi +} + +# Function to check HTTP redirects +check_redirects() { + local url="$1" + local output + + # Follow redirects and capture headers + output=$(curl -sIL -m 10 "$url" 2>/dev/null || echo "CURL_ERROR") + + if [ "$output" = "CURL_ERROR" ]; then + echo "Connection failed" + return 1 + fi + + # Extract all Location headers + local locations=$(echo "$output" | grep -i "^Location:" | sed 's/Location: //i' | tr -d '\r') + + if [ -n "$locations" ]; then + echo "$locations" + else + echo "No redirects" + fi +} + +# Function to identify CDN +identify_cdn() { + local url="$1" + local headers + + headers=$(curl -sI -m 10 "$url" 2>/dev/null || echo "") + + # Check common CDN headers + if echo "$headers" | grep -qi "x-amz-cf-id\|cloudfront"; then + echo "CloudFront" + elif echo "$headers" | grep -qi "x-cache.*fastly\|fastly"; then + echo "Fastly" + elif echo "$headers" | grep -qi "x-akamai\|akamai"; then + echo "Akamai" + elif echo "$headers" | grep -qi "x-cache.*cloudflare\|cf-ray"; then + echo "Cloudflare" + elif echo "$headers" | grep -qi "x-served-by.*varnish"; then + echo "Varnish Cache" + else + echo "Unknown/Direct" + fi +} + +# Function to check if URL requires authentication +check_auth() { + local url="$1" + local status + + status=$(curl -sI -m 10 "$url" 2>/dev/null | grep "^HTTP" | tail -1 | awk '{print $2}') + + case "$status" in + 200|301|302|303|307|308) + echo "Public" + ;; + 401|403) + echo "Auth Required" + ;; + *) + echo "Unknown ($status)" + ;; + esac +} + +# Initialize output files +cat > "$MARKDOWN_OUTPUT" << EOF +# Nexus Upstream Mirror Discovery Report + +**Generated:** $(date '+%Y-%m-%d %H:%M:%S') +**Purpose:** Document actual mirrors, redirects, and CDN endpoints for Nexus proxy configuration + +## Summary + +This report identifies the actual endpoints that upstream repositories use, including: +- DNS resolution and CNAMEs +- HTTP redirects +- CDN providers +- Authentication requirements + +## Detailed Findings + +EOF + +cat > "$CSV_OUTPUT" << 'EOF' +"Component","Base URL","DNS Resolution","HTTP Redirects","CDN Provider","Auth Status","Notes" +EOF + +echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Nexus Upstream Mirror Discovery Tool ║${NC}" +echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "Output files:" +echo -e " - Markdown: ${GREEN}$MARKDOWN_OUTPUT${NC}" +echo -e " - CSV: ${GREEN}$CSV_OUTPUT${NC}" +echo "" + +# Process each URL +total=${#URLS[@]} +current=0 + +for entry in "${URLS[@]}"; do + current=$((current + 1)) + + # Split entry into name and url + name=$(echo "$entry" | cut -d'|' -f1) + url=$(echo "$entry" | cut -d'|' -f2) + domain=$(extract_domain "$url") + + echo -e "\n${YELLOW}[$current/$total]${NC} Processing: ${GREEN}$name${NC}" + echo -e " URL: $url" + + # Gather information + echo -n " DNS: " + dns_info=$(check_dns "$domain") + echo "$dns_info" + + echo -n " Redirects: " + redirect_info=$(check_redirects "$url") + echo "$redirect_info" + + echo -n " CDN: " + cdn_info=$(identify_cdn "$url") + echo "$cdn_info" + + echo -n " Auth: " + auth_info=$(check_auth "$url") + echo "$auth_info" + + # Write to Markdown + cat >> "$MARKDOWN_OUTPUT" << EOF + +### $name + +**Base URL:** \`$url\` + +| Attribute | Value | +|-----------|-------| +| Domain | $domain | +| DNS Resolution | $dns_info | +| HTTP Redirects | $redirect_info | +| CDN Provider | $cdn_info | +| Authentication | $auth_info | + +**Nexus Configuration:** +\`\`\` +Remote URL: $url +\`\`\` + +EOF + + # Write to CSV (escape quotes and commas) + dns_csv=$(echo "$dns_info" | tr '\n' ';' | sed 's/"/\\"/g') + redirect_csv=$(echo "$redirect_info" | tr '\n' ';' | sed 's/"/\\"/g') + + echo "\"$name\",\"$url\",\"$dns_csv\",\"$redirect_csv\",\"$cdn_info\",\"$auth_info\",\"\"" >> "$CSV_OUTPUT" + + # Small delay to be respectful + sleep 0.5 +done + +# Add recommendations section to markdown +cat >> "$MARKDOWN_OUTPUT" << 'EOF' + +## Recommendations for Nexus Proxy Configuration + +### High Priority Mirrors + +Based on the discovery, these repositories should be prioritized for Nexus proxy setup: + +1. **PyPI (files.pythonhosted.org)** - High traffic, CDN-backed +2. **NPM Registry** - Frequent updates, large artifact count +3. **MongoDB Repository** - Mission-critical packages +4. **GitHub/Codeload** - Source tarball dependencies + +### Configuration Notes + +- **Use base URLs** in Nexus proxy configuration (e.g., `https://pypi.org/simple/`) +- Nexus will automatically follow redirects to CDN endpoints +- For repositories with mirrorlists (EPEL), use the primary URL +- Configure appropriate cache TTLs based on update frequency + +### Authentication Requirements + +Repositories requiring authentication should be configured with appropriate credentials in Nexus: +- Itential Registry: Requires authentication token +- Private GitHub repositories: Requires PAT or SSH key + +### GPG Key Handling + +For YUM repositories, import GPG keys separately or host in Nexus raw repository: +- MongoDB: `https://pgp.mongodb.com/server-7.0.asc` +- HashiCorp: Available in repository metadata + +EOF + +echo "" +echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Discovery Complete! ║${NC}" +echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "Reports generated:" +echo -e " 📄 Markdown: ${GREEN}$MARKDOWN_OUTPUT${NC}" +echo -e " 📊 CSV: ${GREEN}$CSV_OUTPUT${NC}" +echo "" +echo -e "Next steps:" +echo -e " 1. Review the generated reports" +echo -e " 2. Configure Nexus proxy repositories using base URLs" +echo -e " 3. Test proxy access from your air-gapped environment" +echo -e " 4. Update your Ansible playbooks to use Nexus URLs" +echo "" + +# Optional: Display summary table +echo -e "${BLUE}Quick Summary:${NC}" +echo "" +printf "%-25s %-15s %-20s\n" "Component" "CDN Provider" "Auth Status" +echo "───────────────────────────────────────────────────────────────" + +for entry in "${URLS[@]}"; do + name=$(echo "$entry" | cut -d'|' -f1) + url=$(echo "$entry" | cut -d'|' -f2) + cdn=$(identify_cdn "$url") + auth=$(check_auth "$url") + printf "%-25s %-15s %-20s\n" "$name" "$cdn" "$auth" +done diff --git a/scripts/test_nexus_proxies.sh b/scripts/test_nexus_proxies.sh new file mode 100644 index 00000000..ec7dc04a --- /dev/null +++ b/scripts/test_nexus_proxies.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# test_nexus_proxies.sh +# Test that Nexus proxies are working correctly + +NEXUS_BASE="https://nexus.yourdomain.com/repository" + +declare -A NEXUS_REPOS=( + ["pypi-proxy"]="simple/" + ["mongodb-proxy"]="repodata/repomd.xml" + ["npm-proxy"]="react" + ["hashicorp-proxy"]="repodata/repomd.xml" +) + +echo "Testing Nexus Proxy Access" +echo "===========================" + +for repo in "${!NEXUS_REPOS[@]}"; do + path="${NEXUS_REPOS[$repo]}" + url="$NEXUS_BASE/$repo/$path" + + echo -n "Testing $repo... " + + status=$(curl -s -o /dev/null -w "%{http_code}" "$url") + + if [[ "$status" == "200" ]]; then + echo "✓ OK" + else + echo "✗ FAILED (HTTP $status)" + fi +done From caa82651e15ddc5b6c2614347eb283dc96a5b0e9 Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Tue, 30 Dec 2025 13:44:36 -0700 Subject: [PATCH 2/4] Fix typo in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 266ead21..f321420c 100644 --- a/README.md +++ b/README.md @@ -226,14 +226,14 @@ the Deployer will either install the required repository or download the package | Itential Platform | | TCP | Itential packages | | Itential Platform | | TCP | Core NPM packages | | Itential Platform | | TCP | Python modules | -| Itential Platform | | TCP | Platform Opensource Adapter packages | +| Itential Platform | | TCP | Platform Opensource Adapters NodeJS source repository | | MongoDB | | TCP | MongoDB YUM RPMs | | MongoDB | | TCP | MongoDB YUM RPMs | | MongoDB | | TCP | MongoDB YUM repository GPG Key | | MongoDB | | TCP | MongoDB YUM repository GPG Key | | MongoDB | | TCP | Python modules | | Redis | | TCP | Redis YUM RPMs
When installing Redis from the Remi repository | -| Redis | | TCP | EPEL YUM RPMsWhen installing Redis from the Remi repository | +| Redis | | TCP | EPEL YUM RPMs
When installing Redis from the Remi repository | | Redis | | TCP | Redis source packages
When installing Redis from source | | Redis | | TCP | Redis source packages
When installing Redis from source | | Vault | | TCP | Vault YUM RPMs | From 09d752c996468f62f0d417a543b19597fb4ae6d2 Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Tue, 30 Dec 2025 13:46:10 -0700 Subject: [PATCH 3/4] Update Nexus repo analysis scripts --- scripts/nexus_mirror_discovery.sh | 66 +++++++++++++++---------------- scripts/test_nexus_proxies.sh | 2 +- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/scripts/nexus_mirror_discovery.sh b/scripts/nexus_mirror_discovery.sh index 0e4097c3..6d76ce40 100755 --- a/scripts/nexus_mirror_discovery.sh +++ b/scripts/nexus_mirror_discovery.sh @@ -71,8 +71,8 @@ check_redirects() { return 1 fi - # Extract all Location headers - local locations=$(echo "$output" | grep -i "^Location:" | sed 's/Location: //i' | tr -d '\r') + # Extract all Location headers and join with arrow separator + local locations=$(echo "$output" | grep -i "^Location:" | sed 's/Location: //i' | tr -d '\r' | tr '\n' ' → ' | sed 's/ → $//') if [ -n "$locations" ]; then echo "$locations" @@ -147,14 +147,14 @@ cat > "$CSV_OUTPUT" << 'EOF' "Component","Base URL","DNS Resolution","HTTP Redirects","CDN Provider","Auth Status","Notes" EOF -echo -e "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${BLUE}║ Nexus Upstream Mirror Discovery Tool ║${NC}" -echo -e "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo -e "Output files:" -echo -e " - Markdown: ${GREEN}$MARKDOWN_OUTPUT${NC}" -echo -e " - CSV: ${GREEN}$CSV_OUTPUT${NC}" -echo "" +printf "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}\n" +printf "${BLUE}║ Nexus Upstream Mirror Discovery Tool ║${NC}\n" +printf "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}\n" +printf "\n" +printf "Output files:\n" +printf " - Markdown: ${GREEN}%s${NC}\n" "$MARKDOWN_OUTPUT" +printf " - CSV: ${GREEN}%s${NC}\n" "$CSV_OUTPUT" +printf "\n" # Process each URL total=${#URLS[@]} @@ -168,23 +168,23 @@ for entry in "${URLS[@]}"; do url=$(echo "$entry" | cut -d'|' -f2) domain=$(extract_domain "$url") - echo -e "\n${YELLOW}[$current/$total]${NC} Processing: ${GREEN}$name${NC}" - echo -e " URL: $url" + printf "\n${YELLOW}[%d/%d]${NC} Processing: ${GREEN}%s${NC}\n" "$current" "$total" "$name" + printf " URL: %s\n" "$url" # Gather information - echo -n " DNS: " + printf " DNS: " dns_info=$(check_dns "$domain") echo "$dns_info" - echo -n " Redirects: " + printf " Redirects: " redirect_info=$(check_redirects "$url") echo "$redirect_info" - echo -n " CDN: " + printf " CDN: " cdn_info=$(identify_cdn "$url") echo "$cdn_info" - echo -n " Auth: " + printf " Auth: " auth_info=$(check_auth "$url") echo "$auth_info" @@ -255,25 +255,25 @@ For YUM repositories, import GPG keys separately or host in Nexus raw repository EOF -echo "" -echo -e "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║ Discovery Complete! ║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo -e "Reports generated:" -echo -e " 📄 Markdown: ${GREEN}$MARKDOWN_OUTPUT${NC}" -echo -e " 📊 CSV: ${GREEN}$CSV_OUTPUT${NC}" -echo "" -echo -e "Next steps:" -echo -e " 1. Review the generated reports" -echo -e " 2. Configure Nexus proxy repositories using base URLs" -echo -e " 3. Test proxy access from your air-gapped environment" -echo -e " 4. Update your Ansible playbooks to use Nexus URLs" -echo "" +printf "\n" +printf "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}\n" +printf "${GREEN}║ Discovery Complete! ║${NC}\n" +printf "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}\n" +printf "\n" +printf "Reports generated:\n" +printf " 📄 Markdown: ${GREEN}%s${NC}\n" "$MARKDOWN_OUTPUT" +printf " 📊 CSV: ${GREEN}%s${NC}\n" "$CSV_OUTPUT" +printf "\n" +printf "Next steps:\n" +printf " 1. Review the generated reports\n" +printf " 2. Configure Nexus proxy repositories using base URLs\n" +printf " 3. Test proxy access from your air-gapped environment\n" +printf " 4. Update your Ansible playbooks to use Nexus URLs\n" +printf "\n" # Optional: Display summary table -echo -e "${BLUE}Quick Summary:${NC}" -echo "" +printf "${BLUE}Quick Summary:${NC}\n" +printf "\n" printf "%-25s %-15s %-20s\n" "Component" "CDN Provider" "Auth Status" echo "───────────────────────────────────────────────────────────────" diff --git a/scripts/test_nexus_proxies.sh b/scripts/test_nexus_proxies.sh index ec7dc04a..c569b65d 100644 --- a/scripts/test_nexus_proxies.sh +++ b/scripts/test_nexus_proxies.sh @@ -18,7 +18,7 @@ for repo in "${!NEXUS_REPOS[@]}"; do path="${NEXUS_REPOS[$repo]}" url="$NEXUS_BASE/$repo/$path" - echo -n "Testing $repo... " + echo "Testing $repo... " status=$(curl -s -o /dev/null -w "%{http_code}" "$url") From fb3ab62abb07399330151b03be06255098e8de9d Mon Sep 17 00:00:00 2001 From: Kevin Velarde Date: Tue, 30 Dec 2025 13:47:34 -0700 Subject: [PATCH 4/4] Remove Nexus repo analysis scripts --- scripts/nexus_mirror_discovery.sh | 286 ------------------------------ scripts/test_nexus_proxies.sh | 30 ---- 2 files changed, 316 deletions(-) delete mode 100755 scripts/nexus_mirror_discovery.sh delete mode 100644 scripts/test_nexus_proxies.sh diff --git a/scripts/nexus_mirror_discovery.sh b/scripts/nexus_mirror_discovery.sh deleted file mode 100755 index 6d76ce40..00000000 --- a/scripts/nexus_mirror_discovery.sh +++ /dev/null @@ -1,286 +0,0 @@ -#!/bin/bash -# nexus_mirror_discovery.sh -# Discovers mirrors, redirects, and CDN endpoints for upstream repositories - -set -eo pipefail - -# Color output for terminal -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Output files -MARKDOWN_OUTPUT="mirror_discovery_$(date +%Y%m%d_%H%M%S).md" -CSV_OUTPUT="mirror_discovery_$(date +%Y%m%d_%H%M%S).csv" - -# URL definitions - format: "name|url" -URLS=( - "PyPI|https://pypi.org/simple/" - "Ansible_Galaxy|https://galaxy.ansible.com/api/" - "Itential_Registry|https://registry.aws.itential.com" - "NPM_Registry|https://registry.npmjs.org/" - "GitHub|https://github.com" - "GitHub_Codeload|https://codeload.github.com" - "MongoDB_Repo|https://repo.mongodb.org/yum/redhat/8/mongodb-org/7.0/x86_64/" - "MongoDB_WWW|https://www.mongodb.org" - "MongoDB_GPG_org|https://pgp.mongodb.org/server-7.0.asc" - "MongoDB_GPG_com|https://pgp.mongodb.com/server-7.0.asc" - "Remi_Repo|http://rpms.remirepo.net/enterprise/8/remi/x86_64/" - "Fedora_EPEL|https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/" - "HashiCorp_Vault|https://rpm.releases.hashicorp.com/RHEL/8/x86_64/stable/" -) - -# Function to extract domain from URL -extract_domain() { - echo "$1" | awk -F[/:] '{print $4}' -} - -# Function to check DNS resolution -check_dns() { - local domain="$1" - local result - - # Get A records - result=$(dig +short "$domain" A 2>/dev/null | head -5 | tr '\n' ',' | sed 's/,$//') - if [ -z "$result" ]; then - echo "No A records" - return - fi - - # Check for CNAME - local cname=$(dig +short "$domain" CNAME 2>/dev/null | tr '\n' ',' | sed 's/,$//') - if [ -n "$cname" ]; then - echo "CNAME: $cname - IPs: $result" - else - echo "Direct: $result" - fi -} - -# Function to check HTTP redirects -check_redirects() { - local url="$1" - local output - - # Follow redirects and capture headers - output=$(curl -sIL -m 10 "$url" 2>/dev/null || echo "CURL_ERROR") - - if [ "$output" = "CURL_ERROR" ]; then - echo "Connection failed" - return 1 - fi - - # Extract all Location headers and join with arrow separator - local locations=$(echo "$output" | grep -i "^Location:" | sed 's/Location: //i' | tr -d '\r' | tr '\n' ' → ' | sed 's/ → $//') - - if [ -n "$locations" ]; then - echo "$locations" - else - echo "No redirects" - fi -} - -# Function to identify CDN -identify_cdn() { - local url="$1" - local headers - - headers=$(curl -sI -m 10 "$url" 2>/dev/null || echo "") - - # Check common CDN headers - if echo "$headers" | grep -qi "x-amz-cf-id\|cloudfront"; then - echo "CloudFront" - elif echo "$headers" | grep -qi "x-cache.*fastly\|fastly"; then - echo "Fastly" - elif echo "$headers" | grep -qi "x-akamai\|akamai"; then - echo "Akamai" - elif echo "$headers" | grep -qi "x-cache.*cloudflare\|cf-ray"; then - echo "Cloudflare" - elif echo "$headers" | grep -qi "x-served-by.*varnish"; then - echo "Varnish Cache" - else - echo "Unknown/Direct" - fi -} - -# Function to check if URL requires authentication -check_auth() { - local url="$1" - local status - - status=$(curl -sI -m 10 "$url" 2>/dev/null | grep "^HTTP" | tail -1 | awk '{print $2}') - - case "$status" in - 200|301|302|303|307|308) - echo "Public" - ;; - 401|403) - echo "Auth Required" - ;; - *) - echo "Unknown ($status)" - ;; - esac -} - -# Initialize output files -cat > "$MARKDOWN_OUTPUT" << EOF -# Nexus Upstream Mirror Discovery Report - -**Generated:** $(date '+%Y-%m-%d %H:%M:%S') -**Purpose:** Document actual mirrors, redirects, and CDN endpoints for Nexus proxy configuration - -## Summary - -This report identifies the actual endpoints that upstream repositories use, including: -- DNS resolution and CNAMEs -- HTTP redirects -- CDN providers -- Authentication requirements - -## Detailed Findings - -EOF - -cat > "$CSV_OUTPUT" << 'EOF' -"Component","Base URL","DNS Resolution","HTTP Redirects","CDN Provider","Auth Status","Notes" -EOF - -printf "${BLUE}╔════════════════════════════════════════════════════════════════╗${NC}\n" -printf "${BLUE}║ Nexus Upstream Mirror Discovery Tool ║${NC}\n" -printf "${BLUE}╚════════════════════════════════════════════════════════════════╝${NC}\n" -printf "\n" -printf "Output files:\n" -printf " - Markdown: ${GREEN}%s${NC}\n" "$MARKDOWN_OUTPUT" -printf " - CSV: ${GREEN}%s${NC}\n" "$CSV_OUTPUT" -printf "\n" - -# Process each URL -total=${#URLS[@]} -current=0 - -for entry in "${URLS[@]}"; do - current=$((current + 1)) - - # Split entry into name and url - name=$(echo "$entry" | cut -d'|' -f1) - url=$(echo "$entry" | cut -d'|' -f2) - domain=$(extract_domain "$url") - - printf "\n${YELLOW}[%d/%d]${NC} Processing: ${GREEN}%s${NC}\n" "$current" "$total" "$name" - printf " URL: %s\n" "$url" - - # Gather information - printf " DNS: " - dns_info=$(check_dns "$domain") - echo "$dns_info" - - printf " Redirects: " - redirect_info=$(check_redirects "$url") - echo "$redirect_info" - - printf " CDN: " - cdn_info=$(identify_cdn "$url") - echo "$cdn_info" - - printf " Auth: " - auth_info=$(check_auth "$url") - echo "$auth_info" - - # Write to Markdown - cat >> "$MARKDOWN_OUTPUT" << EOF - -### $name - -**Base URL:** \`$url\` - -| Attribute | Value | -|-----------|-------| -| Domain | $domain | -| DNS Resolution | $dns_info | -| HTTP Redirects | $redirect_info | -| CDN Provider | $cdn_info | -| Authentication | $auth_info | - -**Nexus Configuration:** -\`\`\` -Remote URL: $url -\`\`\` - -EOF - - # Write to CSV (escape quotes and commas) - dns_csv=$(echo "$dns_info" | tr '\n' ';' | sed 's/"/\\"/g') - redirect_csv=$(echo "$redirect_info" | tr '\n' ';' | sed 's/"/\\"/g') - - echo "\"$name\",\"$url\",\"$dns_csv\",\"$redirect_csv\",\"$cdn_info\",\"$auth_info\",\"\"" >> "$CSV_OUTPUT" - - # Small delay to be respectful - sleep 0.5 -done - -# Add recommendations section to markdown -cat >> "$MARKDOWN_OUTPUT" << 'EOF' - -## Recommendations for Nexus Proxy Configuration - -### High Priority Mirrors - -Based on the discovery, these repositories should be prioritized for Nexus proxy setup: - -1. **PyPI (files.pythonhosted.org)** - High traffic, CDN-backed -2. **NPM Registry** - Frequent updates, large artifact count -3. **MongoDB Repository** - Mission-critical packages -4. **GitHub/Codeload** - Source tarball dependencies - -### Configuration Notes - -- **Use base URLs** in Nexus proxy configuration (e.g., `https://pypi.org/simple/`) -- Nexus will automatically follow redirects to CDN endpoints -- For repositories with mirrorlists (EPEL), use the primary URL -- Configure appropriate cache TTLs based on update frequency - -### Authentication Requirements - -Repositories requiring authentication should be configured with appropriate credentials in Nexus: -- Itential Registry: Requires authentication token -- Private GitHub repositories: Requires PAT or SSH key - -### GPG Key Handling - -For YUM repositories, import GPG keys separately or host in Nexus raw repository: -- MongoDB: `https://pgp.mongodb.com/server-7.0.asc` -- HashiCorp: Available in repository metadata - -EOF - -printf "\n" -printf "${GREEN}╔════════════════════════════════════════════════════════════════╗${NC}\n" -printf "${GREEN}║ Discovery Complete! ║${NC}\n" -printf "${GREEN}╚════════════════════════════════════════════════════════════════╝${NC}\n" -printf "\n" -printf "Reports generated:\n" -printf " 📄 Markdown: ${GREEN}%s${NC}\n" "$MARKDOWN_OUTPUT" -printf " 📊 CSV: ${GREEN}%s${NC}\n" "$CSV_OUTPUT" -printf "\n" -printf "Next steps:\n" -printf " 1. Review the generated reports\n" -printf " 2. Configure Nexus proxy repositories using base URLs\n" -printf " 3. Test proxy access from your air-gapped environment\n" -printf " 4. Update your Ansible playbooks to use Nexus URLs\n" -printf "\n" - -# Optional: Display summary table -printf "${BLUE}Quick Summary:${NC}\n" -printf "\n" -printf "%-25s %-15s %-20s\n" "Component" "CDN Provider" "Auth Status" -echo "───────────────────────────────────────────────────────────────" - -for entry in "${URLS[@]}"; do - name=$(echo "$entry" | cut -d'|' -f1) - url=$(echo "$entry" | cut -d'|' -f2) - cdn=$(identify_cdn "$url") - auth=$(check_auth "$url") - printf "%-25s %-15s %-20s\n" "$name" "$cdn" "$auth" -done diff --git a/scripts/test_nexus_proxies.sh b/scripts/test_nexus_proxies.sh deleted file mode 100644 index c569b65d..00000000 --- a/scripts/test_nexus_proxies.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# test_nexus_proxies.sh -# Test that Nexus proxies are working correctly - -NEXUS_BASE="https://nexus.yourdomain.com/repository" - -declare -A NEXUS_REPOS=( - ["pypi-proxy"]="simple/" - ["mongodb-proxy"]="repodata/repomd.xml" - ["npm-proxy"]="react" - ["hashicorp-proxy"]="repodata/repomd.xml" -) - -echo "Testing Nexus Proxy Access" -echo "===========================" - -for repo in "${!NEXUS_REPOS[@]}"; do - path="${NEXUS_REPOS[$repo]}" - url="$NEXUS_BASE/$repo/$path" - - echo "Testing $repo... " - - status=$(curl -s -o /dev/null -w "%{http_code}" "$url") - - if [[ "$status" == "200" ]]; then - echo "✓ OK" - else - echo "✗ FAILED (HTTP $status)" - fi -done