Merge remote-tracking branch 'origin/develop' into develop

Author: heinrichb

cmd/scrapeycli/main.go

@@ -1,12 +1,15 @@
 package main
 
 import (
-	"flag"
-	"os"
-
-	"github.com/fatih/color"
-	"github.com/heinrichb/scrapey-cli/pkg/config"
-	"github.com/heinrichb/scrapey-cli/pkg/utils"
+    "flag"
+    "fmt"
+    "os"
+    "path"
+
+    "github.com/fatih/color"
+    "github.com/heinrichb/scrapey-cli/pkg/config"
+    "github.com/heinrichb/scrapey-cli/pkg/crawler"
+    "github.com/heinrichb/scrapey-cli/pkg/utils"
 )
 
 /*
@@ -58,27 +61,27 @@ It parses command-line flags, prints a welcome message, loads the configuration,
 applies CLI overrides using a ConfigOverride object, and prints confirmation messages.
 */
 func main() {
-	// Parse CLI flags.
-	flag.Parse()
+    // Parse CLI flags.
+    flag.Parse()
 
 	// Store the verbose flag in global state.
 	config.Verbose = verbose
 
 	// Print a welcome message in cyan using our PrintColored utility.
 	utils.PrintColored("Welcome to Scrapey CLI!", "", color.FgCyan)
 
-	// Default to "configs/default.json" if no config path is provided.
-	if configPath == "" {
-		configPath = "configs/default.json"
-	}
+    // Default to "configs/default.json" if no config path is provided.
+    if configPath == "" {
+        configPath = "configs/default.json"
+    }
 
-	// Attempt to load the configuration from the specified file.
-	cfg, err := config.Load(configPath)
-	if err != nil {
-		// If loading fails, print an error message in red and exit.
-		utils.PrintColored("Failed to load config: ", err.Error(), color.FgRed)
-		os.Exit(1)
-	}
+    // Attempt to load the configuration from the specified file.
+    cfg, err := config.Load(configPath)
+    if err != nil {
+        // If loading fails, print an error message in red and exit.
+        utils.PrintColored("Failed to load config: ", err.Error(), color.FgRed)
+        os.Exit(1)
+    }
 
 	// Construct a partial ConfigOverride struct for CLI overrides.
 	cliOverrides := config.ConfigOverride{}
@@ -128,10 +131,29 @@ func main() {
 
 	// Print which routes will be scraped.
 	utils.PrintColored("Base URL: ", cfg.URL.Base, color.FgYellow)
-	if cfg.URL.IncludeBase {
+    // Create a new Crawler instance
+    c := crawler.New()
+	
+    if cfg.URL.IncludeBase {
 		utils.PrintColored("Including base URL in scraping.", "", color.FgGreen)
+        content, err := c.FetchURL(cfg.URL.Base)
+        if err != nil {
+            utils.PrintColored("Failed to fetch URL: ", err.Error(), color.FgRed)
+            os.Exit(1) // To-Do Handle error without exiting program
+        }
+        // Print the fetched content
+        fmt.Println("Fetched Content:")
+        fmt.Println(content)
 	}
 	for _, route := range cfg.URL.Routes {
 		utils.PrintColored("Scraping route: ", route, color.FgHiBlue)
+        content, err := c.FetchURL(path.Join(cfg.URL.Base,route))
+        if err != nil {
+            utils.PrintColored("Failed to fetch URL: ", err.Error(), color.FgRed)
+            os.Exit(1) // To-Do Handle error without exiting program
+        }
+        // Print the fetched content
+        fmt.Println("Fetched Content:")
+        fmt.Println(content)
 	}
 }

configs/default.json

@@ -3,7 +3,7 @@
 	"url": {
 		"base": "https://example.com",
 		"routes": ["/route1", "/route2", "*"],
-		"includeBase": false
+		"includeBase": true
 	},
 	"parseRules": {
 		"title": "title",

pkg/crawler/crawler.go

@@ -2,6 +2,13 @@
 
 package crawler
 
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
 /*
 Crawler is responsible for fetching HTML content from URLs.
 
@@ -53,5 +60,34 @@ Notes:
 */
 func (c *Crawler) FetchURL(url string) (string, error) {
 	// Stub: return placeholder HTML or empty string for now.
-	return "", nil
+	client := &http.Client{
+		Timeout: 10 * time.Second, // Set timeout
+	}
+
+	// jsonData := `{"key":"value"}`
+
+	// Create a custom request
+	// req, err := http.NewRequest("Post", url, bytes.NewBuffer([]byte(jsonData)))
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		fmt.Printf("Error creating request: %v\n", err)
+		return "", err
+	}
+	// req.Header.Set("Content-Type", "application/json") // Set headers
+
+	// Send the request
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Printf("Error sending request: %v\n", err)
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	// Read and print the response body
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Printf("Error reading response body: %v\n", err)
+		return "", err
+	}
+	return string(body), nil
 }