Skip to content

Commit a34773a

Browse files
heinrichbheinrichb
committed
Updating readme and some comments
1 parent 318b6d4 commit a34773a

4 files changed

Lines changed: 190 additions & 186 deletions

File tree

‎.gitignore‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
1-
# File: .gitignore
2-
# Go binaries for programs and plugins
3-
*.exe
4-
*.exe~
5-
*.dll
6-
*.so
7-
*.dylib
8-
9-
# Build artifacts and local bin directories
10-
bin
11-
build
12-
dist
13-
14-
# Test binaries (built with `go test -c`)
15-
*.test
16-
17-
# Dependency directories
18-
vendor
19-
20-
# Go workspace files
21-
go.work
22-
go.work.sum
23-
24-
# Editor/OS-specific
25-
.DS_Store
26-
.idea
27-
28-
# Environment variables
1+
# File: .gitignore
2+
# Go binaries for programs and plugins
3+
*.exe
4+
*.exe~
5+
*.dll
6+
*.so
7+
*.dylib
8+
9+
# Build artifacts and local bin directories
10+
bin
11+
build
12+
dist
13+
14+
# Test binaries (built with `go test -c`)
15+
*.test
16+
17+
# Dependency directories
18+
vendor
19+
20+
# Go workspace files
21+
go.work
22+
go.work.sum
23+
24+
# Editor/OS-specific
25+
.DS_Store
26+
.idea
27+
28+
# Environment variables
2929
.env

‎Makefile‎

Lines changed: 137 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -1,138 +1,138 @@
1-
# File: Makefile
2-
# Purpose:
3-
# - "install": ensure modules are tidy if go.mod or go.sum changed.
4-
# - "test": run coverage if code changed.
5-
# - "coverage": display coverage summary from coverage.out
6-
# - "build": recompile binary if Go source or mod files changed.
7-
# - "run": executes the compiled binary
8-
# - "tree": display directory structure
9-
# - All skip with "No changes detected, skipping X." if nothing changed.
10-
11-
.PHONY: install test coverage build run tree
12-
13-
# Directories for build artifacts and stamp files
14-
BUILD_DIR := build
15-
STAMPS_DIR := $(BUILD_DIR)/.stamps
16-
17-
# Stamp file paths for each step
18-
INSTALL_STAMP := $(STAMPS_DIR)/install.stamp
19-
BUILD_STAMP := $(STAMPS_DIR)/build.stamp
20-
TEST_STAMP := $(STAMPS_DIR)/test.stamp
21-
22-
BINARY := $(BUILD_DIR)/scrapeycli
23-
24-
# Coverage output
25-
COVER_DIR := ${BUILD_DIR}/coverage
26-
COVER_PROFILE := $(COVER_DIR)/coverage.txt
27-
COVER_HTML := $(COVER_DIR)/coverage.html
28-
29-
# All Go source files (including _test.go files)
30-
GO_FILES := $(shell find . -type f -name '*.go' -not -path "./vendor/*")
31-
32-
# Reusable messages
33-
SKIP_MSG := No changes detected, skipping
34-
CHANGE_MSG := Some files changed; re-running
35-
36-
# ------------------------------------------------------------------------------
37-
# install: ensure go.mod/go.sum are tidy if changed.
38-
# ------------------------------------------------------------------------------
39-
install:
40-
@mkdir -p $(STAMPS_DIR)
41-
@TARGET=install; \
42-
if [ ! -f "$(INSTALL_STAMP)" ] || [ go.mod -nt "$(INSTALL_STAMP)" ] || [ go.sum -nt "$(INSTALL_STAMP)" ]; then \
43-
echo "$(CHANGE_MSG) $$TARGET..."; \
44-
go mod tidy; \
45-
if ! git diff --exit-code go.sum; then \
46-
echo "go.sum updated. Please commit the changes."; \
47-
exit 1; \
48-
fi; \
49-
touch "$(INSTALL_STAMP)"; \
50-
echo "Done with installing."; \
51-
else \
52-
echo "$(SKIP_MSG) $$TARGET."; \
53-
fi
54-
55-
# ------------------------------------------------------------------------------
56-
# test: run tests and update coverage if any Go source (including _test.go files) have changed.
57-
# Ensures gotestsum is installed before running tests.
58-
# Depends on install.
59-
# ------------------------------------------------------------------------------
60-
test: install
61-
@if ! command -v gotestsum >/dev/null 2>&1; then \
62-
echo "Installing gotestsum..."; \
63-
go install gotest.tools/gotestsum@latest; \
64-
fi
65-
@mkdir -p $(COVER_DIR) $(STAMPS_DIR)
66-
@TARGET=test; \
67-
if [ ! -f "$(TEST_STAMP)" ] || [ -n "$$(find $(GO_FILES) -newer "$(TEST_STAMP)" 2>/dev/null)" ]; then \
68-
echo "$(CHANGE_MSG) $$TARGET..."; \
69-
> "$(COVER_PROFILE)"; \
70-
if gotestsum --format short-verbose ./... && \
71-
go test -cover -covermode=atomic -coverpkg=./... -coverprofile="$(COVER_PROFILE)" ./... >/dev/null; then \
72-
if [ -f "$(COVER_PROFILE)" ]; then \
73-
go tool cover -html="$(COVER_PROFILE)" -o "$(COVER_HTML)"; \
74-
echo "Coverage file generated at: $(COVER_PROFILE)"; \
75-
echo "HTML coverage report at: $(COVER_HTML)"; \
76-
else \
77-
echo "ERROR: Coverage file was not generated!"; \
78-
fi; \
79-
touch "$(TEST_STAMP)"; \
80-
else \
81-
echo "Tests failed! Skipping stamp update."; \
82-
exit 1; \
83-
fi; \
84-
else \
85-
echo "$(SKIP_MSG) $$TARGET."; \
86-
fi
87-
88-
# ------------------------------------------------------------------------------
89-
# coverage: displays a colorized coverage summary from the coverage file.
90-
# Depends on test.
91-
# ------------------------------------------------------------------------------
92-
coverage: test
93-
@echo "================== COVERAGE SUMMARY =================="
94-
@go tool cover -func="$(COVER_PROFILE)" | go run ./scripts/coverage_formatter.go
95-
@echo "====================================================="
96-
97-
# ------------------------------------------------------------------------------
98-
# build: compile binary if Go sources changed.
99-
# Depends on install.
100-
# ------------------------------------------------------------------------------
101-
build: install
102-
@mkdir -p $(BUILD_DIR)
103-
@mkdir -p $(STAMPS_DIR)
104-
@TARGET=build; \
105-
if [ ! -f "$(BUILD_STAMP)" ] || [ -n "$$(find $(GO_FILES) -newer "$(BUILD_STAMP)" 2>/dev/null)" ]; then \
106-
echo "$(CHANGE_MSG) $$TARGET..."; \
107-
go build -o $(BINARY) ./cmd/scrapeycli; \
108-
touch "$(BUILD_STAMP)"; \
109-
echo "Done with building."; \
110-
else \
111-
echo "$(SKIP_MSG) $$TARGET."; \
112-
fi
113-
114-
# ------------------------------------------------------------------------------
115-
# run: execute the compiled binary.
116-
# Depends on build.
117-
# ------------------------------------------------------------------------------
118-
run: build
119-
@echo "Running application..."
120-
@$(BINARY)
121-
122-
# ------------------------------------------------------------------------------
123-
# tree: displays directory structure (installs tree if missing).
124-
# ------------------------------------------------------------------------------
125-
tree:
126-
@if ! command -v tree >/dev/null 2>&1; then \
127-
echo "tree command not found. Attempting to install..."; \
128-
OS=$$(uname); \
129-
if [ "$$OS" = "Linux" ]; then \
130-
sudo apt-get update && sudo apt-get install -y tree; \
131-
elif [ "$$OS" = "Darwin" ]; then \
132-
brew install tree; \
133-
else \
134-
echo "Automatic installation not supported on $$OS. Please install manually."; \
135-
exit 1; \
136-
fi; \
137-
fi; \
1+
# File: Makefile
2+
# Purpose:
3+
# - "install": ensure modules are tidy if go.mod or go.sum changed.
4+
# - "test": run coverage if code changed.
5+
# - "coverage": display coverage summary from coverage.out
6+
# - "build": recompile binary if Go source or mod files changed.
7+
# - "run": executes the compiled binary
8+
# - "tree": display directory structure
9+
# - All skip with "No changes detected, skipping X." if nothing changed.
10+
11+
.PHONY: install test coverage build run tree
12+
13+
# Directories for build artifacts and stamp files
14+
BUILD_DIR := build
15+
STAMPS_DIR := $(BUILD_DIR)/.stamps
16+
17+
# Stamp file paths for each step
18+
INSTALL_STAMP := $(STAMPS_DIR)/install.stamp
19+
BUILD_STAMP := $(STAMPS_DIR)/build.stamp
20+
TEST_STAMP := $(STAMPS_DIR)/test.stamp
21+
22+
BINARY := $(BUILD_DIR)/scrapeycli
23+
24+
# Coverage output
25+
COVER_DIR := ${BUILD_DIR}/coverage
26+
COVER_PROFILE := $(COVER_DIR)/coverage.txt
27+
COVER_HTML := $(COVER_DIR)/coverage.html
28+
29+
# All Go source files (including _test.go files)
30+
GO_FILES := $(shell find . -type f -name '*.go' -not -path "./vendor/*")
31+
32+
# Reusable messages
33+
SKIP_MSG := No changes detected, skipping
34+
CHANGE_MSG := Some files changed; re-running
35+
36+
# ------------------------------------------------------------------------------
37+
# install: ensure go.mod/go.sum are tidy if changed.
38+
# ------------------------------------------------------------------------------
39+
install:
40+
@mkdir -p $(STAMPS_DIR)
41+
@TARGET=install; \
42+
if [ ! -f "$(INSTALL_STAMP)" ] || [ go.mod -nt "$(INSTALL_STAMP)" ] || [ go.sum -nt "$(INSTALL_STAMP)" ]; then \
43+
echo "$(CHANGE_MSG) $$TARGET..."; \
44+
go mod tidy; \
45+
if ! git diff --exit-code go.sum; then \
46+
echo "go.sum updated. Please commit the changes."; \
47+
exit 1; \
48+
fi; \
49+
touch "$(INSTALL_STAMP)"; \
50+
echo "Done with installing."; \
51+
else \
52+
echo "$(SKIP_MSG) $$TARGET."; \
53+
fi
54+
55+
# ------------------------------------------------------------------------------
56+
# test: run tests and update coverage if any Go source (including _test.go files) have changed.
57+
# Ensures gotestsum is installed before running tests.
58+
# Depends on install.
59+
# ------------------------------------------------------------------------------
60+
test: install
61+
@if ! command -v gotestsum >/dev/null 2>&1; then \
62+
echo "Installing gotestsum..."; \
63+
go install gotest.tools/gotestsum@latest; \
64+
fi
65+
@mkdir -p $(COVER_DIR) $(STAMPS_DIR)
66+
@TARGET=test; \
67+
if [ ! -f "$(TEST_STAMP)" ] || [ -n "$$(find $(GO_FILES) -newer "$(TEST_STAMP)" 2>/dev/null)" ]; then \
68+
echo "$(CHANGE_MSG) $$TARGET..."; \
69+
> "$(COVER_PROFILE)"; \
70+
if gotestsum --format short-verbose ./... && \
71+
go test -cover -covermode=atomic -coverpkg=./... -coverprofile="$(COVER_PROFILE)" ./... >/dev/null; then \
72+
if [ -f "$(COVER_PROFILE)" ]; then \
73+
go tool cover -html="$(COVER_PROFILE)" -o "$(COVER_HTML)"; \
74+
echo "Coverage file generated at: $(COVER_PROFILE)"; \
75+
echo "HTML coverage report at: $(COVER_HTML)"; \
76+
else \
77+
echo "ERROR: Coverage file was not generated!"; \
78+
fi; \
79+
touch "$(TEST_STAMP)"; \
80+
else \
81+
echo "Tests failed! Skipping stamp update."; \
82+
exit 1; \
83+
fi; \
84+
else \
85+
echo "$(SKIP_MSG) $$TARGET."; \
86+
fi
87+
88+
# ------------------------------------------------------------------------------
89+
# coverage: displays a colorized coverage summary from the coverage file.
90+
# Depends on test.
91+
# ------------------------------------------------------------------------------
92+
coverage: test
93+
@echo "================== COVERAGE SUMMARY =================="
94+
@go tool cover -func="$(COVER_PROFILE)" | go run ./scripts/coverage_formatter.go
95+
@echo "====================================================="
96+
97+
# ------------------------------------------------------------------------------
98+
# build: compile binary if Go sources changed.
99+
# Depends on install.
100+
# ------------------------------------------------------------------------------
101+
build: install
102+
@mkdir -p $(BUILD_DIR)
103+
@mkdir -p $(STAMPS_DIR)
104+
@TARGET=build; \
105+
if [ ! -f "$(BUILD_STAMP)" ] || [ -n "$$(find $(GO_FILES) -newer "$(BUILD_STAMP)" 2>/dev/null)" ]; then \
106+
echo "$(CHANGE_MSG) $$TARGET..."; \
107+
go build -o $(BINARY) ./cmd/scrapeycli; \
108+
touch "$(BUILD_STAMP)"; \
109+
echo "Done with building."; \
110+
else \
111+
echo "$(SKIP_MSG) $$TARGET."; \
112+
fi
113+
114+
# ------------------------------------------------------------------------------
115+
# run: execute the compiled binary.
116+
# Depends on build.
117+
# ------------------------------------------------------------------------------
118+
run: build
119+
@echo "Running application..."
120+
@$(BINARY)
121+
122+
# ------------------------------------------------------------------------------
123+
# tree: displays directory structure (installs tree if missing).
124+
# ------------------------------------------------------------------------------
125+
tree:
126+
@if ! command -v tree >/dev/null 2>&1; then \
127+
echo "tree command not found. Attempting to install..."; \
128+
OS=$$(uname); \
129+
if [ "$$OS" = "Linux" ]; then \
130+
sudo apt-get update && sudo apt-get install -y tree; \
131+
elif [ "$$OS" = "Darwin" ]; then \
132+
brew install tree; \
133+
else \
134+
echo "Automatic installation not supported on $$OS. Please install manually."; \
135+
exit 1; \
136+
fi; \
137+
fi; \
138138
tree -n -I "vendor|.git"

‎README.md‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -76,32 +76,34 @@ scrapey-cli/
7676
│ └── workflows/
7777
│ └── ci.yml
7878
├── .vscode/
79-
│ └── settings.json # VS Code settings (format on save for Go)
79+
│ └── settings.json # VS Code settings (format on save for Go)
8080
├── cmd/
8181
│ └── scrapeycli/
8282
│ └── main.go
8383
├── configs/
84-
│ └── default.json # Default/example configuration file
85-
├── docs/ # Project documentation
84+
│ └── default.json # Default/example configuration file
8685
├── pkg/
8786
│ ├── config/
88-
│ │ └── config.go # Config loading logic
87+
│ │ └── config.go # Config loading logic
8988
│ ├── crawler/
90-
│ │ └── crawler.go # Core web crawling logic
89+
│ │ └── crawler.go # Core web crawling logic
9190
│ ├── parser/
92-
│ │ └── parser.go # HTML parsing logic
91+
│ │ └── parser.go # HTML parsing logic
9392
│ ├── storage/
94-
│ │ └── storage.go # Storage logic
93+
│ │ └── storage.go # Storage logic
9594
│ └── utils/
96-
│ └── utils.go # Utility functions
97-
├── test/ # Optional integration tests
95+
│ ├── printcolor.go # Colorized terminal output utility
96+
│ └── utils.go # Utility functions
97+
├── scripts/
98+
│ └── coverage_formatter.go # Formats and colorizes Go test coverage output
99+
├── test/ # Optional integration tests
100+
│ └── fail_test.go # Test case designed to always fail, used to debug test output
98101
├── .gitignore
99-
├── LICENSE # MIT License file
100-
├── Makefile # Build & run script for CLI (includes targets for build, run, and test)
102+
├── LICENSE # MIT License file
103+
├── Makefile # Build & run script for CLI (includes targets for build, run, and test)
101104
├── go.mod
102105
├── go.sum
103106
└── README.md
104-
105107
```
106108

107109
---

0 commit comments

Comments
 (0)