Skip to content

fix: prevent response file overwrite when -sd flag is used #2226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mzack9999 merged 7 commits into from
Jan 9, 2026
Merged

fix: prevent response file overwrite when -sd flag is used #2226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3cac134
fix: prevent response file overwrite when -sd flag is used
jjhwan-h Jul 31, 2025
63130de
Merge branch 'dev' into pr/2226
Mzack9999 Oct 20, 2025
bd4dc82
fix var declaration
Mzack9999 Oct 20, 2025
a3212c6
fix lint errors
Mzack9999 Oct 20, 2025
ca007ea
fix: correct index file name generation
jjhwan-h Oct 23, 2025
07b45fa
fixing index generation
Mzack9999 Jan 9, 2026
048c43f
Merge branch 'dev' into pr/2226
Mzack9999 Jan 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 83 additions & 26 deletions runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,11 @@ func New(options *Options) (*Runner, error) {
}

if options.StoreResponseDir != "" {
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
// Don't remove index files if skip-dedupe is enabled (we want to append, not truncate)
if !options.SkipDedupe {
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
}
}

httpxOptions := httpx.DefaultOptions
Expand Down Expand Up @@ -499,15 +502,23 @@ func (r *Runner) prepareInputPaths() {
}
}

var duplicateTargetErr = errors.New("duplicate target")

func (r *Runner) prepareInput() {
var numHosts int
// check if input target host(s) have been provided
if len(r.options.InputTargetHost) > 0 {
for _, target := range r.options.InputTargetHost {
expandedTarget, _ := r.countTargetFromRawTarget(target)
if expandedTarget > 0 {
expandedTarget, err := r.countTargetFromRawTarget(target)
if err == nil && expandedTarget > 0 {
numHosts += expandedTarget
r.hm.Set(target, nil) //nolint
r.hm.Set(target, []byte("1")) //nolint
} else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) {
if v, ok := r.hm.Get(target); ok {
cnt, _ := strconv.Atoi(string(v))
_ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1)))
numHosts += 1
}
}
}
}
Expand Down Expand Up @@ -665,10 +676,16 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
for scanner.Scan() {
target := strings.TrimSpace(scanner.Text())
// Used just to get the exact number of targets
expandedTarget, _ := r.countTargetFromRawTarget(target)
if expandedTarget > 0 {
expandedTarget, err := r.countTargetFromRawTarget(target)
if err == nil && expandedTarget > 0 {
numTargets += expandedTarget
r.hm.Set(target, nil) //nolint
r.hm.Set(target, []byte("1")) //nolint
} else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) {
if v, ok := r.hm.Get(target); ok {
cnt, _ := strconv.Atoi(string(v))
_ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1)))
numTargets += 1
}
}
}
err = finput.Close()
Expand All @@ -679,8 +696,9 @@ func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err
if rawTarget == "" {
return 0, nil
}

if _, ok := r.hm.Get(rawTarget); ok {
return 0, nil
return 0, duplicateTargetErr
}

expandedTarget := 0
Expand Down Expand Up @@ -911,7 +929,8 @@ func (r *Runner) RunEnumeration() {
gologger.Fatal().Msgf("Could not create response directory '%s': %s\n", responseDirPath, err)
}
indexPath := filepath.Join(responseDirPath, "index.txt")
if r.options.Resume {
// Append if resume is enabled or skip-dedupe is enabled (never truncate with -sd)
if r.options.Resume || r.options.SkipDedupe {
indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
} else {
indexFile, err = os.Create(indexPath)
Expand All @@ -925,7 +944,8 @@ func (r *Runner) RunEnumeration() {
if r.options.Screenshot {
var err error
indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt")
if r.options.Resume {
// Append if resume is enabled or skip-dedupe is enabled (never truncate with -sd)
if r.options.Resume || r.options.SkipDedupe {
indexScreenshotFile, err = os.OpenFile(indexScreenshotPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
} else {
indexScreenshotFile, err = os.Create(indexScreenshotPath)
Expand Down Expand Up @@ -1117,10 +1137,8 @@ func (r *Runner) RunEnumeration() {
// store responses or chain in directory
if resp.Err == nil {
URL, _ := urlutil.Parse(resp.URL)
domainFile := resp.Method + ":" + URL.EscapedString()
hash := hashes.Sha1([]byte(domainFile))
domainResponseFile := fmt.Sprintf("%s.txt", hash)
screenshotResponseFile := fmt.Sprintf("%s.png", hash)
domainResponseFile := fmt.Sprintf("%s.txt", resp.FileNameHash)
screenshotResponseFile := fmt.Sprintf("%s.png", resp.FileNameHash)
Comment on lines +1140 to +1141
Copy link

@coderabbitai coderabbitai bot Jan 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Get context around lines 1140-1141 to see the conditional protection
sed -n '1130,1160p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 1651

🏁 Script executed:

#!/bin/bash
# Search for FileNameHash assignments and their conditions
rg -n "FileNameHash" runner/runner.go | head -20

Repository: projectdiscovery/httpx

Length of output: 253

🏁 Script executed:

#!/bin/bash
# Check the conditional logic protecting the output writing at lines 1140-1141
sed -n '1100,1150p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 2016

🏁 Script executed:

#!/bin/bash
# Check where FileNameHash is assigned and under what conditions
sed -n '2480,2510p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 1312

🏁 Script executed:

#!/bin/bash
# Search for where fileNameHash variable is created/assigned
rg -n "fileNameHash\s*:=" runner/runner.go -B 5 | head -40

Repository: projectdiscovery/httpx

Length of output: 48

🏁 Script executed:

#!/bin/bash
# Check the full context of where FileNameHash is set in the Response struct
rg -n "FileNameHash" runner/runner.go -B 3 -A 1

Repository: projectdiscovery/httpx

Length of output: 589

🏁 Script executed:

#!/bin/bash
# Find where fileNameHash is computed/assigned before being used in the Result
rg -n "fileNameHash\s*=" runner/runner.go | head -20

Repository: projectdiscovery/httpx

Length of output: 148

🏁 Script executed:

#!/bin/bash
# Check the full analyze function to see if fileNameHash is always set
sed -n '2350,2370p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 935

🏁 Script executed:

#!/bin/bash
# Look for fileNameHash initialization in analyze function
rg -n "fileNameHash" runner/runner.go -B 2 -A 0 | grep -E "^\d+[-:]" | head -30

Repository: projectdiscovery/httpx

Length of output: 48

🏁 Script executed:

#!/bin/bash
# Find the full context of fileNameHash initialization - where it's declared and all assignment paths
rg -n "fileNameHash" runner/runner.go -B 10 | head -80

Repository: projectdiscovery/httpx

Length of output: 1287

🏁 Script executed:

#!/bin/bash
# Check if fileNameHash has a default initialization or is always set
sed -n '2280,2320p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 1299

🏁 Script executed:

#!/bin/bash
# Verify the exact line where fileNameHash is first declared
rg -n "var fileNameHash|fileNameHash :=" runner/runner.go

Repository: projectdiscovery/httpx

Length of output: 48

🏁 Script executed:

#!/bin/bash
# Verify the complete code flow - check if fileNameHash is assigned in all code paths
sed -n '2254,2310p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 2184

🏁 Script executed:

#!/bin/bash
# Check the actual usage of domainResponseFile and screenshotResponseFile at lines 1140-1141
sed -n '1137,1200p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 3485

🏁 Script executed:

#!/bin/bash
# Verify how fileNameHash is handled when StoreResponse and StoreChain are both false
rg -n "StoreResponse|StoreChain" runner/runner.go | grep -E "2254|2256|2299|2300|2301"

Repository: projectdiscovery/httpx

Length of output: 122

🏁 Script executed:

#!/bin/bash
# Verify that scanopts used in analyze function and r.scanopts in output routine refer to the same options
# Check the Runner structure to see how scanopts is stored
rg -n "scanopts" runner/runner.go | grep -E "Runner|this|self" | head -20

Repository: projectdiscovery/httpx

Length of output: 493

🏁 Script executed:

#!/bin/bash
# Confirm the relationship between scanopts and r.scanopts by checking the analyze function signature
sed -n '2200,2260p' runner/runner.go | grep -E "func.*analyze|scanopts"

Repository: projectdiscovery/httpx

Length of output: 409

🏁 Script executed:

#!/bin/bash
# Verify whether domainResponseFile and screenshotResponseFile created at lines 1140-1141 are only used inside the StoreResponse/StoreChain conditional
sed -n '1140,1180p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 2359

🏁 Script executed:

#!/bin/bash
# Final verification: Check if there are any uses of domainResponseFile or screenshotResponseFile outside the conditional blocks
sed -n '1137,1220p' runner/runner.go | cat -n

Repository: projectdiscovery/httpx

Length of output: 4375

Add fallback hash or guard FileNameHash usage against empty values.

The switch from domainFile-based naming to resp.FileNameHash introduces a potential issue: FileNameHash is only set within the if scanopts.StoreResponse || scanopts.StoreChain block in the analyze function (lines 2256-2301). If both conditions are false, FileNameHash remains an empty string. However, the output routine creates domainResponseFile and screenshotResponseFile at lines 1140-1141 using this potentially empty hash, and screenshotResponseFile is later used if r.scanopts.Screenshot is enabled. This can result in screenshot files named only .png without a hash prefix. Either:

  • Ensure FileNameHash is always computed regardless of StoreResponse/StoreChain settings, or
  • Guard screenshotResponseFile usage with a fallback hash if FileNameHash is empty.
🤖 Prompt for AI Agents 
In @runner/runner.go around lines 1140 - 1141, The code builds
domainResponseFile and screenshotResponseFile using resp.FileNameHash which can
be empty because FileNameHash is only set inside analyze when
scanopts.StoreResponse or scanopts.StoreChain are true; fix by either computing
the hash unconditionally in analyze (ensure resp.FileNameHash is assigned
regardless of scanopts) or add a guard/fallback where domainResponseFile and
screenshotResponseFile are constructed (e.g., if resp.FileNameHash == "" use
domainFile or compute a fallback hash), and ensure any use of
screenshotResponseFile when r.scanopts.Screenshot is enabled checks for and uses
that fallback.

hostFilename := strings.ReplaceAll(URL.Host, ":", "_")
domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response")
domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot")
Expand Down Expand Up @@ -1320,14 +1338,28 @@ func (r *Runner) RunEnumeration() {
}
}

if len(r.options.requestURIs) > 0 {
for _, p := range r.options.requestURIs {
scanopts := r.scanopts.Clone()
scanopts.RequestURI = p
r.process(k, wg, r.hp, protocol, scanopts, output)
runProcess := func(times int) {
for i := 0; i < times; i++ {
if len(r.options.requestURIs) > 0 {
for _, p := range r.options.requestURIs {
scanopts := r.scanopts.Clone()
scanopts.RequestURI = p
r.process(k, wg, r.hp, protocol, scanopts, output)
}
} else {
r.process(k, wg, r.hp, protocol, &r.scanopts, output)
}
}
} else {
r.process(k, wg, r.hp, protocol, &r.scanopts, output)
}

if r.options.Stream {
runProcess(1)
} else if v, ok := r.hm.Get(k); ok {
cnt, err := strconv.Atoi(string(v))
if err != nil || cnt <= 0 {
cnt = 1
}
runProcess(cnt)
}

return nil
Expand Down Expand Up @@ -2219,7 +2251,7 @@ retry:
domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response")
responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)

var responsePath string
var responsePath, fileNameHash string
// store response
if scanopts.StoreResponse || scanopts.StoreChain {
if r.options.OmitBody {
Expand All @@ -2240,9 +2272,33 @@ retry:
data = append(data, []byte("\n\n\n")...)
data = append(data, []byte(fullURL)...)
_ = fileutil.CreateFolder(responseBaseDir)
writeErr := os.WriteFile(responsePath, data, 0644)
if writeErr != nil {
gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)

basePath := strings.TrimSuffix(responsePath, ".txt")
var idx int
for idx = 0; ; idx++ {
targetPath := responsePath
if idx > 0 {
targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx)
}
f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err == nil {
_, writeErr := f.Write(data)
_ = f.Close()
if writeErr != nil {
gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr)
}
break
}
if !os.IsExist(err) {
gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err)
break
}
}

if idx == 0 {
fileNameHash = hash
} else {
fileNameHash = fmt.Sprintf("%s_%d", hash, idx)
}
}

Expand Down Expand Up @@ -2433,6 +2489,7 @@ retry:
RequestRaw: requestDump,
Response: resp,
FaviconData: faviconData,
FileNameHash: fileNameHash,
CPE: cpeMatches,
WordPress: wpInfo,
}
Expand Down
7 changes: 5 additions & 2 deletions runner/runner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"testing"
"time"

"github.com/pkg/errors"
_ "github.com/projectdiscovery/fdmax/autofdmax"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/mapcidr/asn"
Expand Down Expand Up @@ -154,7 +155,9 @@ func TestRunner_asn_targets(t *testing.T) {
}

func TestRunner_countTargetFromRawTarget(t *testing.T) {
options := &Options{}
options := &Options{
SkipDedupe: false,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")

Expand All @@ -169,7 +172,7 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) {
err = r.hm.Set(input, nil)
require.Nil(t, err, "could not set value to hm")
got, err = r.countTargetFromRawTarget(input)
require.Nil(t, err, "could not count targets")
require.True(t, errors.Is(err, duplicateTargetErr), "expected duplicate target error")
require.Equal(t, expected, got, "got wrong output")

input = "173.0.84.0/24"
Expand Down
1 change: 1 addition & 0 deletions runner/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ type Result struct {
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"`
FileNameHash string `json:"-" csv:"-" mapstructure:"-"`
CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" mapstructure:"cpe"`
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" mapstructure:"wordpress"`
}
Expand Down
Loading