Skip to content

Commit 58270eb

Browse files
author
Gepeto Escalante
committed
[SP-3848] [scanoss.java] Implement CLI scan configuration parameters
Implemented by Gepeto Escalante.
1 parent 5be762a commit 58270eb

File tree

8 files changed

+645
-6
lines changed

8 files changed

+645
-6
lines changed

‎src/main/java/com/scanoss/Scanner.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import com.scanoss.processor.*;
3232
import com.scanoss.rest.ScanApi;
3333
import com.scanoss.settings.Bom;
34+
import com.scanoss.settings.ScanConfig;
3435
import com.scanoss.settings.ScanossSettings;
3536
import com.scanoss.utils.JsonUtils;
3637
import lombok.*;
@@ -104,6 +105,8 @@ public class Scanner {
104105
private final ScanFileProcessor scanFileProcessor;
105106
private final WfpFileProcessor wfpFileProcessor;
106107
private final ScanossSettings settings;
108+
private final ScanConfig cliScanConfig; // CLI-provided scan config (lowest priority)
109+
private final ScanConfig scanConfig; // Resolved scan config (after priority merge)
107110
private final ScannerPostProcessor postProcessor;
108111
private final FilterConfig filterConfig;
109112
private Predicate<Path> fileFilter;
@@ -116,7 +119,8 @@ private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate,
116119
Integer snippetLimit, String customCert, Proxy proxy,
117120
Winnowing winnowing, ScanApi scanApi,
118121
ScanFileProcessor scanFileProcessor, WfpFileProcessor wfpFileProcessor,
119-
ScanossSettings settings, ScannerPostProcessor postProcessor, FilterConfig filterConfig,
122+
ScanossSettings settings, ScanConfig cliScanConfig, ScanConfig scanConfig,
123+
ScannerPostProcessor postProcessor, FilterConfig filterConfig,
120124
Predicate<Path> fileFilter,
121125
Predicate<Path> folderFilter
122126
) {
@@ -137,20 +141,27 @@ private Scanner(Boolean skipSnippets, Boolean allExtensions, Boolean obfuscate,
137141
this.snippetLimit = snippetLimit;
138142
this.customCert = customCert;
139143
this.proxy = proxy;
144+
this.settings = Objects.requireNonNullElseGet(settings, () -> ScanossSettings.builder().build());
145+
this.cliScanConfig = cliScanConfig;
146+
// Resolve scan config: file_snippet (highest) > settings (middle) > CLI (lowest)
147+
this.scanConfig = this.settings.getResolvedScanConfig(
148+
Objects.requireNonNullElseGet(cliScanConfig, () -> ScanConfig.builder().build()));
140149
this.winnowing = Objects.requireNonNullElseGet(winnowing, () ->
141150
Winnowing.builder().skipSnippets(skipSnippets).allExtensions(allExtensions).obfuscate(obfuscate)
142151
.hpsm(hpsm).snippetLimit(snippetLimit)
152+
.skipHeaders(this.scanConfig.getSkipHeaders() != null && this.scanConfig.getSkipHeaders())
153+
.skipHeadersLimit(this.scanConfig.getSkipHeadersLimit() != null ? this.scanConfig.getSkipHeadersLimit() : 0)
143154
.build());
144155
this.scanApi = Objects.requireNonNullElseGet(scanApi, () ->
145156
ScanApi.builder().url(url).apiKey(apiKey).timeout(timeout).retryLimit(retryLimit).flags(scanFlags)
146-
.sbomType(sbomType).sbom(sbom).customCert(customCert).proxy(proxy).settings(settings)
157+
.sbomType(sbomType).sbom(sbom).customCert(customCert).proxy(proxy).settings(this.settings)
158+
.scanConfig(this.scanConfig)
147159
.build());
148160
this.scanFileProcessor = Objects.requireNonNullElseGet(scanFileProcessor, () ->
149161
ScanFileProcessor.builder().winnowing(this.winnowing).scanApi(this.scanApi).build());
150162
this.wfpFileProcessor = Objects.requireNonNullElseGet(wfpFileProcessor, () -> WfpFileProcessor.builder()
151163
.winnowing(this.winnowing)
152164
.build());
153-
this.settings = Objects.requireNonNullElseGet(settings, () -> ScanossSettings.builder().build());
154165
this.postProcessor = Objects.requireNonNullElseGet(postProcessor, () ->
155166
ScannerPostProcessor.builder().build());
156167

‎src/main/java/com/scanoss/Winnowing.java‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ public class Winnowing {
8080
@Builder.Default
8181
private int snippetLimit = MAX_LONG_LINE_CHARS; // Enable limiting of size of a single line of snippet generation
8282
@Builder.Default
83+
private boolean skipHeaders = false; // Skip license headers, comments and imports at the beginning of files
84+
@Builder.Default
85+
private int skipHeadersLimit = 0; // Maximum number of header lines to skip (0 = auto-detect)
86+
@Builder.Default
8387
private Map<String, String> obfuscationMap = new ConcurrentHashMap<>();
8488

8589
/**
@@ -168,6 +172,12 @@ public String wfpForContents(@NonNull String filename, Boolean binFile, byte[] c
168172
wfpBuilder.append(String.format("hpsm=%s\n", Hpsm.calcHpsm(contents)));
169173
}
170174

175+
int skipLines = 0;
176+
if (this.skipHeaders) {
177+
skipLines = detectHeaderLines(fileContents, this.skipHeadersLimit);
178+
log.trace("Skipping {} header lines for snippet generation: {}", skipLines, filename);
179+
}
180+
171181
String gram = "";
172182
List<Long> window = new ArrayList<>();
173183
char normalized;
@@ -183,7 +193,7 @@ public String wfpForContents(@NonNull String filename, Boolean binFile, byte[] c
183193
} else {
184194
normalized = WinnowingUtils.normalize(c);
185195
}
186-
if (normalized > 0) {
196+
if (normalized > 0 && line > skipLines) {
187197
gram += normalized;
188198
if (gram.length() >= ScanossConstants.GRAM) {
189199
Long gramCRC32 = crc32c(gram);
@@ -312,6 +322,69 @@ private Boolean skipSnippets(@NonNull String filename, char[] contents) {
312322
return false;
313323
}
314324

325+
/**
326+
* Detect the number of header lines at the beginning of a file.
327+
* Header lines include license comment blocks, single-line comments,
328+
* blank lines, and import/package statements.
329+
*
330+
* @param contents file contents as char array
331+
* @param maxLines maximum number of header lines to detect (0 = no limit)
332+
* @return number of header lines detected
333+
*/
334+
int detectHeaderLines(char[] contents, int maxLines) {
335+
int headerLines = 0;
336+
boolean inBlockComment = false;
337+
int lineStart = 0;
338+
339+
for (int i = 0; i <= contents.length; i++) {
340+
if (i == contents.length || contents[i] == '\n') {
341+
String line = new String(contents, lineStart, i - lineStart).trim();
342+
343+
if (inBlockComment) {
344+
headerLines++;
345+
if (line.contains("*/")) {
346+
inBlockComment = false;
347+
}
348+
} else if (line.isEmpty()) {
349+
headerLines++;
350+
} else if (line.startsWith("//") || line.startsWith("#!") || line.startsWith("# ")) {
351+
headerLines++;
352+
} else if (line.startsWith("/*")) {
353+
headerLines++;
354+
if (!line.contains("*/")) {
355+
inBlockComment = true;
356+
}
357+
} else if (line.startsWith("*") || line.startsWith("* ")) {
358+
headerLines++;
359+
} else if (isImportOrPackageLine(line)) {
360+
headerLines++;
361+
} else {
362+
break; // Non-header line found
363+
}
364+
365+
if (maxLines > 0 && headerLines >= maxLines) {
366+
break;
367+
}
368+
369+
lineStart = i + 1;
370+
}
371+
}
372+
373+
return headerLines;
374+
}
375+
376+
/**
377+
* Check if a line is an import or package declaration.
378+
*
379+
* @param line trimmed source line
380+
* @return true if the line is an import/package/include statement
381+
*/
382+
private boolean isImportOrPackageLine(String line) {
383+
return line.startsWith("import ") || line.startsWith("package ") ||
384+
line.startsWith("from ") || line.startsWith("#include ") ||
385+
line.startsWith("using ") || line.startsWith("require ");
386+
}
387+
315388
/**
316389
* Try to detect if this is a text file or not
317390
*

‎src/main/java/com/scanoss/cli/ScanCommandLine.java‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.scanoss.Scanner;
2626
import com.scanoss.exceptions.ScannerException;
2727
import com.scanoss.exceptions.WinnowingException;
28+
import com.scanoss.settings.ScanConfig;
2829
import com.scanoss.settings.ScanossSettings;
2930
import com.scanoss.utils.JsonUtils;
3031
import com.scanoss.utils.ProxyUtils;
@@ -105,6 +106,27 @@ class ScanCommandLine implements Runnable {
105106
@picocli.CommandLine.Option(names = {"-H", "--hpsm"}, description = "Use High Precision Snippet Matching algorithm")
106107
private boolean enableHpsm = false;
107108

109+
@picocli.CommandLine.Option(names = {"--min-snippet-hits"}, description = "Minimum snippet hits required (0 = unset, uses server config)")
110+
private int minSnippetHits = 0;
111+
112+
@picocli.CommandLine.Option(names = {"--min-snippet-lines"}, description = "Minimum snippet lines required (0 = unset, uses server config)")
113+
private int minSnippetLines = 0;
114+
115+
@picocli.CommandLine.Option(names = {"--honour-file-exts"}, description = "Honour file extensions (true|false|unset)", arity = "1")
116+
private String honourFileExts = null;
117+
118+
@picocli.CommandLine.Option(names = {"--ranking"}, description = "Enable/disable ranking (true|false|unset)", arity = "1")
119+
private String ranking = null;
120+
121+
@picocli.CommandLine.Option(names = {"--ranking-threshold"}, description = "Ranking threshold value (-1 = unset, uses server config)")
122+
private int rankingThreshold = -1;
123+
124+
@picocli.CommandLine.Option(names = {"--skip-headers"}, description = "Skip license headers, comments and imports at the beginning of files (applies locally)")
125+
private boolean skipHeaders = false;
126+
127+
@picocli.CommandLine.Option(names = {"--skip-headers-limit"}, description = "Skip limit for license headers (0 = unset, applies locally)")
128+
private int skipHeadersLimit = 0;
129+
108130
@picocli.CommandLine.Parameters(arity = "1", description = "file/folder to scan")
109131
private String fileFolder;
110132

@@ -160,11 +182,12 @@ public void run() {
160182
printMsg(err, String.format("Using flags %s", scanFlags));
161183
}
162184
}
185+
ScanConfig cliScanConfig = buildCliScanConfig();
163186
scanner = Scanner.builder().skipSnippets(skipSnippets).allFolders(allFolders).allExtensions(allExtensions)
164187
.hiddenFilesFolders(allHidden).numThreads(numThreads).url(apiUrl).apiKey(apiKey)
165188
.retryLimit(retryLimit).timeout(Duration.ofSeconds(timeoutLimit)).scanFlags(scanFlags)
166189
.snippetLimit(snippetLimit).customCert(caCertPem).proxy(proxy).hpsm(enableHpsm)
167-
.settings(settings).obfuscate(obfuscate)
190+
.settings(settings).obfuscate(obfuscate).cliScanConfig(cliScanConfig)
168191
.build();
169192

170193
File f = new File(fileFolder);
@@ -198,6 +221,38 @@ private String loadFileToString(@NonNull String filename) {
198221
}
199222
}
200223

224+
/**
225+
* Build a ScanConfig from CLI arguments.
226+
*
227+
* @return ScanConfig populated with CLI-provided values
228+
*/
229+
private ScanConfig buildCliScanConfig() {
230+
ScanConfig.ScanConfigBuilder builder = ScanConfig.builder()
231+
.minSnippetHits(minSnippetHits)
232+
.minSnippetLines(minSnippetLines)
233+
.rankingThreshold(rankingThreshold)
234+
.skipHeaders(skipHeaders)
235+
.skipHeadersLimit(skipHeadersLimit);
236+
237+
builder.honourFileExts(parseTriStateBoolean(honourFileExts));
238+
builder.rankingEnabled(parseTriStateBoolean(ranking));
239+
240+
return builder.build();
241+
}
242+
243+
/**
244+
* Parse a tri-state boolean string value.
245+
*
246+
* @param value the string value ("true", "false", "unset", or null)
247+
* @return Boolean.TRUE, Boolean.FALSE, or null for unset
248+
*/
249+
private static Boolean parseTriStateBoolean(String value) {
250+
if (value == null || value.equalsIgnoreCase("unset")) {
251+
return null;
252+
}
253+
return Boolean.parseBoolean(value);
254+
}
255+
201256
/**
202257
* Scan the specified file and output the results
203258
*

‎src/main/java/com/scanoss/rest/ScanApi.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.scanoss.dto.SbomLegacy;
2626
import com.scanoss.exceptions.ScanApiException;
2727
import com.scanoss.settings.Rule;
28+
import com.scanoss.settings.ScanConfig;
2829
import com.scanoss.settings.ScanossSettings;
2930
import com.scanoss.utils.JsonUtils;
3031
import com.scanoss.utils.PackageDetails;
@@ -72,12 +73,14 @@ public class ScanApi {
7273
private Proxy proxy; // Proxy configuration
7374
private String baseUrl; // SCANOSS base API URI (to used instead of url)
7475
private ScanossSettings settings;
76+
private ScanConfig scanConfig; // Resolved scan configuration parameters
7577
@SuppressWarnings("unused")
7678
private ScanApi(String scanType, Duration timeout, Integer retryLimit, String url, String apiKey, String flags,
7779
String sbomType, String sbom,
7880
OkHttpClient okHttpClient, Map<String, String> headers, String customCert,
79-
Proxy proxy, String baseUrl, ScanossSettings settings) {
81+
Proxy proxy, String baseUrl, ScanossSettings settings, ScanConfig scanConfig) {
8082
this.settings = settings;
83+
this.scanConfig = scanConfig;
8184
this.scanType = scanType;
8285
this.timeout = timeout;
8386
this.retryLimit = retryLimit;
@@ -178,6 +181,24 @@ public String scan(String wfp, String context, int scanID) throws ScanApiExcepti
178181
data.put("type", "identify");
179182
}
180183

184+
// Add scan configuration parameters (only when set/non-default)
185+
if (scanConfig != null) {
186+
if (scanConfig.isMinSnippetHitsSet()) {
187+
data.put("min_snippet_hits", String.valueOf(scanConfig.getMinSnippetHits()));
188+
}
189+
if (scanConfig.isMinSnippetLinesSet()) {
190+
data.put("min_snippet_lines", String.valueOf(scanConfig.getMinSnippetLines()));
191+
}
192+
if (scanConfig.isHonourFileExtsSet()) {
193+
data.put("honour_file_exts", String.valueOf(scanConfig.getHonourFileExts()));
194+
}
195+
if (scanConfig.isRankingEnabledSet()) {
196+
data.put("ranking_enabled", String.valueOf(scanConfig.getRankingEnabled()));
197+
}
198+
if (scanConfig.isRankingThresholdSet()) {
199+
data.put("ranking_threshold", String.valueOf(scanConfig.getRankingThreshold()));
200+
}
201+
}
181202

182203
Request request; // Create multipart request
183204
try {

0 commit comments

Comments
 (0)