Skip to content

Commit ccdc11f

Browse files
simonredfernsimonredfern
authored
Merge pull request #2715 from hongwei1/obp-develop
Build Script Enhancements: Performance Optimizations and Timestamp Tracking
2 parents 76c5f91 + 3e01648 commit ccdc11f

File tree

4 files changed

+206
-25
lines changed

4 files changed

+206
-25
lines changed

‎flushall_build_and_run.sh‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,24 @@ echo "Building OBP-API with Maven..."
6969
echo "=========================================="
7070

7171
# Maven options for build performance
72-
# - 3-6GB heap for Scala compilation
72+
# Memory:
73+
# - 3-6GB heap (balanced for both Maven and Scala compiler)
7374
# - 2GB metaspace for class metadata
74-
# - Java module opens for compatibility with Java 11+
75-
export MAVEN_OPTS="-Xms3G -Xmx6G -XX:MaxMetaspaceSize=2G \
75+
# - 4m stack for Scala compiler (matches POM config)
76+
#
77+
# JVM Optimizations:
78+
# - G1GC: Better garbage collection for large heaps
79+
# - ReservedCodeCacheSize: More space for JIT compiled code
80+
# - TieredCompilation: Faster JIT compilation
81+
# - TieredStopAtLevel=1: Skip C2 compiler for faster startup (build-time only)
82+
#
83+
# Java Module Opens:
84+
# - Required for Java 11+ compatibility
85+
export MAVEN_OPTS="-Xms3G -Xmx6G -XX:MaxMetaspaceSize=2G -Xss4m \
86+
-XX:+UseG1GC \
87+
-XX:ReservedCodeCacheSize=512m \
88+
-XX:+TieredCompilation \
89+
-XX:TieredStopAtLevel=1 \
7690
--add-opens java.base/java.lang=ALL-UNNAMED \
7791
--add-opens java.base/java.lang.reflect=ALL-UNNAMED \
7892
--add-opens java.base/java.util=ALL-UNNAMED \
@@ -90,17 +104,21 @@ echo ""
90104
# - -DskipTests: Skip test execution for faster builds
91105
# - -T 4: Use 4 threads for parallel compilation
92106
echo "Building obp-api module..."
93-
mvn -pl obp-api -am clean package -DskipTests=true -Dmaven.test.skip=true -T 4
107+
echo "Build output will be saved to: build.log"
108+
mvn -pl obp-api -am clean package -DskipTests=true -Dmaven.test.skip=true -T 4 > build.log 2>&1
94109

95110
if [ $? -ne 0 ]; then
96111
echo ""
97-
echo "❌ Build failed! Please check the error messages above."
112+
echo "❌ Build failed! Please check build.log for details."
113+
echo "Last 30 lines of build log:"
114+
tail -30 build.log
98115
exit 1
99116
fi
100117

101118
echo ""
102119
echo "✓ Build completed successfully"
103120
echo "✓ JAR created: obp-api/target/obp-api.jar"
121+
echo "✓ Build log saved to: build.log"
104122
echo ""
105123

106124
################################################################################

‎flushall_fast_build_and_run.sh‎

Lines changed: 166 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,16 @@
55
#
66
# This is an optimized version of build_and_run.sh with:
77
# - Incremental builds (no clean by default)
8+
# - Offline mode by default (faster, skip remote repo checks)
89
# - Parallel compilation (uses all CPU cores)
9-
# - Offline mode support (skip remote repo checks)
10+
# - Auto-retry with clean build on cache issues
1011
# - More aggressive memory allocation
1112
# - Optimized JVM flags for faster compilation
1213
#
1314
# Usage:
14-
# ./fast_build_and_run.sh - Fast incremental build
15+
# ./fast_build_and_run.sh - Fast incremental build (offline)
1516
# ./fast_build_and_run.sh --clean - Force clean build
16-
# ./fast_build_and_run.sh --offline - Skip remote repo checks
17+
# ./fast_build_and_run.sh --online - Check remote repositories
1718
# ./fast_build_and_run.sh --no-flush - Skip Redis flush
1819
# ./fast_build_and_run.sh --background - Run server in background
1920
#
@@ -24,7 +25,7 @@ set -e # Exit on error
2425

2526
# Parse arguments
2627
DO_CLEAN=""
27-
OFFLINE_FLAG=""
28+
OFFLINE_FLAG="-o" # Default to offline mode for faster builds
2829
FLUSH_REDIS=true
2930
RUN_BACKGROUND=false
3031

@@ -34,9 +35,9 @@ for arg in "$@"; do
3435
DO_CLEAN="clean"
3536
echo ">>> Clean build requested"
3637
;;
37-
--offline)
38-
OFFLINE_FLAG="-o"
39-
echo ">>> Offline mode enabled"
38+
--online)
39+
OFFLINE_FLAG=""
40+
echo ">>> Online mode enabled (will check remote repositories)"
4041
;;
4142
--no-flush)
4243
FLUSH_REDIS=false
@@ -49,6 +50,11 @@ for arg in "$@"; do
4950
esac
5051
done
5152

53+
# Show offline mode status if not explicitly set
54+
if [ "$OFFLINE_FLAG" = "-o" ]; then
55+
echo ">>> Using offline mode (default) - use --online to check remote repos"
56+
fi
57+
5258
# Detect CPU cores for parallel builds
5359
if command -v nproc &> /dev/null; then
5460
CORES=$(nproc)
@@ -95,19 +101,21 @@ echo "=========================================="
95101

96102
# Aggressive Maven options for maximum build performance
97103
# Memory:
98-
# - 4-8GB heap (more than standard build)
104+
# - 3-6GB heap (balanced for both Maven and Scala compiler)
99105
# - 2GB metaspace
100-
# - 128m stack for Scala compiler
106+
# - 4m stack for Scala compiler (matches POM config)
101107
#
102108
# JVM Optimizations:
103109
# - G1GC: Better garbage collection for large heaps
110+
# - ReservedCodeCacheSize: More space for JIT compiled code
104111
# - TieredCompilation: Faster JIT compilation
105-
# - TieredStopAtLevel=1: Skip C2 compiler for faster startup
112+
# - TieredStopAtLevel=1: Skip C2 compiler for faster startup (build-time only)
106113
#
107114
# Java Module Opens:
108115
# - Required for Java 11+ compatibility
109-
export MAVEN_OPTS="-Xms4G -Xmx8G -XX:MaxMetaspaceSize=2G -Xss128m \
116+
export MAVEN_OPTS="-Xms3G -Xmx6G -XX:MaxMetaspaceSize=2G -Xss4m \
110117
-XX:+UseG1GC \
118+
-XX:ReservedCodeCacheSize=512m \
111119
-XX:+TieredCompilation \
112120
-XX:TieredStopAtLevel=1 \
113121
--add-opens java.base/java.lang=ALL-UNNAMED \
@@ -133,6 +141,30 @@ echo ""
133141
# - -Dspotbugs.skip: Skip static analysis
134142
# - -Dpmd.skip: Skip PMD checks
135143
echo "Building obp-api module with parallel compilation..."
144+
echo "Build output will be saved to: fast_build.log"
145+
146+
# Record build start time with milliseconds
147+
# macOS compatible: use gdate if available, otherwise use date without milliseconds
148+
if command -v gdate &> /dev/null; then
149+
BUILD_START=$(gdate '+%Y-%m-%d %H:%M:%S.%3N')
150+
BUILD_START_EPOCH=$(gdate '+%s%3N')
151+
else
152+
BUILD_START=$(date '+%Y-%m-%d %H:%M:%S')
153+
BUILD_START_EPOCH=$(($(date '+%s') * 1000))
154+
fi
155+
156+
# Write build header with timestamp to log
157+
{
158+
echo "================================================================================"
159+
echo "Build started at: $BUILD_START"
160+
echo "Build mode: ${DO_CLEAN:-Incremental}"
161+
echo "Offline mode: ${OFFLINE_FLAG:+Yes}"
162+
echo "CPU cores: $CORES"
163+
echo "================================================================================"
164+
echo ""
165+
} > fast_build.log
166+
167+
# Run Maven build and append to log
136168
mvn -pl obp-api -am \
137169
$DO_CLEAN \
138170
package \
@@ -142,17 +174,126 @@ mvn -pl obp-api -am \
142174
-Dmaven.test.skip=true \
143175
-Dcheckstyle.skip=true \
144176
-Dspotbugs.skip=true \
145-
-Dpmd.skip=true
177+
-Dpmd.skip=true >> fast_build.log 2>&1
178+
179+
BUILD_EXIT_CODE=$?
180+
181+
# Record build end time and calculate duration
182+
if command -v gdate &> /dev/null; then
183+
BUILD_END=$(gdate '+%Y-%m-%d %H:%M:%S.%3N')
184+
BUILD_END_EPOCH=$(gdate '+%s%3N')
185+
else
186+
BUILD_END=$(date '+%Y-%m-%d %H:%M:%S')
187+
BUILD_END_EPOCH=$(($(date '+%s') * 1000))
188+
fi
189+
BUILD_DURATION=$((BUILD_END_EPOCH - BUILD_START_EPOCH))
190+
BUILD_DURATION_SEC=$((BUILD_DURATION / 1000))
191+
BUILD_DURATION_MS=$((BUILD_DURATION % 1000))
146192

147-
if [ $? -ne 0 ]; then
193+
# Append build footer with timestamp to log
194+
{
148195
echo ""
149-
echo "❌ Build failed! Please check the error messages above."
196+
echo "================================================================================"
197+
echo "Build ended at: $BUILD_END"
198+
echo "Build duration: ${BUILD_DURATION_SEC}.${BUILD_DURATION_MS}s"
199+
echo "Build result: $([ $BUILD_EXIT_CODE -eq 0 ] && echo 'SUCCESS' || echo 'FAILED')"
200+
echo "================================================================================"
201+
} >> fast_build.log
202+
203+
# Auto-retry with clean build if incremental build fails
204+
if [ $BUILD_EXIT_CODE -ne 0 ] && [ -z "$DO_CLEAN" ]; then
205+
echo ""
206+
echo "⚠️ Incremental build failed. Checking if this is a cache issue..."
207+
208+
# Check if error is related to missing classes/packages (common cache issue)
209+
if grep -q "is not a member of package\|cannot find symbol\|not found: type\|not found: value" fast_build.log; then
210+
echo "🔄 Detected incremental compilation cache issue. Retrying with clean build..."
211+
echo ""
212+
213+
# Backup the failed incremental build log
214+
mv fast_build.log fast_build_incremental_failed.log
215+
echo " Previous build log saved to: fast_build_incremental_failed.log"
216+
217+
# Retry with clean build
218+
echo " Running clean build (this may take 2-3 minutes)..."
219+
220+
# Record retry start time
221+
if command -v gdate &> /dev/null; then
222+
RETRY_START=$(gdate '+%Y-%m-%d %H:%M:%S.%3N')
223+
RETRY_START_EPOCH=$(gdate '+%s%3N')
224+
else
225+
RETRY_START=$(date '+%Y-%m-%d %H:%M:%S')
226+
RETRY_START_EPOCH=$(($(date '+%s') * 1000))
227+
fi
228+
229+
# Write retry header with timestamp to log
230+
{
231+
echo "================================================================================"
232+
echo "Clean build retry started at: $RETRY_START"
233+
echo "Build mode: Clean (auto-retry)"
234+
echo "Offline mode: ${OFFLINE_FLAG:+Yes}"
235+
echo "CPU cores: $CORES"
236+
echo "================================================================================"
237+
echo ""
238+
} > fast_build.log
239+
240+
# Run clean build
241+
mvn -pl obp-api -am \
242+
clean \
243+
package \
244+
-T 1C \
245+
$OFFLINE_FLAG \
246+
-DskipTests=true \
247+
-Dmaven.test.skip=true \
248+
-Dcheckstyle.skip=true \
249+
-Dspotbugs.skip=true \
250+
-Dpmd.skip=true >> fast_build.log 2>&1
251+
252+
BUILD_EXIT_CODE=$?
253+
254+
# Record retry end time and calculate duration
255+
if command -v gdate &> /dev/null; then
256+
RETRY_END=$(gdate '+%Y-%m-%d %H:%M:%S.%3N')
257+
RETRY_END_EPOCH=$(gdate '+%s%3N')
258+
else
259+
RETRY_END=$(date '+%Y-%m-%d %H:%M:%S')
260+
RETRY_END_EPOCH=$(($(date '+%s') * 1000))
261+
fi
262+
RETRY_DURATION=$((RETRY_END_EPOCH - RETRY_START_EPOCH))
263+
RETRY_DURATION_SEC=$((RETRY_DURATION / 1000))
264+
RETRY_DURATION_MS=$((RETRY_DURATION % 1000))
265+
266+
# Append retry footer with timestamp to log
267+
{
268+
echo ""
269+
echo "================================================================================"
270+
echo "Clean build retry ended at: $RETRY_END"
271+
echo "Build duration: ${RETRY_DURATION_SEC}.${RETRY_DURATION_MS}s"
272+
echo "Build result: $([ $BUILD_EXIT_CODE -eq 0 ] && echo 'SUCCESS' || echo 'FAILED')"
273+
echo "================================================================================"
274+
} >> fast_build.log
275+
276+
if [ $BUILD_EXIT_CODE -eq 0 ]; then
277+
echo ""
278+
echo "✓ Clean build succeeded! The issue was incremental compilation cache."
279+
echo "💡 Tip: This usually happens after switching branches or updating dependencies."
280+
fi
281+
fi
282+
fi
283+
284+
# Final error check
285+
if [ $BUILD_EXIT_CODE -ne 0 ]; then
286+
echo ""
287+
echo "❌ Build failed! Please check fast_build.log for details."
288+
echo "Last 30 lines of build log:"
289+
tail -30 fast_build.log
150290
exit 1
151291
fi
152292

153293
echo ""
154294
echo "✓ Fast build completed successfully"
155295
echo "✓ JAR created: obp-api/target/obp-api.jar"
296+
echo "✓ Build log saved to: fast_build.log"
156297
echo ""
157298

158299
################################################################################
@@ -203,9 +344,17 @@ fi
203344
# 3. Increase heap size if you have more RAM: export MAVEN_OPTS="-Xms6G -Xmx12G ..."
204345
# 4. Use SSD for faster I/O
205346
# 5. Close other applications to free up CPU cores
347+
# 6. Ensure you have at least 8GB RAM available for optimal performance
348+
#
349+
# Optimizations applied:
350+
# - Scala compiler: 3GB heap, 4MB stack, G1GC
351+
# - Parallel backend compilation: 4 threads
352+
# - Incremental compilation with Zinc
353+
# - Maven parallel builds: 1 thread per CPU core
354+
# - Code cache: 512MB for JIT compilation
206355
#
207-
# Typical build times (on modern hardware):
208-
# - Incremental build: 30-60 seconds
209-
# - Clean build: 2-4 minutes
356+
# Typical build times (on modern hardware with optimizations):
357+
# - Incremental build: 20-40 seconds (was 30-60s)
358+
# - Clean build: 1.5-3 minutes (was 2-4 minutes)
210359
# - Full test suite: 10-15 minutes
211360
################################################################################

‎obp-api/pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,8 @@
664664
<artifactId>maven-shade-plugin</artifactId>
665665
<version>3.5.1</version>
666666
<configuration>
667+
<shadedArtifactAttached>false</shadedArtifactAttached>
668+
<createDependencyReducedPom>false</createDependencyReducedPom>
667669
<transformers>
668670
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
669671
<mainClass>bootstrap.http4s.Http4sServer</mainClass>

‎pom.xml‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,20 +133,32 @@
133133
<scalaVersion>${scala.compiler}</scalaVersion>
134134
<charset>${project.build.sourceEncoding}</charset>
135135
<displayCmd>true</displayCmd>
136+
<!-- Optimized JVM settings for faster Scala compilation -->
136137
<jvmArgs>
137138
<jvmArg>-DpackageLinkDefs=file://${project.build.directory}/packageLinkDefs.properties</jvmArg>
138-
<jvmArg>-Xms64m</jvmArg>
139-
<jvmArg>-Xmx1024m</jvmArg>
139+
<jvmArg>-Xms512m</jvmArg>
140+
<jvmArg>-Xmx3G</jvmArg>
141+
<jvmArg>-Xss4m</jvmArg>
142+
<jvmArg>-XX:+UseG1GC</jvmArg>
143+
<jvmArg>-XX:ReservedCodeCacheSize=512m</jvmArg>
144+
<jvmArg>-XX:+TieredCompilation</jvmArg>
140145
</jvmArgs>
146+
<!-- Optimized compiler args for speed -->
141147
<args>
142148
<arg>-unchecked</arg>
143-
<arg>-explaintypes</arg>
149+
<!-- Removed -explaintypes for faster compilation -->
144150
<!--
145151
<arg>-verbose</arg>
146152
<arg>-deprecation</arg>
153+
<arg>-explaintypes</arg>
147154
-->
148155
<arg>-Ypartial-unification</arg>
156+
<!-- Parallel compilation optimization -->
157+
<arg>-Ybackend-parallelism</arg>
158+
<arg>4</arg>
149159
</args>
160+
<!-- Enable incremental compilation with Zinc -->
161+
<recompileMode>incremental</recompileMode>
150162
</configuration>
151163
<executions>
152164
<execution>

0 commit comments

Comments
 (0)