Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,46 @@ if(NOT OPENSSL_VERSION OR OPENSSL_VERSION MATCHES "^0[.]")
message(STATUS "WARNING: QFS authentication will not work properly")
endif()

if(NOT DEFINED QFS_OUTPUT_DIR)
set(QFS_OUTPUT_DIR "${KFS_DIR_PREFIX}/output" CACHE PATH
"directory for deployable QFS build output")
endif()
get_filename_component(QFS_OUTPUT_DIR "${QFS_OUTPUT_DIR}" ABSOLUTE
BASE_DIR "${KFS_DIR_PREFIX}")

# Change this to where the install directory is located
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set(CMAKE_INSTALL_PREFIX "." CACHE PATH "install directory prefix" FORCE)
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT OR
CMAKE_INSTALL_PREFIX STREQUAL ".")
set(CMAKE_INSTALL_PREFIX "${QFS_OUTPUT_DIR}" CACHE PATH
"install directory prefix" FORCE)
endif()
message(STATUS "QFS deployable output directory: ${CMAKE_INSTALL_PREFIX}")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${QFS_OUTPUT_DIR}/bin" CACHE PATH
"directory for QFS runtime build output" FORCE)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${QFS_OUTPUT_DIR}/lib" CACHE PATH
"directory for QFS shared library build output" FORCE)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${QFS_OUTPUT_DIR}/lib/static" CACHE PATH
"directory for QFS static library build output" FORCE)
foreach(QFS_OUTPUT_CONFIG DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${QFS_OUTPUT_CONFIG}
"${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" CACHE PATH
"directory for QFS runtime build output" FORCE)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${QFS_OUTPUT_CONFIG}
"${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" CACHE PATH
"directory for QFS shared library build output" FORCE)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${QFS_OUTPUT_CONFIG}
"${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}" CACHE PATH
"directory for QFS static library build output" FORCE)
endforeach()

function(qfs_set_target_runtime_output_dir output_dir)
set_target_properties(${ARGN} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${output_dir}")
foreach(QFS_OUTPUT_CONFIG DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
set_target_properties(${ARGN} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY_${QFS_OUTPUT_CONFIG} "${output_dir}")
endforeach()
endfunction()

# Build with statically linked libraries; the value for this variable has to be
# defined here overwriting whatever is in the cache.
Expand Down
51 changes: 37 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
# Do not assume GNU Make. Keep this makefile as simple as possible.

BUILD_TYPE=release
QFS_OUTPUT_DIR=output
CMAKE_OPTIONS=-D CMAKE_BUILD_TYPE=RelWithDebInfo
CMAKE=cmake
MAKE_OPTIONS=
Expand All @@ -31,31 +32,44 @@ QFSHADOOP_VERSIONS=0.23.11 1.0.4 1.1.2 2.5.1 2.7.2 2.7.7 2.8.5 2.9.2 2.1

QFS_PYTHON_DIR=python-qfs
QFS_PYTHON_WHEEL_DIR=${QFS_PYTHON_DIR}/dist
QFS_PYTHON_TEST_OPTION=test -d ${QFS_PYTHON_WHEEL_DIR} && echo -python-wheel-dir ${QFS_PYTHON_WHEEL_DIR}
QFS_PYTHON_TEST_OPTION=test -d $${qfs_output_dir}/${QFS_PYTHON_WHEEL_DIR} && echo -python-wheel-dir $${qfs_output_dir}/${QFS_PYTHON_WHEEL_DIR}
QFS_MSTRESS_ON=true

.PHONY: all
all: build

.PHONY: dir
dir:
mkdir -p build/${BUILD_TYPE}
mkdir -p build/${BUILD_TYPE} ${QFS_OUTPUT_DIR}

.PHONY: run-cmake
run-cmake: dir
cd build/${BUILD_TYPE} && ${CMAKE} ${CMAKE_OPTIONS} ../..
cd build/${BUILD_TYPE} && \
qfs_output_dir=`cd ../.. && pwd`/${QFS_OUTPUT_DIR} && \
${CMAKE} \
-D QFS_OUTPUT_DIR="$$qfs_output_dir" \
-D CMAKE_INSTALL_PREFIX="$$qfs_output_dir" \
${CMAKE_OPTIONS} ../..

.PHONY: build
build: run-cmake
cd build/${BUILD_TYPE} && $(MAKE) ${MAKE_OPTIONS} install \
`${QFS_MSTRESS_ON} && \
echo ${QFSHADOOP_VERSIONS} | grep '3\.4\.1' >/dev/null 2>&1 && \
mvn --version >/dev/null 2>&1 && echo mstress-bootstrap mstress-tarball`
if ls -1 build/${BUILD_TYPE}/benchmarks/mstress*.tgz >/dev/null 2>&1; then \
mkdir -p ${QFS_OUTPUT_DIR}/benchmarks && \
cp build/${BUILD_TYPE}/benchmarks/mstress*.tgz \
${QFS_OUTPUT_DIR}/benchmarks/; \
fi

.PHONY: java
java: build
./src/java/javabuild.sh ${JAVA_BUILD_OPTIONS} clean
./src/java/javabuild.sh ${JAVA_BUILD_OPTIONS}
if ls -1 build/java/qfs-access/qfs-access*.jar >/dev/null 2>&1; then \
cp build/java/qfs-access/qfs-access*.jar ${QFS_OUTPUT_DIR}/lib/; \
fi

.PHONY: hadoop-jars
hadoop-jars: java
Expand All @@ -67,6 +81,9 @@ hadoop-jars: java
|| exit 1; \
done \
; fi
if ls -1 build/java/hadoop-qfs/hadoop-*.jar >/dev/null 2>&1; then \
cp build/java/hadoop-qfs/hadoop-*.jar ${QFS_OUTPUT_DIR}/lib/; \
fi

.PHONY: go
go: build
Expand All @@ -76,7 +93,7 @@ go: build
exit; \
} \
END { exit ret ? 0 : 1 }'; then \
QFS_BUILD_DIR=`pwd`/build/$(BUILD_TYPE) && \
QFS_BUILD_DIR=`pwd`/${QFS_OUTPUT_DIR} && \
cd src/go && \
CGO_CFLAGS="-I$${QFS_BUILD_DIR}/include" && \
export CGO_CFLAGS && \
Expand All @@ -93,6 +110,7 @@ go: build
.PHONY: tarball
tarball: hadoop-jars python
cd build && \
qfs_output_dir=../${QFS_OUTPUT_DIR}; \
myuname=`uname -s`; \
myarch=`cc -dumpmachine 2>/dev/null | cut -d - -f 1` ; \
[ x"$$myarch" = x ] && \
Expand Down Expand Up @@ -129,16 +147,16 @@ tarball: hadoop-jars python
{ test -d tmpreldir || mkdir tmpreldir; } && \
rm -rf "tmpreldir/$$tarname" && \
mkdir "tmpreldir/$$tarname" && \
cp -r ${BUILD_TYPE}/bin ${BUILD_TYPE}/lib \
${BUILD_TYPE}/include ../scripts ../webui \
cp -r $$qfs_output_dir/bin $$qfs_output_dir/lib \
$$qfs_output_dir/include ../scripts ../webui \
../examples ../benchmarks "tmpreldir/$$tarname/" && \
if ls -1 ./java/qfs-access/qfs-access-*.jar >/dev/null 2>&1; then \
cp ./java/qfs-access/qfs-access*.jar "tmpreldir/$$tarname/lib/"; fi && \
if ls -1 ./java/hadoop-qfs/hadoop-*.jar >/dev/null 2>&1; then \
cp ./java/hadoop-qfs/hadoop-*.jar "tmpreldir/$$tarname/lib/"; fi && \
if ls -1 ${BUILD_TYPE}/${QFS_PYTHON_WHEEL_DIR}/qfs*.whl >/dev/null 2>&1; \
if ls -1 $$qfs_output_dir/${QFS_PYTHON_WHEEL_DIR}/qfs*.whl >/dev/null 2>&1; \
then \
cp ${BUILD_TYPE}/${QFS_PYTHON_WHEEL_DIR}/qfs*.whl \
cp $$qfs_output_dir/${QFS_PYTHON_WHEEL_DIR}/qfs*.whl \
"tmpreldir/$$tarname/lib/"; fi && \
if ls -1 ${BUILD_TYPE}/benchmarks/mstress.tgz > /dev/null 2>&1; then \
cp ${BUILD_TYPE}/benchmarks/mstress.tgz \
Expand All @@ -151,30 +169,35 @@ python: build
if python3 -c 'import sys; exit(0 if sys.version_info >= (3, 6) else 1)' \
>/dev/null 2>&1 && \
python3 -c 'import venv' >/dev/null 2>&1 ; then \
cd build/${BUILD_TYPE} && \
cd ${QFS_OUTPUT_DIR} && \
rm -rf ${QFS_PYTHON_DIR} && \
mkdir ${QFS_PYTHON_DIR} && \
cd ${QFS_PYTHON_DIR} && \
ln -s .. qfs && \
ln -s ../../../src/cc/access/kfs_setup.py setup.py && \
python3 -m venv .venv && \
. .venv/bin/activate && python -m pip install build && \
python -m build -w . ; \
python -m build -w . && \
if ls -1 dist/qfs*.whl >/dev/null 2>&1; then \
cp dist/qfs*.whl ../lib/; \
fi ; \
else \
echo 'python3 module venv is not available'; \
fi

.PHONY: mintest
mintest: hadoop-jars python
cd build/${BUILD_TYPE} && \
qfs_output_dir=`cd ../.. && pwd`/${QFS_OUTPUT_DIR} && \
../../src/test-scripts/qfstest.sh \
`${QFS_PYTHON_TEST_OPTION}` \
-install-prefix . -auth ${QFSTEST_OPTIONS}
-install-prefix "$$qfs_output_dir" -auth ${QFSTEST_OPTIONS}

.PHONY: test
test: mintest
cd build/${BUILD_TYPE} && \
installbindir=`pwd`/bin && \
qfs_output_dir=`cd ../.. && pwd`/${QFS_OUTPUT_DIR} && \
installbindir=$$qfs_output_dir/bin && \
metadir=$$installbindir && \
export metadir && \
chunkdir=$$installbindir && \
Expand All @@ -192,7 +215,7 @@ test: mintest
echo '--------- Test without authentication --------' && \
../../src/test-scripts/qfstest.sh \
`${QFS_PYTHON_TEST_OPTION}` \
-install-prefix . -noauth ${QFSTEST_OPTIONS} ; \
-install-prefix "$$qfs_output_dir" -noauth ${QFSTEST_OPTIONS} ; \
fi

.PHONY: rat
Expand All @@ -201,4 +224,4 @@ rat: dir

.PHONY: clean
clean:
rm -rf build
rm -rf build ${QFS_OUTPUT_DIR}
2 changes: 2 additions & 0 deletions benchmarks/mstress/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#

add_executable(mstress_client EXCLUDE_FROM_ALL mstress_client.cc)
qfs_set_target_runtime_output_dir("${CMAKE_CURRENT_BINARY_DIR}"
mstress_client)

if(USE_STATIC_LIB_LINKAGE)
add_dependencies(mstress_client kfsClient)
Expand Down
3 changes: 3 additions & 0 deletions examples/cc/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,8 @@ else (USE_STATIC_LIB_LINKAGE)
add_dependencies (qfssample kfsClient-shared)
endif (USE_STATIC_LIB_LINKAGE)

qfs_set_target_runtime_output_dir("${QFS_OUTPUT_DIR}/bin/examples"
qfssample)

install (TARGETS qfssample
RUNTIME DESTINATION bin/examples)
2 changes: 2 additions & 0 deletions src/cc/devtools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ foreach (exe_file ${exe_files})
)
endif (USE_STATIC_LIB_LINKAGE)
endforeach (exe_file)
qfs_set_target_runtime_output_dir("${QFS_OUTPUT_DIR}/bin/devtools"
${exe_files})

#
install (TARGETS ${exe_files}
Expand Down
2 changes: 2 additions & 0 deletions src/cc/emulator/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ foreach (exe_file ${exe_files})
kfsEmulator
)
endforeach (exe_file)
qfs_set_target_runtime_output_dir("${QFS_OUTPUT_DIR}/bin/emulator"
${exe_files})

if (CMAKE_SYSTEM_NAME STREQUAL "SunOS")
target_link_libraries(kfsEmulator mtmalloc)
Expand Down
2 changes: 2 additions & 0 deletions src/cc/qcrs/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ add_executable (${rsmktablebin} mktable_main.c)
target_link_libraries (${rstestbin} kfsrs)
add_dependencies (${rstestbin} kfsrs)
add_dependencies (${rsmktablebin} kfsrs)
qfs_set_target_runtime_output_dir("${QFS_OUTPUT_DIR}/bin/devtools"
${rstestbin} ${rsmktablebin})

install (TARGETS kfsrs kfsrs-shared
LIBRARY DESTINATION lib
Expand Down
2 changes: 2 additions & 0 deletions src/cc/qfsc/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ target_link_libraries (qfsc-shared

add_executable (test-qfsc test-qfsc.c)
set_target_properties (test-qfsc PROPERTIES LINKER_LANGUAGE CXX)
qfs_set_target_runtime_output_dir("${QFS_OUTPUT_DIR}/bin/devtools"
test-qfsc)

if (USE_STATIC_LIB_LINKAGE)
add_dependencies (test-qfsc qfsc)
Expand Down
2 changes: 2 additions & 0 deletions src/cc/tools/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ foreach (exe_file ${exe_files})
target_link_libraries (${exe_file} tools-shared)
endif (USE_STATIC_LIB_LINKAGE)
endforeach (exe_file)
qfs_set_target_runtime_output_dir("${QFS_OUTPUT_DIR}/bin/tools"
${exe_files})

#
install (TARGETS ${exe_files} tools
Expand Down