Skip to content

Commit d657698

Browse files
reem-codesreem-codes
committed
release
0 parents  commit d657698

178 files changed

Lines changed: 49100 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: build
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
pull_request:
7+
branches: [ "main" ]
8+
9+
jobs:
10+
build:
11+
runs-on: ${{ matrix.os }}
12+
13+
strategy:
14+
# Set fail-fast to false to ensure that feedback is delivered for all matrix combinations. Consider changing this to true when your workflow is stable.
15+
fail-fast: false
16+
17+
# Set up a matrix to run the following 3 configurations:
18+
# 1. <Windows, Release, latest MSVC compiler toolchain on the default runner image, default generator>
19+
# 2. <Linux, Release, latest GCC compiler toolchain on the default runner image, default generator>
20+
# 3. <Linux, Release, latest Clang compiler toolchain on the default runner image, default generator>
21+
#
22+
# To add more build types (Release, Debug, RelWithDebInfo, etc.) customize the build_type list.
23+
matrix:
24+
os: [ubuntu-latest , windows-latest]
25+
build_type: [Debug]
26+
c_compiler: [gcc, cl]
27+
triplet: [x64-linux, x64-windows]
28+
include:
29+
- os: windows-latest
30+
c_compiler: cl
31+
cpp_compiler: cl
32+
triplet: x64-windows
33+
- os: ubuntu-latest
34+
c_compiler: gcc
35+
cpp_compiler: g++
36+
triplet: x64-linux
37+
exclude:
38+
- os: windows-latest
39+
c_compiler: gcc
40+
- os: windows-latest
41+
triplet: x64-linux
42+
- os: ubuntu-latest
43+
c_compiler: cl
44+
- os: ubuntu-latest
45+
triplet: x64-windows
46+
steps:
47+
# cd current directory
48+
- uses: actions/checkout@v3
49+
50+
# set build folder
51+
- name: Set reusable strings
52+
id: strings
53+
shell: bash
54+
run: |
55+
echo "build-output-dir=${{ github.workspace }}/build" >> "$GITHUB_OUTPUT"
56+
57+
# install dependencies
58+
- name: Install dependencies (Linux)
59+
if: runner.os == 'Linux'
60+
run: sudo apt-get update && sudo apt install doxygen libglfw3 libglfw3-dev freeglut3-dev libxmu-dev libxi-dev libxcursor-dev libxinerama-dev libglew-dev mesa-utils libtbb-dev -y
61+
62+
# vcpkg
63+
- name: vcpkg-action
64+
uses: johnwason/vcpkg-action@v5
65+
id: vcpkg
66+
with:
67+
manifest-dir: ${{ github.workspace }} # Set to directory containing vcpkg.json
68+
triplet: ${{ matrix.triplet }}
69+
token: ${{ github.token }}
70+
71+
# run cmake
72+
- name: Configure CMake
73+
# Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
74+
# See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
75+
run: >
76+
cmake -B ${{ steps.strings.outputs.build-output-dir }}
77+
-DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
78+
-DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
79+
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
80+
-DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
81+
-DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }}
82+
-S ${{ github.workspace }}
83+
84+
# build
85+
- name: Build
86+
# Build your program with the given configuration. Note that --config is needed because the default Windows generator is a multi-config generator (Visual Studio generator).
87+
run: cmake --build ${{ steps.strings.outputs.build-output-dir }} --config ${{ matrix.build_type }}

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
data_structure/.idea
2+
data_structure/bin
3+
data_structure/cmake-build-debug
4+
script/venv
5+
web_visualizer/res/styles/style.css.map

‎.nojekyll‎

Whitespace-only changes.

‎README.md‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Lactea: Web-Based Spectrum-Preserving Multi-Resolution Visualization of the GAIA Star Catalog
2+
[![build](https://github.com/vccvisualization/lactea/actions/workflows/build.yml/badge.svg)](https://github.com/vccvisualization/lactea/actions/workflows/build.yml)
3+
4+
[![Watch the video](https://vccvisualization.org/publications/2025_05_alghamdi_lactea_teaser.png)](https://www.youtube.com/watch?v=mR2bpCEupvk)
5+
6+
This repository contains an open-source, web-based implementation of Lactea, a submitted paper to the EuroVis 2025 conference introducing a spectral visualization, multi-resolution technique for a large star catalog. This repository works with the European Space Agency (ESA)'s GAIA catalog, which contains the spectral information of almost 2 billion stars.
7+
8+
9+
## Download and Process Data
10+
11+
[script](script) folder contains a Python script to download the 3rd release of GAIA. After making a virtual environment, install `gaiaxpy` as well as `numpy` and `pandas`. `gaiaxpy` is used to sample the continuous spectrum.
12+
13+
Once the environment is set up, run the Python script to generate a folder with 3386 files. Lastly, merge them all into one big file before moving to the next step.
14+
15+
An example bash script of all these steps is available at [script/download.sh](script/download.sh).
16+
17+
## Data structure
18+
19+
The [data structure](data_structure) building tool is written in C++. The core library is available at [data_structure/src/lactea_xp_merged](data_structure/src/lactea_xp_merged). Other scripts are also available to sort, sample, build, and chunk the tree.
20+
21+
To build a WebGPU-friendly tree:
22+
23+
### Sort
24+
25+
Run [`raw_star_db_sorter_xp_merged.cpp`](data_structure/src/raw_star_db_sorter_xp_merged/raw_star_db_sorter_xp_merged.cpp) to sort the merged binary file based on the magnitude using [alveko's external_sort](https://github.com/alveko/external_sort) library.
26+
27+
The script takes two arguments:
28+
* input file name: the name of the merged file obtained from the previous step (e.g., `/data/gdr3_gaia_source_xp_extinction.stars`)
29+
* output file name: the sorted filename (e.g., `/data/gdr3_gaia_source_xp_extinction_sorted.stars`)
30+
31+
### [Optional] Create Random Subsets
32+
33+
Make subsets of the data using [`raw_star_db_subset_writer_xp_merged.cpp`](data_structure/src/raw_star_db_subset_writer_xp_merged/raw_star_db_subset_writer_xp_merged.cpp). There are 3 input parameters:
34+
* the sorted file to sample (e.g., `/data/gdr3_gaia_source_xp_extinction_sorted.stars`)
35+
* output file name: (e.g. `/data/gdr3_gaia_source_xp_extinction_1M.stars`)
36+
* number of stars: how many star samples (e.g., `1000000`)
37+
38+
### Build and Chunk the tree
39+
40+
Once the data is sorted (and sampled if desired), build the tree and save it as chunks using [`sandbox.cpp`](data_structure/src/sandbox/sandbox.cpp). 4 arguments are required:
41+
42+
* the sorted file: could be the whole database or a sampled database (e.g., `/data/gdr3_gaia_source_xp_extinction_1M.stars`)
43+
* the output tree file: This only saves the tree structure. (e.g., `/data/gdr3_gaia_source_xp_extinction_1M.str`)
44+
* the chunks output directory: the directory to save each node chunk for WebGPU visualization. (e.g., `/data/gdr3_gaia_source_xp_extinction_1M`)
45+
* The max number of stars per node: In the paper, `1000` stars per node is used. This generates chunks of size `5.6 MB`, which is suitable for fetching on the web
46+
47+
It is also possible to divide this into two subtasks using [`star_tree_builder_xp_merged.cpp`](data_structure/src/star_tree_builder_xp_merged/star_tree_builder_xp_merged.cpp) and [`star_tree_chunker_xp_merged.cpp)`](data_structure/src/star_tree_chunker_xp_merged/star_tree_chunker_xp_merged.cpp)
48+
49+
50+
A sample of the chunked data folder is available in [web_visualizer/res/data](web_visualizer/res/data), using 10,000 samples due to GitHub's file size restriction.
51+
52+
53+
## Visualization
54+
55+
[The visualization](web_visualizer) is implemented using WebGPU for accessibility. [lil-gui](https://lil-gui.georgealways.com/#), [chart js](https://www.chartjs.org/), [d3](https://d3js.org/) and [glmatrix](https://glmatrix.net/) were used.
56+
57+
The shaders are available in [web_visualizer/res/shaders](web_visualizer/res/shaders), while the main JS code is in [web_visualizer/src](web_visualizer/src).
58+
59+
A minimal JS implementation of the [C++ lactea library](data_structure/src/lactea_xp_merged) is in [web_visualizer/src/lactea](web_visualizer/src/lactea). Change line 10 of [web_visualizer/src/lactea/StarTree.js](web_visualizer/src/lactea/StarTree.js) to reflect your served data path.
60+
61+
```
62+
// TODO: put your own data folder
63+
this.path = "http://127.0.0.1:5501/gdr3_gaia_source_xp_extinction/"
64+
```
65+
66+
The components of the visualizer are:
67+
* [`LacteaInterface.js`](web_visualizer/src/visualizer/LacteaInterface.js): this class connects to lactea, builds the viewbounds based on the camera parameters, traverses the tree, and builds the star and patch queues.
68+
* [`LacteaCache.js`](web_visualizer/src/visualizer/LacteaCache.js): a lactea interface instance is created, as well as the star and node caches. `cacheLoad` function manages the queues, file loading, and the CPU and GPU caches.
69+
* [`PipelineStarCompute.js`](web_visualizer/src/visualizer/PipelineStarCompute.js): The star accumulation compute shader.
70+
* [`PipelineNodeCompute.js`](web_visualizer/src/visualizer/PipelineNodeCompute.js): Storing the patch ID into the spectral buffer pipeline
71+
* [`PipelineRender.js`](web_visualizer/src/visualizer/PipelineRender.js): rendering, tone mapping, and the different astronomical tasks pipeline. Once the spectral buffer is filled, any task, such as rendering, image processing, derivative computation or any other astronomical spectral processing task can be computed in a negligible time
72+
* [`LacteaVisualizer.js`](web_visualizer/src/visualizer/LacteaVisualizer.js): entry point of the visualizer
73+
74+
## Citation
75+
76+
To cite this work
77+
78+
```
79+
@article{alghamdi2025Lactea,
80+
title = {Lactea: Web-Based Spectrum-Preserving Multi-Resolution Visualization of the GAIA Star Catalog},
81+
author = {Alghamdi, Reem and Hadwiger, Markus and Reina, Guido and Jaspe-Villanueva, Alberto},
82+
journal = {Computer Graphics Forum (Proceedings of Eurographics Conference on Visualization 2025)},
83+
year = {2025},
84+
volume = {44},
85+
number = {3},
86+
pages = {to appear}
87+
}
88+
```
89+
90+
## Acknowledgement
91+
92+
This work has made use of data from the European Space Agency (ESA) mission [Gaia](https://www.cosmos.esa.int/gaia), processed by the Gaia Data Processing and Analysis Consortium ([DPAC](https://www.cosmos.esa.int/web/gaia/dpac/consortium)).

‎data_structure/CMakeLists.txt‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
2+
CMAKE_MINIMUM_REQUIRED(VERSION 3.7...3.23)
3+
set(CMAKE_CXX_STANDARD 20)
4+
PROJECT(lactea C CXX)
5+
# set(CMAKE_CXX_FLAGS -pthread)
6+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0")
7+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0")
8+
############ Dependencies
9+
10+
find_package (Threads REQUIRED)
11+
find_package(glm CONFIG REQUIRED)
12+
find_package(glew CONFIG REQUIRED)
13+
find_package(glfw3 CONFIG REQUIRED)
14+
find_package(imgui CONFIG REQUIRED)
15+
#find_package(Boost)
16+
find_package(TBB CONFIG REQUIRED)
17+
18+
############## Default target to build
19+
20+
if(NOT CMAKE_BUILD_TYPE)
21+
set(CMAKE_BUILD_TYPE Release)
22+
endif(NOT CMAKE_BUILD_TYPE)
23+
24+
############## Output to the "bin" and "lib" folders
25+
26+
set( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/bin )
27+
set( CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/lib )
28+
foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} )
29+
string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG )
30+
set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_SOURCE_DIR}/bin )
31+
set( CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_SOURCE_DIR}/lib )
32+
endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES )
33+
34+
############## Removing some warnings on Visual Studio
35+
36+
if(MSVC)
37+
add_definitions(/D_CRT_SECURE_NO_WARNINGS)
38+
add_definitions(/D_SCL_SECURE_NO_WARNINGS)
39+
# For generating folders in IDE by same directory structure
40+
file(GLOB_RECURSE _source_list *.cpp* *.h* *.hpp*)
41+
foreach(_source IN ITEMS ${_source_list})
42+
get_filename_component(_source_path "${_source}" PATH)
43+
string(REPLACE "${CMAKE_SOURCE_DIR}/src/" "" _group_path "${_source_path}")
44+
string(REPLACE "/" "\\" _group_path "${_group_path}")
45+
source_group("${_group_path}" FILES "${_source}")
46+
endforeach()
47+
endif()
48+
49+
######## Includes for everybody ############
50+
include_directories( "${PROJECT_SOURCE_DIR}/src" )
51+
52+
######## Files ############
53+
file(GLOB_RECURSE LACTEAXP_MERGE_SRC_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "src/lactea_xp_merged/*.cpp" "src/lactea_xp_merged/*.hpp" "src/lactea_xp_merged/*.c" "src/lactea_xp_merged/*.h")
54+
file(GLOB_RECURSE UTILS_SRC_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "src/utils/*.cpp" "src/utils/*.hpp" "src/utils/*.c" "src/utils/*.h" "src/utils/gl/*.cpp" "src/utils/gl/*.hpp" "src/utils/gl/*.c" "src/utils/gl/*.h" "src/utils/ImGuiFileDialog/ImGuiFileDialog.cpp")
55+
file(GLOB_RECURSE EXTERNAL_SORT_SRC_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "src/external_sort/*.cpp" "src/external_sort/*.hpp")
56+
57+
##############################################################################
58+
59+
60+
############# XP SAMPLED CONTINUOUS AND MERGED VERSIONS ###############
61+
62+
###### Libary LACTEA XP MERGED
63+
add_library( lactea_xp_merged ${LACTEAXP_MERGE_SRC_FILES}
64+
src/utils/portable_io.h
65+
src/utils/portable_io.cpp
66+
)
67+
target_link_libraries(lactea_xp_merged
68+
TBB::tbb
69+
Threads::Threads
70+
)
71+
72+
#### Star sorter XP MERGED
73+
add_executable( lactea_raw_star_db_sorter_xp_merged
74+
${EXTERNAL_SORT_SRC_FILES}
75+
src/raw_star_db_sorter_xp_merged/raw_star_db_sorter_xp_merged.cpp
76+
)
77+
target_link_libraries(lactea_raw_star_db_sorter_xp_merged lactea_xp_merged)
78+
79+
80+
#### Star subset writer sorter XP MERGED
81+
add_executable( lactea_raw_star_db_subset_writer_xp_merged
82+
src/raw_star_db_subset_writer_xp_merged/raw_star_db_subset_writer_xp_merged.cpp
83+
)
84+
target_link_libraries(lactea_raw_star_db_subset_writer_xp_merged lactea_xp_merged)
85+
86+
#### Builder XP MERGED
87+
add_executable( lactea_star_tree_builder_xp_merged
88+
src/star_tree_builder_xp_merged/star_tree_builder_xp_merged.cpp
89+
)
90+
target_link_libraries(lactea_star_tree_builder_xp_merged lactea_xp_merged)
91+
92+
#### Chunker MERGED
93+
add_executable( lactea_star_tree_chunker_xp_merged
94+
src/star_tree_chunker_xp_merged/star_tree_chunker_xp_merged.cpp
95+
)
96+
target_link_libraries(lactea_star_tree_chunker_xp_merged lactea_xp_merged)
97+
98+
99+
#### Sandbox for test
100+
add_executable( lactea_sandbox
101+
src/sandbox/sandbox.cpp
102+
)
103+
target_link_libraries(lactea_sandbox lactea_xp_merged)
104+

‎data_structure/CMakeSettings.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"configurations": [
3+
{
4+
"name": "x64-Debug",
5+
"generator": "Ninja",
6+
"configurationType": "Debug",
7+
"inheritEnvironments": [ "msvc_x64_x64" ],
8+
"buildRoot": "${projectDir}\\out\\build\\${name}",
9+
"installRoot": "${projectDir}\\out\\install\\${name}",
10+
"buildCommandArgs": "",
11+
"ctestCommandArgs": ""
12+
}
13+
]
14+
}

0 commit comments

Comments
 (0)