Merge pull request sleuthkit#3249 from sleuthkit/test-tsk_loaddb

Test tsk loaddb

Author: simsong

.github/workflows/build-unix.yml

@@ -139,23 +139,29 @@ jobs:
           MATRIX_ENABLE_JAVA: ${{ matrix.enable_java }}
         if: ${{ matrix.os == 'macos' }}
         run: |
+          export CPPFLAGS=-I/opt/homebrew/include/
+          export LDFLAGS=-L/opt/homebrew/lib/
           brew update
-          brew install libtool autoconf automake libtool libewf libmagic
+          brew install libtool autoconf automake libtool libewf libmagic afflib
           echo MATRIX_ENABLE_JAVA=$MATRIX_ENABLE_JAVA
           if [ ${MATRIX_ENABLE_JAVA}x == "yesx" ]; then
              echo Installing JAVA
              brew install openjdk@17
-             export JAVA_HOME="$(brew --prefix openjdk)"
-             export PATH="$JAVA_HOME/bin:$PATH"
-             export CPPFLAGS="-I$JAVA_HOME/include"
-             export JNI_CPPFLAGS="-I$JAVA_HOME/include -I$JAVA_HOME/include/darwin"
+             echo JAVA_HOME="$(brew --prefix openjdk)" >> $GITHUB_ENV
+             echo PATH="$JAVA_HOME/bin:$PATH" >> $GITHUB_ENV
+             echo JNI_CPPFLAGS="-I$JAVA_HOME/include -I$JAVA_HOME/include/darwin" >> $GITHUB_ENV
+             export CPPFLAGS="-I$JAVA_HOME/include $CPPFLAGS"
           fi
+          echo search for aff
+          find /opt/homebrew -name 'aff*'
+          echo "CPPFLAGS=$CPPFLAGS" >> $GITHUB_ENV
+          echo "LDFLAGS=$LDFLAGS" >> $GITHUB_ENV
 
       - name: Install Linux packages
         if: ${{ matrix.os == 'linux' }}
         run: |
           sudo apt update
-          sudo apt install -y ant autoconf automake g++ libssl-dev libewf-dev libqcow-dev libvhdi-dev libvmdk-dev libmagic-dev libtool make pkg-config zlib1g-dev
+          sudo apt install -y ant autoconf automake g++ libssl-dev afflib-tools libewf-dev libqcow-dev libvhdi-dev libvmdk-dev libmagic-dev libtool make pkg-config zlib1g-dev
 
       - name: Install Mingw packages and setup for cross-compiling
         if: ${{ matrix.os == 'mingw' }}
@@ -179,17 +185,22 @@ jobs:
 
       - name: Run bootstrap
         run: |
+          echo CPPFLAGS=$CPPFLAGS
+          echo LDFLAGS=$LDFLAGS
           ./bootstrap
 
       - name: Run configure
         run: |
           ./configure ${{ matrix.configure_opts }}
 
-      - name: Get the disk images
+      - name: Unpack and List the test data
         run: |
-          make test/get_images/test_images.txt
-          cat test/get_images/test_images.txt
-          ls -l test/from_brian
+          cd ..
+          pwd
+          git clone https://github.com/sleuthkit/sleuthkit_test_data
+          cd sleuthkit_test_data
+          make unpack
+          find . -ls | grep -v '[.]git'
 
       - name: Run make
         run: |
@@ -219,7 +230,7 @@ jobs:
         run: |
           ./configure ${{ matrix.configure_opts }} --enable-address-sanitizer CFLAGS=-g CXXFLAGS=-g
 
-      - name: Run make with address-sanitizer
+      - name: Check with address-sanitizer
         if: ${{ matrix.address_sanitizer == 'yes' }}
         run: |
           make -j check VERBOSE=1
@@ -237,15 +248,15 @@ jobs:
       - name: Run make for codecov
         if: ${{ matrix.codecov == 'yes' }}
         run: |
-          make -j test/fiwalk/fiwalk_test V=0
-          make -j test/runner V=0
-
-      - name: Run unit tests for codecov
-        if: ${{ matrix.codecov == 'yes' }}
-        run: |
-          test/fiwalk/fiwalk_test -s
-          test/runner -s
+          make -j check V=0
+          #make -j test/fiwalk/fiwalk_test V=0
+          #make -j test/runner V=0
 
+#      - name: Run unit tests for codecov
+#        if: ${{ matrix.codecov == 'yes' }}
+#        run: |
+#          test/fiwalk/fiwalk_test -s
+#          test/runner -s
 
       - name: run gcov
         if: ${{ matrix.codecov == 'yes' }}
@@ -276,10 +287,10 @@ jobs:
             executables/*
           retention-days: 15
 
-#      - name: Run distcheck
-#        run: |
-#          ./configure
-#          make distcheck
-#
-#      - uses: ammaraskar/gcc-problem-matcher@0.2.0
-#        name: GCC Problem Matcher
+      - name: Run distcheck
+        run: |
+          ./configure
+          make distcheck
+
+      - uses: ammaraskar/gcc-problem-matcher@0.2.0
+        name: GCC Problem Matcher

.gitignore

@@ -207,3 +207,4 @@ venv
 wget-log
 TAGS
 test/from_brian/
+*.log

Makefile.am

@@ -634,6 +634,7 @@ check_script_files = \
 	test/img_dump/test_imgs_E01.sh \
 	test/legacy/runtests.sh \
 	test/tools/tool_differ.sh \
+	test/tools/autotools/test_loaddb.sh \
 	test/tools/fstools/test_fls.sh \
 	test/tools/vstools/test_mmls.sh \
 	test/tools/vstools/test_mmls_E01.sh
@@ -647,6 +648,7 @@ TESTS =	\
 	test/fiwalk/fiwalk_test \
 	test/img_dump/test_imgs.sh \
 	test/legacy/runtests.sh \
+	test/tools/autotools/test_loaddb.sh \
 	test/tools/fstools/test_fls.sh \
 	test/tools/vstools/test_mmls.sh
 
@@ -697,33 +699,12 @@ test_runner_SOURCES = \
 	test/runner.cpp \
 	vendors/catch.hpp
 
-EXTRA_test_runner_DEPENDENCIES = test/get_images/test_images.txt
-
 test_fiwalk_fiwalk_test_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/vendors $(CATCH2_CPPFLAGS)
 test_fiwalk_fiwalk_test_LDADD = $(TSK_LIBS)
 test_fiwalk_fiwalk_test_SOURCES = \
 	test/fiwalk/fiwalk_test.cpp \
 	$(tools_fiwalk_src_libfiwalk_la_SOURCES)
 
-EXTRA_test_fiwalk_fiwalk_test_DEPENDENCIES = test/get_images/test_images.txt
-
-# System for getting test images
-# This will be replaced with a sub repo
-
-get_images_inputs = \
-    test/get_images/get_test_images.bash \
-    test/get_images/get_test_images.py \
-    test/get_images/test_images.yaml
-
-test/get_images/test_images.txt: $(get_images_inputs)
-	bash $(top_srcdir)/test/get_images/get_test_images.bash
-
-EXTRA_DIST += $(get_images_inputs)
-
-##
-
-CLEANFILES += test/get_images/test_images.txt
-
 test_img_dump_img_dump_LDADD = $(TSK_LIBS)
 test_img_dump_img_dump_SOURCES = test/img_dump/img_dump.cpp
 

README.md

@@ -15,7 +15,7 @@ the tool or customize it to specific needs.
 The Sleuth Kit uses code from the file system analysis tools of
 The Coroner's Toolkit (TCT) by Wietse Venema and Dan Farmer.  The
 TCT code was modified for platform independence.  In addition,
-support was added for the NTFS (see [wiki/ntfs](http://wiki.sleuthkit.org/index.php?title=NTFS_Implementation_Notes)) 
+support was added for the NTFS (see [wiki/ntfs](http://wiki.sleuthkit.org/index.php?title=NTFS_Implementation_Notes))
 and FAT (see [wiki/fat](http://wiki.sleuthkit.org/index.php?title=FAT_Implementation_Notes)) file systems.  Previously, The Sleuth Kit was
 called The @stake Sleuth Kit (TASK).  The Sleuth Kit is now independent
 of any commercial or academic organizations.
@@ -34,9 +34,9 @@ The Sleuth Kit allows one to analyze a disk or file system image
 created by 'dd', or a similar application that creates a raw image.
 These tools are low-level and each performs a single task.  When
 used together, they can perform a full analysis.  For a more detailed
-description of these tools, refer to [wiki/filesystem](http://wiki.sleuthkit.org/index.php?title=TSK_Tool_Overview). 
+description of these tools, refer to [wiki/filesystem](http://wiki.sleuthkit.org/index.php?title=TSK_Tool_Overview).
 The tools are briefly described in a file system layered approach.  Each
-tool name begins with a letter that is assigned to the layer.  
+tool name begins with a letter that is assigned to the layer.
 
 ### File System Layer:
 A disk contains one or more partitions (or slices).  Each of these
@@ -52,7 +52,7 @@ time, and the details about each "group" in UNIX file systems.
 The content layer of a file system contains the actual file content,
 or data.  Data is stored in large chunks, with names such as blocks,
 fragments, and clusters.  All tools in this layer begin with the letters
-'blk'.  
+'blk'.
 
 The blkcat tool can be used to display the contents of a specific unit of
 the file system (similar to what 'dd' can do with a few arguments).
@@ -75,7 +75,7 @@ descriptive data such as dates and size as well as the addresses of the
 data units.  This layer describes the file in terms that the computer
 can process efficiently.   The structures that the data is stored in
 have names such as inode and directory entry.  All tools in this layer
-begin with an 'i'.  
+begin with an 'i'.
 
 The 'ils' program lists some values of the metadata structures.
 By default, it will only list the unallocated ones.  The 'istat'
@@ -87,15 +87,15 @@ contents of the data units allocated to the metadata structure
 which metadata structure has allocated a given content unit or
 file name.
 
-Refer to the [ntfs wiki](http://wiki.sleuthkit.org/index.php?title=NTFS_Implementation_Notes) 
+Refer to the [ntfs wiki](http://wiki.sleuthkit.org/index.php?title=NTFS_Implementation_Notes)
 for information on addressing metadata attributes in NTFS.
 
 ### Human Interface Layer (file):
 The human interface layer allows one to interact with files in a
 manner that is more convenient than directly with the metadata
 layer.  In some operating systems there are separate structures for
 the metadata and human interface layers while others combine them.
-All tools in this layer begin with the letter 'f'.  
+All tools in this layer begin with the letter 'f'.
 
 The 'fls' program lists file and directory names.  This tool will
 display the names of deleted files as well.  The 'ffind' program will
@@ -125,8 +125,8 @@ Also included is the 'hfind' tool.  The 'hfind' tool allows one to create
 an index of a hash database and perform quick lookups using a binary
 search algorithm.  The 'hfind' tool can perform lookups on the NIST
 National Software Reference Library (NSRL) (www.nsrl.nist.gov) and
-files created from the 'md5' or 'md5sum' command.   Refer to the 
-[wiki/hfind](http://wiki.sleuthkit.org/index.php?title=Hfind) file for more details.  
+files created from the 'md5' or 'md5sum' command.   Refer to the
+[wiki/hfind](http://wiki.sleuthkit.org/index.php?title=Hfind) file for more details.
 
 #### File Type Categories
 Different types of files typically have different internal structure.
@@ -146,6 +146,17 @@ and ignore known good files.  Refer to the [wiki/sorter](http://wiki.sleuthkit.o
 file for more details.
 
 
+## TESTING
+Both unit and end-to-end tests are located in the [test](test/) directory. Small and legacy disk images are located in [test/data](test/data/). Some tests require disk images that are include in the [Github repository](https://github.com/sleuthkit/sleuthkit_test_data); large disk images are distributed as compressed (.E01) images using [git's extensions for large objects](https://git-lfs.com/).  By default, this repo resides at [../sleuthkit_test_data](../sleuthkit_test_data]. However, it can be installed elsewhere by setting the environment variable `SLEUTHKIT_TEST_DATA_DIR`.
+
+If the disk images are not present, tests requiring the disk images will generate a warning but not an error.
+
+- Tests can be run by typing `make check`.
+
+- Tests can be run on a new distribution by typing `make distcheck`.
+
+
+
 ## LICENSE
 There are a variety of licenses used in TSK based on where they
 were first developed.  The licenses are located in the [licenses
@@ -160,14 +171,14 @@ Public License.
 - The modifications to 'mactime' from the original 'mactime' in TCT
 and 'mac-daddy' are released under the Common Public License.
 
-The library uses utilities that were released under MIT and BSD 3-clause. 
+The library uses utilities that were released under MIT and BSD 3-clause.
 
 
 ## INSTALL
 For installation instructions, refer to the INSTALL.txt document.
 
 ## OTHER DOCS
-The [wiki](http://wiki.sleuthkit.org/index.php?title=Main_Page) contains documents that 
+The [wiki](http://wiki.sleuthkit.org/index.php?title=Main_Page) contains documents that
 describe the provided tools in more detail.  The Sleuth Kit Informer is a newsletter that contains
 new documentation and articles.
 
@@ -182,4 +193,3 @@ announcements list.
 Brian Carrier
 
 carrier at sleuthkit dot org
- 

test/README.md

@@ -11,13 +11,22 @@ Goals of unit testing
 4 - Run entirely within GitHub Actions.
 5 - Test with disk images that are publicly available.
 
-
 Disk Images
 -----------
+* The disk images now reside in a [single Github repository](https://github.com/sleuthkit/sleuthkit_test_data) using [git's extensions for large objects](https://git-lfs.com/).
+* By default, this repo resides at [../sleuthkit_test_data](../sleuthkit_test_data). However, it can be installed elsewhere by setting the environment variable `SLEUTHKIT_TEST_DATA_DIR`.
 
-The unit test program is linked with fiwalk (without the fiwalk_main.cpp) so that we can just run fiwalk on disk images and do a full filesystem walk as part of our unit tests.
+Repo:
 
-Disk Images
+Images larger than 10MB should be stored as a compressed data type (e.g. .E01)
+
+
+
+Test Program
+------------
+The unit test program is linked with fiwalk (without the
+fiwalk_main.cpp) so that we can just run fiwalk on disk images and do
+a full filesystem walk as part of our unit tests.
 
 XML should be prepared using fiwalk and then processing with xmlstarlet to remove the `<creator>` section, e.g.:
 

test/data/README.md

@@ -0,0 +1 @@
+Small and legacy disk images are located here, but most should be installed in ../../../sleuthkit_test_data. This location can be changed by setting the environment variable `SLEUTHKIT_TEST_DATA_DIR`.