Skip to content

build mnist files backend #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jackculpepper wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

build mnist files backend #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions examples/mnist/convert_mnist_data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
#include <fstream> // NOLINT(readability/streams)
#include <string>

#include "opencv2/core/core.hpp"
#include "opencv2/opencv.hpp"
#include "caffe/proto/caffe.pb.h"

using namespace caffe; // NOLINT(build/namespaces)
Expand Down Expand Up @@ -94,6 +96,16 @@ void convert_dataset(const char* image_filename, const char* label_filename,
<< "mdb_txn_begin failed";
CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
<< "mdb_open failed. Does the lmdb already exist? ";
} else if (db_backend == "files") { // png files
CHECK_EQ(mkdir(db_path, 0744), 0)
<< "mkdir " << db_path << "failed";
for (int i = 0; i < 10; i++) {
int pathlen = strlen(db_path) + strlen("/0") + 1;
char path[pathlen];
snprintf(path, pathlen, "%s/%c", db_path, '0' + i);
CHECK_EQ(mkdir(path, 0744), 0)
<< "mkdir " << path << "failed";
}
} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
Expand Down Expand Up @@ -131,6 +143,15 @@ void convert_dataset(const char* image_filename, const char* label_filename,
mdb_key.mv_data = reinterpret_cast<void*>(&keystr[0]);
CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS)
<< "mdb_put failed";
} else if (db_backend == "files") { // png files
cv::Mat img = cv::Mat(28, 28, CV_8UC1, pixels);

int pathlen = strlen(db_path) + strlen("/0/12345.png") + 1;
char filename[pathlen];

snprintf(filename, pathlen, "%s/%c/%05d.png", db_path, '0' + label, item_id);
//std::cout << "writing " << filename << std::endl;
cv::imwrite(filename, img);
} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
Expand All @@ -146,6 +167,8 @@ void convert_dataset(const char* image_filename, const char* label_filename,
<< "mdb_txn_commit failed";
CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS)
<< "mdb_txn_begin failed";
} else if (db_backend == "files") { // png files

} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
Expand All @@ -161,6 +184,8 @@ void convert_dataset(const char* image_filename, const char* label_filename,
CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed";
mdb_close(mdb_env, mdb_dbi);
mdb_env_close(mdb_env);
} else if (db_backend == "files") { // files

} else {
LOG(FATAL) << "Unknown db backend " << db_backend;
}
Expand Down
19 changes: 19 additions & 0 deletions examples/mnist/create_mnist.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ DATA=data/mnist
BUILD=build/examples/mnist

BACKEND="lmdb"
BACKEND="files"

echo "Creating ${BACKEND}..."

Expand All @@ -18,4 +19,22 @@ $BUILD/convert_mnist_data.bin $DATA/train-images-idx3-ubyte \
$BUILD/convert_mnist_data.bin $DATA/t10k-images-idx3-ubyte \
$DATA/t10k-labels-idx1-ubyte $EXAMPLE/mnist_test_${BACKEND} --backend=${BACKEND}

if [ "$BACKEND" == "files" ]
then
for t in train test
do
rm -f $EXAMPLE/mnist_${t}.txt

for d in 0 1 2 3 4 5 6 7 8 9
do
for f in $EXAMPLE/mnist_${t}_${BACKEND}/$d/*.png
do
echo "$f $d" >>$EXAMPLE/mnist_${t}_a.txt
done
done

shuf $EXAMPLE/mnist_${t}_a.txt >$EXAMPLE/mnist_${t}_b.txt
done
fi

echo "Done."