Merge pull request #223 from oliverfn/dev

1.0.5-of

Author: oracle58

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## 1.0.5
 -  Fixed `getBalance`: `RoundViewModel.get()` returns null on `index`=0, thus NPE was thrown when `references` were not passed and the first round hadn't been completed. In our implementation the snapshot is already constructed based on relative confirmations, thus it suffices for `getBalances` to respond with balance according to `latestSnapshot`.
+-  Updated `previousEpochsSpentAddresses` resource files 
 
 ## 1.0.4
 -  Fixed several zmq publish statements in which an incorrect format was specified.

README.md

@@ -6,7 +6,7 @@
 
 Pendulum is a quorum based [Tangle](https://github.com/iotaledger/iri/) implementation designed towards reliable timekeeping and high-throughput messaging.
 
--   **Latest release:** 1.0.4 pre-release
+-   **Latest release:** 1.0.5 pre-release
 -   **License:** GPLv3
 
 Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)!

src/main/java/net/helix/pendulum/Pendulum.java

@@ -138,14 +138,12 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException
                 ? new LocalSnapshotManagerImpl()
                 : null;
         milestoneService = new MilestoneServiceImpl();
-        //validatorService = new ValidatorServiceImpl();
         validatorManagerService = new ValidatorManagerServiceImpl();
         latestMilestoneTracker = new MilestoneTrackerImpl();
         candidateTracker = new CandidateTrackerImpl();
         latestSolidMilestoneTracker = new LatestSolidMilestoneTrackerImpl();
         seenMilestonesRetriever = new SeenMilestonesRetrieverImpl();
         milestoneSolidifier = new MilestoneSolidifierImpl();
-        //validatorSolidifier = new ValidatorSolidifierImpl();
         candidateSolidifier = new CandidateSolidifierImpl();
         transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled()
                 ? new AsyncTransactionPruner()
@@ -219,7 +217,6 @@ public void init() throws Exception {
         }
     }
 
-    //TODO: bundleValidator should be passed to: milestoneService, spentAddressService and ledgerService.
     private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException {
         //snapshot provider must be initialized first
         //because we check whether spent addresses data exists
@@ -238,7 +235,6 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx
                 latestMilestoneTracker);
         seenMilestonesRetriever.init(tangle, snapshotProvider, transactionRequester);
         milestoneSolidifier.init(snapshotProvider, transactionValidator);
-        //validatorSolidifier.init(snapshotProvider, transactionValidator);
         candidateSolidifier.init(snapshotProvider, transactionValidator);
         ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, configuration);
         if (transactionPruner != null) {

src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java

@@ -1040,7 +1040,7 @@ public interface Defaults {
 
         //Snapshot
         boolean LOCAL_SNAPSHOTS_ENABLED = true;
-        boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = true;
+        boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = false;
         int LOCAL_SNAPSHOTS_PRUNING_DELAY = 50000;
         int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10;
         int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 1000;

src/main/java/net/helix/pendulum/controllers/RoundViewModel.java

@@ -269,9 +269,6 @@ public static Set<Hash> getMilestoneTrunk(Tangle tangle, TransactionViewModel tr
             // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash
             trunk.add(transaction.getTrunkTransactionHash());
         }
-        if (log.isTraceEnabled()) {
-            log.trace("trunk: {}", PendulumUtils.logHashList(trunk, 8));
-        }
         return trunk;
     }
 
@@ -310,9 +307,6 @@ public static Set<Hash> getMilestoneBranch(Tangle tangle, TransactionViewModel t
                 }
             }
         }
-        if (log.isTraceEnabled()) {
-            log.trace("Milestone branch: {}", PendulumUtils.logHashList(branch, 8));
-        }
         return branch;
     }
 

src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java

@@ -145,6 +145,9 @@ public void trackLatestSolidMilestones() throws MilestoneException {
         try {
             int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex();
             RoundViewModel nextRound;
+            if (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) {
+                log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Round = " + milestoneTracker.getCurrentRoundIndex() + ", Still in Round = " + (currentSolidRoundIndex == milestoneTracker.getCurrentRoundIndex() - 1) + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()));
+            }
             while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex())
                     && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()))) {
 
@@ -154,6 +157,7 @@ public void trackLatestSolidMilestones() throws MilestoneException {
                     // round has finished without milestones
                     RoundViewModel latest = RoundViewModel.latest(tangle);
                     if (latest != null && latest.index() > currentSolidRoundIndex + 1 && isRoundSolid(latest)) {
+                        log.delegate().trace("Round #" + (currentSolidRoundIndex + 1) + " has finished without milestones");
                         nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>());
                         nextRound.store(tangle);
                         tracer.trace("created and stored empty round: {}", nextRound.index());
@@ -164,6 +168,7 @@ public void trackLatestSolidMilestones() throws MilestoneException {
                         break;
                     }
                 }
+                log.delegate().trace("Round = " + nextRound.index() + ", solid = " + isRoundSolid(nextRound));
                 if (isRoundSolid(nextRound)) {
                     tracer.trace("solid round: {}", nextRound.toString());
                     // TODO: Ask Oliver about these classes?
@@ -173,6 +178,7 @@ public void trackLatestSolidMilestones() throws MilestoneException {
                     logChange(currentSolidRoundIndex);
                     currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex();
                     tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index());
+                    log.delegate().trace("ctx= " + nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)) + ", round=" +  nextRound.index());
                 }
             }
         } catch (Exception e) {

src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java

@@ -5,8 +5,6 @@
 import net.helix.pendulum.model.Hash;
 import net.helix.pendulum.model.HashFactory;
 import net.helix.pendulum.service.snapshot.*;
-import net.helix.pendulum.service.spentaddresses.SpentAddressesException;
-import net.helix.pendulum.service.spentaddresses.SpentAddressesProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,7 +99,7 @@ public class SnapshotProviderImpl implements SnapshotProvider {
      * @return the initialized instance itself to allow chaining
      *
      */
-    public SnapshotProviderImpl init(SnapshotConfig config) throws SnapshotException, SpentAddressesException {
+    public SnapshotProviderImpl init(SnapshotConfig config) throws SnapshotException {
         this.config = config;
         File pathToLocalSnapshotDir = Paths.get(this.config.getLocalSnapshotsBasePath()).toFile();
         if (!pathToLocalSnapshotDir.exists() || !pathToLocalSnapshotDir.isDirectory()) {
@@ -201,7 +199,7 @@ public void shutdown() {
      *
      * @throws SnapshotException if anything goes wrong while loading the snapshots
      */
-    private void loadSnapshots() throws SnapshotException, SpentAddressesException {
+    private void loadSnapshots() throws SnapshotException {
         initialSnapshot = loadLocalSnapshot();
         if (initialSnapshot == null) {
             initialSnapshot = loadBuiltInSnapshot();
@@ -220,7 +218,7 @@ private void loadSnapshots() throws SnapshotException, SpentAddressesException {
      * @return local snapshot of the node
      * @throws SnapshotException if local snapshot files exist but are malformed
      */
-      private Snapshot loadLocalSnapshot() throws SnapshotException, SpentAddressesException {
+      private Snapshot loadLocalSnapshot() throws SnapshotException {
           if (config.getLocalSnapshotsEnabled()) {
               String fileSeperator = System.getProperty("file.separator");
               File localSnapshotFile = new File(
@@ -231,8 +229,6 @@ private Snapshot loadLocalSnapshot() throws SnapshotException, SpentAddressesExc
               );
               if (localSnapshotFile.exists() && localSnapshotFile.isFile() && localSnapshotMetaDataFile.exists() &&
                       localSnapshotMetaDataFile.isFile()) {
-                  //TODO: enable this for mainnet-1.0. This will cause issues on a testnet where value transfers might not be guaranteed during a local snapshot und thus would always be thrown.
-                  //assertSpentAddressesDbExist();
                   SnapshotState snapshotState = readSnapshotStatefromFile(localSnapshotFile.getAbsolutePath());
                   if (!snapshotState.hasCorrectSupply()) {
                       throw new SnapshotException("the snapshot state file has an invalid supply");
@@ -245,25 +241,9 @@ private Snapshot loadLocalSnapshot() throws SnapshotException, SpentAddressesExc
                   return new SnapshotImpl(snapshotState, snapshotMetaData);
               }
           }
-
           return null;
       }
 
-    private void assertSpentAddressesDbExist() throws SpentAddressesException {
-        try {
-            File spentAddressFolder = new File(SpentAddressesProvider.SPENT_ADDRESSES_DB);
-            //If there is at least one file in the db the check should pass
-            if (Files.newDirectoryStream(spentAddressFolder.toPath(), "*.sst").iterator().hasNext()) {
-                return;
-            }
-        }
-        catch (IOException e){
-            throw new SpentAddressesException("Can't load " + SpentAddressesProvider.SPENT_ADDRESSES_DB + " folder", e);
-        }
-
-        throw new SpentAddressesException(SpentAddressesProvider.SPENT_ADDRESSES_DB + " folder has no sst files");
-    }
-
     /**
      * Loads the builtin snapshot (last global snapshot) that is embedded in the jar (if a different path is provided it
      * can also load from the disk).

src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java

@@ -517,15 +517,6 @@ private void cleanupOldData(PendulumConfig config, TransactionPruner transaction
     private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot newSnapshot, PendulumConfig config)
             throws SnapshotException {
 
-        try {
-            spentAddressesService.persistSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(),
-                    newSnapshot.getIndex());
-
-        } catch (Exception e) {
-            throw new SnapshotException(e);
-        }
-
-        // TODO: Add timestamp and hash to this snapshot so it can be saved into hashdir/timestamp-filename
         snapshotProvider.writeSnapshotToDisk(newSnapshot, config.getLocalSnapshotsBasePath() );
 
         snapshotProvider.getLatestSnapshot().lockWrite();

src/main/java/net/helix/pendulum/service/spentaddresses/SpentAddressesService.java

@@ -20,15 +20,6 @@ public interface SpentAddressesService {
      */
     boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException;
 
-    /**
-     * Calculates and persists all spent addresses in between a range that were validly signed
-     *
-     * @param fromMilestoneIndex the lower bound milestone index (inclusive)
-     * @param toMilestoneIndex the upper bound milestone index (exclusive)
-     * @throws Exception when anything went wrong whilst calculating.
-     */
-    void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception;
-
     /**
      * Persist all the verifiable spent from a given list of transactions
      * @param transactions transactions to obtain spends from

src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesServiceImpl.java

@@ -2,18 +2,15 @@
 
 import net.helix.pendulum.BundleValidator;
 import net.helix.pendulum.controllers.AddressViewModel;
-import net.helix.pendulum.controllers.RoundViewModel;
 import net.helix.pendulum.controllers.TransactionViewModel;
 import net.helix.pendulum.model.Hash;
 import net.helix.pendulum.service.snapshot.SnapshotProvider;
-import net.helix.pendulum.service.snapshot.impl.SnapshotServiceImpl;
 import net.helix.pendulum.service.spentaddresses.SpentAddressesException;
 import net.helix.pendulum.service.spentaddresses.SpentAddressesProvider;
 import net.helix.pendulum.service.spentaddresses.SpentAddressesService;
 import net.helix.pendulum.service.tipselection.TailFinder;
 import net.helix.pendulum.service.tipselection.impl.TailFinderImpl;
 import net.helix.pendulum.storage.Tangle;
-import net.helix.pendulum.utils.dag.DAGHelper;
 import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -80,42 +77,6 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept
         return false;
     }
 
-    @Override
-    public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws SpentAddressesException {
-        Set<Hash> addressesToCheck = new HashSet<>();
-        try {
-            for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) {
-                RoundViewModel currentMilestone = RoundViewModel.get(tangle, i);
-                if (currentMilestone != null) {
-                    for (Hash confirmedTip : currentMilestone.getConfirmedTips(tangle, 1)) {
-                        DAGHelper.get(tangle).traverseApprovees(
-                                confirmedTip,
-                                transactionViewModel -> transactionViewModel.snapshotIndex() >= currentMilestone.index(),
-                                transactionViewModel -> addressesToCheck.add(transactionViewModel.getAddressHash())
-                        );
-                    }
-                }
-            }
-        } catch (Exception e) {
-            throw new SpentAddressesException(e);
-        }
-
-        //Can only throw runtime exceptions in streams
-        try {
-            spentAddressesProvider.saveAddressesBatch(addressesToCheck.stream()
-                    .filter(ThrowingPredicate.unchecked(this::wasAddressSpentFrom))
-                    .collect(Collectors.toList()));
-            log.trace("spentAddressesService.persistSpentAddresses += 1");
-        } catch (RuntimeException e) {
-            if (e.getCause() instanceof SpentAddressesException) {
-                throw (SpentAddressesException) e.getCause();
-            } else {
-                throw e;
-            }
-        }
-
-    }
-
     @Override
     public void persistSpentAddresses(Collection<TransactionViewModel> transactions) throws SpentAddressesException {
         try {