/*_############################################################################
  _## 
  _##  SNMP4J-Agent-DB 3 - MOXodusPersistence.java  
  _## 
  _##  Copyright (C) 2017-2025 Frank Fock (SNMP4J.org)
  _##  
  _##  Licensed under the Apache License, Version 2.0 (the "License");
  _##  you may not use this file except in compliance with the License.
  _##  You may obtain a copy of the License at
  _##  
  _##      http://www.apache.org/licenses/LICENSE-2.0
  _##  
  _##  Unless required by applicable law or agreed to in writing, software
  _##  distributed under the License is distributed on an "AS IS" BASIS,
  _##  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  _##  See the License for the specific language governing permissions and
  _##  limitations under the License.
  _##  
  _##########################################################################*/
package org.snmp4j.agent.db;

import jetbrains.exodus.ArrayByteIterable;
import jetbrains.exodus.ByteIterable;
import jetbrains.exodus.ByteIterator;
import jetbrains.exodus.bindings.CompressedUnsignedLongArrayByteIterable;
import jetbrains.exodus.env.*;
import org.snmp4j.SNMP4JSettings;
import org.snmp4j.agent.*;
import org.snmp4j.agent.io.ImportMode;
import org.snmp4j.agent.mo.*;
import org.snmp4j.agent.util.MOScopePriorityComparator;
import org.snmp4j.asn1.BER;
import org.snmp4j.asn1.BERInputStream;
import org.snmp4j.asn1.BEROutputStream;
import org.snmp4j.log.LogAdapter;
import org.snmp4j.log.LogFactory;
import org.snmp4j.security.SecretOctetString;
import org.snmp4j.smi.*;

import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.util.*;

/**
 * The {@link MOXodusPersistence} class provides persistent storage for SNMP4J-Agent using the
 * {@link MOXodusPersistenceProvider} wrapper that actually implements the
 * {@link org.snmp4j.agent.io.MOPersistenceProvider} interface of SNMP4J-Agent.
 * As a storage engine, the Xodus open source (Apache 2 License) DB is used.
 * See <a href="https://github.com/JetBrains/xodus">https://github.com/JetBrains/xodus</a>} for details.
 * The database approach has the following advantages compared to the standard sequential persistence provider coming
 * with SNMP4J-Agent:
 * <ul>
 *     <li>Only changed objects are written again to disk. The default DefaultMOPersistenceProvider needs to save all
 *     objects in a sequence.</li>
 *     <li>DB size is smaller - if changes are limited to approximately less than 40% of the MIB objects during
 *     runtime.</li>
 *     <li>Agent shutdown is much faster because no objects need to be saved anymore.</li>
 *     <li>No data loss if agent is killed.</li>
 * </ul>
 * The following sample code from {@link org.snmp4j.agent.db.sample.SampleAgent} illustrates how this class is created
 * and assigned to the agent during its initialization.
 *
 * <pre>
 *     File configFile = new File(myConfigDir);
 *     MOXodusPersistence moXodusPersistence = new MOXodusPersistence(moServers, Environments.newInstance(configFile));
 *     MOXodusPersistenceProvider moXodusPersistenceProvider = new MOXodusPersistenceProvider(moXodusPersistence);
 *     OctetString defaultEngineID = new OctetString(MPv3.createLocalEngineID());
 *     OctetString engineID = moXodusPersistenceProvider.getEngineId(defaultEngineID);
 *     ...
 *     agent = new AgentConfigManager(engineID, messageDispatcher, null, moServers, ThreadPool.create("SampleAgent", 3),
 *                                    (defaultEngineID == engineID) ? configurationFactory : null,
 *                                    moXodusPersistenceProvider,
 *                                    new EngineBootsCounterFile(bootCounterFile), null, dhKickstartParameters);
 *     agent.addAgentStateListener(new AgentStateListener() {
 *         public void agentStateChanged(AgentConfigManager agentConfigManager, AgentState newState) {
 *             switch (newState.getState()) {
 *                 case AgentState.STATE_INITIALIZED:
 *                     moXodusPersistence.registerChangeListenersWithServer(server);
 *                     break;
 *                 case AgentState.STATE_SHUTDOWN:
 *                     moXodusPersistence.unregisterChangeListenersWithServer(server);
 *                     break;
 *             }
 *         }
 *     });
 *
 * </pre>
 *
 * @author Frank Fock
 * @version 3.5.0
 * @since 3.0
 */
public class MOXodusPersistence implements MOChangeListener {

    private static final LogAdapter logger = LogFactory.getLogger(MOXodusPersistence.class);

    /**
     * The default encryption algorithm identifier used by {@link MOXodusPersistence} for protecting
     * {@link SecretOctetString} values stored to disk. The format is:
     * <pre>
     *   &lt;KDF&gt;:&lt;iterations&gt;:&lt;cipherTransformation&gt;:&lt;keyBits&gt;
     * </pre>
     * The default uses PBKDF2 with HMAC-SHA-256 (600.000 iterations as recommended by OWASP) for key
     * derivation and AES in GCM mode with a 256-bit key for authenticated encryption.
     * @since 3.8.0
     */
    public static final String DEFAULT_ENCRYPTION_ALGORITHM =
            "PBKDF2WithHmacSHA256:600000:AES/GCM/NoPadding:256";

    private static final int GCM_IV_BYTES = 12;
    private static final int GCM_TAG_BITS = 128;
    private static final int CBC_IV_BYTES = 16;
    private static final int AES_BLOCK_BYTES = 16;
    private static final int SALT_BYTES = 16;

    private static final SecureRandom SECURE_RANDOM = new SecureRandom();

    /**
     * The {@link SavingStrategy} defines how and when persistent data is saved.
     */
    public enum SavingStrategy {
        /**
         * The default strategy saves changed data only if a corresponding {@link MOChangeEvent} has been received.
         */
        onChangeEventsOnly,
        /**
         * Save only when the {@link #save()} method is being called and the persistent data do no longer match the
         * data in the agent.
         */
        checkForModificationsOnSave,
        /**
         * Always save everything when {@link #save()} is being called without checking for modifications.
         */
        fullDumpOnSave
    }

    private final MOServer[] moServers;
    private final Environment environment;
    private boolean ignoreChangeListenerEvents;
    private int continuousChangeListening = 0;
    private SavingStrategy savingStrategy = SavingStrategy.onChangeEventsOnly;
    private MOScopeComparator moScopeComparator = new MOScopeComparator();

    /**
     * List of {@link ManagedObject} OIDs that will be kept persistent even if their
     * {@link RandomAccessManagedObject#isVolatile()} returns {@code true}.
     */
    private Map<OID, Boolean> persistenceExceptionList;

    /**
     * Master password used to derive the symmetric key that protects {@link SecretOctetString} values when
     * they are written to the persistent store. If {@code null}, encryption is disabled and
     * {@link SecretOctetString} values are stored unencrypted (the historical behaviour).
     */
    private char[] masterPassword;

    /**
     * Identifier of the encryption algorithm used for newly written {@link SecretOctetString} values.
     * Stored inside the BER sequence of each encrypted secret, so the algorithm can be changed without
     * losing the ability to decrypt previously stored data. Default: {@link #DEFAULT_ENCRYPTION_ALGORITHM}.
     */
    private String defaultEncryptionAlgorithm = DEFAULT_ENCRYPTION_ALGORITHM;

    /**
     * When {@code true}, encrypted secrets encountered during a load with no master password set are
     * returned as opaque {@link SecretOctetString} values containing the raw BER octet stream of the
     * encrypted sequence. When {@code false} (the default), encountering such data triggers a fatal log
     * and a {@link RuntimeException}, aborting the load.
     */
    private boolean ignoreEncryptedDataWithoutPassword = false;

    /**
     * Creates a new {@link MOXodusPersistence} from an array of {@link MOServer} instances and an Xodus
     * {@link Environment}. The data of modified objects are stored whenever a corresponding {@link MOChangeEvent} is
     * received.
     * @param moServers
     *    the {@link ManagedObject} servers of the agent to be supported with persistent storage capabilities by this
     *    object.
     * @param environment
     *    the Xodus environment that actually holds the persistent data.
     */
    public MOXodusPersistence(MOServer[] moServers, Environment environment) {
        this(moServers, environment, SavingStrategy.onChangeEventsOnly);
    }

    /**
     * Creates a new {@link MOXodusPersistence} from an array of {@link MOServer} instances and an Xodus
     * {@link Environment}.
     * @param moServers
     *    the {@link ManagedObject} servers of the agent to be supported with persistent storage capabilities by this
     *    object.
     * @param environment
     *    the Xodus environment that actually holds the persistent data.
     * @param savingStrategy
     *    defines when and how modified objects of the agent should be saved into persistent storage.
     */
    public MOXodusPersistence(MOServer[] moServers, Environment environment, SavingStrategy savingStrategy) {
        this.moServers = moServers;
        this.environment = environment;
        this.savingStrategy = savingStrategy;
    }

    /**
     * Creates a new {@link MOXodusPersistence} from an array of {@link MOServer} instances and an Xodus
     * {@link Environment}.
     * @param moServers
     *    the {@link ManagedObject} servers of the agent to be supported with persistent storage capabilities by this
     *    object.
     * @param environment
     *    the Xodus environment that actually holds the persistent data.
     * @param savingStrategy
     *    defines when and how modified objects of the agent should be saved into persistent storage.
     * @param moScopeComparator
     *    defines the order in which {@link ManagedObject}s are stored and restored from persistent storage.
     *    If {@code null}, the {@link MOScopeComparator} is used that sorts the registered managed objects
     *    by their lower bound OID in ascending order.
     * @since 3.5.0
     */
    public MOXodusPersistence(MOServer[] moServers, Environment environment, SavingStrategy savingStrategy,
                              MOScopeComparator moScopeComparator) {
        this(moServers, environment, savingStrategy);
        if (moScopeComparator != null) this.moScopeComparator = moScopeComparator;
    }

    /**
     * Creates a new {@link MOXodusPersistence} from an array of {@link MOServer} instances, an Xodus
     * {@link Environment}, and a master password used to encrypt every {@link SecretOctetString}
     * value before it is written to persistent storage. The agent's data is restored using the same
     * master password to decrypt those values again when {@link #load(ImportMode)} (or one of its
     * overloads) is invoked.
     * <p>
     * Existing constructors of this class create instances <em>without</em> encryption; passing a non-null
     * master password is the only way to activate encryption. The default encryption algorithm is
     * {@link #DEFAULT_ENCRYPTION_ALGORITHM} and can be changed via
     * {@link #setDefaultEncryptionAlgorithm(String)} for newly written data. Already stored secrets carry
     * their own algorithm identifier and can be decrypted independently of the current default.
     *
     * @param moServers
     *    the {@link ManagedObject} servers of the agent to be supported with persistent storage capabilities.
     * @param environment
     *    the Xodus environment that actually holds the persistent data.
     * @param masterPassword
     *    the master password used to derive the symmetric key that protects {@link SecretOctetString}
     *    values. The array is defensively copied. Pass {@code null} to disable encryption (in which case
     *    you should use one of the password-less constructors instead).
     * @since 3.8.0
     */
    public MOXodusPersistence(MOServer[] moServers, Environment environment, char[] masterPassword) {
        this(moServers, environment, SavingStrategy.onChangeEventsOnly);
        this.masterPassword = (masterPassword != null) ? masterPassword.clone() : null;
    }

    /**
     * Creates a new {@link MOXodusPersistence} with explicit {@link SavingStrategy}, comparator and
     * master password. See {@link #MOXodusPersistence(MOServer[], Environment, char[])} for the
     * encryption semantics.
     *
     * @param moServers
     *    the {@link ManagedObject} servers of the agent to be supported with persistent storage capabilities.
     * @param environment
     *    the Xodus environment that actually holds the persistent data.
     * @param savingStrategy
     *    defines when and how modified objects of the agent should be saved into persistent storage.
     * @param moScopeComparator
     *    defines the order in which {@link ManagedObject}s are stored and restored from persistent storage.
     *    If {@code null}, a default {@link MOScopeComparator} is used.
     * @param masterPassword
     *    the master password used to derive the symmetric key that protects {@link SecretOctetString}
     *    values. The array is defensively copied. Pass {@code null} to disable encryption.
     * @since 3.8.0
     */
    public MOXodusPersistence(MOServer[] moServers, Environment environment, SavingStrategy savingStrategy,
                              MOScopeComparator moScopeComparator, char[] masterPassword) {
        this(moServers, environment, savingStrategy, moScopeComparator);
        this.masterPassword = (masterPassword != null) ? masterPassword.clone() : null;
    }

    /**
     * Gets the saving strategy as defined by {@link SavingStrategy}.
     * @return
     *    the current managed object saving strategy.
     */
    public SavingStrategy getSavingStrategy() {
        return savingStrategy;
    }

    /**
     * Sets the saving strategy for this persistence provider as defined by {@link SavingStrategy}.
     * @param savingStrategy
     *    the new managed object saving strategy.
     */
    public void setSavingStrategy(SavingStrategy savingStrategy) {
        this.savingStrategy = savingStrategy;
    }

    /**
     * Check whether change listener events are actually suppressed (i.e. ignored) or not.
     * @return
     *    {@code true} if change listener events will be ignored.
     */
    public boolean isIgnoreChangeListenerEvents() {
        return ignoreChangeListenerEvents;
    }

    /**
     * Defines whether {@link MOChangeEvent}s should be ignored or not. This method can be used to disable persistent
     * storage activities when the default strategy {@link SavingStrategy#onChangeEventsOnly} is active and other bulk
     * operations change MIB data in the agent. When activating the processing of {@link MOChangeEvent}s is activated
     * again by setting this value to {@code false}, the missed events will not be processed again. Thus, if data has
     * changed that need to be persistent, the {@link #save()} has to be called with strategy
     * {@link SavingStrategy#checkForModificationsOnSave} or {@link SavingStrategy#fullDumpOnSave} manually.
     *
     * @param ignoreChangeListenerEvents
     *    {@code true} to disable event processing and saving changes triggered by {@link MOChangeEvent}s.
     */
    public void setIgnoreChangeListenerEvents(boolean ignoreChangeListenerEvents) {
        this.ignoreChangeListenerEvents = ignoreChangeListenerEvents;
    }

    /**
     * Returns {@code true} if a master password is configured and {@link SecretOctetString} values will
     * therefore be encrypted on save and decrypted on load.
     * @return
     *    whether encryption of {@link SecretOctetString} values is active.
     * @since 3.8.0
     */
    public boolean isEncryptionEnabled() {
        return masterPassword != null;
    }

    /**
     * Returns the encryption algorithm identifier used to protect {@link SecretOctetString} values that
     * are <em>newly</em> written to persistent storage. Values that have already been stored carry their
     * own algorithm identifier and remain decryptable independent of this setting.
     * @return
     *    an algorithm identifier of the form {@code <KDF>:<iterations>:<cipherTransformation>:<keyBits>},
     *    by default {@link #DEFAULT_ENCRYPTION_ALGORITHM}.
     * @since 3.8.0
     */
    public String getDefaultEncryptionAlgorithm() {
        return defaultEncryptionAlgorithm;
    }

    /**
     * Sets the encryption algorithm identifier used to protect {@link SecretOctetString} values that are
     * subsequently written to persistent storage. The value must be parseable by
     * {@link #parseEncryptionAlgorithm(String)}.
     * @param defaultEncryptionAlgorithm
     *    a non-null algorithm identifier of the form
     *    {@code <KDF>:<iterations>:<cipherTransformation>:<keyBits>}.
     * @since 3.8.0
     */
    public void setDefaultEncryptionAlgorithm(String defaultEncryptionAlgorithm) {
        if (defaultEncryptionAlgorithm == null) {
            throw new NullPointerException("defaultEncryptionAlgorithm must not be null");
        }
        // validate eagerly so misconfigurations surface before the first save
        parseEncryptionAlgorithm(defaultEncryptionAlgorithm);
        this.defaultEncryptionAlgorithm = defaultEncryptionAlgorithm;
    }

    /**
     * Returns the flag controlling how encrypted secrets are handled when loading data without a master
     * password set. See {@link #setIgnoreEncryptedDataWithoutPassword(boolean)} for details.
     * @return
     *    {@code true} if encrypted secrets are passed through as opaque {@link SecretOctetString}s when
     *    no master password is set; {@code false} (the default) if loading aborts in that situation.
     * @since 3.8.0
     */
    public boolean isIgnoreEncryptedDataWithoutPassword() {
        return ignoreEncryptedDataWithoutPassword;
    }

    /**
     * Controls how the persistence loader reacts when it encounters an encrypted {@link SecretOctetString}
     * but no master password has been configured. If {@code true}, the raw BER encoded sequence
     * representing the encrypted value is wrapped into a {@link SecretOctetString} and returned to the
     * agent as-is &mdash; allowing data to be re-saved untouched when the password is unknown. If
     * {@code false} (the default), the loader logs a fatal error and aborts with a {@link RuntimeException}.
     * @param ignoreEncryptedDataWithoutPassword
     *    {@code true} to suppress the fatal error and return the encrypted bytes as a {@link SecretOctetString}.
     * @since 3.8.0
     */
    public void setIgnoreEncryptedDataWithoutPassword(boolean ignoreEncryptedDataWithoutPassword) {
        this.ignoreEncryptedDataWithoutPassword = ignoreEncryptedDataWithoutPassword;
    }

    /**
     * Gets the Xodus {@link Environment} used by this persistent probvider.
     * @return
     *    a {@link Environment}.
     */
    public Environment getEnvironment() {
        return environment;
    }

    /**
     * Return {@code true} if the {@link #registerChangeListenersWithServer(MOServer)} for any server, but
     * {@link #unregisterChangeListenersWithServer(MOServer)} has not been called yet.
     * @return
     *    {@code true} if the {@link #registerChangeListenersWithServer(MOServer)} for any server.
     */
    public boolean isContinuousChangeListening() {
        return continuousChangeListening > 1;
    }

    /**
     * Register this object as {@link MOChangeListener} on all {@link RandomAccessManagedObject} instances in the
     * provided {@link MOServer}.
     *
     * @param moServer
     *   a {@link MOServer} holding {@link RandomAccessManagedObject}s that should be persisted.
     *
     */
    public synchronized void registerChangeListenersWithServer(MOServer moServer) {
        DefaultMOServer.registerChangeListener(moServer, this, mo -> mo instanceof RandomAccessManagedObject);
        continuousChangeListening = 1;
    }

    /**
     * Removes a former registration of this object as {@link MOChangeListener} on all {@link RandomAccessManagedObject}
     * instances in the provided {@link MOServer}.
     *
     * @param moServer
     *   a {@link MOServer} holding {@link RandomAccessManagedObject}s that should not be persisted anymore.
     *
     */
    public synchronized void unregisterChangeListenersWithServer(MOServer moServer) {
        DefaultMOServer.unregisterChangeListener(moServer, this, mo -> mo instanceof RandomAccessManagedObject);
        continuousChangeListening = 0;
    }

    /**
     * Checks if there is already MIB data stored for the specified context.
     * To check the default context ({@code null}), please use the empty {@link OctetString}. This method should be
     * called before calling {@link #load(ImportMode)} because afterwards it will return {@code true} for all contexts
     * that were present in {@link #getMOServer()} and for the default context (empty context).
     * @param context
     *    a non-null context string. The empty (zero length) {@link OctetString} represents the default context.
     * @return
     *    {@code true} if there has been data stored for this context - even if no {@link RandomAccessManagedObject}
     *    actually has stored any data.
     * @since 3.0.1
     */
    public synchronized boolean isContextLoadable(OctetString context) {
        final Transaction txn = environment.beginReadonlyTransaction();
        try {
            return environment.storeExists(storeNameFromContext(context), txn);
        } finally {
            txn.abort();
        }
    }

    public void load(ImportMode importMode) {
        load(importMode, null);
    }

    /**
     * Load the contents of all {@link RandomAccessManagedObject}s using
     * {@link RandomAccessManagedObject#importInstance(OID, List, ImportMode)} calls. The provided {@link ImportMode}
     * thereby defines how the data handles existing data.
     * Data is loaded for all contexts and managed objects found in the {link MOServer} instances provided during
     * object creation.
     * While loading, the member {@link #ignoreChangeListenerEvents} is set to {@code true} to ignore updates caused
     * by loading data into the {@link RandomAccessManagedObject} instances.
     * Use {@link #setMoScopeComparator(MOScopeComparator)} to modify the order in which {@link ManagedObject}s
     * are loaded.
     * Since version 3.5.0, objects are loaded in the order defined by the {@link MOPriorityProvider} given in order to
     * facilitate loading and especially initialization of {@link ManagedObject}s that augment or extend base tables
     * or depend on each other. For each context, a load order defined by the content of single boot
     * {@link ManagedObject} can be defined. In most cases, a single order defined in the default ({@code null}) context
     * is sufficient.
     *
     * @param importMode
     *    controls how existing data is used or not used during import.
     * @param priorityProvider
     *    if not {@code null}, it provides the order for loading and initializing managed objects by
     *    {@link MOPriorityProvider#getPriorityMap(OctetString)}.
     */
    public synchronized void load(ImportMode importMode, MOPriorityProvider priorityProvider) {
        load(importMode, priorityProvider, false);
    }

    /**
     * Load the contents of all {@link RandomAccessManagedObject}s using
     * {@link RandomAccessManagedObject#importInstance(OID, List, ImportMode)} calls. The provided {@link ImportMode}
     * thereby defines how the data handles existing data.
     * Data is loaded for all contexts and managed objects found in the {link MOServer} instances provided during
     * object creation.
     * While loading, the member {@link #ignoreChangeListenerEvents} is set to {@code true} to ignore updates caused
     * by loading data into the {@link RandomAccessManagedObject} instances.
     * Use {@link #setMoScopeComparator(MOScopeComparator)} to modify the order in which {@link ManagedObject}s
     * are loaded.
     * Since version 3.5.0, objects are loaded in the order defined by the {@link MOPriorityProvider} given in order to
     * facilitate loading and especially initialization of {@link ManagedObject}s that augment or extend base tables
     * or depend on each other. For each context, a load order defined by the content of single boot
     * {@link ManagedObject} can be defined. In most cases, a single order defined in the default ({@code null}) context
     * is sufficient.
     *
     * @param importMode
     *    controls how existing data is used or not used during import.
     * @param priorityProvider
     *    if not {@code null}, it provides the order for loading and initializing managed objects by
     *    {@link MOPriorityProvider#getPriorityMap(OctetString)}.
     * @param includeVolatile
     *    if {@code true}, {@link RandomAccessManagedObject}s with {@link RandomAccessManagedObject#isVolatile()}
     *    {@code true} will be saved too, otherwise those objects will not be saved.
     * @since 3.6.0
     */
    public synchronized void load(ImportMode importMode, MOPriorityProvider priorityProvider, boolean includeVolatile) {
        try {
            MOScopeComparator moScopeComparator = this.moScopeComparator;
            setIgnoreChangeListenerEvents(true);
            continuousChangeListening = (continuousChangeListening > 0) ? continuousChangeListening = 2 : 0;
            final Transaction txn = environment.beginReadonlyTransaction();
            for (MOServer moServer : moServers) {
                Map<OctetString, Store> stores = new HashMap<>();
                ManagedObject<?> bootMO = runSyncOnBootMO(txn, moServer, stores, importMode, priorityProvider, null);
                for (OctetString context : moServer.getContexts()) {
                    try {
                        if (!stores.containsKey(context)) {
                            stores.put(context, createStore(txn, context));
                        }
                        MOScopeComparator moScopeComparatorCtx =
                                getMoScopeComparatorForContext(importMode, priorityProvider, moScopeComparator,
                                        txn, moServer, stores, bootMO, context);
                        Iterator<Map.Entry<MOScope, ManagedObject<?>>> moIterator =
                                moServer.iterator(moScopeComparatorCtx, null);
                        runSynchronization(stores, txn, importMode, moIterator, includeVolatile);
                    } catch (ReadonlyTransactionException rotex) {
                        logger.info("No persistent data for context '" + context + "' context found");
                    }
                }
                if (moServer.isContextSupported(null)) {
                    try {
                        // Store already put above in runSyncOnBootMO:
                        //stores.put(new OctetString(), environment.openStore("", StoreConfig.WITHOUT_DUPLICATES, txn));
                        MOScopeComparator moScopeComparatorCtx =
                                getMoScopeComparatorForContext(importMode, priorityProvider, moScopeComparator,
                                        txn, moServer, stores, bootMO, null);
                        Iterator<Map.Entry<MOScope, ManagedObject<?>>> moIterator =
                                moServer.iterator(moScopeComparatorCtx, mo -> mo != bootMO);
                        runSynchronization(stores, txn, importMode, moIterator, includeVolatile);
                    } catch (ReadonlyTransactionException rotex) {
                        logger.info("No persistent data for context default context found");
                    }
                }
            }
            txn.abort();
        }
        finally {
            setIgnoreChangeListenerEvents(false);
        }
    }

    private MOScopeComparator getMoScopeComparatorForContext(ImportMode importMode, MOPriorityProvider priorityProvider,
                                                             MOScopeComparator moScopeComparator, Transaction txn,
                                                             MOServer moServer, Map<OctetString, Store> stores,
                                                             ManagedObject<?> bootMO, OctetString context) {
        if (bootMO != null) {
            ManagedObject<?> contextBootMO = priorityProvider.getBootManagedObject(context);
            if (contextBootMO != bootMO) {
                runSyncOnBootMO(txn, moServer, stores, importMode, priorityProvider, context);
            }
            SortedMap<OID, Integer> priorityMap = priorityProvider.getPriorityMap(context);
            moScopeComparator = new MOScopePriorityComparator(priorityMap);
            if (logger.isDebugEnabled()) {
                logger.debug("Using priority map "+priorityMap+" for context "+context);
            }
        }
        return moScopeComparator;
    }

    /**
     * Run synchronisation on the boot {@link ManagedObject} (i.e. the {@link ManagedObject} restored before any
     * other.
     * @param txn
     *    the {@link Transaction}.
     * @param moServer
     *    the {@link MOServer}.
     * @param stores
     *    the stores containing the persistent data.
     * @param importMode
     *    the {@link ImportMode} to be used.
     * @param priorityProvider
     *    the {@link MOPriorityProvider} that defines the restore priorities.
     * @param context
     *    the context to restore
     * @return
     *    the boot {@link ManagedObject}.
     */
    protected ManagedObject<?> runSyncOnBootMO(Transaction txn, MOServer moServer, Map<OctetString, Store> stores,
                                               ImportMode importMode, MOPriorityProvider priorityProvider,
                                               OctetString context) {
        MOScope scope = null;
        ManagedObject<?> bootMO = null;
        if (priorityProvider != null) {
            bootMO = priorityProvider.getBootManagedObject(context);
            scope = moServer.getRegisteredScopes(bootMO).get(context);
        }
        boolean contextSupported = moServer.isContextSupported(context);
        if (contextSupported) {
            try {
                stores.put((context == null) ? new OctetString() : context, createStore(txn, context));
                if (bootMO != null) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Boot MO for context '" + context + "' is '" + bootMO + "', loading it.");
                    }
                    try {
                        Map<MOScope, ManagedObject<?>> bootTask = new HashMap<>();
                        bootTask.put(scope, bootMO);
                        runSynchronization(stores, txn, importMode, bootTask.entrySet().iterator(), true);
                        if (logger.isDebugEnabled()) {
                            logger.debug("Boot MO loaded '" + bootMO + "' for context '" + context + "', order is: " +
                                    priorityProvider.getPriorityMap(context));
                        }
                    } catch (ReadonlyTransactionException rotex) {
                        logger.info("No persistent data for context " + ((context == null) ? "<default>" : context) + " context found");
                    }
                }
            } catch (ReadonlyTransactionException rotex) {
                logger.warn("No persistent data for context " + ((context == null) ? "<default>" : context) + " context found/accessible");
            }
        }
        return bootMO;
    }

    /**
     * Ceates a {@link Store} for a context.
     * @param txn
     *    the {@link Transaction}.
     * @param context
     *    the context.
     * @return
     *    a new opened {@link Store};
     */
    protected Store createStore(Transaction txn, OctetString context) {
        return environment.openStore(storeNameFromContext(context), StoreConfig.WITHOUT_DUPLICATES, txn);
    }

    /**
     * Return a string store name for the provided SNMPv3 context.
     * @param context
     *    a context name or {@code null} for the default context.
     * @return
     *    a store name, by default {@code context == null ? "" : context.toHexString()}.
     * @since 3.0.1
     */
    protected String storeNameFromContext(OctetString context) {
        return context == null ? "" : context.toHexString();
    }

    /**
     * Get the associated {@link MOServer}s.
     * @return
     *    an array of {@link MOServer} instances.
     */
    public MOServer[] getMOServer() {
        return moServers;
    }

    /**
     * Gets the {@link MOScopeComparator} that defines the default order in which {@link ManagedObject}s are stored
     * and restored from persistent storage (unless the parameter {@link MOPriorityProvider} is used).
     * @return
     *    a {@link MOScopeComparator} instance.
     */
    public synchronized MOScopeComparator getMoScopeComparator() {
        return moScopeComparator;
    }

    /**
     * Sets the {@link MOScopeComparator} and by this defines the default order in which {@link ManagedObject}s are
     * stored and restored from persistent storage as long as parameter {@link MOPriorityProvider} of
     * {@link #save(MOPriorityProvider)} and {@link #load(ImportMode, MOPriorityProvider)} are not used.
     * If set to {@code null}, the {@link MOScopeComparator} is used that sorts the registered managed objects
     * by their lower bound OID in ascending order.
     * @param moScopeComparator
     *    defines the order in which {@link ManagedObject}s are stored and restored from persistent storage.
     *    If {@code null}, the {@link MOScopeComparator} is used that sorts the registered managed objects
     *    by their lower bound OID in ascending order.
     */
    public synchronized void setMoScopeComparator(MOScopeComparator moScopeComparator) {
        this.moScopeComparator = (moScopeComparator == null) ? new MOScopeComparator(): moScopeComparator;
    }

    /**
     * Run synchronisation between in memory data and on disk data.
     * @param stores
     *    the context stores.
     * @param txn
     *    the {@link Transaction}.
     * @param importMode
     *    the import mode to use.
     * @param moIterator
     *    the object iterator over the {@link ManagedObject}s to be synchronized.
     * @param includeVolatile
     *    if {@code true}, {@link RandomAccessManagedObject}s with {@link RandomAccessManagedObject#isVolatile()}
     *    {@code true} will be saved too, otherwise those objects will not be saved.
     */
    protected void runSynchronization(Map<OctetString, Store> stores, Transaction txn, ImportMode importMode,
                                      Iterator<Map.Entry<MOScope, ManagedObject<?>>> moIterator,
                                      boolean includeVolatile) {
        // import mode == null -> export
        while (moIterator.hasNext()) {
            Map.Entry<MOScope, ManagedObject<?>> entry = moIterator.next();
            if (entry.getValue() instanceof MOScalar) {
                logger.debug("MOScalar " + entry.getValue());
            }
            if (entry.getValue() instanceof RandomAccessManagedObject) {
                RandomAccessManagedObject<?> randomAccessManagedObject = (RandomAccessManagedObject<?>) entry.getValue();
                if (!isPersistent(randomAccessManagedObject, randomAccessManagedObject.getScope().getLowerBound(),
                        null, includeVolatile)) {
                    continue;
                }
                OctetString context = new OctetString();
                if (entry.getKey() instanceof MOContextScope) {
                    context = ((MOContextScope) entry.getKey()).getContext();
                }
                Store store = stores.get(context);
                if (store == null) {
                    logger.info("DB store for context '"+context+"' not found, creating it.");
                    store = createStore(txn, context);
                }
                Cursor cursor = store.openCursor(txn);
                OID oid = entry.getKey().getLowerBound();
                HashSet<OID> exported = null;
                final ByteIterable objectKey = cursor.getSearchKeyRange(getKey(oid, new OID()));
                if (objectKey != null) {
                    OID instanceOID;
                    while ((instanceOID = getKeyOid(cursor.getKey())).startsWith(oid)) {
                        OID index = instanceOID.getSuffix(oid);
                        ByteIterable rawValues = cursor.getValue();
                        List<VariableBinding> rawVBS = decodeInstanceData(rawValues);
                        if (rawVBS != null && !rawVBS.isEmpty()) {
                            if (importMode == null) {
                                if (exported == null) {
                                    exported = new HashSet<>(randomAccessManagedObject.instanceCount());
                                }
                                List<VariableBinding> exportVbs = null;
                                if (!randomAccessManagedObject.isVolatile(index)) {
                                    exportVbs = randomAccessManagedObject.exportInstance(index);
                                }
                                if (exportVbs == null) {
                                    cursor.deleteCurrent();
                                } else {
                                    if (savingStrategy == SavingStrategy.checkForModificationsOnSave) {
                                        byte[] exportRawValues = encodeInstanceData(exportVbs);
                                        ArrayByteIterable exportByteIterable = new ArrayByteIterable(exportRawValues);
                                        if (rawValues.compareTo(exportByteIterable) != 0) {
                                            ByteIterable key = getKey(oid, index);
                                            if (logger.isDebugEnabled()) {
                                                logger.debug("Saving modified " + context + ":" + oid + "|" + index +
                                                        " ("+key+") = " +
                                                        (SNMP4JSettings.isSecretLoggingEnabled() ?
                                                            new OctetString(exportRawValues).toHexString() +
                                                            " <- " + exportVbs : exportVbs.size()));
                                            }
                                            store.put(txn, key, new ArrayByteIterable(rawValues));
                                            exported.add(index);
                                        }
                                    }
                                    else {
                                        putInstanceData(txn, context, store, oid, index, exportVbs);
                                        exported.add(index);
                                    }
                                }
                            } else {
                                if (logger.isDebugEnabled()) {
                                    logger.debug("Loading data for " + oid + " with index " + index +
                                            " ("+cursor.getKey()+"): " +
                                            (SNMP4JSettings.isSecretLoggingEnabled() ? rawVBS : rawVBS.size()));
                                }
                                randomAccessManagedObject.importInstance(index, rawVBS, importMode);
                            }
                        } else {
                            // unknown & unsupported entity
                            if (logger.isWarnEnabled()) {
                                logger.warn("Unable to load persistent data: " +
                                        new OctetString(rawValues.getBytesUnsafe()).toHexString());
                            }
                            break;
                        }
                        if (!cursor.getNext()) {
                            break;
                        }
                    }
                }
                // export new instances
                if (importMode == null) {
                    // export mode
                    for (Iterator<OID> oidIterator = randomAccessManagedObject.instanceIterator();
                         oidIterator.hasNext(); ) {
                        OID nextInstanceOID = oidIterator.next();
                        if (exported == null || !exported.contains(nextInstanceOID)) {
                            List<VariableBinding> vbs = null;
                            if (!randomAccessManagedObject.isVolatile(nextInstanceOID)) {
                                vbs = randomAccessManagedObject.exportInstance(nextInstanceOID);
                            }
                            if (vbs != null) {
                                putInstanceData(txn, context, store, oid, nextInstanceOID, vbs);
                            }
                        }
                    }
                }
                cursor.close();
            }
        }
    }

    private void putInstanceData(Transaction txn, OctetString context, Store store, OID oid,
                                 OID index, List<VariableBinding> vbs) {
        byte[] rawValues = encodeInstanceData(vbs);
        ByteIterable key = getKey(oid, index);
        if (logger.isDebugEnabled()) {
            logger.debug("Saving " + context + ":" + oid + "|" + index + " ("+key+") = " +
                    (SNMP4JSettings.isSecretLoggingEnabled() ?
                        new OctetString(rawValues).toHexString() + " <- " + vbs : vbs.size()));
        }
        store.put(txn, key, new ArrayByteIterable(rawValues));
    }

    /**
     * Decode BER instance data from raw byte stream.
     * @param rawData
     *    the BER encoded data.
     * @return
     *    the decoded list of {@link VariableBinding}s.
     */
    protected List<VariableBinding> decodeInstanceData(ByteIterable rawData) {
        BERInputStream inputStream =
                new BERInputStream(ByteBuffer.wrap(rawData.getBytesUnsafe(), 0, rawData.getLength()));
        try {
            BER.MutableByte pduType;
            pduType = new BER.MutableByte();
            int vbLength = BER.decodeHeader(inputStream, pduType);
            if (pduType.getValue() != BER.SEQUENCE) {
                throw new IOException("Encountered invalid tag, SEQUENCE expected: " +
                        pduType.getValue());
            }
            // rest read count
            int startPos = (int) inputStream.getPosition();
            ArrayList<VariableBinding> variableBindings = new ArrayList<>();
            while (inputStream.getPosition() - startPos < vbLength) {
                VariableBinding vb = decodeVariableBinding(inputStream);
                variableBindings.add(vb);
            }
            if (inputStream.getPosition() - startPos != vbLength) {
                throw new IOException("Length of VB sequence (" + vbLength +
                        ") does not match real length: " +
                        ((int) inputStream.getPosition() - startPos));
            }
            return variableBindings;
        } catch (IOException e) {
            logger.error(e);
        }
        return null;
    }

    /**
     * Encode instance data from a list of {@link VariableBinding}s.
     * @param vbs
     *    a list of {@link VariableBinding}s.
     * @return
     *    a BER encoded byte array.
     */
    protected byte[] encodeInstanceData(List<VariableBinding> vbs) {
        ArrayList<VariableBinding> exports = new ArrayList<>(vbs.size());
//        exports.add(new VariableBinding(instanceSubID, new Integer32(0)));
        exports.addAll(vbs);
        // Pre-encrypt SecretOctetString instances so the size accounting and the on-the-wire bytes are
        // guaranteed to agree (the salt and IV that drive the ciphertext length are random per encryption
        // call). For non-encrypted variables the map stays null / empty and there is no overhead.
        IdentityHashMap<VariableBinding, byte[]> encryptedCache = null;
        if (masterPassword != null) {
            for (VariableBinding vb : exports) {
                if (vb.getVariable() instanceof SecretOctetString) {
                    if (encryptedCache == null) {
                        encryptedCache = new IdentityHashMap<>();
                    }
                    encryptedCache.put(vb, encryptSecretToBerSequence((SecretOctetString) vb.getVariable()));
                }
            }
        }
        int vbLength = 0;
        for (VariableBinding vb : exports) {
            int indexOIDLength = getIndexOIDLength(vb.getOid().getValue());
            int variableLength = (encryptedCache != null && encryptedCache.containsKey(vb))
                    ? encryptedCache.get(vb).length
                    : vb.getVariable().getBERLength();
            int subLength = indexOIDLength + BER.getBERLengthOfLength(indexOIDLength) + 1 + variableLength;
            vbLength += BER.getBERLengthOfLength(subLength) + 1 + subLength;
        }
        BEROutputStream berOutputStream =
                new BEROutputStream(ByteBuffer.allocate(vbLength + BER.getBERLengthOfLength(vbLength) + 1));
        try {
            BER.encodeHeader(berOutputStream, BER.SEQUENCE, vbLength);
            for (VariableBinding vb : exports) {
                byte[] preEncrypted = (encryptedCache != null) ? encryptedCache.get(vb) : null;
                encodeVariableBinding(vb, berOutputStream, preEncrypted);
            }
            berOutputStream.flush();
            berOutputStream.close();
            return berOutputStream.getBuffer().array();
        } catch (IOException e) {
            logger.error(e);
            return null;
        }
    }

    /**
     * Decode a {@link VariableBinding} from an {@link BERInputStream}.
     * @param inputStream
     *    the {@link BERInputStream}
     * @return
     *    the decoded {@link VariableBinding}.
     * @throws IOException
     *    if BER encoding is wrong.
     */
    protected VariableBinding decodeVariableBinding(BERInputStream inputStream) throws IOException {
        BER.MutableByte type = new BER.MutableByte();
        int length = BER.decodeHeader(inputStream, type);
        if (type.getValue() != BER.SEQUENCE) {
            throw new IOException("Invalid sequence encoding: " + type.getValue());
        }
        OID index = new OID(decodeIndexOID(inputStream, type));
        // Peek the next tag byte: when an encrypted SecretOctetString has been written, the value of the
        // VariableBinding is itself a BER SEQUENCE (the three-element "{algorithm, secret, salt}" envelope)
        // rather than one of the standard SMI tags. We must distinguish without consuming the tag from
        // the stream.
        inputStream.mark(1);
        int peeked = inputStream.read();
        inputStream.reset();
        Variable variable;
        if (peeked != -1 && ((byte) peeked) == BER.SEQUENCE) {
            variable = decodeEncryptedSecretOctetString(inputStream);
        } else {
            variable = AbstractVariable.createFromBER(inputStream);
        }
        return new VariableBinding(index, variable);
    }

    /**
     * Encode a {@link VariableBinding} to a {@link BEROutputStream}. If a master password has been
     * configured (see {@link #MOXodusPersistence(MOServer[], Environment, char[])}) and the value of
     * {@code vb} is a {@link SecretOctetString}, the value is encrypted before being written. The
     * encrypted representation replaces the original {@link Variable} with a BER {@link BER#SEQUENCE}
     * consisting of three OCTET STRINGs (algorithm identifier, ciphertext with IV prefix, KDF salt).
     * @param vb
     *    a {@link VariableBinding}.
     * @param outputStream
     *    a {@link BEROutputStream}.
     * @throws IOException
     *    on a buffer overflow.
     */
    protected void encodeVariableBinding(VariableBinding vb, BEROutputStream outputStream) throws IOException {
        byte[] preEncrypted = null;
        if (masterPassword != null && vb.getVariable() instanceof SecretOctetString) {
            preEncrypted = encryptSecretToBerSequence((SecretOctetString) vb.getVariable());
        }
        encodeVariableBinding(vb, outputStream, preEncrypted);
    }

    /**
     * Encode a {@link VariableBinding} where the {@link Variable} part may already be available as a
     * pre-computed BER byte sequence. This overload is used internally to avoid encrypting twice (once
     * during size calculation, once during the actual write).
     * @param vb
     *    the {@link VariableBinding}.
     * @param outputStream
     *    the target {@link BEROutputStream}.
     * @param preEncodedVariable
     *    if non-null, the bytes are written verbatim in place of {@code vb.getVariable().encodeBER(...)};
     *    if null, the variable encodes itself.
     * @throws IOException
     *    on a buffer overflow.
     * @since 3.8.0
     */
    protected void encodeVariableBinding(VariableBinding vb, BEROutputStream outputStream,
                                         byte[] preEncodedVariable) throws IOException {
        Variable variable = vb.getVariable();
        int indexOIDLength = getIndexOIDLength(vb.getOid().getValue());
        int variableLength = (preEncodedVariable != null) ? preEncodedVariable.length : variable.getBERLength();
        int length = indexOIDLength + BER.getBERLengthOfLength(indexOIDLength) + variableLength;
        BER.encodeHeader(outputStream, BER.SEQUENCE, length);
        encodeIndexOID(outputStream, BER.OID, vb.getOid().getValue());
        if (preEncodedVariable != null) {
            outputStream.write(preEncodedVariable);
        } else {
            variable.encodeBER(outputStream);
        }
    }

    /**
     * Gets the key for an OID and instance ID.
     * @param oid
     *    a OID.
     * @param instanceID
     *    an instance ID.
     * @return
     *   a byte stream encoded unique key.
     */
    protected ByteIterable getKey(OID oid, OID instanceID) {
        if (instanceID == null || instanceID.size() == 0) {
            return  CompressedUnsignedLongArrayByteIterable.getIterable(oid.toUnsignedLongArray());
        }
        OID instanceOID = new OID(oid.getValue(), instanceID.getValue());
        return CompressedUnsignedLongArrayByteIterable.getIterable(instanceOID.toUnsignedLongArray());
    }

    /**
     * Gets the OID of a key.
     * @param key
     *    a byte stream containing a key.
     * @return
     *    the OID portion of the key.
     */
    protected OID getKeyOid(ByteIterable key) {
        int len = key.getLength();
        ByteIterator byteIterator = key.iterator();
        int bytesPerLong = byteIterator.next();
        int[] values = new int[(len-1)/bytesPerLong];
        for (int i = 0; i < values.length; ++i) {
            values[i] = (int) byteIterator.nextLong(bytesPerLong);
        }
        return new OID(values);
    }

    /**
     * Saves the data of the {@link MOServer}s associated with this instance to persistent storage depending on
     * the currently configured {@link SavingStrategy}. If that strategy is {@link SavingStrategy#onChangeEventsOnly},
     * calling this method will have no effect, except that it sets {@link #setIgnoreChangeListenerEvents(boolean)} to
     * {@code false} in any case.
     * Use {@link #setMoScopeComparator(MOScopeComparator)} to modify the order in which {@link ManagedObject}s
     * are stored.
     */
    public void save() {
        save(null);
    }

    /**
     * Saves the data of the {@link MOServer}s associated with this instance to persistent storage depending on
     * the currently configured {@link SavingStrategy}. If that strategy is {@link SavingStrategy#onChangeEventsOnly},
     * calling this method will have no effect, except that it sets {@link #setIgnoreChangeListenerEvents(boolean)} to
     * {@code false} in any case.
     * Use {@link #setMoScopeComparator(MOScopeComparator)} to modify the order in which {@link ManagedObject}s
     * are stored.
     * Since version 3.5.0, objects are saved in the order defined by the {@link MOPriorityProvider} given in order to
     * facilitate saving of {@link ManagedObject}s that augment or extend base tables or depend on each other.
     *
     * @param priorityProvider
     *    if not {@code null}, it provides the order for saving the managed objects by
     *    {@link MOPriorityProvider#getPriorityMap(OctetString)}.
     */
    public synchronized void save(MOPriorityProvider priorityProvider) {
        try {
            if (savingStrategy != SavingStrategy.onChangeEventsOnly || !isContinuousChangeListening()) {
                saveFullDump(priorityProvider, false);
            }
        }
        finally {
            setIgnoreChangeListenerEvents(false);
        }
    }

    /**
     * Saves the data of the {@link MOServer}s associated with this instance to persistent storage independent of the
     * currently set {@link SavingStrategy}. In other words, any non-volatile {@link ManagedObject} will be saved to
     * persistent storage.
     * @param priorityProvider
     *    if not {@code null}, it provides the order for saving the managed objects by
     *    {@link MOPriorityProvider#getPriorityMap(OctetString)}.
     * @param includeVolatile
     *    if {@code true}, {@link RandomAccessManagedObject}s with {@link RandomAccessManagedObject#isVolatile()}
     *    {@code true} will be saved too, otherwise those objects will not be saved.
     * @since 3.6.0
     */
    public synchronized void saveFullDump(MOPriorityProvider priorityProvider, boolean includeVolatile) {
        for (MOServer moServer : moServers) {
            final Transaction txn = environment.beginExclusiveTransaction();
            Map<OctetString, Store> stores = new HashMap<>();
            for (OctetString context : moServer.getContexts()) {
                MOScopeComparator moScopeComparator = this.moScopeComparator;
                if (!stores.containsKey(context)) {
                    stores.put(context, createStore(txn, context));
                    moScopeComparator = getContextComparator(priorityProvider, moScopeComparator, context);
                }
                Iterator<Map.Entry<MOScope, ManagedObject<?>>> moIterator = moServer.iterator(moScopeComparator, null);
                runSynchronization(stores, txn, null, moIterator, includeVolatile);
            }
            if (moServer.isContextSupported(null)) {
                stores.put(new OctetString(), environment.openStore("",
                        StoreConfig.WITHOUT_DUPLICATES, txn));
                MOScopeComparator moScopeComparator = this.moScopeComparator;
                moScopeComparator = getContextComparator(priorityProvider, moScopeComparator, null);
                Iterator<Map.Entry<MOScope, ManagedObject<?>>> moIterator =
                        moServer.iterator(moScopeComparator, null);
                runSynchronization(stores, txn, null, moIterator, includeVolatile);
            }
            txn.flush();
            txn.commit();
        }
    }

    private MOScopeComparator getContextComparator(MOPriorityProvider priorityProvider,
                                                   MOScopeComparator moScopeComparator,
                                                   OctetString context) {
        if (priorityProvider == null) {
            return moScopeComparator;
        }
        SortedMap<OID, Integer> priorityMap = priorityProvider.getPriorityMap(context);
        if (priorityMap != null) {
            moScopeComparator = new MOScopePriorityComparator(priorityMap);
            if (logger.isDebugEnabled()) {
                logger.debug("Priority map for context '"+context+"' is: "+
                        priorityProvider.getPriorityMap(context));
            }
        }
        return moScopeComparator;
    }

    /**
     * Encodes an index OID.
     * @param os
     *    the {@link BEROutputStream} to encode to.
     * @param type
     *    the type of the index OID (i.e. {@link OID#getSyntax()}.
     * @param oid
     *    the OID value to encode.
     * @throws IOException
     *    if the output stream throws an {@link IOException}.
     */
    public static void encodeIndexOID(OutputStream os, byte type, int[] oid)
            throws IOException {
        BER.encodeHeader(os, type, getIndexOIDLength(oid));

        int encodedLength = oid.length;
        int rpos = 0;

        while (encodedLength-- > 0) {
            BER.encodeSubID(os, oid[rpos++]);
        }
    }

    /**
     * Get the index OID length from a index OID raw value.
     * @param value
     *    the OID raw value (see {@link OID#getValue()}.
     * @return
     *    the length of the index.
     */
    public static int getIndexOIDLength(int[] value) {
        int length = 0;
        for (int j : value) {
            length += BER.getSubIDLength(j);
        }
        return length;
    }


    /**
     * Decode a index {@link OID} from a BER input stream.
     * @param is
     *   the {@link BERInputStream} containing the OID.
     * @param type
     *   the object type read is returned here.
     * @return
     *   the decoded OID.
     * @throws IOException
     *   if the encoded is wrong.
     */
    public static int[] decodeIndexOID(BERInputStream is, BER.MutableByte type)
            throws IOException {
        int subidentifier;
        int length;

        // get the type
        type.setValue((byte) is.read());
        if (type.getValue() != 0x06) {
            throw new IOException("Wrong type. Not an OID: " + type.getValue() +
                    BER.getPositionMessage(is));
        }
        length = BER.decodeLength(is);

        int[] oid = new int[length];
        // in SNMP pos = 1, but we want to encode any unsigned int at first/second position!
        int pos = 0;
        while (length > 0) {
            subidentifier = 0;
            int b;
            do {    /* shift and add in low order 7 bits */
                int next = is.read();
                if (next < 0) {
                    throw new IOException("Unexpected end of input stream" +
                            BER.getPositionMessage(is));
                }
                b = next & 0xFF;
                subidentifier = (subidentifier << 7) + (b & ~BER.ASN_BIT8);
                length--;
            } while ((length > 0) && ((b & BER.ASN_BIT8) != 0));    /* last byte has high bit clear */
            oid[pos++] = subidentifier;
        }
        if (pos < oid.length) {
            int[] value = new int[pos];
            System.arraycopy(oid, 0, value, 0, pos);
            return value;
        }
        return oid;
    }


    /**
     * A ManagedObject change is being prepared. To cancel preparation set the
     * deny reason to a SNMPv2/v3 error status.
     *
     * @param changeEvent
     *         the change event object.
     */
    public void beforePrepareMOChange(MOChangeEvent changeEvent) {

    }

    /**
     * A change has been prepared. Setting the deny reason of the supplied event
     * object will be ignored.
     *
     * @param changeEvent
     *         the change event object.
     */
    public void afterPrepareMOChange(MOChangeEvent changeEvent) {

    }

    /**
     * A ManagedObject change is being committed. To cancel the commit phase set
     * the deny reason to a SNMPv2/v3 error status.
     * <p>
     * NOTE: Canceling the commit phase must be avoided. Setting a deny reason
     * has only an effect if {@link MOChangeEvent#isDeniable()} returns
     * {@code true}. Otherwise, you will need to throw an exception.
     *
     * @param changeEvent
     *         the change event object.
     */
    public void beforeMOChange(MOChangeEvent changeEvent) {

    }

    /**
     * Gets the current exception list for keeping {@link RandomAccessManagedObject}s persistent regardless their
     * {@link RandomAccessManagedObject#isVolatile()} value or not. A {@link RandomAccessManagedObject} is kept
     * exceptionally persistent if its OID is key of the returned list and its value is {@code true}. If the value
     * is {@code false}, the corresponding {@link RandomAccessManagedObject} will not be stored/restored even if it is
     * non-volatile (i.e., {@link RandomAccessManagedObject#isVolatile()} returns {@code false}).
     * @return
     *    the exception list. If {@code null} is returned, there are no exceptions defined/active (same effect as an
     *    empty exception list).
     * @since 3.6.0
     */
    public Map<OID, Boolean> getPersistenceExceptionList() {
        return persistenceExceptionList;
    }

    /**
     * Sets the current exception list for keeping {@link RandomAccessManagedObject}s persistent regardless their
     * {@link RandomAccessManagedObject#isVolatile()} value or not. A {@link RandomAccessManagedObject} is kept
     * exceptionally persistent if its OID is key of the provided list and its value is {@code true}. If the value
     * is {@code false}, the corresponding {@link RandomAccessManagedObject} will not be stored/restored even if it is
     * non-volatile (i.e., {@link RandomAccessManagedObject#isVolatile()} returns {@code false}).
     * For {@link RandomAccessManagedObject}s with a rang eof OIDs, the lower bound of its {@link MOScope} has to be
     * provided.
     * @param persistenceExceptionList
     *    the exception list. If {@code null} is provided, there are no exceptions defined/active (same effect as an
     *    empty exception list).
     * @since 3.6.0
     */
    public void setPersistenceExceptionList(Map<OID, Boolean> persistenceExceptionList) {
        this.persistenceExceptionList = persistenceExceptionList;
    }

    /**
     * A change has been committed. Setting the deny reason of the supplied event
     * object will be ignored.
     *
     * @param changeEvent
     *         the change event object.
     */
    public void afterMOChange(MOChangeEvent changeEvent) {
        if (logger.isDebugEnabled()) {
            logger.debug("Managed object " + changeEvent.getChangedObject().getScope() + " changed");
        }
        if (!isIgnoreChangeListenerEvents()) {
            ManagedObject<?> managedObject = changeEvent.getChangedObject();
            if (managedObject instanceof RandomAccessManagedObject) {
                RandomAccessManagedObject<?> randomAccessManagedObject = (RandomAccessManagedObject<?>) managedObject;
                OID instanceID = changeEvent.getOID();
                if (changeEvent.getOidType() != MOChangeEvent.OidType.index) {
                    instanceID = instanceID.getSuffix(managedObject.getScope().getLowerBound());
                }
                if (!isPersistent(randomAccessManagedObject, randomAccessManagedObject.getScope().getLowerBound(),
                        instanceID, false)) {
                    return;
                }
                Set<OctetString> contexts = getContexts(managedObject);
                if (!contexts.isEmpty()) {
                    @SuppressWarnings("unchecked")
                    List<VariableBinding> vbs = randomAccessManagedObject.exportInstance(instanceID);
                    final Transaction txn = environment.beginTransaction();
                    if (changeEvent.getModification() == null) {
                        for (OctetString context : contexts) {
                            Store store = createStore(txn, context);
                            putInstanceData(txn, context, store, managedObject.getScope().getLowerBound(),
                                    instanceID, vbs);
                        }
                    } else {
                        switch (changeEvent.getModification()) {
                            case removed:
                                for (OctetString context : contexts) {
                                    Store store = createStore(txn, context);
                                    store.delete(txn, getKey(managedObject.getScope().getLowerBound(), instanceID));
                                    if (logger.isDebugEnabled()) {
                                        logger.debug("Removed managed object " +
                                                changeEvent.getChangedObject() + "|" + instanceID);
                                    }
                                }
                                break;
                            case added:
                            case updated:
                                for (OctetString context : contexts) {
                                    Store store = createStore(txn, context);
                                    putInstanceData(txn, context, store, managedObject.getScope().getLowerBound(),
                                            instanceID, vbs);
                                }
                                break;
                        }
                    }
                    txn.flush();
                    txn.commit();
                } else {
                    logger.warn("Managed object " + changeEvent.getChangedObject() +
                            " is not registered to any MOServer known to " + this);
                }
            }
        }
        else if (logger.isDebugEnabled()) {
            logger.debug("Ignored change event "+changeEvent);
        }
    }

    private boolean isPersistent(RandomAccessManagedObject<?> managedObject, OID instanceOID, OID instanceSubID, boolean includeVolatile) {
        if (includeVolatile ||
                ((instanceSubID != null) ? managedObject.isVolatile(instanceSubID) : managedObject.isVolatile())) {
            if (isExceptionalPersistent(instanceOID)) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Change of volatile " + managedObject + " is accepted by exception list");
                }
            }
            else if (instanceSubID == null) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Ignored change of " + managedObject + " because it is volatile");
                }
                return false;
            }
            else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Sub-instance " + instanceSubID + " from " + managedObject + " is volatile");
                }
                return false;
            }
        }
        else if (isExceptionalNonPersistent(instanceOID)) {
            if (logger.isDebugEnabled()) {
                logger.debug("Ignored change of " + managedObject +
                        " because it is not accepted by exception list");
            }
            return false;
        }
        return true;
    }

    private boolean isExceptionalPersistent(OID oid) {
        return persistenceExceptionList != null && Objects.requireNonNullElse(persistenceExceptionList.get(oid), false);
    }

    private boolean isExceptionalNonPersistent(OID oid) {
        return persistenceExceptionList != null && !Objects.requireNonNullElse(persistenceExceptionList.get(oid), true);
    }

    /**
     * Get the contexts registered with all {@link MOServer} for the given {@link ManagedObject}.
     * @param managedObject
     *    the managed object.
     * @return
     *    the set of registered contexts, see also {@link MOServer#getRegisteredContexts(ManagedObject)}.
     */
    protected Set<OctetString> getContexts(ManagedObject<?> managedObject) {
        Set<OctetString> contexts = new HashSet<>();
        for (MOServer moServer : moServers) {
            contexts.addAll(Arrays.asList(moServer.getRegisteredContexts(managedObject)));
        }
        return contexts;
    }

    /**
     * Encrypts a {@link SecretOctetString} value and returns the BER-encoded SEQUENCE that replaces the
     * variable on the wire. The returned bytes form a complete BER object (tag + length + content) with
     * three OCTET STRING children: the algorithm identifier, the ciphertext (with the cipher IV prefixed),
     * and the random PBKDF2 salt.
     * @param secret
     *    the value to encrypt. Must not be null.
     * @return
     *    a BER-encoded SEQUENCE replacing the {@link Variable} portion of the {@link VariableBinding}.
     * @since 3.8.0
     */
    protected byte[] encryptSecretToBerSequence(SecretOctetString secret) {
        if (masterPassword == null) {
            throw new IllegalStateException("Cannot encrypt SecretOctetString: no master password configured");
        }
        EncryptionSpec spec = parseEncryptionAlgorithm(defaultEncryptionAlgorithm);
        byte[] salt = new byte[SALT_BYTES];
        byte[] iv = new byte[spec.ivBytes];
        SECURE_RANDOM.nextBytes(salt);
        SECURE_RANDOM.nextBytes(iv);
        byte[] plaintext = secret.getValue();
        if (plaintext == null) {
            plaintext = new byte[0];
        }
        try {
            SecretKey key = deriveKey(masterPassword, salt, spec);
            Cipher cipher = Cipher.getInstance(spec.cipherTransformation);
            if (spec.gcm) {
                cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv));
            } else {
                cipher.init(Cipher.ENCRYPT_MODE, key, new IvParameterSpec(iv));
            }
            byte[] ciphertext = cipher.doFinal(plaintext);
            byte[] cipherBlob = new byte[iv.length + ciphertext.length];
            System.arraycopy(iv, 0, cipherBlob, 0, iv.length);
            System.arraycopy(ciphertext, 0, cipherBlob, iv.length, ciphertext.length);

            byte[] algBytes = defaultEncryptionAlgorithm.getBytes(StandardCharsets.UTF_8);
            int algBer = 1 + BER.getBERLengthOfLength(algBytes.length) + algBytes.length;
            int cipherBer = 1 + BER.getBERLengthOfLength(cipherBlob.length) + cipherBlob.length;
            int saltBer = 1 + BER.getBERLengthOfLength(salt.length) + salt.length;
            int contentLen = algBer + cipherBer + saltBer;
            int totalLen = 1 + BER.getBERLengthOfLength(contentLen) + contentLen;

            BEROutputStream out = new BEROutputStream(ByteBuffer.allocate(totalLen));
            BER.encodeHeader(out, BER.SEQUENCE, contentLen);
            BER.encodeString(out, BER.OCTETSTRING, algBytes);
            BER.encodeString(out, BER.OCTETSTRING, cipherBlob);
            BER.encodeString(out, BER.OCTETSTRING, salt);
            out.flush();
            return out.getBuffer().array();
        } catch (GeneralSecurityException | IOException ex) {
            throw new RuntimeException("Failed to encrypt SecretOctetString with algorithm '" +
                    defaultEncryptionAlgorithm + "': " + ex.getMessage(), ex);
        }
    }

    /**
     * Decodes a BER SEQUENCE that wraps an encrypted {@link SecretOctetString} and returns the decrypted
     * value as {@link SecretOctetString}. The input stream must be positioned at the SEQUENCE tag.
     * <p>
     * If no master password is configured, behaviour depends on
     * {@link #isIgnoreEncryptedDataWithoutPassword()}: when {@code true}, the raw BER octet stream of the
     * sequence (including header) is returned wrapped in a {@link SecretOctetString} so that the data can
     * be passed through unchanged. When {@code false}, a fatal log message is emitted and a
     * {@link RuntimeException} is thrown to abort the load.
     * @param inputStream
     *    a {@link BERInputStream} positioned at the SEQUENCE tag of an encrypted secret.
     * @return
     *    the decrypted {@link SecretOctetString}, or the opaque raw bytes if no password is set and the
     *    ignore-flag has been activated.
     * @throws IOException
     *    if the input stream is malformed or decryption fails.
     * @since 3.8.0
     */
    protected Variable decodeEncryptedSecretOctetString(BERInputStream inputStream) throws IOException {
        long sequenceStart = inputStream.getPosition();
        BER.MutableByte type = new BER.MutableByte();
        int seqLen = BER.decodeHeader(inputStream, type);
        if (type.getValue() != BER.SEQUENCE) {
            throw new IOException("Expected SEQUENCE for encrypted SecretOctetString, got: " + type.getValue());
        }
        long contentStart = inputStream.getPosition();
        long sequenceEnd = contentStart + seqLen;

        if (masterPassword == null) {
            if (ignoreEncryptedDataWithoutPassword) {
                int rawLen = (int) (sequenceEnd - sequenceStart);
                byte[] raw = new byte[rawLen];
                ByteBuffer buf = inputStream.getBuffer();
                buf.position((int) sequenceStart);
                buf.get(raw);
                if (logger.isInfoEnabled()) {
                    logger.info("Encountered encrypted SecretOctetString without master password " +
                            "configured; passing through " + rawLen + " raw BER bytes as SecretOctetString");
                }
                return new SecretOctetString(raw);
            }
            logger.fatal("Encountered encrypted SecretOctetString in persistent data but no master " +
                    "password is configured on MOXodusPersistence. Aborting load to avoid silent data loss. " +
                    "Provide the master password to the constructor, or set " +
                    "setIgnoreEncryptedDataWithoutPassword(true) to pass encrypted bytes through unchanged.");
            throw new RuntimeException("Persistent data contains encrypted SecretOctetString values but no " +
                    "master password was provided to MOXodusPersistence");
        }

        byte[] algBytes = readOctetString(inputStream);
        byte[] cipherBlob = readOctetString(inputStream);
        byte[] salt = readOctetString(inputStream);
        String algorithmId = new String(algBytes, StandardCharsets.UTF_8);
        EncryptionSpec spec = parseEncryptionAlgorithm(algorithmId);
        if (cipherBlob.length < spec.ivBytes) {
            throw new IOException("Encrypted secret too short for algorithm '" + algorithmId +
                    "': expected at least " + spec.ivBytes + " IV bytes, got " + cipherBlob.length);
        }
        byte[] iv = Arrays.copyOfRange(cipherBlob, 0, spec.ivBytes);
        byte[] ciphertext = Arrays.copyOfRange(cipherBlob, spec.ivBytes, cipherBlob.length);
        try {
            SecretKey key = deriveKey(masterPassword, salt, spec);
            Cipher cipher = Cipher.getInstance(spec.cipherTransformation);
            if (spec.gcm) {
                cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv));
            } else {
                cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
            }
            byte[] plaintext = cipher.doFinal(ciphertext);
            return new SecretOctetString(plaintext);
        } catch (GeneralSecurityException ex) {
            throw new IOException("Failed to decrypt SecretOctetString with algorithm '" + algorithmId +
                    "': " + ex.getMessage(), ex);
        }
    }

    private static byte[] readOctetString(BERInputStream inputStream) throws IOException {
        BER.MutableByte type = new BER.MutableByte();
        byte[] data = BER.decodeString(inputStream, type);
        if (type.getValue() != BER.OCTETSTRING) {
            throw new IOException("Expected OCTET STRING inside encrypted sequence, got: " + type.getValue());
        }
        return data;
    }

    /**
     * Derives an AES {@link SecretKey} from {@code password} and {@code salt} using the KDF named in
     * {@code spec}. The returned key is wrapped in a {@link SecretKeySpec} that uses the cipher algorithm
     * (the part before the first "/" in the transformation string) so that it can be passed to
     * {@link Cipher#init(int, java.security.Key, java.security.spec.AlgorithmParameterSpec)}.
     * @param password
     *    the master password (defensively cleared inside the PBE key spec).
     * @param salt
     *    the per-value random salt.
     * @param spec
     *    the parsed encryption algorithm specification.
     * @return
     *    a {@link SecretKey} usable with the configured cipher.
     * @throws GeneralSecurityException
     *    if the JCE provider does not support the requested KDF or key size.
     * @since 3.8.0
     */
    protected SecretKey deriveKey(char[] password, byte[] salt, EncryptionSpec spec)
            throws GeneralSecurityException {
        PBEKeySpec keySpec = new PBEKeySpec(password, salt, spec.iterations, spec.keyBits);
        try {
            SecretKeyFactory factory = SecretKeyFactory.getInstance(spec.kdf);
            SecretKey derived = factory.generateSecret(keySpec);
            String[] parts = spec.cipherTransformation.split("/");
            return new SecretKeySpec(derived.getEncoded(), parts[0]);
        } finally {
            keySpec.clearPassword();
        }
    }

    /**
     * Parses a {@link MOXodusPersistence} encryption algorithm identifier into a structured
     * {@link EncryptionSpec}. The expected format is
     * {@code <KDF>:<iterations>:<cipherTransformation>:<keyBits>}, e.g.
     * {@code PBKDF2WithHmacSHA256:600000:AES/GCM/NoPadding:256}.
     * @param id
     *    the algorithm identifier.
     * @return
     *    the parsed {@link EncryptionSpec}.
     * @throws IllegalArgumentException
     *    if {@code id} does not have the expected format or contains invalid values.
     * @since 3.8.0
     */
    protected EncryptionSpec parseEncryptionAlgorithm(String id) {
        if (id == null) {
            throw new IllegalArgumentException("Encryption algorithm identifier must not be null");
        }
        String[] tokens = id.split(":");
        if (tokens.length != 4) {
            throw new IllegalArgumentException("Invalid encryption algorithm format '" + id +
                    "'. Expected '<KDF>:<iterations>:<cipherTransformation>:<keyBits>'");
        }
        int iterations;
        int keyBits;
        try {
            iterations = Integer.parseInt(tokens[1]);
            keyBits = Integer.parseInt(tokens[3]);
        } catch (NumberFormatException nfe) {
            throw new IllegalArgumentException("Invalid number in encryption algorithm '" + id + "': " +
                    nfe.getMessage(), nfe);
        }
        return new EncryptionSpec(id, tokens[0], iterations, tokens[2], keyBits);
    }

    /**
     * A parsed representation of an algorithm identifier as accepted by
     * {@link #setDefaultEncryptionAlgorithm(String)}. Subclasses may override
     * {@link MOXodusPersistence#parseEncryptionAlgorithm(String)} to extend the supported set.
     * @since 3.8.0
     */
    protected static class EncryptionSpec {
        public final String algorithmId;
        public final String kdf;
        public final int iterations;
        public final String cipherTransformation;
        public final int keyBits;
        public final int ivBytes;
        public final boolean gcm;

        public EncryptionSpec(String algorithmId, String kdf, int iterations,
                              String cipherTransformation, int keyBits) {
            this.algorithmId = algorithmId;
            this.kdf = kdf;
            this.iterations = iterations;
            this.cipherTransformation = cipherTransformation;
            this.keyBits = keyBits;
            String[] parts = cipherTransformation.split("/");
            String mode = (parts.length >= 2) ? parts[1] : "";
            this.gcm = "GCM".equalsIgnoreCase(mode);
            this.ivBytes = this.gcm ? GCM_IV_BYTES : CBC_IV_BYTES;
        }
    }
}
