diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..796b96d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/build diff --git a/build.gradle b/build.gradle index df967cd..23db49f 100644 --- a/build.gradle +++ b/build.gradle @@ -1,33 +1,64 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. -apply plugin: 'java' -version="1.3" +buildscript { + repositories { + google() + jcenter() -targetCompatibility = 1.8 -sourceCompatibility = 1.8 + } + dependencies { + classpath 'com.android.tools.build:gradle:3.5.0-beta05' -jar.baseName='BigArrayList' - -repositories { - mavenCentral() + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } } -configurations { - extraLibs -} +allprojects { + repositories { + google() + jcenter() -dependencies { - testCompile group: 'junit', name: 'junit', version: '4.4' - compile group: 'de.ruedigermoeller', name: 'fst', version: '2.50' - extraLibs group: 'de.ruedigermoeller', name: 'fst', version: '2.50' - configurations.compile.extendsFrom(configurations.extraLibs) + } } +apply plugin: 'com.android.library' + +android { + compileSdkVersion 29 -jar { - from { - configurations.extraLibs.collect { it.isDirectory() ? it : zipTree(it) } + defaultConfig { + + minSdkVersion 19 + targetSdkVersion 29 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' } + } + compileOptions { + sourceCompatibility = 1.8 + targetCompatibility = 1.8 + } } +configurations { + extraLibs +} +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation 'androidx.appcompat:appcompat:1.0.2' + testImplementation 'junit:junit:4.12' + androidTestImplementation 'androidx.test.ext:junit:1.1.1' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + api group: 'de.ruedigermoeller', name: 'fst', version: '2.57' + extraLibs group: 'de.ruedigermoeller', name: 'fst', version: '2.57' + configurations.compile.extendsFrom(configurations.extraLibs) +} diff --git a/gradle.properties b/gradle.properties index e69de29..199d16e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -0,0 +1,20 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx1536m +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Automatically convert third-party libraries to use AndroidX +android.enableJetifier=true + diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..cccdd3d --- /dev/null +++ b/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..f955316 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/lib/commons-logging-1.1.1.jar b/lib/commons-logging-1.1.1.jar deleted file mode 100644 index 1deef14..0000000 Binary files a/lib/commons-logging-1.1.1.jar and /dev/null differ diff --git a/lib/fst-2.50.jar b/lib/fst-2.50.jar deleted file mode 100644 index b03db23..0000000 Binary files a/lib/fst-2.50.jar and /dev/null differ diff --git a/lib/jackson-core-2.8.8.jar b/lib/jackson-core-2.8.8.jar deleted file mode 100644 index 2a66a6f..0000000 Binary files a/lib/jackson-core-2.8.8.jar and /dev/null differ diff --git a/lib/java-util-1.9.0.jar b/lib/java-util-1.9.0.jar deleted file mode 100644 index c4856fd..0000000 Binary files a/lib/java-util-1.9.0.jar and /dev/null differ diff --git a/lib/javassist-3.21.0-GA.jar b/lib/javassist-3.21.0-GA.jar deleted file mode 100644 index 64549c4..0000000 Binary files a/lib/javassist-3.21.0-GA.jar and /dev/null differ diff --git a/lib/json-io-2.5.1.jar b/lib/json-io-2.5.1.jar deleted file mode 100644 index d920d97..0000000 Binary files a/lib/json-io-2.5.1.jar and /dev/null differ diff --git a/lib/objenesis-2.5.1.jar b/lib/objenesis-2.5.1.jar deleted file mode 100644 index 31b245b..0000000 Binary files a/lib/objenesis-2.5.1.jar and /dev/null differ diff --git a/local.properties b/local.properties new file mode 100644 index 0000000..b32fef8 --- /dev/null +++ b/local.properties @@ -0,0 +1,10 @@ +## This file is automatically generated by Android Studio. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file should *NOT* be checked into Version Control Systems, +# as it contains information specific to your local configuration. +# +# Location of the SDK. This is only used by Gradle. +# For customization when using a Version Control System, please read the +# header note. +sdk.dir=C\:\\Users\\Pascal\\AppData\\Local\\Android\\Sdk diff --git a/oldlibs/commons-logging-1.1.1.jar b/oldlibs/commons-logging-1.1.1.jar new file mode 100644 index 0000000..e69de29 diff --git a/oldlibs/fst-2.50.jar b/oldlibs/fst-2.50.jar new file mode 100644 index 0000000..e69de29 diff --git a/oldlibs/jackson-core-2.8.8.jar b/oldlibs/jackson-core-2.8.8.jar new file mode 100644 index 0000000..e69de29 diff --git a/oldlibs/java-util-1.9.0.jar b/oldlibs/java-util-1.9.0.jar new file mode 100644 index 0000000..e69de29 diff --git a/oldlibs/javassist-3.21.0-GA.jar b/oldlibs/javassist-3.21.0-GA.jar new file mode 100644 index 0000000..e69de29 diff --git a/oldlibs/json-io-2.5.1.jar b/oldlibs/json-io-2.5.1.jar new file mode 100644 index 0000000..e69de29 diff --git a/oldlibs/objenesis-2.5.1.jar b/oldlibs/objenesis-2.5.1.jar new file mode 100644 index 0000000..e69de29 diff --git a/proguard-rules.pro b/proguard-rules.pro new file mode 100644 index 0000000..f1b4245 --- /dev/null +++ b/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..dbfb4cd --- /dev/null +++ b/settings.gradle @@ -0,0 +1,2 @@ +//include ':app' +rootProject.name = 'BigArrayList' diff --git a/src/main/AndroidManifest.xml b/src/main/AndroidManifest.xml new file mode 100644 index 0000000..f02f0c8 --- /dev/null +++ b/src/main/AndroidManifest.xml @@ -0,0 +1,11 @@ + + + + diff --git a/src/main/java/com/dselent/bigarraylist/BigArrayList.java b/src/main/java/com/dselent/bigarraylist/BigArrayList.java index 6979ec0..8c16147 100644 --- a/src/main/java/com/dselent/bigarraylist/BigArrayList.java +++ b/src/main/java/com/dselent/bigarraylist/BigArrayList.java @@ -2,40 +2,47 @@ /* * BigArrayList * Copyright (C) 2015 Douglas Selent - * + * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with this program. If not, see . -*/ + */ package com.dselent.bigarraylist; +import android.annotation.SuppressLint; + +import androidx.core.util.Consumer; + import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; /** - * A BigArrayList acts the same way a regular {@link java.util.ArrayList} would for data sizes that cannot fit in memory all at once. - * This class can be used just like an ArrayList, with all the file I/O managed automatically and internally. + * A BigArrayList acts the same way a regular {@link ArrayList} would for data sizes that cannot fit in memory all at + * once. This class can be used just like an ArrayList, with all the file I/O managed automatically and internally. *

* The size and number of cache blocks to be stored in memory can be specified. *

- * BigArrayList uses an internal ArrayList of ArrayLists (A list of cache blocks) to map data in memory to disk. - * The mapping is managed and maintained by the CacheMapping class. - * The File I/O uses Object input/output streams for classes that implement the serializable interface. - * An LRU cache policy is used to determine which block of data so swap out of memory. + * BigArrayList uses an internal ArrayList of ArrayLists (A list of cache blocks) to map data in memory to disk. The + * mapping is managed and maintained by the CacheMapping class. The File I/O uses Object input/output streams for + * classes that implement the serializable interface. An LRU cache policy is used to determine which block of data so + * swap out of memory. *

* A BigArrayList currently supports adding elements to the end of the list and setting/getting elements *

@@ -45,39 +52,38 @@ * //Construct an instance of BigArrayList with 4 cache blocks of size 100,000 each * //to store data in a folder called "memory" relative to the programs starting file path location * BigArrayList arrayList = new BigArrayList("memory", 100000, 4); - * + * * //Add elements to the list * for(long i=0; i<1000000; i++) * { * arrayList.add(i); * } - * + * * //Set and get elements * arrayList.set(10, 100l); * long getElement = arrayList.get(10); - * + * * //Clear data from disk when done * arrayList.clearMemory(); * } * *

- * @author Douglas Selent * * @param Generic type + * @author Douglas Selent */ -public class BigArrayList -{ +@SuppressLint("NewApi") +public class BigArrayList { /** - * The ArrayList of cache blocks. - * Each ArrayList corresponds to a cache block currently in memory + * The ArrayList of cache blocks. Each ArrayList corresponds to a cache block currently in memory */ private List> arrayLists; - + /** * The SoftMapping object used to map the soft indices to the hard indices */ private final SoftMapping softMapping; - + /** * The CacheMapping object used to map contents in memory to contents on disk */ @@ -87,12 +93,12 @@ public class BigArrayList * Size of the whole list including what is and is not currently in memory */ private long wholeListSize; - + /** * Default size of cache block = 1,000,000 */ private static final int DEFAULT_BLOCK_SIZE = 1000000; - + /** * Default number of cache blocks = 2 */ @@ -102,7 +108,7 @@ public class BigArrayList * The minimum size of a cache block = 5 elements */ private static final int MIN_CACHE_SIZE = 5; - + /** * The maximum size of a cache block = the integer limit of 2^31 - 1 */ @@ -112,7 +118,7 @@ public class BigArrayList * The minimum number of cache blocks = 2 */ private static final int MIN_CACHE_BLOCKS = 2; - + /** * The maximum number of cache blocks = the integer limit of 2^31 - 1 */ @@ -122,7 +128,7 @@ public class BigArrayList * The size of the cache blocks */ private int blockSize; - + /** * The number of cache blocks */ @@ -130,47 +136,44 @@ public class BigArrayList //all methods should check this and throw an exception if false /** - * Whether or not the BigArrayList is still a live object. - * The contents of the BigArrayList must be manually cleared from disk using {@link #clearMemory()}. - * Doing so, will result in the BigArrayList object still existing in memory - * but it will be unusable since all the references point to content that has been deleted. - * In this case the BigArrayList is considered a dead object even though it is not technically considered dead by Java. + * Whether or not the BigArrayList is still a live object. The contents of the BigArrayList must be manually cleared + * from disk using {@link #clearMemory()}. Doing so, will result in the BigArrayList object still existing in memory + * but it will be unusable since all the references point to content that has been deleted. In this case the + * BigArrayList is considered a dead object even though it is not technically considered dead by Java. */ private boolean liveObject; - + /** * Type of serialization to use */ private IOTypes ioType; - + /** * Possible IO serialization types - * + * * @author Doug */ - public enum IOTypes - { + public enum IOTypes { OBJECT, MMAP_OBJECT, FST_OBJECT, - MMAP_FST_OBJECT; + MMAP_FST_OBJECT } /** - * Constructs a BigArrayList with default values for the number of cache block, size of each cache block, and disk path. + * Constructs a BigArrayList with default values for the number of cache block, size of each cache block, and disk + * path. */ - public BigArrayList() - { + public BigArrayList() { blockSize = DEFAULT_BLOCK_SIZE; cacheBlocks = DEFAULT_CACHE_BLOCKS; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -183,21 +186,19 @@ public BigArrayList() /** * Constructor that specifies where BigArrayList should write to disk. - * + * * @param memoryPath The folder path to write to */ - public BigArrayList(String memoryPath) - { + public BigArrayList(String memoryPath) { blockSize = DEFAULT_BLOCK_SIZE; cacheBlocks = DEFAULT_CACHE_BLOCKS; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks, memoryPath); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -207,25 +208,23 @@ public BigArrayList(String memoryPath) this.ioType = IOTypes.FST_OBJECT; liveObject = true; } - + /** * Constructor that specifies where BigArrayList should write to disk. - * + * * @param memoryPath The folder path to write to - * @param ioType The type of IO to use + * @param ioType The type of IO to use */ - public BigArrayList(String memoryPath, IOTypes ioType) - { + public BigArrayList(String memoryPath, IOTypes ioType) { blockSize = DEFAULT_BLOCK_SIZE; cacheBlocks = DEFAULT_CACHE_BLOCKS; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks, memoryPath); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -238,32 +237,28 @@ public BigArrayList(String memoryPath, IOTypes ioType) /** * Constructor that specifies the size and number of cache blocks. - * - * @param blockSize Size of each cache block + * + * @param blockSize Size of each cache block * @param cacheBlocks Number of cache blocks stored in memory at a given time */ - public BigArrayList(int blockSize, int cacheBlocks) - { - if(blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) - { + public BigArrayList(int blockSize, int cacheBlocks) { + if (blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) { throw new IllegalArgumentException("Cache size is " + blockSize + " but must be >= " + MIN_CACHE_SIZE + " and <= " + MAX_CACHE_SIZE); } - if(cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) - { - throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); + if (cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) { + throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); } - + this.blockSize = blockSize; this.cacheBlocks = cacheBlocks; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -273,36 +268,32 @@ public BigArrayList(int blockSize, int cacheBlocks) this.ioType = IOTypes.FST_OBJECT; liveObject = true; } - + /** * Constructor that specifies the size and number of cache blocks. - * - * @param blockSize Size of each cache block + * + * @param blockSize Size of each cache block * @param cacheBlocks Number of cache blocks stored in memory at a given time - * @param ioType The type of IO to use + * @param ioType The type of IO to use */ - public BigArrayList(int blockSize, int cacheBlocks, IOTypes ioType) - { - if(blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) - { + public BigArrayList(int blockSize, int cacheBlocks, IOTypes ioType) { + if (blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) { throw new IllegalArgumentException("Cache size is " + blockSize + " but must be >= " + MIN_CACHE_SIZE + " and <= " + MAX_CACHE_SIZE); } - if(cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) - { - throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); + if (cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) { + throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); } - + this.blockSize = blockSize; this.cacheBlocks = cacheBlocks; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -313,36 +304,31 @@ public BigArrayList(int blockSize, int cacheBlocks, IOTypes ioType) liveObject = true; } - /** * Constructor that specifies the size and number of cache blocks as well as the folder path to write to. - * - * @param blockSize cacheSize Size of each cache block + * + * @param blockSize cacheSize Size of each cache block * @param cacheBlocks Number of cache blocks stored in memory at a given time - * @param memoryPath The folder path to write to + * @param memoryPath The folder path to write to */ - public BigArrayList(int blockSize, int cacheBlocks, String memoryPath) - { - if(blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) - { + public BigArrayList(int blockSize, int cacheBlocks, String memoryPath) { + if (blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) { throw new IllegalArgumentException("Cache size is " + blockSize + " but must be >= " + MIN_CACHE_SIZE + " and <= " + MAX_CACHE_SIZE); } - if(cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) - { - throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); + if (cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) { + throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); } - + this.blockSize = blockSize; this.cacheBlocks = cacheBlocks; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks, memoryPath); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -352,37 +338,33 @@ public BigArrayList(int blockSize, int cacheBlocks, String memoryPath) this.ioType = IOTypes.FST_OBJECT; liveObject = true; } - + /** * Constructor that specifies the size and number of cache blocks as well as the folder path to write to. - * - * @param blockSize cacheSize Size of each cache block + * + * @param blockSize cacheSize Size of each cache block * @param cacheBlocks Number of cache blocks stored in memory at a given time - * @param memoryPath The folder path to write to - * @param ioType The type of IO to use + * @param memoryPath The folder path to write to + * @param ioType The type of IO to use */ - public BigArrayList(int blockSize, int cacheBlocks, String memoryPath, IOTypes ioType) - { - if(blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) - { + public BigArrayList(int blockSize, int cacheBlocks, String memoryPath, IOTypes ioType) { + if (blockSize < MIN_CACHE_SIZE || blockSize > MAX_CACHE_SIZE) { throw new IllegalArgumentException("Cache size is " + blockSize + " but must be >= " + MIN_CACHE_SIZE + " and <= " + MAX_CACHE_SIZE); } - if(cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) - { - throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); + if (cacheBlocks < MIN_CACHE_BLOCKS || cacheBlocks > MAX_CACHE_BLOCKS) { + throw new IllegalArgumentException("Number of cache blocks is " + cacheBlocks + " but must be >= " + MIN_CACHE_BLOCKS + " and <= " + MAX_CACHE_BLOCKS); } - + this.blockSize = blockSize; this.cacheBlocks = cacheBlocks; - + softMapping = new SoftMapping(); cacheMapping = new CacheMapping(this, blockSize, cacheBlocks, memoryPath); - + arrayLists = new ArrayList>(); - for(int i=0; i arrayList = new ArrayList(); arrayList.ensureCapacity(blockSize); arrayLists.add(arrayList); @@ -393,148 +375,141 @@ public BigArrayList(int blockSize, int cacheBlocks, String memoryPath, IOTypes i liveObject = true; } - //////////////////////////////////////////////////////////////////////////////////////////////// - + /** TODO slow stream implementation */ + public Stream stream() { + AtomicInteger n = new AtomicInteger(0); + return Stream.generate(() -> get(n.getAndIncrement())).limit(this.size()); +// return Arrays.stream(array); + } + + /** TODO Or slow foreach implementation */ + public void forEach(Consumer action) + { + for (int i = 0; i < this.size(); i++) + action.accept(this.get(i)); + } + /** * @return Returns the associated CacheMapping object */ - protected CacheMapping getCacheMapping() - { + protected CacheMapping getCacheMapping() { return cacheMapping; } - + /** * @return Returns the number of blocks in memory at a time */ - public int getNumberOfBlocks() - { + public int getNumberOfBlocks() { return cacheBlocks; } - + /** * @return Returns the block size */ - public int getBlockSize() - { - - return blockSize; - } - + public int getBlockSize() { return blockSize; } + /** * @return Returns the number of used cache blocks based on the size of the list */ - protected int getNumberOfUsedBlocks() - { + protected int getNumberOfUsedBlocks() { long blockSizeLong = blockSize; long usedBlocks = (long) Math.ceil(this.size() * 1.0 / blockSizeLong * 1.0); - + //safe cast, I really doubt there will ever be over 2^31 - 1 blocks - return (int)usedBlocks; + return (int) usedBlocks; } - + /** * Returns the minimum of the number of used cache blocks based on the list size or the parameter size - * + * * @param index A virtual size index * @return The number of used cache blocks */ - protected int getNumberOfUsedBlocks(long index) - { + protected int getNumberOfUsedBlocks(long index) { long blockSizeLong = blockSize; long usedVirtualBlocks = (long) Math.ceil(index * 1.0 / blockSizeLong * 1.0); long usedRealBlocks = getNumberOfUsedBlocks(); long usedBlocks = Math.max(usedRealBlocks, usedVirtualBlocks); - + //safe cast, I really doubt there will ever be over 2^31 - 1 blocks - return (int)usedBlocks; + return (int) usedBlocks; } - /** * @return Returns the file location of the memory storage */ - public String getFilePath() - { + public String getFilePath() { return cacheMapping.getFileAccessor().getMemoryFilePath(); } - + /** * Sets the ArrayList at the given index - * - * @param index The index of the ArrayList/Cache block to set + * + * @param index The index of the ArrayList/Cache block to set * @param arrayList The new ArrayList/Cache block */ - public void setList(int index, List arrayList) - { + public void setList(int index, List arrayList) { arrayLists.set(index, arrayList); } /** * Returns the ArrayList at the given index - * + * * @param index The index of the ArrayList/Cache block to get * @return The ArrayList/Cache block */ - protected List getList(int index) - { + protected List getList(int index) { return arrayLists.get(index); } /** * Used by the CacheMapping class for swapping caches. + * * @param index The index of the cache block to clear data from */ - protected void clearList(int index) - { + protected void clearList(int index) { arrayLists.get(index).clear(); } /** - * * @return The size of the BigArrayList */ - public long size() - { + public long size() { return wholeListSize; } - + /** - * * @return The IO serialization type */ - public IOTypes getIOType() - { + public IOTypes getIOType() { return ioType; } /** * Returns the size of the ArrayList at the specified index - * + * * @param index The index of the ArrayList/Cache block * @return The size of ArrayList/Cache block */ - protected int getArraySize(int index) - { + protected int getArraySize(int index) { return arrayLists.get(index).size(); } /** * Whether or no the object is live + * * @return True if the object is live, false otherwise */ - public boolean isLive() - { + public boolean isLive() { return liveObject; } /** - * Used to delete the memory file. - * The object should not be used anymore once this method is called - * + * Used to delete the memory file. The object should not be used anymore once this method is called + * * @throws IOException For I/O error */ - public void clearMemory() throws IOException - { + public void clearMemory() throws IOException { cacheMapping.clearMemory(); liveObject = false; } @@ -542,341 +517,282 @@ public void clearMemory() throws IOException /** * Flushes all data in memory to disk */ - public void flushMemory() - { + public void flushMemory() { cacheMapping.flushCache(); } - + /** * Sorts the BigArrayList. Note that the usage mimics Collections.sort() except that the sorted list is returned. - * The caller must set their list to equal the return value (similar to String concatenation), ex: sortedList = BigArrayList.sort(sortedList); - * + * The caller must set their list to equal the return value (similar to String concatenation), ex: sortedList = + * BigArrayList.sort(sortedList); + * * @param unsortedList The list to be sorted * @return The list in sorted order * @throws IOException */ - public static & Serializable> BigArrayList sort(BigArrayList unsortedList) throws IOException - { + @SuppressLint("NewApi") + public static & Serializable> BigArrayList sort(BigArrayList unsortedList) throws IOException { return sort(unsortedList, Comparator.naturalOrder()); } - + /** * Sorts the BigArrayList. Note that the usage mimics Collections.sort() except that the sorted list is returned. - * The caller must set their list to equal the return value (similar to String concatenation), ex: sortedList = BigArrayList.sort(sortedList); - * + * The caller must set their list to equal the return value (similar to String concatenation), ex: sortedList = + * BigArrayList.sort(sortedList); + * * @param unsortedList unsortedList The list to be sorted - * @param comparator How to compare the elements in the list + * @param comparator How to compare the elements in the list * @return The list in sorted order * @throws IOException */ - public static BigArrayList sort(BigArrayList unsortedList, Comparator comparator) throws IOException - { - if(unsortedList.size() <= 1) - { + public static BigArrayList sort(BigArrayList unsortedList, Comparator comparator) throws IOException { + if (unsortedList.size() <= 1) { return unsortedList; - } - else - { + } else { unsortedList.purgeActionBuffer(); - + CacheMapping unsortedCacheMapping = unsortedList.getCacheMapping(); int blockSize = unsortedList.getBlockSize(); int cacheBlocks = unsortedList.getNumberOfBlocks(); int usedCacheBlocks = unsortedList.getNumberOfUsedBlocks(); String filePath = unsortedList.getFilePath(); IOTypes ioType = unsortedList.getIOType(); - + BigArrayList sortedList = new BigArrayList(blockSize, cacheBlocks, filePath, ioType); - - for(int i=usedCacheBlocks-1; i>=0; i--) - { + + for (int i = usedCacheBlocks - 1; i >= 0; i--) { int cacheBlockSpot = -1; - - if(!unsortedCacheMapping.isFileInCache(i)) - { + + if (!unsortedCacheMapping.isFileInCache(i)) { unsortedCacheMapping.bringFileIntoCache(i); } - + cacheBlockSpot = unsortedCacheMapping.getCacheBlockSpot(i); unsortedCacheMapping.setDirtyBit(cacheBlockSpot, true); Collections.sort(unsortedList.getList(cacheBlockSpot), comparator); - } - - if(usedCacheBlocks > 1) - { + } + + if (usedCacheBlocks > 1) { int currentRun = 0; - long totalRuns = 64 - Long.numberOfLeadingZeros(usedCacheBlocks-1); - - while(currentRun < totalRuns) - { + long totalRuns = 64 - Long.numberOfLeadingZeros(usedCacheBlocks - 1); + + while (currentRun < totalRuns) { sortedList = merge(unsortedList, comparator, currentRun); - + unsortedList.clearMemory(); - + unsortedList = sortedList; currentRun++; } - } - else - { + } else { sortedList = unsortedList; } - + return sortedList; } } - + /** * Internal function used to sort. This is the step to merge two sorted pieces together into a single sorted list. - * + * * @param unsortedList The list - * @param comparator How to compare the elements - * @param currentRun What run step the merge is being used on. This is to determine what merge pieces to merge and their sizes + * @param comparator How to compare the elements + * @param currentRun What run step the merge is being used on. This is to determine what merge pieces to merge + * and their sizes * @return The unsorted list with the sorted merged pieces. */ - private static BigArrayList merge(BigArrayList unsortedList, Comparator comparator, int currentRun) - { + private static BigArrayList merge(BigArrayList unsortedList, Comparator comparator, int currentRun) { int blockSize = unsortedList.getBlockSize(); int cacheBlocks = unsortedList.getNumberOfBlocks(); long usedCacheBlocksLong = unsortedList.getNumberOfUsedBlocks(); String filePath = unsortedList.getFilePath(); IOTypes ioType = unsortedList.getIOType(); - + BigArrayList sortedList = new BigArrayList(blockSize, cacheBlocks, filePath, ioType); - + int blockIncrement = ipow(2, currentRun); - - for(long i=0; i= mergePiece3*blockSize) - { - secondElementEnd = mergePiece3*blockSize; - } - else - { + + if (unsortedList.size() >= mergePiece3 * blockSize) { + secondElementEnd = mergePiece3 * blockSize; + } else { secondElementEnd = unsortedList.size(); } - + totalMergeElements = secondElementEnd - firstElementStart; - } - else - { - if(unsortedList.size() >= (i+blockIncrement)*blockSize) - { - firstElementEnd = (i+blockIncrement)*blockSize; - } - else - { + } else { + if (unsortedList.size() >= (i + blockIncrement) * blockSize) { + firstElementEnd = (i + blockIncrement) * blockSize; + } else { firstElementEnd = unsortedList.size(); } - + totalMergeElements = firstElementEnd - firstElementStart; } - - + long index1 = firstElementStart; long index2 = secondElementStart; - - if(mergePiece2 == -1 || mergePiece2 >= usedCacheBlocksLong) - { + + if (mergePiece2 == -1 || mergePiece2 >= usedCacheBlocksLong) { T element1 = unsortedList.get(index1); - while(currentMergeElements < totalMergeElements) - { + while (currentMergeElements < totalMergeElements) { sortedList.add(element1); index1++; - - if(index1 < firstElementEnd) - { + + if (index1 < firstElementEnd) { element1 = unsortedList.get(index1); } - + currentMergeElements++; } - } - else - { + } else { T element1 = unsortedList.get(index1); T element2 = unsortedList.get(index2); - - while(currentMergeElements < totalMergeElements) - { - if(index2 >= secondElementEnd) - { + + while (currentMergeElements < totalMergeElements) { + if (index2 >= secondElementEnd) { sortedList.add(element1); index1++; - - if(index1 < firstElementEnd) - { + + if (index1 < firstElementEnd) { element1 = unsortedList.get(index1); } - } - else if(index1 >= firstElementEnd) - { + } else if (index1 >= firstElementEnd) { sortedList.add(element2); index2++; - - if(index2 < secondElementEnd) - { + + if (index2 < secondElementEnd) { element2 = unsortedList.get(index2); } - } - else - { - if(comparator.compare(element1, element2) <= 0) - { + } else { + if (comparator.compare(element1, element2) <= 0) { sortedList.add(element1); index1++; - - if(index1 < firstElementEnd) - { + + if (index1 < firstElementEnd) { element1 = unsortedList.get(index1); } - } - else - { + } else { sortedList.add(element2); index2++; - - if(index2 < secondElementEnd) - { + + if (index2 < secondElementEnd) { element2 = unsortedList.get(index2); } } } - + currentMergeElements++; } } } - + return sortedList; } - - + //skipping bound checks, trusting the caller to know that the result will not be greater an integer + /** - * Internal function used by sorting. - * modified function from http://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int - * + * Internal function used by sorting. modified function from http://stackoverflow.com/questions/101439/the-most-efficient-way-to-implement-an-integer-based-power-function-powint-int + * * @param base Base number - * @param exp Exponent + * @param exp Exponent * @return Ceiling of base^exp as a power of two */ - private static int ipow(int base, int exp) - { - int result = 1; - - while (exp != 0) - { - if ((exp & 1) == 1) - { - result *= base; - } - - exp >>= 1; - base *= base; - } - - return result; + private static int ipow(int base, int exp) { + int result = 1; + + while (exp != 0) { + if ((exp & 1) == 1) { + result *= base; + } + + exp >>= 1; + base *= base; + } + + return result; } /** * Purges all actions in the queue */ - private void purgeActionBuffer() - { - if(softMapping.getBufferSize() > 0) - { - long startIndex = softMapping.getShiftIndex(0); - + private void purgeActionBuffer() { + if (softMapping.getBufferSize() > 0) { + long startIndex = softMapping.getShiftIndex(0); + //first index int fileNumber = cacheMapping.getFileNumber(startIndex); - int nextFileNumber = fileNumber+1; + int nextFileNumber = fileNumber + 1; long virtualSize = wholeListSize + softMapping.getLastShiftAmount(); int usedCacheBlocks = getNumberOfUsedBlocks(virtualSize); - + int cacheBlockSpot = -1; int nextCacheBlockSpot = -1; - if(!cacheMapping.isFileInCache(fileNumber)) - { + if (!cacheMapping.isFileInCache(fileNumber)) { cacheBlockSpot = cacheMapping.bringFileIntoCache(fileNumber); - } - else - { + } else { cacheBlockSpot = cacheMapping.getCacheBlockSpot(fileNumber); cacheMapping.updateUsedList(cacheBlockSpot); } - + //assume there will be changes, this assumption is not always true? cacheMapping.setDirtyBit(cacheBlockSpot, true); - - if(!cacheMapping.isFileInCache(nextFileNumber)) - { + + if (!cacheMapping.isFileInCache(nextFileNumber)) { nextCacheBlockSpot = cacheMapping.bringFileIntoCache(nextFileNumber); cacheMapping.setDirtyBit(nextCacheBlockSpot, true); - } - else - { + } else { nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); cacheMapping.setDirtyBit(nextCacheBlockSpot, true); } - + //SHOULD FIND MAX SHIFT TOO - while(nextFileNumber < usedCacheBlocks) - { + while (nextFileNumber < usedCacheBlocks) { List cacheBlock = arrayLists.get(cacheBlockSpot); List nextCacheBlock = arrayLists.get(nextCacheBlockSpot); - + //SHOULD FIND MAX SHIFT TOO - while(cacheBlock.size() < getBlockSize() && nextFileNumber < usedCacheBlocks) - { - if(nextCacheBlock.size() > 0) - { + while (cacheBlock.size() < getBlockSize() && nextFileNumber < usedCacheBlocks) { + if (nextCacheBlock.size() > 0) { cacheBlock.add(nextCacheBlock.remove(0)); cacheMapping.removeEntry(nextCacheBlockSpot); cacheMapping.addEntry(cacheBlockSpot); - } - else - { + } else { cacheMapping.updateUsedList(cacheBlockSpot); nextFileNumber++; - + //not reached end of blocks - if(nextFileNumber < usedCacheBlocks) - { + if (nextFileNumber < usedCacheBlocks) { //next file is not in cache - if(!cacheMapping.isFileInCache(nextFileNumber)) - { + if (!cacheMapping.isFileInCache(nextFileNumber)) { nextCacheBlockSpot = cacheMapping.bringFileIntoCache(nextFileNumber); - } - else - { + } else { nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); } - + nextCacheBlock = arrayLists.get(nextCacheBlockSpot); cacheMapping.setDirtyBit(nextCacheBlockSpot, true); } @@ -884,34 +800,26 @@ private void purgeActionBuffer() } fileNumber++; - - if(fileNumber == nextFileNumber) - { + + if (fileNumber == nextFileNumber) { nextFileNumber = fileNumber + 1; } - + //if not at end - if(nextFileNumber < usedCacheBlocks) - { - if(!cacheMapping.isFileInCache(fileNumber)) - { + if (nextFileNumber < usedCacheBlocks) { + if (!cacheMapping.isFileInCache(fileNumber)) { cacheBlockSpot = cacheMapping.bringFileIntoCache(fileNumber); - } - else - { + } else { cacheBlockSpot = cacheMapping.getCacheBlockSpot(fileNumber); cacheMapping.updateUsedList(cacheBlockSpot); } cacheMapping.setDirtyBit(cacheBlockSpot, true); - - if(!cacheMapping.isFileInCache(nextFileNumber)) - { + + if (!cacheMapping.isFileInCache(nextFileNumber)) { nextCacheBlockSpot = cacheMapping.bringFileIntoCache(nextFileNumber); - } - else - { - nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); + } else { + nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); } cacheMapping.setDirtyBit(nextCacheBlockSpot, true); @@ -922,181 +830,146 @@ private void purgeActionBuffer() } } - /** * Purges action queue for all consecutive blocks in cache starting at startIndex - * + * * @param startIndex The block index to start at */ - private void purgeActionBufferInCache(long startIndex) - { - if(softMapping.getBufferSize() > 0) - { + private void purgeActionBufferInCache(long startIndex) { + if (softMapping.getBufferSize() > 0) { boolean done = false; - + //first index int fileNumber = cacheMapping.getFileNumber(startIndex); - int nextFileNumber = fileNumber+1; + int nextFileNumber = fileNumber + 1; long virtualSize = wholeListSize + softMapping.getLastShiftAmount(); int usedCacheBlocks = getNumberOfUsedBlocks(virtualSize); - + int cacheBlockSpot = -1; int nextCacheBlockSpot = -1; - if(!cacheMapping.isFileInCache(fileNumber)) - { + if (!cacheMapping.isFileInCache(fileNumber)) { cacheBlockSpot = cacheMapping.bringFileIntoCache(fileNumber); - } - else - { + } else { cacheBlockSpot = cacheMapping.getCacheBlockSpot(fileNumber); } - + //assume there will be changes, this assumption is not always true? cacheMapping.setDirtyBit(cacheBlockSpot, true); - - if(!cacheMapping.isFileInCache(nextFileNumber)) - { + + if (!cacheMapping.isFileInCache(nextFileNumber)) { done = true; - } - else - { + } else { nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); cacheMapping.setDirtyBit(nextCacheBlockSpot, true); } - + long lastIndexInBlock = cacheMapping.getLastIndexInFile(fileNumber); long lastIndexInNextBlock = cacheMapping.getLastIndexInFile(nextFileNumber); - - while(nextFileNumber < usedCacheBlocks && !done) - { + + while (nextFileNumber < usedCacheBlocks && !done) { long shiftAmount = softMapping.getCurrentShiftAmount(lastIndexInBlock); softMapping.removeShift(lastIndexInBlock); - + //count of shifts done into current cache block long shiftCount = 0; - + //count of shifts done from next cache block long nextShiftCount = 0; - + List cacheBlock = arrayLists.get(cacheBlockSpot); List nextCacheBlock = arrayLists.get(nextCacheBlockSpot); - + //shift down to current block - while(cacheBlock.size() < getBlockSize() && !done) - { - if(nextCacheBlock.size() > 0) - { + while (cacheBlock.size() < getBlockSize() && !done) { + if (nextCacheBlock.size() > 0) { cacheBlock.add(nextCacheBlock.remove(0)); cacheMapping.removeEntry(nextCacheBlockSpot); cacheMapping.addEntry(cacheBlockSpot); shiftCount++; nextShiftCount++; - } - else - { + } else { //next block is empty, reset nextShiftCount - if(nextShiftCount > 0) - { + if (nextShiftCount > 0) { softMapping.addShift(lastIndexInNextBlock, nextShiftCount); nextShiftCount = 0; } nextFileNumber++; - + //not reached end of blocks - if(nextFileNumber < usedCacheBlocks) - { + if (nextFileNumber < usedCacheBlocks) { //next file is not in cache - if(!cacheMapping.isFileInCache(nextFileNumber)) - { + if (!cacheMapping.isFileInCache(nextFileNumber)) { done = true; - } - else - { + } else { nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); nextCacheBlock = arrayLists.get(nextCacheBlockSpot); lastIndexInNextBlock = cacheMapping.getLastIndexInFile(nextFileNumber); cacheMapping.setDirtyBit(nextCacheBlockSpot, true); } - } - else - { + } else { done = true; } } } - + //may have ended loop without doing all shifts for current block //add back in - if(shiftAmount - shiftCount > 0) - { - softMapping.addShift(lastIndexInBlock, shiftAmount-shiftCount); - } - else - { + if (shiftAmount - shiftCount > 0) { + softMapping.addShift(lastIndexInBlock, shiftAmount - shiftCount); + } else { //current block = full - + fileNumber++; - - if(fileNumber == nextFileNumber) - { + + if (fileNumber == nextFileNumber) { nextFileNumber = fileNumber + 1; } - + //if not at end - if(nextFileNumber < usedCacheBlocks) - { + if (nextFileNumber < usedCacheBlocks) { //add remaining shifts to next block //no shifts to add to current block since it was filled softMapping.addShift(lastIndexInNextBlock, nextShiftCount); - - if(!cacheMapping.isFileInCache(fileNumber)) - { + + if (!cacheMapping.isFileInCache(fileNumber)) { done = true; - } - else - { + } else { cacheBlockSpot = cacheMapping.getCacheBlockSpot(fileNumber); lastIndexInBlock = cacheMapping.getLastIndexInFile(fileNumber); cacheMapping.setDirtyBit(cacheBlockSpot, true); } - - if(!cacheMapping.isFileInCache(nextFileNumber)) - { + + if (!cacheMapping.isFileInCache(nextFileNumber)) { done = true; - } - else - { + } else { nextCacheBlockSpot = cacheMapping.getCacheBlockSpot(nextFileNumber); lastIndexInNextBlock = cacheMapping.getLastIndexInFile(nextFileNumber); cacheMapping.setDirtyBit(nextCacheBlockSpot, true); } } - } + } } } } - + /////////////////////////////////////////////////////////////////////////////////////////////////// /** - * Adds an element to the end of the list. - * Analogous to the add method of the ArrayList class - * + * Adds an element to the end of the list. Analogous to the add method of the ArrayList class + * * @param element The element to add * @return If the element was added or not */ - public boolean add(E element) - { + public boolean add(E element) { boolean added = false; long adjustedIndex = softMapping.getAdjustedIndex(wholeListSize); int lastFile = cacheMapping.getFileNumber(adjustedIndex); int cacheBlockSpot = -1; - if(!cacheMapping.isFileInCache(lastFile)) - { + if (!cacheMapping.isFileInCache(lastFile)) { //bring last file into cache cacheMapping.bringFileIntoCache(lastFile); } @@ -1105,21 +978,17 @@ public boolean add(E element) //no -1 check, assumed it was brought in //if last file is not full - if(!cacheMapping.isCacheFull(cacheBlockSpot)) - { + if (!cacheMapping.isCacheFull(cacheBlockSpot)) { //add to last array list and update cache entry added = arrayLists.get(cacheBlockSpot).add(element); - if(added) - { + if (added) { cacheMapping.addEntry(cacheBlockSpot); cacheMapping.setDirtyBit(cacheBlockSpot, true); wholeListSize++; } - - } - else - { + + } else { throw new RuntimeException("Failed to add " + element + " at the end of the list"); } @@ -1127,28 +996,23 @@ public boolean add(E element) } /** - * Gets an element at the specified index. - * Analogous to the get method of the ArrayList class - * + * Gets an element at the specified index. Analogous to the get method of the ArrayList class + * * @param index The index * @return The element */ - public E get(long index) - { - if(index < 0 || index >= wholeListSize) - { + public E get(long index) { + if (index < 0 || index >= wholeListSize) { throw new IndexOutOfBoundsException(" " + index + " "); } - //if index not in cache and not greater than max - //bring corresponding file in cache + //bring corresponding file in cache long adjustedIndex = softMapping.getAdjustedIndex(index); int fileNumber = cacheMapping.getFileNumber(adjustedIndex); - if(!cacheMapping.isFileInCache(fileNumber)) - { + if (!cacheMapping.isFileInCache(fileNumber)) { cacheMapping.bringFileIntoCache(fileNumber); } @@ -1165,107 +1029,95 @@ public E get(long index) /** * Analogous to the get method of the ArrayList class - * + * * @param index The index * @return Returns the element at the specified index */ - public E get(int index) - { + public E get(int index) { long longIndex = index; return get(longIndex); } - - + /** * Analogous to the remove method of the ArrayList class - * + * * @param index The index * @return Returns the element removed at the specified index */ - public E remove(long index) - { - if(index < 0 || index >= wholeListSize) - { + public E remove(long index) { + if (index < 0 || index >= wholeListSize) { throw new IndexOutOfBoundsException(" " + index + " "); } - + //can possibly add something to the buffer //safest place to clear the buffer is here - if(softMapping.isBufferFull() || softMapping.isShiftMaxed()) - { + if (softMapping.isBufferFull() || softMapping.isShiftMaxed()) { purgeActionBuffer(); } - + long adjustedIndex = softMapping.getAdjustedIndex(index); int fileNumber = cacheMapping.getFileNumber(adjustedIndex); - if(!cacheMapping.isFileInCache(fileNumber)) - { + if (!cacheMapping.isFileInCache(fileNumber)) { cacheMapping.bringFileIntoCache(fileNumber); } - + int cacheBlockSpot = cacheMapping.getCacheBlockSpot(fileNumber); int spotInCache = cacheMapping.getSpotInCache(adjustedIndex); long virtualSize = wholeListSize + softMapping.getLastShiftAmount(); int usedCacheBlocks = getNumberOfUsedBlocks(virtualSize); - - List cacheBlock = arrayLists.get(cacheBlockSpot); + + List cacheBlock = arrayLists.get(cacheBlockSpot); E element = cacheBlock.remove(spotInCache); cacheMapping.removeEntry(cacheBlockSpot); cacheMapping.setDirtyBit(cacheBlockSpot, true); - + //need to shift other lists down to the one where an element was just removed - //update SoftMapping for the remove action + //update SoftMapping for the remove action long lastIndexInBlock = cacheMapping.getLastIndexInFile(fileNumber); - + //if this block is the last block, no need to care about unnecessary shifts - if((fileNumber+1) < usedCacheBlocks) - { + if ((fileNumber + 1) < usedCacheBlocks) { softMapping.addShift(lastIndexInBlock, 1); purgeActionBufferInCache(lastIndexInBlock); } wholeListSize--; - + return element; } /** * Analogous to the remove method of the ArrayList class - * + * * @param index The index * @return Returns the element removed at the specified index */ - public E remove(int index) - { + public E remove(int index) { long longIndex = index; return remove(longIndex); } /** - * Sets the element at the specified index - * Analogous to the set method of the ArrayList class - * - * @param index The index + * Sets the element at the specified index Analogous to the set method of the ArrayList class + * + * @param index The index * @param element The new element * @return The new element at the specified index */ - public E set(long index, E element) - { - if(index < 0 || index >= wholeListSize) - { + public E set(long index, E element) { + if (index < 0 || index >= wholeListSize) { throw new IndexOutOfBoundsException(" " + index + " "); } //if index not in cache and not greater than max - //bring corresponding file in cache + //bring corresponding file in cache long adjustedIndex = softMapping.getAdjustedIndex(index); - int fileNumber = cacheMapping.getFileNumber(adjustedIndex); + int fileNumber = cacheMapping.getFileNumber(adjustedIndex); - if(!cacheMapping.isFileInCache(fileNumber)) - { + if (!cacheMapping.isFileInCache(fileNumber)) { cacheMapping.bringFileIntoCache(fileNumber); } @@ -1282,38 +1134,32 @@ public E set(long index, E element) } /** - * Sets the element at the specified index - * Analogous to the set method of the ArrayList class - * - * @param index The index + * Sets the element at the specified index Analogous to the set method of the ArrayList class + * + * @param index The index * @param element The new element * @return The new element at the specified index */ - public E set(int index, E element) - { + public E set(int index, E element) { long longIndex = index; return set(longIndex, element); } - - + /** * Returns if the list is empty. - * + * * @return True is the list is empty, false otherwise */ - public boolean isEmpty() - { + public boolean isEmpty() { boolean empty = true; - - if(size() > 0) - { + + if (size() > 0) { empty = false; } - + return empty; } - //hashCode cannot be implemented correctly due to contents being on disk and out of sight from memory /* (non-Javadoc) @@ -1321,51 +1167,34 @@ public boolean isEmpty() */ @SuppressWarnings("rawtypes") @Override - public boolean equals(Object otherObject) - { - + public boolean equals(Object otherObject) { + boolean isEqual = true; - - if(otherObject == null) - { + + if (otherObject == null) { isEqual = false; - } - else if (this == otherObject) - { + } else if (this == otherObject) { isEqual = true; - } - else if(!(otherObject instanceof BigArrayList)) - { + } else if (!(otherObject instanceof BigArrayList)) { isEqual = false; - } - else - { + } else { BigArrayList otherBigArrayList = (BigArrayList) otherObject; - - if(wholeListSize != otherBigArrayList.size()) - { + + if (wholeListSize != otherBigArrayList.size()) { isEqual = false; - } - else if(liveObject != otherBigArrayList.isLive()) - { + } else if (liveObject != otherBigArrayList.isLive()) { isEqual = false; - } - else if(ioType != otherBigArrayList.ioType) - { + } else if (ioType != otherBigArrayList.ioType) { isEqual = false; - } - else - { - for(long i=0; i. -*/ + */ package com.dselent.bigarraylist; import java.io.IOException; import java.io.Serializable; - /** - * Class that manages the mapping from files on disk to elements in memory for the BigArrayList class. - * Uses an LRU policy at the cache block level to determine which cache block should be swapped out next. - * - * @author Douglas Selent + * Class that manages the mapping from files on disk to elements in memory for the BigArrayList class. Uses an LRU + * policy at the cache block level to determine which cache block should be swapped out next. * * @param Generic type + * @author Douglas Selent */ -class CacheMapping -{ +class CacheMapping { /** * Array storing the next spot to add to for each cache @@ -40,22 +37,19 @@ class CacheMapping private int[] cacheTableSpots; /** - * The file number each ArrayList/Cache block is currently storing - * Index is the same as the ArrayList index + * The file number each ArrayList/Cache block is currently storing Index is the same as the ArrayList index */ private int[] cacheTableFiles; /** - * Array of when each block was last used - * Stores a list of block numbers sorted from least recently used to most recently used - * most recent = end of array + * Array of when each block was last used Stores a list of block numbers sorted from least recently used to most + * recently used most recent = end of array */ private int[] mostRecentlyUsedList; /** - * Array for each cache block for whether or not it's data has changed - * If clean, then it does not need to be written to disk when swapped out - * If dirty, then it does need to be written to disk when swapped out + * Array for each cache block for whether or not it's data has changed If clean, then it does not need to be written + * to disk when swapped out If dirty, then it does need to be written to disk when swapped out */ private boolean[] dirtyBits; @@ -63,62 +57,57 @@ class CacheMapping * Reference to the associated BigArrayList object */ private BigArrayList bigArrayList; - + /** * Reference to the associated FileAccessor object */ private FileAccessor fileAccessor; - /** * Constructs a CacheMapping object for the BigArrayList with the following parameters - * - * @param blockSize Size of each cache block + * + * @param blockSize Size of each cache block * @param cacheBlocks Number of cache blocks - * @param theList Associated BigArrayList - * @param memoryPath The file path to store contents on disk + * @param theList Associated BigArrayList + * @param memoryPath The file path to store contents on disk */ - protected CacheMapping(BigArrayList theList, int blockSize, int cacheBlocks, String memoryPath) - { + protected CacheMapping(BigArrayList theList, int blockSize, int cacheBlocks, String memoryPath) { cacheTableSpots = new int[cacheBlocks]; cacheTableFiles = new int[cacheBlocks]; mostRecentlyUsedList = new int[cacheBlocks]; dirtyBits = new boolean[cacheBlocks]; - for(int i=0; i(memoryPath); } - + /** * Constructs a CacheMapping object for the BigArrayList with the following parameters - * - * @param blockSize Size of each cache block + * + * @param blockSize Size of each cache block * @param cacheBlocks Number of cache blocks - * @param theList Associated BigArrayList + * @param theList Associated BigArrayList */ - protected CacheMapping(BigArrayList theList, int blockSize, int cacheBlocks) - { + protected CacheMapping(BigArrayList theList, int blockSize, int cacheBlocks) { cacheTableSpots = new int[cacheBlocks]; cacheTableFiles = new int[cacheBlocks]; mostRecentlyUsedList = new int[cacheBlocks]; dirtyBits = new boolean[cacheBlocks]; - for(int i=0; i(); } @@ -128,33 +117,27 @@ protected CacheMapping(BigArrayList theList, int blockSize, int cacheBlocks) /** * @return Returns the fileAccessor object */ - protected FileAccessor getFileAccessor() - { + protected FileAccessor getFileAccessor() { return fileAccessor; } - - - /** * Sets the index to add the next element to for the given cache block - * + * * @param cacheBlockIndex The index for the cache block - * @param indexToAdd The next index to add to for the specified cache block + * @param indexToAdd The next index to add to for the specified cache block */ - private void setCacheTableSpots(int cacheBlockIndex, int indexToAdd) - { + private void setCacheTableSpots(int cacheBlockIndex, int indexToAdd) { cacheTableSpots[cacheBlockIndex] = indexToAdd; } /** * Sets the mapping for cache blocks that are in memory to the file number on disk - * - * @param index The index of the cache block + * + * @param index The index of the cache block * @param fileNumber The file number */ - private void setCacheTableFiles(int index, int fileNumber) - { + private void setCacheTableFiles(int index, int fileNumber) { cacheTableFiles[index] = fileNumber; } @@ -162,12 +145,10 @@ private void setCacheTableFiles(int index, int fileNumber) * @param block The cache block * @return Returns if the cache block is full or not */ - protected boolean isCacheFull(int block) - { + protected boolean isCacheFull(int block) { boolean full = false; - if(cacheTableSpots[block] >= bigArrayList.getBlockSize()) - { + if (cacheTableSpots[block] >= bigArrayList.getBlockSize()) { full = true; } @@ -176,61 +157,53 @@ protected boolean isCacheFull(int block) /** * Sets the dirty bit for the given cache block index - * + * * @param blockIndex Index of the cache block - * @param dirty Whether or not the block of cache is dirty + * @param dirty Whether or not the block of cache is dirty */ - protected void setDirtyBit(int blockIndex, boolean dirty) - { + protected void setDirtyBit(int blockIndex, boolean dirty) { dirtyBits[blockIndex] = dirty; } /** - * Called by the add method of BigArrayList - * Updates meta data associated with adding an element - * + * Called by the add method of BigArrayList Updates meta data associated with adding an element + * * @param cacheBlockIndex Index of the cache block */ - protected void addEntry(int cacheBlockIndex) - { + protected void addEntry(int cacheBlockIndex) { cacheTableSpots[cacheBlockIndex]++; updateUsedList(cacheBlockIndex); } - + /** - * Called by the remove method of BigArrayList - * Updates meta data associated with removing an element - * + * Called by the remove method of BigArrayList Updates meta data associated with removing an element + * * @param cacheBlockIndex Index of the cache block */ - protected void removeEntry(int cacheBlockIndex) - { + protected void removeEntry(int cacheBlockIndex) { cacheTableSpots[cacheBlockIndex]--; updateUsedList(cacheBlockIndex); } /** - * * @param index The index of the element * @return Returns the file number where the element at this index would be */ - protected int getFileNumber(long index) - { + protected int getFileNumber(long index) { long blockSizeLong = bigArrayList.getBlockSize(); long longFileNumber = index / blockSizeLong; - + //safe cast, I really doubt there will ever be over 2^31 - 1 files - return (int)longFileNumber; + return (int) longFileNumber; } - + /** * Returns the last element index for the given file number - * + * * @param fileNumber Number of the file * @return Last element index in the file */ - protected long getLastIndexInFile(int fileNumber) - { + protected long getLastIndexInFile(int fileNumber) { long blockSizeLong = bigArrayList.getBlockSize(); long fileNumberLong = fileNumber; long index = (blockSizeLong * fileNumberLong) + blockSizeLong - 1; @@ -239,18 +212,15 @@ protected long getLastIndexInFile(int fileNumber) /** * Returns the cacheTableFiles spot where the current file/cache block is being held - * + * * @param fileNumber The file number * @return Returns the cacheTableFiles spot where the current file/cache block is being held */ - protected int getCacheBlockSpot(int fileNumber) - { + protected int getCacheBlockSpot(int fileNumber) { int blockSpot = -1; - for(int i=0; i 2^31 - 1 elements in an ArrayList - return (int)spotInFile; + return (int) spotInFile; } - + /** * Returns if the file is in cache or not - * + * * @param fileNumber The file number index * @return Returns true if the contents of the file are in cache and false otherwise */ - protected boolean isFileInCache(int fileNumber) - { + protected boolean isFileInCache(int fileNumber) { boolean inCache = false; - for(int i=0; i= 0) + System.arraycopy(mostRecentlyUsedList, shiftPosition + 1, mostRecentlyUsedList, shiftPosition, mostRecentlyUsedList.length - 1 - shiftPosition); mostRecentlyUsedList[newPosition] = blockNumber; @@ -365,12 +319,9 @@ protected void updateUsedList(int blockNumber) /** * @param blockIndex Index to remove from the list */ - private void removeFromUsedList(int blockIndex) - { - for(int i=0; i. -*/ + */ package com.dselent.bigarraylist; -import java.io.*; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; - import org.nustaq.serialization.FSTConfiguration; import org.nustaq.serialization.FSTObjectInput; import org.nustaq.serialization.FSTObjectOutput; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.RandomAccessFile; +import java.io.Serializable; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; //FST //https://github.com/RuedigerMoeller/fast-serialization/wiki/Serialization /** - * Class that reads and writes the contents of the BigArrayList to/from disk. - * Each BigArrayList is given a unique identifier so multiple BigArrayList objects can be used at once. + * Class that reads and writes the contents of the BigArrayList to/from disk. Each BigArrayList is given a unique + * identifier so multiple BigArrayList objects can be used at once. *

* The files are names with the following convention *

* (filePath)(memory instance)"_memory_"(file/block number)(file extension) *

- * Examples:
- * "memory\0_memory_0.jobj
- * "memory\0_memory_1.jobj
- * - * @author Douglas Selent + * Examples:
"memory\0_memory_0.jobj
"memory\0_memory_1.jobj
* * @param Generic type + * @author Douglas Selent */ -class FileAccessor -{ +class FileAccessor { /** * Unique identifier for the files associated with a specific instance of BigArrayList */ private static int nextMemoryInstance = 0; - + /** * Default folder path string = "memory" */ private final String DEFAULT_MEMORY_FILE_PATH = "memory"; - + /** * Default file extension = ".jobj" */ private final String DEFAULT_MEMORY_FILE_EXTENSION = ".jobj"; - + /** * Default buffer size for file I/O = 262,144 bytes */ @@ -82,53 +88,47 @@ class FileAccessor * The File object holding the folder path for the memory contents */ private final File memoryFolder; - + /** * A string for the file path */ private String memoryPath; - + /** * File extension */ private String memoryExtension; - + /** * Memory instance for this object */ private int memoryInstance; - + /** * FST configuration for using FST object serialization */ private static FSTConfiguration fstConfiguration; - + /** * Constructs a FileAccessor object with the default file path to store contents on disk */ - public FileAccessor() - { + public FileAccessor() { memoryPath = DEFAULT_MEMORY_FILE_PATH; memoryExtension = DEFAULT_MEMORY_FILE_EXTENSION; memoryFolder = new File(memoryPath); - try - { - if(!memoryFolder.exists()) - { + try { + if (!memoryFolder.exists()) { memoryFolder.mkdir(); } memoryInstance = findMemoryInstance(); - } - catch(Exception e) - { + } catch (Exception e) { e.printStackTrace(); System.exit(-1); } - - if(fstConfiguration == null) - { + + if (fstConfiguration == null) { fstConfiguration = FSTConfiguration.createDefaultConfiguration(); } @@ -136,50 +136,42 @@ public FileAccessor() /** * Constructs a FileAccessor object with the specified file path to store contents on disk - * + * * @param memoryPath The folder path to read and write to */ - public FileAccessor(String memoryPath) - { + public FileAccessor(String memoryPath) { this.memoryPath = memoryPath; memoryExtension = DEFAULT_MEMORY_FILE_EXTENSION; memoryFolder = new File(memoryPath); - try - { - if(!memoryFolder.exists()) - { + try { + if (!memoryFolder.exists()) { memoryFolder.mkdir(); } memoryInstance = findMemoryInstance(); - } - catch(Exception e) - { + } catch (Exception e) { e.printStackTrace(); System.exit(-1); } - - if(fstConfiguration == null) - { + + if (fstConfiguration == null) { fstConfiguration = FSTConfiguration.createDefaultConfiguration(); } } /** * Returns if the file exists or not - * + * * @param fileNumber The file number * @return Returns true if the file with the given file number exists false otherwise */ - protected boolean doesFileExist(int fileNumber) - { + protected boolean doesFileExist(int fileNumber) { boolean exists = false; File file = new File(memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension); - if(file.exists()) - { + if (file.exists()) { exists = true; } @@ -188,49 +180,41 @@ protected boolean doesFileExist(int fileNumber) /** * Finds a unique instance number for this FileAccessor object - * + *

* Searches numbers in ascending order to find the corresponding named file that does not exist. + * * @return Returns the unique memory instance that this FileAccessor object can use */ - private int findMemoryInstance() - { + private int findMemoryInstance() { int memoryInstanceNumber = nextMemoryInstance; File memoryFile = new File(memoryPath + File.separator + memoryInstanceNumber + "_memory_" + "0" + memoryExtension); - while(memoryFile.exists()) - { + while (memoryFile.exists()) { memoryInstanceNumber++; memoryFile = new File(memoryPath + File.separator + memoryInstanceNumber + "_memory_" + "0" + memoryExtension); } - nextMemoryInstance = memoryInstanceNumber+1; + nextMemoryInstance = memoryInstanceNumber + 1; return memoryInstanceNumber; } /** * Creates a file with the given file number - * + * * @param fileNumber The file number */ - protected void createFile(int fileNumber) - { + protected void createFile(int fileNumber) { int memoryInstance = findMemoryInstance(); File file = new File(memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension); - try - { - if(!file.exists()) - { + try { + if (!file.exists()) { file.createNewFile(); - } - else - { + } else { throw new Exception("File already exists " + file.toString()); } - } - catch(Exception e) - { + } catch (Exception e) { e.printStackTrace(); System.exit(-1); } @@ -238,481 +222,394 @@ protected void createFile(int fileNumber) /** * Deletes the file with the given path - * + * * @param filePath The file path */ - private void deleteFile(String filePath) - { + private void deleteFile(String filePath) { File file = new File(filePath); - if(file.exists()) - { + if (file.exists()) { file.delete(); } } /** * Deletes the file with the given number - * + * * @param fileNumber The file number */ - protected void deleteFile(int fileNumber) - { + protected void deleteFile(int fileNumber) { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; deleteFile(filePath); } /** * Returns the file path where the BigArrayList is writing to disk - * + * * @return Returns the file path where the BigArrayList is writing to disk */ - protected String getMemoryFilePath() - { + protected String getMemoryFilePath() { return memoryPath; } - /** - * Reads the contents of a cache block into memory using buffered I/O with standard object streams - * Reads in the size (number of elements) as the first object - * + * Reads the contents of a cache block into memory using buffered I/O with standard object streams Reads in the size + * (number of elements) as the first object + * * @param fileNumber The file to read from - * @param cacheSpot The location in cache to read into (the ArrayList index) - * @param arrayList The BigArrayList object - * @throws IOException For I/O errors + * @param cacheSpot The location in cache to read into (the ArrayList index) + * @param arrayList The BigArrayList object + * @throws IOException For I/O errors * @throws ClassNotFoundException If no such class exists */ @SuppressWarnings("unchecked") //must use unchecked warning because ObjectInputStream doesn't use generic typing - protected void readFromFileObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException - { + protected void readFromFileObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; File file = new File(filePath); - - if(file.exists()) - { + + if (file.exists()) { FileInputStream fileInputStream = new FileInputStream(filePath); BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream, BUFFER_SIZE); ObjectInputStream objectInputStream = new ObjectInputStream(bufferedInputStream); - - try - { - arrayList.setList(cacheSpot, (ArrayList)objectInputStream.readObject()); - } - catch(IOException ioe) - { + + try { + arrayList.setList(cacheSpot, (ArrayList) objectInputStream.readObject()); + } catch (IOException ioe) { throw ioe; - } - catch (ClassNotFoundException ce) - { + } catch (ClassNotFoundException ce) { throw ce; - } - finally - { + } finally { //explicitly closing all streams to be safe fileInputStream.close(); bufferedInputStream.close(); objectInputStream.close(); } - } - else - { + } else { arrayList.setList(cacheSpot, new ArrayList()); } } - + /** - * Reads the contents of a cache block into memory using memory mapped files with object streams - * Reads in the size (number of elements) as the first object + * Reads the contents of a cache block into memory using memory mapped files with object streams Reads in the size + * (number of elements) as the first object + * * @param fileNumber The file to read from - * @param cacheSpot The location in cache to read into (the ArrayList index) - * @param arrayList The BigArrayList object - * @throws IOException For I/O errors + * @param cacheSpot The location in cache to read into (the ArrayList index) + * @param arrayList The BigArrayList object + * @throws IOException For I/O errors * @throws ClassNotFoundException If no such class exists */ //must clean up byte buffer @SuppressWarnings("unchecked") - protected void readFromFileMMapObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException - { - String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; - File file = new File(filePath); - - if(file.exists()) - { - RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); - - long tempFileLength = tempFile.length(); - int fileLength = -1; - - if(tempFile.length() > Integer.MAX_VALUE) - { + protected void readFromFileMMapObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException { + String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; + File file = new File(filePath); + + if (file.exists()) { + RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); + + long tempFileLength = tempFile.length(); + int fileLength = -1; + + if (tempFile.length() > Integer.MAX_VALUE) { + tempFile.close(); + throw new IllegalArgumentException(tempFileLength + " cannot be cast to an int"); + } else { + fileLength = (int) tempFileLength; + + MappedByteBuffer tempByteBuffer = tempFile.getChannel() + .map(FileChannel.MapMode.READ_ONLY, 0, tempFile.length()); + tempByteBuffer.load(); + + byte[] byteArray = new byte[fileLength]; + tempByteBuffer.get(byteArray); + + ByteArrayInputStream bais = new ByteArrayInputStream(byteArray); + ObjectInputStream objectInputStream = new ObjectInputStream(bais); + + try { + arrayList.setList(cacheSpot, (ArrayList) objectInputStream.readObject()); + } catch (IOException ioe) { + throw ioe; + } finally { + + bais.close(); + objectInputStream.close(); + + tempByteBuffer.clear(); + tempByteBuffer = null; tempFile.close(); - throw new IllegalArgumentException(tempFileLength + " cannot be cast to an int"); - } - else - { - fileLength = (int) tempFileLength; - - MappedByteBuffer tempByteBuffer = tempFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, tempFile.length()); - tempByteBuffer.load(); - - byte[] byteArray = new byte[fileLength]; - tempByteBuffer.get(byteArray); - - ByteArrayInputStream bais = new ByteArrayInputStream(byteArray); - ObjectInputStream objectInputStream = new ObjectInputStream(bais); - - try - { - arrayList.setList(cacheSpot, (ArrayList)objectInputStream.readObject()); - } - catch(IOException ioe) - { - throw ioe; - } - finally - { - - bais.close(); - objectInputStream.close(); - - tempByteBuffer.clear(); - tempByteBuffer = null; - tempFile.close(); - System.gc(); - } + System.gc(); } } - else - { - arrayList.setList(cacheSpot, new ArrayList()); - } + } else { + arrayList.setList(cacheSpot, new ArrayList()); + } } - + /** * Reads a cache block from disk using FST object streams - * + * * @param fileNumber The file number to read from - * @param cacheSpot The block to write to disk - * @param arrayList The BigArrayList - * @throws IOException For I/O errors + * @param cacheSpot The block to write to disk + * @param arrayList The BigArrayList + * @throws IOException For I/O errors * @throws ClassNotFoundException If no such class exists */ @SuppressWarnings("unchecked") - protected void readFromFileFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException - { + protected void readFromFileFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; File file = new File(filePath); - - if(file.exists()) - { + + if (file.exists()) { FileInputStream fileInputStream = new FileInputStream(filePath); FSTObjectInput fstObjectInputStream = new FSTObjectInput(fileInputStream); - - try - { - arrayList.setList(cacheSpot, (ArrayList)fstObjectInputStream.readObject()); - } - catch(IOException ioe) - { + + try { + arrayList.setList(cacheSpot, (ArrayList) fstObjectInputStream.readObject()); + } catch (IOException ioe) { throw ioe; - } - catch (ClassNotFoundException ce) - { + } catch (ClassNotFoundException ce) { throw ce; - } - finally - { + } finally { //need to close both streams, I think this is a java bug //expected behavior = closing the fstObjectInputStream would also close the fileInputStream, but it does not //Consequence = unable to delete files fileInputStream.close(); fstObjectInputStream.close(); } - } - else - { + } else { arrayList.setList(cacheSpot, new ArrayList()); } } - + /** - * Reads the contents of a cache block into memory using memory mapped files with FST serialization - * Reads in the size (number of elements) as the first object - * + * Reads the contents of a cache block into memory using memory mapped files with FST serialization Reads in the + * size (number of elements) as the first object + * * @param fileNumber The file to read from - * @param cacheSpot The location in cache to read into (the ArrayList index) - * @param arrayList The BigArrayList object - * @throws IOException For I/O errors + * @param cacheSpot The location in cache to read into (the ArrayList index) + * @param arrayList The BigArrayList object + * @throws IOException For I/O errors * @throws ClassNotFoundException If no such class exists */ //must clean up byte buffer @SuppressWarnings("unchecked") - protected void readFromFileMMapFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException - { - String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; - File file = new File(filePath); - - if(file.exists()) - { - RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); - - long tempFileLength = tempFile.length(); - int fileLength = -1; - - if(tempFile.length() > Integer.MAX_VALUE) - { + protected void readFromFileMMapFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException, ClassNotFoundException { + String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; + File file = new File(filePath); + + if (file.exists()) { + RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); + + long tempFileLength = tempFile.length(); + int fileLength = -1; + + if (tempFile.length() > Integer.MAX_VALUE) { + tempFile.close(); + throw new IllegalArgumentException(tempFileLength + " cannot be cast to an int"); + } else { + fileLength = (int) tempFileLength; + + MappedByteBuffer tempByteBuffer; + + try { + tempByteBuffer = tempFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, tempFile.length()); + tempByteBuffer.load(); + + byte[] byteArray = new byte[fileLength]; + tempByteBuffer.get(byteArray); + arrayList.setList(cacheSpot, (ArrayList) fstConfiguration.asObject(byteArray)); + + tempByteBuffer.clear(); + } catch (IOException ioe) { + throw ioe; + } finally { + + tempByteBuffer = null; tempFile.close(); - throw new IllegalArgumentException(tempFileLength + " cannot be cast to an int"); - } - else - { - fileLength = (int) tempFileLength; - - MappedByteBuffer tempByteBuffer; - - try - { - tempByteBuffer = tempFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, tempFile.length()); - tempByteBuffer.load(); - - byte[] byteArray = new byte[fileLength]; - tempByteBuffer.get(byteArray); - arrayList.setList(cacheSpot, (ArrayList)fstConfiguration.asObject(byteArray)); - - tempByteBuffer.clear(); - } - catch(IOException ioe) - { - throw ioe; - } - finally - { - - tempByteBuffer = null; - tempFile.close(); - System.gc(); - } + System.gc(); } } - else - { - arrayList.setList(cacheSpot, new ArrayList()); - } + } else { + arrayList.setList(cacheSpot, new ArrayList()); + } } /** * Writes a cache block to disk using buffered I/O with standard object streams - * + * * @param fileNumber The file number to write to - * @param cacheSpot The block to write to disk - * @param arrayList The BigArrayList + * @param cacheSpot The block to write to disk + * @param arrayList The BigArrayList * @throws IOException For I/O errors */ - protected void writeToFileObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException - { + protected void writeToFileObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; File tempFile = new File(filePath); - - if(!arrayList.getList(cacheSpot).isEmpty()) - { + + if (!arrayList.getList(cacheSpot).isEmpty()) { tempFile.deleteOnExit(); - + FileOutputStream fileOutputStream = new FileOutputStream(filePath); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream, BUFFER_SIZE); ObjectOutputStream objectOutputStream = new ObjectOutputStream(bufferedOutputStream); - - try - { + + try { objectOutputStream.writeObject(arrayList.getList(cacheSpot)); - objectOutputStream.flush(); - } - catch(IOException ioe) - { + objectOutputStream.flush(); + } catch (IOException ioe) { throw ioe; - } - finally - { + } finally { //explicitly closing all streams to be safe fileOutputStream.close(); bufferedOutputStream.close(); objectOutputStream.close(); } - } - else - { + } else { tempFile.delete(); } - } - + } + /** * Writes a cache block to disk using memory mapped files with object streams *

- * Will not automatically delete files when the program ends. Programmer must call {@link com.dselent.bigarraylist.BigArrayList#clearMemory} - * + * Will not automatically delete files when the program ends. Programmer must call {@link + * com.dselent.bigarraylist.BigArrayList#clearMemory} + * * @param fileNumber The file number to write to - * @param cacheSpot The block to write to disk - * @param arrayList The BigArrayList + * @param cacheSpot The block to write to disk + * @param arrayList The BigArrayList * @throws IOException For I/O errors */ //must clean up byte buffer and suggest to garbage collect it or else the files cannot be deleted - protected void writeToFileMMapObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException - { + protected void writeToFileMMapObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; - if(!arrayList.getList(cacheSpot).isEmpty()) - { - RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); - + if (!arrayList.getList(cacheSpot).isEmpty()) { + RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream objectOutputStream = new ObjectOutputStream(baos); MappedByteBuffer tempByteBuffer; - - try - { + + try { objectOutputStream.writeObject(arrayList.getList(cacheSpot)); objectOutputStream.flush(); - + byte[] byteArray = baos.toByteArray(); tempByteBuffer = tempFile.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, byteArray.length); tempByteBuffer.put(byteArray); tempByteBuffer.force(); tempByteBuffer.clear(); - } - catch(IOException ioe) - { + } catch (IOException ioe) { throw ioe; - } - finally - { + } finally { objectOutputStream.close(); - + tempByteBuffer = null; tempFile.close(); System.gc(); } - } - else - { + } else { File tempFile = new File(filePath); tempFile.delete(); } } - + /** * Writes a cache block to disk using FST object output streams - * + * * @param fileNumber The file number to write to - * @param cacheSpot The block to write to disk - * @param arrayList The BigArrayList + * @param cacheSpot The block to write to disk + * @param arrayList The BigArrayList * @throws IOException For I/O errors */ - protected void writeToFileFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException - { + protected void writeToFileFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; File tempFile = new File(filePath); - - if(!arrayList.getList(cacheSpot).isEmpty()) - { + + if (!arrayList.getList(cacheSpot).isEmpty()) { tempFile.deleteOnExit(); - + FileOutputStream fileOutputStream = new FileOutputStream(filePath); FSTObjectOutput fstObjectOutputStream = new FSTObjectOutput(fileOutputStream); - try - { - byte byteArray[] = fstConfiguration.asByteArray(arrayList.getList(cacheSpot)); + try { + byte[] byteArray = fstConfiguration.asByteArray(arrayList.getList(cacheSpot)); fstObjectOutputStream.write(byteArray); fstObjectOutputStream.flush(); - } - catch(IOException ioe) - { + } catch (IOException ioe) { throw ioe; - } - finally - { + } finally { fileOutputStream.close(); fstObjectOutputStream.close(); } - } - else - { + } else { tempFile.delete(); } } - /** * Writes a cache block to disk using memory mapped files with FST object output streams *

- * Will not automatically delete files when the program ends. Programmer must call {@link com.dselent.bigarraylist.BigArrayList#clearMemory} - * + * Will not automatically delete files when the program ends. Programmer must call {@link + * com.dselent.bigarraylist.BigArrayList#clearMemory} + * * @param fileNumber The file number to write to - * @param cacheSpot The block to write to disk - * @param arrayList The BigArrayList + * @param cacheSpot The block to write to disk + * @param arrayList The BigArrayList * @throws IOException For I/O errors */ - protected void writeToFileMMapFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException - { + protected void writeToFileMMapFSTObject(int fileNumber, int cacheSpot, BigArrayList arrayList) throws IOException { String filePath = memoryPath + File.separator + memoryInstance + "_memory_" + fileNumber + memoryExtension; - if(!arrayList.getList(cacheSpot).isEmpty()) - { + if (!arrayList.getList(cacheSpot).isEmpty()) { RandomAccessFile tempFile = new RandomAccessFile(filePath, "rw"); MappedByteBuffer tempByteBuffer; - - try - { - byte byteArray[] = fstConfiguration.asByteArray(arrayList.getList(cacheSpot)); + + try { + byte[] byteArray = fstConfiguration.asByteArray(arrayList.getList(cacheSpot)); tempByteBuffer = tempFile.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, byteArray.length); tempByteBuffer.put(byteArray); tempByteBuffer.force(); tempByteBuffer.clear(); - } - catch(IOException ioe) - { + } catch (IOException ioe) { throw ioe; - } - finally - { + } finally { tempByteBuffer = null; tempFile.close(); System.gc(); } - } - else - { + } else { File tempFile = new File(filePath); tempFile.delete(); } } - + /** * Deletes all files associated with the current BigArrayList object + * * @throws IOException When the file cannot be deleted */ - protected void clearMemory() throws IOException - { + protected void clearMemory() throws IOException { //get all files associated with this memory instance and delete them File[] fileList = memoryFolder.listFiles(); - for(int i=0; i. -*/ + */ package com.dselent.bigarraylist; import java.nio.BufferOverflowException; import java.util.Arrays; - //why are negative numbers being stored? /** - * Class that manages the mapping from virtual indices to physical indices in memory - * Since the remove operation is coalesced some elements are not physically removed until later. - * This means a virtual index must be mapped to the physical index of the desired element. - * For example if element 3 is virtually removed, the virtual index for the new element 3 maps to the - * physical index of 4, where element 3 is currently stored. - * - * @author Douglas Selent + * Class that manages the mapping from virtual indices to physical indices in memory Since the remove operation is + * coalesced some elements are not physically removed until later. This means a virtual index must be mapped to the + * physical index of the desired element. For example if element 3 is virtually removed, the virtual index for the new + * element 3 maps to the physical index of 4, where element 3 is currently stored. * * @param Generic type + * @author Douglas Selent */ -class SoftMapping -{ +class SoftMapping { /** * Minimum buffer size, or the minimum number of unique shift operations that can be stored */ private static final int MIN_BUFFER_SIZE = 0; - + /** * Maximum buffer size, or the maximum number of unique shift operations that can be stored */ private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE; - + //not allowing users to specify their own sizes yet, just to make it simpler for them //size of number of unique shift places that can be stored - + /** * Default buffer size, or the default number of unique shift operations that can be stored */ private static final int DEFAULT_BUFFER_SIZE = 10000; - + //number of cumulative shifts that can be stored //elements that have been removed but not deleted from disk //this is to limit the amount of disk space wasted - + /** * Default number of shifts that can be stored */ private static final int DEFAULT_SHIFT_ELEMENT_SIZE = 10000; - + /** * Size of the first dimension of the shift buffer (number of columns / pieces of information to store) */ private static final int INFORMATION_PIECES = 3; - + /** * Stores the information related to the virtual index to physical mapping in terms of shift offsets. - * - * 0 = current index number - * 1 = current shift amount for the index - * 2 = cumulative shift amount index + *

+ * 0 = current index number 1 = current shift amount for the index 2 = cumulative shift amount index */ private long[][] shiftBuffer; @@ -83,315 +77,257 @@ class SoftMapping * Current size of the shift buffer */ private int currentBufferSize; - + /** * Default constructor */ - protected SoftMapping() - { + protected SoftMapping() { shiftBuffer = new long[INFORMATION_PIECES][DEFAULT_BUFFER_SIZE]; currentBufferSize = 0; } /** * Constructor that takes the number of unique shift operations that can be stored - * + * * @param bufferSize The size of unique shifts that can be stored */ - protected SoftMapping(int bufferSize) - { - if(bufferSize < MIN_BUFFER_SIZE || bufferSize > MAX_BUFFER_SIZE) - { + protected SoftMapping(int bufferSize) { + if (bufferSize < MIN_BUFFER_SIZE || bufferSize > MAX_BUFFER_SIZE) { throw new IllegalArgumentException("Buffer size is " + bufferSize + " but must be >= " + MIN_BUFFER_SIZE + " and <= " + MAX_BUFFER_SIZE); } shiftBuffer = new long[INFORMATION_PIECES][bufferSize]; currentBufferSize = 0; } - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - + /** * @return Returns the current size of the shift buffer */ - protected int getBufferSize() - { + protected int getBufferSize() { return currentBufferSize; } - + /** * @param bufferIndex Index of the last element for a cache block * @return Returns the number of shifts at the given index */ - protected long getShiftIndex(int bufferIndex) - { + protected long getShiftIndex(int bufferIndex) { return shiftBuffer[0][bufferIndex]; } - + /** * @return Returns the total number of shifts in the buffer */ - protected long getLastShiftAmount() - { + protected long getLastShiftAmount() { long lastShiftAmount = 0; - - if(currentBufferSize > 0) - { - lastShiftAmount = shiftBuffer[2][currentBufferSize-1]; + + if (currentBufferSize > 0) { + lastShiftAmount = shiftBuffer[2][currentBufferSize - 1]; } - + return lastShiftAmount; } - + //whenever skipping over a cache block with shifts, need to add current shift of that block + /** * Calculates the physical index for the given virtual index - * + * * @param adjustedIndex Virtual index * @return Returns the physical index */ - protected long getAdjustedIndex(long adjustedIndex) - { - if(currentBufferSize != 0) - { + protected long getAdjustedIndex(long adjustedIndex) { + if (currentBufferSize != 0) { boolean done = false; boolean firstPass = true; int previousIndex = -1; - - while(!done) - { - int searchIndex = Arrays.binarySearch(shiftBuffer[0], 0, currentBufferSize, adjustedIndex); - - if(firstPass) - { - if(searchIndex >= 0) - { + + while (!done) { + int searchIndex = Arrays.binarySearch(shiftBuffer[0], 0, currentBufferSize, adjustedIndex); + + if (firstPass) { + if (searchIndex >= 0) { adjustedIndex = adjustedIndex + shiftBuffer[2][searchIndex]; previousIndex = searchIndex; - } - else if(searchIndex == -1) - { + } else if (searchIndex == -1) { //if end index of first block - block shifts < index - if(shiftBuffer[0][0] - shiftBuffer[1][0] < adjustedIndex) - { + if (shiftBuffer[0][0] - shiftBuffer[1][0] < adjustedIndex) { adjustedIndex = adjustedIndex + shiftBuffer[2][0]; previousIndex = 0; - } - else - { + } else { done = true; - } - } - else - { - //if end index of next block - next block shifts < index - if(currentBufferSize > (-searchIndex)-1 && shiftBuffer[0][(-searchIndex)-1] - shiftBuffer[2][(-searchIndex)-1] < adjustedIndex) - { - adjustedIndex = adjustedIndex + shiftBuffer[2][(-searchIndex)-1]; - previousIndex = (-searchIndex)-1; } - else - { - adjustedIndex = adjustedIndex + shiftBuffer[2][(-searchIndex)-2]; + } else { + //if end index of next block - next block shifts < index + if (currentBufferSize > (-searchIndex) - 1 && shiftBuffer[0][(-searchIndex) - 1] - shiftBuffer[2][(-searchIndex) - 1] < adjustedIndex) { + adjustedIndex = adjustedIndex + shiftBuffer[2][(-searchIndex) - 1]; + previousIndex = (-searchIndex) - 1; + } else { + adjustedIndex = adjustedIndex + shiftBuffer[2][(-searchIndex) - 2]; //previousIndex = (-searchIndex)-2; done = true; } - + } - + firstPass = false; - } - else - { - if(searchIndex >= 0) - { + } else { + if (searchIndex >= 0) { //add previous shifts and current - for(int i=previousIndex+1; i<=searchIndex; i++) - { + for (int i = previousIndex + 1; i <= searchIndex; i++) { adjustedIndex = adjustedIndex + shiftBuffer[1][i]; } previousIndex = searchIndex; - } - else - { + } else { //add previous shifts - for(int i=previousIndex+1; i<(-searchIndex)-1; i++) - { + for (int i = previousIndex + 1; i < (-searchIndex) - 1; i++) { adjustedIndex = adjustedIndex + shiftBuffer[1][i]; } - + //if end index of next block - next block shifts < index - if(currentBufferSize > (-searchIndex)-1 && shiftBuffer[0][(-searchIndex)-1] - shiftBuffer[1][(-searchIndex)-1] < adjustedIndex) - { - adjustedIndex = adjustedIndex + shiftBuffer[1][(-searchIndex)-1]; - previousIndex = (-searchIndex)-1; - } - else - { + if (currentBufferSize > (-searchIndex) - 1 && shiftBuffer[0][(-searchIndex) - 1] - shiftBuffer[1][(-searchIndex) - 1] < adjustedIndex) { + adjustedIndex = adjustedIndex + shiftBuffer[1][(-searchIndex) - 1]; + previousIndex = (-searchIndex) - 1; + } else { done = true; } } - } + } } - } - + } + return adjustedIndex; } - /** * Adds the number of shifts for the given position - * + * * @param position The position to add the shift to - * @param shift The shift amount to add + * @param shift The shift amount to add */ - protected void addShift(long position, long shift) - { - if(shift == 0) - { + protected void addShift(long position, long shift) { + if (shift == 0) { throw new IllegalArgumentException("Shift cannot be zero"); } - + //if buffer is not full - - if(currentBufferSize < shiftBuffer[0].length) - { + + if (currentBufferSize < shiftBuffer[0].length) { //search for position to update or add to buffer int searchIndex = Arrays.binarySearch(shiftBuffer[0], 0, currentBufferSize, position); - - if(searchIndex >= 0) - { + + if (searchIndex >= 0) { shiftBuffer[1][searchIndex] = shiftBuffer[1][searchIndex] + shift; - } - else - { + } else { searchIndex = (searchIndex * -1) - 1; - - for(int i=currentBufferSize; i>searchIndex; i--) - { - shiftBuffer[0][i] = shiftBuffer[0][i-1]; - shiftBuffer[1][i] = shiftBuffer[1][i-1]; + + for (int i = currentBufferSize; i > searchIndex; i--) { + shiftBuffer[0][i] = shiftBuffer[0][i - 1]; + shiftBuffer[1][i] = shiftBuffer[1][i - 1]; } - + shiftBuffer[0][searchIndex] = position; shiftBuffer[1][searchIndex] = shift; - + currentBufferSize++; } - + updateShifts(searchIndex, currentBufferSize); - } - else - { + } else { throw new BufferOverflowException(); } } - + /** * Updates the shift buffer. This is used to keep the cumulative statistics correct. - * + * * @param startIndex Start index - * @param endIndex End index + * @param endIndex End index */ - private void updateShifts(int startIndex, int endIndex) - { - for(int i=startIndex; i= shiftBuffer[0].length) - { + + if (currentBufferSize >= shiftBuffer[0].length) { full = true; } - + return full; } - + /** * @return Returns if the shift buffer is full and no more shifts (unique or not unique) can be stored */ - protected boolean isShiftMaxed() - { + protected boolean isShiftMaxed() { boolean maxed = false; - - if(currentBufferSize > 0 && shiftBuffer[2][currentBufferSize-1] >= DEFAULT_SHIFT_ELEMENT_SIZE) - { + + if (currentBufferSize > 0 && shiftBuffer[2][currentBufferSize - 1] >= DEFAULT_SHIFT_ELEMENT_SIZE) { maxed = true; } - + return maxed; } } \ No newline at end of file diff --git a/src/main/java/examples/SimpleExample.java b/src/main/java/examples/SimpleExample.java index 99c9497..d852fe2 100644 --- a/src/main/java/examples/SimpleExample.java +++ b/src/main/java/examples/SimpleExample.java @@ -1,36 +1,39 @@ package examples; +import com.dselent.bigarraylist.BigArrayList; + import java.io.IOException; +import java.util.Arrays; -import com.dselent.bigarraylist.BigArrayList; +public class SimpleExample { + public static void main(String[] args) throws IOException { + //create a BigArrayList of Longs + //cache block size = 1 million + //cache blocks = 4 + BigArrayList bal = new BigArrayList(1000000, 4); + + BigArrayList balS = new BigArrayList<>(1000000, 4); + + + //add 10 million elements + for (long i = 0; i < 10000000; i++) { + bal.add(i); + balS.add(new String[]{String.valueOf(i), String.valueOf(i+1)}); + } + + //get the element at index 5 + System.out.println(bal.get(5)); + System.out.println(Arrays.toString(balS.get(5))); + + //set the element at index 5 + bal.set(5, 100L); + + //get the element at index 5 + System.out.println(bal.get(5)); -public class SimpleExample -{ - public static void main(String args[]) throws IOException - { - //create a BigArrayList of Longs - //cache block size = 1 million - //cache blocks = 4 - BigArrayList bal = new BigArrayList(1000000, 4); - - //add 10 million elements - for(long i=0; i<10000000; i++) - { - bal.add(i); - } - - //get the element at index 5 - System.out.println(bal.get(5)); - - //set the element at index 5 - bal.set(5, 100l); - - //get the element at index 5 - System.out.println(bal.get(5)); - - //clear contents on disk - bal.clearMemory(); - } + //clear contents on disk + bal.clearMemory(); + } } diff --git a/src/main/java/test/BigArrayListTest.java b/src/test/java/com/example/bigarraylist/BigArrayListTest.java similarity index 81% rename from src/main/java/test/BigArrayListTest.java rename to src/test/java/com/example/bigarraylist/BigArrayListTest.java index 873f37b..acd5566 100644 --- a/src/main/java/test/BigArrayListTest.java +++ b/src/test/java/com/example/bigarraylist/BigArrayListTest.java @@ -1,4 +1,8 @@ -package test; +package com.example.bigarraylist; + +import com.dselent.bigarraylist.BigArrayList; + +import junit.framework.TestCase; import java.io.IOException; import java.util.ArrayList; @@ -6,14 +10,9 @@ import java.util.List; import java.util.Random; -import com.dselent.bigarraylist.BigArrayList; - -import junit.framework.TestCase; - -public class BigArrayListTest extends TestCase -{ +public class BigArrayListTest extends TestCase { private static final int NUMBER_OF_OPERATIONS = 4; - + private int testRuns; private int minBlockSize; private int maxBlockSize; @@ -21,19 +20,17 @@ public class BigArrayListTest extends TestCase private int maxCacheBlocks; private int minActions; private int maxActions; - + private Random random; private BigArrayList bigArrayList; - - public BigArrayListTest(String testName) - { + + public BigArrayListTest(String testName) { super(testName); } - - protected void setUp() throws Exception - { + + protected void setUp() throws Exception { super.setUp(); - + //modify number of test runs as desired testRuns = 10; minBlockSize = 5; @@ -42,61 +39,52 @@ protected void setUp() throws Exception maxCacheBlocks = 20; minActions = 2; maxActions = 100000; - + random = new Random(0); } - - protected void tearDown() throws Exception - { + + protected void tearDown() throws Exception { super.tearDown(); - - if(bigArrayList != null) - { + + if (bigArrayList != null) { bigArrayList.clearMemory(); } } - + /** * Monte-carlo test case. Tests random operations on BigArrayLists with parameters randomized within ranges. */ - public void testBigArrayList() - { - for(int i=0; i(blockSize, cacheBlocks); List arrayList = new ArrayList(); - for(int j=0; jTesting documentation + */ +public class ExampleUnitTest { + @Test + public void addition_isCorrect() { + assertEquals(4, 2 + 2); + } +} \ No newline at end of file