diff --git a/ALOISampling/Readme.md b/ALOISampling/Readme.md new file mode 100644 index 0000000..068f79a --- /dev/null +++ b/ALOISampling/Readme.md @@ -0,0 +1,16 @@ +Instructions on preparing the ALOI samples +========================================== + +- Download the 192x144 databases for illumination direction, illumination color and viewing direction as well as the masks from http://aloi.science.uva.nl/ +- Put all archives into the same folder +- Place the scripts provided here in the same folder +- make all scripts executable +- execute createAloiSamples.sh +-- the first argument defines how much training image sequences will be created +-- the second argument defines how much test image sequences will be created + +After finishing, whiten the images with the provided MatLab script + +Dependencies: +- imagemagick +- GNU parallel diff --git a/ALOISampling/Readme.md~ b/ALOISampling/Readme.md~ new file mode 100644 index 0000000..2b24079 --- /dev/null +++ b/ALOISampling/Readme.md~ @@ -0,0 +1,14 @@ +Instructions on preparing the ALOI samples +========================================== + +- Download the 192x144 databases for illumination direction, illumination color and viewing direction as well as the masks from http://aloi.science.uva.nl/ +- Put all archives into the same folder +- Place the scripts provided here in the same folder +- make all scripts executable +- execute createAloiSamples.sh +-- the first argument defines how much training image sequences will be created +-- the second argument defines how much test image sequences will be created + +Dependencies: +- imagemagick +- GNU parallel diff --git a/ALOISampling/createAloiSamples.sh b/ALOISampling/createAloiSamples.sh new file mode 100755 index 0000000..5bf8606 --- /dev/null +++ b/ALOISampling/createAloiSamples.sh @@ -0,0 +1,36 @@ +#!/bin/bash +if [ "$#" -lt 2 ]; then + numTestSequences=1000 + if [ "$#" -lt 1 ]; then + numTrainingSequences=10000 + else + numTrainingSequences=${1} + fi +else + numTestSequences=${2} +fi + +tar -xf aloi_grey_red4_view.tar +VIEWDIR=aloi_red4_view +mv grey4 $VIEWDIR + +tar -xf aloi_grey_red4_col.tar +COLDIR=aloi_red4_col +mv grey4 $COLDIR + +tar -xf aloi_grey_red4_ill.tar +ILLDIR=aloi_red4_ill +mv grey4 $ILLDIR + +tar -xf aloi_mask4.tar +MASKDIR=aloi_mask4 +mv mask4 $MASKDIR + +mkdir output +./erodeMasks.sh +./createOutlines.sh +./cropBackground.sh +echo "Creating training sequences" +seq 1 $numTrainingSequences | parallel -eta ./createMotionSequence.sh {} "training" +echo "Creating test sequences" +seq 1 $numTestSequences | parallel -eta ./createMotionSequence.sh {} "test" diff --git a/ALOISampling/createAloiSamples.sh~ b/ALOISampling/createAloiSamples.sh~ new file mode 100755 index 0000000..75b74cf --- /dev/null +++ b/ALOISampling/createAloiSamples.sh~ @@ -0,0 +1,36 @@ +#!/bin/bash +if [ "$#" -lt 2 ]; then + numTestSequences=1000 + if [ "$#" -lt 1 ]; then + numTrainingSequences=10000 + else + numTrainingSequences=${1} + fi +else + numTestSequences=${2} +fi + +tar -xf aloi_grey_red4_view.tar +VIEWDIR=aloi_red4_view +mv grey4 $VIEWDIR + +tar -xf aloi_grey_red4_col.tar +COLDIR=aloi_red4_col +mv grey4 $COLDIR + +tar -xf aloi_grey_red4_ill.tar +ILLDIR=aloi_red4_ill +mv grey4 $ILLDIR + +tar -xf aloi_mask4.tar +MASKDIR=aloi_mask4 +mv mask4 $MASKDIR + +mkdir output +./erodeMasks +./createOutlines +./cropBackground +echo "Creating training sequences" +seq 1 $numTrainingSequences | parallel -eta ./createMotionSequence.sh {} "training" +echo "Creating test sequences" +seq 1 $numTestSequences | parallel -eta ./createMotionSequence.sh {} "test" diff --git a/ALOISampling/createMotionSequence.sh b/ALOISampling/createMotionSequence.sh new file mode 100755 index 0000000..fd4a8b1 --- /dev/null +++ b/ALOISampling/createMotionSequence.sh @@ -0,0 +1,331 @@ +#!/bin/bash + +### Author: Benjamin Mitzkus +### This script generates random motion sequences where one or two objects +### move randomly on a background + +## General Information: +## Vectors are always given as arrays in the notation a[0]=x, a[1]=y + +ITERATION=${1} + +# directory where the image patches are saved +OUTPUTDIR="output/${2}" + +# subdirectory for occlusion patches +OCCOUTPUT=${OUTPUTDIR}/occlusion + +# subdirectory for non-occlusion patches +NOOCCOUTPUT=${OUTPUTDIR}/motion + +# directory where the image files are saved +IMAGEDIR="output/croppedImages" + +# directory where the outline masks are saved +OUTLINEDIR="output/aloi_mask4_outline" + +# list of image files +FILES=($(find ${IMAGEDIR} -name "*.png")) + +# number of image files in $IMAGELIST +NUMFILES=${#FILES[@]} + +# folder for temporary data, will be deleted on exit +TEMPDIR="tmp" + +# size of the patches, always quadratic +PATCHSIZE=16 + +# number of frames for each sequence +NUMFRAMES=4 + +# boundaries for the velocity with which the images will move +MINVELOCITY=1.5 #Idea here: roughly sqrt(2) which means at least one pixel movement per frame +MAXVELOCITY=3 + +# seed to control pseudorandom number generation +SEED=0 +SEED=$(awk "BEGIN{print ${SEED} + ($ITERATION*13)}") + +### Help Functions + +## random number generation +# Parameters: 1: return variable, 2: real or integer (r or i), 3: lower boundary, 4: upper boundary +function getRandomNumber { + local AWKSCRIPT=" { srand(${SEED}); print rand() } " + local rnd=`echo | awk "${AWKSCRIPT}"` + echo $rnd >> rnd.log + let SEED+=1 + local lower=${3} + local upper=${4} + local ret + if [ ${2} == "i" ] + then + ret=$(awk "BEGIN{print ${rnd} * 1000000}") + ret=${ret%.*} + ret=$(awk "BEGIN{print ${ret}%(${upper}-(${lower}))+${lower}}") + elif [ ${2} == "r" ] + then + ret=$(awk "BEGIN{print (${rnd}*(${upper}-(${lower})))+${lower}}") + fi + + eval "$1=${ret}" # return random number +} + +## normalizing a vector +# Parameters: 1: return variable +function normalizeVector { + local vector; declare -a vector + local vectorname=$1[@] + vector=("${!vectorname}") + local length=$(awk "BEGIN{print sqrt((${vector[0]} * ${vector[0]}) + (${vector[1]} * ${vector[1]}))}") + + if [ ${length} == 0 ];then + return 0 #avoid 0division here + fi + local x=$(awk "BEGIN{print ${vector[0]}/${length}}") + local y=$(awk "BEGIN{print ${vector[1]}/${length}}") + + eval "${1}[0]=${x}" + eval "${1}[1]=${y}" + +} + +## searches $IMAGEDIR for .png files and choses one randomly +# Parameters: 1: return variable +function getRandomImagePath { + local RANDOMNUMBER + getRandomNumber RANDOMNUMBER i 1 ${NUMFILES} + eval "$1=${FILES[${RANDOMNUMBER}]}" +} + +## finds the outline mask to a given image +# Parameters: 1: Image, 2: return variable +function findOutline { + local path; declare -a path + path=( $(echo ${1} | tr "/" "\n") ) + local file + case "${path[2]}" in + aloi_red4_col) + file=${path[3]}_c1.png + ;; + + aloi_red4_ill) + file=${path[3]}_${path[4] : (-6)} + ;; + + aloi_red4_view) + local angle=$(echo ${path[4]} | cut -d"_" -f2) + if [ ${angle} == "r0.png" ] + then + file=${path[3]}_c1.png + else + file=${path[3]}_${angle} + fi + ;; + esac + + eval "${2}=${OUTLINEDIR}/${path[3]}/${file}" +} + +## returns a random point on the white outline of a mask +# Parameters: 1: return array, 2: path to outline image +function findOutlinePoint { + local outlinepoints=($(convert ${2} txt:- | grep FFFFFF | sed "s/ //g" | cut -d":" -f1)) + local numpoints=${#outlinepoints[@]} + + local randomPoint + getRandomNumber randomPoint i 1 ${numpoints} + + local outlineX=$(echo ${outlinepoints[${randomPoint}]} | cut -d"," -f1) + local outlineY=$(echo ${outlinepoints[${randomPoint}]} | cut -d"," -f2) + + eval "${1}[0]=${outlineX}" + eval "${1}[1]=${outlineY}" +} + +## calculates the center of a given image (integer, rounded down) +# Parameters: 1: return variable, 2: image +function calculateImageCenter { + local IMAGESIZE=$(identify ${2} | cut -d" " -f3) + local width=$(echo ${IMAGESIZE} | cut -d"x" -f1) + local height=$(echo ${IMAGESIZE} | cut -d"x" -f2) + local midX=$(awk "BEGIN{print int(${width}/2)}") + local midY=$(awk "BEGIN{print int(${height}/2)}") + + eval "${1}[0]=${midX}" + eval "${1}[1]=${midY}" +} + +## calculates the direction vector, images will move from the center to the outlinepoint +## with some random velocity between 0 and $MAXVELOCITY +# Parameters: 1: return variable, 2: image center, 3: outline point +function calculateMovement { +local centername=${2}[@] +local outlinename=${3}[@] + +local imageCenter=("${!centername}") +local outlinePoint=("${!outlinename}") + +local direction; declare -a direction +direction[0]=$(awk "BEGIN{print ${outlinePoint[0]} - ${imageCenter[0]}}") +direction[1]=$(awk "BEGIN{print ${outlinePoint[1]} - ${imageCenter[1]}}") + +normalizeVector direction + +local velocity +getRandomNumber velocity r ${MINVELOCITY} ${MAXVELOCITY} +local movement; declare -a movement + +movement[0]=$(awk "BEGIN{print ${direction[0]} * ${velocity}}") +movement[1]=$(awk "BEGIN{print ${direction[1]} * ${velocity}}") +eval "${1}[0]=${movement[0]}" +eval "${1}[1]=${movement[1]}" +} + +## finds a background patch with less than 10% transparent pixels +# Parameters: 1: return variable image path 2:return variable position of patch +function findBackgroundPatch { + local numPixels=$(awk "BEGIN{print ${PATCHSIZE}*${PATCHSIZE}}") + local maxTransparentPixels=$(awk "BEGIN{print int(${numPixels}*0.1)}") + local numTransparentPixels=${numPixels} + local transparentPixels; declare -a transparentPixels + + + while [ ${numTransparentPixels} -gt ${maxTransparentPixels} ] + do + local width=0 + local height=0 + local size + local image + local x + local y + while [ ${width} -lt ${PATCHSIZE} -o ${height} -lt ${PATCHSIZE} ] + do + getRandomImagePath image + size=$(identify ${image} | cut -d" " -f3) + width=$(echo ${size} | cut -d"x" -f1) + height=$(echo ${size} | cut -d"x" -f2) + done + getRandomNumber x i 0 $(awk "BEGIN{print ${width}-${PATCHSIZE} + 1}") + getRandomNumber y i 0 $(awk "BEGIN{print ${height}-${PATCHSIZE} + 1}") + transparentPixels=($(convert ${image} -crop "${PATCHSIZE}x${PATCHSIZE}+${x}+${y}" txt:- | grep "#......00")) + numTransparentPixels=${#transparentPixels[@]} + done + eval "${1}=${image}" + eval "${2}[0]=${x}" + eval "${2}[1]=${y}" +} + +### Data structure management + +## creating a temp folder which will be deleted on exit +function createTmpDir { + if [ -e ${TEMPDIR} ] + then + deleteTmpDir + fi + mkdir ${TEMPDIR} +} + +function deleteTmpDir { + rm -r ${TEMPDIR} +} + +## Manages the creation of all folders +function mkDirs { + createTmpDir + if [ ! -d ${OUTPUTDIR} ] + then + mkdir ${OUTPUTDIR} + mkdir ${OCCOUTPUT} + mkdir ${NOOCCOUTPUT} + fi +} + +## creates a motion sequence where $NUMOBJECTS random objects move in random directions +## in $NUMFRAMES frames and overlap in some of them +# Parameters: 1: patchNumber +function createMotionSequence { + # get one or two random image paths + local IMAGE + getRandomImagePath IMAGE + local OUTLINE + findOutline ${IMAGE} OUTLINE + local OUTLINEPOINT; declare -a OUTLINEPOINT + findOutlinePoint OUTLINEPOINT ${OUTLINE} + + local IMAGECENTER; declare -a IMAGECENTER + calculateImageCenter IMAGECENTER ${IMAGE} + + local MOVEMENT; declare -a MOVEMENT + calculateMovement MOVEMENT IMAGECENTER OUTLINEPOINT + + # calculate the point where the objects meet + local STARTPOINT; declare -a STARTPOINT + getRandomNumber STARTPOINT[0] i 0 $(awk "BEGIN{print int(${PATCHSIZE/4}) + 1}") + getRandomNumber STARTPOINT[1] i 0 $(awk "BEGIN{print int(${PATCHSIZE/4}) + 1}") + + if [ $(echo "${MOVEMENT[0]}<0" | bc -l) ] #moving left, STARTPOINT should be in right half + then + STARTPOINT[0]=$(awk "BEGIN{print ${PATCHSIZE} - ${STARTPOINT[0]}}") + fi + if [ $(echo "${MOVEMENT[1]}<0" | bc -l) ] #moving up, STARTPOINT should be in lower half + then + STARTPOINT[1]=$(awk "BEGIN{print ${PATCHSIZE} - ${STARTPOINT[1]}}") + fi + + local xOffset=$(awk "BEGIN{print ${STARTPOINT[0]} - ${OUTLINEPOINT[0]}}") + local yOffset=$(awk "BEGIN{print ${STARTPOINT[1]} - ${OUTLINEPOINT[1]}}") + + local background="${TEMPDIR}/${1}background.png" + local size="${PATCHSIZE}x${PATCHSIZE}" + + local backgroundPatch + local patchPos + findBackgroundPatch backgroundPatch patchPos + convert ${backgroundPatch} -crop "${PATCHSIZE}x${PATCHSIZE}+${patchPos[0]}+${patchPos[1]} " png:- | composite png:- -size ${size} xc:white ${background} + + for frame in `seq 1 ${NUMFRAMES}`; do + local occFile="${TEMPDIR}/${1}occFrame${frame}.png" + local nooccFile="${TEMPDIR}/${1}nooccFrame${frame}.png" + local xPosition=$(awk "BEGIN{print ${xOffset} + (${frame} * ${MOVEMENT[0]})}") + local xPosition=$(echo ${xPosition} | awk '{printf "%.0f\n", $1}') # rounding to integer + local yPosition=$(awk "BEGIN{print ${yOffset} + (${frame} * ${MOVEMENT[1]})}") + local yPosition=$(echo ${yPosition} | awk '{printf "%.0f\n", $1}') # rounding to integer + + if [ ${xPosition} -ge 0 ] + then + xPosition="+${xPosition}" + fi + + if [ ${yPosition} -ge 0 ] + then + yPosition="+${yPosition}" + fi + + + composite -geometry "${xPosition}${yPosition}" ${IMAGE} ${background} png:- | convert :- -colorspace Gray ${occFile} + composite -geometry "${xPosition}${yPosition}" ${IMAGE} -size ${size} xc:white png:- | convert :- -colorspace Gray ${nooccFile} + + done + + + local imageNr=$(echo ${IMAGE} | grep -o '[^/]*$' |cut -d"_" -f1) + local backgroundNr=$(echo ${backgroundPatch} | grep -o '[^/]*$' | cut -d"_" -f1) + local occOutputfile=${OCCOUTPUT}/${1}_${imageNr}_${backgroundNr}.png + local nooccOutputfile=${NOOCCOUTPUT}/${1}_${imageNr}.png + cp ${TEMPDIR}/${1}occFrame1.png ${occOutputfile} + cp ${TEMPDIR}/${1}nooccFrame1.png ${nooccOutputfile} + local i + for i in `seq 2 ${NUMFRAMES}` + do + convert ${occOutputfile} ${TEMPDIR}/${1}occFrame${i}.png +append ${occOutputfile} + convert ${nooccOutputfile} ${TEMPDIR}/${1}nooccFrame${i}.png +append ${nooccOutputfile} + done +} + +### FUNCTION CALLS STARTING HERE ### +mkDirs +createMotionSequence ${ITERATION} +deleteTmpDir diff --git a/ALOISampling/createOutlines.sh b/ALOISampling/createOutlines.sh new file mode 100755 index 0000000..f3bd34c --- /dev/null +++ b/ALOISampling/createOutlines.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +MASKDIR="output/aloi_mask4_eroded" +OUTPUTDIR="output/aloi_mask4_outline" + +function createfolders { + if [ -e ${OUTPUTDIR} ] + then + rm -r ${OUTPUTDIR} + fi + + mkdir ${OUTPUTDIR} + + for i in `seq 1 1000`; do + mkdir ${OUTPUTDIR}/${i} + done +} + +createfolders + +for item in $(find ${MASKDIR}/ -name "*.png");do +echo "converting ${item}" +output=$(echo ${item} | sed s/eroded/outline/g) +convert ${item} -morphology edgein disk:1 ${output} +done diff --git a/ALOISampling/crop.sh b/ALOISampling/crop.sh new file mode 100755 index 0000000..feeb845 --- /dev/null +++ b/ALOISampling/crop.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +MASKDIR="output/aloi_mask4_eroded" #insert path to folder with the masks here +VIEWDIR=aloi_red4_view #insert path to view changing images here +COLDIR=aloi_red4_col #insert path to color changing images here +ILLDIR=aloi_red4_ill #insert path to illuminance changing images here +OUTPUTDIR="output/croppedImages" #insert path to output directory here +NUMBEROFOBJECTS=1000 #the first x objects will be processed, 1000 is full + +function usemask { #uses the mask on the image, inputfile first, mask second, outputpath third +convert ${1} ${2} -alpha Off -compose CopyOpacity -composite ${3} +} + +function usemaskoncol { #processes all col images of the given object number +echo "Processing `echo ${COLDIR}/${1}`" +local i + for i in $(ls ${COLDIR}/${1}); do + usemask ${COLDIR}/${1}/${i} ${MASKDIR}/${1}/${1}_c1.png ${OUTPUTDIR}/${COLDIR}/${1}/${i} + done +} + +function usemaskonview { #processes all view images of the given object number +echo "Processing `echo ${VIEWDIR}/${1}`" + +#first image gets dealt with separately because of different mask name + usemask ${VIEWDIR}/${1}/${1}_r0.png ${MASKDIR}/${1}/${1}_c1.png ${OUTPUTDIR}/${VIEWDIR}/${1}/${1}_r0.png + +#processing other images now + local angle=5 + while [ ${angle} -le 355 ]; do + IMAGE=${VIEWDIR}/${1}/${1}_r${angle}.png + MASK=${MASKDIR}/${1}/${1}_r${angle}.png + OUTPUTFILE=${OUTPUTDIR}/${VIEWDIR}/${1}/${1}_r${angle}.png + usemask ${IMAGE} ${MASK} ${OUTPUTFILE} + let angle=${angle}+5 + done +} + +function usemaskonill { #processes all ill images of the given object number +echo "Processing `echo ${ILLDIR}/${1}`" +local c +local i + for c in `seq 1 3`; do # filter for 3 different camera angles + for i in $(ls ${ILLDIR}/${1} | grep c${c}); do + usemask ${ILLDIR}/${1}/${i} ${MASKDIR}/${1}/${1}_c${c}.png ${OUTPUTDIR}/${ILLDIR}/${1}/${i} + done + done +} + +usemaskoncol ${1} +usemaskonview ${1} +usemaskonill ${1} diff --git a/ALOISampling/cropBackground.sh b/ALOISampling/cropBackground.sh new file mode 100755 index 0000000..018933d --- /dev/null +++ b/ALOISampling/cropBackground.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +MASKDIR="output/aloi_mask4_eroded" #insert path to folder with the masks here +VIEWDIR=aloi_red4_view #insert path to view changing images here +COLDIR=aloi_red4_col #insert path to color changing images here +ILLDIR=aloi_red4_ill #insert path to illuminance changing images here +OUTPUTDIR="output/croppedImages" #insert path to output directory here +NUMBEROFOBJECTS=1000 #the first x objects will be processed, 1000 is full + + +function usemask { #uses the mask on the image, inputfile first, mask second, outputpath third +convert ${1} ${2} -alpha Off -compose CopyOpacity -composite ${3} +} + + +function createfolders { + mkdir ${OUTPUTDIR} + mkdir ${OUTPUTDIR}/${VIEWDIR} + mkdir ${OUTPUTDIR}/${COLDIR} + mkdir ${OUTPUTDIR}/${ILLDIR} + + local i + for i in `seq 1 ${NUMBEROFOBJECTS}`; do + mkdir ${OUTPUTDIR}/${VIEWDIR}/${i} + mkdir ${OUTPUTDIR}/${COLDIR}/${i} + mkdir ${OUTPUTDIR}/${ILLDIR}/${i} + done +} + +function usemaskoncol { #processes all col images of the given object number +echo "Processing `echo ${COLDIR}/${1}`" +local i + for i in $(ls ${COLDIR}/${1}); do + usemask ${COLDIR}/${1}/${i} ${MASKDIR}/${1}/${1}_c1.png ${OUTPUTDIR}/${COLDIR}/${1}/${i} + done +} + +function usemaskonview { #processes all view images of the given object number +echo "Processing `echo ${VIEWDIR}/${1}`" + +#first image gets dealt with separately because of different mask name + usemask ${VIEWDIR}/${1}/${1}_r0.png ${MASKDIR}/${1}/${1}_c1.png ${OUTPUTDIR}/${VIEWDIR}/${1}/${1}_r0.png + +#processing other images now + local angle=5 + while [ ${angle} -le 355 ]; do + IMAGE=${VIEWDIR}/${1}/${1}_r${angle}.png + MASK=${MASKDIR}/${1}/${1}_r${angle}.png + OUTPUTFILE=${OUTPUTDIR}/${VIEWDIR}/${1}/${1}_r${angle}.png + usemask ${IMAGE} ${MASK} ${OUTPUTFILE} + let angle=${angle}+5 + done +} + +function usemaskonill { #processes all ill images of the given object number +echo "Processing `echo ${ILLDIR}/${1}`" +local c +local i + for c in `seq 1 3`; do # filter for 3 different camera angles + for i in $(ls ${ILLDIR}/${1} | grep c${c}); do + usemask ${ILLDIR}/${1}/${i} ${MASKDIR}/${1}/${1}_c${c}.png ${OUTPUTDIR}/${ILLDIR}/${1}/${i} + done + done +} + +createfolders + +seq 1 $NUMBEROFOBJECTS | parallel ./crop.sh {} diff --git a/ALOISampling/erodeMasks.sh b/ALOISampling/erodeMasks.sh new file mode 100755 index 0000000..6dfde25 --- /dev/null +++ b/ALOISampling/erodeMasks.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +MASKDIR=aloi_mask4 +OUTPUTDIR="output/aloi_mask4_eroded" + +function createfolders { + if [ -e ${OUTPUTDIR} ] + then + rm -r ${OUTPUTDIR} + fi + + mkdir ${OUTPUTDIR} + + for i in `seq 1 1000`; do + mkdir ${OUTPUTDIR}/${i} + done +} + +createfolders + +for i in `seq 1 1000`; do + for j in $(ls ${MASKDIR}/${i}); do + convert ${MASKDIR}/${i}/${j} -morphology Erode Octagon ${OUTPUTDIR}/${i}/${j} + done +done + +cd $MASKDIR +find . -name "*.png" | parallel -eta convert {} -morphology Erode Octagon ../$OUTPUTDIR/{} diff --git a/Data/ALOItest.zip b/Data/ALOItest.zip new file mode 100644 index 0000000..817962c --- /dev/null +++ b/Data/ALOItest.zip Binary files differ diff --git a/Data/ALOItraining.zip b/Data/ALOItraining.zip new file mode 100644 index 0000000..a38322d --- /dev/null +++ b/Data/ALOItraining.zip Binary files differ diff --git a/Data/Readme.md b/Data/Readme.md new file mode 100644 index 0000000..152a730 --- /dev/null +++ b/Data/Readme.md @@ -0,0 +1,4 @@ +Data Samples +============ + +This folder contains the training and testing data that was used in my work. This data can be reproduced with the provided scripts. diff --git a/Data/Readme.md~ b/Data/Readme.md~ new file mode 100644 index 0000000..f6e307d --- /dev/null +++ b/Data/Readme.md~ @@ -0,0 +1,4 @@ +Data Samples +============ + +This folder contains the training and testing data that was used in my work. This data can be reproduced with the provided scripts diff --git a/Data/SintelTraining.zip b/Data/SintelTraining.zip new file mode 100644 index 0000000..6760f97 --- /dev/null +++ b/Data/SintelTraining.zip Binary files differ diff --git a/Data/Sinteltest.zip b/Data/Sinteltest.zip new file mode 100644 index 0000000..4b31203 --- /dev/null +++ b/Data/Sinteltest.zip Binary files differ diff --git a/MatLab-Code/Readme.md b/MatLab-Code/Readme.md new file mode 100644 index 0000000..63f6f76 --- /dev/null +++ b/MatLab-Code/Readme.md @@ -0,0 +1,4 @@ +MatLab Scripts +============== + +These scripts can be used for whitening the data. diff --git a/MatLab-Code/Readme.md~ b/MatLab-Code/Readme.md~ new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/MatLab-Code/Readme.md~ diff --git a/MatLab-Code/loadImages.m b/MatLab-Code/loadImages.m new file mode 100644 index 0000000..ad2e134 --- /dev/null +++ b/MatLab-Code/loadImages.m @@ -0,0 +1,33 @@ +function [C, names] = loadImages(dirPath, fileType) +% load images from a folder +poolobj = gcp('nocreate'); % If no pool, do not create new one. +poolcreated = false; +if isempty(poolobj) + poolobj = parpool; + poolcreated=true; +end + +contents = dir([dirPath fileType]); +if length(contents) < 1 + error('No Images found in specified folder'); +end + +names = {contents.name}; + +firstIm = imread([dirPath contents(1).name]); + +[m, n] = size(firstIm); +C=zeros(length(contents), m, n); + +parfor idx = 1:length(contents) + C(idx, :, :) = imread([dirPath contents(idx).name]); +end + +disp('Loading completed'); + +% if this function created a pool object, it will be deleted after use +if(poolcreated) + delete(poolobj); +end + +end \ No newline at end of file diff --git a/MatLab-Code/whitenImages.m b/MatLab-Code/whitenImages.m new file mode 100644 index 0000000..6486bf1 --- /dev/null +++ b/MatLab-Code/whitenImages.m @@ -0,0 +1,43 @@ +% Credit for this script goes to xcorr.net +% the script was essentially taken from https://xcorr.net/2011/05/27/whiten-a-matrix-matlab-code/ + + +%% create Pool + +pool=parpool(); + +%% Load Data +imgPath = ''; %path to the folder where the images are found +imgType = '*.png'; % change based on image type + +[nonWhite, fileNames] = loadImages(imgPath, imgType); +%% Process Data + +mX = bsxfun(@minus,nonWhite,mean(nonWhite)); %remove mean +fX = fft(fft(mX,[],2),[],3); %fourier transform of the images +spectr = sqrt(mean(abs(fX).^2)); %Mean spectrum +wX = ifft(ifft(bsxfun(@times,fX,1./spectr),[],2),[],3); %whitened X + + + +disp('Whitening complete') + +%% Get correct output format + +rwX = real(wX); + +rwX=rwX-min(rwX(:)); % shift data such that the smallest element of A is 0 + +rwX=rwX/max(rwX(:)); % normalize the shifted data to 1 + +disp('Correct output format') +%% Save Data +writePath = ''; % provide output path here + +parfor idx = 1:length(occlFileNames) + imwrite(reshape(rwX(idx,:,:), [16, 64]), [writePath fileNames{idx}]); +end + +disp('Writing complete') + +delete(pool); diff --git a/SintelSampling/Readme.md b/SintelSampling/Readme.md new file mode 100644 index 0000000..67fb015 --- /dev/null +++ b/SintelSampling/Readme.md @@ -0,0 +1,15 @@ +Instructions on preparing the Sintel samples +============================================ +- Download the full Sintel MPI dataset from http://sintel.is.tue.mpg.de/ +- Extract the archive +- Place the scripts provided here in the same folder +- Make all scripts executable +- execute createSintelSamples.sh +-- the first argument defines how much training image sequences will be created +-- the second argument defines how much test image sequences will be created + +After finishing, whiten the images with the provided MatLab script + +Dependencies: +- imagemagick +- GNU parallel diff --git a/SintelSampling/Readme.md~ b/SintelSampling/Readme.md~ new file mode 100644 index 0000000..4c2748f --- /dev/null +++ b/SintelSampling/Readme.md~ @@ -0,0 +1,13 @@ +Instructions on preparing the Sintel samples +============================================ +- Download the full Sintel MPI dataset from http://sintel.is.tue.mpg.de/ +- Extract the archive +- Place the scripts provided here in the same folder +- Make all scripts executable +- execute createSintelSamples.sh +-- the first argument defines how much training image sequences will be created +-- the second argument defines how much test image sequences will be created + +Dependencies: +- imagemagick +- GNU parallel diff --git a/SintelSampling/createSintelSamples.sh b/SintelSampling/createSintelSamples.sh new file mode 100755 index 0000000..1233840 --- /dev/null +++ b/SintelSampling/createSintelSamples.sh @@ -0,0 +1,34 @@ +#!/bin/bash +if [ "$#" -lt 2 ]; then + numTestSequences=1000 + if [ "$#" -lt 1 ]; then + numTrainingSequences=10000 + else + numTrainingSequences=${1} + fi +else + numTestSequences=${2} + numTrainingSequences=${1} +fi + +mkdir output + +TRAININGDATA="training" # folder with the training data +OCCLUSIONDATA="${TRAININGDATA}/occlusions" # folder with the occlusion maps +OUTPUTDIR="output" # where to save all processing steps and the final output + +### Transform images to grayscale +GRAYSCALEOUTPUT=${OUTPUTDIR}/SintelGray + +## if you want to skip this step, comment lines below +## however, GRAYSCALEOUTPUT must be set to the directory with the grayscale transformed images + +mkdir $GRAYSCALEOUTPUT +#./transformGrayscale.sh $TRAININGDATA $GRAYSCALEOUTPUT + +### END grayscale transform + +echo "Creating training sequences" +./createTrainingData.sh $numTrainingSequences "training" +echo "Creating test sequences" +./createTrainingData.sh $numTestSequences "test" diff --git a/SintelSampling/createTrainingData.sh b/SintelSampling/createTrainingData.sh new file mode 100755 index 0000000..be3333b --- /dev/null +++ b/SintelSampling/createTrainingData.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +NUMTRIALS=$1 # number of trials +SINTELDIR="$(pwd)" # root folder of unpacked sintel data +TRAININGDATA="${SINTELDIR}/training" # folder with the training data +OCCLUSIONDATA="${TRAININGDATA}/occlusions" # folder with the occlusion maps +OUTPUTDIR="$(pwd)/output" # where to save all processing steps and the final output +GRAYSCALEOUTPUT=${OUTPUTDIR}/SintelGray +mkdir $OUTPUTDIR/$2 + + +### Create trials +export OCCLOUTPUTDIR="$OUTPUTDIR/$2/occlusion" +mkdir $OCCLOUTPUTDIR +export OCCLUSIONMAPDIR="$OUTPUTDIR/$2/occlusionMaps" +mkdir $OCCLUSIONMAPDIR +export MOTIONOUTPUTDIR="$OUTPUTDIR/$2/motion" +mkdir $MOTIONOUTPUTDIR +export MOTIONMAPDIR="$OUTPUTDIR/$2/motionMaps" +mkdir $MOTIONMAPDIR + + +## create occlusion trials and occlusion maps +# if you want to skip this step, comment lines below +mkdir parallelres +seq 1 $NUMTRIALS | parallel --eta --joblog occlusion.log ./createTrial.sh $GRAYSCALEOUTPUT $OCCLUSIONDATA "occlusion" {} $NUMTRIALS +find parallelres -name "stdout*" | xargs cat | sort | echo -e "trial, xOffset, yOffset, sequence, startFrame\n$(cat -)" > OcclCreation.log +rm -r parallelres + +## create motion trials and corresponding occlusion maps +# if you want to skip this step, comment lines below +mkdir parallelres +seq 1 $NUMTRIALS | parallel --eta --joblog motion.log ./createTrial.sh $GRAYSCALEOUTPUT $OCCLUSIONDATA "motion" {} $NUMTRIALS +find parallelres -name "stdout*" | xargs cat | sort | echo -e "trial, xOffset, yOffset, sequence, startFrame\n$(cat -)" > motionCreation.log +rm -r parallelres + + +# END creation of occlusion data diff --git a/SintelSampling/createTrial.sh b/SintelSampling/createTrial.sh new file mode 100755 index 0000000..5f70797 --- /dev/null +++ b/SintelSampling/createTrial.sh @@ -0,0 +1,141 @@ +#!/bin/bash +INPUTDIR=$1 +OCCLUSIONDIR=$2 +TYPE=$3 +TRIAL=$4 +SEED=$4 +NUMTRIALS=$5 + + +case $TYPE in + "motion") + OUTPUTDIR=$MOTIONOUTPUTDIR + MAPDIR=$MOTIONMAPDIR + minOcclusionScale="0" + maxOcclusionScale="0.05" + ;; + "occlusion") + OUTPUTDIR=$OCCLOUTPUTDIR + MAPDIR=$OCCLUSIONMAPDIR + minOcclusionScale="0.3" + maxOcclusionScale="1" + ;; + *) + echo "Unknown Type $TYPE, exiting" + exit + ;; +esac +LENGTH=${#NUMTRIALS} + +minToleratedStandardDev=5 + +# list of animated sequences +SEQUENCES=($(ls $INPUTDIR)) + +numTimeSteps=4 + +# dimensions (pixel) of input +width=1024 +height=436 + +# dimensions (pixel) of output (per timestep) +xPatchSize=32 +yPatchSize=32 + + + + +## Help Functions + +## random number generation +# Parameters: 1: return variable, 2: lower boundary, 3: upper boundary +function getRandomNumber { + local AWKSCRIPT=" { srand(${SEED}); print rand() } " + local rnd=`echo | awk "${AWKSCRIPT}"` + let SEED+=1 + local lower=${2} + local upper=${3} + local ret + ret=$(awk "BEGIN{print ${rnd} * 1000000}") + ret=${ret%.*} + ret=$(awk "BEGIN{print ${ret}%(${upper}-(${lower}))+${lower}}") + + eval "$1=${ret}" # return random number +} + +getRandomNumber SEED 0 999999 + +numOcclusionPixels=-1 +standardDev=0 +minToleratedOcclusionPixels=$(awk "BEGIN{print int(${xPatchSize}*${yPatchSize}*(${numTimeSteps}- 1)*${minOcclusionScale})}") +maxToleratedOcclusionPixels=$(awk "BEGIN{print int(${xPatchSize}*${yPatchSize}*(${numTimeSteps}- 1)*${maxOcclusionScale})}") + +# create trial with occlusion/motion happening, check for patches with too low standard Dev +while [ $standardDev -lt $minToleratedStandardDev ] +do + # choose a random animated sequence + getRandomNumber rand 1 ${#SEQUENCES[@]} + let rand=$rand-1 + sequence=${SEQUENCES[$rand]} + occlusionDir=${OCCLUSIONDIR}/$sequence + + # calculate last possible frame (numframes - numTimeSteps + 2) since occlusionmaps have always 1 frame less than actual movie frames + frames=($(ls $occlusionDir)) + + numFrames=${#frames[@]} + let lastPossibleFrame=$numFrames-$numTimeSteps+2 + + # get random starting frame + getRandomNumber startFrameNr 1 $lastPossibleFrame + let startFrameNr=$startFrameNr-1 + + # get random coordinates + let xMax=$width-$xPatchSize-1 + let yMax=$height-$yPatchSize-1 + getRandomNumber xOffset 0 $xMax + getRandomNumber yOffset 0 $yMax + + usedframes="" + for i in `seq 0 $(awk "BEGIN{print $numTimeSteps - 2}")`; do + let frameNr=$i+$startFrameNr + usedframes="${usedframes} $occlusionDir/${frames[$frameNr]}" + done + + trialNrLz="$(printf "%0${LENGTH}d" ${TRIAL})" + fileName="$trialNrLz.png" + outputFile="$MAPDIR/${fileName}" + + convert $usedframes -crop "${xPatchSize}x${yPatchSize}+${xOffset}+${yOffset}" +append $outputFile + numOcclusionPixels=$(convert $outputFile txt:- | grep "#FFFFFF" | wc -l) + + + + convert $outputFile -resize 48x16 ${outputFile} + + # check if occluding pixels are in bounds for occlusion/motion trial + if [ $numOcclusionPixels -lt $minToleratedOcclusionPixels ] || [ $numOcclusionPixels -gt $maxToleratedOcclusionPixels ] + then + continue; + fi + + # create trial to that occlusion + usedframes="" + + sequenceDir=$INPUTDIR/$sequence + + for i in `seq 0 $(awk "BEGIN{print $numTimeSteps - 1}")`; do + let frameNr=$i+$startFrameNr + usedframes="${usedframes} $sequenceDir/${frames[$frameNr]}" + done + + outputFile="$OUTPUTDIR/${fileName}" + + convert $usedframes -crop "${xPatchSize}x${yPatchSize}+${xOffset}+${yOffset}" +append $outputFile + + convert $outputFile -resize 64x16 ${outputFile} + standardDev=$(identify -verbose $outputFile | grep "standard deviation" | cut -d":" -f2 | cut -d" " -f2) + standardDev=$(awk "BEGIN{print int($standardDev)}") +done + + +printf "%s, %d, %d, %s, %d\n" $trialNrLz $xOffset $yOffset $sequence $startFrameNr > "parallelres/stdout$trialNrLz" diff --git a/SintelSampling/transformGrayscale.sh b/SintelSampling/transformGrayscale.sh new file mode 100755 index 0000000..3b4dac8 --- /dev/null +++ b/SintelSampling/transformGrayscale.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +SINTELDIR=$1 +OUTPUTDIR=$2 + +DATASET="clean" + +cp -r ${SINTELDIR}/${DATASET}/* ${OUTPUTDIR} + +echo "Transforming images to grayscale:" +find ${OUTPUTDIR} -name "*.png" | parallel --eta convert {} -colorspace Gray {} +echo "Finished!" diff --git a/SparseCodingOfDynamicOcclusions_Thesis_Benjamin_Mitzkus.pdf b/SparseCodingOfDynamicOcclusions_Thesis_Benjamin_Mitzkus.pdf new file mode 100644 index 0000000..f737cee --- /dev/null +++ b/SparseCodingOfDynamicOcclusions_Thesis_Benjamin_Mitzkus.pdf Binary files differ diff --git a/paramFiles/Readme.md b/paramFiles/Readme.md new file mode 100644 index 0000000..279675b --- /dev/null +++ b/paramFiles/Readme.md @@ -0,0 +1,4 @@ +Parameter Files +=============== + +These files were used as parameter files for the petavision runs. In order to reproduce the findings several parameters need to be adjusted to the setup the toolbox is running on (mostly path variables and batching). Before starting the run, translate the parameters to petavision input format by typing lua > diff --git a/paramFiles/Readme.md~ b/paramFiles/Readme.md~ new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/paramFiles/Readme.md~ diff --git a/paramFiles/testParams.lua b/paramFiles/testParams.lua new file mode 100644 index 0000000..a428c1d --- /dev/null +++ b/paramFiles/testParams.lua @@ -0,0 +1,330 @@ +package.path = package.path .. ";" .. "/home/neuralnet/workspace/OpenPV/parameterWrapper/?.lua"; --neds to be adjusted +local pv = require "PVModule"; +--local subnets = require "PVSubnets"; + +local nbatch = 1; --Batch size of learning +local nxSize = 64; --Cifar is 32 x 32 +local nySize = 16; +local xpatchSize = nxSize; +local ypatchSize = nySize; +local displayPeriod = 1000; --Number of timesteps to find sparse approximation +local numEpochs = 1; --Number of times to run through dataset +local numImages = 1000; --Total number of images in dataset +local stopTime = numImages * displayPeriod; +local writeStep = displayPeriod; +local initialWriteTime = displayPeriod; + +local inputPath = ; -- path to a .txt file with all paths to input images +local outputPath = "output/"; +local checkpointPeriod = (displayPeriod * (numImages + 1)); -- How often to write checkpoints + +local numBasisVectors = 512; --1024; --2048; -- depending on completeness of used set of basis functions +local basisVectorFile = "V1ToError_W.pvp"; --the weights from the respective training run +local plasticityFlag = false; --Determines if we are learning weights, false because we are in testing condition +local momentumTau = 50; --The momentum parameter. A single weight update will last for momentumTau timesteps. +local dWMax = 0.05; --The learning rate +local VThresh = 0.035; -- .005; --The threshold, or lambda, of the network +local AMin = 0; +local AMax = infinity; +local AShift = VThresh; --This being equal to VThresh is a soft threshold +local VWidth = 0; +local timeConstantTau = math.ceil(displayPeriod/15); --The integration tau for sparse approximation +local weightInit = math.sqrt((1/xpatchSize)*(1/ypatchSize)*(1/3)); + +-- Base table variable to store +local pvParameters = { + + --Layers------------------------------------------------------------ + -------------------------------------------------------------------- + column = { + groupType = "HyPerCol"; + startTime = 0; + dt = 1; + dtAdaptFlag = true; + useAdaptMethodExp1stOrder = true; + dtAdaptController = "V1EnergyProbe"; + dtAdaptTriggerLayerName = "Input"; + dtAdaptTriggerOffset = 0; + dtScaleMax = 1; --1.0; -- minimum value for the maximum time scale, regardless of tau_eff + dtScaleMin = 0.1; -- default time scale to use after image flips or when something is wacky + dtChangeMax = 0.1; -- determines fraction of tau_effective to which to set the time step, can be a small percentage as tau_eff can be huge + dtChangeMin = 0.05; -- percentage increase in the maximum allowed time scale whenever the time scale equals the current maximum + dtMinToleratedTimeScale = 0.0001; + stopTime = stopTime; + progressInterval = (displayPeriod * 10); + writeProgressToErr = true; + verifyWrites = false; + outputPath = outputPath; + printParamsFilename = "occlusion.params"; + randomSeed = 1234567890; + nx = nxSize; + ny = nySize; + nbatch = nbatch; + filenamesContainLayerNames = 2; + filenamesContainConnectionNames = 2; + initializeFromCheckpointDir = ""; + defaultInitializeFromCheckpointFlag = false; + checkpointWrite = true; + checkpointWriteDir = outputPath .. "/Checkpoints"; --The checkpoint output directory + checkpointWriteTriggerMode = "step"; + checkpointWriteStepInterval = checkpointPeriod; --How often to checkpoint + deleteOlderCheckpoints = false; + suppressNonplasticCheckpoints = false; + writeTimescales = true; + errorOnNotANumber = false; + }; + + Input = { + groupType = "Movie"; + nxScale = 1; + nyScale = 1; + nf = 1; + phase = 0; + mirrorBCflag = true; + initializeFromCheckpointFlag = false; + writeStep = writeStep; + initialWriteTime = initialWriteTime; + sparseLayer = false; + updateGpu = false; + dataType = nil; + inputPath = inputPath; + offsetAnchor = "tl"; + offsetX = 0; + offsetY = 0; + writeImages = 0; + inverseFlag = false; + normalizeLuminanceFlag = true; + normalizeStdDev = true; + jitterFlag = 0; + useImageBCflag = false; + padValue = 0; + autoResizeFlag = false; + displayPeriod = displayPeriod; + echoFramePathnameFlag = true; + batchMethod = "byImage"; + start_frame_index = {0.000000}; + writeFrameToTimestamp = true; + flipOnTimescaleError = true; + resetToStartOnLoop = false; + }; + + InputError = { + groupType = "ANNLayer"; + nxScale = 1; + nyScale = 1; + nf = 1; + phase = 1; + mirrorBCflag = false; + valueBC = 0; + initializeFromCheckpointFlag = false; + InitVType = "ZeroV"; + triggerLayerName = NULL; + writeStep = displayPeriod; + initialWriteTime = initialWriteTime; + sparseLayer = false; + updateGpu = false; + dataType = nil; + VThresh = -infinity; + AMin = -infinity; + AMax = infinity; + AShift = 0; + clearGSynInterval = 0; + useMask = false; + }; + + V1 = { + groupType = "HyPerLCALayer"; + nxScale = 1/nxSize; + nyScale = 1/nySize; + nf = numBasisVectors; + phase = 2; + mirrorBCflag = false; + valueBC = 0; + initializeFromCheckpointFlag = false; + InitVType = "ConstantV"; + valueV = VThresh; + --InitVType = "InitVFromFile"; + --Vfilename = "V1_V.pvp"; + triggerLayerName = NULL; + writeStep = displayPeriod; + initialWriteTime = initialWriteTime; + sparseLayer = true; + writeSparseValues = true; + updateGpu = true; + dataType = nil; + VThresh = VThresh; + AMin = AMin; + AMax = AMax; + AShift = AShift; + VWidth = VWidth; + clearGSynInterval = 0; + timeConstantTau = timeConstantTau; + selfInteract = true; + }; + + InputRecon = { + groupType = "ANNLayer"; + nxScale = 1; + nyScale = 1; + nf = 1; + phase = 3; + mirrorBCflag = false; + valueBC = 0; + initializeFromCheckpointFlag = false; + InitVType = "ZeroV"; + triggerLayerName = NULL; + writeStep = writeStep; + initialWriteTime = initialWriteTime; + sparseLayer = false; + updateGpu = false; + dataType = nil; + VThresh = -infinity; + AMin = -infinity; + AMax = infinity; + AShift = 0; + VWidth = 0; + clearGSynInterval = 0; + }; + +--Connections ------------------------------------------------------ +-------------------------------------------------------------------- + + InputToError = { + groupType = "RescaleConn"; + preLayerName = "Input"; + postLayerName = "InputError"; + channelCode = 0; + delay = {0.000000}; + scale = weightInit; + }; + + ErrorToV1 = { + groupType = "TransposeConn"; + preLayerName = "InputError"; + postLayerName = "V1"; + channelCode = 0; + delay = {0.000000}; + convertRateToSpikeCount = false; + receiveGpu = true; + updateGSynFromPostPerspective = true; + pvpatchAccumulateType = "convolve"; + writeStep = -1; + writeCompressedCheckpoints = false; + selfFlag = false; + gpuGroupIdx = -1; + originalConnName = "V1ToError"; + }; + + V1ToError = { + groupType = "MomentumConn"; + preLayerName = "V1"; + postLayerName = "InputError"; + channelCode = -1; + delay = {0.000000}; + numAxonalArbors = 1; + plasticityFlag = plasticityFlag; + convertRateToSpikeCount = false; + receiveGpu = false; -- non-sparse -> non-sparse + sharedWeights = true; + --weightInitType = "UniformRandomWeight"; + wMinInit = -1; + wMaxInit = 1; + sparseFraction = 0.9; + weightInitType = "FileWeight"; + initWeightsFile = basisVectorFile; + useListOfArborFiles = false; + combineWeightFiles = false; + initializeFromCheckpointFlag = false; + triggerLayerName = "Input"; + triggerOffset = 0; + updateGSynFromPostPerspective = false; -- Should be false from V1 (sparse layer) to Error (not sparse). Otherwise every input from pre will be calculated (Instead of only active ones) + pvpatchAccumulateType = "convolve"; + writeStep = -1; + initialWriteTime = initialWriteTime; + writeCompressedCheckpoints = false; + selfFlag = false; + nxp = xpatchSize; + nyp = ypatchSize; + shrinkPatches = false; + normalizeMethod = "normalizeL2"; + strength = 1; + normalizeArborsIndividually = false; + normalizeOnInitialize = true; + normalizeOnWeightUpdate = true; + rMinX = 0; + rMinY = 0; + nonnegativeConstraintFlag = false; + normalize_cutoff = 0; + normalizeFromPostPerspective = false; + minL2NormTolerated = 0; + dWMax = dWMax; + keepKernelsSynchronized = true; + useMask = false; + momentumTau = momentumTau; --The momentum parameter. A single weight update will last for momentumTau timesteps. + momentumMethod = "viscosity"; + momentumDecay = 0; + }; + + V1ToRecon = { + groupType = "CloneConn"; + preLayerName = "V1"; + postLayerName = "InputRecon"; + channelCode = 0; + delay = {0.000000}; + convertRateToSpikeCount = false; + receiveGpu = false; + updateGSynFromPostPerspective = false; + pvpatchAccumulateType = "convolve"; + writeCompressedCheckpoints = false; + selfFlag = false; + originalConnName = "V1ToError"; + }; + + ReconToError = { + groupType = "IdentConn"; + preLayerName = "InputRecon"; + postLayerName = "InputError"; + channelCode = 1; + delay = {0.000000}; + initWeightsFile = nil; + }; + + --Probes------------------------------------------------------------ + -------------------------------------------------------------------- + + V1EnergyProbe = { + groupType = "ColumnEnergyProbe"; + message = nil; + textOutputFlag = true; + probeOutputFile = "V1EnergyProbe.txt"; + triggerLayerName = nil; + energyProbe = nil; + }; + + InputErrorL2NormEnergyProbe = { + groupType = "L2NormProbe"; + targetLayer = "InputError"; + message = nil; + textOutputFlag = true; + probeOutputFile = "InputErrorL2NormEnergyProbe.txt"; + energyProbe = "V1EnergyProbe"; + coefficient = 0.5; + maskLayerName = nil; + exponent = 2; + }; + + V1L1NormEnergyProbe = { + groupType = "L1NormProbe"; + targetLayer = "V1"; + message = nil; + textOutputFlag = true; + probeOutputFile = "V1L1NormEnergyProbe.txt"; + energyProbe = "V1EnergyProbe"; + coefficient = 0.025; + maskLayerName = nil; + }; + +} --End of pvParameters + +-- Print out PetaVision approved parameter file to the console +pv.printConsole(pvParameters) diff --git a/paramFiles/trainingParams.lua b/paramFiles/trainingParams.lua new file mode 100644 index 0000000..fc606d0 --- /dev/null +++ b/paramFiles/trainingParams.lua @@ -0,0 +1,330 @@ +package.path = package.path .. ";" .. "/home/neuralnetlab/workspace/OpenPV/parameterWrapper/?.lua"; -- needs to be adjusted +local pv = require "PVModule"; +--local subnets = require "PVSubnets"; + +local nbatch = 32; --Batch size of learning +local nxSize = 64; --Cifar is 32 x 32 +local nySize = 16; +local xpatchSize = nxSize; +local ypatchSize = nySize; +local displayPeriod = 100; --Number of timesteps to find sparse approximation +local numEpochs = 3; --Number of times to run through dataset +local numImages = 100000; --Total number of images in dataset +local stopTime = math.ceil((numImages * numEpochs) / nbatch) * displayPeriod; +local writeStep = displayPeriod*100; +local initialWriteTime = displayPeriod; + +local inputPath = ; --path to a .txt file with all paths to input images +local outputPath = "output/"; +local checkpointPeriod = (displayPeriod * 1000); -- How often to write checkpoints + +local numBasisVectors = 512; --1024; --2048; --depending on how many basis functions were trained +local basisVectorFile = nil; --nil for initial weights, otherwise, specifies the weights file to load. Change init parameter in MomentumConn +local plasticityFlag = true; --Determines if we are learning weights or holding them constant +local momentumTau = 50; --The momentum parameter. A single weight update will last for momentumTau timesteps. +local dWMax = 0.01; -- 0.05; -- 0.10; --The learning rate +local VThresh = .035; --.05; --.065; --The threshold, or lambda, of the network +local AMin = 0; +local AMax = infinity; +local AShift = VThresh; --This being equal to VThresh is a soft threshold +local VWidth = 0; +local timeConstantTau = math.ceil(displayPeriod/15); --The integration tau for sparse approximation +local weightInit = math.sqrt((1/xpatchSize)*(1/ypatchSize)*(1/3)); + +-- Base table variable to store +local pvParameters = { + + --Layers------------------------------------------------------------ + -------------------------------------------------------------------- + column = { + groupType = "HyPerCol"; + startTime = 0; + dt = 1; + dtAdaptFlag = true; + useAdaptMethodExp1stOrder = true; + dtAdaptController = "V1EnergyProbe"; + dtAdaptTriggerLayerName = "Input"; + dtAdaptTriggerOffset = 0; + dtScaleMax = 1; --1.0; -- minimum value for the maximum time scale, regardless of tau_eff + dtScaleMin = 0.1; -- default time scale to use after image flips or when something is wacky + dtChangeMax = 0.1; -- determines fraction of tau_effective to which to set the time step, can be a small percentage as tau_eff can be huge + dtChangeMin = 0.05; -- percentage increase in the maximum allowed time scale whenever the time scale equals the current maximum + dtMinToleratedTimeScale = 0.0001; + stopTime = stopTime; + progressInterval = (displayPeriod * 10); + writeProgressToErr = true; + verifyWrites = false; + outputPath = outputPath; + printParamsFilename = "occlusion.params"; + randomSeed = 1234567890; + nx = nxSize; + ny = nySize; + nbatch = nbatch; + filenamesContainLayerNames = 2; + filenamesContainConnectionNames = 2; + initializeFromCheckpointDir = ""; + defaultInitializeFromCheckpointFlag = false; + checkpointWrite = true; + checkpointWriteDir = outputPath .. "/Checkpoints"; --The checkpoint output directory + checkpointWriteTriggerMode = "step"; + checkpointWriteStepInterval = checkpointPeriod; --How often to checkpoint + deleteOlderCheckpoints = false; + suppressNonplasticCheckpoints = false; + writeTimescales = true; + errorOnNotANumber = false; + }; + + Input = { + groupType = "Movie"; + nxScale = 1; + nyScale = 1; + nf = 1; + phase = 0; + mirrorBCflag = true; + initializeFromCheckpointFlag = false; + writeStep = writeStep; + initialWriteTime = initialWriteTime; + sparseLayer = false; + updateGpu = false; + dataType = nil; + inputPath = inputPath; + offsetAnchor = "tl"; + offsetX = 0; + offsetY = 0; + writeImages = 0; + inverseFlag = false; + normalizeLuminanceFlag = true; + normalizeStdDev = true; + jitterFlag = 0; + useImageBCflag = false; + padValue = 0; + autoResizeFlag = false; + displayPeriod = displayPeriod; + echoFramePathnameFlag = true; + batchMethod = "byImage"; + start_frame_index = {0.000000}; + writeFrameToTimestamp = true; + flipOnTimescaleError = true; + resetToStartOnLoop = false; + }; + + InputError = { + groupType = "ANNLayer"; + nxScale = 1; + nyScale = 1; + nf = 1; + phase = 1; + mirrorBCflag = false; + valueBC = 0; + initializeFromCheckpointFlag = false; + InitVType = "ZeroV"; + triggerLayerName = NULL; + writeStep = displayPeriod; + initialWriteTime = initialWriteTime; + sparseLayer = false; + updateGpu = false; + dataType = nil; + VThresh = -infinity; + AMin = -infinity; + AMax = infinity; + AShift = 0; + clearGSynInterval = 0; + useMask = false; + }; + + V1 = { + groupType = "HyPerLCALayer"; + nxScale = 1/nxSize; + nyScale = 1/nySize; + nf = numBasisVectors; + phase = 2; + mirrorBCflag = false; + valueBC = 0; + initializeFromCheckpointFlag = false; + InitVType = "ConstantV"; + valueV = VThresh; + --InitVType = "InitVFromFile"; + --Vfilename = "V1_V.pvp"; + triggerLayerName = NULL; + writeStep = displayPeriod; + initialWriteTime = initialWriteTime; + sparseLayer = true; + writeSparseValues = true; + updateGpu = true; + dataType = nil; + VThresh = VThresh; + AMin = AMin; + AMax = AMax; + AShift = AShift; + VWidth = VWidth; + clearGSynInterval = 0; + timeConstantTau = timeConstantTau; + selfInteract = true; + }; + + InputRecon = { + groupType = "ANNLayer"; + nxScale = 1; + nyScale = 1; + nf = 1; + phase = 3; + mirrorBCflag = false; + valueBC = 0; + initializeFromCheckpointFlag = false; + InitVType = "ZeroV"; + triggerLayerName = NULL; + writeStep = writeStep; + initialWriteTime = initialWriteTime; + sparseLayer = false; + updateGpu = false; + dataType = nil; + VThresh = -infinity; + AMin = -infinity; + AMax = infinity; + AShift = 0; + VWidth = 0; + clearGSynInterval = 0; + }; + +--Connections ------------------------------------------------------ +-------------------------------------------------------------------- + + InputToError = { + groupType = "RescaleConn"; + preLayerName = "Input"; + postLayerName = "InputError"; + channelCode = 0; + delay = {0.000000}; + scale = weightInit; + }; + + ErrorToV1 = { + groupType = "TransposeConn"; + preLayerName = "InputError"; + postLayerName = "V1"; + channelCode = 0; + delay = {0.000000}; + convertRateToSpikeCount = false; + receiveGpu = true; + updateGSynFromPostPerspective = true; + pvpatchAccumulateType = "convolve"; + writeStep = -1; + writeCompressedCheckpoints = false; + selfFlag = false; + gpuGroupIdx = -1; + originalConnName = "V1ToError"; + }; + + V1ToError = { + groupType = "MomentumConn"; + preLayerName = "V1"; + postLayerName = "InputError"; + channelCode = -1; + delay = {0.000000}; + numAxonalArbors = 1; + plasticityFlag = plasticityFlag; + convertRateToSpikeCount = false; + receiveGpu = false; -- non-sparse -> non-sparse + sharedWeights = true; + weightInitType = "UniformRandomWeight"; + wMinInit = -1; + wMaxInit = 1; + sparseFraction = 0.9; + --weightInitType = "FileWeight"; + --initWeightsFile = basisVectorFile; + useListOfArborFiles = false; + combineWeightFiles = false; + initializeFromCheckpointFlag = false; + triggerLayerName = "Input"; + triggerOffset = 0; + updateGSynFromPostPerspective = false; -- Should be false from V1 (sparse layer) to Error (not sparse). Otherwise every input from pre will be calculated (Instead of only active ones) + pvpatchAccumulateType = "convolve"; + writeStep = writeStep; + initialWriteTime = initialWriteTime; + writeCompressedCheckpoints = false; + selfFlag = false; + nxp = xpatchSize; + nyp = ypatchSize; + shrinkPatches = false; + normalizeMethod = "normalizeL2"; + strength = 1; + normalizeArborsIndividually = false; + normalizeOnInitialize = true; + normalizeOnWeightUpdate = true; + rMinX = 0; + rMinY = 0; + nonnegativeConstraintFlag = false; + normalize_cutoff = 0; + normalizeFromPostPerspective = false; + minL2NormTolerated = 0; + dWMax = dWMax; + keepKernelsSynchronized = true; + useMask = false; + momentumTau = momentumTau; --The momentum parameter. A single weight update will last for momentumTau timesteps. + momentumMethod = "viscosity"; + momentumDecay = 0; + }; + + V1ToRecon = { + groupType = "CloneConn"; + preLayerName = "V1"; + postLayerName = "InputRecon"; + channelCode = 0; + delay = {0.000000}; + convertRateToSpikeCount = false; + receiveGpu = false; + updateGSynFromPostPerspective = false; + pvpatchAccumulateType = "convolve"; + writeCompressedCheckpoints = false; + selfFlag = false; + originalConnName = "V1ToError"; + }; + + ReconToError = { + groupType = "IdentConn"; + preLayerName = "InputRecon"; + postLayerName = "InputError"; + channelCode = 1; + delay = {0.000000}; + initWeightsFile = nil; + }; + + --Probes------------------------------------------------------------ + -------------------------------------------------------------------- + + V1EnergyProbe = { + groupType = "ColumnEnergyProbe"; + message = nil; + textOutputFlag = true; + probeOutputFile = "V1EnergyProbe.txt"; + triggerLayerName = nil; + energyProbe = nil; + }; + + InputErrorL2NormEnergyProbe = { + groupType = "L2NormProbe"; + targetLayer = "InputError"; + message = nil; + textOutputFlag = true; + probeOutputFile = "InputErrorL2NormEnergyProbe.txt"; + energyProbe = "V1EnergyProbe"; + coefficient = 0.5; + maskLayerName = nil; + exponent = 2; + }; + + V1L1NormEnergyProbe = { + groupType = "L1NormProbe"; + targetLayer = "V1"; + message = nil; + textOutputFlag = true; + probeOutputFile = "V1L1NormEnergyProbe.txt"; + energyProbe = "V1EnergyProbe"; + coefficient = 0.025; + maskLayerName = nil; + }; + +} --End of pvParameters + +-- Print out PetaVision approved parameter file to the console +pv.printConsole(pvParameters)