diff --git a/README.md b/README.md index b31a2b1..884077d 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,31 @@ # Creation for weight matrices for initialising the first layer of a neural network ## Dependencies & Environment - Conda needs to be installed - Setup the conda environment `weightmatrices` by running: ``` bash setup_dependencies.sh ``` ## Usage - To run the main script(s) that produce the weight matrices run: ``` bash run.sh ``` - If you want to run specific scripts, do not forget to activate the environment by running: + If you want to directly run `main.py` to use command line args (e.g. `python main.py --methods 'pca' --nhidden 10 25`), do not forget to activate the environment by running: ``` conda activate weightmatrices ``` ## Remarks By default, Datasets are downloaded and put into a /datasets directory. To avoid this you can: 1. Change the root variable in the torchvision data loader function. 2. Create the /datasets folder and corresponding subfolder and put the datasets (or a link) in there. diff --git a/env.yml b/env.yml index 2b5e8bc..a6d14d0 100644 --- a/env.yml +++ b/env.yml @@ -1,59 +1,72 @@ name: weightmatrices channels: - pytorch - conda-forge - brian-team - defaults dependencies: - blas=1.0=mkl - - ca-certificates=2020.4.5.2=hecda079_0 - - certifi=2020.4.5.2=py37_0 + - ca-certificates=2020.6.20=hecda079_0 + - certifi=2020.6.20=py37hc8dfbb8_0 + - cloudpickle=1.4.1=py_0 - cycler=0.10.0=py_2 + - cytoolz=0.10.1=py37h0b31af3_0 + - dask-core=2.19.0=py_0 + - decorator=4.4.2=py_0 - freetype=2.10.2=ha233b18_0 + - imagecodecs-lite=2019.12.3=py37h10e2902_1 + - imageio=2.8.0=py_0 - intel-openmp=2019.4=233 - joblib=0.15.1=py_0 - jpeg=9b=he5867d9_2 - kiwisolver=1.2.0=py37ha1cc60f_0 - libcxx=10.0.0=1 - libedit=3.1.20191231=haf1e3a3_0 - libffi=3.3=h0a44026_1 - libgfortran=3.0.1=h93005f0_2 - libpng=1.6.37=ha441bb4_0 - libtiff=4.1.0=hcb84e12_1 - llvm-openmp=10.0.0=h28b9765_0 - lz4-c=1.9.2=h0a44026_0 - matplotlib=3.2.1=0 - matplotlib-base=3.2.1=py37hddda452_0 - mkl=2019.4=233 - mkl-service=2.3.0=py37hfbe908c_0 - mkl_fft=1.1.0=py37hc64f4ea_0 - mkl_random=1.1.1=py37h959d312_0 - ncurses=6.2=h0a44026_1 + - networkx=2.4=py_1 - ninja=1.9.0=py37h04f5b5a_0 - numpy=1.18.1=py37h7241aed_0 - numpy-base=1.18.1=py37h3304bdc_1 - olefile=0.46=py37_0 - openssl=1.1.1g=h0b31af3_0 - pillow=7.1.2=py37h4655f20_0 - pip=20.1.1=py37_1 - pyparsing=2.4.7=pyh9f0ad1d_0 - python=3.7.7=hf48f09d_4 - python-dateutil=2.8.1=py_0 - python_abi=3.7=1_cp37m - pytorch=1.5.1=py3.7_0 + - pywavelets=1.1.1=py37h10e2902_1 + - pyyaml=5.3.1=py37h9bfed18_0 - readline=8.0=h1de35cc_0 + - scikit-image=0.17.2=py37h94625e5_1 - scikit-learn=0.22.1=py37h27c97d8_0 - scipy=1.4.1=py37h9fa6033_0 - setuptools=47.3.0=py37_0 - six=1.15.0=py_0 - sqlite=3.32.2=hffcf06c_0 + - tifffile=2020.6.3=py_0 - tk=8.6.10=hb0a8c7a_0 + - toolz=0.10.0=py_0 - torchvision=0.6.1=py37_cpu - tornado=6.0.4=py37h9bfed18_1 - tqdm=4.46.1=py_0 - wheel=0.34.2=py37_0 - xz=5.2.5=h1de35cc_0 + - yaml=0.2.5=h0b31af3_0 - zlib=1.2.11=h1de35cc_3 - zstd=1.4.4=h1990bb4_3 prefix: /Users/Bernd/anaconda3/envs/weightmatrices diff --git a/main.py b/main.py index aceddd4..2103d74 100644 --- a/main.py +++ b/main.py @@ -1,33 +1,79 @@ + import torch import numpy as np import random import matplotlib matplotlib.use('TkAgg') import matplotlib.pyplot as plt - +import argparse # own package from weightmatrices.utils import utils -from weightmatrices.algos import pca -n_h = 100 # replace by list of n_hs -################################################################################ +# read command line args and kwargs +parser = argparse.ArgumentParser() +parser.add_argument("--nhidden", nargs="*", type=int, help="number of hidden neurons", default=100) +parser.add_argument("--methods", nargs="*", help="methods to be applied to create weight matrix", default='all') +parser.add_argument("--save", type=bool, help="whether to save results or not", default=True) +args = parser.parse_args() + +# data import +print("loading data") data_loader = utils.load_data() +data_matrix = utils.getbigdatamatrix(data_loader) +n_in_features = data_matrix.shape[1] + + +# weight matrix creation + +if 'pca' in args.methods or args.methods == 'all': + from weightmatrices.algos import pca + print("creating weight matrix using PCA") + for n_h in args.nhidden: + if n_h <= n_in_features: # Number of requested components <= input dimensionality + W_pca = pca.get_weightmatrices_pca(data_loader, n_h) + if args.save: + utils.saveweightmatrix('pca'+str(n_h), W_pca) -W = pca.get_weightmatrices_pca(data_loader, n_h) +if 'ica' in args.methods or args.methods == 'all': + from weightmatrices.algos import ica + print("creating weight matrix using ICA") + for n_h in args.nhidden: + if n_h <= n_in_features: # Number of requested components <= input dimensionality + W_ica = ica.get_weightmatrices_ica(data_matrix, n_h) + if args.save: + utils.saveweightmatrix('ica'+str(n_h), W_ica) -plt.imshow(W[random.sample(range(0, n_h), 1)[0], :].reshape(28, 28), cmap = 'gray') -plt.show() +if 'sc' in args.methods or args.methods == 'all': + from weightmatrices.algos import sc + print("creating weight matrix using SC") + for n_h in args.nhidden: + W_sc = sc.get_weightmatrices_sc(data_matrix, n_h, getsparsity=True) + if args.save: + utils.saveweightmatrix('sc'+str(n_h), W_sc) -################################################################################ +if 'rg' in args.methods or args.methods == 'all': + from weightmatrices.algos import rg + print("creating weight matrix using RG") + for n_h in args.nhidden: + W_rg = rg.get_weightmatrices_rg(data_matrix, n_h) + if args.save: + utils.saveweightmatrix('rg'+str(n_h), W_rg) -import os -if not os.path.exists('./results'): - os.mkdir('./results') +if 'rp' in args.methods or args.methods == 'all': + from weightmatrices.algos import rp + print("creating weight matrix using RP") + for n_h in args.nhidden: + W_rp = rp.get_weightmatrices_rp(data_matrix, n_h) + if args.save: + utils.saveweightmatrix('rp'+str(n_h), W_rp) -np.save('./results/pca_test.npy', W) +# plotting +#plt.ion() +#plt.imshow(W_sc[random.sample(range(0, args.nhidden), 1)[0], :].reshape(28, 28), cmap = 'gray') +#plt.show() # jump to interactive mode import code code.interact(local=locals()) diff --git a/run.sh b/run.sh index ec2af33..96dbd94 100644 --- a/run.sh +++ b/run.sh @@ -1,4 +1,4 @@ echo "Activating environment..." source activate weightmatrices -echo "Creating weight matrices on EMNIST for algos: PCA, ICA, ..." -python main.py +echo "Creating weight matrices..." +python main.py --methods 'all' --nhidden 10 25 50 100 250 500 1000 diff --git a/weightmatrices/algos/ica.py b/weightmatrices/algos/ica.py index bf8c240..b11d239 100644 --- a/weightmatrices/algos/ica.py +++ b/weightmatrices/algos/ica.py @@ -1,16 +1,17 @@ - -import time -from tqdm import tqdm +# Independent Component Analysis from sklearn.decomposition import FastICA from weightmatrices.utils import utils -def get_ica_trafo_matrix(data_loader, n_h): - transformer = FastICA(n_components=n_h) - +def get_ica_trafo_matrix(data_matrix, n_h): + transformer = FastICA(n_components=n_h, tol=0.001) + data_transformed = transformer.fit_transform(data_matrix) + return transformer.components_ -def get_weightmatrices_ica(data_loader, n_h): +def get_weightmatrices_ica(data_matrix, n_h): print("Creating weigth matrix for ICA for "+str(n_h)+" hidden neurons...") - s = data_loader.dataset.data.shape - n_in_features = s[-1]*s[-2] + n_in_features = data_matrix.shape[1] assert n_h <= n_in_features, "Number of requested independent components higher than input dimensionality!" + W = get_ica_trafo_matrix(data_matrix, n_h) + W = utils.normalise_weightmatrix(W) + return W diff --git a/weightmatrices/algos/pca.py b/weightmatrices/algos/pca.py index eef03e0..3186e46 100644 --- a/weightmatrices/algos/pca.py +++ b/weightmatrices/algos/pca.py @@ -1,23 +1,24 @@ +# Principal Component Analysis import time from tqdm import tqdm from sklearn.decomposition import IncrementalPCA from weightmatrices.utils import utils def get_pca_trafo_matrix(data_loader, n_h): transformer = IncrementalPCA(n_components=n_h) for (d, t) in tqdm(data_loader): s = d.shape transformer.partial_fit(d.numpy().reshape(data_loader.batch_size, s[-1]*s[-2])) return transformer.components_ def get_weightmatrices_pca(data_loader, n_h): - print("Creating weigth matrix for PCA for "+str(n_h)+" hidden neurons...") + print("creating weigth matrix for PCA for "+str(n_h)+" hidden neurons...") s = data_loader.dataset.data.shape n_in_features = s[-1]*s[-2] assert n_h <= n_in_features, "Number of requested principal components higher than input dimensionality!" W = get_pca_trafo_matrix(data_loader, n_h) W = utils.normalise_weightmatrix(W) return W diff --git a/weightmatrices/algos/rg.py b/weightmatrices/algos/rg.py new file mode 100644 index 0000000..9619e41 --- /dev/null +++ b/weightmatrices/algos/rg.py @@ -0,0 +1,30 @@ +# Random Gabors + +import numpy as np +import random +from weightmatrices.utils import utils + +def get_gabor_kernel(lbda, theta, psi, sigma, gamma, N): + w = np.zeros((N, N)) + rm = np.array([[np.cos(theta), np.sin(theta)],[-np.sin(theta), np.cos(theta)]]) + for x in range(N): + for y in range(N): + r = np.matmul(rm, np.array([x, y])-(N/2)) + w[x, y] = np.exp(-(r[0]**2 + gamma**2*r[1]**2)/(2*sigma**2)) * np.cos(2*np.pi*r[0]/lbda + psi) + + return w + +# these are heuristic boundaries for random Gabors +def get_random_gabor(N): + return get_gabor_kernel((2*N-N/4)*random.random()+N/4, + 2*np.pi*random.random(), 2*np.pi*random.random(), + (N-N/8)*random.random()+N/8, random.random(), N) + + +def get_weightmatrices_rg(data_matrix, n_h): + N = int(np.sqrt(data_matrix.shape[1])) + W = np.zeros((n_h, N**2)) + for i in range(n_h): + W[i, :] = get_random_gabor(N).flatten() + W = utils.normalise_weightmatrix(W) + return W diff --git a/weightmatrices/algos/rp.py b/weightmatrices/algos/rp.py new file mode 100644 index 0000000..df4a3f9 --- /dev/null +++ b/weightmatrices/algos/rp.py @@ -0,0 +1,10 @@ +# Random Projections + +import numpy as np +from weightmatrices.utils import utils + +def get_weightmatrices_rp(data_matrix, n_h): + N = data_matrix.shape[1] + W = np.random.randn(n_h, N) + W = utils.normalise_weightmatrix(W) + return W diff --git a/weightmatrices/algos/sc.py b/weightmatrices/algos/sc.py index 3fa98bd..46ce3b2 100644 --- a/weightmatrices/algos/sc.py +++ b/weightmatrices/algos/sc.py @@ -1,21 +1,36 @@ +# Sparse Coding import time from tqdm import tqdm +import numpy as np from sklearn.decomposition import dict_learning_online from weightmatrices.utils import utils +def get_sparsity(code): + return np.sum(code == 0.) / code.size +def get_sc_trafo_matrix(data_matrix, n_h, getsparsity=True): + dictout = dict_learning_online(data_matrix, n_components=n_h, alpha=1, n_iter=100, + return_code=getsparsity, dict_init=None, callback=None, + batch_size=3, verbose=False, shuffle=True, + n_jobs=None, method='cd', iter_offset=0, + random_state=None, return_inner_stats=False, + inner_stats=None, return_n_iter=False, + positive_dict=False, positive_code=True, + method_max_iter=1000) -def get_weightmatrices_sc(data_loader): - X = ? - out_tuple = dict_learning_online(X, n_components=2, *, alpha=1, n_iter=100, - return_code=True, dict_init=None, callback=None, - batch_size=3, verbose=False, shuffle=True, - n_jobs=None, method='lars', iter_offset=0, - random_state=None, return_inner_stats=False, - inner_stats=None, return_n_iter=False, - positive_dict=False, positive_code=False, - method_max_iter=1000): - W = ? + if getsparsity: + code = dictout[0] + print("sparsity for "+str(n_h)+" hidden neurons: "+str(get_sparsity(code))) + dictionary = dictout[1] + else: + dictionary = dictout + return dictionary + + +def get_weightmatrices_sc(data_matrix, n_h, getsparsity=True): + print("creating weigth matrix for SC for "+str(n_h)+" hidden neurons...") + W = get_sc_trafo_matrix(data_matrix, n_h, getsparsity=getsparsity) + W = utils.normalise_weightmatrix(W) return W diff --git a/weightmatrices/utils/utils.py b/weightmatrices/utils/utils.py index d145f9c..d56f186 100644 --- a/weightmatrices/utils/utils.py +++ b/weightmatrices/utils/utils.py @@ -1,24 +1,38 @@ import torch +from tqdm import tqdm +import numpy as np from torchvision import datasets, transforms from sklearn.preprocessing import normalize def load_data(dataset='EMNIST', batch_size=100000): if dataset=='EMNIST': trans = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) data = datasets.EMNIST('./datasets/', 'bymerge', transform=trans, download = True) data_loader = torch.utils.data.DataLoader( dataset=data, batch_size=batch_size, shuffle=False, drop_last=True) return data_loader +def saveweightmatrix(filename, W): + import os + if not os.path.exists('./results'): + os.mkdir('./results') + + np.save('./results/'+filename, W) + def getbigdatamatrix(data_loader): - # Needed for ICA and SC and stuff... - # loop over data_loader and concatenation - return X + print("create big data matrix out of data loader (this takes a minute or so)...") + X = [] + for (d, t) in tqdm(data_loader): + s = d.shape + X.append(d.numpy().reshape(data_loader.batch_size, s[-1]*s[-2])) + + return np.vstack(X) + def normalise_weightmatrix(W): return normalize(W)