commit 25fa687f7f63ca17f013dfe5fb7cfeea6e82c929 Author: raphael Date: Thu Jul 22 22:24:50 2021 +0200 initial commit, adds notes.md for instructions, adds data, adds python code diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..f4241e0 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,37 @@ +FROM tensorflow/tensorflow:1.13.2-gpu + +## Install updates and network tool +RUN apt-get update -y && apt-get upgrade -y && apt install net-tools -y + +## Install basic functions +RUN apt-get install sudo -y + +## Install git +RUN apt-get install git -y + +## Install python requirements +COPY requirements.txt . +RUN pip install -r requirements.txt + +## Create user and group +ARG HOST_USER_UID=1000 +ARG HOST_USER_GID=1000 +RUN groupadd -g $HOST_USER_GID containergroup +RUN useradd -m -l -u $HOST_USER_UID -g $HOST_USER_GID containeruser + +## Passwordless sudo for user +RUN usermod -aG sudo containeruser +RUN echo "containeruser ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/containeruser && \ + chmod 0440 /etc/sudoers.d/containeruser + +## Activate User +USER containeruser + +## Set working directory +WORKDIR /home/containeruser + +## Workaround for vscode bug +ENV HOME=/home/containeruser + +## Keep container running forever +CMD tail -f /dev/null diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..9258f8b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +{ + "name": "siamese", + "dockerComposeFile": "docker-compose.yml", + "workspaceMount": "/workspace", + "workspaceFolder": "/workspace", + "service": "devcontainer", + "shutdownAction": "stopCompose", + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker" + ] +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..e016851 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,22 @@ +version: '2.3' +services: + devcontainer: + + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + HOST_USER_UID: 1000 + HOST_USER_GID: 1000 + + network_mode: host + environment: + - DISPLAY=$DISPLAY + runtime: nvidia + + volumes: + - ..:/workspace + - ~/.gitconfig:/home/containeruser/.gitconfig + - ~/.ssh:/home/containeruser/.ssh + + command: sleep infinity diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..efd5831 --- /dev/null +++ b/.gitignore @@ -0,0 +1,114 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +/env + +# idea files +.idea/ + +# model checkpoint data +checkpoint +model_checkpoint +siamese_checkpoint diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..11b1e4e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 aspamers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0701f2d --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +# Siamese Neural Network for Keras + +This project provides a lightweight, easy to use and flexible siamese neural network module for use with the Keras +framework. + +Siamese neural networks are used to generate embeddings that describe inter and extra class relationships. +This makes Siamese Networks like many other similarity learning algorithms suitable as a pre-training step for many +classification problems. + +An example of the siamese network module being used to produce a noteworthy 99.85% validation performance on the MNIST +dataset with no data augmentation and minimal modification from the Keras example is provided. + +## Installation + +Create and activate a virtual environment for the project. +```sh +$ virtualenv env +$ source env/bin/activate +``` + +To install the module directly from GitHub: +``` +$ pip install git+https://github.com/aspamers/siamese +``` + +The module will install keras and numpy but no back-end (like tensorflow). This is deliberate since it leaves the module +decoupled from any back-end and gives you a chance to install whatever backend you prefer. + +To install tensorflow: +``` +$ pip install tensorflow +``` + +To install tensorflow with gpu support: +``` +$ pip install tensorflow-gpu +``` + +## To run examples + +With the activated virtual environment with the installed python package run the following commands. + +To run the mnist baseline example: +``` +$ python mnist_example.py +``` + +To run the mnist siamese pretrained example: +``` +$ python mnist_siamese_example.py +``` + +## Usage +For detailed usage examples please refer to the examples and unit test modules. If the instructions are not sufficient +feel free to make a request for improvements. + +- Import the module +```python +from siamese import SiameseNetwork +``` + +- Load or generate some data. +```python +x_train = np.random.rand(100, 3) +y_train = np.random.randint(num_classes, size=100) + +x_test = np.random.rand(30, 3) +y_test = np.random.randint(num_classes, size=30) +``` + +- Design a base model +```python +def create_base_model(input_shape): + model_input = Input(shape=input_shape) + embedding = Flatten()(model_input) + embedding = Dense(128)(embedding) + return Model(model_input, embedding) +``` + +- Design a head model +```python +def create_head_model(embedding_shape): + embedding_a = Input(shape=embedding_shape) + embedding_b = Input(shape=embedding_shape) + + head = Concatenate()([embedding_a, embedding_b]) + head = Dense(4)(head) + head = BatchNormalization()(head) + head = Activation(activation='sigmoid')(head) + + head = Dense(1)(head) + head = BatchNormalization()(head) + head = Activation(activation='sigmoid')(head) + + return Model([embedding_a, embedding_b], head) +``` +- Create an instance of the SiameseNetwork class +```python +base_model = create_base_model(input_shape) +head_model = create_head_model(base_model.output_shape) +siamese_network = SiameseNetwork(base_model, head_model) +``` + +- Compile the model +```python +siamese_network.compile(loss='binary_crossentropy', optimizer=keras.optimizers.adam()) +``` + +- Train the model +```python +siamese_network.fit(x_train, y_train, + validation_data=(x_test, y_test), + batch_size=64, + epochs=epochs) +``` + +## Development Environment +Create and activate a test virtual environment for the project. +```sh +$ virtualenv env +$ source env/bin/activate +``` + +Install requirements +```sh +$ pip install -r requirements.txt +``` + +Install the backend of your choice. +``` +$ pip install tensorflow +``` + +Run tests +```sh +$ pytest tests/test_siamese.py +``` + +## Development container +To set up the vscode development container follow the instructions at the link provided: +https://github.com/aspamers/vscode-devcontainer + +You will also need to install the nvidia docker gpu passthrough layer: +https://github.com/NVIDIA/nvidia-docker diff --git a/data b/data new file mode 120000 index 0000000..3e1c8cc --- /dev/null +++ b/data @@ -0,0 +1 @@ +/home/creation/files/data/ \ No newline at end of file diff --git a/mnist_example.py b/mnist_example.py new file mode 100644 index 0000000..f14a0c1 --- /dev/null +++ b/mnist_example.py @@ -0,0 +1,94 @@ +""" +This is a modified version of the Keras mnist example. +https://keras.io/examples/mnist_cnn/ + +Instead of using a fixed number of epochs this version continues to train +until the stop criteria is reached. + +Model performance should be around 99.4% after training. +""" + +from __future__ import print_function +import keras +from keras.datasets import mnist +from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Activation +from keras import backend as K +from keras.callbacks import ModelCheckpoint, EarlyStopping +from keras.models import Model +from keras.layers import Input, Flatten, Dense + +batch_size = 128 +num_classes = 10 +epochs = 999999 + +# input image dimensions +img_rows, img_cols = 28, 28 + +# the data, split between train and test sets +(x_train, y_train), (x_test, y_test) = mnist.load_data() + +if K.image_data_format() == 'channels_first': + x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) + x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) + input_shape = (1, img_rows, img_cols) +else: + x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) + x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) + input_shape = (img_rows, img_cols, 1) + +x_train = x_train.astype('float32') +x_test = x_test.astype('float32') +x_train /= 255 +x_test /= 255 + +y_train = keras.utils.to_categorical(y_train, num_classes) +y_test = keras.utils.to_categorical(y_test, num_classes) + + +def create_base_network(input_shape): + input = Input(shape=input_shape) + x = Conv2D(32, kernel_size=(3, 3), + input_shape=input_shape)(input) + x = BatchNormalization()(x) + x = Activation(activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + x = Conv2D(64, kernel_size=(3, 3))(x) + x = BatchNormalization()(x) + x = Activation(activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + x = Flatten()(x) + x = Dense(128)(x) + x = BatchNormalization()(x) + x = Activation(activation='relu')(x) + x = Dense(num_classes)(x) + x = BatchNormalization()(x) + x = Activation(activation='softmax')(x) + return Model(input, x) + + +model = create_base_network(input_shape) +model.compile(loss=keras.losses.categorical_crossentropy, + optimizer=keras.optimizers.adam(), + metrics=['accuracy']) + +checkpoint_path = "./checkpoint" + +callbacks = [ + EarlyStopping(monitor='val_acc', patience=10, verbose=0), + ModelCheckpoint(checkpoint_path, + monitor='val_acc', + save_best_only=True, + verbose=0) +] +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=epochs, + verbose=1, + callbacks=callbacks, + validation_data=(x_test, y_test)) + +model.load_weights(checkpoint_path) + +score = model.evaluate(x_test, y_test, verbose=0) +print('Test loss:', score[0]) +print('Test accuracy:', score[1]) diff --git a/mnist_siamese_example.py b/mnist_siamese_example.py new file mode 100644 index 0000000..737f6ea --- /dev/null +++ b/mnist_siamese_example.py @@ -0,0 +1,299 @@ +""" +This is a modified version of the Keras mnist example. +https://keras.io/examples/mnist_cnn/ + +Instead of using a fixed number of epochs this version continues to train until a stop criteria is reached. + +A siamese neural network is used to pre-train an embedding for the network. The resulting embedding is then extended +with a softmax output layer for categorical predictions. + +Model performance should be around 99.84% after training. The resulting model is identical in structure to the one in +the example yet shows considerable improvement in relative error confirming that the embedding learned by the siamese +network is useful. +""" + +from __future__ import print_function +import tensorflow.keras as keras +from tensorflow.keras.datasets import mnist +from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Activation, Concatenate +from tensorflow.keras import backend as K +from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping +from tensorflow.keras.models import Model +from tensorflow.keras.layers import Input, Flatten, Dense + +from siamese import SiameseNetwork + +import os, math, numpy as np +from PIL import Image + +import pdb + +batch_size = 128 +num_classes = 131 +epochs = 999999 + +# input image dimensions +img_rows, img_cols = 28, 28 + +def createTrainingData(): + base_dir = '../towards/data/fruits-360/Training/' + train_test_split = 0.7 + no_of_files_in_each_class = 80 + + #Read all the folders in the directory + folder_list = os.listdir(base_dir) + print( len(folder_list), "categories found in the dataset") + + #Declare training array + cat_list = [] + x = [] + names = [] + y = [] + y_label = 0 + + #Using just 5 images per category + for folder_name in folder_list: + files_list = os.listdir(os.path.join(base_dir, folder_name)) + temp=[] + for file_name in files_list[:no_of_files_in_each_class]: + temp.append(len(x)) + x.append(np.asarray(Image.open(os.path.join(base_dir, folder_name, file_name)).convert('RGB').resize((img_rows, img_cols)))) + names.append(folder_name + "/" + file_name) + y.append(y_label) + y_label+=1 + cat_list.append(temp) + + cat_list = np.asarray(cat_list) + x = np.asarray(x)/255.0 + y = np.asarray(y) + print('X, Y shape',x.shape, y.shape, cat_list.shape) + + + #Training Split + x_train, y_train, cat_train, x_val, y_val, cat_test = [], [], [], [], [], [] + + train_split = math.floor((train_test_split) * no_of_files_in_each_class) + test_split = math.floor((1-train_test_split) * no_of_files_in_each_class) + + train_count = 0 + test_count = 0 + for i in range(len(x)-1): + if i % no_of_files_in_each_class == 0: + cat_train.append([]) + cat_test.append([]) + class_train_count = 1 + class_test_count = 1 + + if i % math.floor(1/train_test_split) == 0 and class_test_count < test_split: + x_val.append(x[i]) + y_val.append(y[i]) + cat_test[-1].append(test_count) + test_count += 1 + class_test_count += 1 + + elif class_train_count < train_split: + x_train.append(x[i]) + y_train.append(y[i]) + cat_train[-1].append(train_count) + train_count += 1 + class_train_count += 1 + + + x_val = np.array(x_val) + y_val = np.array(y_val) + x_train = np.array(x_train) + y_train = np.array(y_train) + cat_train = np.array(cat_train) + cat_test = np.array(cat_test) + + + print('X&Y shape of training data :',x_train.shape, 'and', + y_train.shape, cat_train.shape) + print('X&Y shape of testing data :' , x_val.shape, 'and', + y_val.shape, cat_test.shape) + + return (x_train, y_train), (x_val, y_val), cat_train + + +# the data, split between train and test sets +# (x_train, y_train), (x_test, y_test) = mnist.load_data() +# channels = 1 + +(x_train, y_train), (x_test, y_test), cat_train = createTrainingData() +channels = 3 + +if K.image_data_format() == 'channels_first': + x_train = x_train.reshape(x_train.shape[0], channels, img_rows, img_cols) + x_test = x_test.reshape(x_test.shape[0], channels, img_rows, img_cols) + input_shape = (channels, img_rows, img_cols) +else: + x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, channels) + x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, channels) + input_shape = (img_rows, img_cols, channels) + +x_train = x_train.astype('float32') +x_test = x_test.astype('float32') +x_train /= 255 +x_test /= 255 + +pdb.set_trace() + +def create_own_base_model(input_shape): + model_input = Input(shape=input_shape) + + embedding = Conv2D(32, kernel_size=(10, 10), input_shape=input_shape)(model_input) + embedding = MaxPooling2D(pool_size=(2, 2))(embedding) + embedding = Conv2D(64, kernel_size=(7, 7))(embedding) + embedding = MaxPooling2D(pool_size=(2, 2))(embedding) + embedding = Conv2D(128, kernel_size=(4, 4))(embedding) + embedding = MaxPooling2D(pool_size=(2, 2))(embedding) + embedding = Conv2D(256, kernel_size=(4, 4))(embedding) + embedding = MaxPooling2D(pool_size=(2, 2))(embedding) + embedding = Flatten()(embedding) + embedding = Dense(4096, activation='sigmoid')(embedding) + embedding = BatchNormalization()(embedding) + embedding = Activation(activation='relu')(embedding) + + return Model(model_input, embedding) + +def create_base_model(input_shape): + model_input = Input(shape=input_shape) + + embedding = Conv2D(32, kernel_size=(3, 3), input_shape=input_shape)(model_input) + embedding = BatchNormalization()(embedding) + embedding = Activation(activation='relu')(embedding) + embedding = MaxPooling2D(pool_size=(2, 2))(embedding) + embedding = Conv2D(64, kernel_size=(3, 3))(embedding) + embedding = BatchNormalization()(embedding) + embedding = Activation(activation='relu')(embedding) + embedding = MaxPooling2D(pool_size=(2, 2))(embedding) + embedding = Flatten()(embedding) + embedding = Dense(128)(embedding) + embedding = BatchNormalization()(embedding) + embedding = Activation(activation='relu')(embedding) + + return Model(model_input, embedding) + + +def create_head_model(embedding_shape): + embedding_a = Input(shape=embedding_shape[1:]) + embedding_b = Input(shape=embedding_shape[1:]) + + head = Concatenate()([embedding_a, embedding_b]) + head = Dense(8)(head) + head = BatchNormalization()(head) + head = Activation(activation='sigmoid')(head) + + head = Dense(1)(head) + head = BatchNormalization()(head) + head = Activation(activation='sigmoid')(head) + + return Model([embedding_a, embedding_b], head) + +def get_batch(x_train, y_train, x_test, y_test, cat_train, batch_size=64): + + temp_x = x_train + temp_cat_list = cat_train + start=0 + batch_x=[] + + batch_y = np.zeros(batch_size) + batch_y[int(batch_size/2):] = 1 + np.random.shuffle(batch_y) + + class_list = np.random.randint(start, len(cat_train), batch_size) + batch_x.append(np.zeros((batch_size, 100, 100, 3))) + batch_x.append(np.zeros((batch_size, 100, 100, 3))) + + for i in range(0, batch_size): + batch_x[0][i] = temp_x[np.random.choice(temp_cat_list[class_list[i]])] + #If train_y has 0 pick from the same class, else pick from any other class + if batch_y[i]==0: + r = np.random.choice(temp_cat_list[class_list[i]]) + batch_x[1][i] = temp_x[r] + + else: + temp_list = np.append(temp_cat_list[:class_list[i]].flatten(), temp_cat_list[class_list[i]+1:].flatten()) + batch_x[1][i] = temp_x[np.random.choice(temp_list)] + + return(batch_x, batch_y) + + +num_classes = 131 +epochs = 2000 + +base_model = create_base_model(input_shape) +head_model = create_head_model(base_model.output_shape) + +siamese_network = SiameseNetwork(base_model, head_model) +siamese_network.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) + +siamese_checkpoint_path = "./siamese_checkpoint" + +siamese_callbacks = [ + # EarlyStopping(monitor='val_accuracy', patience=10, verbose=0), + ModelCheckpoint(siamese_checkpoint_path, monitor='val_accuracy', save_best_only=True, verbose=0) +] + +# batch_size = 64 +# for epoch in range(1, epochs): +# batch_x, batch_y = get_batch(x_train, y_train, x_test, y_test, cat_train, train_size, batch_size) +# loss = siamese_network.train_on_batch(batch_x, batch_y) +# print('Epoch:', epoch, ', Loss:', loss) + +siamese_network.fit(x_train, y_train, + validation_data=(x_test, y_test), + batch_size=45, + epochs=epochs, + callbacks=siamese_callbacks) + +# try: +# siamese_network = keras.models.load_model(siamese_checkpoint_path) +# except Exception as e: +# print(e) +# print("!!!!!!") +# siamese_network.load_weights(siamese_checkpoint_path) + +embedding = base_model.outputs[-1] + +y_train = keras.utils.to_categorical(y_train) +y_test = keras.utils.to_categorical(y_test) + +# Add softmax layer to the pre-trained embedding network +embedding = Dense(num_classes)(embedding) +embedding = BatchNormalization()(embedding) +embedding = Activation(activation='sigmoid')(embedding) + +model = Model(base_model.inputs[0], embedding) +model.compile(loss=keras.losses.binary_crossentropy, + optimizer=keras.optimizers.Adam(), + metrics=['accuracy']) + +model_checkpoint_path = "./model_checkpoint" + +model__callbacks = [ + # EarlyStopping(monitor='val_accuracy', patience=10, verbose=0), + ModelCheckpoint(model_checkpoint_path, monitor='val_accuracy', save_best_only=True, verbose=0) +] + +# for e in range(1, epochs): +# batch_x, batch_y = get_batch(x_train, y_train, x_test, y_test, cat_train, train_size, batch_size) +# loss = model.train_on_batch(batch_x, batch_y) +# print('Epoch:', epoch, ', Loss:', loss) + +model.fit(x_train, y_train, + batch_size=128, + epochs=epochs, + callbacks=model__callbacks, + validation_data=(x_test, y_test)) +# try: +# model = keras.models.load_model(model_checkpoint_path) +# except Exception as e: +# print(e) +# print("!!!!!!") + +# model.load_weights(model_checkpoint_path) + +score = model.evaluate(x_test, y_test, verbose=0) +print('Test loss:', score[0]) +print('Test accuracy:', score[1]) diff --git a/model.png b/model.png new file mode 100644 index 0000000..0257743 Binary files /dev/null and b/model.png differ diff --git a/mymodel.png b/mymodel.png new file mode 100644 index 0000000..200eda1 Binary files /dev/null and b/mymodel.png differ diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..9585e3e --- /dev/null +++ b/notes.md @@ -0,0 +1,39 @@ +the steps taken so far, which lead to a successfull detection of an image + +- train the model defined in mnist_siamese_example, which uses the 'siamese.py' model to + create a siamese keras model. + + - in this mnist siamese example, the data collection has been updated form the mnist drawing + sample to the fruit sample. Lots of work went into setting the arrays up correctly, because the + example from towards data science did not correctly seperate the classes. He had originally used + 91 classes for teching and the rest for testing, where I now use images of every class for + teaching _and_ training. + + - The images were shrunken down to 28 x 28 so the model defined in the siamese example could be used + without adaption + + - in this example, there is two teachings going on, once he trains the siamese model (which is saved under + 'siamese_checkpoint' and then he reteaches a new model based on this one, with some additonal layers ontop + + I'm not yet sure what these do [todo] but 'I'll figure it out. + +- after you've successfully trained the model, it's now saved to 'model_checkpoint' or 'siamese_checkpoint' + +- The following steps can be used to classify two images: + Note, that it was so far only tested using images in a 'pdb' shell from the mnist_siamese_example script + +``` +import tensorflow.keras as keras +from PIL import image +model = keras.models.load_model('./siamese_checkpoint') +image1 = np.asarray(Image.open('../towards/data/fruits-360/Training/Avocado/r_254_100.jpg').convert('RGB').resize((28, 28))) / 255 / 255 +image2 = np.asarray(Image.open('../towards/data/fruits-360/Training/Avocado/r_250_100.jpg').convert('RGB').resize((28, 28))) / 255 / 255 +# note that the double division through 255 is only because the model bas taught with this double division, depends on +# the input numbers of course + +output = model.predict([np.array([image2]), np.array([image1])]) +# Note here, that the cast to np.array is nencessary - otherwise the input vector is malformed + +print(output) +``` + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e0225fb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +keras==2.2.4 +numpy==1.16.4 +pytest==4.6.4 +pep8==1.7.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1914485 --- /dev/null +++ b/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup + +setup( + name='siamese', + version='0.1', + packages=[''], + url='https://github.com/aspamers/siamese', + license='MIT', + author='Abram Spamers', + author_email='aspamers@gmail.com', + install_requires=[ + 'keras', 'numpy', + ], + description='An easy to use Keras Siamese Neural Network implementation' +) diff --git a/siamese.py b/siamese.py new file mode 100644 index 0000000..9f8f9ca --- /dev/null +++ b/siamese.py @@ -0,0 +1,291 @@ +""" +Siamese neural network module. +""" + +import random, math +import numpy as np + +from tensorflow.keras.layers import Input +from tensorflow.keras.models import Model + +import pdb + + +class SiameseNetwork: + """ + A simple and lightweight siamese neural network implementation. + + The SiameseNetwork class requires the base and head model to be defined via the constructor. The class exposes + public methods that allow it to behave similarly to a regular Keras model by passing kwargs through to the + underlying keras model object where possible. This allows Keras features like callbacks and metrics to be used. + """ + def __init__(self, base_model, head_model): + """ + Construct the siamese model class with the following structure. + + ------------------------------------------------------------------- + input1 -> base_model | + --> embedding --> head_model --> binary output + input2 -> base_model | + ------------------------------------------------------------------- + + :param base_model: The embedding model. + * Input shape must be equal to that of data. + :param head_model: The discriminator model. + * Input shape must be equal to that of embedding + * Output shape must be equal to 1.. + """ + # Set essential parameters + self.base_model = base_model + self.head_model = head_model + + # Get input shape from base model + self.input_shape = self.base_model.input_shape[1:] + + # Initialize siamese model + self.siamese_model = None + self.__initialize_siamese_model() + + def compile(self, *args, **kwargs): + """ + Configures the model for training. + + Passes all arguments to the underlying Keras model compile function. + """ + self.siamese_model.compile(*args, **kwargs) + + def train_on_batch(self, *args, **kwargs): + return self.siamese_model.train_on_batch(args[0], args[1]) + + def fit(self, *args, **kwargs): + """ + Trains the model on data generated batch-by-batch using the siamese network generator function. + + Redirects arguments to the fit_generator function. + """ + x_train = args[0] + y_train = args[1] + x_test, y_test = kwargs.pop('validation_data') + batch_size = kwargs.pop('batch_size') + + train_generator = self.__pair_generator(x_train, y_train, batch_size) + train_steps = math.floor(max(len(x_train) / batch_size, 1)) + test_generator = self.__pair_generator(x_test, y_test, batch_size) + test_steps = math.floor(max(len(x_test) / batch_size, 1)) + + pdb.set_trace() + + self.siamese_model.fit(train_generator, + steps_per_epoch=train_steps, + validation_data=test_generator, + validation_steps=test_steps, **kwargs) + + def fit_generator(self, x_train, y_train, x_test, y_test, batch_size, *args, **kwargs): + """ + Trains the model on data generated batch-by-batch using the siamese network generator function. + + :param x_train: Training input data. + :param y_train: Training output data. + :param x_test: Validation input data. + :param y_test: Validation output data. + :param batch_size: Number of pairs to generate per batch. + """ + train_generator = self.__pair_generator(x_train, y_train, batch_size) + train_steps = max(len(x_train) / batch_size, 1) + test_generator = self.__pair_generator(x_test, y_test, batch_size) + test_steps = max(len(x_test) / batch_size, 1) + self.siamese_model.fit_generator(train_generator, + steps_per_epoch=train_steps, + validation_data=test_generator, + validation_steps=test_steps, + *args, **kwargs) + + + def load_weights(self, checkpoint_path): + """ + Load siamese model weights. This also affects the reference to the base and head models. + + :param checkpoint_path: Path to the checkpoint file. + """ + self.siamese_model.load_weights(checkpoint_path) + + def evaluate(self, *args, **kwargs): + """ + Evaluate the siamese network with the same generator that is used to train it. Passes arguments through to the + underlying Keras function so that callbacks etc can be used. + + Redirects arguments to the evaluate_generator function. + + :return: A tuple of scores + """ + x = args[0] + y = args[1] + batch_size = kwargs.pop('batch_size') + + generator = self.__pair_generator(x, y, batch_size) + steps = len(x) / batch_size + return self.siamese_model.evaluate_generator(generator, steps=steps, **kwargs) + + def evaluate_generator(self, x, y, batch_size, *args, **kwargs): + """ + Evaluate the siamese network with the same generator that is used to train it. Passes arguments through to the + underlying Keras function so that callbacks etc can be used. + + :param x: Input data + :param y: Class labels + :param batch_size: Number of pairs to generate per batch. + :return: A tuple of scores + """ + generator = self.__pair_generator(x, y, batch_size=batch_size) + steps = len(x) / batch_size + return self.siamese_model.evaluate_generator(generator, steps=steps, *args, **kwargs) + + def __initialize_siamese_model(self): + """ + Create the siamese model structure using the supplied base and head model. + """ + input_a = Input(shape=self.input_shape) + input_b = Input(shape=self.input_shape) + + processed_a = self.base_model(input_a) + processed_b = self.base_model(input_b) + + head = self.head_model([processed_a, processed_b]) + self.siamese_model = Model([input_a, input_b], head) + + def __create_pairs(self, x, class_indices, batch_size, num_classes): + """ + Create a numpy array of positive and negative pairs and their associated labels. + + :param x: Input data + :param class_indices: A python list of lists that contains each of the indices in the input data that belong + to each class. It is used to find and access elements in the input data that belong to a desired class. + * Example usage: + * element_index = class_indices[class][index] + * element = x[element_index] + :param batch_size: The number of pair samples to create. + :param num_classes: number of classes in the supplied input data + :return: A tuple of (Numpy array of pairs, Numpy array of labels) + """ + num_pairs = batch_size / 2 + positive_pairs, positive_labels = self.__create_positive_pairs(x, class_indices, num_pairs, num_classes) + negative_pairs, negative_labels = self.__create_negative_pairs(x, class_indices, num_pairs, num_classes) + return np.array(positive_pairs + negative_pairs), np.array(positive_labels + negative_labels) + + def __create_positive_pairs(self, x, class_indices, num_positive_pairs, num_classes): + """ + Create a list of positive pairs and labels. A positive pair is defined as two input samples of the same class. + + :param x: Input data + :param class_indices: A python list of lists that contains each of the indices in the input data that belong + to each class. It is used to find and access elements in the input data that belong to a desired class. + * Example usage: + * element_index = class_indices[class][index] + * element = x[element_index] + :param num_positive_pairs: The number of positive pair samples to create. + :param num_classes: number of classes in the supplied input data + :return: A tuple of (python list of positive pairs, python list of positive labels) + """ + positive_pairs = [] + positive_labels = [] + + for _ in range(int(num_positive_pairs)): + class_1 = random.randint(0, num_classes - 1) + num_elements = len(class_indices[class_1]) + + if num_elements == 0: + return [], [] + index_1, index_2 = self.__randint_unequal(0, num_elements - 1) + + element_index_1, element_index_2 = class_indices[class_1][index_1], class_indices[class_1][index_2] + positive_pairs.append([x[element_index_1], x[element_index_2]]) + positive_labels.append([1.0]) + return positive_pairs, positive_labels + + def __create_negative_pairs(self, x, class_indices, num_negative_pairs, num_classes): + """ + Create a list of negative pairs and labels. A negative pair is defined as two input samples of different class. + + :param x: Input data + :param class_indices: A python list of lists that contains each of the indices in the input data that belong + to each class. It is used to find and access elements in the input data that belong to a desired class. + * Example usage: + * element_index = class_indices[class][index] + * element = x[element_index] + :param num_negative_pairs: The number of negative pair samples to create. + :param num_classes: number of classes in the supplied input data + :return: A tuple of (python list of negative pairs, python list of negative labels) + """ + negative_pairs = [] + negative_labels = [] + + if num_classes == 0: + return [], [] + + for _ in range(int(num_negative_pairs)): + cls_1, cls_2 = self.__randint_unequal(0, num_classes - 1) + + try: + index_1 = random.randint(0, len(class_indices[cls_1]) - 1) + index_2 = random.randint(0, len(class_indices[cls_2]) - 1) + except Exception as e: + print(e) + pdb.set_trace() + + + element_index_1, element_index_2 = class_indices[cls_1][index_1], class_indices[cls_2][index_2] + negative_pairs.append([x[element_index_1], x[element_index_2]]) + negative_labels.append([0.0]) + return negative_pairs, negative_labels + + def __pair_generator(self, x, y, batch_size): + """ + Creates a python generator that produces pairs from the original input data. + :param x: Input data + :param y: Integer class labels + :param batch_size: The number of pair samples to create per batch. + :return: + """ + class_indices, num_classes = self.__get_class_indices(y) + while True: + pairs, labels = self.__create_pairs(x, class_indices, batch_size, num_classes) + + # The siamese network expects two inputs and one output. Split the pairs into a list of inputs. + yield [pairs[:, 0], pairs[:, 1]], labels + + def __get_class_indices(self, y): + """ + Create a python list of lists that contains each of the indices in the input data that belong + to each class. It is used to find and access elements in the input data that belong to a desired class. + * Example usage: + * element_index = class_indices[class][index] + * element = x[element_index] + :param y: Integer class labels + :return: Python list of lists + """ + num_classes = np.max(y) + 1 + return [np.where(y == i)[0] for i in range(num_classes)], num_classes + + @staticmethod + def __randint_unequal(lower, upper): + """ + Get two random integers that are not equal. + + Note: In some cases (such as there being only one sample of a class) there may be an endless loop here. This + will only happen on fairly exotic datasets though. May have to address in future. + :param lower: Lower limit inclusive of the random integer. + :param upper: Upper limit inclusive of the random integer. Need to use -1 for random indices. + :return: Tuple of (integer, integer) + """ + + int_1 = random.randint(lower, upper) + int_2 = random.randint(lower, upper) + + tries = 0 + while int_1 == int_2: + tries += 1 + if tries > 10: + break + int_1 = random.randint(lower, upper) + int_2 = random.randint(lower, upper) + return int_1, int_2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_siamese.py b/tests/test_siamese.py new file mode 100644 index 0000000..d569dcf --- /dev/null +++ b/tests/test_siamese.py @@ -0,0 +1,80 @@ + +""" +Tests for the siamese neural network module +""" + +import numpy as np +import keras +from keras import Model, Input +from keras.layers import Concatenate, Dense, BatchNormalization, Activation + +from siamese import SiameseNetwork + + +def test_siamese(): + """ + Test that all components the siamese network work correctly by executing a + training run against generated data. + """ + + num_classes = 5 + input_shape = (3,) + epochs = 1000 + + # Generate some data + x_train = np.random.rand(100, 3) + y_train = np.random.randint(num_classes, size=100) + + x_test = np.random.rand(30, 3) + y_test = np.random.randint(num_classes, size=30) + + # Define base and head model + def create_base_model(input_shape): + model_input = Input(shape=input_shape) + + embedding = Dense(4)(model_input) + embedding = BatchNormalization()(embedding) + embedding = Activation(activation='relu')(embedding) + + return Model(model_input, embedding) + + def create_head_model(embedding_shape): + embedding_a = Input(shape=embedding_shape) + embedding_b = Input(shape=embedding_shape) + + head = Concatenate()([embedding_a, embedding_b]) + head = Dense(4)(head) + head = BatchNormalization()(head) + head = Activation(activation='sigmoid')(head) + + head = Dense(1)(head) + head = BatchNormalization()(head) + head = Activation(activation='sigmoid')(head) + + return Model([embedding_a, embedding_b], head) + + # Create siamese neural network + base_model = create_base_model(input_shape) + head_model = create_head_model(base_model.output_shape) + siamese_network = SiameseNetwork(base_model, head_model) + + # Prepare siamese network for training + siamese_network.compile(loss='binary_crossentropy', + optimizer=keras.optimizers.adam()) + + # Evaluate network before training to establish a baseline + score_before = siamese_network.evaluate_generator( + x_train, y_train, batch_size=64 + ) + + # Train network + siamese_network.fit(x_train, y_train, + validation_data=(x_test, y_test), + batch_size=64, + epochs=epochs) + + # Evaluate network + score_after = siamese_network.evaluate(x_train, y_train, batch_size=64) + + # Ensure that the training loss score improved as a result of the training + assert(score_before > score_after)