;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;; LUSH Lisp Universal Shell
;;;   Copyright (C) 2002 Leon Bottou, Yann Le Cun, AT&T Corp, NECI.
;;; Includes parts of TL3:
;;;   Copyright (C) 1987-1999 Leon Bottou and Neuristique.
;;; Includes selected parts of SN3.2:
;;;   Copyright (C) 1991-2001 AT&T Corp.
;;;
;;; This program is free software; you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 2 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program; if not, write to the Free Software
;;; Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA
;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; $Id: lenet5.lsh,v 1.6 2003/11/10 18:14:48 leonb Exp $
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

#? Lenet5 Convolutional Neural Network Demo
;; {<author> Yann LeCun, December 2002}
;; This is an example of how to train a convolutional
;; network on the MNIST database (handwritten digits).
;; The database can be obtained at http://yann.lecun.com
;; A paper describing an experiment similar to the one 
;; demonstrated here is described in
;; LeCun, Bottou, Bengio, Haffner: "gradient-based learning
;; applied to document recognition", Proceedings of the IEEE, Nov 1998.
;; This paper is also available at the above URL.
;; This demo assumes that the MNIST data files are in
;; LUSHDIR/local/mnist. If you installed them someplace else,
;; simply do {<c> (setq *mnist-dir* "your-mnist-directory")}
;; before loading the present demo. 
;; The MNIST datafiles are:
;; {<ul>
;;  {<li> <train-images-idx3-ubyte>: training set, images, 60000 samples.}
;;  {<li> <train-labels-idx1-ubyte>: training set, labels, 60000 samples.}
;;  {<li> <t10k-images-idx3-ubyte>: testing set, images, 10000 samples.}
;;  {<li> <t10k-labels-idx1-ubyte>: testing set, labels, 10000 samples.}
;; }


(libload "gblearn2/net-lenet5")
(libload "gblearn2/gb-trainers")
(libload "gblearn2/gb-meters")
(libload "dsource-mnist")

(when (not *mnist-dir*)
      (setq *mnist-dir* (concat lushdir "/local/mnist")))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; a function to load the MNIST database.
(de build-databases-mnist (trsize tesize image-dir)
  (setq trainingdb
        (new dsource-idx3l-narrow
             (new dsource-mnist
                  (load-matrix (concat-fname image-dir "train-images-idx3-ubyte"))
                  (load-matrix (concat-fname image-dir "train-labels-idx1-ubyte"))
		  32 32 0 0.01)
             trsize 0))
  (setq testingdb
        (new dsource-idx3l-narrow
             (new dsource-mnist
                  (load-matrix (concat-fname image-dir "t10k-images-idx3-ubyte"))
                  (load-matrix (concat-fname image-dir "t10k-labels-idx1-ubyte"))
		  32 32 0 0.01)
             tesize (- 5000 (* 0.5 tesize)))) ())


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; number of classes
(setq nclasses 10)
;; the target values for mean-squared error training
;; are +target for the output corresponding to the class 
;; being shown and -target for the other outputs.
(setq target 1)

;; fill matrix with 1-of-n code
(setq labels (int-matrix nclasses))
(setq targets (float-matrix nclasses nclasses))
(idx-f2dotc targets 1.5 targets)
(targets ()() (- target))
(for (i 0 (- nclasses 1)) 
     (targets i i target)
     (labels i i))

;; create trainable parameter
(setq theparam (new idx1-ddparam 0 60000))

;; create the network
(setq thenet
      (new idx3-supervised-module
	   (new-lenet5 32 32 5 5 2 2 5 5 2 2 120 10 theparam)
	   (new edist-cost labels 1 1 targets)
	   (new max-classer labels)))

;; create the trainer
(setq thetrainer
      (new supervised-gradient thenet theparam))

;; a classifier-meter measures classification errors
(setq trainmeter (new classifier-meter))
(setq testmeter  (new classifier-meter))

;; initialize the network weights
(==> :thenet:machine forget 1 2)

;; build databases
;; the first two arguments are the sizes of the training
;; and testing sets. The 3rd arg is the directory where
;; the MNIST file reside (change this for your local setup).

(build-databases-mnist 2000 1000 *mnist-dir*) ;; SMALL DEMO
;; (build-databases-mnist 60000 10000 *mnist-dir*) ;; FULL MNIST


;; estimate second derivative on 100 iterations, using mu=0.02
;; and set individual espilons
(printf "computing diagonal hessian and learning rates\n")
(==> thetrainer compute-diaghessian trainingdb 100 0.02)

;; do training iterations 
(printf "training with %d training samples and %d test samples\n" 
	(==> trainingdb size)
	(==> testingdb size))

;; this goes at about 25 examples per second on a PIIIM 800MHz
(de doit (n)
  (repeat n
    (==> thetrainer train trainingdb trainmeter 0.0001 0)
    (printf "training: ") (flush)
    (==> thetrainer test trainingdb trainmeter)
    (==> trainmeter display)
    (printf " testing: ") (flush)
    (==> thetrainer test testingdb testmeter)
    (==> testmeter display)  
    ()))

(print (time (doit 5)))

